public inbox for git-commits@fedoraproject.org
help / color / mirror / Atom feed
* [rpms/llvm] f44: Backport s390x unaligned memops improvements to llvm 22
@ 2026-06-09 12:54 Paul Murphy
0 siblings, 0 replies; only message in thread
From: Paul Murphy @ 2026-06-09 12:54 UTC (permalink / raw)
To: git-commits
A new commit has been pushed.
Repo : rpms/llvm
Branch : f44
Commit : 3ca7f7b2c1ce6bcb4f40a6315274db0e2ac17432
Author : Paul Murphy <murp@redhat.com>
Date : 2026-06-09T09:46:50-03:00
Stats : +4159/-0 in 2 file(s)
URL : https://src.fedoraproject.org/rpms/llvm/c/3ca7f7b2c1ce6bcb4f40a6315274db0e2ac17432?branch=f44
Log:
Backport s390x unaligned memops improvements to llvm 22
Apply proposed PR #196359, which did not meet the stricter upstream
backport requirements.
(cherry picked from commit ae15be4898a8b612d86e1585e07b176ed8ee1e36)
---
diff --git a/0001-SystemZ-Avoid-unaligned-VL-VST-s-with-memcpy-memmove.patch b/0001-SystemZ-Avoid-unaligned-VL-VST-s-with-memcpy-memmove.patch
new file mode 100644
index 0000000..95a7dfb
--- /dev/null
+++ b/0001-SystemZ-Avoid-unaligned-VL-VST-s-with-memcpy-memmove.patch
@@ -0,0 +1,4156 @@
+From deb79107cdc640de8ccf4512a01512e7a8dcb72e Mon Sep 17 00:00:00 2001
+From: Jonas Paulsson <paulson1@linux.ibm.com>
+Date: Tue, 28 Apr 2026 19:09:09 +0200
+Subject: [PATCH] [SystemZ] Avoid unaligned VL/VST:s with
+ memcpy/memmove/memset. (#187100)
+
+This is a squash of the proposed backport PR #196359 and its dependencies
+to llvm 22:
+
+[SystemZ] Improved testing for memcpy/memmove/memset. (#194682)
+[SystemZ] Avoid unaligned VL/VST:s with memcpy/memmove/memset. (#187100)
+[SystemZ] Remove superfluous args in tests. (#196022)
+Fix memmove-01.ll test (to use libcalls in some cases).
+---
+ .../Target/SystemZ/SystemZISelLowering.cpp | 26 +-
+ llvm/test/CodeGen/SystemZ/memcpy-03.ll | 648 +++++-
+ llvm/test/CodeGen/SystemZ/memmove-01.ll | 1090 +++++++++
+ llvm/test/CodeGen/SystemZ/memset-08.ll | 2018 +++++++++++++++--
+ 4 files changed, 3453 insertions(+), 329 deletions(-)
+ create mode 100644 llvm/test/CodeGen/SystemZ/memmove-01.ll
+
+diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+index c3ded986fd68..78862759c880 100644
+--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
++++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+@@ -830,6 +830,10 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
+ MaxStoresPerMemcpy = Subtarget.hasVector() ? 2 : 0;
+ MaxStoresPerMemcpyOptSize = 0;
+
++ // Same with memmove.
++ MaxStoresPerMemmove = Subtarget.hasVector() ? 2 : 0;
++ MaxStoresPerMemmoveOptSize = 0;
++
+ // The main memset sequence is a byte store followed by an MVC.
+ // Two STC or MV..I stores win over that, but the kind of fused stores
+ // generated by target-independent code don't when the byte value is
+@@ -1473,17 +1477,21 @@ bool SystemZTargetLowering::findOptimalMemOpLowering(
+ LLVMContext &Context, std::vector<EVT> &MemOps, unsigned Limit,
+ const MemOp &Op, unsigned DstAS, unsigned SrcAS,
+ const AttributeList &FuncAttributes) const {
++
++ assert(Limit != ~0U &&
++ "Expected EmitTargetCodeForMemXXX() to handle AlwaysInline cases.");
++
++ if (Op.isZeroMemset())
++ return false; // Memset zero: Use XC.
++
+ const int MVCFastLen = 16;
++ // Use MVC up to 16 bytes. Small memset uses STC/MVI for first byte.
++ if ((Op.isMemset() ? Op.size() - 1 : Op.size()) <= MVCFastLen)
++ return false;
+
+- if (Limit != ~unsigned(0)) {
+- // Don't expand Op into scalar loads/stores in these cases:
+- if (Op.isMemcpy() && Op.allowOverlap() && Op.size() <= MVCFastLen)
+- return false; // Small memcpy: Use MVC
+- if (Op.isMemset() && Op.size() - 1 <= MVCFastLen)
+- return false; // Small memset (first byte with STC/MVI): Use MVC
+- if (Op.isZeroMemset())
+- return false; // Memset zero: Use XC
+- }
++ // Avoid unaligned VL/VST:s.
++ if (!Op.isAligned(Align(8)) || (Op.size() >= 25 && Op.size() <= 31))
++ return false;
+
+ return TargetLowering::findOptimalMemOpLowering(Context, MemOps, Limit, Op,
+ DstAS, SrcAS, FuncAttributes);
+diff --git a/llvm/test/CodeGen/SystemZ/memcpy-03.ll b/llvm/test/CodeGen/SystemZ/memcpy-03.ll
+index c703aef27532..213764e79ffb 100644
+--- a/llvm/test/CodeGen/SystemZ/memcpy-03.ll
++++ b/llvm/test/CodeGen/SystemZ/memcpy-03.ll
+@@ -1,217 +1,661 @@
+ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+-; RUN: llc -mcpu=z15 < %s -mtriple=s390x-linux-gnu | FileCheck %s
++; RUN: llc -mcpu=z17 < %s -mtriple=s390x-linux-gnu | FileCheck %s
+ ;
+-; Test memcpys of small constant lengths that should not be done with MVC.
++; Test non-volatile memcpys of small constant lengths in both aligned and
++; unaligned cases.
+
+ declare void @llvm.memcpy.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1) nounwind
+
+-define void @fun16(ptr %Src, ptr %Dst, i8 %val) {
++define void @fun1(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun1:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(1,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 1, i1 false)
++ ret void
++}
++
++define void @fun1_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun1_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(1,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 1, i1 false)
++ ret void
++}
++
++define void @fun2(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun2:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(2,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 2, i1 false)
++ ret void
++}
++
++define void @fun2_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun2_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(2,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 2, i1 false)
++ ret void
++}
++
++define void @fun3(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun3:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(3,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 3, i1 false)
++ ret void
++}
++
++define void @fun3_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun3_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(3,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 3, i1 false)
++ ret void
++}
++
++define void @fun4(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun4:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(4,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 4, i1 false)
++ ret void
++}
++
++define void @fun4_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun4_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(4,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 4, i1 false)
++ ret void
++}
++
++define void @fun5(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun5:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(5,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 5, i1 false)
++ ret void
++}
++
++define void @fun5_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun5_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(5,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 5, i1 false)
++ ret void
++}
++
++define void @fun6(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun6:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(6,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 6, i1 false)
++ ret void
++}
++
++define void @fun6_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun6_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(6,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 6, i1 false)
++ ret void
++}
++
++define void @fun7(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun7:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(7,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 7, i1 false)
++ ret void
++}
++
++define void @fun7_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun7_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(7,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 7, i1 false)
++ ret void
++}
++
++define void @fun8(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun8:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(8,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 8, i1 false)
++ ret void
++}
++
++define void @fun8_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun8_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(8,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 8, i1 false)
++ ret void
++}
++
++define void @fun9(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun9:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(9,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 9, i1 false)
++ ret void
++}
++
++define void @fun9_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun9_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(9,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 9, i1 false)
++ ret void
++}
++
++define void @fun10(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun10:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(10,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 10, i1 false)
++ ret void
++}
++
++define void @fun10_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun10_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(10,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 10, i1 false)
++ ret void
++}
++
++define void @fun11(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun11:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(11,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 11, i1 false)
++ ret void
++}
++
++define void @fun11_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun11_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(11,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 11, i1 false)
++ ret void
++}
++
++define void @fun12(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun12:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(12,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 12, i1 false)
++ ret void
++}
++
++define void @fun12_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun12_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(12,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 12, i1 false)
++ ret void
++}
++
++define void @fun13(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun13:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(13,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 13, i1 false)
++ ret void
++}
++
++define void @fun13_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun13_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(13,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 13, i1 false)
++ ret void
++}
++
++define void @fun14(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun14:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(14,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 14, i1 false)
++ ret void
++}
++
++define void @fun14_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun14_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(14,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 14, i1 false)
++ ret void
++}
++
++define void @fun15(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun15:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(15,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 15, i1 false)
++ ret void
++}
++
++define void @fun15_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun15_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(15,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 15, i1 false)
++ ret void
++}
++
++define void @fun16(ptr %Dst, ptr %Src) {
+ ; CHECK-LABEL: fun16:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: mvc 0(16,%r3), 0(%r2)
++; CHECK-NEXT: mvc 0(16,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 16, i1 false)
++ ret void
++}
++
++define void @fun16_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun16_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(16,%r2), 0(%r3)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 16, i1 false)
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 16, i1 false)
+ ret void
+ }
+
+-define void @fun17(ptr %Src, ptr %Dst, i8 %val) {
++define void @fun17(ptr %Dst, ptr %Src) {
+ ; CHECK-LABEL: fun17:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: lb %r0, 16(%r2)
+-; CHECK-NEXT: stc %r0, 16(%r3)
+-; CHECK-NEXT: vl %v0, 0(%r2), 4
+-; CHECK-NEXT: vst %v0, 0(%r3), 4
++; CHECK-NEXT: lb %r0, 16(%r3)
++; CHECK-NEXT: stc %r0, 16(%r2)
++; CHECK-NEXT: vl %v0, 0(%r3), 3
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 17, i1 false)
++ ret void
++}
++
++define void @fun17_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun17_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(17,%r2), 0(%r3)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 17, i1 false)
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 17, i1 false)
+ ret void
+ }
+
+-define void @fun18(ptr %Src, ptr %Dst, i8 %val) {
++define void @fun17_unalignedSrc(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun17_unalignedSrc:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(17,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 4 %Src, i64 17, i1 false)
++ ret void
++}
++
++define void @fun18(ptr %Dst, ptr %Src) {
+ ; CHECK-LABEL: fun18:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: lh %r0, 16(%r2)
+-; CHECK-NEXT: sth %r0, 16(%r3)
+-; CHECK-NEXT: vl %v0, 0(%r2), 4
+-; CHECK-NEXT: vst %v0, 0(%r3), 4
++; CHECK-NEXT: lh %r0, 16(%r3)
++; CHECK-NEXT: sth %r0, 16(%r2)
++; CHECK-NEXT: vl %v0, 0(%r3), 3
++; CHECK-NEXT: vst %v0, 0(%r2), 3
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 18, i1 false)
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 18, i1 false)
+ ret void
+ }
+
+-define void @fun19(ptr %Src, ptr %Dst, i8 %val) {
++define void @fun18_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun18_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(18,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 18, i1 false)
++ ret void
++}
++
++define void @fun18_unalignedDst(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun18_unalignedDst:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(18,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 2 %Dst, ptr align 16 %Src, i64 18, i1 false)
++ ret void
++}
++
++define void @fun19(ptr %Dst, ptr %Src) {
+ ; CHECK-LABEL: fun19:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: l %r0, 15(%r2)
+-; CHECK-NEXT: st %r0, 15(%r3)
+-; CHECK-NEXT: vl %v0, 0(%r2), 4
+-; CHECK-NEXT: vst %v0, 0(%r3), 4
++; CHECK-NEXT: l %r0, 15(%r3)
++; CHECK-NEXT: st %r0, 15(%r2)
++; CHECK-NEXT: vl %v0, 0(%r3), 3
++; CHECK-NEXT: vst %v0, 0(%r2), 3
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 19, i1 false)
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 19, i1 false)
+ ret void
+ }
+
+-define void @fun20(ptr %Src, ptr %Dst, i8 %val) {
++define void @fun19_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun19_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(19,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 19, i1 false)
++ ret void
++}
++
++define void @fun20(ptr %Dst, ptr %Src) {
+ ; CHECK-LABEL: fun20:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: l %r0, 16(%r2)
+-; CHECK-NEXT: st %r0, 16(%r3)
++; CHECK-NEXT: l %r0, 16(%r3)
++; CHECK-NEXT: st %r0, 16(%r2)
++; CHECK-NEXT: vl %v0, 0(%r3), 3
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 20, i1 false)
++ ret void
++}
++
++define void @fun20_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun20_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(20,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 20, i1 false)
++ ret void
++}
++
++define void @fun20_localDst(ptr %Src) {
++; CHECK-LABEL: fun20_localDst:
++; CHECK: # %bb.0:
++; CHECK-NEXT: aghi %r15, -184
++; CHECK-NEXT: .cfi_def_cfa_offset 344
+ ; CHECK-NEXT: vl %v0, 0(%r2), 4
+-; CHECK-NEXT: vst %v0, 0(%r3), 4
++; CHECK-NEXT: vst %v0, 164(%r15), 4
++; CHECK-NEXT: mvc 180(4,%r15), 16(%r2)
++; CHECK-NEXT: aghi %r15, 184
+ ; CHECK-NEXT: br %r14
++ %Dst = alloca [20 x i8]
+ call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 20, i1 false)
+ ret void
+ }
+
+-define void @fun21(ptr %Src, ptr %Dst, i8 %val) {
++define void @fun21(ptr %Dst, ptr %Src) {
+ ; CHECK-LABEL: fun21:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: lg %r0, 13(%r2)
+-; CHECK-NEXT: stg %r0, 13(%r3)
+-; CHECK-NEXT: vl %v0, 0(%r2), 4
+-; CHECK-NEXT: vst %v0, 0(%r3), 4
++; CHECK-NEXT: lg %r0, 13(%r3)
++; CHECK-NEXT: stg %r0, 13(%r2)
++; CHECK-NEXT: vl %v0, 0(%r3), 3
++; CHECK-NEXT: vst %v0, 0(%r2), 3
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 21, i1 false)
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 21, i1 false)
+ ret void
+ }
+
+-define void @fun22(ptr %Src, ptr %Dst, i8 %val) {
++define void @fun21_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun21_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(21,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 21, i1 false)
++ ret void
++}
++
++define void @fun22(ptr %Dst, ptr %Src) {
+ ; CHECK-LABEL: fun22:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: lg %r0, 14(%r2)
+-; CHECK-NEXT: stg %r0, 14(%r3)
+-; CHECK-NEXT: vl %v0, 0(%r2), 4
+-; CHECK-NEXT: vst %v0, 0(%r3), 4
++; CHECK-NEXT: lg %r0, 14(%r3)
++; CHECK-NEXT: stg %r0, 14(%r2)
++; CHECK-NEXT: vl %v0, 0(%r3), 3
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 22, i1 false)
++ ret void
++}
++
++define void @fun22_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun22_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(22,%r2), 0(%r3)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 22, i1 false)
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 22, i1 false)
+ ret void
+ }
+
+-define void @fun23(ptr %Src, ptr %Dst, i8 %val) {
++define void @fun23(ptr %Dst, ptr %Src) {
+ ; CHECK-LABEL: fun23:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: lg %r0, 15(%r2)
+-; CHECK-NEXT: stg %r0, 15(%r3)
+-; CHECK-NEXT: vl %v0, 0(%r2), 4
+-; CHECK-NEXT: vst %v0, 0(%r3), 4
++; CHECK-NEXT: lg %r0, 15(%r3)
++; CHECK-NEXT: stg %r0, 15(%r2)
++; CHECK-NEXT: vl %v0, 0(%r3), 3
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 23, i1 false)
++ ret void
++}
++
++define void @fun23_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun23_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(23,%r2), 0(%r3)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 23, i1 false)
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 23, i1 false)
+ ret void
+ }
+
+-define void @fun24(ptr %Src, ptr %Dst, i8 %val) {
++define void @fun24(ptr %Dst, ptr %Src) {
+ ; CHECK-LABEL: fun24:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: lg %r0, 16(%r2)
+-; CHECK-NEXT: stg %r0, 16(%r3)
+-; CHECK-NEXT: vl %v0, 0(%r2), 4
+-; CHECK-NEXT: vst %v0, 0(%r3), 4
++; CHECK-NEXT: lg %r0, 16(%r3)
++; CHECK-NEXT: stg %r0, 16(%r2)
++; CHECK-NEXT: vl %v0, 0(%r3), 3
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 24, i1 false)
++ ret void
++}
++
++define void @fun24_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun24_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(24,%r2), 0(%r3)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 24, i1 false)
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 24, i1 false)
+ ret void
+ }
+
+-define void @fun25(ptr %Src, ptr %Dst, i8 %val) {
++define void @fun25(ptr %Dst, ptr %Src) {
+ ; CHECK-LABEL: fun25:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vl %v0, 9(%r2)
+-; CHECK-NEXT: vst %v0, 9(%r3)
+-; CHECK-NEXT: vl %v0, 0(%r2), 4
+-; CHECK-NEXT: vst %v0, 0(%r3), 4
++; CHECK-NEXT: mvc 0(25,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 25, i1 false)
++ ret void
++}
++
++define void @fun25_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun25_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(25,%r2), 0(%r3)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 25, i1 false)
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 25, i1 false)
+ ret void
+ }
+
+-define void @fun26(ptr %Src, ptr %Dst, i8 %val) {
++define void @fun26(ptr %Dst, ptr %Src) {
+ ; CHECK-LABEL: fun26:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vl %v0, 10(%r2)
+-; CHECK-NEXT: vst %v0, 10(%r3)
+-; CHECK-NEXT: vl %v0, 0(%r2), 4
+-; CHECK-NEXT: vst %v0, 0(%r3), 4
++; CHECK-NEXT: mvc 0(26,%r2), 0(%r3)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 26, i1 false)
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 26, i1 false)
+ ret void
+ }
+
+-define void @fun27(ptr %Src, ptr %Dst, i8 %val) {
++define void @fun26_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun26_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(26,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 26, i1 false)
++ ret void
++}
++
++define void @fun27(ptr %Dst, ptr %Src) {
+ ; CHECK-LABEL: fun27:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vl %v0, 11(%r2)
+-; CHECK-NEXT: vst %v0, 11(%r3)
+-; CHECK-NEXT: vl %v0, 0(%r2), 4
+-; CHECK-NEXT: vst %v0, 0(%r3), 4
++; CHECK-NEXT: mvc 0(27,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 27, i1 false)
++ ret void
++}
++
++define void @fun27_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun27_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(27,%r2), 0(%r3)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 27, i1 false)
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 27, i1 false)
+ ret void
+ }
+
+-define void @fun28(ptr %Src, ptr %Dst, i8 %val) {
++define void @fun28(ptr %Dst, ptr %Src) {
+ ; CHECK-LABEL: fun28:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vl %v0, 12(%r2)
+-; CHECK-NEXT: vst %v0, 12(%r3)
+-; CHECK-NEXT: vl %v0, 0(%r2), 4
+-; CHECK-NEXT: vst %v0, 0(%r3), 4
++; CHECK-NEXT: mvc 0(28,%r2), 0(%r3)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 28, i1 false)
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 28, i1 false)
+ ret void
+ }
+
+-define void @fun29(ptr %Src, ptr %Dst, i8 %val) {
++define void @fun28_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun28_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(28,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 28, i1 false)
++ ret void
++}
++
++define void @fun29(ptr %Dst, ptr %Src) {
+ ; CHECK-LABEL: fun29:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vl %v0, 13(%r2)
+-; CHECK-NEXT: vst %v0, 13(%r3)
+-; CHECK-NEXT: vl %v0, 0(%r2), 4
+-; CHECK-NEXT: vst %v0, 0(%r3), 4
++; CHECK-NEXT: mvc 0(29,%r2), 0(%r3)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 29, i1 false)
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 29, i1 false)
+ ret void
+ }
+
+-define void @fun30(ptr %Src, ptr %Dst, i8 %val) {
++define void @fun29_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun29_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(29,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 29, i1 false)
++ ret void
++}
++
++define void @fun30(ptr %Dst, ptr %Src) {
+ ; CHECK-LABEL: fun30:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vl %v0, 14(%r2)
+-; CHECK-NEXT: vst %v0, 14(%r3)
+-; CHECK-NEXT: vl %v0, 0(%r2), 4
+-; CHECK-NEXT: vst %v0, 0(%r3), 4
++; CHECK-NEXT: mvc 0(30,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 30, i1 false)
++ ret void
++}
++
++define void @fun30_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun30_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(30,%r2), 0(%r3)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 30, i1 false)
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 30, i1 false)
+ ret void
+ }
+
+-define void @fun31(ptr %Src, ptr %Dst, i8 %val) {
++define void @fun31(ptr %Dst, ptr %Src) {
+ ; CHECK-LABEL: fun31:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vl %v0, 15(%r2)
+-; CHECK-NEXT: vst %v0, 15(%r3)
+-; CHECK-NEXT: vl %v0, 0(%r2), 4
+-; CHECK-NEXT: vst %v0, 0(%r3), 4
++; CHECK-NEXT: mvc 0(31,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 31, i1 false)
++ ret void
++}
++
++define void @fun31_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun31_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(31,%r2), 0(%r3)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 31, i1 false)
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 31, i1 false)
+ ret void
+ }
+
+-define void @fun32(ptr %Src, ptr %Dst, i8 %val) {
++define void @fun32(ptr %Dst, ptr %Src) {
+ ; CHECK-LABEL: fun32:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vl %v0, 16(%r2), 4
+-; CHECK-NEXT: vst %v0, 16(%r3), 4
+-; CHECK-NEXT: vl %v0, 0(%r2), 4
+-; CHECK-NEXT: vst %v0, 0(%r3), 4
++; CHECK-NEXT: vl %v0, 16(%r3), 3
++; CHECK-NEXT: vst %v0, 16(%r2), 3
++; CHECK-NEXT: vl %v0, 0(%r3), 3
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 32, i1 false)
++ ret void
++}
++
++define void @fun32_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun32_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(32,%r2), 0(%r3)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 32, i1 false)
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 32, i1 false)
+ ret void
+ }
+
+-define void @fun33(ptr %Src, ptr %Dst, i8 %val) {
++define void @fun33(ptr %Dst, ptr %Src) {
+ ; CHECK-LABEL: fun33:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: mvc 0(33,%r3), 0(%r2)
++; CHECK-NEXT: mvc 0(33,%r2), 0(%r3)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 33, i1 false)
++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 33, i1 false)
+ ret void
+ }
+
++define void @fun33_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun33_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvc 0(33,%r2), 0(%r3)
++; CHECK-NEXT: br %r14
++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 33, i1 false)
++ ret void
++}
+diff --git a/llvm/test/CodeGen/SystemZ/memmove-01.ll b/llvm/test/CodeGen/SystemZ/memmove-01.ll
+new file mode 100644
+index 000000000000..b9c60fbe59a7
+--- /dev/null
++++ b/llvm/test/CodeGen/SystemZ/memmove-01.ll
+@@ -0,0 +1,1090 @@
++; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
++; RUN: llc -mcpu=z17 < %s -mtriple=s390x-linux-gnu | FileCheck %s
++;
++; Test non-volatile memmoves of small constant lengths in both aligned and
++; unaligned cases.
++
++declare void @llvm.memmove.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1) nounwind
++
++define void @fun1(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun1:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 1
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 1, i1 false)
++ ret void
++}
++
++define void @fun1_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun1_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 1
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 1, i1 false)
++ ret void
++}
++
++define void @fun2(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun2:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 2
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 2, i1 false)
++ ret void
++}
++
++define void @fun2_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun2_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 2
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 2, i1 false)
++ ret void
++}
++
++define void @fun3(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun3:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 3
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 3, i1 false)
++ ret void
++}
++
++define void @fun3_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun3_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 3
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 3, i1 false)
++ ret void
++}
++
++define void @fun4(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun4:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 4
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 4, i1 false)
++ ret void
++}
++
++define void @fun4_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun4_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 4
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 4, i1 false)
++ ret void
++}
++
++define void @fun5(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun5:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 5
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 5, i1 false)
++ ret void
++}
++
++define void @fun5_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun5_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 5
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 5, i1 false)
++ ret void
++}
++
++define void @fun6(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun6:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 6
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 6, i1 false)
++ ret void
++}
++
++define void @fun6_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun6_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 6
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 6, i1 false)
++ ret void
++}
++
++define void @fun7(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun7:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 7
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 7, i1 false)
++ ret void
++}
++
++define void @fun7_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun7_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 7
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 7, i1 false)
++ ret void
++}
++
++define void @fun8(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun8:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 8
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 8, i1 false)
++ ret void
++}
++
++define void @fun8_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun8_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 8
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 8, i1 false)
++ ret void
++}
++
++define void @fun9(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun9:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 9
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 9, i1 false)
++ ret void
++}
++
++define void @fun9_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun9_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 9
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 9, i1 false)
++ ret void
++}
++
++define void @fun10(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun10:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 10
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 10, i1 false)
++ ret void
++}
++
++define void @fun10_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun10_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 10
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 10, i1 false)
++ ret void
++}
++
++define void @fun11(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun11:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 11
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 11, i1 false)
++ ret void
++}
++
++define void @fun11_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun11_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 11
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 11, i1 false)
++ ret void
++}
++
++define void @fun12(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun12:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 12
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 12, i1 false)
++ ret void
++}
++
++define void @fun12_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun12_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 12
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 12, i1 false)
++ ret void
++}
++
++define void @fun13(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun13:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 13
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 13, i1 false)
++ ret void
++}
++
++define void @fun13_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun13_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 13
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 13, i1 false)
++ ret void
++}
++
++define void @fun14(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun14:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 14
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 14, i1 false)
++ ret void
++}
++
++define void @fun14_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun14_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 14
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 14, i1 false)
++ ret void
++}
++
++define void @fun15(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun15:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 15
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 15, i1 false)
++ ret void
++}
++
++define void @fun15_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun15_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 15
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 15, i1 false)
++ ret void
++}
++
++define void @fun16(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun16:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 16
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 16, i1 false)
++ ret void
++}
++
++define void @fun16_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun16_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 16
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 16, i1 false)
++ ret void
++}
++
++define void @fun17(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun17:
++; CHECK: # %bb.0:
++; CHECK-NEXT: lb %r0, 16(%r3)
++; CHECK-NEXT: vl %v0, 0(%r3), 3
++; CHECK-NEXT: stc %r0, 16(%r2)
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 17, i1 false)
++ ret void
++}
++
++define void @fun17_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun17_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 17
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 17, i1 false)
++ ret void
++}
++
++define void @fun17_unalignedSrc(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun17_unalignedSrc:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 17
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 16 %Dst, ptr align 4 %Src, i64 17, i1 false)
++ ret void
++}
++
++define void @fun18(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun18:
++; CHECK: # %bb.0:
++; CHECK-NEXT: lh %r0, 16(%r3)
++; CHECK-NEXT: vl %v0, 0(%r3), 3
++; CHECK-NEXT: sth %r0, 16(%r2)
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 18, i1 false)
++ ret void
++}
++
++define void @fun18_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun18_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 18
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 18, i1 false)
++ ret void
++}
++
++define void @fun18_unalignedDst(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun18_unalignedDst:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 18
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 2 %Dst, ptr align 16 %Src, i64 18, i1 false)
++ ret void
++}
++
++define void @fun19(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun19:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 19
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 19, i1 false)
++ ret void
++}
++
++define void @fun19_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun19_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 19
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 19, i1 false)
++ ret void
++}
++
++define void @fun20(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun20:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vl %v0, 0(%r3), 3
++; CHECK-NEXT: l %r0, 16(%r3)
++; CHECK-NEXT: st %r0, 16(%r2)
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 20, i1 false)
++ ret void
++}
++
++define void @fun20_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun20_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 20
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 20, i1 false)
++ ret void
++}
++
++define void @fun20_localDst(ptr %Src) {
++; CHECK-LABEL: fun20_localDst:
++; CHECK: # %bb.0:
++; CHECK-NEXT: aghi %r15, -184
++; CHECK-NEXT: .cfi_def_cfa_offset 344
++; CHECK-NEXT: vl %v0, 0(%r2), 4
++; CHECK-NEXT: vst %v0, 164(%r15), 4
++; CHECK-NEXT: mvc 180(4,%r15), 16(%r2)
++; CHECK-NEXT: aghi %r15, 184
++; CHECK-NEXT: br %r14
++ %Dst = alloca [20 x i8]
++ call void @llvm.memmove.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 20, i1 false)
++ ret void
++}
++
++define void @fun21(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun21:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 21
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 21, i1 false)
++ ret void
++}
++
++define void @fun21_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun21_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 21
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 21, i1 false)
++ ret void
++}
++
++define void @fun22(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun22:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 22
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 22, i1 false)
++ ret void
++}
++
++define void @fun22_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun22_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 22
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 22, i1 false)
++ ret void
++}
++
++define void @fun23(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun23:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 23
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 23, i1 false)
++ ret void
++}
++
++define void @fun23_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun23_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 23
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 23, i1 false)
++ ret void
++}
++
++define void @fun24(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun24:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vl %v0, 0(%r3), 3
++; CHECK-NEXT: lg %r0, 16(%r3)
++; CHECK-NEXT: stg %r0, 16(%r2)
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 24, i1 false)
++ ret void
++}
++
++define void @fun24_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun24_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 24
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 24, i1 false)
++ ret void
++}
++
++define void @fun25(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun25:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 25
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 25, i1 false)
++ ret void
++}
++
++define void @fun25_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun25_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 25
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 25, i1 false)
++ ret void
++}
++
++define void @fun26(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun26:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 26
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 26, i1 false)
++ ret void
++}
++
++define void @fun26_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun26_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 26
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 26, i1 false)
++ ret void
++}
++
++define void @fun27(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun27:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 27
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 27, i1 false)
++ ret void
++}
++
++define void @fun27_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun27_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 27
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 27, i1 false)
++ ret void
++}
++
++define void @fun28(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun28:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 28
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 28, i1 false)
++ ret void
++}
++
++define void @fun28_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun28_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 28
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 28, i1 false)
++ ret void
++}
++
++define void @fun29(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun29:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 29
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 29, i1 false)
++ ret void
++}
++
++define void @fun29_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun29_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 29
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 29, i1 false)
++ ret void
++}
++
++define void @fun30(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun30:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 30
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 30, i1 false)
++ ret void
++}
++
++define void @fun30_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun30_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 30
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 30, i1 false)
++ ret void
++}
++
++define void @fun31(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun31:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 31
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 31, i1 false)
++ ret void
++}
++
++define void @fun31_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun31_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 31
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 31, i1 false)
++ ret void
++}
++
++define void @fun32(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun32:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vl %v0, 0(%r3), 3
++; CHECK-NEXT: vl %v1, 16(%r3), 3
++; CHECK-NEXT: vst %v1, 16(%r2), 3
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 32, i1 false)
++ ret void
++}
++
++define void @fun32_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun32_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 32
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 32, i1 false)
++ ret void
++}
++
++define void @fun33(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun33:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 33
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 33, i1 false)
++ ret void
++}
++
++define void @fun33_unaligned(ptr %Dst, ptr %Src) {
++; CHECK-LABEL: fun33_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
++; CHECK-NEXT: .cfi_offset %r14, -48
++; CHECK-NEXT: .cfi_offset %r15, -40
++; CHECK-NEXT: aghi %r15, -160
++; CHECK-NEXT: .cfi_def_cfa_offset 320
++; CHECK-NEXT: lghi %r4, 33
++; CHECK-NEXT: brasl %r14, memmove@PLT
++; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
++; CHECK-NEXT: br %r14
++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 33, i1 false)
++ ret void
++}
+diff --git a/llvm/test/CodeGen/SystemZ/memset-08.ll b/llvm/test/CodeGen/SystemZ/memset-08.ll
+index 931230983368..43c50320e011 100644
+--- a/llvm/test/CodeGen/SystemZ/memset-08.ll
++++ b/llvm/test/CodeGen/SystemZ/memset-08.ll
+@@ -1,230 +1,1702 @@
+ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+ ; RUN: llc -mcpu=z15 %s -mtriple=s390x-linux-gnu -o - | FileCheck %s
+ ;
+-; Test memsets of small constant lengths, that should not be done with MVC.
++; Test non-volatile memsets of small constant lengths in both aligned and
++; unaligned cases.
+
+ declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg)
+
++define void @reg1(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg1:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 1, i1 false)
++ ret void
++}
++
++define void @reg1_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg1_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 1, i1 false)
++ ret void
++}
++
++define void @reg2(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg2:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 1(%r2)
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 2, i1 false)
++ ret void
++}
++
++define void @reg2_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg2_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 1(%r2)
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 2, i1 false)
++ ret void
++}
++
++define void @reg3(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg3:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(2,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 3, i1 false)
++ ret void
++}
++
++define void @reg3_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg3_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(2,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 3, i1 false)
++ ret void
++}
++
++define void @reg4(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg4:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(3,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 4, i1 false)
++ ret void
++}
++
++define void @reg4_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg4_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(3,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 4, i1 false)
++ ret void
++}
++
++define void @reg5(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg5:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(4,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 5, i1 false)
++ ret void
++}
++
++define void @reg5_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg5_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(4,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 5, i1 false)
++ ret void
++}
++
++define void @reg6(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg6:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(5,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 6, i1 false)
++ ret void
++}
++
++define void @reg6_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg6_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(5,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 6, i1 false)
++ ret void
++}
++
++define void @reg7(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg7:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(6,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 7, i1 false)
++ ret void
++}
++
++define void @reg7_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg7_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(6,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 7, i1 false)
++ ret void
++}
++
++define void @reg8(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg8:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(7,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 8, i1 false)
++ ret void
++}
++
++define void @reg8_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg8_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(7,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 8, i1 false)
++ ret void
++}
++
++define void @reg9(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg9:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(8,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 9, i1 false)
++ ret void
++}
++
++define void @reg9_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg9_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(8,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 9, i1 false)
++ ret void
++}
++
++define void @reg10(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg10:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(9,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 10, i1 false)
++ ret void
++}
++
++define void @reg10_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg10_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(9,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 10, i1 false)
++ ret void
++}
++
++define void @reg11(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg11:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(10,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 11, i1 false)
++ ret void
++}
++
++define void @reg11_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg11_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(10,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 11, i1 false)
++ ret void
++}
++
++define void @reg12(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg12:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(11,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 12, i1 false)
++ ret void
++}
++
++define void @reg12_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg12_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(11,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 12, i1 false)
++ ret void
++}
++
++define void @reg13(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg13:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(12,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 13, i1 false)
++ ret void
++}
++
++define void @reg13_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg13_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(12,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 13, i1 false)
++ ret void
++}
++
++define void @reg14(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg14:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(13,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 14, i1 false)
++ ret void
++}
++
++define void @reg14_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg14_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(13,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 14, i1 false)
++ ret void
++}
++
++define void @reg15(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg15:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(14,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 15, i1 false)
++ ret void
++}
++
++define void @reg15_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg15_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(14,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 15, i1 false)
++ ret void
++}
++
++define void @reg16(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg16:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(15,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 16, i1 false)
++ ret void
++}
++
++define void @reg16_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg16_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(15,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 16, i1 false)
++ ret void
++}
++
+ define void @reg17(ptr %Dst, i8 %val) {
+ ; CHECK-LABEL: reg17:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: stc %r3, 0(%r2)
+-; CHECK-NEXT: mvc 1(16,%r2), 0(%r2)
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(16,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 17, i1 false)
++ ret void
++}
++
++define void @reg17_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg17_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(16,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 17, i1 false)
++ ret void
++}
++
++define void @reg18(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg18:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vlvgp %v0, %r3, %r3
++; CHECK-NEXT: vrepb %v0, %v0, 7
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: vsteh %v0, 16(%r2), 0
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 18, i1 false)
++ ret void
++}
++
++define void @reg18_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg18_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(17,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 18, i1 false)
++ ret void
++}
++
++define void @reg19(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg19:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vlvgp %v0, %r3, %r3
++; CHECK-NEXT: vrepb %v0, %v0, 7
++; CHECK-NEXT: vstef %v0, 15(%r2), 0
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 19, i1 false)
++ ret void
++}
++
++define void @reg19_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg19_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(18,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 19, i1 false)
++ ret void
++}
++
++define void @reg20(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg20:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vlvgp %v0, %r3, %r3
++; CHECK-NEXT: vrepb %v0, %v0, 7
++; CHECK-NEXT: vstef %v0, 16(%r2), 0
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 20, i1 false)
++ ret void
++}
++
++define void @reg20_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg20_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(19,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 20, i1 false)
++ ret void
++}
++
++define void @reg20_localDst(i8 %val) {
++; CHECK-LABEL: reg20_localDst:
++; CHECK: # %bb.0:
++; CHECK-NEXT: aghi %r15, -184
++; CHECK-NEXT: .cfi_def_cfa_offset 344
++; CHECK-NEXT: vlvgp %v0, %r2, %r2
++; CHECK-NEXT: vrepb %v0, %v0, 7
++; CHECK-NEXT: vstef %v0, 180(%r15), 0
++; CHECK-NEXT: vst %v0, 164(%r15), 4
++; CHECK-NEXT: aghi %r15, 184
++; CHECK-NEXT: br %r14
++ %Dst = alloca [20 x i8]
++ call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 20, i1 false)
++ ret void
++}
++
++define void @reg21(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg21:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vlvgp %v0, %r3, %r3
++; CHECK-NEXT: vrepb %v0, %v0, 7
++; CHECK-NEXT: vsteg %v0, 13(%r2), 0
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 21, i1 false)
++ ret void
++}
++
++define void @reg21_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg21_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(20,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 21, i1 false)
++ ret void
++}
++
++define void @reg22(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg22:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vlvgp %v0, %r3, %r3
++; CHECK-NEXT: vrepb %v0, %v0, 7
++; CHECK-NEXT: vsteg %v0, 14(%r2), 0
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 22, i1 false)
++ ret void
++}
++
++define void @reg22_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg22_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(21,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 22, i1 false)
++ ret void
++}
++
++define void @reg23(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg23:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vlvgp %v0, %r3, %r3
++; CHECK-NEXT: vrepb %v0, %v0, 7
++; CHECK-NEXT: vsteg %v0, 15(%r2), 0
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 23, i1 false)
++ ret void
++}
++
++define void @reg23_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg23_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(22,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 23, i1 false)
++ ret void
++}
++
++define void @reg24(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg24:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vlvgp %v0, %r3, %r3
++; CHECK-NEXT: vrepb %v0, %v0, 7
++; CHECK-NEXT: vsteg %v0, 16(%r2), 0
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 24, i1 false)
++ ret void
++}
++
++define void @reg24_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg24_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(23,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 24, i1 false)
++ ret void
++}
++
++define void @reg25(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg25:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(24,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 25, i1 false)
++ ret void
++}
++
++define void @reg25_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg25_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(24,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 25, i1 false)
++ ret void
++}
++
++define void @reg26(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg26:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(25,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 26, i1 false)
++ ret void
++}
++
++define void @reg26_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg26_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(25,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 26, i1 false)
++ ret void
++}
++
++define void @reg27(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg27:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(26,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 27, i1 false)
++ ret void
++}
++
++define void @reg27_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg27_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(26,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 27, i1 false)
++ ret void
++}
++
++define void @reg28(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg28:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(27,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 28, i1 false)
++ ret void
++}
++
++define void @reg28_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg28_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(27,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 28, i1 false)
++ ret void
++}
++
++define void @reg29(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg29:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(28,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 29, i1 false)
++ ret void
++}
++
++define void @reg29_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg29_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(28,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 29, i1 false)
++ ret void
++}
++
++define void @reg30(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg30:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(29,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 30, i1 false)
++ ret void
++}
++
++define void @reg30_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg30_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(29,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 30, i1 false)
++ ret void
++}
++
++define void @reg31(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg31:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(30,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 31, i1 false)
++ ret void
++}
++
++define void @reg31_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg31_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(30,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 31, i1 false)
++ ret void
++}
++
++define void @reg32(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg32:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vlvgp %v0, %r3, %r3
++; CHECK-NEXT: vrepb %v0, %v0, 7
++; CHECK-NEXT: vst %v0, 16(%r2), 3
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 32, i1 false)
++ ret void
++}
++
++define void @reg32_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg32_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(31,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 32, i1 false)
++ ret void
++}
++
++define void @reg33(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg33:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(32,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 33, i1 false)
++ ret void
++}
++
++define void @reg33_unaligned(ptr %Dst, i8 %val) {
++; CHECK-LABEL: reg33_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: stc %r3, 0(%r2)
++; CHECK-NEXT: mvc 1(32,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 33, i1 false)
++ ret void
++}
++
++;; Immediate value
++
++define void @imm1(ptr %Dst) {
++; CHECK-LABEL: imm1:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 1
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 1, i64 1, i1 false)
++ ret void
++}
++
++define void @imm1_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm1_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 255
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 -1, i64 1, i1 false)
++ ret void
++}
++
++define void @imm2(ptr %Dst) {
++; CHECK-LABEL: imm2:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvhhi 0(%r2), 514
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 2, i64 2, i1 false)
++ ret void
++}
++
++define void @imm2_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm2_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvhhi 0(%r2), 514
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 2, i64 2, i1 false)
++ ret void
++}
++
++define void @imm3(ptr %Dst) {
++; CHECK-LABEL: imm3:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 2(%r2), 3
++; CHECK-NEXT: mvhhi 0(%r2), 771
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 3, i64 3, i1 false)
++ ret void
++}
++
++define void @imm3_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm3_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 2(%r2), 3
++; CHECK-NEXT: mvhhi 0(%r2), 771
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 3, i64 3, i1 false)
++ ret void
++}
++
++define void @imm4(ptr %Dst) {
++; CHECK-LABEL: imm4:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vrepib %v0, 4
++; CHECK-NEXT: vstef %v0, 0(%r2), 0
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 4, i64 4, i1 false)
++ ret void
++}
++
++define void @imm4_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm4_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vrepib %v0, 4
++; CHECK-NEXT: vstef %v0, 0(%r2), 0
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 4, i64 4, i1 false)
++ ret void
++}
++
++define void @imm5(ptr %Dst) {
++; CHECK-LABEL: imm5:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 5
++; CHECK-NEXT: mvc 1(4,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 5, i64 5, i1 false)
++ ret void
++}
++
++define void @imm5_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm5_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 5
++; CHECK-NEXT: mvc 1(4,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 5, i64 5, i1 false)
++ ret void
++}
++
++define void @imm6(ptr %Dst) {
++; CHECK-LABEL: imm6:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 6
++; CHECK-NEXT: mvc 1(5,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 6, i64 6, i1 false)
++ ret void
++}
++
++define void @imm6_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm6_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 6
++; CHECK-NEXT: mvc 1(5,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 6, i64 6, i1 false)
++ ret void
++}
++
++define void @imm7(ptr %Dst) {
++; CHECK-LABEL: imm7:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 7
++; CHECK-NEXT: mvc 1(6,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 7, i64 7, i1 false)
++ ret void
++}
++
++define void @imm7_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm7_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 7
++; CHECK-NEXT: mvc 1(6,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 7, i64 7, i1 false)
++ ret void
++}
++
++define void @imm8(ptr %Dst) {
++; CHECK-LABEL: imm8:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 8
++; CHECK-NEXT: mvc 1(7,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 8, i64 8, i1 false)
++ ret void
++}
++
++define void @imm8_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm8_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 8
++; CHECK-NEXT: mvc 1(7,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 8, i64 8, i1 false)
++ ret void
++}
++
++define void @imm9(ptr %Dst) {
++; CHECK-LABEL: imm9:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 9
++; CHECK-NEXT: mvc 1(8,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 9, i64 9, i1 false)
++ ret void
++}
++
++define void @imm9_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm9_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 9
++; CHECK-NEXT: mvc 1(8,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 9, i64 9, i1 false)
++ ret void
++}
++
++define void @imm10(ptr %Dst) {
++; CHECK-LABEL: imm10:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 10
++; CHECK-NEXT: mvc 1(9,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 10, i64 10, i1 false)
++ ret void
++}
++
++define void @imm10_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm10_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 10
++; CHECK-NEXT: mvc 1(9,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 10, i64 10, i1 false)
++ ret void
++}
++
++define void @imm11(ptr %Dst) {
++; CHECK-LABEL: imm11:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 11
++; CHECK-NEXT: mvc 1(10,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 11, i64 11, i1 false)
++ ret void
++}
++
++define void @imm11_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm11_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 11
++; CHECK-NEXT: mvc 1(10,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 11, i64 11, i1 false)
++ ret void
++}
++
++define void @imm12(ptr %Dst) {
++; CHECK-LABEL: imm12:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 12
++; CHECK-NEXT: mvc 1(11,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 12, i64 12, i1 false)
++ ret void
++}
++
++define void @imm12_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm12_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 12
++; CHECK-NEXT: mvc 1(11,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 12, i64 12, i1 false)
++ ret void
++}
++
++define void @imm13(ptr %Dst) {
++; CHECK-LABEL: imm13:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 13
++; CHECK-NEXT: mvc 1(12,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 13, i64 13, i1 false)
++ ret void
++}
++
++define void @imm13_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm13_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 13
++; CHECK-NEXT: mvc 1(12,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 13, i64 13, i1 false)
++ ret void
++}
++
++define void @imm14(ptr %Dst) {
++; CHECK-LABEL: imm14:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 14
++; CHECK-NEXT: mvc 1(13,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 14, i64 14, i1 false)
++ ret void
++}
++
++define void @imm14_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm14_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 14
++; CHECK-NEXT: mvc 1(13,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 14, i64 14, i1 false)
++ ret void
++}
++
++define void @imm15(ptr %Dst) {
++; CHECK-LABEL: imm15:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 15
++; CHECK-NEXT: mvc 1(14,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 15, i64 15, i1 false)
++ ret void
++}
++
++define void @imm15_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm15_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 15
++; CHECK-NEXT: mvc 1(14,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 15, i64 15, i1 false)
++ ret void
++}
++
++define void @imm16(ptr %Dst) {
++; CHECK-LABEL: imm16:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 16
++; CHECK-NEXT: mvc 1(15,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 16, i64 16, i1 false)
++ ret void
++}
++
++define void @imm16_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm16_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 16
++; CHECK-NEXT: mvc 1(15,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 16, i64 16, i1 false)
++ ret void
++}
++
++define void @imm17(ptr %Dst) {
++; CHECK-LABEL: imm17:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 17
++; CHECK-NEXT: mvc 1(16,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 17, i64 17, i1 false)
++ ret void
++}
++
++define void @imm17_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm17_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 17
++; CHECK-NEXT: mvc 1(16,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 17, i64 17, i1 false)
++ ret void
++}
++
++define void @imm18(ptr %Dst) {
++; CHECK-LABEL: imm18:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vrepib %v0, 18
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: mvhhi 16(%r2), 4626
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 18, i64 18, i1 false)
++ ret void
++}
++
++define void @imm18_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm18_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 18
++; CHECK-NEXT: mvc 1(17,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 18, i64 18, i1 false)
++ ret void
++}
++
++define void @imm19(ptr %Dst) {
++; CHECK-LABEL: imm19:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vrepib %v0, 19
++; CHECK-NEXT: vstef %v0, 15(%r2), 0
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 19, i64 19, i1 false)
++ ret void
++}
++
++define void @imm19_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm19_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 19
++; CHECK-NEXT: mvc 1(18,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 19, i64 19, i1 false)
++ ret void
++}
++
++define void @imm20(ptr %Dst) {
++; CHECK-LABEL: imm20:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vrepib %v0, 20
++; CHECK-NEXT: vstef %v0, 16(%r2), 0
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 20, i64 20, i1 false)
++ ret void
++}
++
++define void @imm20_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm20_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 20
++; CHECK-NEXT: mvc 1(19,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 20, i64 20, i1 false)
++ ret void
++}
++
++define void @imm20_localDst() {
++; CHECK-LABEL: imm20_localDst:
++; CHECK: # %bb.0:
++; CHECK-NEXT: aghi %r15, -184
++; CHECK-NEXT: .cfi_def_cfa_offset 344
++; CHECK-NEXT: vrepib %v0, 20
++; CHECK-NEXT: vstef %v0, 180(%r15), 0
++; CHECK-NEXT: vst %v0, 164(%r15), 4
++; CHECK-NEXT: aghi %r15, 184
++; CHECK-NEXT: br %r14
++ %Dst = alloca [20 x i8]
++ call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 20, i64 20, i1 false)
++ ret void
++}
++
++define void @imm21(ptr %Dst) {
++; CHECK-LABEL: imm21:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vrepib %v0, 21
++; CHECK-NEXT: vsteg %v0, 13(%r2), 0
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 21, i64 21, i1 false)
++ ret void
++}
++
++define void @imm21_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm21_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 21
++; CHECK-NEXT: mvc 1(20,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 21, i64 21, i1 false)
++ ret void
++}
++
++define void @imm22(ptr %Dst) {
++; CHECK-LABEL: imm22:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vrepib %v0, 22
++; CHECK-NEXT: vsteg %v0, 14(%r2), 0
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 22, i64 22, i1 false)
++ ret void
++}
++
++define void @imm22_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm22_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 22
++; CHECK-NEXT: mvc 1(21,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 22, i64 22, i1 false)
++ ret void
++}
++
++define void @imm23(ptr %Dst) {
++; CHECK-LABEL: imm23:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vrepib %v0, 23
++; CHECK-NEXT: vsteg %v0, 15(%r2), 0
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 23, i64 23, i1 false)
++ ret void
++}
++
++define void @imm23_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm23_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 23
++; CHECK-NEXT: mvc 1(22,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 23, i64 23, i1 false)
++ ret void
++}
++
++define void @imm24(ptr %Dst) {
++; CHECK-LABEL: imm24:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vrepib %v0, 24
++; CHECK-NEXT: vsteg %v0, 16(%r2), 0
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 24, i64 24, i1 false)
++ ret void
++}
++
++define void @imm24_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm24_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 24
++; CHECK-NEXT: mvc 1(23,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 24, i64 24, i1 false)
++ ret void
++}
++
++define void @imm25(ptr %Dst) {
++; CHECK-LABEL: imm25:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 25
++; CHECK-NEXT: mvc 1(24,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 25, i64 25, i1 false)
++ ret void
++}
++
++define void @imm25_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm25_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 25
++; CHECK-NEXT: mvc 1(24,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 25, i64 25, i1 false)
++ ret void
++}
++
++define void @imm26(ptr %Dst) {
++; CHECK-LABEL: imm26:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 26
++; CHECK-NEXT: mvc 1(25,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 26, i64 26, i1 false)
++ ret void
++}
++
++define void @imm26_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm26_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 26
++; CHECK-NEXT: mvc 1(25,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 26, i64 26, i1 false)
++ ret void
++}
++
++define void @imm27(ptr %Dst) {
++; CHECK-LABEL: imm27:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 27
++; CHECK-NEXT: mvc 1(26,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 27, i64 27, i1 false)
++ ret void
++}
++
++define void @imm27_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm27_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 27
++; CHECK-NEXT: mvc 1(26,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 27, i64 27, i1 false)
++ ret void
++}
++
++define void @imm28(ptr %Dst) {
++; CHECK-LABEL: imm28:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 28
++; CHECK-NEXT: mvc 1(27,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 28, i64 28, i1 false)
++ ret void
++}
++
++define void @imm28_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm28_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 28
++; CHECK-NEXT: mvc 1(27,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 28, i64 28, i1 false)
++ ret void
++}
++
++define void @imm29(ptr %Dst) {
++; CHECK-LABEL: imm29:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 29
++; CHECK-NEXT: mvc 1(28,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 29, i64 29, i1 false)
++ ret void
++}
++
++define void @imm29_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm29_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 29
++; CHECK-NEXT: mvc 1(28,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 29, i64 29, i1 false)
++ ret void
++}
++
++define void @imm30(ptr %Dst) {
++; CHECK-LABEL: imm30:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 30
++; CHECK-NEXT: mvc 1(29,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 30, i64 30, i1 false)
++ ret void
++}
++
++define void @imm30_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm30_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 30
++; CHECK-NEXT: mvc 1(29,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 30, i64 30, i1 false)
++ ret void
++}
++
++define void @imm31(ptr %Dst) {
++; CHECK-LABEL: imm31:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 31
++; CHECK-NEXT: mvc 1(30,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 31, i64 31, i1 false)
++ ret void
++}
++
++define void @imm31_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm31_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 31
++; CHECK-NEXT: mvc 1(30,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 31, i64 31, i1 false)
++ ret void
++}
++
++define void @imm32(ptr %Dst) {
++; CHECK-LABEL: imm32:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vrepib %v0, 32
++; CHECK-NEXT: vst %v0, 16(%r2), 3
++; CHECK-NEXT: vst %v0, 0(%r2), 3
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 32, i64 32, i1 false)
++ ret void
++}
++
++define void @imm32_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm32_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 32
++; CHECK-NEXT: mvc 1(31,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 32, i64 32, i1 false)
++ ret void
++}
++
++define void @imm33(ptr %Dst) {
++; CHECK-LABEL: imm33:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 33
++; CHECK-NEXT: mvc 1(32,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 33, i64 33, i1 false)
++ ret void
++}
++
++define void @imm33_unaligned(ptr %Dst) {
++; CHECK-LABEL: imm33_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 33
++; CHECK-NEXT: mvc 1(32,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 33, i64 33, i1 false)
++ ret void
++}
++
++;; zero
++
++define void @zero1(ptr %Dst) {
++; CHECK-LABEL: zero1:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 0
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 1, i1 false)
++ ret void
++}
++
++define void @zero1_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero1_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvi 0(%r2), 0
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 1, i1 false)
++ ret void
++}
++
++define void @zero2(ptr %Dst) {
++; CHECK-LABEL: zero2:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvhhi 0(%r2), 0
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 2, i1 false)
++ ret void
++}
++
++define void @zero2_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero2_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvhhi 0(%r2), 0
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 17, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 2, i1 false)
+ ret void
+ }
+
+-define void @reg18(ptr %Dst, i8 %val) {
+-; CHECK-LABEL: reg18:
++define void @zero3(ptr %Dst) {
++; CHECK-LABEL: zero3:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vlvgp %v0, %r3, %r3
+-; CHECK-NEXT: vrepb %v0, %v0, 7
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
+-; CHECK-NEXT: vsteh %v0, 16(%r2), 0
++; CHECK-NEXT: mvi 2(%r2), 0
++; CHECK-NEXT: mvhhi 0(%r2), 0
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 18, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 3, i1 false)
+ ret void
+ }
+
+-define void @reg19(ptr %Dst, i8 %val) {
+-; CHECK-LABEL: reg19:
++define void @zero3_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero3_unaligned:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vlvgp %v0, %r3, %r3
+-; CHECK-NEXT: vrepb %v0, %v0, 7
+-; CHECK-NEXT: vstef %v0, 15(%r2), 0
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: mvi 2(%r2), 0
++; CHECK-NEXT: mvhhi 0(%r2), 0
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 19, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 3, i1 false)
+ ret void
+ }
+
+-define void @reg20(ptr %Dst, i8 %val) {
+-; CHECK-LABEL: reg20:
++define void @zero4(ptr %Dst) {
++; CHECK-LABEL: zero4:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vlvgp %v0, %r3, %r3
+-; CHECK-NEXT: vrepb %v0, %v0, 7
+-; CHECK-NEXT: vstef %v0, 16(%r2), 0
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: mvhi 0(%r2), 0
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 20, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 4, i1 false)
+ ret void
+ }
+
+-define void @reg21(ptr %Dst, i8 %val) {
+-; CHECK-LABEL: reg21:
++define void @zero4_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero4_unaligned:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vlvgp %v0, %r3, %r3
+-; CHECK-NEXT: vrepb %v0, %v0, 7
+-; CHECK-NEXT: vsteg %v0, 13(%r2), 0
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: mvhi 0(%r2), 0
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 21, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 4, i1 false)
+ ret void
+ }
+
+-define void @reg22(ptr %Dst, i8 %val) {
+-; CHECK-LABEL: reg22:
++define void @zero5(ptr %Dst) {
++; CHECK-LABEL: zero5:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vlvgp %v0, %r3, %r3
+-; CHECK-NEXT: vrepb %v0, %v0, 7
+-; CHECK-NEXT: vsteg %v0, 14(%r2), 0
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: mvi 4(%r2), 0
++; CHECK-NEXT: mvhi 0(%r2), 0
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 22, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 5, i1 false)
+ ret void
+ }
+
+-define void @reg23(ptr %Dst, i8 %val) {
+-; CHECK-LABEL: reg23:
++define void @zero5_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero5_unaligned:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vlvgp %v0, %r3, %r3
+-; CHECK-NEXT: vrepb %v0, %v0, 7
+-; CHECK-NEXT: vsteg %v0, 15(%r2), 0
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: mvi 4(%r2), 0
++; CHECK-NEXT: mvhi 0(%r2), 0
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 23, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 5, i1 false)
+ ret void
+ }
+
+-define void @reg24(ptr %Dst, i8 %val) {
+-; CHECK-LABEL: reg24:
++define void @zero6(ptr %Dst) {
++; CHECK-LABEL: zero6:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vlvgp %v0, %r3, %r3
+-; CHECK-NEXT: vrepb %v0, %v0, 7
+-; CHECK-NEXT: vsteg %v0, 16(%r2), 0
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: mvhhi 4(%r2), 0
++; CHECK-NEXT: mvhi 0(%r2), 0
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 24, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 6, i1 false)
+ ret void
+ }
+
+-define void @reg25(ptr %Dst, i8 %val) {
+-; CHECK-LABEL: reg25:
++define void @zero6_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero6_unaligned:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vlvgp %v0, %r3, %r3
+-; CHECK-NEXT: vrepb %v0, %v0, 7
+-; CHECK-NEXT: vst %v0, 9(%r2)
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: mvhhi 4(%r2), 0
++; CHECK-NEXT: mvhi 0(%r2), 0
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 25, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 6, i1 false)
+ ret void
+ }
+
+-define void @reg26(ptr %Dst, i8 %val) {
+-; CHECK-LABEL: reg26:
++define void @zero7(ptr %Dst) {
++; CHECK-LABEL: zero7:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vlvgp %v0, %r3, %r3
+-; CHECK-NEXT: vrepb %v0, %v0, 7
+-; CHECK-NEXT: vst %v0, 10(%r2)
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: xc 0(7,%r2), 0(%r2)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 26, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 7, i1 false)
+ ret void
+ }
+
+-define void @reg27(ptr %Dst, i8 %val) {
+-; CHECK-LABEL: reg27:
++define void @zero7_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero7_unaligned:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vlvgp %v0, %r3, %r3
+-; CHECK-NEXT: vrepb %v0, %v0, 7
+-; CHECK-NEXT: vst %v0, 11(%r2)
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: xc 0(7,%r2), 0(%r2)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 27, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 7, i1 false)
+ ret void
+ }
+
+-define void @reg28(ptr %Dst, i8 %val) {
+-; CHECK-LABEL: reg28:
++define void @zero8(ptr %Dst) {
++; CHECK-LABEL: zero8:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vlvgp %v0, %r3, %r3
+-; CHECK-NEXT: vrepb %v0, %v0, 7
+-; CHECK-NEXT: vst %v0, 12(%r2)
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: mvghi 0(%r2), 0
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 28, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 8, i1 false)
+ ret void
+ }
+
+-define void @reg29(ptr %Dst, i8 %val) {
+-; CHECK-LABEL: reg29:
++define void @zero8_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero8_unaligned:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vlvgp %v0, %r3, %r3
+-; CHECK-NEXT: vrepb %v0, %v0, 7
+-; CHECK-NEXT: vst %v0, 13(%r2)
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: mvi 0(%r2), 8
++; CHECK-NEXT: mvc 1(7,%r2), 0(%r2)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 29, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 8, i64 8, i1 false)
+ ret void
+ }
+
+-define void @reg30(ptr %Dst, i8 %val) {
+-; CHECK-LABEL: reg30:
++define void @zero9(ptr %Dst) {
++; CHECK-LABEL: zero9:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vlvgp %v0, %r3, %r3
+-; CHECK-NEXT: vrepb %v0, %v0, 7
+-; CHECK-NEXT: vst %v0, 14(%r2)
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: mvi 8(%r2), 0
++; CHECK-NEXT: mvghi 0(%r2), 0
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 30, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 9, i1 false)
+ ret void
+ }
+
+-define void @reg31(ptr %Dst, i8 %val) {
+-; CHECK-LABEL: reg31:
++define void @zero9_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero9_unaligned:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vlvgp %v0, %r3, %r3
+-; CHECK-NEXT: vrepb %v0, %v0, 7
+-; CHECK-NEXT: vst %v0, 15(%r2)
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: mvi 8(%r2), 0
++; CHECK-NEXT: mvghi 0(%r2), 0
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 31, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 9, i1 false)
+ ret void
+ }
+
+-define void @reg32(ptr %Dst, i8 %val) {
+-; CHECK-LABEL: reg32:
++define void @zero10(ptr %Dst) {
++; CHECK-LABEL: zero10:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vlvgp %v0, %r3, %r3
+-; CHECK-NEXT: vrepb %v0, %v0, 7
+-; CHECK-NEXT: vst %v0, 16(%r2), 4
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: mvhhi 8(%r2), 0
++; CHECK-NEXT: mvghi 0(%r2), 0
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 32, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 10, i1 false)
+ ret void
+ }
+
+-define void @reg33(ptr %Dst, i8 %val) {
+-; CHECK-LABEL: reg33:
++define void @zero10_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero10_unaligned:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: stc %r3, 0(%r2)
+-; CHECK-NEXT: mvc 1(32,%r2), 0(%r2)
++; CHECK-NEXT: mvhhi 8(%r2), 0
++; CHECK-NEXT: mvghi 0(%r2), 0
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 33, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 10, i1 false)
+ ret void
+ }
+
+-;; Immediate value
++define void @zero11(ptr %Dst) {
++; CHECK-LABEL: zero11:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(11,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 11, i1 false)
++ ret void
++}
+
+-define void @imm17(ptr %Dst) {
+-; CHECK-LABEL: imm17:
++define void @zero11_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero11_unaligned:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: mvi 0(%r2), 1
+-; CHECK-NEXT: mvc 1(16,%r2), 0(%r2)
++; CHECK-NEXT: xc 0(11,%r2), 0(%r2)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 1, i64 17, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 11, i1 false)
+ ret void
+ }
+
+-define void @imm18(ptr %Dst) {
+-; CHECK-LABEL: imm18:
++define void @zero12(ptr %Dst) {
++; CHECK-LABEL: zero12:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvhi 8(%r2), 0
++; CHECK-NEXT: mvghi 0(%r2), 0
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 12, i1 false)
++ ret void
++}
++
++define void @zero12_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero12_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: mvhi 8(%r2), 0
++; CHECK-NEXT: mvghi 0(%r2), 0
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 12, i1 false)
++ ret void
++}
++
++define void @zero13(ptr %Dst) {
++; CHECK-LABEL: zero13:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(13,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 13, i1 false)
++ ret void
++}
++
++define void @zero13_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero13_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(13,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 13, i1 false)
++ ret void
++}
++
++define void @zero14(ptr %Dst) {
++; CHECK-LABEL: zero14:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(14,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 14, i1 false)
++ ret void
++}
++
++define void @zero14_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero14_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(14,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 14, i1 false)
++ ret void
++}
++
++define void @zero15(ptr %Dst) {
++; CHECK-LABEL: zero15:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(15,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 15, i1 false)
++ ret void
++}
++
++define void @zero15_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero15_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(15,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 15, i1 false)
++ ret void
++}
++
++define void @zero16(ptr %Dst) {
++; CHECK-LABEL: zero16:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vgbm %v0, 65535
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
+-; CHECK-NEXT: mvhhi 16(%r2), -1
++; CHECK-NEXT: vgbm %v0, 0
++; CHECK-NEXT: vst %v0, 0(%r2), 3
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 -1, i64 18, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 16, i1 false)
++ ret void
++}
++
++define void @zero16_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero16_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: vgbm %v0, 0
++; CHECK-NEXT: vst %v0, 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 16, i1 false)
++ ret void
++}
++
++define void @zero17(ptr %Dst) {
++; CHECK-LABEL: zero17:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(17,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 17, i1 false)
++ ret void
++}
++
++define void @zero17_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero17_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(17,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 17, i1 false)
+ ret void
+ }
+
+@@ -233,95 +1705,155 @@ define void @zero18(ptr %Dst) {
+ ; CHECK: # %bb.0:
+ ; CHECK-NEXT: xc 0(18,%r2), 0(%r2)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 0, i64 18, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 18, i1 false)
+ ret void
+ }
+
+-define void @imm19(ptr %Dst) {
+-; CHECK-LABEL: imm19:
++define void @zero18_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero18_unaligned:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vrepib %v0, 1
+-; CHECK-NEXT: vstef %v0, 15(%r2), 0
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: xc 0(18,%r2), 0(%r2)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 1, i64 19, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 18, i1 false)
+ ret void
+ }
+
+-define void @imm20(ptr %Dst) {
+-; CHECK-LABEL: imm20:
++define void @zero19(ptr %Dst) {
++; CHECK-LABEL: zero19:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vgbm %v0, 65535
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
+-; CHECK-NEXT: mvhi 16(%r2), -1
++; CHECK-NEXT: xc 0(19,%r2), 0(%r2)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 -1, i64 20, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 19, i1 false)
+ ret void
+ }
+
+-define void @imm21(ptr %Dst) {
+-; CHECK-LABEL: imm21:
++define void @zero19_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero19_unaligned:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vrepib %v0, 1
+-; CHECK-NEXT: vsteg %v0, 13(%r2), 0
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: xc 0(19,%r2), 0(%r2)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 1, i64 21, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 19, i1 false)
+ ret void
+ }
+
+-define void @imm22(ptr %Dst) {
+-; CHECK-LABEL: imm22:
++define void @zero20(ptr %Dst) {
++; CHECK-LABEL: zero20:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vgbm %v0, 65535
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
+-; CHECK-NEXT: mvghi 14(%r2), -1
++; CHECK-NEXT: xc 0(20,%r2), 0(%r2)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 -1, i64 22, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 20, i1 false)
+ ret void
+ }
+
+-define void @imm23(ptr %Dst) {
+-; CHECK-LABEL: imm23:
++define void @zero20_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero20_unaligned:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vrepib %v0, 1
+-; CHECK-NEXT: vsteg %v0, 15(%r2), 0
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: xc 0(20,%r2), 0(%r2)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 1, i64 23, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 20, i1 false)
+ ret void
+ }
+
+-define void @imm24(ptr %Dst) {
+-; CHECK-LABEL: imm24:
++define void @zero20_localDst() {
++; CHECK-LABEL: zero20_localDst:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vgbm %v0, 65535
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
+-; CHECK-NEXT: mvghi 16(%r2), -1
++; CHECK-NEXT: aghi %r15, -184
++; CHECK-NEXT: .cfi_def_cfa_offset 344
++; CHECK-NEXT: xc 164(20,%r15), 164(%r15)
++; CHECK-NEXT: aghi %r15, 184
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 -1, i64 24, i1 false)
++ %Dst = alloca [20 x i8]
++ call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 0, i64 20, i1 false)
+ ret void
+ }
+
+-define void @imm25(ptr %Dst) {
+-; CHECK-LABEL: imm25:
++define void @zero21(ptr %Dst) {
++; CHECK-LABEL: zero21:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vrepib %v0, 1
+-; CHECK-NEXT: vst %v0, 9(%r2)
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: xc 0(21,%r2), 0(%r2)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 1, i64 25, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 21, i1 false)
+ ret void
+ }
+
+-define void @imm26(ptr %Dst) {
+-; CHECK-LABEL: imm26:
++define void @zero21_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero21_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(21,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 21, i1 false)
++ ret void
++}
++
++define void @zero22(ptr %Dst) {
++; CHECK-LABEL: zero22:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(22,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 22, i1 false)
++ ret void
++}
++
++define void @zero22_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero22_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(22,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 22, i1 false)
++ ret void
++}
++
++define void @zero23(ptr %Dst) {
++; CHECK-LABEL: zero23:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(23,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 23, i1 false)
++ ret void
++}
++
++define void @zero23_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero23_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(23,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 23, i1 false)
++ ret void
++}
++
++define void @zero24(ptr %Dst) {
++; CHECK-LABEL: zero24:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(24,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 24, i1 false)
++ ret void
++}
++
++define void @zero24_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero24_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(24,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 24, i1 false)
++ ret void
++}
++
++define void @zero25(ptr %Dst) {
++; CHECK-LABEL: zero25:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(25,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 25, i1 false)
++ ret void
++}
++
++define void @zero25_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero25_unaligned:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vgbm %v0, 65535
+-; CHECK-NEXT: vst %v0, 10(%r2)
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: xc 0(25,%r2), 0(%r2)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 -1, i64 26, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 25, i1 false)
+ ret void
+ }
+
+@@ -330,73 +1862,106 @@ define void @zero26(ptr %Dst) {
+ ; CHECK: # %bb.0:
+ ; CHECK-NEXT: xc 0(26,%r2), 0(%r2)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 0, i64 26, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 26, i1 false)
+ ret void
+ }
+
+-define void @imm27(ptr %Dst) {
+-; CHECK-LABEL: imm27:
++define void @zero26_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero26_unaligned:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vrepib %v0, 1
+-; CHECK-NEXT: vst %v0, 11(%r2)
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: xc 0(26,%r2), 0(%r2)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 1, i64 27, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 26, i1 false)
+ ret void
+ }
+
+-define void @imm28(ptr %Dst) {
+-; CHECK-LABEL: imm28:
++define void @zero27(ptr %Dst) {
++; CHECK-LABEL: zero27:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vgbm %v0, 65535
+-; CHECK-NEXT: vst %v0, 12(%r2)
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: xc 0(27,%r2), 0(%r2)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 -1, i64 28, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 27, i1 false)
+ ret void
+ }
+
+-define void @imm29(ptr %Dst) {
+-; CHECK-LABEL: imm29:
++define void @zero27_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero27_unaligned:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vrepib %v0, 1
+-; CHECK-NEXT: vst %v0, 13(%r2)
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: xc 0(27,%r2), 0(%r2)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 1, i64 29, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 27, i1 false)
+ ret void
+ }
+
+-define void @imm30(ptr %Dst) {
+-; CHECK-LABEL: imm30:
++define void @zero28(ptr %Dst) {
++; CHECK-LABEL: zero28:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vgbm %v0, 65535
+-; CHECK-NEXT: vst %v0, 14(%r2)
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: xc 0(28,%r2), 0(%r2)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 -1, i64 30, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 28, i1 false)
+ ret void
+ }
+
+-define void @imm31(ptr %Dst) {
+-; CHECK-LABEL: imm31:
++define void @zero28_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero28_unaligned:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vrepib %v0, 1
+-; CHECK-NEXT: vst %v0, 15(%r2)
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: xc 0(28,%r2), 0(%r2)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 1, i64 31, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 28, i1 false)
+ ret void
+ }
+
+-define void @imm32(ptr %Dst) {
+-; CHECK-LABEL: imm32:
++define void @zero29(ptr %Dst) {
++; CHECK-LABEL: zero29:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(29,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 29, i1 false)
++ ret void
++}
++
++define void @zero29_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero29_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(29,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 29, i1 false)
++ ret void
++}
++
++define void @zero30(ptr %Dst) {
++; CHECK-LABEL: zero30:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(30,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 30, i1 false)
++ ret void
++}
++
++define void @zero30_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero30_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(30,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 30, i1 false)
++ ret void
++}
++
++define void @zero31(ptr %Dst) {
++; CHECK-LABEL: zero31:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(31,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 31, i1 false)
++ ret void
++}
++
++define void @zero31_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero31_unaligned:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: vgbm %v0, 65535
+-; CHECK-NEXT: vst %v0, 16(%r2), 4
+-; CHECK-NEXT: vst %v0, 0(%r2), 4
++; CHECK-NEXT: xc 0(31,%r2), 0(%r2)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 -1, i64 32, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 31, i1 false)
+ ret void
+ }
+
+@@ -405,16 +1970,33 @@ define void @zero32(ptr %Dst) {
+ ; CHECK: # %bb.0:
+ ; CHECK-NEXT: xc 0(32,%r2), 0(%r2)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 0, i64 32, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 32, i1 false)
+ ret void
+ }
+
+-define void @imm33(ptr %Dst) {
+-; CHECK-LABEL: imm33:
++define void @zero32_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero32_unaligned:
+ ; CHECK: # %bb.0:
+-; CHECK-NEXT: mvi 0(%r2), 1
+-; CHECK-NEXT: mvc 1(32,%r2), 0(%r2)
++; CHECK-NEXT: xc 0(32,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 32, i1 false)
++ ret void
++}
++
++define void @zero33(ptr %Dst) {
++; CHECK-LABEL: zero33:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(33,%r2), 0(%r2)
++; CHECK-NEXT: br %r14
++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 33, i1 false)
++ ret void
++}
++
++define void @zero33_unaligned(ptr %Dst) {
++; CHECK-LABEL: zero33_unaligned:
++; CHECK: # %bb.0:
++; CHECK-NEXT: xc 0(33,%r2), 0(%r2)
+ ; CHECK-NEXT: br %r14
+- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 1, i64 33, i1 false)
++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 33, i1 false)
+ ret void
+ }
+--
+2.54.0
+
diff --git a/llvm.spec b/llvm.spec
index f1f0d15..86798f7 100644
--- a/llvm.spec
+++ b/llvm.spec
@@ -503,6 +503,9 @@ Patch103: 0001-Workaround-a-bug-in-ORC-on-ppc64le.patch
Patch104: 0001-Driver-Give-devtoolset-path-precedence-over-Installe.patch
#endregion CLANG patches
+# s390x fix for unaligned memory access performance regressions.
+Patch2210: 0001-SystemZ-Avoid-unaligned-VL-VST-s-with-memcpy-memmove.patch
+
#region LLD patches
Patch106: 0001-19-Always-build-shared-libs-for-LLD.patch
Patch2103: 0001-lld-Adjust-compressed-debug-level-test-for-s390x-wit.patch
^ permalink raw reply related [flat|nested] only message in thread
only message in thread, other threads:[~2026-06-09 12:54 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2026-06-09 12:54 [rpms/llvm] f44: Backport s390x unaligned memops improvements to llvm 22 Paul Murphy
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox