; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK

define arm_aapcs_vfpcc <2 x i32> @vmulhs_v2i32(<2 x i32> %s0, <2 x i32> %s1) {
; CHECK-LABEL: vmulhs_v2i32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmullb.s32 q2, q0, q1
; CHECK-NEXT:    vmov r0, s11
; CHECK-NEXT:    vmov r1, s9
; CHECK-NEXT:    vmov q0[2], q0[0], r1, r0
; CHECK-NEXT:    asrs r0, r0, #31
; CHECK-NEXT:    asrs r1, r1, #31
; CHECK-NEXT:    vmov q0[3], q0[1], r1, r0
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <2 x i32> %s0 to <2 x i64>
  %s1s = sext <2 x i32> %s1 to <2 x i64>
  %m = mul <2 x i64> %s0s, %s1s
  %s = ashr <2 x i64> %m, <i64 32, i64 32>
  %s2 = trunc <2 x i64> %s to <2 x i32>
  ret <2 x i32> %s2
}

define arm_aapcs_vfpcc <2 x i32> @vmulhu_v2i32(<2 x i32> %s0, <2 x i32> %s1) {
; CHECK-LABEL: vmulhu_v2i32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmullb.u32 q2, q0, q1
; CHECK-NEXT:    vldr s1, .LCPI1_0
; CHECK-NEXT:    vmov.f32 s0, s9
; CHECK-NEXT:    vmov.f32 s2, s11
; CHECK-NEXT:    vmov.f32 s3, s1
; CHECK-NEXT:    bx lr
; CHECK-NEXT:    .p2align 2
; CHECK-NEXT:  @ %bb.1:
; CHECK-NEXT:  .LCPI1_0:
; CHECK-NEXT:    .long 0x00000000 @ float 0
entry:
  %s0s = zext <2 x i32> %s0 to <2 x i64>
  %s1s = zext <2 x i32> %s1 to <2 x i64>
  %m = mul <2 x i64> %s0s, %s1s
  %s = lshr <2 x i64> %m, <i64 32, i64 32>
  %s2 = trunc <2 x i64> %s to <2 x i32>
  ret <2 x i32> %s2
}

define arm_aapcs_vfpcc <4 x i32> @vmulhs_v4i32(<4 x i32> %s0, <4 x i32> %s1) {
; CHECK-LABEL: vmulhs_v4i32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmulh.s32 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <4 x i32> %s0 to <4 x i64>
  %s1s = sext <4 x i32> %s1 to <4 x i64>
  %m = mul <4 x i64> %s0s, %s1s
  %s = ashr <4 x i64> %m, <i64 32, i64 32, i64 32, i64 32>
  %s2 = trunc <4 x i64> %s to <4 x i32>
  ret <4 x i32> %s2
}

define arm_aapcs_vfpcc <4 x i32> @vmulhu_v4i32(<4 x i32> %s0, <4 x i32> %s1) {
; CHECK-LABEL: vmulhu_v4i32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmulh.u32 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <4 x i32> %s0 to <4 x i64>
  %s1s = zext <4 x i32> %s1 to <4 x i64>
  %m = mul <4 x i64> %s0s, %s1s
  %s = lshr <4 x i64> %m, <i64 32, i64 32, i64 32, i64 32>
  %s2 = trunc <4 x i64> %s to <4 x i32>
  ret <4 x i32> %s2
}

define arm_aapcs_vfpcc <8 x i32> @vmulhs_v8i32(<8 x i32> %s0, <8 x i32> %s1) {
; CHECK-LABEL: vmulhs_v8i32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmulh.s32 q0, q0, q2
; CHECK-NEXT:    vmulh.s32 q1, q1, q3
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <8 x i32> %s0 to <8 x i64>
  %s1s = sext <8 x i32> %s1 to <8 x i64>
  %m = mul <8 x i64> %s0s, %s1s
  %s = ashr <8 x i64> %m, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
  %s2 = trunc <8 x i64> %s to <8 x i32>
  ret <8 x i32> %s2
}

define arm_aapcs_vfpcc <8 x i32> @vmulhu_v8i32(<8 x i32> %s0, <8 x i32> %s1) {
; CHECK-LABEL: vmulhu_v8i32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmulh.u32 q0, q0, q2
; CHECK-NEXT:    vmulh.u32 q1, q1, q3
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <8 x i32> %s0 to <8 x i64>
  %s1s = zext <8 x i32> %s1 to <8 x i64>
  %m = mul <8 x i64> %s0s, %s1s
  %s = lshr <8 x i64> %m, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
  %s2 = trunc <8 x i64> %s to <8 x i32>
  ret <8 x i32> %s2
}

define arm_aapcs_vfpcc <16 x i32> @vmulhs_v16i32(<16 x i32> %s0, <16 x i32> %s1) {
; CHECK-LABEL: vmulhs_v16i32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .vsave {d8, d9}
; CHECK-NEXT:    vpush {d8, d9}
; CHECK-NEXT:    add r0, sp, #16
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    add r0, sp, #32
; CHECK-NEXT:    vmulh.s32 q0, q0, q4
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    add r0, sp, #48
; CHECK-NEXT:    vmulh.s32 q1, q1, q4
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    add r0, sp, #64
; CHECK-NEXT:    vmulh.s32 q2, q2, q4
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    vmulh.s32 q3, q3, q4
; CHECK-NEXT:    vpop {d8, d9}
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <16 x i32> %s0 to <16 x i64>
  %s1s = sext <16 x i32> %s1 to <16 x i64>
  %m = mul <16 x i64> %s0s, %s1s
  %s = ashr <16 x i64> %m, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
  %s2 = trunc <16 x i64> %s to <16 x i32>
  ret <16 x i32> %s2
}

define arm_aapcs_vfpcc <16 x i32> @vmulhu_v16i32(<16 x i32> %s0, <16 x i32> %s1) {
; CHECK-LABEL: vmulhu_v16i32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .vsave {d8, d9}
; CHECK-NEXT:    vpush {d8, d9}
; CHECK-NEXT:    add r0, sp, #16
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    add r0, sp, #32
; CHECK-NEXT:    vmulh.u32 q0, q0, q4
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    add r0, sp, #48
; CHECK-NEXT:    vmulh.u32 q1, q1, q4
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    add r0, sp, #64
; CHECK-NEXT:    vmulh.u32 q2, q2, q4
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    vmulh.u32 q3, q3, q4
; CHECK-NEXT:    vpop {d8, d9}
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <16 x i32> %s0 to <16 x i64>
  %s1s = zext <16 x i32> %s1 to <16 x i64>
  %m = mul <16 x i64> %s0s, %s1s
  %s = lshr <16 x i64> %m, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
  %s2 = trunc <16 x i64> %s to <16 x i32>
  ret <16 x i32> %s2
}

define arm_aapcs_vfpcc <4 x i16> @vmulhs_v4i16(<4 x i16> %s0, <4 x i16> %s1) {
; CHECK-LABEL: vmulhs_v4i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmullb.s16 q0, q0, q1
; CHECK-NEXT:    vshr.s32 q0, q0, #16
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <4 x i16> %s0 to <4 x i32>
  %s1s = sext <4 x i16> %s1 to <4 x i32>
  %m = mul <4 x i32> %s0s, %s1s
  %s = ashr <4 x i32> %m, <i32 16, i32 16, i32 16, i32 16>
  %s2 = trunc <4 x i32> %s to <4 x i16>
  ret <4 x i16> %s2
}

define arm_aapcs_vfpcc <4 x i16> @vmulhu_v4i16(<4 x i16> %s0, <4 x i16> %s1) {
; CHECK-LABEL: vmulhu_v4i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmullb.u16 q0, q0, q1
; CHECK-NEXT:    vshr.u32 q0, q0, #16
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <4 x i16> %s0 to <4 x i32>
  %s1s = zext <4 x i16> %s1 to <4 x i32>
  %m = mul <4 x i32> %s0s, %s1s
  %s = lshr <4 x i32> %m, <i32 16, i32 16, i32 16, i32 16>
  %s2 = trunc <4 x i32> %s to <4 x i16>
  ret <4 x i16> %s2
}

define arm_aapcs_vfpcc <8 x i16> @vmulhs_v8i16(<8 x i16> %s0, <8 x i16> %s1) {
; CHECK-LABEL: vmulhs_v8i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmulh.s16 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <8 x i16> %s0 to <8 x i32>
  %s1s = sext <8 x i16> %s1 to <8 x i32>
  %m = mul <8 x i32> %s0s, %s1s
  %s = ashr <8 x i32> %m, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %s2 = trunc <8 x i32> %s to <8 x i16>
  ret <8 x i16> %s2
}

define arm_aapcs_vfpcc <8 x i16> @vmulhu_v8i16(<8 x i16> %s0, <8 x i16> %s1) {
; CHECK-LABEL: vmulhu_v8i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmulh.u16 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <8 x i16> %s0 to <8 x i32>
  %s1s = zext <8 x i16> %s1 to <8 x i32>
  %m = mul <8 x i32> %s0s, %s1s
  %s = lshr <8 x i32> %m, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %s2 = trunc <8 x i32> %s to <8 x i16>
  ret <8 x i16> %s2
}

define arm_aapcs_vfpcc <16 x i16> @vmulhs_v16i16(<16 x i16> %s0, <16 x i16> %s1) {
; CHECK-LABEL: vmulhs_v16i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmulh.s16 q0, q0, q2
; CHECK-NEXT:    vmulh.s16 q1, q1, q3
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <16 x i16> %s0 to <16 x i32>
  %s1s = sext <16 x i16> %s1 to <16 x i32>
  %m = mul <16 x i32> %s0s, %s1s
  %s = ashr <16 x i32> %m, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %s2 = trunc <16 x i32> %s to <16 x i16>
  ret <16 x i16> %s2
}

define arm_aapcs_vfpcc <16 x i16> @vmulhu_v16i16(<16 x i16> %s0, <16 x i16> %s1) {
; CHECK-LABEL: vmulhu_v16i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmulh.u16 q0, q0, q2
; CHECK-NEXT:    vmulh.u16 q1, q1, q3
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <16 x i16> %s0 to <16 x i32>
  %s1s = zext <16 x i16> %s1 to <16 x i32>
  %m = mul <16 x i32> %s0s, %s1s
  %s = lshr <16 x i32> %m, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %s2 = trunc <16 x i32> %s to <16 x i16>
  ret <16 x i16> %s2
}

define arm_aapcs_vfpcc <32 x i16> @vmulhs_v32i16(<32 x i16> %s0, <32 x i16> %s1) {
; CHECK-LABEL: vmulhs_v32i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .vsave {d8, d9}
; CHECK-NEXT:    vpush {d8, d9}
; CHECK-NEXT:    add r0, sp, #16
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    add r0, sp, #32
; CHECK-NEXT:    vmulh.s16 q0, q0, q4
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    add r0, sp, #48
; CHECK-NEXT:    vmulh.s16 q1, q1, q4
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    add r0, sp, #64
; CHECK-NEXT:    vmulh.s16 q2, q2, q4
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    vmulh.s16 q3, q3, q4
; CHECK-NEXT:    vpop {d8, d9}
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <32 x i16> %s0 to <32 x i32>
  %s1s = sext <32 x i16> %s1 to <32 x i32>
  %m = mul <32 x i32> %s0s, %s1s
  %s = ashr <32 x i32> %m, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %s2 = trunc <32 x i32> %s to <32 x i16>
  ret <32 x i16> %s2
}

define arm_aapcs_vfpcc <32 x i16> @vmulhu_v32i16(<32 x i16> %s0, <32 x i16> %s1) {
; CHECK-LABEL: vmulhu_v32i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .vsave {d8, d9}
; CHECK-NEXT:    vpush {d8, d9}
; CHECK-NEXT:    add r0, sp, #16
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    add r0, sp, #32
; CHECK-NEXT:    vmulh.u16 q0, q0, q4
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    add r0, sp, #48
; CHECK-NEXT:    vmulh.u16 q1, q1, q4
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    add r0, sp, #64
; CHECK-NEXT:    vmulh.u16 q2, q2, q4
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    vmulh.u16 q3, q3, q4
; CHECK-NEXT:    vpop {d8, d9}
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <32 x i16> %s0 to <32 x i32>
  %s1s = zext <32 x i16> %s1 to <32 x i32>
  %m = mul <32 x i32> %s0s, %s1s
  %s = lshr <32 x i32> %m, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %s2 = trunc <32 x i32> %s to <32 x i16>
  ret <32 x i16> %s2
}

define arm_aapcs_vfpcc <4 x i8> @vmulhs_v4i8(<4 x i8> %s0, <4 x i8> %s1) {
; CHECK-LABEL: vmulhs_v4i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmovlb.s8 q1, q1
; CHECK-NEXT:    vmovlb.s8 q0, q0
; CHECK-NEXT:    vmovlb.s16 q1, q1
; CHECK-NEXT:    vmovlb.s16 q0, q0
; CHECK-NEXT:    vmul.i32 q0, q0, q1
; CHECK-NEXT:    vshr.s32 q0, q0, #8
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <4 x i8> %s0 to <4 x i16>
  %s1s = sext <4 x i8> %s1 to <4 x i16>
  %m = mul <4 x i16> %s0s, %s1s
  %s = ashr <4 x i16> %m, <i16 8, i16 8, i16 8, i16 8>
  %s2 = trunc <4 x i16> %s to <4 x i8>
  ret <4 x i8> %s2
}

define arm_aapcs_vfpcc <4 x i8> @vmulhu_v4i8(<4 x i8> %s0, <4 x i8> %s1) {
; CHECK-LABEL: vmulhu_v4i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmov.i32 q2, #0xff
; CHECK-NEXT:    vand q1, q1, q2
; CHECK-NEXT:    vand q0, q0, q2
; CHECK-NEXT:    vmul.i32 q0, q0, q1
; CHECK-NEXT:    vshr.u32 q0, q0, #8
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <4 x i8> %s0 to <4 x i16>
  %s1s = zext <4 x i8> %s1 to <4 x i16>
  %m = mul <4 x i16> %s0s, %s1s
  %s = lshr <4 x i16> %m, <i16 8, i16 8, i16 8, i16 8>
  %s2 = trunc <4 x i16> %s to <4 x i8>
  ret <4 x i8> %s2
}

define arm_aapcs_vfpcc <8 x i8> @vmulhs_v8i8(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-LABEL: vmulhs_v8i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmullb.s8 q0, q0, q1
; CHECK-NEXT:    vshr.s16 q0, q0, #8
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <8 x i8> %s0 to <8 x i16>
  %s1s = sext <8 x i8> %s1 to <8 x i16>
  %m = mul <8 x i16> %s0s, %s1s
  %s = ashr <8 x i16> %m, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
  %s2 = trunc <8 x i16> %s to <8 x i8>
  ret <8 x i8> %s2
}

define arm_aapcs_vfpcc <8 x i8> @vmulhu_v8i8(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-LABEL: vmulhu_v8i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmullb.u8 q0, q0, q1
; CHECK-NEXT:    vshr.u16 q0, q0, #8
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <8 x i8> %s0 to <8 x i16>
  %s1s = zext <8 x i8> %s1 to <8 x i16>
  %m = mul <8 x i16> %s0s, %s1s
  %s = lshr <8 x i16> %m, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
  %s2 = trunc <8 x i16> %s to <8 x i8>
  ret <8 x i8> %s2
}

define arm_aapcs_vfpcc <16 x i8> @vmulhs_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
; CHECK-LABEL: vmulhs_v16i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmulh.s8 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <16 x i8> %s0 to <16 x i16>
  %s1s = sext <16 x i8> %s1 to <16 x i16>
  %m = mul <16 x i16> %s0s, %s1s
  %s = ashr <16 x i16> %m, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
  %s2 = trunc <16 x i16> %s to <16 x i8>
  ret <16 x i8> %s2
}

define arm_aapcs_vfpcc <16 x i8> @vmulhu_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
; CHECK-LABEL: vmulhu_v16i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmulh.u8 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <16 x i8> %s0 to <16 x i16>
  %s1s = zext <16 x i8> %s1 to <16 x i16>
  %m = mul <16 x i16> %s0s, %s1s
  %s = lshr <16 x i16> %m, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
  %s2 = trunc <16 x i16> %s to <16 x i8>
  ret <16 x i8> %s2
}

define arm_aapcs_vfpcc <32 x i8> @vmulhs_v32i8(<32 x i8> %s0, <32 x i8> %s1) {
; CHECK-LABEL: vmulhs_v32i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmulh.s8 q0, q0, q2
; CHECK-NEXT:    vmulh.s8 q1, q1, q3
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <32 x i8> %s0 to <32 x i16>
  %s1s = sext <32 x i8> %s1 to <32 x i16>
  %m = mul <32 x i16> %s0s, %s1s
  %s = ashr <32 x i16> %m, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
  %s2 = trunc <32 x i16> %s to <32 x i8>
  ret <32 x i8> %s2
}

define arm_aapcs_vfpcc <32 x i8> @vmulhu_v32i8(<32 x i8> %s0, <32 x i8> %s1) {
; CHECK-LABEL: vmulhu_v32i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmulh.u8 q0, q0, q2
; CHECK-NEXT:    vmulh.u8 q1, q1, q3
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <32 x i8> %s0 to <32 x i16>
  %s1s = zext <32 x i8> %s1 to <32 x i16>
  %m = mul <32 x i16> %s0s, %s1s
  %s = lshr <32 x i16> %m, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
  %s2 = trunc <32 x i16> %s to <32 x i8>
  ret <32 x i8> %s2
}

define arm_aapcs_vfpcc <64 x i8> @vmulhs_v64i8(<64 x i8> %s0, <64 x i8> %s1) {
; CHECK-LABEL: vmulhs_v64i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .vsave {d8, d9}
; CHECK-NEXT:    vpush {d8, d9}
; CHECK-NEXT:    add r0, sp, #16
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    add r0, sp, #32
; CHECK-NEXT:    vmulh.s8 q0, q0, q4
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    add r0, sp, #48
; CHECK-NEXT:    vmulh.s8 q1, q1, q4
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    add r0, sp, #64
; CHECK-NEXT:    vmulh.s8 q2, q2, q4
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    vmulh.s8 q3, q3, q4
; CHECK-NEXT:    vpop {d8, d9}
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <64 x i8> %s0 to <64 x i16>
  %s1s = sext <64 x i8> %s1 to <64 x i16>
  %m = mul <64 x i16> %s0s, %s1s
  %s = ashr <64 x i16> %m, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
  %s2 = trunc <64 x i16> %s to <64 x i8>
  ret <64 x i8> %s2
}

define arm_aapcs_vfpcc <64 x i8> @vmulhu_v64i8(<64 x i8> %s0, <64 x i8> %s1) {
; CHECK-LABEL: vmulhu_v64i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .vsave {d8, d9}
; CHECK-NEXT:    vpush {d8, d9}
; CHECK-NEXT:    add r0, sp, #16
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    add r0, sp, #32
; CHECK-NEXT:    vmulh.u8 q0, q0, q4
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    add r0, sp, #48
; CHECK-NEXT:    vmulh.u8 q1, q1, q4
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    add r0, sp, #64
; CHECK-NEXT:    vmulh.u8 q2, q2, q4
; CHECK-NEXT:    vldrw.u32 q4, [r0]
; CHECK-NEXT:    vmulh.u8 q3, q3, q4
; CHECK-NEXT:    vpop {d8, d9}
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <64 x i8> %s0 to <64 x i16>
  %s1s = zext <64 x i8> %s1 to <64 x i16>
  %m = mul <64 x i16> %s0s, %s1s
  %s = lshr <64 x i16> %m, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
  %s2 = trunc <64 x i16> %s to <64 x i8>
  ret <64 x i8> %s2
}

define void @vmulh_s8(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture %z, i32 %n) {
; CHECK-LABEL: vmulh_s8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #64
; CHECK-NEXT:  .LBB26_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrb.u8 q0, [r0], #16
; CHECK-NEXT:    vldrb.u8 q1, [r1], #16
; CHECK-NEXT:    vmulh.s8 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB26_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i8, ptr %x, i32 %index
  %wide.load = load <16 x i8>, ptr %0, align 1
  %1 = sext <16 x i8> %wide.load to <16 x i16>
  %2 = getelementptr inbounds i8, ptr %y, i32 %index
  %wide.load17 = load <16 x i8>, ptr %2, align 1
  %3 = sext <16 x i8> %wide.load17 to <16 x i16>
  %4 = mul nsw <16 x i16> %3, %1
  %5 = lshr <16 x i16> %4, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
  %6 = trunc <16 x i16> %5 to <16 x i8>
  %7 = getelementptr inbounds i8, ptr %z, i32 %index
  store <16 x i8> %6, ptr %7, align 1
  %index.next = add i32 %index, 16
  %8 = icmp eq i32 %index.next, 1024
  br i1 %8, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vmulh_s16(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture %z, i32 %n) {
; CHECK-LABEL: vmulh_s16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #128
; CHECK-NEXT:  .LBB27_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
; CHECK-NEXT:    vldrh.u16 q1, [r1], #16
; CHECK-NEXT:    vmulh.s16 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB27_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i16, ptr %x, i32 %index
  %wide.load = load <8 x i16>, ptr %0, align 2
  %1 = sext <8 x i16> %wide.load to <8 x i32>
  %2 = getelementptr inbounds i16, ptr %y, i32 %index
  %wide.load17 = load <8 x i16>, ptr %2, align 2
  %3 = sext <8 x i16> %wide.load17 to <8 x i32>
  %4 = mul nsw <8 x i32> %3, %1
  %5 = lshr <8 x i32> %4, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %6 = trunc <8 x i32> %5 to <8 x i16>
  %7 = getelementptr inbounds i16, ptr %z, i32 %index
  store <8 x i16> %6, ptr %7, align 2
  %index.next = add i32 %index, 8
  %8 = icmp eq i32 %index.next, 1024
  br i1 %8, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vmulh_s32(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture %z, i32 %n) {
; CHECK-LABEL: vmulh_s32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #256
; CHECK-NEXT:  .LBB28_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
; CHECK-NEXT:    vmulh.s32 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB28_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i32, ptr %x, i32 %index
  %wide.load = load <4 x i32>, ptr %0, align 4
  %1 = sext <4 x i32> %wide.load to <4 x i64>
  %2 = getelementptr inbounds i32, ptr %y, i32 %index
  %wide.load17 = load <4 x i32>, ptr %2, align 4
  %3 = sext <4 x i32> %wide.load17 to <4 x i64>
  %4 = mul nsw <4 x i64> %3, %1
  %5 = lshr <4 x i64> %4, <i64 32, i64 32, i64 32, i64 32>
  %6 = trunc <4 x i64> %5 to <4 x i32>
  %7 = getelementptr inbounds i32, ptr %z, i32 %index
  store <4 x i32> %6, ptr %7, align 4
  %index.next = add i32 %index, 4
  %8 = icmp eq i32 %index.next, 1024
  br i1 %8, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vmulh_u8(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture %z, i32 %n) {
; CHECK-LABEL: vmulh_u8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #64
; CHECK-NEXT:  .LBB29_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrb.u8 q0, [r0], #16
; CHECK-NEXT:    vldrb.u8 q1, [r1], #16
; CHECK-NEXT:    vmulh.u8 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB29_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i8, ptr %x, i32 %index
  %wide.load = load <16 x i8>, ptr %0, align 1
  %1 = zext <16 x i8> %wide.load to <16 x i16>
  %2 = getelementptr inbounds i8, ptr %y, i32 %index
  %wide.load17 = load <16 x i8>, ptr %2, align 1
  %3 = zext <16 x i8> %wide.load17 to <16 x i16>
  %4 = mul nuw <16 x i16> %3, %1
  %5 = lshr <16 x i16> %4, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
  %6 = trunc <16 x i16> %5 to <16 x i8>
  %7 = getelementptr inbounds i8, ptr %z, i32 %index
  store <16 x i8> %6, ptr %7, align 1
  %index.next = add i32 %index, 16
  %8 = icmp eq i32 %index.next, 1024
  br i1 %8, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vmulh_u16(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture %z, i32 %n) {
; CHECK-LABEL: vmulh_u16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #128
; CHECK-NEXT:  .LBB30_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
; CHECK-NEXT:    vldrh.u16 q1, [r1], #16
; CHECK-NEXT:    vmulh.u16 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB30_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i16, ptr %x, i32 %index
  %wide.load = load <8 x i16>, ptr %0, align 2
  %1 = zext <8 x i16> %wide.load to <8 x i32>
  %2 = getelementptr inbounds i16, ptr %y, i32 %index
  %wide.load17 = load <8 x i16>, ptr %2, align 2
  %3 = zext <8 x i16> %wide.load17 to <8 x i32>
  %4 = mul nuw <8 x i32> %3, %1
  %5 = lshr <8 x i32> %4, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %6 = trunc <8 x i32> %5 to <8 x i16>
  %7 = getelementptr inbounds i16, ptr %z, i32 %index
  store <8 x i16> %6, ptr %7, align 2
  %index.next = add i32 %index, 8
  %8 = icmp eq i32 %index.next, 1024
  br i1 %8, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vmulh_u32(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture %z, i32 %n) {
; CHECK-LABEL: vmulh_u32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #256
; CHECK-NEXT:  .LBB31_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
; CHECK-NEXT:    vmulh.u32 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB31_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i32, ptr %x, i32 %index
  %wide.load = load <4 x i32>, ptr %0, align 4
  %1 = zext <4 x i32> %wide.load to <4 x i64>
  %2 = getelementptr inbounds i32, ptr %y, i32 %index
  %wide.load17 = load <4 x i32>, ptr %2, align 4
  %3 = zext <4 x i32> %wide.load17 to <4 x i64>
  %4 = mul nuw <4 x i64> %3, %1
  %5 = lshr <4 x i64> %4, <i64 32, i64 32, i64 32, i64 32>
  %6 = trunc <4 x i64> %5 to <4 x i32>
  %7 = getelementptr inbounds i32, ptr %z, i32 %index
  store <4 x i32> %6, ptr %7, align 4
  %index.next = add i32 %index, 4
  %8 = icmp eq i32 %index.next, 1024
  br i1 %8, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}


define void @vmulh_s32_pred(ptr noalias nocapture %d, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y, i32 %n) {
; CHECK-LABEL: vmulh_s32_pred:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    cmp r3, #1
; CHECK-NEXT:    it lt
; CHECK-NEXT:    poplt {r7, pc}
; CHECK-NEXT:  .LBB32_1: @ %vector.ph
; CHECK-NEXT:    dlstp.32 lr, r3
; CHECK-NEXT:  .LBB32_2: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
; CHECK-NEXT:    vldrw.u32 q1, [r2], #16
; CHECK-NEXT:    vmulh.s32 q0, q1, q0
; CHECK-NEXT:    vstrw.32 q0, [r0], #16
; CHECK-NEXT:    letp lr, .LBB32_2
; CHECK-NEXT:  @ %bb.3: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  %cmp10 = icmp sgt i32 %n, 0
  br i1 %cmp10, label %vector.ph, label %for.cond.cleanup

vector.ph:                                        ; preds = %entry
  %n.rnd.up = add i32 %n, 3
  %n.vec = and i32 %n.rnd.up, -4
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %n)
  %0 = getelementptr inbounds i32, ptr %x, i32 %index
  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %0, i32 4, <4 x i1> %active.lane.mask, <4 x i32> poison)
  %1 = sext <4 x i32> %wide.masked.load to <4 x i64>
  %2 = getelementptr inbounds i32, ptr %y, i32 %index
  %wide.masked.load12 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %2, i32 4, <4 x i1> %active.lane.mask, <4 x i32> poison)
  %3 = sext <4 x i32> %wide.masked.load12 to <4 x i64>
  %4 = mul nsw <4 x i64> %3, %1
  %5 = lshr <4 x i64> %4, <i64 32, i64 32, i64 32, i64 32>
  %6 = trunc <4 x i64> %5 to <4 x i32>
  %7 = getelementptr inbounds i32, ptr %d, i32 %index
  call void @llvm.masked.store.v4i32.p0(<4 x i32> %6, ptr %7, i32 4, <4 x i1> %active.lane.mask)
  %index.next = add i32 %index, 4
  %8 = icmp eq i32 %index.next, %n.vec
  br i1 %8, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body, %entry
  ret void
}

define void @vmulh_u32_pred(ptr noalias nocapture %d, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y, i32 %n) {
; CHECK-LABEL: vmulh_u32_pred:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    cmp r3, #1
; CHECK-NEXT:    it lt
; CHECK-NEXT:    poplt {r7, pc}
; CHECK-NEXT:  .LBB33_1: @ %vector.ph
; CHECK-NEXT:    dlstp.32 lr, r3
; CHECK-NEXT:  .LBB33_2: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
; CHECK-NEXT:    vldrw.u32 q1, [r2], #16
; CHECK-NEXT:    vmulh.u32 q0, q1, q0
; CHECK-NEXT:    vstrw.32 q0, [r0], #16
; CHECK-NEXT:    letp lr, .LBB33_2
; CHECK-NEXT:  @ %bb.3: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  %cmp10 = icmp sgt i32 %n, 0
  br i1 %cmp10, label %vector.ph, label %for.cond.cleanup

vector.ph:                                        ; preds = %entry
  %n.rnd.up = add i32 %n, 3
  %n.vec = and i32 %n.rnd.up, -4
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %n)
  %0 = getelementptr inbounds i32, ptr %x, i32 %index
  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %0, i32 4, <4 x i1> %active.lane.mask, <4 x i32> poison)
  %1 = zext <4 x i32> %wide.masked.load to <4 x i64>
  %2 = getelementptr inbounds i32, ptr %y, i32 %index
  %wide.masked.load12 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %2, i32 4, <4 x i1> %active.lane.mask, <4 x i32> poison)
  %3 = zext <4 x i32> %wide.masked.load12 to <4 x i64>
  %4 = mul nuw <4 x i64> %3, %1
  %5 = lshr <4 x i64> %4, <i64 32, i64 32, i64 32, i64 32>
  %6 = trunc <4 x i64> %5 to <4 x i32>
  %7 = getelementptr inbounds i32, ptr %d, i32 %index
  call void @llvm.masked.store.v4i32.p0(<4 x i32> %6, ptr %7, i32 4, <4 x i1> %active.lane.mask)
  %index.next = add i32 %index, 4
  %8 = icmp eq i32 %index.next, %n.vec
  br i1 %8, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body, %entry
  ret void
}

define void @vmulh_s16_pred(ptr noalias nocapture %d, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y, i32 %n) {
; CHECK-LABEL: vmulh_s16_pred:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    cmp r3, #1
; CHECK-NEXT:    it lt
; CHECK-NEXT:    poplt {r7, pc}
; CHECK-NEXT:  .LBB34_1: @ %vector.ph
; CHECK-NEXT:    dlstp.16 lr, r3
; CHECK-NEXT:  .LBB34_2: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrh.u16 q0, [r1], #16
; CHECK-NEXT:    vldrh.u16 q1, [r2], #16
; CHECK-NEXT:    vmulh.s16 q0, q1, q0
; CHECK-NEXT:    vstrh.16 q0, [r0], #16
; CHECK-NEXT:    letp lr, .LBB34_2
; CHECK-NEXT:  @ %bb.3: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  %cmp10 = icmp sgt i32 %n, 0
  br i1 %cmp10, label %vector.ph, label %for.cond.cleanup

vector.ph:                                        ; preds = %entry
  %n.rnd.up = add i32 %n, 7
  %n.vec = and i32 %n.rnd.up, -8
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %n)
  %0 = getelementptr inbounds i16, ptr %x, i32 %index
  %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %0, i32 2, <8 x i1> %active.lane.mask, <8 x i16> poison)
  %1 = sext <8 x i16> %wide.masked.load to <8 x i32>
  %2 = getelementptr inbounds i16, ptr %y, i32 %index
  %wide.masked.load12 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %2, i32 2, <8 x i1> %active.lane.mask, <8 x i16> poison)
  %3 = sext <8 x i16> %wide.masked.load12 to <8 x i32>
  %4 = mul nsw <8 x i32> %3, %1
  %5 = lshr <8 x i32> %4, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %6 = trunc <8 x i32> %5 to <8 x i16>
  %7 = getelementptr inbounds i16, ptr %d, i32 %index
  call void @llvm.masked.store.v8i16.p0(<8 x i16> %6, ptr %7, i32 2, <8 x i1> %active.lane.mask)
  %index.next = add i32 %index, 8
  %8 = icmp eq i32 %index.next, %n.vec
  br i1 %8, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body, %entry
  ret void
}

define void @vmulh_u16_pred(ptr noalias nocapture %d, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y, i32 %n) {
; CHECK-LABEL: vmulh_u16_pred:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    cmp r3, #1
; CHECK-NEXT:    it lt
; CHECK-NEXT:    poplt {r7, pc}
; CHECK-NEXT:  .LBB35_1: @ %vector.ph
; CHECK-NEXT:    dlstp.16 lr, r3
; CHECK-NEXT:  .LBB35_2: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrh.u16 q0, [r1], #16
; CHECK-NEXT:    vldrh.u16 q1, [r2], #16
; CHECK-NEXT:    vmulh.u16 q0, q1, q0
; CHECK-NEXT:    vstrh.16 q0, [r0], #16
; CHECK-NEXT:    letp lr, .LBB35_2
; CHECK-NEXT:  @ %bb.3: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  %cmp10 = icmp sgt i32 %n, 0
  br i1 %cmp10, label %vector.ph, label %for.cond.cleanup

vector.ph:                                        ; preds = %entry
  %n.rnd.up = add i32 %n, 7
  %n.vec = and i32 %n.rnd.up, -8
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %n)
  %0 = getelementptr inbounds i16, ptr %x, i32 %index
  %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %0, i32 2, <8 x i1> %active.lane.mask, <8 x i16> poison)
  %1 = zext <8 x i16> %wide.masked.load to <8 x i32>
  %2 = getelementptr inbounds i16, ptr %y, i32 %index
  %wide.masked.load12 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %2, i32 2, <8 x i1> %active.lane.mask, <8 x i16> poison)
  %3 = zext <8 x i16> %wide.masked.load12 to <8 x i32>
  %4 = mul nuw <8 x i32> %3, %1
  %5 = lshr <8 x i32> %4, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %6 = trunc <8 x i32> %5 to <8 x i16>
  %7 = getelementptr inbounds i16, ptr %d, i32 %index
  call void @llvm.masked.store.v8i16.p0(<8 x i16> %6, ptr %7, i32 2, <8 x i1> %active.lane.mask)
  %index.next = add i32 %index, 8
  %8 = icmp eq i32 %index.next, %n.vec
  br i1 %8, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body, %entry
  ret void
}

define void @vmulh_s8_pred(ptr noalias nocapture %d, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y, i32 %n) {
; CHECK-LABEL: vmulh_s8_pred:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    cmp r3, #1
; CHECK-NEXT:    it lt
; CHECK-NEXT:    poplt {r7, pc}
; CHECK-NEXT:  .LBB36_1: @ %vector.ph
; CHECK-NEXT:    dlstp.8 lr, r3
; CHECK-NEXT:  .LBB36_2: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrb.u8 q0, [r1], #16
; CHECK-NEXT:    vldrb.u8 q1, [r2], #16
; CHECK-NEXT:    vmulh.s8 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r0], #16
; CHECK-NEXT:    letp lr, .LBB36_2
; CHECK-NEXT:  @ %bb.3: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  %cmp10 = icmp sgt i32 %n, 0
  br i1 %cmp10, label %vector.ph, label %for.cond.cleanup

vector.ph:                                        ; preds = %entry
  %n.rnd.up = add i32 %n, 15
  %n.vec = and i32 %n.rnd.up, -16
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %n)
  %0 = getelementptr inbounds i8, ptr %x, i32 %index
  %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %0, i32 1, <16 x i1> %active.lane.mask, <16 x i8> poison)
  %1 = sext <16 x i8> %wide.masked.load to <16 x i16>
  %2 = getelementptr inbounds i8, ptr %y, i32 %index
  %wide.masked.load12 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %2, i32 1, <16 x i1> %active.lane.mask, <16 x i8> poison)
  %3 = sext <16 x i8> %wide.masked.load12 to <16 x i16>
  %4 = mul nsw <16 x i16> %3, %1
  %5 = lshr <16 x i16> %4, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
  %6 = trunc <16 x i16> %5 to <16 x i8>
  %7 = getelementptr inbounds i8, ptr %d, i32 %index
  call void @llvm.masked.store.v16i8.p0(<16 x i8> %6, ptr %7, i32 1, <16 x i1> %active.lane.mask)
  %index.next = add i32 %index, 16
  %8 = icmp eq i32 %index.next, %n.vec
  br i1 %8, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body, %entry
  ret void
}

define void @vmulh_u8_pred(ptr noalias nocapture %d, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y, i32 %n) {
; CHECK-LABEL: vmulh_u8_pred:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    cmp r3, #1
; CHECK-NEXT:    it lt
; CHECK-NEXT:    poplt {r7, pc}
; CHECK-NEXT:  .LBB37_1: @ %vector.ph
; CHECK-NEXT:    dlstp.8 lr, r3
; CHECK-NEXT:  .LBB37_2: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrb.u8 q0, [r1], #16
; CHECK-NEXT:    vldrb.u8 q1, [r2], #16
; CHECK-NEXT:    vmulh.u8 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r0], #16
; CHECK-NEXT:    letp lr, .LBB37_2
; CHECK-NEXT:  @ %bb.3: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  %cmp10 = icmp sgt i32 %n, 0
  br i1 %cmp10, label %vector.ph, label %for.cond.cleanup

vector.ph:                                        ; preds = %entry
  %n.rnd.up = add i32 %n, 15
  %n.vec = and i32 %n.rnd.up, -16
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %n)
  %0 = getelementptr inbounds i8, ptr %x, i32 %index
  %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %0, i32 1, <16 x i1> %active.lane.mask, <16 x i8> poison)
  %1 = zext <16 x i8> %wide.masked.load to <16 x i16>
  %2 = getelementptr inbounds i8, ptr %y, i32 %index
  %wide.masked.load12 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %2, i32 1, <16 x i1> %active.lane.mask, <16 x i8> poison)
  %3 = zext <16 x i8> %wide.masked.load12 to <16 x i16>
  %4 = mul nuw <16 x i16> %3, %1
  %5 = lshr <16 x i16> %4, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
  %6 = trunc <16 x i16> %5 to <16 x i8>
  %7 = getelementptr inbounds i8, ptr %d, i32 %index
  call void @llvm.masked.store.v16i8.p0(<16 x i8> %6, ptr %7, i32 1, <16 x i1> %active.lane.mask)
  %index.next = add i32 %index, 16
  %8 = icmp eq i32 %index.next, %n.vec
  br i1 %8, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body, %entry
  ret void
}


define arm_aapcs_vfpcc i16 @vmulhs_reduce_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
; CHECK-LABEL: vmulhs_reduce_v16i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmulh.s8 q0, q0, q1
; CHECK-NEXT:    vaddv.s8 r0, q0
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <16 x i8> %s0 to <16 x i16>
  %s1s = sext <16 x i8> %s1 to <16 x i16>
  %m = mul <16 x i16> %s0s, %s1s
  %s = ashr <16 x i16> %m, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
  %result = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %s)
  ret i16 %result
}

define arm_aapcs_vfpcc i16 @vmulhu_reduce_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
; CHECK-LABEL: vmulhu_reduce_v16i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmulh.u8 q0, q0, q1
; CHECK-NEXT:    vaddv.s8 r0, q0
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <16 x i8> %s0 to <16 x i16>
  %s1s = zext <16 x i8> %s1 to <16 x i16>
  %m = mul <16 x i16> %s0s, %s1s
  %s = ashr <16 x i16> %m, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
  %result = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %s)
  ret i16 %result
}

define arm_aapcs_vfpcc <4 x i32> @vmulhs_kb_v4i32(<4 x i32> %s0, <4 x i64> %s1) {
; CHECK-LABEL: vmulhs_kb_v4i32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmov.f32 s4, s2
; CHECK-NEXT:    vmov r1, s9
; CHECK-NEXT:    vmov r2, s5
; CHECK-NEXT:    vmov.f32 s6, s3
; CHECK-NEXT:    vmov.f32 s10, s1
; CHECK-NEXT:    vmov r0, s4
; CHECK-NEXT:    smmul r0, r0, r1
; CHECK-NEXT:    vmov r1, s0
; CHECK-NEXT:    smmul r1, r1, r2
; CHECK-NEXT:    vmov r2, s7
; CHECK-NEXT:    vmov q0[2], q0[0], r1, r0
; CHECK-NEXT:    vmov r0, s6
; CHECK-NEXT:    vmov r1, s11
; CHECK-NEXT:    smmul r0, r0, r1
; CHECK-NEXT:    vmov r1, s10
; CHECK-NEXT:    smmul r1, r1, r2
; CHECK-NEXT:    vmov q0[3], q0[1], r1, r0
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <4 x i32> %s0 to <4 x i64>
  %s1s = ashr <4 x i64> %s1, <i64 32, i64 32, i64 32, i64 32>
  %m = mul <4 x i64> %s0s, %s1s
  %s = ashr <4 x i64> %m, <i64 32, i64 32, i64 32, i64 32>
  %s2 = trunc <4 x i64> %s to <4 x i32>
  ret <4 x i32> %s2
}

define arm_aapcs_vfpcc <4 x i32> @vmulhu_kb_v4i32(<4 x i32> %s0, <4 x i64> %s1) {
; CHECK-LABEL: vmulhu_kb_v4i32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmov.f32 s4, s2
; CHECK-NEXT:    vmov r1, s9
; CHECK-NEXT:    vmov r2, s5
; CHECK-NEXT:    vmov.f32 s6, s3
; CHECK-NEXT:    vmov.f32 s10, s1
; CHECK-NEXT:    vmov r0, s4
; CHECK-NEXT:    umull r0, r1, r0, r1
; CHECK-NEXT:    vmov r0, s0
; CHECK-NEXT:    umull r0, r2, r0, r2
; CHECK-NEXT:    vmov r0, s6
; CHECK-NEXT:    vmov q0[2], q0[0], r2, r1
; CHECK-NEXT:    vmov r1, s11
; CHECK-NEXT:    vmov r2, s7
; CHECK-NEXT:    umull r0, r1, r0, r1
; CHECK-NEXT:    vmov r0, s10
; CHECK-NEXT:    umull r0, r2, r0, r2
; CHECK-NEXT:    vmov q0[3], q0[1], r2, r1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <4 x i32> %s0 to <4 x i64>
  %s1s = lshr <4 x i64> %s1, <i64 32, i64 32, i64 32, i64 32>
  %m = mul <4 x i64> %s0s, %s1s
  %s = lshr <4 x i64> %m, <i64 32, i64 32, i64 32, i64 32>
  %s2 = trunc <4 x i64> %s to <4 x i32>
  ret <4 x i32> %s2
}

define arm_aapcs_vfpcc <4 x i32> @vmulhs_kbc_v4i32(<4 x i32> %s0, <4 x i64> %s1) {
; CHECK-LABEL: vmulhs_kbc_v4i32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmov.f32 s4, s2
; CHECK-NEXT:    vmov r1, s9
; CHECK-NEXT:    vmov r2, s5
; CHECK-NEXT:    vmov.f32 s6, s3
; CHECK-NEXT:    vmov.f32 s10, s1
; CHECK-NEXT:    vmov r0, s4
; CHECK-NEXT:    smmul r0, r1, r0
; CHECK-NEXT:    vmov r1, s0
; CHECK-NEXT:    smmul r1, r2, r1
; CHECK-NEXT:    vmov r2, s7
; CHECK-NEXT:    vmov q0[2], q0[0], r1, r0
; CHECK-NEXT:    vmov r0, s6
; CHECK-NEXT:    vmov r1, s11
; CHECK-NEXT:    smmul r0, r1, r0
; CHECK-NEXT:    vmov r1, s10
; CHECK-NEXT:    smmul r1, r2, r1
; CHECK-NEXT:    vmov q0[3], q0[1], r1, r0
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <4 x i32> %s0 to <4 x i64>
  %s1s = ashr <4 x i64> %s1, <i64 32, i64 32, i64 32, i64 32>
  %m = mul <4 x i64> %s1s, %s0s
  %s = ashr <4 x i64> %m, <i64 32, i64 32, i64 32, i64 32>
  %s2 = trunc <4 x i64> %s to <4 x i32>
  ret <4 x i32> %s2
}

define arm_aapcs_vfpcc <4 x i32> @vmulhu_kbc_v4i32(<4 x i32> %s0, <4 x i64> %s1) {
; CHECK-LABEL: vmulhu_kbc_v4i32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmov.f32 s4, s2
; CHECK-NEXT:    vmov r1, s9
; CHECK-NEXT:    vmov r2, s5
; CHECK-NEXT:    vmov.f32 s6, s3
; CHECK-NEXT:    vmov.f32 s10, s1
; CHECK-NEXT:    vmov r0, s4
; CHECK-NEXT:    umull r0, r1, r1, r0
; CHECK-NEXT:    vmov r0, s0
; CHECK-NEXT:    umull r0, r2, r2, r0
; CHECK-NEXT:    vmov r0, s6
; CHECK-NEXT:    vmov q0[2], q0[0], r2, r1
; CHECK-NEXT:    vmov r1, s11
; CHECK-NEXT:    vmov r2, s7
; CHECK-NEXT:    umull r0, r1, r1, r0
; CHECK-NEXT:    vmov r0, s10
; CHECK-NEXT:    umull r0, r2, r2, r0
; CHECK-NEXT:    vmov q0[3], q0[1], r2, r1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <4 x i32> %s0 to <4 x i64>
  %s1s = lshr <4 x i64> %s1, <i64 32, i64 32, i64 32, i64 32>
  %m = mul <4 x i64> %s1s, %s0s
  %s = lshr <4 x i64> %m, <i64 32, i64 32, i64 32, i64 32>
  %s2 = trunc <4 x i64> %s to <4 x i32>
  ret <4 x i32> %s2
}

define arm_aapcs_vfpcc <8 x i16> @vmulhs_kb_v8i16(<8 x i16> %s0, <8 x i32> %s1) {
; CHECK-LABEL: vmulhs_kb_v8i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .vsave {d8, d9}
; CHECK-NEXT:    vpush {d8, d9}
; CHECK-NEXT:    vmov.f32 s12, s5
; CHECK-NEXT:    vmovlt.s16 q4, q0
; CHECK-NEXT:    vmov.f32 s13, s7
; CHECK-NEXT:    vmovlb.s16 q0, q0
; CHECK-NEXT:    vmov.f32 s5, s6
; CHECK-NEXT:    vmov.f32 s14, s9
; CHECK-NEXT:    vmov.f32 s15, s11
; CHECK-NEXT:    vmov.f32 s6, s8
; CHECK-NEXT:    vshr.s32 q3, q3, #16
; CHECK-NEXT:    vmov.f32 s7, s10
; CHECK-NEXT:    vmul.i32 q3, q4, q3
; CHECK-NEXT:    vshr.s32 q1, q1, #16
; CHECK-NEXT:    vshr.u32 q3, q3, #16
; CHECK-NEXT:    vmul.i32 q0, q0, q1
; CHECK-NEXT:    vshr.u32 q0, q0, #16
; CHECK-NEXT:    vmovnt.i32 q0, q3
; CHECK-NEXT:    vpop {d8, d9}
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <8 x i16> %s0 to <8 x i32>
  %s1s = ashr <8 x i32> %s1, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %m = mul <8 x i32> %s0s, %s1s
  %s = ashr <8 x i32> %m, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %s2 = trunc <8 x i32> %s to <8 x i16>
  ret <8 x i16> %s2
}

define arm_aapcs_vfpcc <8 x i16> @vmulhu_kb_v8i16(<8 x i16> %s0, <8 x i32> %s1) {
; CHECK-LABEL: vmulhu_kb_v8i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .vsave {d8, d9}
; CHECK-NEXT:    vpush {d8, d9}
; CHECK-NEXT:    vmov.f32 s12, s5
; CHECK-NEXT:    vmovlt.u16 q4, q0
; CHECK-NEXT:    vmov.f32 s13, s7
; CHECK-NEXT:    vmovlb.u16 q0, q0
; CHECK-NEXT:    vmov.f32 s5, s6
; CHECK-NEXT:    vmov.f32 s14, s9
; CHECK-NEXT:    vmov.f32 s15, s11
; CHECK-NEXT:    vmov.f32 s6, s8
; CHECK-NEXT:    vshr.u32 q3, q3, #16
; CHECK-NEXT:    vmov.f32 s7, s10
; CHECK-NEXT:    vmul.i32 q3, q4, q3
; CHECK-NEXT:    vshr.u32 q1, q1, #16
; CHECK-NEXT:    vshr.u32 q3, q3, #16
; CHECK-NEXT:    vmul.i32 q0, q0, q1
; CHECK-NEXT:    vshr.u32 q0, q0, #16
; CHECK-NEXT:    vmovnt.i32 q0, q3
; CHECK-NEXT:    vpop {d8, d9}
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <8 x i16> %s0 to <8 x i32>
  %s1s = lshr <8 x i32> %s1, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %m = mul <8 x i32> %s0s, %s1s
  %s = lshr <8 x i32> %m, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %s2 = trunc <8 x i32> %s to <8 x i16>
  ret <8 x i16> %s2
}

define arm_aapcs_vfpcc <8 x i16> @vmulhs_kbc_v8i16(<8 x i16> %s0, <8 x i32> %s1) {
; CHECK-LABEL: vmulhs_kbc_v8i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .vsave {d8, d9}
; CHECK-NEXT:    vpush {d8, d9}
; CHECK-NEXT:    vmov.f32 s12, s5
; CHECK-NEXT:    vmovlt.s16 q4, q0
; CHECK-NEXT:    vmov.f32 s13, s7
; CHECK-NEXT:    vmovlb.s16 q0, q0
; CHECK-NEXT:    vmov.f32 s5, s6
; CHECK-NEXT:    vmov.f32 s14, s9
; CHECK-NEXT:    vmov.f32 s15, s11
; CHECK-NEXT:    vmov.f32 s6, s8
; CHECK-NEXT:    vshr.s32 q3, q3, #16
; CHECK-NEXT:    vmov.f32 s7, s10
; CHECK-NEXT:    vmul.i32 q3, q3, q4
; CHECK-NEXT:    vshr.s32 q1, q1, #16
; CHECK-NEXT:    vshr.u32 q3, q3, #16
; CHECK-NEXT:    vmul.i32 q0, q1, q0
; CHECK-NEXT:    vshr.u32 q0, q0, #16
; CHECK-NEXT:    vmovnt.i32 q0, q3
; CHECK-NEXT:    vpop {d8, d9}
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <8 x i16> %s0 to <8 x i32>
  %s1s = ashr <8 x i32> %s1, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %m = mul <8 x i32> %s1s, %s0s
  %s = ashr <8 x i32> %m, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %s2 = trunc <8 x i32> %s to <8 x i16>
  ret <8 x i16> %s2
}

define arm_aapcs_vfpcc <8 x i16> @vmulhu_kbc_v8i16(<8 x i16> %s0, <8 x i32> %s1) {
; CHECK-LABEL: vmulhu_kbc_v8i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .vsave {d8, d9}
; CHECK-NEXT:    vpush {d8, d9}
; CHECK-NEXT:    vmov.f32 s12, s5
; CHECK-NEXT:    vmovlt.u16 q4, q0
; CHECK-NEXT:    vmov.f32 s13, s7
; CHECK-NEXT:    vmovlb.u16 q0, q0
; CHECK-NEXT:    vmov.f32 s5, s6
; CHECK-NEXT:    vmov.f32 s14, s9
; CHECK-NEXT:    vmov.f32 s15, s11
; CHECK-NEXT:    vmov.f32 s6, s8
; CHECK-NEXT:    vshr.u32 q3, q3, #16
; CHECK-NEXT:    vmov.f32 s7, s10
; CHECK-NEXT:    vmul.i32 q3, q3, q4
; CHECK-NEXT:    vshr.u32 q1, q1, #16
; CHECK-NEXT:    vshr.u32 q3, q3, #16
; CHECK-NEXT:    vmul.i32 q0, q1, q0
; CHECK-NEXT:    vshr.u32 q0, q0, #16
; CHECK-NEXT:    vmovnt.i32 q0, q3
; CHECK-NEXT:    vpop {d8, d9}
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <8 x i16> %s0 to <8 x i32>
  %s1s = lshr <8 x i32> %s1, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %m = mul <8 x i32> %s1s, %s0s
  %s = lshr <8 x i32> %m, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %s2 = trunc <8 x i32> %s to <8 x i16>
  ret <8 x i16> %s2
}
