; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 -mattr=prefer-256-bit | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 -mattr=prefer-256-bit,avx512vbmi | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI ; Make sure CPUs default to prefer-256-bit. avx512vnni isn't interesting as it just adds an isel peephole for vpmaddwd+vpaddd ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=cascadelake | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=cooperlake | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=cannonlake | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=icelake-client | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=icelake-server | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI ; This file primarily contains tests for specific places in X86ISelLowering.cpp that needed be made aware of the legalizer not allowing 512-bit vectors due to prefer-256-bit even though AVX512 is enabled. define dso_local void @add256(<16 x i32>* %a, <16 x i32>* %b, <16 x i32>* %c) "min-legal-vector-width"="256" { ; CHECK-LABEL: add256: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa (%rdi), %ymm0 ; CHECK-NEXT: vmovdqa 32(%rdi), %ymm1 ; CHECK-NEXT: vpaddd 32(%rsi), %ymm1, %ymm1 ; CHECK-NEXT: vpaddd (%rsi), %ymm0, %ymm0 ; CHECK-NEXT: vmovdqa %ymm0, (%rdx) ; CHECK-NEXT: vmovdqa %ymm1, 32(%rdx) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %d = load <16 x i32>, <16 x i32>* %a %e = load <16 x i32>, <16 x i32>* %b %f = add <16 x i32> %d, %e store <16 x i32> %f, <16 x i32>* %c ret void } define dso_local void @add512(<16 x i32>* %a, <16 x i32>* %b, <16 x i32>* %c) "min-legal-vector-width"="512" { ; CHECK-LABEL: add512: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa64 (%rdi), %zmm0 ; CHECK-NEXT: vpaddd (%rsi), %zmm0, %zmm0 ; CHECK-NEXT: vmovdqa64 %zmm0, (%rdx) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %d = load <16 x i32>, <16 x i32>* %a %e = load <16 x i32>, <16 x i32>* %b %f = add <16 x i32> %d, %e store <16 x i32> %f, <16 x i32>* %c ret void } define dso_local void @avg_v64i8_256(<64 x i8>* %a, <64 x i8>* %b) "min-legal-vector-width"="256" { ; CHECK-LABEL: avg_v64i8_256: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa (%rsi), %ymm0 ; CHECK-NEXT: vmovdqa 32(%rsi), %ymm1 ; CHECK-NEXT: vpavgb (%rdi), %ymm0, %ymm0 ; CHECK-NEXT: vpavgb 32(%rdi), %ymm1, %ymm1 ; CHECK-NEXT: vmovdqu %ymm1, (%rax) ; CHECK-NEXT: vmovdqu %ymm0, (%rax) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %1 = load <64 x i8>, <64 x i8>* %a %2 = load <64 x i8>, <64 x i8>* %b %3 = zext <64 x i8> %1 to <64 x i32> %4 = zext <64 x i8> %2 to <64 x i32> %5 = add nuw nsw <64 x i32> %3, %6 = add nuw nsw <64 x i32> %5, %4 %7 = lshr <64 x i32> %6, %8 = trunc <64 x i32> %7 to <64 x i8> store <64 x i8> %8, <64 x i8>* undef, align 4 ret void } define dso_local void @avg_v64i8_512(<64 x i8>* %a, <64 x i8>* %b) "min-legal-vector-width"="512" { ; CHECK-LABEL: avg_v64i8_512: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa64 (%rsi), %zmm0 ; CHECK-NEXT: vpavgb (%rdi), %zmm0, %zmm0 ; CHECK-NEXT: vmovdqu64 %zmm0, (%rax) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %1 = load <64 x i8>, <64 x i8>* %a %2 = load <64 x i8>, <64 x i8>* %b %3 = zext <64 x i8> %1 to <64 x i32> %4 = zext <64 x i8> %2 to <64 x i32> %5 = add nuw nsw <64 x i32> %3, %6 = add nuw nsw <64 x i32> %5, %4 %7 = lshr <64 x i32> %6, %8 = trunc <64 x i32> %7 to <64 x i8> store <64 x i8> %8, <64 x i8>* undef, align 4 ret void } define dso_local void @pmaddwd_32_256(<32 x i16>* %APtr, <32 x i16>* %BPtr, <16 x i32>* %CPtr) "min-legal-vector-width"="256" { ; CHECK-LABEL: pmaddwd_32_256: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa (%rdi), %ymm0 ; CHECK-NEXT: vmovdqa 32(%rdi), %ymm1 ; CHECK-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1 ; CHECK-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0 ; CHECK-NEXT: vmovdqa %ymm0, (%rdx) ; CHECK-NEXT: vmovdqa %ymm1, 32(%rdx) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %A = load <32 x i16>, <32 x i16>* %APtr %B = load <32 x i16>, <32 x i16>* %BPtr %a = sext <32 x i16> %A to <32 x i32> %b = sext <32 x i16> %B to <32 x i32> %m = mul nsw <32 x i32> %a, %b %odd = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> %even = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> %ret = add <16 x i32> %odd, %even store <16 x i32> %ret, <16 x i32>* %CPtr ret void } define dso_local void @pmaddwd_32_512(<32 x i16>* %APtr, <32 x i16>* %BPtr, <16 x i32>* %CPtr) "min-legal-vector-width"="512" { ; CHECK-LABEL: pmaddwd_32_512: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa64 (%rdi), %zmm0 ; CHECK-NEXT: vpmaddwd (%rsi), %zmm0, %zmm0 ; CHECK-NEXT: vmovdqa64 %zmm0, (%rdx) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %A = load <32 x i16>, <32 x i16>* %APtr %B = load <32 x i16>, <32 x i16>* %BPtr %a = sext <32 x i16> %A to <32 x i32> %b = sext <32 x i16> %B to <32 x i32> %m = mul nsw <32 x i32> %a, %b %odd = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> %even = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> %ret = add <16 x i32> %odd, %even store <16 x i32> %ret, <16 x i32>* %CPtr ret void } define dso_local void @psubus_64i8_max_256(<64 x i8>* %xptr, <64 x i8>* %yptr, <64 x i8>* %zptr) "min-legal-vector-width"="256" { ; CHECK-LABEL: psubus_64i8_max_256: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa (%rdi), %ymm0 ; CHECK-NEXT: vmovdqa 32(%rdi), %ymm1 ; CHECK-NEXT: vpsubusb 32(%rsi), %ymm1, %ymm1 ; CHECK-NEXT: vpsubusb (%rsi), %ymm0, %ymm0 ; CHECK-NEXT: vmovdqa %ymm0, (%rdx) ; CHECK-NEXT: vmovdqa %ymm1, 32(%rdx) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %x = load <64 x i8>, <64 x i8>* %xptr %y = load <64 x i8>, <64 x i8>* %yptr %cmp = icmp ult <64 x i8> %x, %y %max = select <64 x i1> %cmp, <64 x i8> %y, <64 x i8> %x %res = sub <64 x i8> %max, %y store <64 x i8> %res, <64 x i8>* %zptr ret void } define dso_local void @psubus_64i8_max_512(<64 x i8>* %xptr, <64 x i8>* %yptr, <64 x i8>* %zptr) "min-legal-vector-width"="512" { ; CHECK-LABEL: psubus_64i8_max_512: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa64 (%rdi), %zmm0 ; CHECK-NEXT: vpsubusb (%rsi), %zmm0, %zmm0 ; CHECK-NEXT: vmovdqa64 %zmm0, (%rdx) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %x = load <64 x i8>, <64 x i8>* %xptr %y = load <64 x i8>, <64 x i8>* %yptr %cmp = icmp ult <64 x i8> %x, %y %max = select <64 x i1> %cmp, <64 x i8> %y, <64 x i8> %x %res = sub <64 x i8> %max, %y store <64 x i8> %res, <64 x i8>* %zptr ret void } define dso_local i32 @_Z9test_charPcS_i_256(i8* nocapture readonly, i8* nocapture readonly, i32) "min-legal-vector-width"="256" { ; CHECK-LABEL: _Z9test_charPcS_i_256: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: movl %edx, %eax ; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0 ; CHECK-NEXT: xorl %ecx, %ecx ; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; CHECK-NEXT: vpxor %xmm2, %xmm2, %xmm2 ; CHECK-NEXT: .p2align 4, 0x90 ; CHECK-NEXT: .LBB8_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vpmovsxbw 16(%rdi,%rcx), %ymm3 ; CHECK-NEXT: vpmovsxbw (%rdi,%rcx), %ymm4 ; CHECK-NEXT: vpmovsxbw 16(%rsi,%rcx), %ymm5 ; CHECK-NEXT: vpmaddwd %ymm3, %ymm5, %ymm3 ; CHECK-NEXT: vpaddd %ymm2, %ymm3, %ymm2 ; CHECK-NEXT: vpmovsxbw (%rsi,%rcx), %ymm3 ; CHECK-NEXT: vpmaddwd %ymm4, %ymm3, %ymm3 ; CHECK-NEXT: vpaddd %ymm1, %ymm3, %ymm1 ; CHECK-NEXT: addq $32, %rcx ; CHECK-NEXT: cmpq %rcx, %rax ; CHECK-NEXT: jne .LBB8_1 ; CHECK-NEXT: # %bb.2: # %middle.block ; CHECK-NEXT: vpaddd %ymm0, %ymm1, %ymm1 ; CHECK-NEXT: vpaddd %ymm0, %ymm2, %ymm0 ; CHECK-NEXT: vpaddd %ymm0, %ymm1, %ymm0 ; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm1 ; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1] ; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: vmovd %xmm0, %eax ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq entry: %3 = zext i32 %2 to i64 br label %vector.body vector.body: %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ] %vec.phi = phi <32 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ] %4 = getelementptr inbounds i8, i8* %0, i64 %index %5 = bitcast i8* %4 to <32 x i8>* %wide.load = load <32 x i8>, <32 x i8>* %5, align 1 %6 = sext <32 x i8> %wide.load to <32 x i32> %7 = getelementptr inbounds i8, i8* %1, i64 %index %8 = bitcast i8* %7 to <32 x i8>* %wide.load14 = load <32 x i8>, <32 x i8>* %8, align 1 %9 = sext <32 x i8> %wide.load14 to <32 x i32> %10 = mul nsw <32 x i32> %9, %6 %11 = add nsw <32 x i32> %10, %vec.phi %index.next = add i64 %index, 32 %12 = icmp eq i64 %index.next, %3 br i1 %12, label %middle.block, label %vector.body middle.block: %rdx.shuf1 = shufflevector <32 x i32> %11, <32 x i32> undef, <32 x i32> %bin.rdx1 = add <32 x i32> %11, %rdx.shuf1 %rdx.shuf = shufflevector <32 x i32> %bin.rdx1, <32 x i32> undef, <32 x i32> %bin.rdx = add <32 x i32> %bin.rdx1, %rdx.shuf %rdx.shuf15 = shufflevector <32 x i32> %bin.rdx, <32 x i32> undef, <32 x i32> %bin.rdx32 = add <32 x i32> %bin.rdx, %rdx.shuf15 %rdx.shuf17 = shufflevector <32 x i32> %bin.rdx32, <32 x i32> undef, <32 x i32> %bin.rdx18 = add <32 x i32> %bin.rdx32, %rdx.shuf17 %rdx.shuf19 = shufflevector <32 x i32> %bin.rdx18, <32 x i32> undef, <32 x i32> %bin.rdx20 = add <32 x i32> %bin.rdx18, %rdx.shuf19 %13 = extractelement <32 x i32> %bin.rdx20, i32 0 ret i32 %13 } define dso_local i32 @_Z9test_charPcS_i_512(i8* nocapture readonly, i8* nocapture readonly, i32) "min-legal-vector-width"="512" { ; CHECK-LABEL: _Z9test_charPcS_i_512: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: movl %edx, %eax ; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0 ; CHECK-NEXT: xorl %ecx, %ecx ; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; CHECK-NEXT: .p2align 4, 0x90 ; CHECK-NEXT: .LBB9_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vpmovsxbw (%rdi,%rcx), %zmm2 ; CHECK-NEXT: vpmovsxbw (%rsi,%rcx), %zmm3 ; CHECK-NEXT: vpmaddwd %zmm2, %zmm3, %zmm2 ; CHECK-NEXT: vpaddd %zmm1, %zmm2, %zmm1 ; CHECK-NEXT: addq $32, %rcx ; CHECK-NEXT: cmpq %rcx, %rax ; CHECK-NEXT: jne .LBB9_1 ; CHECK-NEXT: # %bb.2: # %middle.block ; CHECK-NEXT: vpaddd %zmm0, %zmm1, %zmm0 ; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm1 ; CHECK-NEXT: vpaddd %zmm1, %zmm0, %zmm0 ; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm1 ; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1] ; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: vmovd %xmm0, %eax ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq entry: %3 = zext i32 %2 to i64 br label %vector.body vector.body: %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ] %vec.phi = phi <32 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ] %4 = getelementptr inbounds i8, i8* %0, i64 %index %5 = bitcast i8* %4 to <32 x i8>* %wide.load = load <32 x i8>, <32 x i8>* %5, align 1 %6 = sext <32 x i8> %wide.load to <32 x i32> %7 = getelementptr inbounds i8, i8* %1, i64 %index %8 = bitcast i8* %7 to <32 x i8>* %wide.load14 = load <32 x i8>, <32 x i8>* %8, align 1 %9 = sext <32 x i8> %wide.load14 to <32 x i32> %10 = mul nsw <32 x i32> %9, %6 %11 = add nsw <32 x i32> %10, %vec.phi %index.next = add i64 %index, 32 %12 = icmp eq i64 %index.next, %3 br i1 %12, label %middle.block, label %vector.body middle.block: %rdx.shuf1 = shufflevector <32 x i32> %11, <32 x i32> undef, <32 x i32> %bin.rdx1 = add <32 x i32> %11, %rdx.shuf1 %rdx.shuf = shufflevector <32 x i32> %bin.rdx1, <32 x i32> undef, <32 x i32> %bin.rdx = add <32 x i32> %bin.rdx1, %rdx.shuf %rdx.shuf15 = shufflevector <32 x i32> %bin.rdx, <32 x i32> undef, <32 x i32> %bin.rdx32 = add <32 x i32> %bin.rdx, %rdx.shuf15 %rdx.shuf17 = shufflevector <32 x i32> %bin.rdx32, <32 x i32> undef, <32 x i32> %bin.rdx18 = add <32 x i32> %bin.rdx32, %rdx.shuf17 %rdx.shuf19 = shufflevector <32 x i32> %bin.rdx18, <32 x i32> undef, <32 x i32> %bin.rdx20 = add <32 x i32> %bin.rdx18, %rdx.shuf19 %13 = extractelement <32 x i32> %bin.rdx20, i32 0 ret i32 %13 } @a = dso_local global [1024 x i8] zeroinitializer, align 16 @b = dso_local global [1024 x i8] zeroinitializer, align 16 define dso_local i32 @sad_16i8_256() "min-legal-vector-width"="256" { ; CHECK-LABEL: sad_16i8_256: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0 ; CHECK-NEXT: movq $-1024, %rax # imm = 0xFC00 ; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; CHECK-NEXT: .p2align 4, 0x90 ; CHECK-NEXT: .LBB10_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vmovdqu a+1024(%rax), %xmm2 ; CHECK-NEXT: vpsadbw b+1024(%rax), %xmm2, %xmm2 ; CHECK-NEXT: vpaddd %ymm1, %ymm2, %ymm1 ; CHECK-NEXT: addq $4, %rax ; CHECK-NEXT: jne .LBB10_1 ; CHECK-NEXT: # %bb.2: # %middle.block ; CHECK-NEXT: vpaddd %ymm0, %ymm1, %ymm0 ; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm1 ; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1] ; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: vmovd %xmm0, %eax ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq entry: br label %vector.body vector.body: %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %10, %vector.body ] %0 = getelementptr inbounds [1024 x i8], [1024 x i8]* @a, i64 0, i64 %index %1 = bitcast i8* %0 to <16 x i8>* %wide.load = load <16 x i8>, <16 x i8>* %1, align 4 %2 = zext <16 x i8> %wide.load to <16 x i32> %3 = getelementptr inbounds [1024 x i8], [1024 x i8]* @b, i64 0, i64 %index %4 = bitcast i8* %3 to <16 x i8>* %wide.load1 = load <16 x i8>, <16 x i8>* %4, align 4 %5 = zext <16 x i8> %wide.load1 to <16 x i32> %6 = sub nsw <16 x i32> %2, %5 %7 = icmp sgt <16 x i32> %6, %8 = sub nsw <16 x i32> zeroinitializer, %6 %9 = select <16 x i1> %7, <16 x i32> %6, <16 x i32> %8 %10 = add nsw <16 x i32> %9, %vec.phi %index.next = add i64 %index, 4 %11 = icmp eq i64 %index.next, 1024 br i1 %11, label %middle.block, label %vector.body middle.block: %rdx.shuf = shufflevector <16 x i32> %10, <16 x i32> undef, <16 x i32> %bin.rdx = add <16 x i32> %10, %rdx.shuf %rdx.shuf2 = shufflevector <16 x i32> %bin.rdx, <16 x i32> undef, <16 x i32> %bin.rdx2 = add <16 x i32> %bin.rdx, %rdx.shuf2 %rdx.shuf3 = shufflevector <16 x i32> %bin.rdx2, <16 x i32> undef, <16 x i32> %bin.rdx3 = add <16 x i32> %bin.rdx2, %rdx.shuf3 %rdx.shuf4 = shufflevector <16 x i32> %bin.rdx3, <16 x i32> undef, <16 x i32> %bin.rdx4 = add <16 x i32> %bin.rdx3, %rdx.shuf4 %12 = extractelement <16 x i32> %bin.rdx4, i32 0 ret i32 %12 } define dso_local i32 @sad_16i8_512() "min-legal-vector-width"="512" { ; CHECK-LABEL: sad_16i8_512: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0 ; CHECK-NEXT: movq $-1024, %rax # imm = 0xFC00 ; CHECK-NEXT: .p2align 4, 0x90 ; CHECK-NEXT: .LBB11_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vmovdqu a+1024(%rax), %xmm1 ; CHECK-NEXT: vpsadbw b+1024(%rax), %xmm1, %xmm1 ; CHECK-NEXT: vpaddd %zmm0, %zmm1, %zmm0 ; CHECK-NEXT: addq $4, %rax ; CHECK-NEXT: jne .LBB11_1 ; CHECK-NEXT: # %bb.2: # %middle.block ; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm1 ; CHECK-NEXT: vpaddd %zmm1, %zmm0, %zmm0 ; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm1 ; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1] ; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: vmovd %xmm0, %eax ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq entry: br label %vector.body vector.body: %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ] %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %10, %vector.body ] %0 = getelementptr inbounds [1024 x i8], [1024 x i8]* @a, i64 0, i64 %index %1 = bitcast i8* %0 to <16 x i8>* %wide.load = load <16 x i8>, <16 x i8>* %1, align 4 %2 = zext <16 x i8> %wide.load to <16 x i32> %3 = getelementptr inbounds [1024 x i8], [1024 x i8]* @b, i64 0, i64 %index %4 = bitcast i8* %3 to <16 x i8>* %wide.load1 = load <16 x i8>, <16 x i8>* %4, align 4 %5 = zext <16 x i8> %wide.load1 to <16 x i32> %6 = sub nsw <16 x i32> %2, %5 %7 = icmp sgt <16 x i32> %6, %8 = sub nsw <16 x i32> zeroinitializer, %6 %9 = select <16 x i1> %7, <16 x i32> %6, <16 x i32> %8 %10 = add nsw <16 x i32> %9, %vec.phi %index.next = add i64 %index, 4 %11 = icmp eq i64 %index.next, 1024 br i1 %11, label %middle.block, label %vector.body middle.block: %rdx.shuf = shufflevector <16 x i32> %10, <16 x i32> undef, <16 x i32> %bin.rdx = add <16 x i32> %10, %rdx.shuf %rdx.shuf2 = shufflevector <16 x i32> %bin.rdx, <16 x i32> undef, <16 x i32> %bin.rdx2 = add <16 x i32> %bin.rdx, %rdx.shuf2 %rdx.shuf3 = shufflevector <16 x i32> %bin.rdx2, <16 x i32> undef, <16 x i32> %bin.rdx3 = add <16 x i32> %bin.rdx2, %rdx.shuf3 %rdx.shuf4 = shufflevector <16 x i32> %bin.rdx3, <16 x i32> undef, <16 x i32> %bin.rdx4 = add <16 x i32> %bin.rdx3, %rdx.shuf4 %12 = extractelement <16 x i32> %bin.rdx4, i32 0 ret i32 %12 } define dso_local void @sbto16f32_256(<16 x i16> %a, <16 x float>* %res) "min-legal-vector-width"="256" { ; CHECK-LABEL: sbto16f32_256: ; CHECK: # %bb.0: ; CHECK-NEXT: vpmovw2m %ymm0, %k0 ; CHECK-NEXT: kshiftrw $8, %k0, %k1 ; CHECK-NEXT: vpmovm2d %k1, %ymm0 ; CHECK-NEXT: vcvtdq2ps %ymm0, %ymm0 ; CHECK-NEXT: vpmovm2d %k0, %ymm1 ; CHECK-NEXT: vcvtdq2ps %ymm1, %ymm1 ; CHECK-NEXT: vmovaps %ymm1, (%rdi) ; CHECK-NEXT: vmovaps %ymm0, 32(%rdi) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %mask = icmp slt <16 x i16> %a, zeroinitializer %1 = sitofp <16 x i1> %mask to <16 x float> store <16 x float> %1, <16 x float>* %res ret void } define dso_local void @sbto16f32_512(<16 x i16> %a, <16 x float>* %res) "min-legal-vector-width"="512" { ; CHECK-LABEL: sbto16f32_512: ; CHECK: # %bb.0: ; CHECK-NEXT: vpmovw2m %ymm0, %k0 ; CHECK-NEXT: vpmovm2d %k0, %zmm0 ; CHECK-NEXT: vcvtdq2ps %zmm0, %zmm0 ; CHECK-NEXT: vmovaps %zmm0, (%rdi) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %mask = icmp slt <16 x i16> %a, zeroinitializer %1 = sitofp <16 x i1> %mask to <16 x float> store <16 x float> %1, <16 x float>* %res ret void } define dso_local void @sbto16f64_256(<16 x i16> %a, <16 x double>* %res) "min-legal-vector-width"="256" { ; CHECK-LABEL: sbto16f64_256: ; CHECK: # %bb.0: ; CHECK-NEXT: vpmovw2m %ymm0, %k0 ; CHECK-NEXT: kshiftrw $8, %k0, %k1 ; CHECK-NEXT: vpmovm2d %k1, %ymm0 ; CHECK-NEXT: vcvtdq2pd %xmm0, %ymm1 ; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm0 ; CHECK-NEXT: vcvtdq2pd %xmm0, %ymm0 ; CHECK-NEXT: vpmovm2d %k0, %ymm2 ; CHECK-NEXT: vcvtdq2pd %xmm2, %ymm3 ; CHECK-NEXT: vextracti128 $1, %ymm2, %xmm2 ; CHECK-NEXT: vcvtdq2pd %xmm2, %ymm2 ; CHECK-NEXT: vmovaps %ymm2, 32(%rdi) ; CHECK-NEXT: vmovaps %ymm3, (%rdi) ; CHECK-NEXT: vmovaps %ymm0, 96(%rdi) ; CHECK-NEXT: vmovaps %ymm1, 64(%rdi) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %mask = icmp slt <16 x i16> %a, zeroinitializer %1 = sitofp <16 x i1> %mask to <16 x double> store <16 x double> %1, <16 x double>* %res ret void } define dso_local void @sbto16f64_512(<16 x i16> %a, <16 x double>* %res) "min-legal-vector-width"="512" { ; CHECK-LABEL: sbto16f64_512: ; CHECK: # %bb.0: ; CHECK-NEXT: vpmovw2m %ymm0, %k0 ; CHECK-NEXT: vpmovm2d %k0, %zmm0 ; CHECK-NEXT: vcvtdq2pd %ymm0, %zmm1 ; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm0 ; CHECK-NEXT: vcvtdq2pd %ymm0, %zmm0 ; CHECK-NEXT: vmovaps %zmm0, 64(%rdi) ; CHECK-NEXT: vmovaps %zmm1, (%rdi) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %mask = icmp slt <16 x i16> %a, zeroinitializer %1 = sitofp <16 x i1> %mask to <16 x double> store <16 x double> %1, <16 x double>* %res ret void } define dso_local void @ubto16f32_256(<16 x i16> %a, <16 x float>* %res) "min-legal-vector-width"="256" { ; CHECK-LABEL: ubto16f32_256: ; CHECK: # %bb.0: ; CHECK-NEXT: vpmovw2m %ymm0, %k0 ; CHECK-NEXT: kshiftrw $8, %k0, %k1 ; CHECK-NEXT: vpmovm2d %k1, %ymm0 ; CHECK-NEXT: vpsrld $31, %ymm0, %ymm0 ; CHECK-NEXT: vcvtdq2ps %ymm0, %ymm0 ; CHECK-NEXT: vpmovm2d %k0, %ymm1 ; CHECK-NEXT: vpsrld $31, %ymm1, %ymm1 ; CHECK-NEXT: vcvtdq2ps %ymm1, %ymm1 ; CHECK-NEXT: vmovaps %ymm1, (%rdi) ; CHECK-NEXT: vmovaps %ymm0, 32(%rdi) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %mask = icmp slt <16 x i16> %a, zeroinitializer %1 = uitofp <16 x i1> %mask to <16 x float> store <16 x float> %1, <16 x float>* %res ret void } define dso_local void @ubto16f32_512(<16 x i16> %a, <16 x float>* %res) "min-legal-vector-width"="512" { ; CHECK-LABEL: ubto16f32_512: ; CHECK: # %bb.0: ; CHECK-NEXT: vpmovw2m %ymm0, %k0 ; CHECK-NEXT: vpmovm2d %k0, %zmm0 ; CHECK-NEXT: vpsrld $31, %zmm0, %zmm0 ; CHECK-NEXT: vcvtdq2ps %zmm0, %zmm0 ; CHECK-NEXT: vmovaps %zmm0, (%rdi) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %mask = icmp slt <16 x i16> %a, zeroinitializer %1 = uitofp <16 x i1> %mask to <16 x float> store <16 x float> %1, <16 x float>* %res ret void } define dso_local void @ubto16f64_256(<16 x i16> %a, <16 x double>* %res) "min-legal-vector-width"="256" { ; CHECK-LABEL: ubto16f64_256: ; CHECK: # %bb.0: ; CHECK-NEXT: vpmovw2m %ymm0, %k0 ; CHECK-NEXT: kshiftrw $8, %k0, %k1 ; CHECK-NEXT: vpmovm2d %k1, %ymm0 ; CHECK-NEXT: vpsrld $31, %ymm0, %ymm0 ; CHECK-NEXT: vcvtdq2pd %xmm0, %ymm1 ; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm0 ; CHECK-NEXT: vcvtdq2pd %xmm0, %ymm0 ; CHECK-NEXT: vpmovm2d %k0, %ymm2 ; CHECK-NEXT: vpsrld $31, %ymm2, %ymm2 ; CHECK-NEXT: vcvtdq2pd %xmm2, %ymm3 ; CHECK-NEXT: vextracti128 $1, %ymm2, %xmm2 ; CHECK-NEXT: vcvtdq2pd %xmm2, %ymm2 ; CHECK-NEXT: vmovaps %ymm2, 32(%rdi) ; CHECK-NEXT: vmovaps %ymm3, (%rdi) ; CHECK-NEXT: vmovaps %ymm0, 96(%rdi) ; CHECK-NEXT: vmovaps %ymm1, 64(%rdi) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %mask = icmp slt <16 x i16> %a, zeroinitializer %1 = uitofp <16 x i1> %mask to <16 x double> store <16 x double> %1, <16 x double>* %res ret void } define dso_local void @ubto16f64_512(<16 x i16> %a, <16 x double>* %res) "min-legal-vector-width"="512" { ; CHECK-LABEL: ubto16f64_512: ; CHECK: # %bb.0: ; CHECK-NEXT: vpmovw2m %ymm0, %k0 ; CHECK-NEXT: vpmovm2d %k0, %zmm0 ; CHECK-NEXT: vpsrld $31, %zmm0, %zmm0 ; CHECK-NEXT: vcvtdq2pd %ymm0, %zmm1 ; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm0 ; CHECK-NEXT: vcvtdq2pd %ymm0, %zmm0 ; CHECK-NEXT: vmovaps %zmm0, 64(%rdi) ; CHECK-NEXT: vmovaps %zmm1, (%rdi) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %mask = icmp slt <16 x i16> %a, zeroinitializer %1 = uitofp <16 x i1> %mask to <16 x double> store <16 x double> %1, <16 x double>* %res ret void } define <16 x i16> @test_16f32toub_256(<16 x float>* %ptr, <16 x i16> %passthru) "min-legal-vector-width"="256" { ; CHECK-LABEL: test_16f32toub_256: ; CHECK: # %bb.0: ; CHECK-NEXT: vcvttps2dq (%rdi), %ymm1 ; CHECK-NEXT: vpslld $31, %ymm1, %ymm1 ; CHECK-NEXT: vpmovd2m %ymm1, %k0 ; CHECK-NEXT: vcvttps2dq 32(%rdi), %ymm1 ; CHECK-NEXT: vpslld $31, %ymm1, %ymm1 ; CHECK-NEXT: vpmovd2m %ymm1, %k1 ; CHECK-NEXT: kunpckbw %k0, %k1, %k1 ; CHECK-NEXT: vmovdqu16 %ymm0, %ymm0 {%k1} {z} ; CHECK-NEXT: retq %a = load <16 x float>, <16 x float>* %ptr %mask = fptoui <16 x float> %a to <16 x i1> %select = select <16 x i1> %mask, <16 x i16> %passthru, <16 x i16> zeroinitializer ret <16 x i16> %select } define <16 x i16> @test_16f32toub_512(<16 x float>* %ptr, <16 x i16> %passthru) "min-legal-vector-width"="512" { ; CHECK-LABEL: test_16f32toub_512: ; CHECK: # %bb.0: ; CHECK-NEXT: vcvttps2dq (%rdi), %zmm1 ; CHECK-NEXT: vpslld $31, %zmm1, %zmm1 ; CHECK-NEXT: vpmovd2m %zmm1, %k1 ; CHECK-NEXT: vmovdqu16 %ymm0, %ymm0 {%k1} {z} ; CHECK-NEXT: retq %a = load <16 x float>, <16 x float>* %ptr %mask = fptoui <16 x float> %a to <16 x i1> %select = select <16 x i1> %mask, <16 x i16> %passthru, <16 x i16> zeroinitializer ret <16 x i16> %select } define <16 x i16> @test_16f32tosb_256(<16 x float>* %ptr, <16 x i16> %passthru) "min-legal-vector-width"="256" { ; CHECK-LABEL: test_16f32tosb_256: ; CHECK: # %bb.0: ; CHECK-NEXT: vcvttps2dq (%rdi), %ymm1 ; CHECK-NEXT: vpmovd2m %ymm1, %k0 ; CHECK-NEXT: vcvttps2dq 32(%rdi), %ymm1 ; CHECK-NEXT: vpmovd2m %ymm1, %k1 ; CHECK-NEXT: kunpckbw %k0, %k1, %k1 ; CHECK-NEXT: vmovdqu16 %ymm0, %ymm0 {%k1} {z} ; CHECK-NEXT: retq %a = load <16 x float>, <16 x float>* %ptr %mask = fptosi <16 x float> %a to <16 x i1> %select = select <16 x i1> %mask, <16 x i16> %passthru, <16 x i16> zeroinitializer ret <16 x i16> %select } define <16 x i16> @test_16f32tosb_512(<16 x float>* %ptr, <16 x i16> %passthru) "min-legal-vector-width"="512" { ; CHECK-LABEL: test_16f32tosb_512: ; CHECK: # %bb.0: ; CHECK-NEXT: vcvttps2dq (%rdi), %zmm1 ; CHECK-NEXT: vpmovd2m %zmm1, %k1 ; CHECK-NEXT: vmovdqu16 %ymm0, %ymm0 {%k1} {z} ; CHECK-NEXT: retq %a = load <16 x float>, <16 x float>* %ptr %mask = fptosi <16 x float> %a to <16 x i1> %select = select <16 x i1> %mask, <16 x i16> %passthru, <16 x i16> zeroinitializer ret <16 x i16> %select } define dso_local void @mul256(<64 x i8>* %a, <64 x i8>* %b, <64 x i8>* %c) "min-legal-vector-width"="256" { ; CHECK-AVX512-LABEL: mul256: ; CHECK-AVX512: # %bb.0: ; CHECK-AVX512-NEXT: vmovdqa (%rdi), %ymm0 ; CHECK-AVX512-NEXT: vmovdqa 32(%rdi), %ymm1 ; CHECK-AVX512-NEXT: vmovdqa (%rsi), %ymm2 ; CHECK-AVX512-NEXT: vmovdqa 32(%rsi), %ymm3 ; CHECK-AVX512-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] ; CHECK-AVX512-NEXT: vpunpckhbw {{.*#+}} ymm5 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] ; CHECK-AVX512-NEXT: vpmullw %ymm4, %ymm5, %ymm4 ; CHECK-AVX512-NEXT: vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] ; CHECK-AVX512-NEXT: vpand %ymm5, %ymm4, %ymm4 ; CHECK-AVX512-NEXT: vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] ; CHECK-AVX512-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] ; CHECK-AVX512-NEXT: vpmullw %ymm3, %ymm1, %ymm1 ; CHECK-AVX512-NEXT: vpand %ymm5, %ymm1, %ymm1 ; CHECK-AVX512-NEXT: vpackuswb %ymm4, %ymm1, %ymm1 ; CHECK-AVX512-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] ; CHECK-AVX512-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] ; CHECK-AVX512-NEXT: vpmullw %ymm3, %ymm4, %ymm3 ; CHECK-AVX512-NEXT: vpand %ymm5, %ymm3, %ymm3 ; CHECK-AVX512-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] ; CHECK-AVX512-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] ; CHECK-AVX512-NEXT: vpmullw %ymm2, %ymm0, %ymm0 ; CHECK-AVX512-NEXT: vpand %ymm5, %ymm0, %ymm0 ; CHECK-AVX512-NEXT: vpackuswb %ymm3, %ymm0, %ymm0 ; CHECK-AVX512-NEXT: vmovdqa %ymm0, (%rdx) ; CHECK-AVX512-NEXT: vmovdqa %ymm1, 32(%rdx) ; CHECK-AVX512-NEXT: vzeroupper ; CHECK-AVX512-NEXT: retq ; ; CHECK-VBMI-LABEL: mul256: ; CHECK-VBMI: # %bb.0: ; CHECK-VBMI-NEXT: vmovdqa (%rdi), %ymm0 ; CHECK-VBMI-NEXT: vmovdqa 32(%rdi), %ymm1 ; CHECK-VBMI-NEXT: vmovdqa (%rsi), %ymm2 ; CHECK-VBMI-NEXT: vmovdqa 32(%rsi), %ymm3 ; CHECK-VBMI-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] ; CHECK-VBMI-NEXT: vpunpckhbw {{.*#+}} ymm5 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] ; CHECK-VBMI-NEXT: vpmullw %ymm4, %ymm5, %ymm4 ; CHECK-VBMI-NEXT: vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] ; CHECK-VBMI-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] ; CHECK-VBMI-NEXT: vpmullw %ymm3, %ymm1, %ymm1 ; CHECK-VBMI-NEXT: vmovdqa {{.*#+}} ymm3 = [0,2,4,6,8,10,12,14,32,34,36,38,40,42,44,46,16,18,20,22,24,26,28,30,48,50,52,54,56,58,60,62] ; CHECK-VBMI-NEXT: vpermt2b %ymm4, %ymm3, %ymm1 ; CHECK-VBMI-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] ; CHECK-VBMI-NEXT: vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] ; CHECK-VBMI-NEXT: vpmullw %ymm4, %ymm5, %ymm4 ; CHECK-VBMI-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] ; CHECK-VBMI-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23] ; CHECK-VBMI-NEXT: vpmullw %ymm2, %ymm0, %ymm0 ; CHECK-VBMI-NEXT: vpermt2b %ymm4, %ymm3, %ymm0 ; CHECK-VBMI-NEXT: vmovdqa %ymm0, (%rdx) ; CHECK-VBMI-NEXT: vmovdqa %ymm1, 32(%rdx) ; CHECK-VBMI-NEXT: vzeroupper ; CHECK-VBMI-NEXT: retq %d = load <64 x i8>, <64 x i8>* %a %e = load <64 x i8>, <64 x i8>* %b %f = mul <64 x i8> %d, %e store <64 x i8> %f, <64 x i8>* %c ret void } define dso_local void @mul512(<64 x i8>* %a, <64 x i8>* %b, <64 x i8>* %c) "min-legal-vector-width"="512" { ; CHECK-AVX512-LABEL: mul512: ; CHECK-AVX512: # %bb.0: ; CHECK-AVX512-NEXT: vmovdqa64 (%rdi), %zmm0 ; CHECK-AVX512-NEXT: vmovdqa64 (%rsi), %zmm1 ; CHECK-AVX512-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63] ; CHECK-AVX512-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63] ; CHECK-AVX512-NEXT: vpmullw %zmm2, %zmm3, %zmm2 ; CHECK-AVX512-NEXT: vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] ; CHECK-AVX512-NEXT: vpandq %zmm3, %zmm2, %zmm2 ; CHECK-AVX512-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55] ; CHECK-AVX512-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55] ; CHECK-AVX512-NEXT: vpmullw %zmm1, %zmm0, %zmm0 ; CHECK-AVX512-NEXT: vpandq %zmm3, %zmm0, %zmm0 ; CHECK-AVX512-NEXT: vpackuswb %zmm2, %zmm0, %zmm0 ; CHECK-AVX512-NEXT: vmovdqa64 %zmm0, (%rdx) ; CHECK-AVX512-NEXT: vzeroupper ; CHECK-AVX512-NEXT: retq ; ; CHECK-VBMI-LABEL: mul512: ; CHECK-VBMI: # %bb.0: ; CHECK-VBMI-NEXT: vmovdqa64 (%rdi), %zmm0 ; CHECK-VBMI-NEXT: vmovdqa64 (%rsi), %zmm1 ; CHECK-VBMI-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63] ; CHECK-VBMI-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63] ; CHECK-VBMI-NEXT: vpmullw %zmm2, %zmm3, %zmm2 ; CHECK-VBMI-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55] ; CHECK-VBMI-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55] ; CHECK-VBMI-NEXT: vpmullw %zmm1, %zmm0, %zmm0 ; CHECK-VBMI-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,2,4,6,8,10,12,14,64,66,68,70,72,74,76,78,16,18,20,22,24,26,28,30,80,82,84,86,88,90,92,94,32,34,36,38,40,42,44,46,96,98,100,102,104,106,108,110,48,50,52,54,56,58,60,62,112,114,116,118,120,122,124,126] ; CHECK-VBMI-NEXT: vpermi2b %zmm2, %zmm0, %zmm1 ; CHECK-VBMI-NEXT: vmovdqa64 %zmm1, (%rdx) ; CHECK-VBMI-NEXT: vzeroupper ; CHECK-VBMI-NEXT: retq %d = load <64 x i8>, <64 x i8>* %a %e = load <64 x i8>, <64 x i8>* %b %f = mul <64 x i8> %d, %e store <64 x i8> %f, <64 x i8>* %c ret void } ; This threw an assertion at one point. define <4 x i32> @mload_v4i32(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i32> %dst) "min-legal-vector-width"="256" { ; CHECK-LABEL: mload_v4i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vptestnmd %xmm0, %xmm0, %k1 ; CHECK-NEXT: vpblendmd (%rdi), %xmm1, %xmm0 {%k1} ; CHECK-NEXT: retq %mask = icmp eq <4 x i32> %trigger, zeroinitializer %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1> %mask, <4 x i32> %dst) ret <4 x i32> %res } declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32, <4 x i1>, <4 x i32>) define <16 x i32> @trunc_v16i64_v16i32(<16 x i64>* %x) nounwind "min-legal-vector-width"="256" { ; CHECK-LABEL: trunc_v16i64_v16i32: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa (%rdi), %ymm0 ; CHECK-NEXT: vmovdqa 32(%rdi), %ymm1 ; CHECK-NEXT: vmovdqa 64(%rdi), %ymm2 ; CHECK-NEXT: vmovdqa 96(%rdi), %ymm3 ; CHECK-NEXT: vpmovqd %ymm0, %xmm0 ; CHECK-NEXT: vpmovqd %ymm1, %xmm1 ; CHECK-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; CHECK-NEXT: vpmovqd %ymm2, %xmm1 ; CHECK-NEXT: vpmovqd %ymm3, %xmm2 ; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 ; CHECK-NEXT: retq %a = load <16 x i64>, <16 x i64>* %x %b = trunc <16 x i64> %a to <16 x i32> ret <16 x i32> %b } define <16 x i8> @trunc_v16i64_v16i8(<16 x i64>* %x) nounwind "min-legal-vector-width"="256" { ; CHECK-LABEL: trunc_v16i64_v16i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa (%rdi), %ymm0 ; CHECK-NEXT: vmovdqa 32(%rdi), %ymm1 ; CHECK-NEXT: vmovdqa 64(%rdi), %ymm2 ; CHECK-NEXT: vmovdqa 96(%rdi), %ymm3 ; CHECK-NEXT: vpmovqb %ymm3, %xmm3 ; CHECK-NEXT: vpmovqb %ymm2, %xmm2 ; CHECK-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] ; CHECK-NEXT: vpmovqb %ymm1, %xmm1 ; CHECK-NEXT: vpmovqb %ymm0, %xmm0 ; CHECK-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %a = load <16 x i64>, <16 x i64>* %x %b = trunc <16 x i64> %a to <16 x i8> ret <16 x i8> %b } define <16 x i8> @trunc_v16i32_v16i8(<16 x i32>* %x) nounwind "min-legal-vector-width"="256" { ; CHECK-LABEL: trunc_v16i32_v16i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa (%rdi), %ymm0 ; CHECK-NEXT: vmovdqa 32(%rdi), %ymm1 ; CHECK-NEXT: vpmovdb %ymm1, %xmm1 ; CHECK-NEXT: vpmovdb %ymm0, %xmm0 ; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %a = load <16 x i32>, <16 x i32>* %x %b = trunc <16 x i32> %a to <16 x i8> ret <16 x i8> %b } define <8 x i8> @trunc_v8i64_v8i8(<8 x i64>* %x) nounwind "min-legal-vector-width"="256" { ; CHECK-LABEL: trunc_v8i64_v8i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa (%rdi), %ymm0 ; CHECK-NEXT: vmovdqa 32(%rdi), %ymm1 ; CHECK-NEXT: vpmovqb %ymm1, %xmm1 ; CHECK-NEXT: vpmovqb %ymm0, %xmm0 ; CHECK-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %a = load <8 x i64>, <8 x i64>* %x %b = trunc <8 x i64> %a to <8 x i8> ret <8 x i8> %b } define <8 x i16> @trunc_v8i64_v8i16(<8 x i64>* %x) nounwind "min-legal-vector-width"="256" { ; CHECK-LABEL: trunc_v8i64_v8i16: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa (%rdi), %ymm0 ; CHECK-NEXT: vmovdqa 32(%rdi), %ymm1 ; CHECK-NEXT: vpmovqw %ymm1, %xmm1 ; CHECK-NEXT: vpmovqw %ymm0, %xmm0 ; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %a = load <8 x i64>, <8 x i64>* %x %b = trunc <8 x i64> %a to <8 x i16> ret <8 x i16> %b } define <8 x i32> @trunc_v8i64_v8i32_zeroes(<8 x i64>* %x) nounwind "min-legal-vector-width"="256" { ; CHECK-LABEL: trunc_v8i64_v8i32_zeroes: ; CHECK: # %bb.0: ; CHECK-NEXT: vpsrlq $48, 32(%rdi), %ymm0 ; CHECK-NEXT: vpsrlq $48, (%rdi), %ymm1 ; CHECK-NEXT: vpackusdw %ymm0, %ymm1, %ymm0 ; CHECK-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3] ; CHECK-NEXT: retq %a = load <8 x i64>, <8 x i64>* %x %b = lshr <8 x i64> %a, %c = trunc <8 x i64> %b to <8 x i32> ret <8 x i32> %c } define <16 x i16> @trunc_v16i32_v16i16_zeroes(<16 x i32>* %x) nounwind "min-legal-vector-width"="256" { ; CHECK-LABEL: trunc_v16i32_v16i16_zeroes: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa (%rdi), %ymm1 ; CHECK-NEXT: vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31] ; CHECK-NEXT: vpermi2w 32(%rdi), %ymm1, %ymm0 ; CHECK-NEXT: retq %a = load <16 x i32>, <16 x i32>* %x %b = lshr <16 x i32> %a, %c = trunc <16 x i32> %b to <16 x i16> ret <16 x i16> %c } define <32 x i8> @trunc_v32i16_v32i8_zeroes(<32 x i16>* %x) nounwind "min-legal-vector-width"="256" { ; CHECK-AVX512-LABEL: trunc_v32i16_v32i8_zeroes: ; CHECK-AVX512: # %bb.0: ; CHECK-AVX512-NEXT: vpsrlw $8, 32(%rdi), %ymm0 ; CHECK-AVX512-NEXT: vpsrlw $8, (%rdi), %ymm1 ; CHECK-AVX512-NEXT: vpackuswb %ymm0, %ymm1, %ymm0 ; CHECK-AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3] ; CHECK-AVX512-NEXT: retq ; ; CHECK-VBMI-LABEL: trunc_v32i16_v32i8_zeroes: ; CHECK-VBMI: # %bb.0: ; CHECK-VBMI-NEXT: vmovdqa (%rdi), %ymm1 ; CHECK-VBMI-NEXT: vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63] ; CHECK-VBMI-NEXT: vpermi2b 32(%rdi), %ymm1, %ymm0 ; CHECK-VBMI-NEXT: retq %a = load <32 x i16>, <32 x i16>* %x %b = lshr <32 x i16> %a, %c = trunc <32 x i16> %b to <32 x i8> ret <32 x i8> %c } define <8 x i32> @trunc_v8i64_v8i32_sign(<8 x i64>* %x) nounwind "min-legal-vector-width"="256" { ; CHECK-LABEL: trunc_v8i64_v8i32_sign: ; CHECK: # %bb.0: ; CHECK-NEXT: vpsraq $48, 32(%rdi), %ymm0 ; CHECK-NEXT: vpsraq $48, (%rdi), %ymm1 ; CHECK-NEXT: vpmovqd %ymm1, %xmm1 ; CHECK-NEXT: vpmovqd %ymm0, %xmm0 ; CHECK-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 ; CHECK-NEXT: retq %a = load <8 x i64>, <8 x i64>* %x %b = ashr <8 x i64> %a, %c = trunc <8 x i64> %b to <8 x i32> ret <8 x i32> %c } define <16 x i16> @trunc_v16i32_v16i16_sign(<16 x i32>* %x) nounwind "min-legal-vector-width"="256" { ; CHECK-LABEL: trunc_v16i32_v16i16_sign: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa (%rdi), %ymm1 ; CHECK-NEXT: vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31] ; CHECK-NEXT: vpermi2w 32(%rdi), %ymm1, %ymm0 ; CHECK-NEXT: retq %a = load <16 x i32>, <16 x i32>* %x %b = ashr <16 x i32> %a, %c = trunc <16 x i32> %b to <16 x i16> ret <16 x i16> %c } define <32 x i8> @trunc_v32i16_v32i8_sign(<32 x i16>* %x) nounwind "min-legal-vector-width"="256" { ; CHECK-AVX512-LABEL: trunc_v32i16_v32i8_sign: ; CHECK-AVX512: # %bb.0: ; CHECK-AVX512-NEXT: vpsrlw $8, 32(%rdi), %ymm0 ; CHECK-AVX512-NEXT: vpsrlw $8, (%rdi), %ymm1 ; CHECK-AVX512-NEXT: vpackuswb %ymm0, %ymm1, %ymm0 ; CHECK-AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3] ; CHECK-AVX512-NEXT: retq ; ; CHECK-VBMI-LABEL: trunc_v32i16_v32i8_sign: ; CHECK-VBMI: # %bb.0: ; CHECK-VBMI-NEXT: vmovdqa (%rdi), %ymm1 ; CHECK-VBMI-NEXT: vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63] ; CHECK-VBMI-NEXT: vpermi2b 32(%rdi), %ymm1, %ymm0 ; CHECK-VBMI-NEXT: retq %a = load <32 x i16>, <32 x i16>* %x %b = ashr <32 x i16> %a, %c = trunc <32 x i16> %b to <32 x i8> ret <32 x i8> %c } define dso_local void @zext_v16i8_v16i64(<16 x i8> %x, <16 x i64>* %y) nounwind "min-legal-vector-width"="256" { ; CHECK-LABEL: zext_v16i8_v16i64: ; CHECK: # %bb.0: ; CHECK-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero ; CHECK-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3] ; CHECK-NEXT: vpmovzxwq {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero ; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm1 ; CHECK-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3] ; CHECK-NEXT: vpmovzxwq {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero ; CHECK-NEXT: vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero ; CHECK-NEXT: vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero ; CHECK-NEXT: vmovdqa %ymm0, (%rdi) ; CHECK-NEXT: vmovdqa %ymm1, 64(%rdi) ; CHECK-NEXT: vmovdqa %ymm3, 96(%rdi) ; CHECK-NEXT: vmovdqa %ymm2, 32(%rdi) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %a = zext <16 x i8> %x to <16 x i64> store <16 x i64> %a, <16 x i64>* %y ret void } define dso_local void @sext_v16i8_v16i64(<16 x i8> %x, <16 x i64>* %y) nounwind "min-legal-vector-width"="256" { ; CHECK-LABEL: sext_v16i8_v16i64: ; CHECK: # %bb.0: ; CHECK-NEXT: vpmovsxbw %xmm0, %ymm1 ; CHECK-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3] ; CHECK-NEXT: vpmovsxwq %xmm2, %ymm2 ; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm1 ; CHECK-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3] ; CHECK-NEXT: vpmovsxwq %xmm3, %ymm3 ; CHECK-NEXT: vpmovsxwq %xmm1, %ymm1 ; CHECK-NEXT: vpmovsxbq %xmm0, %ymm0 ; CHECK-NEXT: vmovdqa %ymm0, (%rdi) ; CHECK-NEXT: vmovdqa %ymm1, 64(%rdi) ; CHECK-NEXT: vmovdqa %ymm3, 96(%rdi) ; CHECK-NEXT: vmovdqa %ymm2, 32(%rdi) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %a = sext <16 x i8> %x to <16 x i64> store <16 x i64> %a, <16 x i64>* %y ret void } define dso_local void @vselect_split_v8i16_setcc(<8 x i16> %s, <8 x i16> %t, <8 x i64>* %p, <8 x i64>* %q, <8 x i64>* %r) "min-legal-vector-width"="256" { ; CHECK-LABEL: vselect_split_v8i16_setcc: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa (%rsi), %ymm2 ; CHECK-NEXT: vmovdqa 32(%rsi), %ymm3 ; CHECK-NEXT: vpcmpeqw %xmm1, %xmm0, %k1 ; CHECK-NEXT: kshiftrb $4, %k1, %k2 ; CHECK-NEXT: vmovdqa64 32(%rdi), %ymm3 {%k2} ; CHECK-NEXT: vmovdqa64 (%rdi), %ymm2 {%k1} ; CHECK-NEXT: vmovdqa %ymm2, (%rdx) ; CHECK-NEXT: vmovdqa %ymm3, 32(%rdx) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %x = load <8 x i64>, <8 x i64>* %p %y = load <8 x i64>, <8 x i64>* %q %a = icmp eq <8 x i16> %s, %t %b = select <8 x i1> %a, <8 x i64> %x, <8 x i64> %y store <8 x i64> %b, <8 x i64>* %r ret void } define dso_local void @vselect_split_v8i32_setcc(<8 x i32> %s, <8 x i32> %t, <8 x i64>* %p, <8 x i64>* %q, <8 x i64>* %r) "min-legal-vector-width"="256" { ; CHECK-LABEL: vselect_split_v8i32_setcc: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa (%rsi), %ymm2 ; CHECK-NEXT: vmovdqa 32(%rsi), %ymm3 ; CHECK-NEXT: vpcmpeqd %ymm1, %ymm0, %k1 ; CHECK-NEXT: kshiftrb $4, %k1, %k2 ; CHECK-NEXT: vmovdqa64 32(%rdi), %ymm3 {%k2} ; CHECK-NEXT: vmovdqa64 (%rdi), %ymm2 {%k1} ; CHECK-NEXT: vmovdqa %ymm2, (%rdx) ; CHECK-NEXT: vmovdqa %ymm3, 32(%rdx) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %x = load <8 x i64>, <8 x i64>* %p %y = load <8 x i64>, <8 x i64>* %q %a = icmp eq <8 x i32> %s, %t %b = select <8 x i1> %a, <8 x i64> %x, <8 x i64> %y store <8 x i64> %b, <8 x i64>* %r ret void } define dso_local void @vselect_split_v16i8_setcc(<16 x i8> %s, <16 x i8> %t, <16 x i32>* %p, <16 x i32>* %q, <16 x i32>* %r) "min-legal-vector-width"="256" { ; CHECK-LABEL: vselect_split_v16i8_setcc: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa (%rsi), %ymm2 ; CHECK-NEXT: vmovdqa 32(%rsi), %ymm3 ; CHECK-NEXT: vpcmpeqb %xmm1, %xmm0, %k1 ; CHECK-NEXT: kshiftrw $8, %k1, %k2 ; CHECK-NEXT: vmovdqa32 32(%rdi), %ymm3 {%k2} ; CHECK-NEXT: vmovdqa32 (%rdi), %ymm2 {%k1} ; CHECK-NEXT: vmovdqa %ymm2, (%rdx) ; CHECK-NEXT: vmovdqa %ymm3, 32(%rdx) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %x = load <16 x i32>, <16 x i32>* %p %y = load <16 x i32>, <16 x i32>* %q %a = icmp eq <16 x i8> %s, %t %b = select <16 x i1> %a, <16 x i32> %x, <16 x i32> %y store <16 x i32> %b, <16 x i32>* %r ret void } define dso_local void @vselect_split_v16i16_setcc(<16 x i16> %s, <16 x i16> %t, <16 x i32>* %p, <16 x i32>* %q, <16 x i32>* %r) "min-legal-vector-width"="256" { ; CHECK-LABEL: vselect_split_v16i16_setcc: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa (%rsi), %ymm2 ; CHECK-NEXT: vmovdqa 32(%rsi), %ymm3 ; CHECK-NEXT: vpcmpeqw %ymm1, %ymm0, %k1 ; CHECK-NEXT: kshiftrw $8, %k1, %k2 ; CHECK-NEXT: vmovdqa32 32(%rdi), %ymm3 {%k2} ; CHECK-NEXT: vmovdqa32 (%rdi), %ymm2 {%k1} ; CHECK-NEXT: vmovdqa %ymm2, (%rdx) ; CHECK-NEXT: vmovdqa %ymm3, 32(%rdx) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %x = load <16 x i32>, <16 x i32>* %p %y = load <16 x i32>, <16 x i32>* %q %a = icmp eq <16 x i16> %s, %t %b = select <16 x i1> %a, <16 x i32> %x, <16 x i32> %y store <16 x i32> %b, <16 x i32>* %r ret void } define <16 x i8> @trunc_packus_v16i32_v16i8(<16 x i32>* %p) "min-legal-vector-width"="256" { ; CHECK-LABEL: trunc_packus_v16i32_v16i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa (%rdi), %ymm0 ; CHECK-NEXT: vpackusdw 32(%rdi), %ymm0, %ymm0 ; CHECK-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3] ; CHECK-NEXT: vpmovuswb %ymm0, %xmm0 ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %a = load <16 x i32>, <16 x i32>* %p %b = icmp slt <16 x i32> %a, %c = select <16 x i1> %b, <16 x i32> %a, <16 x i32> %d = icmp sgt <16 x i32> %c, zeroinitializer %e = select <16 x i1> %d, <16 x i32> %c, <16 x i32> zeroinitializer %f = trunc <16 x i32> %e to <16 x i8> ret <16 x i8> %f } define dso_local void @trunc_packus_v16i32_v16i8_store(<16 x i32>* %p, <16 x i8>* %q) "min-legal-vector-width"="256" { ; CHECK-LABEL: trunc_packus_v16i32_v16i8_store: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa (%rdi), %ymm0 ; CHECK-NEXT: vpackusdw 32(%rdi), %ymm0, %ymm0 ; CHECK-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3] ; CHECK-NEXT: vpmovuswb %ymm0, (%rsi) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %a = load <16 x i32>, <16 x i32>* %p %b = icmp slt <16 x i32> %a, %c = select <16 x i1> %b, <16 x i32> %a, <16 x i32> %d = icmp sgt <16 x i32> %c, zeroinitializer %e = select <16 x i1> %d, <16 x i32> %c, <16 x i32> zeroinitializer %f = trunc <16 x i32> %e to <16 x i8> store <16 x i8> %f, <16 x i8>* %q ret void } define <64 x i1> @v64i1_argument_return(<64 x i1> %x) "min-legal-vector-width"="256" { ; CHECK-LABEL: v64i1_argument_return: ; CHECK: # %bb.0: ; CHECK-NEXT: retq ret <64 x i1> %x } define dso_local void @v64i1_shuffle(<64 x i8>* %x, <64 x i8>* %y) "min-legal-vector-width"="256" { ; CHECK-LABEL: v64i1_shuffle: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: vmovdqa (%rdi), %ymm1 ; CHECK-NEXT: vmovdqa 32(%rdi), %ymm0 ; CHECK-NEXT: vptestnmb %ymm1, %ymm1, %k0 ; CHECK-NEXT: kshiftrd $1, %k0, %k1 ; CHECK-NEXT: movq $-3, %rax ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftlq $63, %k0, %k2 ; CHECK-NEXT: kshiftrq $62, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-5, %rax ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $3, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $61, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-9, %rax ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $2, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $60, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-17, %rax ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $5, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $59, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-33, %rax ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $4, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $58, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-65, %rax ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $7, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $57, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-129, %rax ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $6, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $56, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-257, %rax # imm = 0xFEFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $9, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $55, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-513, %rax # imm = 0xFDFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $8, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $54, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-1025, %rax # imm = 0xFBFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $11, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $53, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-2049, %rax # imm = 0xF7FF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $10, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $52, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-4097, %rax # imm = 0xEFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $13, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $51, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-8193, %rax # imm = 0xDFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $12, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $50, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-16385, %rax # imm = 0xBFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $15, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $49, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-32769, %rax # imm = 0xFFFF7FFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $14, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $48, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-65537, %rax # imm = 0xFFFEFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $17, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $47, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-131073, %rax # imm = 0xFFFDFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $16, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $46, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-262145, %rax # imm = 0xFFFBFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $19, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $45, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-524289, %rax # imm = 0xFFF7FFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $18, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $44, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-1048577, %rax # imm = 0xFFEFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $21, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $43, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-2097153, %rax # imm = 0xFFDFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $20, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $42, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-4194305, %rax # imm = 0xFFBFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $23, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $41, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-8388609, %rax # imm = 0xFF7FFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $22, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $40, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-16777217, %rax # imm = 0xFEFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $25, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $39, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-33554433, %rax # imm = 0xFDFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $24, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $38, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-67108865, %rax # imm = 0xFBFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $27, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $37, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-134217729, %rax # imm = 0xF7FFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $26, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $36, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-268435457, %rax # imm = 0xEFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $29, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $35, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-536870913, %rax # imm = 0xDFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $28, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $34, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movq $-1073741825, %rax # imm = 0xBFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k1 ; CHECK-NEXT: kshiftrd $31, %k0, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $33, %k2, %k2 ; CHECK-NEXT: korq %k2, %k1, %k1 ; CHECK-NEXT: movabsq $-2147483649, %rax # imm = 0xFFFFFFFF7FFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k1, %k2 ; CHECK-NEXT: vptestnmb %ymm0, %ymm0, %k1 ; CHECK-NEXT: kshiftrd $30, %k0, %k0 ; CHECK-NEXT: kshiftlq $63, %k0, %k0 ; CHECK-NEXT: kshiftrq $32, %k0, %k0 ; CHECK-NEXT: korq %k0, %k2, %k0 ; CHECK-NEXT: movabsq $-4294967297, %rax # imm = 0xFFFFFFFEFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $1, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $31, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-8589934593, %rax # imm = 0xFFFFFFFDFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftlq $63, %k1, %k2 ; CHECK-NEXT: kshiftrq $30, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-17179869185, %rax # imm = 0xFFFFFFFBFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $3, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $29, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-34359738369, %rax # imm = 0xFFFFFFF7FFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $2, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $28, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-68719476737, %rax # imm = 0xFFFFFFEFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $5, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $27, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-137438953473, %rax # imm = 0xFFFFFFDFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $4, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $26, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-274877906945, %rax # imm = 0xFFFFFFBFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $7, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $25, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-549755813889, %rax # imm = 0xFFFFFF7FFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $6, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $24, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-1099511627777, %rax # imm = 0xFFFFFEFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $9, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $23, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-2199023255553, %rax # imm = 0xFFFFFDFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $8, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $22, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-4398046511105, %rax # imm = 0xFFFFFBFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $11, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $21, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-8796093022209, %rax # imm = 0xFFFFF7FFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $10, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $20, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-17592186044417, %rax # imm = 0xFFFFEFFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $13, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $19, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-35184372088833, %rax # imm = 0xFFFFDFFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $12, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $18, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-70368744177665, %rax # imm = 0xFFFFBFFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $15, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $17, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-140737488355329, %rax # imm = 0xFFFF7FFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $14, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $16, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-281474976710657, %rax # imm = 0xFFFEFFFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $17, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $15, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-562949953421313, %rax # imm = 0xFFFDFFFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $16, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $14, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-1125899906842625, %rax # imm = 0xFFFBFFFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $19, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $13, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-2251799813685249, %rax # imm = 0xFFF7FFFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $18, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $12, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-4503599627370497, %rax # imm = 0xFFEFFFFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $21, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $11, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-9007199254740993, %rax # imm = 0xFFDFFFFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $20, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $10, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-18014398509481985, %rax # imm = 0xFFBFFFFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $23, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $9, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-36028797018963969, %rax # imm = 0xFF7FFFFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $22, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $8, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-72057594037927937, %rax # imm = 0xFEFFFFFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $25, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $7, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-144115188075855873, %rax # imm = 0xFDFFFFFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $24, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $6, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-288230376151711745, %rax # imm = 0xFBFFFFFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $27, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $5, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-576460752303423489, %rax # imm = 0xF7FFFFFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $26, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $4, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-1152921504606846977, %rax # imm = 0xEFFFFFFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $29, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $3, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-2305843009213693953, %rax # imm = 0xDFFFFFFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $28, %k1, %k2 ; CHECK-NEXT: kshiftlq $63, %k2, %k2 ; CHECK-NEXT: kshiftrq $2, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: movabsq $-4611686018427387905, %rax # imm = 0xBFFFFFFFFFFFFFFF ; CHECK-NEXT: kmovq %rax, %k2 ; CHECK-NEXT: kandq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $31, %k1, %k2 ; CHECK-NEXT: kshiftlq $62, %k2, %k2 ; CHECK-NEXT: korq %k2, %k0, %k0 ; CHECK-NEXT: kshiftrd $30, %k1, %k1 ; CHECK-NEXT: kshiftlq $1, %k0, %k0 ; CHECK-NEXT: kshiftrq $1, %k0, %k0 ; CHECK-NEXT: kshiftlq $63, %k1, %k1 ; CHECK-NEXT: korq %k1, %k0, %k1 ; CHECK-NEXT: vmovdqu8 %ymm1, (%rsi) {%k1} ; CHECK-NEXT: kshiftrq $32, %k1, %k1 ; CHECK-NEXT: vmovdqu8 %ymm0, 32(%rsi) {%k1} ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq entry: %a = load <64 x i8>, <64 x i8>* %x %b = icmp eq <64 x i8> %a, zeroinitializer %shuf = shufflevector <64 x i1> %b, <64 x i1> undef, <64 x i32> call void @llvm.masked.store.v64i8.p0v64i8(<64 x i8> %a, <64 x i8>* %y, i32 1, <64 x i1> %shuf) ret void } declare void @llvm.masked.store.v64i8.p0v64i8(<64 x i8>, <64 x i8>*, i32, <64 x i1>) @mem64_dst = dso_local global i64 0, align 8 @mem64_src = dso_local global i64 0, align 8 define dso_local i32 @v64i1_inline_asm() "min-legal-vector-width"="256" { ; CHECK-LABEL: v64i1_inline_asm: ; CHECK: # %bb.0: ; CHECK-NEXT: kmovq {{.*}}(%rip), %k0 ; CHECK-NEXT: #APP ; CHECK-NEXT: #NO_APP ; CHECK-NEXT: kmovq %k0, {{.*}}(%rip) ; CHECK-NEXT: movl -{{[0-9]+}}(%rsp), %eax ; CHECK-NEXT: retq %1 = alloca i32, align 4 %2 = load i64, i64* @mem64_src, align 8 %3 = call i64 asm "", "=k,k,~{dirflag},~{fpsr},~{flags}"(i64 %2) store i64 %3, i64* @mem64_dst, align 8 %4 = load i32, i32* %1, align 4 ret i32 %4 } define dso_local void @cmp_v8i64_sext(<8 x i64>* %xptr, <8 x i64>* %yptr, <8 x i64>* %zptr) "min-legal-vector-width"="256" { ; CHECK-LABEL: cmp_v8i64_sext: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa (%rsi), %ymm0 ; CHECK-NEXT: vmovdqa 32(%rsi), %ymm1 ; CHECK-NEXT: vpcmpgtq 32(%rdi), %ymm1, %ymm1 ; CHECK-NEXT: vpcmpgtq (%rdi), %ymm0, %ymm0 ; CHECK-NEXT: vmovdqa %ymm0, (%rdx) ; CHECK-NEXT: vmovdqa %ymm1, 32(%rdx) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %x = load <8 x i64>, <8 x i64>* %xptr %y = load <8 x i64>, <8 x i64>* %yptr %cmp = icmp slt <8 x i64> %x, %y %ext = sext <8 x i1> %cmp to <8 x i64> store <8 x i64> %ext, <8 x i64>* %zptr ret void } define dso_local void @cmp_v8i64_zext(<8 x i64>* %xptr, <8 x i64>* %yptr, <8 x i64>* %zptr) "min-legal-vector-width"="256" { ; CHECK-LABEL: cmp_v8i64_zext: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa (%rsi), %ymm0 ; CHECK-NEXT: vmovdqa 32(%rsi), %ymm1 ; CHECK-NEXT: vpcmpgtq 32(%rdi), %ymm1, %ymm1 ; CHECK-NEXT: vpcmpgtq (%rdi), %ymm0, %ymm0 ; CHECK-NEXT: vpsrlq $63, %ymm1, %ymm1 ; CHECK-NEXT: vpsrlq $63, %ymm0, %ymm0 ; CHECK-NEXT: vmovdqa %ymm0, (%rdx) ; CHECK-NEXT: vmovdqa %ymm1, 32(%rdx) ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %x = load <8 x i64>, <8 x i64>* %xptr %y = load <8 x i64>, <8 x i64>* %yptr %cmp = icmp slt <8 x i64> %x, %y %ext = zext <8 x i1> %cmp to <8 x i64> store <8 x i64> %ext, <8 x i64>* %zptr ret void } define <16 x i8> @var_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind "min-legal-vector-width"="256" { ; CHECK-LABEL: var_rotate_v16i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vmovdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] ; CHECK-NEXT: vpsubb %xmm1, %xmm2, %xmm2 ; CHECK-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero ; CHECK-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero ; CHECK-NEXT: vpsllvw %ymm1, %ymm0, %ymm1 ; CHECK-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero ; CHECK-NEXT: vpsrlvw %ymm2, %ymm0, %ymm0 ; CHECK-NEXT: vpor %ymm0, %ymm1, %ymm0 ; CHECK-NEXT: vpmovwb %ymm0, %xmm0 ; CHECK-NEXT: vzeroupper ; CHECK-NEXT: retq %b8 = sub <16 x i8> , %b %shl = shl <16 x i8> %a, %b %lshr = lshr <16 x i8> %a, %b8 %or = or <16 x i8> %shl, %lshr ret <16 x i8> %or } define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind "min-legal-vector-width"="256" { ; CHECK-LABEL: var_rotate_v32i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vpsllw $4, %ymm0, %ymm2 ; CHECK-NEXT: vpsrlw $4, %ymm0, %ymm3 ; CHECK-NEXT: vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm3 ; CHECK-NEXT: vpsllw $5, %ymm1, %ymm1 ; CHECK-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; CHECK-NEXT: vpsllw $2, %ymm0, %ymm2 ; CHECK-NEXT: vpsrlw $6, %ymm0, %ymm3 ; CHECK-NEXT: vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm3 ; CHECK-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; CHECK-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; CHECK-NEXT: vpsrlw $7, %ymm0, %ymm2 ; CHECK-NEXT: vpaddb %ymm0, %ymm0, %ymm3 ; CHECK-NEXT: vpternlogq $248, {{.*}}(%rip), %ymm2, %ymm3 ; CHECK-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; CHECK-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; CHECK-NEXT: retq %b8 = sub <32 x i8> , %b %shl = shl <32 x i8> %a, %b %lshr = lshr <32 x i8> %a, %b8 %or = or <32 x i8> %shl, %lshr ret <32 x i8> %or } define <32 x i8> @splatvar_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind "min-legal-vector-width"="256" { ; CHECK-AVX512-LABEL: splatvar_rotate_v32i8: ; CHECK-AVX512: # %bb.0: ; CHECK-AVX512-NEXT: vpbroadcastb %xmm1, %xmm1 ; CHECK-AVX512-NEXT: vpand {{.*}}(%rip), %xmm1, %xmm1 ; CHECK-AVX512-NEXT: vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero ; CHECK-AVX512-NEXT: vpsllw %xmm2, %ymm0, %ymm3 ; CHECK-AVX512-NEXT: vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] ; CHECK-AVX512-NEXT: vpsubb %xmm1, %xmm4, %xmm1 ; CHECK-AVX512-NEXT: vpcmpeqd %xmm4, %xmm4, %xmm4 ; CHECK-AVX512-NEXT: vpsllw %xmm2, %xmm4, %xmm2 ; CHECK-AVX512-NEXT: vpbroadcastb %xmm2, %ymm2 ; CHECK-AVX512-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero ; CHECK-AVX512-NEXT: vpsrlw %xmm1, %ymm0, %ymm5 ; CHECK-AVX512-NEXT: vpand %ymm2, %ymm3, %ymm2 ; CHECK-AVX512-NEXT: vpsrlw %xmm1, %xmm4, %xmm0 ; CHECK-AVX512-NEXT: vpsrlw $8, %xmm0, %xmm0 ; CHECK-AVX512-NEXT: vpbroadcastb %xmm0, %ymm0 ; CHECK-AVX512-NEXT: vpternlogq $236, %ymm5, %ymm2, %ymm0 ; CHECK-AVX512-NEXT: retq ; ; CHECK-VBMI-LABEL: splatvar_rotate_v32i8: ; CHECK-VBMI: # %bb.0: ; CHECK-VBMI-NEXT: vpbroadcastb %xmm1, %xmm1 ; CHECK-VBMI-NEXT: vpand {{.*}}(%rip), %xmm1, %xmm1 ; CHECK-VBMI-NEXT: vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero ; CHECK-VBMI-NEXT: vpsllw %xmm2, %ymm0, %ymm3 ; CHECK-VBMI-NEXT: vpcmpeqd %xmm4, %xmm4, %xmm4 ; CHECK-VBMI-NEXT: vpsllw %xmm2, %xmm4, %xmm2 ; CHECK-VBMI-NEXT: vpbroadcastb %xmm2, %ymm2 ; CHECK-VBMI-NEXT: vpand %ymm2, %ymm3, %ymm2 ; CHECK-VBMI-NEXT: vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8] ; CHECK-VBMI-NEXT: vpsubb %xmm1, %xmm3, %xmm1 ; CHECK-VBMI-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero ; CHECK-VBMI-NEXT: vpsrlw %xmm1, %ymm0, %ymm3 ; CHECK-VBMI-NEXT: vpsrlw %xmm1, %xmm4, %xmm0 ; CHECK-VBMI-NEXT: vmovdqa {{.*#+}} ymm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ; CHECK-VBMI-NEXT: vpermb %ymm0, %ymm1, %ymm0 ; CHECK-VBMI-NEXT: vpternlogq $236, %ymm3, %ymm2, %ymm0 ; CHECK-VBMI-NEXT: retq %splat = shufflevector <32 x i8> %b, <32 x i8> undef, <32 x i32> zeroinitializer %splat8 = sub <32 x i8> , %splat %shl = shl <32 x i8> %a, %splat %lshr = lshr <32 x i8> %a, %splat8 %or = or <32 x i8> %shl, %lshr ret <32 x i8> %or } define <32 x i8> @constant_rotate_v32i8(<32 x i8> %a) nounwind "min-legal-vector-width"="256" { ; CHECK-AVX512-LABEL: constant_rotate_v32i8: ; CHECK-AVX512: # %bb.0: ; CHECK-AVX512-NEXT: vpsllw $4, %ymm0, %ymm1 ; CHECK-AVX512-NEXT: vpand {{.*}}(%rip), %ymm1, %ymm1 ; CHECK-AVX512-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256] ; CHECK-AVX512-NEXT: # ymm2 = mem[0,1,0,1] ; CHECK-AVX512-NEXT: vpblendvb %ymm2, %ymm1, %ymm0, %ymm1 ; CHECK-AVX512-NEXT: vpsllw $2, %ymm1, %ymm3 ; CHECK-AVX512-NEXT: vpand {{.*}}(%rip), %ymm3, %ymm3 ; CHECK-AVX512-NEXT: vpaddb %ymm2, %ymm2, %ymm2 ; CHECK-AVX512-NEXT: vpblendvb %ymm2, %ymm3, %ymm1, %ymm1 ; CHECK-AVX512-NEXT: vpaddb %ymm1, %ymm1, %ymm3 ; CHECK-AVX512-NEXT: vpaddb %ymm2, %ymm2, %ymm2 ; CHECK-AVX512-NEXT: vpblendvb %ymm2, %ymm3, %ymm1, %ymm1 ; CHECK-AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2 ; CHECK-AVX512-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31] ; CHECK-AVX512-NEXT: vpsllvw {{.*}}(%rip), %ymm3, %ymm3 ; CHECK-AVX512-NEXT: vpsrlw $8, %ymm3, %ymm3 ; CHECK-AVX512-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23] ; CHECK-AVX512-NEXT: vpsllvw {{.*}}(%rip), %ymm0, %ymm0 ; CHECK-AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0 ; CHECK-AVX512-NEXT: vpackuswb %ymm3, %ymm0, %ymm0 ; CHECK-AVX512-NEXT: vpor %ymm0, %ymm1, %ymm0 ; CHECK-AVX512-NEXT: retq ; ; CHECK-VBMI-LABEL: constant_rotate_v32i8: ; CHECK-VBMI: # %bb.0: ; CHECK-VBMI-NEXT: vpsllw $4, %ymm0, %ymm1 ; CHECK-VBMI-NEXT: vpand {{.*}}(%rip), %ymm1, %ymm1 ; CHECK-VBMI-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256] ; CHECK-VBMI-NEXT: # ymm2 = mem[0,1,0,1] ; CHECK-VBMI-NEXT: vpblendvb %ymm2, %ymm1, %ymm0, %ymm1 ; CHECK-VBMI-NEXT: vpsllw $2, %ymm1, %ymm3 ; CHECK-VBMI-NEXT: vpand {{.*}}(%rip), %ymm3, %ymm3 ; CHECK-VBMI-NEXT: vpaddb %ymm2, %ymm2, %ymm2 ; CHECK-VBMI-NEXT: vpblendvb %ymm2, %ymm3, %ymm1, %ymm1 ; CHECK-VBMI-NEXT: vpaddb %ymm1, %ymm1, %ymm3 ; CHECK-VBMI-NEXT: vpaddb %ymm2, %ymm2, %ymm2 ; CHECK-VBMI-NEXT: vpblendvb %ymm2, %ymm3, %ymm1, %ymm1 ; CHECK-VBMI-NEXT: vpxor %xmm2, %xmm2, %xmm2 ; CHECK-VBMI-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31] ; CHECK-VBMI-NEXT: vpsllvw {{.*}}(%rip), %ymm3, %ymm3 ; CHECK-VBMI-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23] ; CHECK-VBMI-NEXT: vpsllvw {{.*}}(%rip), %ymm0, %ymm0 ; CHECK-VBMI-NEXT: vmovdqa {{.*#+}} ymm2 = [1,3,5,7,9,11,13,15,33,35,37,39,41,43,45,47,17,19,21,23,25,27,29,31,49,51,53,55,57,59,61,63] ; CHECK-VBMI-NEXT: vpermi2b %ymm3, %ymm0, %ymm2 ; CHECK-VBMI-NEXT: vpor %ymm2, %ymm1, %ymm0 ; CHECK-VBMI-NEXT: retq %shl = shl <32 x i8> %a, %lshr = lshr <32 x i8> %a, %or = or <32 x i8> %shl, %lshr ret <32 x i8> %or } define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind "min-legal-vector-width"="256" { ; CHECK-LABEL: splatconstant_rotate_v32i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vpsllw $4, %ymm0, %ymm1 ; CHECK-NEXT: vpsrlw $4, %ymm0, %ymm0 ; CHECK-NEXT: vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0 ; CHECK-NEXT: retq %shl = shl <32 x i8> %a, %lshr = lshr <32 x i8> %a, %or = or <32 x i8> %shl, %lshr ret <32 x i8> %or } define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind "min-legal-vector-width"="256" { ; CHECK-LABEL: splatconstant_rotate_mask_v32i8: ; CHECK: # %bb.0: ; CHECK-NEXT: vpsllw $4, %ymm0, %ymm1 ; CHECK-NEXT: vpsrlw $4, %ymm0, %ymm0 ; CHECK-NEXT: vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0 ; CHECK-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 ; CHECK-NEXT: retq %shl = shl <32 x i8> %a, %lshr = lshr <32 x i8> %a, %rmask = and <32 x i8> %lshr, %lmask = and <32 x i8> %shl, %or = or <32 x i8> %lmask, %rmask ret <32 x i8> %or }