191 lines
9.6 KiB
LLVM
191 lines
9.6 KiB
LLVM
|
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
|
||
|
|
|
||
|
|
|
||
|
|
; RUN: opt --arm-mve-gather-scatter-lowering -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp %s -S -o 2>/dev/null - | FileCheck %s
|
||
|
|
|
||
|
|
define arm_aapcs_vfpcc void @push_out_add_sub_block(i32* noalias nocapture readonly %data, i32* noalias nocapture %dst, i32 %n.vec) {
|
||
|
|
; CHECK-LABEL: @push_out_add_sub_block(
|
||
|
|
; CHECK-NEXT: vector.ph:
|
||
|
|
; CHECK-NEXT: [[IND_END:%.*]] = shl i32 [[N_VEC:%.*]], 1
|
||
|
|
; CHECK-NEXT: [[PUSHEDOUTADD:%.*]] = add <4 x i32> <i32 0, i32 2, i32 4, i32 6>, <i32 6, i32 6, i32 6, i32 6>
|
||
|
|
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
|
||
|
|
; CHECK: vector.body:
|
||
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY_END:%.*]] ]
|
||
|
|
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[PUSHEDOUTADD]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY_END]] ]
|
||
|
|
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX]], 50
|
||
|
|
; CHECK-NEXT: br i1 [[TMP0]], label [[LOWER_BLOCK:%.*]], label [[END:%.*]]
|
||
|
|
; CHECK: lower.block:
|
||
|
|
; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0i32.v4i32(i32* [[DATA:%.*]], <4 x i32> [[VEC_IND]], i32 32, i32 2, i32 1)
|
||
|
|
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[INDEX]]
|
||
|
|
; CHECK-NEXT: [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>*
|
||
|
|
; CHECK-NEXT: store <4 x i32> [[TMP1]], <4 x i32>* [[TMP3]], align 4
|
||
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
|
||
|
|
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 8, i32 8, i32 8, i32 8>
|
||
|
|
; CHECK-NEXT: br label [[VECTOR_BODY_END]]
|
||
|
|
; CHECK: vector.body.end:
|
||
|
|
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
|
||
|
|
; CHECK-NEXT: br i1 [[TMP4]], label [[END]], label [[VECTOR_BODY]]
|
||
|
|
; CHECK: end:
|
||
|
|
; CHECK-NEXT: ret void
|
||
|
|
;
|
||
|
|
|
||
|
|
vector.ph:
|
||
|
|
%ind.end = shl i32 %n.vec, 1
|
||
|
|
br label %vector.body
|
||
|
|
|
||
|
|
vector.body: ; preds = %vector.body, %vector.ph
|
||
|
|
%index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body.end ]
|
||
|
|
%vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body.end ]
|
||
|
|
%0 = icmp eq i32 %index, 50
|
||
|
|
br i1 %0, label %lower.block, label %end
|
||
|
|
|
||
|
|
lower.block: ; preds = %vector.body
|
||
|
|
%1 = add <4 x i32> %vec.ind, <i32 6, i32 6, i32 6, i32 6>
|
||
|
|
%2 = getelementptr inbounds i32, i32* %data, <4 x i32> %1
|
||
|
|
%wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %2, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
|
||
|
|
%3 = getelementptr inbounds i32, i32* %dst, i32 %index
|
||
|
|
%4 = bitcast i32* %3 to <4 x i32>*
|
||
|
|
store <4 x i32> %wide.masked.gather, <4 x i32>* %4, align 4
|
||
|
|
%index.next = add i32 %index, 4
|
||
|
|
%vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>
|
||
|
|
br label %vector.body.end
|
||
|
|
|
||
|
|
vector.body.end: ; preds = %lower.block
|
||
|
|
%5 = icmp eq i32 %index.next, %n.vec
|
||
|
|
br i1 %5, label %end, label %vector.body
|
||
|
|
|
||
|
|
end:
|
||
|
|
ret void;
|
||
|
|
}
|
||
|
|
|
||
|
|
define arm_aapcs_vfpcc void @push_out_mul_sub_block(i32* noalias nocapture readonly %data, i32* noalias nocapture %dst, i32 %n.vec) {
|
||
|
|
; CHECK-LABEL: @push_out_mul_sub_block(
|
||
|
|
; CHECK-NEXT: vector.ph:
|
||
|
|
; CHECK-NEXT: [[IND_END:%.*]] = shl i32 [[N_VEC:%.*]], 1
|
||
|
|
; CHECK-NEXT: [[PUSHEDOUTMUL:%.*]] = mul <4 x i32> <i32 0, i32 2, i32 4, i32 6>, <i32 3, i32 3, i32 3, i32 3>
|
||
|
|
; CHECK-NEXT: [[PRODUCT:%.*]] = mul <4 x i32> <i32 8, i32 8, i32 8, i32 8>, <i32 3, i32 3, i32 3, i32 3>
|
||
|
|
; CHECK-NEXT: [[PUSHEDOUTADD:%.*]] = add <4 x i32> [[PUSHEDOUTMUL]], <i32 6, i32 6, i32 6, i32 6>
|
||
|
|
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
|
||
|
|
; CHECK: vector.body:
|
||
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY_END:%.*]] ]
|
||
|
|
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[PUSHEDOUTADD]], [[VECTOR_PH]] ], [ [[INCREMENTPUSHEDOUTMUL:%.*]], [[VECTOR_BODY_END]] ]
|
||
|
|
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX]], 50
|
||
|
|
; CHECK-NEXT: br i1 [[TMP0]], label [[LOWER_BLOCK:%.*]], label [[END:%.*]]
|
||
|
|
; CHECK: lower.block:
|
||
|
|
; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0i32.v4i32(i32* [[DATA:%.*]], <4 x i32> [[VEC_IND]], i32 32, i32 2, i32 1)
|
||
|
|
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[INDEX]]
|
||
|
|
; CHECK-NEXT: [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>*
|
||
|
|
; CHECK-NEXT: store <4 x i32> [[TMP1]], <4 x i32>* [[TMP3]], align 4
|
||
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
|
||
|
|
; CHECK-NEXT: br label [[VECTOR_BODY_END]]
|
||
|
|
; CHECK: vector.body.end:
|
||
|
|
; CHECK-NEXT: [[INCREMENTPUSHEDOUTMUL]] = add <4 x i32> [[VEC_IND]], [[PRODUCT]]
|
||
|
|
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
|
||
|
|
; CHECK-NEXT: br i1 [[TMP4]], label [[END]], label [[VECTOR_BODY]]
|
||
|
|
; CHECK: end:
|
||
|
|
; CHECK-NEXT: ret void
|
||
|
|
;
|
||
|
|
|
||
|
|
vector.ph:
|
||
|
|
%ind.end = shl i32 %n.vec, 1
|
||
|
|
br label %vector.body
|
||
|
|
|
||
|
|
vector.body: ; preds = %vector.body, %vector.ph
|
||
|
|
%index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body.end ]
|
||
|
|
%vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body.end ]
|
||
|
|
%0 = icmp eq i32 %index, 50
|
||
|
|
br i1 %0, label %lower.block, label %end
|
||
|
|
|
||
|
|
lower.block: ; preds = %vector.body
|
||
|
|
%1 = mul <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3>
|
||
|
|
%2 = add <4 x i32> %1, <i32 6, i32 6, i32 6, i32 6>
|
||
|
|
%3 = getelementptr inbounds i32, i32* %data, <4 x i32> %2
|
||
|
|
%wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %3, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
|
||
|
|
%4 = getelementptr inbounds i32, i32* %dst, i32 %index
|
||
|
|
%5 = bitcast i32* %4 to <4 x i32>*
|
||
|
|
store <4 x i32> %wide.masked.gather, <4 x i32>* %5, align 4
|
||
|
|
%index.next = add i32 %index, 4
|
||
|
|
%vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>
|
||
|
|
br label %vector.body.end
|
||
|
|
|
||
|
|
vector.body.end: ; preds = %lower.block
|
||
|
|
%6 = icmp eq i32 %index.next, %n.vec
|
||
|
|
br i1 %6, label %end, label %vector.body
|
||
|
|
|
||
|
|
end:
|
||
|
|
ret void;
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
define arm_aapcs_vfpcc void @push_out_mul_sub_loop(i32* noalias nocapture readonly %data, i32* noalias nocapture %dst, i32 %n.vec) {
|
||
|
|
; CHECK-LABEL: @push_out_mul_sub_loop(
|
||
|
|
; CHECK-NEXT: vector.ph:
|
||
|
|
; CHECK-NEXT: [[IND_END:%.*]] = shl i32 [[N_VEC:%.*]], 2
|
||
|
|
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
|
||
|
|
; CHECK: vector.body:
|
||
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY_END:%.*]] ]
|
||
|
|
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY_END]] ]
|
||
|
|
; CHECK-NEXT: br label [[VECTOR_2_PH:%.*]]
|
||
|
|
; CHECK: vector.2.ph:
|
||
|
|
; CHECK-NEXT: br label [[VECTOR_2_BODY:%.*]]
|
||
|
|
; CHECK: vector.2.body:
|
||
|
|
; CHECK-NEXT: [[TMP0:%.*]] = mul <4 x i32> [[VEC_IND]], <i32 3, i32 3, i32 3, i32 3>
|
||
|
|
; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[TMP0]], <i32 6, i32 6, i32 6, i32 6>
|
||
|
|
; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0i32.v4i32(i32* [[DATA:%.*]], <4 x i32> [[TMP1]], i32 32, i32 2, i32 1)
|
||
|
|
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i32 [[INDEX]]
|
||
|
|
; CHECK-NEXT: [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
|
||
|
|
; CHECK-NEXT: store <4 x i32> [[TMP2]], <4 x i32>* [[TMP4]], align 4
|
||
|
|
; CHECK-NEXT: br label [[VECTOR_2_BODY_END:%.*]]
|
||
|
|
; CHECK: vector.2.body.end:
|
||
|
|
; CHECK-NEXT: [[INDEX_2_NEXT:%.*]] = add i32 [[INDEX]], 4
|
||
|
|
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_2_NEXT]], 15
|
||
|
|
; CHECK-NEXT: br i1 [[TMP5]], label [[VECTOR_BODY_END]], label [[VECTOR_2_BODY]]
|
||
|
|
; CHECK: vector.body.end:
|
||
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
|
||
|
|
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 8, i32 8, i32 8, i32 8>
|
||
|
|
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
|
||
|
|
; CHECK-NEXT: br i1 [[TMP6]], label [[END:%.*]], label [[VECTOR_BODY]]
|
||
|
|
; CHECK: end:
|
||
|
|
; CHECK-NEXT: ret void
|
||
|
|
;
|
||
|
|
|
||
|
|
vector.ph:
|
||
|
|
%ind.end = shl i32 %n.vec, 2
|
||
|
|
br label %vector.body
|
||
|
|
|
||
|
|
vector.body: ; preds = %vector.body, %vector.ph
|
||
|
|
%index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body.end ]
|
||
|
|
%vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body.end ]
|
||
|
|
br label %vector.2.ph
|
||
|
|
|
||
|
|
vector.2.ph:
|
||
|
|
br label %vector.2.body
|
||
|
|
|
||
|
|
vector.2.body: ; preds = %vector.body
|
||
|
|
%index.2 = phi i32 [ 0, %vector.2.ph ], [ %index.2.next, %vector.2.body.end ]
|
||
|
|
%0 = mul <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3>
|
||
|
|
%1 = add <4 x i32> %0, <i32 6, i32 6, i32 6, i32 6>
|
||
|
|
%2 = getelementptr inbounds i32, i32* %data, <4 x i32> %1
|
||
|
|
%wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %2, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
|
||
|
|
%3 = getelementptr inbounds i32, i32* %dst, i32 %index
|
||
|
|
%4 = bitcast i32* %3 to <4 x i32>*
|
||
|
|
store <4 x i32> %wide.masked.gather, <4 x i32>* %4, align 4
|
||
|
|
br label %vector.2.body.end
|
||
|
|
|
||
|
|
vector.2.body.end: ; preds = %lower.block
|
||
|
|
%index.2.next = add i32 %index, 4
|
||
|
|
%5 = icmp eq i32 %index.2.next, 15
|
||
|
|
br i1 %5, label %vector.body.end, label %vector.2.body
|
||
|
|
|
||
|
|
vector.body.end: ; preds = %lower.block
|
||
|
|
%index.next = add i32 %index, 4
|
||
|
|
%vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>
|
||
|
|
%6 = icmp eq i32 %index.next, %n.vec
|
||
|
|
br i1 %6, label %end, label %vector.body
|
||
|
|
|
||
|
|
end:
|
||
|
|
ret void;
|
||
|
|
}
|
||
|
|
|
||
|
|
declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>)
|