-multiclass SS41I_binop_rm_int8_y<bits<8> opc, string OpcodeStr,
- Intrinsic IntId, X86FoldableSchedWrite Sched> {
- def Yrr : SS48I<opc, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src),
- !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
- [(set VR256:$dst, (IntId VR128:$src))]>, Sched<[Sched]>;
-
- def Yrm : SS48I<opc, MRMSrcMem, (outs VR256:$dst), (ins i32mem:$src),
- !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
- [(set VR256:$dst,
- (IntId (bitconvert (v2i64 (scalar_to_vector (loadi64 addr:$src))))))]>,
- Sched<[Sched.Folded]>;
-}
-
-let Predicates = [HasAVX] in {
-defm VPMOVSXBD : SS41I_binop_rm_int4<0x21, "vpmovsxbd", int_x86_sse41_pmovsxbd,
- DEFAULT_ITINS_SHUFFLESCHED>, VEX;
-defm VPMOVSXWQ : SS41I_binop_rm_int4<0x24, "vpmovsxwq", int_x86_sse41_pmovsxwq,
- DEFAULT_ITINS_SHUFFLESCHED>, VEX;
-defm VPMOVZXBD : SS41I_binop_rm_int4<0x31, "vpmovzxbd", int_x86_sse41_pmovzxbd,
- DEFAULT_ITINS_SHUFFLESCHED>, VEX;
-defm VPMOVZXWQ : SS41I_binop_rm_int4<0x34, "vpmovzxwq", int_x86_sse41_pmovzxwq,
- DEFAULT_ITINS_SHUFFLESCHED>, VEX;
-}
-
-let Predicates = [HasAVX2] in {
-defm VPMOVSXBD : SS41I_binop_rm_int8_y<0x21, "vpmovsxbd",
- int_x86_avx2_pmovsxbd, WriteShuffle>,
- VEX, VEX_L;
-defm VPMOVSXWQ : SS41I_binop_rm_int8_y<0x24, "vpmovsxwq",
- int_x86_avx2_pmovsxwq, WriteShuffle>,
- VEX, VEX_L;
-defm VPMOVZXBD : SS41I_binop_rm_int8_y<0x31, "vpmovzxbd",
- int_x86_avx2_pmovzxbd, WriteShuffle>,
- VEX, VEX_L;
-defm VPMOVZXWQ : SS41I_binop_rm_int8_y<0x34, "vpmovzxwq",
- int_x86_avx2_pmovzxwq, WriteShuffle>,
- VEX, VEX_L;
-}
-
-defm PMOVSXBD : SS41I_binop_rm_int4<0x21, "pmovsxbd", int_x86_sse41_pmovsxbd,
- SSE_INTALU_ITINS_SHUFF_P>;
-defm PMOVSXWQ : SS41I_binop_rm_int4<0x24, "pmovsxwq", int_x86_sse41_pmovsxwq,
- SSE_INTALU_ITINS_SHUFF_P>;
-defm PMOVZXBD : SS41I_binop_rm_int4<0x31, "pmovzxbd", int_x86_sse41_pmovzxbd,
- SSE_INTALU_ITINS_SHUFF_P>;
-defm PMOVZXWQ : SS41I_binop_rm_int4<0x34, "pmovzxwq", int_x86_sse41_pmovzxwq,
- SSE_INTALU_ITINS_SHUFF_P>;
-
-let Predicates = [HasAVX] in {
- // Common patterns involving scalar load
- def : Pat<(int_x86_sse41_pmovsxbd (vzmovl_v4i32 addr:$src)),
- (VPMOVSXBDrm addr:$src)>;
- def : Pat<(int_x86_sse41_pmovsxwq (vzmovl_v4i32 addr:$src)),
- (VPMOVSXWQrm addr:$src)>;
-
- def : Pat<(int_x86_sse41_pmovzxbd (vzmovl_v4i32 addr:$src)),
- (VPMOVZXBDrm addr:$src)>;
- def : Pat<(int_x86_sse41_pmovzxwq (vzmovl_v4i32 addr:$src)),
- (VPMOVZXWQrm addr:$src)>;
-}
-
-let Predicates = [UseSSE41] in {
- // Common patterns involving scalar load
- def : Pat<(int_x86_sse41_pmovsxbd (vzmovl_v4i32 addr:$src)),
- (PMOVSXBDrm addr:$src)>;
- def : Pat<(int_x86_sse41_pmovsxwq (vzmovl_v4i32 addr:$src)),
- (PMOVSXWQrm addr:$src)>;
-
- def : Pat<(int_x86_sse41_pmovzxbd (vzmovl_v4i32 addr:$src)),
- (PMOVZXBDrm addr:$src)>;
- def : Pat<(int_x86_sse41_pmovzxwq (vzmovl_v4i32 addr:$src)),
- (PMOVZXWQrm addr:$src)>;
-}
-
-multiclass SS41I_binop_rm_int2<bits<8> opc, string OpcodeStr, Intrinsic IntId,
- X86FoldableSchedWrite Sched> {
- def rr : SS48I<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
- !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
- [(set VR128:$dst, (IntId VR128:$src))]>, Sched<[Sched]>;
-
- // Expecting a i16 load any extended to i32 value.
- def rm : SS48I<opc, MRMSrcMem, (outs VR128:$dst), (ins i16mem:$src),
- !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
- [(set VR128:$dst, (IntId (bitconvert
- (v4i32 (scalar_to_vector (loadi16_anyext addr:$src))))))]>,
- Sched<[Sched.Folded]>;
-}
-
-multiclass SS41I_binop_rm_int4_y<bits<8> opc, string OpcodeStr,
- Intrinsic IntId, X86FoldableSchedWrite Sched> {
- def Yrr : SS48I<opc, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src),
- !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
- [(set VR256:$dst, (IntId VR128:$src))]>, Sched<[Sched]>;
-
- // Expecting a i16 load any extended to i32 value.
- def Yrm : SS48I<opc, MRMSrcMem, (outs VR256:$dst), (ins i16mem:$src),
- !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
- [(set VR256:$dst, (IntId (bitconvert
- (v4i32 (scalar_to_vector (loadi32 addr:$src))))))]>,
- Sched<[Sched.Folded]>;
-}
-
-let Predicates = [HasAVX] in {
-defm VPMOVSXBQ : SS41I_binop_rm_int2<0x22, "vpmovsxbq", int_x86_sse41_pmovsxbq,
- WriteShuffle>, VEX;
-defm VPMOVZXBQ : SS41I_binop_rm_int2<0x32, "vpmovzxbq", int_x86_sse41_pmovzxbq,
- WriteShuffle>, VEX;
-}
-let Predicates = [HasAVX2] in {
-defm VPMOVSXBQ : SS41I_binop_rm_int4_y<0x22, "vpmovsxbq", int_x86_avx2_pmovsxbq,
- WriteShuffle>, VEX, VEX_L;
-defm VPMOVZXBQ : SS41I_binop_rm_int4_y<0x32, "vpmovzxbq", int_x86_avx2_pmovzxbq,
- WriteShuffle>, VEX, VEX_L;
-}
-defm PMOVSXBQ : SS41I_binop_rm_int2<0x22, "pmovsxbq", int_x86_sse41_pmovsxbq,
- WriteShuffle>;
-defm PMOVZXBQ : SS41I_binop_rm_int2<0x32, "pmovzxbq", int_x86_sse41_pmovzxbq,
- WriteShuffle>;
-
-let Predicates = [HasAVX2] in {
- def : Pat<(v16i16 (X86vsext (v16i8 VR128:$src))), (VPMOVSXBWYrr VR128:$src)>;
- def : Pat<(v8i32 (X86vsext (v16i8 VR128:$src))), (VPMOVSXBDYrr VR128:$src)>;
- def : Pat<(v4i64 (X86vsext (v16i8 VR128:$src))), (VPMOVSXBQYrr VR128:$src)>;
-
- def : Pat<(v8i32 (X86vsext (v8i16 VR128:$src))), (VPMOVSXWDYrr VR128:$src)>;
- def : Pat<(v4i64 (X86vsext (v8i16 VR128:$src))), (VPMOVSXWQYrr VR128:$src)>;
-
- def : Pat<(v4i64 (X86vsext (v4i32 VR128:$src))), (VPMOVSXDQYrr VR128:$src)>;
-
- def : Pat<(v16i16 (X86vsext (v32i8 VR256:$src))),
- (VPMOVSXBWYrr (EXTRACT_SUBREG VR256:$src, sub_xmm))>;
- def : Pat<(v8i32 (X86vsext (v32i8 VR256:$src))),
- (VPMOVSXBDYrr (EXTRACT_SUBREG VR256:$src, sub_xmm))>;
- def : Pat<(v4i64 (X86vsext (v32i8 VR256:$src))),
- (VPMOVSXBQYrr (EXTRACT_SUBREG VR256:$src, sub_xmm))>;
-
- def : Pat<(v8i32 (X86vsext (v16i16 VR256:$src))),
- (VPMOVSXWDYrr (EXTRACT_SUBREG VR256:$src, sub_xmm))>;
- def : Pat<(v4i64 (X86vsext (v16i16 VR256:$src))),
- (VPMOVSXWQYrr (EXTRACT_SUBREG VR256:$src, sub_xmm))>;
-
- def : Pat<(v4i64 (X86vsext (v8i32 VR256:$src))),
- (VPMOVSXDQYrr (EXTRACT_SUBREG VR256:$src, sub_xmm))>;
-
- def : Pat<(v8i32 (X86vsext (v8i16 (bitconvert (v2i64 (load addr:$src)))))),
- (VPMOVSXWDYrm addr:$src)>;
- def : Pat<(v4i64 (X86vsext (v4i32 (bitconvert (v2i64 (load addr:$src)))))),
- (VPMOVSXDQYrm addr:$src)>;
-
- def : Pat<(v8i32 (X86vsext (v16i8 (bitconvert (v2i64
- (scalar_to_vector (loadi64 addr:$src))))))),
- (VPMOVSXBDYrm addr:$src)>;
- def : Pat<(v8i32 (X86vsext (v16i8 (bitconvert (v2f64
- (scalar_to_vector (loadf64 addr:$src))))))),
- (VPMOVSXBDYrm addr:$src)>;
-
- def : Pat<(v4i64 (X86vsext (v8i16 (bitconvert (v2i64
- (scalar_to_vector (loadi64 addr:$src))))))),
- (VPMOVSXWQYrm addr:$src)>;
- def : Pat<(v4i64 (X86vsext (v8i16 (bitconvert (v2f64
- (scalar_to_vector (loadf64 addr:$src))))))),
- (VPMOVSXWQYrm addr:$src)>;
-
- def : Pat<(v4i64 (X86vsext (v16i8 (bitconvert (v4i32
- (scalar_to_vector (loadi32 addr:$src))))))),
- (VPMOVSXBQYrm addr:$src)>;
-}
-
-let Predicates = [HasAVX] in {
- // Common patterns involving scalar load
- def : Pat<(int_x86_sse41_pmovsxbq
- (bitconvert (v4i32 (X86vzmovl
- (v4i32 (scalar_to_vector (loadi32 addr:$src))))))),
- (VPMOVSXBQrm addr:$src)>;
-
- def : Pat<(int_x86_sse41_pmovzxbq
- (bitconvert (v4i32 (X86vzmovl
- (v4i32 (scalar_to_vector (loadi32 addr:$src))))))),
- (VPMOVZXBQrm addr:$src)>;
-}
-
-let Predicates = [UseSSE41] in {
- def : Pat<(v8i16 (X86vsext (v16i8 VR128:$src))), (PMOVSXBWrr VR128:$src)>;
- def : Pat<(v4i32 (X86vsext (v16i8 VR128:$src))), (PMOVSXBDrr VR128:$src)>;
- def : Pat<(v2i64 (X86vsext (v16i8 VR128:$src))), (PMOVSXBQrr VR128:$src)>;
-
- def : Pat<(v4i32 (X86vsext (v8i16 VR128:$src))), (PMOVSXWDrr VR128:$src)>;
- def : Pat<(v2i64 (X86vsext (v8i16 VR128:$src))), (PMOVSXWQrr VR128:$src)>;
-
- def : Pat<(v2i64 (X86vsext (v4i32 VR128:$src))), (PMOVSXDQrr VR128:$src)>;
-
- // Common patterns involving scalar load
- def : Pat<(int_x86_sse41_pmovsxbq
- (bitconvert (v4i32 (X86vzmovl
- (v4i32 (scalar_to_vector (loadi32 addr:$src))))))),
- (PMOVSXBQrm addr:$src)>;
-
- def : Pat<(int_x86_sse41_pmovzxbq
- (bitconvert (v4i32 (X86vzmovl
- (v4i32 (scalar_to_vector (loadi32 addr:$src))))))),
- (PMOVZXBQrm addr:$src)>;
-
- def : Pat<(v4i32 (X86vsext (v8i16 (bitconvert (v2i64
- (scalar_to_vector (loadi64 addr:$src))))))),
- (PMOVSXWDrm addr:$src)>;
- def : Pat<(v4i32 (X86vsext (v8i16 (bitconvert (v2f64
- (scalar_to_vector (loadf64 addr:$src))))))),
- (PMOVSXWDrm addr:$src)>;
- def : Pat<(v4i32 (X86vsext (v16i8 (bitconvert (v4i32
- (scalar_to_vector (loadi32 addr:$src))))))),
- (PMOVSXBDrm addr:$src)>;
- def : Pat<(v2i64 (X86vsext (v8i16 (bitconvert (v4i32
- (scalar_to_vector (loadi32 addr:$src))))))),
- (PMOVSXWQrm addr:$src)>;
- def : Pat<(v2i64 (X86vsext (v16i8 (bitconvert (v4i32
- (scalar_to_vector (extloadi32i16 addr:$src))))))),
- (PMOVSXBQrm addr:$src)>;
- def : Pat<(v2i64 (X86vsext (v4i32 (bitconvert (v2i64
- (scalar_to_vector (loadi64 addr:$src))))))),
- (PMOVSXDQrm addr:$src)>;
- def : Pat<(v2i64 (X86vsext (v4i32 (bitconvert (v2f64
- (scalar_to_vector (loadf64 addr:$src))))))),
- (PMOVSXDQrm addr:$src)>;
- def : Pat<(v8i16 (X86vsext (v16i8 (bitconvert (v2i64
- (scalar_to_vector (loadi64 addr:$src))))))),
- (PMOVSXBWrm addr:$src)>;
- def : Pat<(v8i16 (X86vsext (v16i8 (bitconvert (v2f64
- (scalar_to_vector (loadf64 addr:$src))))))),
- (PMOVSXBWrm addr:$src)>;
-}
-
-let Predicates = [HasAVX2] in {
- def : Pat<(v16i16 (X86vzext (v16i8 VR128:$src))), (VPMOVZXBWYrr VR128:$src)>;
- def : Pat<(v8i32 (X86vzext (v16i8 VR128:$src))), (VPMOVZXBDYrr VR128:$src)>;
- def : Pat<(v4i64 (X86vzext (v16i8 VR128:$src))), (VPMOVZXBQYrr VR128:$src)>;
-
- def : Pat<(v8i32 (X86vzext (v8i16 VR128:$src))), (VPMOVZXWDYrr VR128:$src)>;
- def : Pat<(v4i64 (X86vzext (v8i16 VR128:$src))), (VPMOVZXWQYrr VR128:$src)>;
-
- def : Pat<(v4i64 (X86vzext (v4i32 VR128:$src))), (VPMOVZXDQYrr VR128:$src)>;
-
- def : Pat<(v16i16 (X86vzext (v32i8 VR256:$src))),
- (VPMOVZXBWYrr (EXTRACT_SUBREG VR256:$src, sub_xmm))>;
- def : Pat<(v8i32 (X86vzext (v32i8 VR256:$src))),
- (VPMOVZXBDYrr (EXTRACT_SUBREG VR256:$src, sub_xmm))>;
- def : Pat<(v4i64 (X86vzext (v32i8 VR256:$src))),
- (VPMOVZXBQYrr (EXTRACT_SUBREG VR256:$src, sub_xmm))>;
-
- def : Pat<(v8i32 (X86vzext (v16i16 VR256:$src))),
- (VPMOVZXWDYrr (EXTRACT_SUBREG VR256:$src, sub_xmm))>;
- def : Pat<(v4i64 (X86vzext (v16i16 VR256:$src))),
- (VPMOVZXWQYrr (EXTRACT_SUBREG VR256:$src, sub_xmm))>;
-
- def : Pat<(v4i64 (X86vzext (v8i32 VR256:$src))),
- (VPMOVZXDQYrr (EXTRACT_SUBREG VR256:$src, sub_xmm))>;
-}
-
-let Predicates = [HasAVX] in {
- def : Pat<(v8i16 (X86vzext (v16i8 VR128:$src))), (VPMOVZXBWrr VR128:$src)>;
- def : Pat<(v4i32 (X86vzext (v16i8 VR128:$src))), (VPMOVZXBDrr VR128:$src)>;
- def : Pat<(v2i64 (X86vzext (v16i8 VR128:$src))), (VPMOVZXBQrr VR128:$src)>;
-
- def : Pat<(v4i32 (X86vzext (v8i16 VR128:$src))), (VPMOVZXWDrr VR128:$src)>;
- def : Pat<(v2i64 (X86vzext (v8i16 VR128:$src))), (VPMOVZXWQrr VR128:$src)>;
-
- def : Pat<(v2i64 (X86vzext (v4i32 VR128:$src))), (VPMOVZXDQrr VR128:$src)>;
-
- def : Pat<(v8i16 (X86vzext (v16i8 (bitconvert (v2i64 (scalar_to_vector (loadi64 addr:$src))))))),
- (VPMOVZXBWrm addr:$src)>;
- def : Pat<(v8i16 (X86vzext (v16i8 (bitconvert (v2f64 (scalar_to_vector (loadf64 addr:$src))))))),
- (VPMOVZXBWrm addr:$src)>;
- def : Pat<(v4i32 (X86vzext (v16i8 (bitconvert (v4i32 (scalar_to_vector (loadi32 addr:$src))))))),
- (VPMOVZXBDrm addr:$src)>;
- def : Pat<(v2i64 (X86vzext (v16i8 (bitconvert (v4i32 (scalar_to_vector (loadi16_anyext addr:$src))))))),
- (VPMOVZXBQrm addr:$src)>;
-
- def : Pat<(v4i32 (X86vzext (v8i16 (bitconvert (v2i64 (scalar_to_vector (loadi64 addr:$src))))))),
- (VPMOVZXWDrm addr:$src)>;
- def : Pat<(v4i32 (X86vzext (v8i16 (bitconvert (v2f64 (scalar_to_vector (loadf64 addr:$src))))))),
- (VPMOVZXWDrm addr:$src)>;
- def : Pat<(v2i64 (X86vzext (v8i16 (bitconvert (v4i32 (scalar_to_vector (loadi32 addr:$src))))))),
- (VPMOVZXWQrm addr:$src)>;
-
- def : Pat<(v2i64 (X86vzext (v4i32 (bitconvert (v2i64 (scalar_to_vector (loadi64 addr:$src))))))),
- (VPMOVZXDQrm addr:$src)>;
- def : Pat<(v2i64 (X86vzext (v4i32 (bitconvert (v2f64 (scalar_to_vector (loadf64 addr:$src))))))),
- (VPMOVZXDQrm addr:$src)>;
- def : Pat<(v2i64 (X86vzext (v4i32 (bitconvert (v2i64 (X86vzload addr:$src)))))),
- (VPMOVZXDQrm addr:$src)>;
-
- def : Pat<(v8i16 (X86vsext (v16i8 VR128:$src))), (VPMOVSXBWrr VR128:$src)>;
- def : Pat<(v4i32 (X86vsext (v16i8 VR128:$src))), (VPMOVSXBDrr VR128:$src)>;
- def : Pat<(v2i64 (X86vsext (v16i8 VR128:$src))), (VPMOVSXBQrr VR128:$src)>;
-
- def : Pat<(v4i32 (X86vsext (v8i16 VR128:$src))), (VPMOVSXWDrr VR128:$src)>;
- def : Pat<(v2i64 (X86vsext (v8i16 VR128:$src))), (VPMOVSXWQrr VR128:$src)>;
-
- def : Pat<(v2i64 (X86vsext (v4i32 VR128:$src))), (VPMOVSXDQrr VR128:$src)>;
-
- def : Pat<(v4i32 (X86vsext (v8i16 (bitconvert (v2i64
- (scalar_to_vector (loadi64 addr:$src))))))),
- (VPMOVSXWDrm addr:$src)>;
- def : Pat<(v2i64 (X86vsext (v4i32 (bitconvert (v2i64
- (scalar_to_vector (loadi64 addr:$src))))))),
- (VPMOVSXDQrm addr:$src)>;
- def : Pat<(v4i32 (X86vsext (v8i16 (bitconvert (v2f64
- (scalar_to_vector (loadf64 addr:$src))))))),
- (VPMOVSXWDrm addr:$src)>;
- def : Pat<(v2i64 (X86vsext (v4i32 (bitconvert (v2f64
- (scalar_to_vector (loadf64 addr:$src))))))),
- (VPMOVSXDQrm addr:$src)>;
- def : Pat<(v8i16 (X86vsext (v16i8 (bitconvert (v2i64
- (scalar_to_vector (loadi64 addr:$src))))))),
- (VPMOVSXBWrm addr:$src)>;
- def : Pat<(v8i16 (X86vsext (v16i8 (bitconvert (v2f64
- (scalar_to_vector (loadf64 addr:$src))))))),
- (VPMOVSXBWrm addr:$src)>;
-
- def : Pat<(v4i32 (X86vsext (v16i8 (bitconvert (v4i32
- (scalar_to_vector (loadi32 addr:$src))))))),
- (VPMOVSXBDrm addr:$src)>;
- def : Pat<(v2i64 (X86vsext (v8i16 (bitconvert (v4i32
- (scalar_to_vector (loadi32 addr:$src))))))),
- (VPMOVSXWQrm addr:$src)>;
- def : Pat<(v2i64 (X86vsext (v16i8 (bitconvert (v4i32
- (scalar_to_vector (extloadi32i16 addr:$src))))))),
- (VPMOVSXBQrm addr:$src)>;