//===-- X86InstrAVX512.td - AVX512 Instruction Set ---------*- tablegen -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This file describes the X86 AVX512 instruction set, defining the // instructions, and properties of the instructions which are needed for code // generation, machine code emission, and analysis. // //===----------------------------------------------------------------------===// // Group template arguments that can be derived from the vector type (EltNum x // EltVT). These are things like the register class for the writemask, etc. // The idea is to pass one of these as the template argument rather than the // individual arguments. // The template is also used for scalar types, in this case numelts is 1. class X86VectorVTInfo { RegisterClass RC = rc; ValueType EltVT = eltvt; int NumElts = numelts; // Corresponding mask register class. RegisterClass KRC = !cast("VK" # NumElts); // Corresponding write-mask register class. RegisterClass KRCWM = !cast("VK" # NumElts # "WM"); // The GPR register class that can hold the write mask. Use GR8 for fewer // than 8 elements. Use shift-right and equal to work around the lack of // !lt in tablegen. RegisterClass MRC = !cast("GR" # !if (!eq (!srl(NumElts, 3), 0), 8, NumElts)); // Suffix used in the instruction mnemonic. string Suffix = suffix; // VTName is a string name for vector VT. For vector types it will be // v # NumElts # EltVT, so for vector of 8 elements of i32 it will be v8i32 // It is a little bit complex for scalar types, where NumElts = 1. // In this case we build v4f32 or v2f64 string VTName = "v" # !if (!eq (NumElts, 1), !if (!eq (EltVT.Size, 32), 4, !if (!eq (EltVT.Size, 64), 2, NumElts)), NumElts) # EltVT; // The vector VT. ValueType VT = !cast(VTName); string EltTypeName = !cast(EltVT); // Size of the element type in bits, e.g. 32 for v16i32. string EltSizeName = !subst("i", "", !subst("f", "", EltTypeName)); int EltSize = EltVT.Size; // "i" for integer types and "f" for floating-point types string TypeVariantName = !subst(EltSizeName, "", EltTypeName); // Size of RC in bits, e.g. 512 for VR512. int Size = VT.Size; // The corresponding memory operand, e.g. i512mem for VR512. X86MemOperand MemOp = !cast(TypeVariantName # Size # "mem"); X86MemOperand ScalarMemOp = !cast(EltVT # "mem"); // Load patterns // Note: For 128/256-bit integer VT we choose loadv2i64/loadv4i64 // due to load promotion during legalization PatFrag LdFrag = !cast("load" # !if (!eq (TypeVariantName, "i"), !if (!eq (Size, 128), "v2i64", !if (!eq (Size, 256), "v4i64", VTName)), VTName)); PatFrag AlignedLdFrag = !cast("alignedload" # !if (!eq (TypeVariantName, "i"), !if (!eq (Size, 128), "v2i64", !if (!eq (Size, 256), "v4i64", !if (!eq (Size, 512), !if (!eq (EltSize, 64), "v8i64", "v16i32"), VTName))), VTName)); PatFrag ScalarLdFrag = !cast("load" # EltVT); // The corresponding float type, e.g. v16f32 for v16i32 // Note: For EltSize < 32, FloatVT is illegal and TableGen // fails to compile, so we choose FloatVT = VT ValueType FloatVT = !cast( !if (!eq (!srl(EltSize,5),0), VTName, !if (!eq(TypeVariantName, "i"), "v" # NumElts # "f" # EltSize, VTName))); // The string to specify embedded broadcast in assembly. string BroadcastStr = "{1to" # NumElts # "}"; // 8-bit compressed displacement tuple/subvector format. This is only // defined for NumElts <= 8. CD8VForm CD8TupleForm = !if (!eq (!srl(NumElts, 4), 0), !cast("CD8VT" # NumElts), ?); SubRegIndex SubRegIdx = !if (!eq (Size, 128), sub_xmm, !if (!eq (Size, 256), sub_ymm, ?)); Domain ExeDomain = !if (!eq (EltTypeName, "f32"), SSEPackedSingle, !if (!eq (EltTypeName, "f64"), SSEPackedDouble, SSEPackedInt)); RegisterClass FRC = !if (!eq (EltTypeName, "f32"), FR32X, FR64X); // A vector type of the same width with element type i32. This is used to // create the canonical constant zero node ImmAllZerosV. ValueType i32VT = !cast("v" # !srl(Size, 5) # "i32"); dag ImmAllZerosV = (VT (bitconvert (i32VT immAllZerosV))); string ZSuffix = !if (!eq (Size, 128), "Z128", !if (!eq (Size, 256), "Z256", "Z")); } def v64i8_info : X86VectorVTInfo<64, i8, VR512, "b">; def v32i16_info : X86VectorVTInfo<32, i16, VR512, "w">; def v16i32_info : X86VectorVTInfo<16, i32, VR512, "d">; def v8i64_info : X86VectorVTInfo<8, i64, VR512, "q">; def v16f32_info : X86VectorVTInfo<16, f32, VR512, "ps">; def v8f64_info : X86VectorVTInfo<8, f64, VR512, "pd">; // "x" in v32i8x_info means RC = VR256X def v32i8x_info : X86VectorVTInfo<32, i8, VR256X, "b">; def v16i16x_info : X86VectorVTInfo<16, i16, VR256X, "w">; def v8i32x_info : X86VectorVTInfo<8, i32, VR256X, "d">; def v4i64x_info : X86VectorVTInfo<4, i64, VR256X, "q">; def v8f32x_info : X86VectorVTInfo<8, f32, VR256X, "ps">; def v4f64x_info : X86VectorVTInfo<4, f64, VR256X, "pd">; def v16i8x_info : X86VectorVTInfo<16, i8, VR128X, "b">; def v8i16x_info : X86VectorVTInfo<8, i16, VR128X, "w">; def v4i32x_info : X86VectorVTInfo<4, i32, VR128X, "d">; def v2i64x_info : X86VectorVTInfo<2, i64, VR128X, "q">; def v4f32x_info : X86VectorVTInfo<4, f32, VR128X, "ps">; def v2f64x_info : X86VectorVTInfo<2, f64, VR128X, "pd">; // We map scalar types to the smallest (128-bit) vector type // with the appropriate element type. This allows to use the same masking logic. def i32x_info : X86VectorVTInfo<1, i32, GR32, "si">; def i64x_info : X86VectorVTInfo<1, i64, GR64, "sq">; def f32x_info : X86VectorVTInfo<1, f32, VR128X, "ss">; def f64x_info : X86VectorVTInfo<1, f64, VR128X, "sd">; class AVX512VLVectorVTInfo { X86VectorVTInfo info512 = i512; X86VectorVTInfo info256 = i256; X86VectorVTInfo info128 = i128; } def avx512vl_i8_info : AVX512VLVectorVTInfo; def avx512vl_i16_info : AVX512VLVectorVTInfo; def avx512vl_i32_info : AVX512VLVectorVTInfo; def avx512vl_i64_info : AVX512VLVectorVTInfo; def avx512vl_f32_info : AVX512VLVectorVTInfo; def avx512vl_f64_info : AVX512VLVectorVTInfo; // This multiclass generates the masking variants from the non-masking // variant. It only provides the assembly pieces for the masking variants. // It assumes custom ISel patterns for masking which can be provided as // template arguments. multiclass AVX512_maskable_custom O, Format F, dag Outs, dag Ins, dag MaskingIns, dag ZeroMaskingIns, string OpcodeStr, string AttSrcAsm, string IntelSrcAsm, list Pattern, list MaskingPattern, list ZeroMaskingPattern, string MaskingConstraint = "", InstrItinClass itin = NoItinerary, bit IsCommutable = 0> { let isCommutable = IsCommutable in def NAME: AVX512; // Prefer over VMOV*rrk Pat<> let AddedComplexity = 20 in def NAME#k: AVX512, EVEX_K { // In case of the 3src subclass this is overridden with a let. string Constraints = MaskingConstraint; } let AddedComplexity = 30 in // Prefer over VMOV*rrkz Pat<> def NAME#kz: AVX512, EVEX_KZ; } // Common base class of AVX512_maskable and AVX512_maskable_3src. multiclass AVX512_maskable_common O, Format F, X86VectorVTInfo _, dag Outs, dag Ins, dag MaskingIns, dag ZeroMaskingIns, string OpcodeStr, string AttSrcAsm, string IntelSrcAsm, dag RHS, dag MaskingRHS, SDNode Select = vselect, string MaskingConstraint = "", InstrItinClass itin = NoItinerary, bit IsCommutable = 0> : AVX512_maskable_custom; // This multiclass generates the unconditional/non-masking, the masking and // the zero-masking variant of the vector instruction. In the masking case, the // perserved vector elements come from a new dummy input operand tied to $dst. multiclass AVX512_maskable O, Format F, X86VectorVTInfo _, dag Outs, dag Ins, string OpcodeStr, string AttSrcAsm, string IntelSrcAsm, dag RHS, InstrItinClass itin = NoItinerary, bit IsCommutable = 0> : AVX512_maskable_common; // This multiclass generates the unconditional/non-masking, the masking and // the zero-masking variant of the scalar instruction. multiclass AVX512_maskable_scalar O, Format F, X86VectorVTInfo _, dag Outs, dag Ins, string OpcodeStr, string AttSrcAsm, string IntelSrcAsm, dag RHS, InstrItinClass itin = NoItinerary, bit IsCommutable = 0> : AVX512_maskable_common; // Similar to AVX512_maskable but in this case one of the source operands // ($src1) is already tied to $dst so we just use that for the preserved // vector elements. NOTE that the NonTiedIns (the ins dag) should exclude // $src1. multiclass AVX512_maskable_3src O, Format F, X86VectorVTInfo _, dag Outs, dag NonTiedIns, string OpcodeStr, string AttSrcAsm, string IntelSrcAsm, dag RHS> : AVX512_maskable_common; // Similar to AVX512_maskable_3rc but in this case the input VT for the tied // operand differs from the output VT. This requires a bitconvert on // the preserved vector going into the vselect. multiclass AVX512_maskable_3src_cast O, Format F, X86VectorVTInfo OutVT, X86VectorVTInfo InVT, dag Outs, dag NonTiedIns, string OpcodeStr, string AttSrcAsm, string IntelSrcAsm, dag RHS> : AVX512_maskable_common; multiclass AVX512_maskable_3src_scalar O, Format F, X86VectorVTInfo _, dag Outs, dag NonTiedIns, string OpcodeStr, string AttSrcAsm, string IntelSrcAsm, dag RHS> : AVX512_maskable_common; multiclass AVX512_maskable_in_asm O, Format F, X86VectorVTInfo _, dag Outs, dag Ins, string OpcodeStr, string AttSrcAsm, string IntelSrcAsm, list Pattern> : AVX512_maskable_custom; // Instruction with mask that puts result in mask register, // like "compare" and "vptest" multiclass AVX512_maskable_custom_cmp O, Format F, dag Outs, dag Ins, dag MaskingIns, string OpcodeStr, string AttSrcAsm, string IntelSrcAsm, list Pattern, list MaskingPattern, string Round = "", InstrItinClass itin = NoItinerary> { def NAME: AVX512; def NAME#k: AVX512, EVEX_K; } multiclass AVX512_maskable_common_cmp O, Format F, X86VectorVTInfo _, dag Outs, dag Ins, dag MaskingIns, string OpcodeStr, string AttSrcAsm, string IntelSrcAsm, dag RHS, dag MaskingRHS, string Round = "", InstrItinClass itin = NoItinerary> : AVX512_maskable_custom_cmp; multiclass AVX512_maskable_cmp O, Format F, X86VectorVTInfo _, dag Outs, dag Ins, string OpcodeStr, string AttSrcAsm, string IntelSrcAsm, dag RHS, string Round = "", InstrItinClass itin = NoItinerary> : AVX512_maskable_common_cmp; multiclass AVX512_maskable_cmp_alt O, Format F, X86VectorVTInfo _, dag Outs, dag Ins, string OpcodeStr, string AttSrcAsm, string IntelSrcAsm> : AVX512_maskable_custom_cmp; // Bitcasts between 512-bit vector types. Return the original type since // no instruction is needed for the conversion let Predicates = [HasAVX512] in { def : Pat<(v8f64 (bitconvert (v8i64 VR512:$src))), (v8f64 VR512:$src)>; def : Pat<(v8f64 (bitconvert (v16i32 VR512:$src))), (v8f64 VR512:$src)>; def : Pat<(v8f64 (bitconvert (v32i16 VR512:$src))), (v8f64 VR512:$src)>; def : Pat<(v8f64 (bitconvert (v64i8 VR512:$src))), (v8f64 VR512:$src)>; def : Pat<(v8f64 (bitconvert (v16f32 VR512:$src))), (v8f64 VR512:$src)>; def : Pat<(v16f32 (bitconvert (v8i64 VR512:$src))), (v16f32 VR512:$src)>; def : Pat<(v16f32 (bitconvert (v16i32 VR512:$src))), (v16f32 VR512:$src)>; def : Pat<(v16f32 (bitconvert (v32i16 VR512:$src))), (v16f32 VR512:$src)>; def : Pat<(v16f32 (bitconvert (v64i8 VR512:$src))), (v16f32 VR512:$src)>; def : Pat<(v16f32 (bitconvert (v8f64 VR512:$src))), (v16f32 VR512:$src)>; def : Pat<(v8i64 (bitconvert (v16i32 VR512:$src))), (v8i64 VR512:$src)>; def : Pat<(v8i64 (bitconvert (v32i16 VR512:$src))), (v8i64 VR512:$src)>; def : Pat<(v8i64 (bitconvert (v64i8 VR512:$src))), (v8i64 VR512:$src)>; def : Pat<(v8i64 (bitconvert (v8f64 VR512:$src))), (v8i64 VR512:$src)>; def : Pat<(v8i64 (bitconvert (v16f32 VR512:$src))), (v8i64 VR512:$src)>; def : Pat<(v16i32 (bitconvert (v8i64 VR512:$src))), (v16i32 VR512:$src)>; def : Pat<(v16i32 (bitconvert (v16f32 VR512:$src))), (v16i32 VR512:$src)>; def : Pat<(v16i32 (bitconvert (v32i16 VR512:$src))), (v16i32 VR512:$src)>; def : Pat<(v16i32 (bitconvert (v64i8 VR512:$src))), (v16i32 VR512:$src)>; def : Pat<(v16i32 (bitconvert (v8f64 VR512:$src))), (v16i32 VR512:$src)>; def : Pat<(v32i16 (bitconvert (v8i64 VR512:$src))), (v32i16 VR512:$src)>; def : Pat<(v32i16 (bitconvert (v16i32 VR512:$src))), (v32i16 VR512:$src)>; def : Pat<(v32i16 (bitconvert (v64i8 VR512:$src))), (v32i16 VR512:$src)>; def : Pat<(v32i16 (bitconvert (v8f64 VR512:$src))), (v32i16 VR512:$src)>; def : Pat<(v32i16 (bitconvert (v16f32 VR512:$src))), (v32i16 VR512:$src)>; def : Pat<(v32i16 (bitconvert (v16f32 VR512:$src))), (v32i16 VR512:$src)>; def : Pat<(v64i8 (bitconvert (v8i64 VR512:$src))), (v64i8 VR512:$src)>; def : Pat<(v64i8 (bitconvert (v16i32 VR512:$src))), (v64i8 VR512:$src)>; def : Pat<(v64i8 (bitconvert (v32i16 VR512:$src))), (v64i8 VR512:$src)>; def : Pat<(v64i8 (bitconvert (v8f64 VR512:$src))), (v64i8 VR512:$src)>; def : Pat<(v64i8 (bitconvert (v16f32 VR512:$src))), (v64i8 VR512:$src)>; def : Pat<(v2i64 (bitconvert (v4i32 VR128X:$src))), (v2i64 VR128X:$src)>; def : Pat<(v2i64 (bitconvert (v8i16 VR128X:$src))), (v2i64 VR128X:$src)>; def : Pat<(v2i64 (bitconvert (v16i8 VR128X:$src))), (v2i64 VR128X:$src)>; def : Pat<(v2i64 (bitconvert (v2f64 VR128X:$src))), (v2i64 VR128X:$src)>; def : Pat<(v2i64 (bitconvert (v4f32 VR128X:$src))), (v2i64 VR128X:$src)>; def : Pat<(v4i32 (bitconvert (v2i64 VR128X:$src))), (v4i32 VR128X:$src)>; def : Pat<(v4i32 (bitconvert (v8i16 VR128X:$src))), (v4i32 VR128X:$src)>; def : Pat<(v4i32 (bitconvert (v16i8 VR128X:$src))), (v4i32 VR128X:$src)>; def : Pat<(v4i32 (bitconvert (v2f64 VR128X:$src))), (v4i32 VR128X:$src)>; def : Pat<(v4i32 (bitconvert (v4f32 VR128X:$src))), (v4i32 VR128X:$src)>; def : Pat<(v8i16 (bitconvert (v2i64 VR128X:$src))), (v8i16 VR128X:$src)>; def : Pat<(v8i16 (bitconvert (v4i32 VR128X:$src))), (v8i16 VR128X:$src)>; def : Pat<(v8i16 (bitconvert (v16i8 VR128X:$src))), (v8i16 VR128X:$src)>; def : Pat<(v8i16 (bitconvert (v2f64 VR128X:$src))), (v8i16 VR128X:$src)>; def : Pat<(v8i16 (bitconvert (v4f32 VR128X:$src))), (v8i16 VR128X:$src)>; def : Pat<(v16i8 (bitconvert (v2i64 VR128X:$src))), (v16i8 VR128X:$src)>; def : Pat<(v16i8 (bitconvert (v4i32 VR128X:$src))), (v16i8 VR128X:$src)>; def : Pat<(v16i8 (bitconvert (v8i16 VR128X:$src))), (v16i8 VR128X:$src)>; def : Pat<(v16i8 (bitconvert (v2f64 VR128X:$src))), (v16i8 VR128X:$src)>; def : Pat<(v16i8 (bitconvert (v4f32 VR128X:$src))), (v16i8 VR128X:$src)>; def : Pat<(v4f32 (bitconvert (v2i64 VR128X:$src))), (v4f32 VR128X:$src)>; def : Pat<(v4f32 (bitconvert (v4i32 VR128X:$src))), (v4f32 VR128X:$src)>; def : Pat<(v4f32 (bitconvert (v8i16 VR128X:$src))), (v4f32 VR128X:$src)>; def : Pat<(v4f32 (bitconvert (v16i8 VR128X:$src))), (v4f32 VR128X:$src)>; def : Pat<(v4f32 (bitconvert (v2f64 VR128X:$src))), (v4f32 VR128X:$src)>; def : Pat<(v2f64 (bitconvert (v2i64 VR128X:$src))), (v2f64 VR128X:$src)>; def : Pat<(v2f64 (bitconvert (v4i32 VR128X:$src))), (v2f64 VR128X:$src)>; def : Pat<(v2f64 (bitconvert (v8i16 VR128X:$src))), (v2f64 VR128X:$src)>; def : Pat<(v2f64 (bitconvert (v16i8 VR128X:$src))), (v2f64 VR128X:$src)>; def : Pat<(v2f64 (bitconvert (v4f32 VR128X:$src))), (v2f64 VR128X:$src)>; // Bitcasts between 256-bit vector types. Return the original type since // no instruction is needed for the conversion def : Pat<(v4f64 (bitconvert (v8f32 VR256X:$src))), (v4f64 VR256X:$src)>; def : Pat<(v4f64 (bitconvert (v8i32 VR256X:$src))), (v4f64 VR256X:$src)>; def : Pat<(v4f64 (bitconvert (v4i64 VR256X:$src))), (v4f64 VR256X:$src)>; def : Pat<(v4f64 (bitconvert (v16i16 VR256X:$src))), (v4f64 VR256X:$src)>; def : Pat<(v4f64 (bitconvert (v32i8 VR256X:$src))), (v4f64 VR256X:$src)>; def : Pat<(v8f32 (bitconvert (v8i32 VR256X:$src))), (v8f32 VR256X:$src)>; def : Pat<(v8f32 (bitconvert (v4i64 VR256X:$src))), (v8f32 VR256X:$src)>; def : Pat<(v8f32 (bitconvert (v4f64 VR256X:$src))), (v8f32 VR256X:$src)>; def : Pat<(v8f32 (bitconvert (v32i8 VR256X:$src))), (v8f32 VR256X:$src)>; def : Pat<(v8f32 (bitconvert (v16i16 VR256X:$src))), (v8f32 VR256X:$src)>; def : Pat<(v4i64 (bitconvert (v8f32 VR256X:$src))), (v4i64 VR256X:$src)>; def : Pat<(v4i64 (bitconvert (v8i32 VR256X:$src))), (v4i64 VR256X:$src)>; def : Pat<(v4i64 (bitconvert (v4f64 VR256X:$src))), (v4i64 VR256X:$src)>; def : Pat<(v4i64 (bitconvert (v32i8 VR256X:$src))), (v4i64 VR256X:$src)>; def : Pat<(v4i64 (bitconvert (v16i16 VR256X:$src))), (v4i64 VR256X:$src)>; def : Pat<(v32i8 (bitconvert (v4f64 VR256X:$src))), (v32i8 VR256X:$src)>; def : Pat<(v32i8 (bitconvert (v4i64 VR256X:$src))), (v32i8 VR256X:$src)>; def : Pat<(v32i8 (bitconvert (v8f32 VR256X:$src))), (v32i8 VR256X:$src)>; def : Pat<(v32i8 (bitconvert (v8i32 VR256X:$src))), (v32i8 VR256X:$src)>; def : Pat<(v32i8 (bitconvert (v16i16 VR256X:$src))), (v32i8 VR256X:$src)>; def : Pat<(v8i32 (bitconvert (v32i8 VR256X:$src))), (v8i32 VR256X:$src)>; def : Pat<(v8i32 (bitconvert (v16i16 VR256X:$src))), (v8i32 VR256X:$src)>; def : Pat<(v8i32 (bitconvert (v8f32 VR256X:$src))), (v8i32 VR256X:$src)>; def : Pat<(v8i32 (bitconvert (v4i64 VR256X:$src))), (v8i32 VR256X:$src)>; def : Pat<(v8i32 (bitconvert (v4f64 VR256X:$src))), (v8i32 VR256X:$src)>; def : Pat<(v16i16 (bitconvert (v8f32 VR256X:$src))), (v16i16 VR256X:$src)>; def : Pat<(v16i16 (bitconvert (v8i32 VR256X:$src))), (v16i16 VR256X:$src)>; def : Pat<(v16i16 (bitconvert (v4i64 VR256X:$src))), (v16i16 VR256X:$src)>; def : Pat<(v16i16 (bitconvert (v4f64 VR256X:$src))), (v16i16 VR256X:$src)>; def : Pat<(v16i16 (bitconvert (v32i8 VR256X:$src))), (v16i16 VR256X:$src)>; } // // AVX-512: VPXOR instruction writes zero to its upper part, it's safe build zeros. // let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1, isPseudo = 1, Predicates = [HasAVX512] in { def AVX512_512_SET0 : I<0, Pseudo, (outs VR512:$dst), (ins), "", [(set VR512:$dst, (v16f32 immAllZerosV))]>; } let Predicates = [HasAVX512] in { def : Pat<(v8i64 immAllZerosV), (AVX512_512_SET0)>; def : Pat<(v16i32 immAllZerosV), (AVX512_512_SET0)>; def : Pat<(v8f64 immAllZerosV), (AVX512_512_SET0)>; } //===----------------------------------------------------------------------===// // AVX-512 - VECTOR INSERT // multiclass vinsert_for_size { let hasSideEffects = 0, ExeDomain = To.ExeDomain in { defm rr : AVX512_maskable, AVX512AIi8Base, EVEX_4V; let mayLoad = 1 in defm rm : AVX512_maskable, AVX512AIi8Base, EVEX_4V, EVEX_CD8; } } multiclass vinsert_for_size_lowering p> { let Predicates = p in { def : Pat<(vinsert_insert:$ins (To.VT To.RC:$src1), (From.VT From.RC:$src2), (iPTR imm)), (To.VT (!cast(InstrStr#"rr") To.RC:$src1, From.RC:$src2, (INSERT_get_vinsert_imm To.RC:$ins)))>; def : Pat<(vinsert_insert:$ins (To.VT To.RC:$src1), (From.VT (bitconvert (From.LdFrag addr:$src2))), (iPTR imm)), (To.VT (!cast(InstrStr#"rm") To.RC:$src1, addr:$src2, (INSERT_get_vinsert_imm To.RC:$ins)))>; } } multiclass vinsert_for_type { let Predicates = [HasVLX] in defm NAME # "32x4Z256" : vinsert_for_size, X86VectorVTInfo< 8, EltVT32, VR256X>, vinsert128_insert>, EVEX_V256; defm NAME # "32x4Z" : vinsert_for_size, X86VectorVTInfo<16, EltVT32, VR512>, vinsert128_insert>, EVEX_V512; defm NAME # "64x4Z" : vinsert_for_size, X86VectorVTInfo< 8, EltVT64, VR512>, vinsert256_insert>, VEX_W, EVEX_V512; let Predicates = [HasVLX, HasDQI] in defm NAME # "64x2Z256" : vinsert_for_size, X86VectorVTInfo< 4, EltVT64, VR256X>, vinsert128_insert>, VEX_W, EVEX_V256; let Predicates = [HasDQI] in { defm NAME # "64x2Z" : vinsert_for_size, X86VectorVTInfo< 8, EltVT64, VR512>, vinsert128_insert>, VEX_W, EVEX_V512; defm NAME # "32x8Z" : vinsert_for_size, X86VectorVTInfo<16, EltVT32, VR512>, vinsert256_insert>, EVEX_V512; } } defm VINSERTF : vinsert_for_type; defm VINSERTI : vinsert_for_type; // Codegen pattern with the alternative types, // Only add this if 64x2 and its friends are not supported natively via AVX512DQ. defm : vinsert_for_size_lowering<"VINSERTF32x4Z256", v2f64x_info, v4f64x_info, vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX, NoDQI]>; defm : vinsert_for_size_lowering<"VINSERTI32x4Z256", v2i64x_info, v4i64x_info, vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX, NoDQI]>; defm : vinsert_for_size_lowering<"VINSERTF32x4Z", v2f64x_info, v8f64_info, vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512, NoDQI]>; defm : vinsert_for_size_lowering<"VINSERTI32x4Z", v2i64x_info, v8i64_info, vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512, NoDQI]>; defm : vinsert_for_size_lowering<"VINSERTF64x4Z", v8f32x_info, v16f32_info, vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512, NoDQI]>; defm : vinsert_for_size_lowering<"VINSERTI64x4Z", v8i32x_info, v16i32_info, vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512, NoDQI]>; // Codegen pattern with the alternative types insert VEC128 into VEC256 defm : vinsert_for_size_lowering<"VINSERTI32x4Z256", v8i16x_info, v16i16x_info, vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX]>; defm : vinsert_for_size_lowering<"VINSERTI32x4Z256", v16i8x_info, v32i8x_info, vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX]>; // Codegen pattern with the alternative types insert VEC128 into VEC512 defm : vinsert_for_size_lowering<"VINSERTI32x4Z", v8i16x_info, v32i16_info, vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512]>; defm : vinsert_for_size_lowering<"VINSERTI32x4Z", v16i8x_info, v64i8_info, vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512]>; // Codegen pattern with the alternative types insert VEC256 into VEC512 defm : vinsert_for_size_lowering<"VINSERTI64x4Z", v16i16x_info, v32i16_info, vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512]>; defm : vinsert_for_size_lowering<"VINSERTI64x4Z", v32i8x_info, v64i8_info, vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512]>; // vinsertps - insert f32 to XMM def VINSERTPSzrr : AVX512AIi8<0x21, MRMSrcReg, (outs VR128X:$dst), (ins VR128X:$src1, VR128X:$src2, u8imm:$src3), "vinsertps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", [(set VR128X:$dst, (X86insertps VR128X:$src1, VR128X:$src2, imm:$src3))]>, EVEX_4V; def VINSERTPSzrm: AVX512AIi8<0x21, MRMSrcMem, (outs VR128X:$dst), (ins VR128X:$src1, f32mem:$src2, u8imm:$src3), "vinsertps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", [(set VR128X:$dst, (X86insertps VR128X:$src1, (v4f32 (scalar_to_vector (loadf32 addr:$src2))), imm:$src3))]>, EVEX_4V, EVEX_CD8<32, CD8VT1>; //===----------------------------------------------------------------------===// // AVX-512 VECTOR EXTRACT //--- multiclass vextract_for_size_first_position_lowering { // A subvector extract from the first vector position is // a subregister copy that needs no instruction. def NAME # To.NumElts: Pat<(To.VT (extract_subvector (From.VT From.RC:$src),(iPTR 0))), (To.VT (EXTRACT_SUBREG (From.VT From.RC:$src), To.SubRegIdx))>; } multiclass vextract_for_size : vextract_for_size_first_position_lowering { let hasSideEffects = 0, ExeDomain = To.ExeDomain in { // use AVX512_maskable_in_asm (AVX512_maskable can't be used due to // vextract_extract), we interesting only in patterns without mask, // intrinsics pattern match generated bellow. defm rr : AVX512_maskable_in_asm, AVX512AIi8Base, EVEX; let mayStore = 1 in { def rm : AVX512AIi8, EVEX; def rmk : AVX512AIi8, EVEX_K, EVEX; }//mayStore = 1 } // Intrinsic call with masking. def : Pat<(!cast("int_x86_avx512_mask_vextract" # To.EltTypeName # "x" # To.NumElts # "_" # From.Size) From.RC:$src1, (iPTR imm:$idx), To.RC:$src0, To.MRC:$mask), (!cast(NAME # To.EltSize # "x" # To.NumElts # From.ZSuffix # "rrk") To.RC:$src0, (COPY_TO_REGCLASS To.MRC:$mask, To.KRCWM), From.RC:$src1, imm:$idx)>; // Intrinsic call with zero-masking. def : Pat<(!cast("int_x86_avx512_mask_vextract" # To.EltTypeName # "x" # To.NumElts # "_" # From.Size) From.RC:$src1, (iPTR imm:$idx), To.ImmAllZerosV, To.MRC:$mask), (!cast(NAME # To.EltSize # "x" # To.NumElts # From.ZSuffix # "rrkz") (COPY_TO_REGCLASS To.MRC:$mask, To.KRCWM), From.RC:$src1, imm:$idx)>; // Intrinsic call without masking. def : Pat<(!cast("int_x86_avx512_mask_vextract" # To.EltTypeName # "x" # To.NumElts # "_" # From.Size) From.RC:$src1, (iPTR imm:$idx), To.ImmAllZerosV, (i8 -1)), (!cast(NAME # To.EltSize # "x" # To.NumElts # From.ZSuffix # "rr") From.RC:$src1, imm:$idx)>; } // Codegen pattern for the alternative types multiclass vextract_for_size_lowering p> : vextract_for_size_first_position_lowering { let Predicates = p in def : Pat<(vextract_extract:$ext (From.VT From.RC:$src1), (iPTR imm)), (To.VT (!cast(InstrStr#"rr") From.RC:$src1, (EXTRACT_get_vextract_imm To.RC:$ext)))>; } multiclass vextract_for_type { defm NAME # "32x4Z" : vextract_for_size, X86VectorVTInfo< 4, EltVT32, VR128X>, vextract128_extract>, EVEX_V512, EVEX_CD8<32, CD8VT4>; defm NAME # "64x4Z" : vextract_for_size, X86VectorVTInfo< 4, EltVT64, VR256X>, vextract256_extract>, VEX_W, EVEX_V512, EVEX_CD8<64, CD8VT4>; let Predicates = [HasVLX] in defm NAME # "32x4Z256" : vextract_for_size, X86VectorVTInfo< 4, EltVT32, VR128X>, vextract128_extract>, EVEX_V256, EVEX_CD8<32, CD8VT4>; let Predicates = [HasVLX, HasDQI] in defm NAME # "64x2Z256" : vextract_for_size, X86VectorVTInfo< 2, EltVT64, VR128X>, vextract128_extract>, VEX_W, EVEX_V256, EVEX_CD8<64, CD8VT2>; let Predicates = [HasDQI] in { defm NAME # "64x2Z" : vextract_for_size, X86VectorVTInfo< 2, EltVT64, VR128X>, vextract128_extract>, VEX_W, EVEX_V512, EVEX_CD8<64, CD8VT2>; defm NAME # "32x8Z" : vextract_for_size, X86VectorVTInfo< 8, EltVT32, VR256X>, vextract256_extract>, EVEX_V512, EVEX_CD8<32, CD8VT8>; } } defm VEXTRACTF : vextract_for_type; defm VEXTRACTI : vextract_for_type; // extract_subvector codegen patterns with the alternative types. // Only add this if 64x2 and its friends are not supported natively via AVX512DQ. defm : vextract_for_size_lowering<"VEXTRACTF32x4Z", v8f64_info, v2f64x_info, vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512, NoDQI]>; defm : vextract_for_size_lowering<"VEXTRACTI32x4Z", v8i64_info, v2i64x_info, vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512, NoDQI]>; defm : vextract_for_size_lowering<"VEXTRACTF64x4Z", v16f32_info, v8f32x_info, vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512, NoDQI]>; defm : vextract_for_size_lowering<"VEXTRACTI64x4Z", v16i32_info, v8i32x_info, vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512, NoDQI]>; defm : vextract_for_size_lowering<"VEXTRACTF32x4Z256", v4f64x_info, v2f64x_info, vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX, NoDQI]>; defm : vextract_for_size_lowering<"VEXTRACTI32x4Z256", v4i64x_info, v2i64x_info, vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX, NoDQI]>; // Codegen pattern with the alternative types extract VEC128 from VEC512 defm : vextract_for_size_lowering<"VEXTRACTI32x4Z", v32i16_info, v8i16x_info, vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512]>; defm : vextract_for_size_lowering<"VEXTRACTI32x4Z", v64i8_info, v16i8x_info, vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512]>; // Codegen pattern with the alternative types extract VEC256 from VEC512 defm : vextract_for_size_lowering<"VEXTRACTI64x4Z", v32i16_info, v16i16x_info, vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512]>; defm : vextract_for_size_lowering<"VEXTRACTI64x4Z", v64i8_info, v32i8x_info, vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512]>; // A 128-bit subvector insert to the first 512-bit vector position // is a subregister copy that needs no instruction. def : Pat<(insert_subvector undef, (v2i64 VR128X:$src), (iPTR 0)), (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), (INSERT_SUBREG (v4i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm), sub_ymm)>; def : Pat<(insert_subvector undef, (v2f64 VR128X:$src), (iPTR 0)), (INSERT_SUBREG (v8f64 (IMPLICIT_DEF)), (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm), sub_ymm)>; def : Pat<(insert_subvector undef, (v4i32 VR128X:$src), (iPTR 0)), (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm), sub_ymm)>; def : Pat<(insert_subvector undef, (v4f32 VR128X:$src), (iPTR 0)), (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), (INSERT_SUBREG (v8f32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm), sub_ymm)>; def : Pat<(insert_subvector undef, (v4i64 VR256X:$src), (iPTR 0)), (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>; def : Pat<(insert_subvector undef, (v4f64 VR256X:$src), (iPTR 0)), (INSERT_SUBREG (v8f64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>; def : Pat<(insert_subvector undef, (v8i32 VR256X:$src), (iPTR 0)), (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>; def : Pat<(insert_subvector undef, (v8f32 VR256X:$src), (iPTR 0)), (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>; def : Pat<(insert_subvector undef, (v16i16 VR256X:$src), (iPTR 0)), (INSERT_SUBREG (v32i16 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>; def : Pat<(insert_subvector undef, (v32i8 VR256X:$src), (iPTR 0)), (INSERT_SUBREG (v64i8 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>; // vextractps - extract 32 bits from XMM def VEXTRACTPSzrr : AVX512AIi8<0x17, MRMDestReg, (outs GR32:$dst), (ins VR128X:$src1, u8imm:$src2), "vextractps\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set GR32:$dst, (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2))]>, EVEX; def VEXTRACTPSzmr : AVX512AIi8<0x17, MRMDestMem, (outs), (ins f32mem:$dst, VR128X:$src1, u8imm:$src2), "vextractps\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(store (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2), addr:$dst)]>, EVEX, EVEX_CD8<32, CD8VT1>; //===---------------------------------------------------------------------===// // AVX-512 BROADCAST //--- multiclass avx512_broadcast_rm opc, string OpcodeStr, X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo> { defm r : AVX512_maskable, T8PD, EVEX; let mayLoad = 1 in defm m : AVX512_maskable, T8PD, EVEX, EVEX_CD8; } multiclass avx512_fp_broadcast_vl opc, string OpcodeStr, AVX512VLVectorVTInfo _> { defm Z : avx512_broadcast_rm, EVEX_V512; let Predicates = [HasVLX] in { defm Z256 : avx512_broadcast_rm, EVEX_V256; } } let ExeDomain = SSEPackedSingle in { defm VBROADCASTSS : avx512_fp_broadcast_vl<0x18, "vbroadcastss", avx512vl_f32_info>; let Predicates = [HasVLX] in { defm VBROADCASTSSZ128 : avx512_broadcast_rm<0x18, "vbroadcastss", v4f32x_info, v4f32x_info>, EVEX_V128; } } let ExeDomain = SSEPackedDouble in { defm VBROADCASTSD : avx512_fp_broadcast_vl<0x19, "vbroadcastsd", avx512vl_f64_info>, VEX_W; } // avx512_broadcast_pat introduces patterns for broadcast with a scalar argument. // Later, we can canonize broadcast instructions before ISel phase and // eliminate additional patterns on ISel. // SrcRC_v and SrcRC_s are RegisterClasses for vector and scalar // representations of source multiclass avx512_broadcast_pat { def : Pat<(_.VT (OpNode (_.EltVT SrcRC_s:$src))), (!cast(InstName##"r") (COPY_TO_REGCLASS SrcRC_s:$src, SrcRC_v))>; let AddedComplexity = 30 in { def : Pat<(_.VT (vselect _.KRCWM:$mask, (OpNode (_.EltVT SrcRC_s:$src)), _.RC:$src0)), (!cast(InstName##"rk") _.RC:$src0, _.KRCWM:$mask, (COPY_TO_REGCLASS SrcRC_s:$src, SrcRC_v))>; def : Pat<(_.VT(vselect _.KRCWM:$mask, (OpNode (_.EltVT SrcRC_s:$src)), _.ImmAllZerosV)), (!cast(InstName##"rkz") _.KRCWM:$mask, (COPY_TO_REGCLASS SrcRC_s:$src, SrcRC_v))>; } } defm : avx512_broadcast_pat<"VBROADCASTSSZ", X86VBroadcast, v16f32_info, VR128X, FR32X>; defm : avx512_broadcast_pat<"VBROADCASTSDZ", X86VBroadcast, v8f64_info, VR128X, FR64X>; let Predicates = [HasVLX] in { defm : avx512_broadcast_pat<"VBROADCASTSSZ256", X86VBroadcast, v8f32x_info, VR128X, FR32X>; defm : avx512_broadcast_pat<"VBROADCASTSSZ128", X86VBroadcast, v4f32x_info, VR128X, FR32X>; defm : avx512_broadcast_pat<"VBROADCASTSDZ256", X86VBroadcast, v4f64x_info, VR128X, FR64X>; } def : Pat<(v16f32 (X86VBroadcast (loadf32 addr:$src))), (VBROADCASTSSZm addr:$src)>; def : Pat<(v8f64 (X86VBroadcast (loadf64 addr:$src))), (VBROADCASTSDZm addr:$src)>; def : Pat<(int_x86_avx512_vbroadcast_ss_512 addr:$src), (VBROADCASTSSZm addr:$src)>; def : Pat<(int_x86_avx512_vbroadcast_sd_512 addr:$src), (VBROADCASTSDZm addr:$src)>; multiclass avx512_int_broadcast_reg opc, X86VectorVTInfo _, RegisterClass SrcRC> { defm r : AVX512_maskable_in_asm, T8PD, EVEX; } multiclass avx512_int_broadcast_reg_vl opc, AVX512VLVectorVTInfo _, RegisterClass SrcRC, Predicate prd> { let Predicates = [prd] in defm Z : avx512_int_broadcast_reg, EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : avx512_int_broadcast_reg, EVEX_V256; defm Z128 : avx512_int_broadcast_reg, EVEX_V128; } } defm VPBROADCASTBr : avx512_int_broadcast_reg_vl<0x7A, avx512vl_i8_info, GR32, HasBWI>; defm VPBROADCASTWr : avx512_int_broadcast_reg_vl<0x7B, avx512vl_i16_info, GR32, HasBWI>; defm VPBROADCASTDr : avx512_int_broadcast_reg_vl<0x7C, avx512vl_i32_info, GR32, HasAVX512>; defm VPBROADCASTQr : avx512_int_broadcast_reg_vl<0x7C, avx512vl_i64_info, GR64, HasAVX512>, VEX_W; def : Pat <(v16i32 (X86vzext VK16WM:$mask)), (VPBROADCASTDrZrkz VK16WM:$mask, (i32 (MOV32ri 0x1)))>; def : Pat <(v8i64 (X86vzext VK8WM:$mask)), (VPBROADCASTQrZrkz VK8WM:$mask, (i64 (MOV64ri 0x1)))>; def : Pat<(v16i32 (X86VBroadcast (i32 GR32:$src))), (VPBROADCASTDrZr GR32:$src)>; def : Pat<(v8i64 (X86VBroadcast (i64 GR64:$src))), (VPBROADCASTQrZr GR64:$src)>; def : Pat<(v16i32 (int_x86_avx512_pbroadcastd_i32_512 (i32 GR32:$src))), (VPBROADCASTDrZr GR32:$src)>; def : Pat<(v8i64 (int_x86_avx512_pbroadcastq_i64_512 (i64 GR64:$src))), (VPBROADCASTQrZr GR64:$src)>; def : Pat<(v16i32 (int_x86_avx512_mask_pbroadcast_d_gpr_512 (i32 GR32:$src), (v16i32 immAllZerosV), (i16 GR16:$mask))), (VPBROADCASTDrZrkz (COPY_TO_REGCLASS GR16:$mask, VK16WM), GR32:$src)>; def : Pat<(v8i64 (int_x86_avx512_mask_pbroadcast_q_gpr_512 (i64 GR64:$src), (bc_v8i64 (v16i32 immAllZerosV)), (i8 GR8:$mask))), (VPBROADCASTQrZrkz (COPY_TO_REGCLASS GR8:$mask, VK8WM), GR64:$src)>; // Provide aliases for broadcast from the same register class that // automatically does the extract. multiclass avx512_int_broadcast_rm_lowering { def : Pat<(DestInfo.VT (X86VBroadcast (SrcInfo.VT SrcInfo.RC:$src))), (!cast(NAME#DestInfo.ZSuffix#"r") (EXTRACT_SUBREG (SrcInfo.VT SrcInfo.RC:$src), sub_xmm))>; } multiclass avx512_int_broadcast_rm_vl opc, string OpcodeStr, AVX512VLVectorVTInfo _, Predicate prd> { let Predicates = [prd] in { defm Z : avx512_broadcast_rm, avx512_int_broadcast_rm_lowering<_.info512, _.info256>, EVEX_V512; // Defined separately to avoid redefinition. defm Z_Alt : avx512_int_broadcast_rm_lowering<_.info512, _.info512>; } let Predicates = [prd, HasVLX] in { defm Z256 : avx512_broadcast_rm, avx512_int_broadcast_rm_lowering<_.info256, _.info256>, EVEX_V256; defm Z128 : avx512_broadcast_rm, EVEX_V128; } } defm VPBROADCASTB : avx512_int_broadcast_rm_vl<0x78, "vpbroadcastb", avx512vl_i8_info, HasBWI>; defm VPBROADCASTW : avx512_int_broadcast_rm_vl<0x79, "vpbroadcastw", avx512vl_i16_info, HasBWI>; defm VPBROADCASTD : avx512_int_broadcast_rm_vl<0x58, "vpbroadcastd", avx512vl_i32_info, HasAVX512>; defm VPBROADCASTQ : avx512_int_broadcast_rm_vl<0x59, "vpbroadcastq", avx512vl_i64_info, HasAVX512>, VEX_W; multiclass avx512_subvec_broadcast_rm opc, string OpcodeStr, X86VectorVTInfo _Dst, X86VectorVTInfo _Src> { let mayLoad = 1 in defm rm : AVX512_maskable, AVX5128IBase, EVEX; } defm VBROADCASTI32X4 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti32x4", v16i32_info, v4i32x_info>, EVEX_V512, EVEX_CD8<32, CD8VT4>; defm VBROADCASTF32X4 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf32x4", v16f32_info, v4f32x_info>, EVEX_V512, EVEX_CD8<32, CD8VT4>; defm VBROADCASTI64X4 : avx512_subvec_broadcast_rm<0x5b, "vbroadcasti64x4", v8i64_info, v4i64x_info>, VEX_W, EVEX_V512, EVEX_CD8<64, CD8VT4>; defm VBROADCASTF64X4 : avx512_subvec_broadcast_rm<0x1b, "vbroadcastf64x4", v8f64_info, v4f64x_info>, VEX_W, EVEX_V512, EVEX_CD8<64, CD8VT4>; let Predicates = [HasVLX] in { defm VBROADCASTI32X4Z256 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti32x4", v8i32x_info, v4i32x_info>, EVEX_V256, EVEX_CD8<32, CD8VT4>; defm VBROADCASTF32X4Z256 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf32x4", v8f32x_info, v4f32x_info>, EVEX_V256, EVEX_CD8<32, CD8VT4>; } let Predicates = [HasVLX, HasDQI] in { defm VBROADCASTI64X2Z128 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti64x2", v4i64x_info, v2i64x_info>, VEX_W, EVEX_V256, EVEX_CD8<64, CD8VT2>; defm VBROADCASTF64X2Z128 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf64x2", v4f64x_info, v2f64x_info>, VEX_W, EVEX_V256, EVEX_CD8<64, CD8VT2>; } let Predicates = [HasDQI] in { defm VBROADCASTI64X2 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti64x2", v8i64_info, v2i64x_info>, VEX_W, EVEX_V512, EVEX_CD8<64, CD8VT2>; defm VBROADCASTI32X8 : avx512_subvec_broadcast_rm<0x5b, "vbroadcasti32x8", v16i32_info, v8i32x_info>, EVEX_V512, EVEX_CD8<32, CD8VT8>; defm VBROADCASTF64X2 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf64x2", v8f64_info, v2f64x_info>, VEX_W, EVEX_V512, EVEX_CD8<64, CD8VT2>; defm VBROADCASTF32X8 : avx512_subvec_broadcast_rm<0x1b, "vbroadcastf32x8", v16f32_info, v8f32x_info>, EVEX_V512, EVEX_CD8<32, CD8VT8>; } multiclass avx512_broadcast_32x2 opc, string OpcodeStr, X86VectorVTInfo _Dst, X86VectorVTInfo _Src, SDNode OpNode = X86SubVBroadcast> { defm r : AVX512_maskable, T8PD, EVEX; let mayLoad = 1 in defm m : AVX512_maskable, T8PD, EVEX, EVEX_CD8<_Src.EltSize, CD8VT2>; } multiclass avx512_common_broadcast_32x2 opc, string OpcodeStr, AVX512VLVectorVTInfo _> { let Predicates = [HasDQI] in defm Z : avx512_broadcast_32x2, EVEX_V512; let Predicates = [HasDQI, HasVLX] in defm Z256 : avx512_broadcast_32x2, EVEX_V256; } multiclass avx512_common_broadcast_i32x2 opc, string OpcodeStr, AVX512VLVectorVTInfo _> : avx512_common_broadcast_32x2 { let Predicates = [HasDQI, HasVLX] in defm Z128 : avx512_broadcast_32x2, EVEX_V128; } defm VPBROADCASTI32X2 : avx512_common_broadcast_i32x2<0x59, "vbroadcasti32x2", avx512vl_i32_info>; defm VPBROADCASTF32X2 : avx512_common_broadcast_32x2<0x19, "vbroadcastf32x2", avx512vl_f32_info>; def : Pat<(v16f32 (X86VBroadcast (v16f32 VR512:$src))), (VBROADCASTSSZr (EXTRACT_SUBREG (v16f32 VR512:$src), sub_xmm))>; def : Pat<(v16f32 (X86VBroadcast (v8f32 VR256X:$src))), (VBROADCASTSSZr (EXTRACT_SUBREG (v8f32 VR256X:$src), sub_xmm))>; def : Pat<(v8f64 (X86VBroadcast (v8f64 VR512:$src))), (VBROADCASTSDZr (EXTRACT_SUBREG (v8f64 VR512:$src), sub_xmm))>; def : Pat<(v8f64 (X86VBroadcast (v4f64 VR256X:$src))), (VBROADCASTSDZr (EXTRACT_SUBREG (v4f64 VR256X:$src), sub_xmm))>; def : Pat<(v16f32 (int_x86_avx512_vbroadcast_ss_ps_512 (v4f32 VR128X:$src))), (VBROADCASTSSZr VR128X:$src)>; def : Pat<(v8f64 (int_x86_avx512_vbroadcast_sd_pd_512 (v2f64 VR128X:$src))), (VBROADCASTSDZr VR128X:$src)>; // Provide fallback in case the load node that is used in the patterns above // is used by additional users, which prevents the pattern selection. def : Pat<(v16f32 (X86VBroadcast FR32X:$src)), (VBROADCASTSSZr (COPY_TO_REGCLASS FR32X:$src, VR128X))>; def : Pat<(v8f64 (X86VBroadcast FR64X:$src)), (VBROADCASTSDZr (COPY_TO_REGCLASS FR64X:$src, VR128X))>; //===----------------------------------------------------------------------===// // AVX-512 BROADCAST MASK TO VECTOR REGISTER //--- multiclass avx512_mask_broadcastm opc, string OpcodeStr, X86VectorVTInfo _, RegisterClass KRC> { def rr : AVX512XS8I, EVEX; } multiclass avx512_mask_broadcast opc, string OpcodeStr, AVX512VLVectorVTInfo VTInfo, RegisterClass KRC> { let Predicates = [HasCDI] in defm Z : avx512_mask_broadcastm, EVEX_V512; let Predicates = [HasCDI, HasVLX] in { defm Z256 : avx512_mask_broadcastm, EVEX_V256; defm Z128 : avx512_mask_broadcastm, EVEX_V128; } } defm VPBROADCASTMW2D : avx512_mask_broadcast<0x3A, "vpbroadcastmw2d", avx512vl_i32_info, VK16>; defm VPBROADCASTMB2Q : avx512_mask_broadcast<0x2A, "vpbroadcastmb2q", avx512vl_i64_info, VK8>, VEX_W; //===----------------------------------------------------------------------===// // -- VPERMI2 - 3 source operands form -- multiclass avx512_perm_i opc, string OpcodeStr, X86VectorVTInfo _, X86VectorVTInfo IdxVT> { let Constraints = "$src1 = $dst" in { defm rr: AVX512_maskable_3src_cast, EVEX_4V, AVX5128IBase; let mayLoad = 1 in defm rm: AVX512_maskable_3src_cast, EVEX_4V, AVX5128IBase; } } multiclass avx512_perm_i_mb opc, string OpcodeStr, X86VectorVTInfo _, X86VectorVTInfo IdxVT> { let mayLoad = 1, Constraints = "$src1 = $dst" in defm rmb: AVX512_maskable_3src_cast, AVX5128IBase, EVEX_4V, EVEX_B; } multiclass avx512_perm_i_sizes opc, string OpcodeStr, AVX512VLVectorVTInfo VTInfo, AVX512VLVectorVTInfo ShuffleMask> { defm NAME: avx512_perm_i, avx512_perm_i_mb, EVEX_V512; let Predicates = [HasVLX] in { defm NAME#128: avx512_perm_i, avx512_perm_i_mb, EVEX_V128; defm NAME#256: avx512_perm_i, avx512_perm_i_mb, EVEX_V256; } } multiclass avx512_perm_i_sizes_w opc, string OpcodeStr, AVX512VLVectorVTInfo VTInfo, AVX512VLVectorVTInfo Idx> { let Predicates = [HasBWI] in defm NAME: avx512_perm_i, EVEX_V512; let Predicates = [HasBWI, HasVLX] in { defm NAME#128: avx512_perm_i, EVEX_V128; defm NAME#256: avx512_perm_i, EVEX_V256; } } defm VPERMI2D : avx512_perm_i_sizes<0x76, "vpermi2d", avx512vl_i32_info, avx512vl_i32_info>, EVEX_CD8<32, CD8VF>; defm VPERMI2Q : avx512_perm_i_sizes<0x76, "vpermi2q", avx512vl_i64_info, avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>; defm VPERMI2W : avx512_perm_i_sizes_w<0x75, "vpermi2w", avx512vl_i16_info, avx512vl_i16_info>, VEX_W, EVEX_CD8<16, CD8VF>; defm VPERMI2PS : avx512_perm_i_sizes<0x77, "vpermi2ps", avx512vl_f32_info, avx512vl_i32_info>, EVEX_CD8<32, CD8VF>; defm VPERMI2PD : avx512_perm_i_sizes<0x77, "vpermi2pd", avx512vl_f64_info, avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>; // VPERMT2 multiclass avx512_perm_t opc, string OpcodeStr, X86VectorVTInfo _, X86VectorVTInfo IdxVT> { let Constraints = "$src1 = $dst" in { defm rr: AVX512_maskable_3src, EVEX_4V, AVX5128IBase; let mayLoad = 1 in defm rm: AVX512_maskable_3src, EVEX_4V, AVX5128IBase; } } multiclass avx512_perm_t_mb opc, string OpcodeStr, X86VectorVTInfo _, X86VectorVTInfo IdxVT> { let mayLoad = 1, Constraints = "$src1 = $dst" in defm rmb: AVX512_maskable_3src, AVX5128IBase, EVEX_4V, EVEX_B; } multiclass avx512_perm_t_sizes opc, string OpcodeStr, AVX512VLVectorVTInfo VTInfo, AVX512VLVectorVTInfo ShuffleMask> { defm NAME: avx512_perm_t, avx512_perm_t_mb, EVEX_V512; let Predicates = [HasVLX] in { defm NAME#128: avx512_perm_t, avx512_perm_t_mb, EVEX_V128; defm NAME#256: avx512_perm_t, avx512_perm_t_mb, EVEX_V256; } } multiclass avx512_perm_t_sizes_w opc, string OpcodeStr, AVX512VLVectorVTInfo VTInfo, AVX512VLVectorVTInfo Idx> { let Predicates = [HasBWI] in defm NAME: avx512_perm_t, EVEX_V512; let Predicates = [HasBWI, HasVLX] in { defm NAME#128: avx512_perm_t, EVEX_V128; defm NAME#256: avx512_perm_t, EVEX_V256; } } defm VPERMT2D : avx512_perm_t_sizes<0x7E, "vpermt2d", avx512vl_i32_info, avx512vl_i32_info>, EVEX_CD8<32, CD8VF>; defm VPERMT2Q : avx512_perm_t_sizes<0x7E, "vpermt2q", avx512vl_i64_info, avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>; defm VPERMT2W : avx512_perm_t_sizes_w<0x7D, "vpermt2w", avx512vl_i16_info, avx512vl_i16_info>, VEX_W, EVEX_CD8<16, CD8VF>; defm VPERMT2PS : avx512_perm_t_sizes<0x7F, "vpermt2ps", avx512vl_f32_info, avx512vl_i32_info>, EVEX_CD8<32, CD8VF>; defm VPERMT2PD : avx512_perm_t_sizes<0x7F, "vpermt2pd", avx512vl_f64_info, avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>; //===----------------------------------------------------------------------===// // AVX-512 - BLEND using mask // multiclass avx512_blendmask opc, string OpcodeStr, X86VectorVTInfo _> { let ExeDomain = _.ExeDomain in { def rr : AVX5128I, EVEX_4V; def rrk : AVX5128I, EVEX_4V, EVEX_K; def rrkz : AVX5128I, EVEX_4V, EVEX_KZ; let mayLoad = 1 in { def rm : AVX5128I, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>; def rmk : AVX5128I, EVEX_4V, EVEX_K, EVEX_CD8<_.EltSize, CD8VF>; def rmkz : AVX5128I, EVEX_4V, EVEX_KZ, EVEX_CD8<_.EltSize, CD8VF>; } } } multiclass avx512_blendmask_rmb opc, string OpcodeStr, X86VectorVTInfo _> { def rmbk : AVX5128I, EVEX_4V, EVEX_K, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>; def rmb : AVX5128I, EVEX_4V, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>; } multiclass blendmask_dq opc, string OpcodeStr, AVX512VLVectorVTInfo VTInfo> { defm Z : avx512_blendmask , avx512_blendmask_rmb , EVEX_V512; let Predicates = [HasVLX] in { defm Z256 : avx512_blendmask, avx512_blendmask_rmb , EVEX_V256; defm Z128 : avx512_blendmask, avx512_blendmask_rmb , EVEX_V128; } } multiclass blendmask_bw opc, string OpcodeStr, AVX512VLVectorVTInfo VTInfo> { let Predicates = [HasBWI] in defm Z : avx512_blendmask , EVEX_V512; let Predicates = [HasBWI, HasVLX] in { defm Z256 : avx512_blendmask , EVEX_V256; defm Z128 : avx512_blendmask , EVEX_V128; } } defm VBLENDMPS : blendmask_dq <0x65, "vblendmps", avx512vl_f32_info>; defm VBLENDMPD : blendmask_dq <0x65, "vblendmpd", avx512vl_f64_info>, VEX_W; defm VPBLENDMD : blendmask_dq <0x64, "vpblendmd", avx512vl_i32_info>; defm VPBLENDMQ : blendmask_dq <0x64, "vpblendmq", avx512vl_i64_info>, VEX_W; defm VPBLENDMB : blendmask_bw <0x66, "vpblendmb", avx512vl_i8_info>; defm VPBLENDMW : blendmask_bw <0x66, "vpblendmw", avx512vl_i16_info>, VEX_W; let Predicates = [HasAVX512] in { def : Pat<(v8f32 (vselect (v8i1 VK8WM:$mask), (v8f32 VR256X:$src1), (v8f32 VR256X:$src2))), (EXTRACT_SUBREG (v16f32 (VBLENDMPSZrrk (COPY_TO_REGCLASS VK8WM:$mask, VK16WM), (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)), (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>; def : Pat<(v8i32 (vselect (v8i1 VK8WM:$mask), (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))), (EXTRACT_SUBREG (v16i32 (VPBLENDMDZrrk (COPY_TO_REGCLASS VK8WM:$mask, VK16WM), (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)), (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>; } //===----------------------------------------------------------------------===// // Compare Instructions //===----------------------------------------------------------------------===// // avx512_cmp_scalar - AVX512 CMPSS and CMPSD multiclass avx512_cmp_scalar{ defm rr_Int : AVX512_maskable_cmp<0xC2, MRMSrcReg, _, (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2, AVXCC:$cc), "vcmp${cc}"#_.Suffix, "$src2, $src1", "$src1, $src2", (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), imm:$cc)>, EVEX_4V; let mayLoad = 1 in defm rm_Int : AVX512_maskable_cmp<0xC2, MRMSrcMem, _, (outs _.KRC:$dst), (ins _.RC:$src1, _.MemOp:$src2, AVXCC:$cc), "vcmp${cc}"#_.Suffix, "$src2, $src1", "$src1, $src2", (OpNode (_.VT _.RC:$src1), (_.VT (scalar_to_vector (_.ScalarLdFrag addr:$src2))), imm:$cc)>, EVEX_4V, EVEX_CD8<_.EltSize, CD8VT1>; defm rrb_Int : AVX512_maskable_cmp<0xC2, MRMSrcReg, _, (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2, AVXCC:$cc), "vcmp${cc}"#_.Suffix, "{sae}, $src2, $src1", "$src1, $src2,{sae}", (OpNodeRnd (_.VT _.RC:$src1), (_.VT _.RC:$src2), imm:$cc, (i32 FROUND_NO_EXC))>, EVEX_4V, EVEX_B; // Accept explicit immediate argument form instead of comparison code. let isAsmParserOnly = 1, hasSideEffects = 0 in { defm rri_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _, (outs VK1:$dst), (ins _.RC:$src1, _.RC:$src2, u8imm:$cc), "vcmp"#_.Suffix, "$cc, $src2, $src1", "$src1, $src2, $cc">, EVEX_4V; defm rmi_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcMem, _, (outs _.KRC:$dst), (ins _.RC:$src1, _.MemOp:$src2, u8imm:$cc), "vcmp"#_.Suffix, "$cc, $src2, $src1", "$src1, $src2, $cc">, EVEX_4V, EVEX_CD8<_.EltSize, CD8VT1>; defm rrb_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _, (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2, u8imm:$cc), "vcmp"#_.Suffix, "$cc,{sae}, $src2, $src1","$src1, $src2,{sae}, $cc">, EVEX_4V, EVEX_B; }// let isAsmParserOnly = 1, hasSideEffects = 0 let isCodeGenOnly = 1 in { def rr : AVX512Ii8<0xC2, MRMSrcReg, (outs _.KRC:$dst), (ins _.FRC:$src1, _.FRC:$src2, AVXCC:$cc), !strconcat("vcmp${cc}", _.Suffix, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set _.KRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2, imm:$cc))], IIC_SSE_ALU_F32S_RR>, EVEX_4V; let mayLoad = 1 in def rm : AVX512Ii8<0xC2, MRMSrcMem, (outs _.KRC:$dst), (ins _.FRC:$src1, _.ScalarMemOp:$src2, AVXCC:$cc), !strconcat("vcmp${cc}", _.Suffix, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set _.KRC:$dst, (OpNode _.FRC:$src1, (_.ScalarLdFrag addr:$src2), imm:$cc))], IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_CD8<_.EltSize, CD8VT1>; } } let Predicates = [HasAVX512] in { defm VCMPSSZ : avx512_cmp_scalar, AVX512XSIi8Base; defm VCMPSDZ : avx512_cmp_scalar, AVX512XDIi8Base, VEX_W; } multiclass avx512_icmp_packed opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { def rr : AVX512BI, EVEX_4V; let mayLoad = 1 in def rm : AVX512BI, EVEX_4V; def rrk : AVX512BI, EVEX_4V, EVEX_K; let mayLoad = 1 in def rmk : AVX512BI, EVEX_4V, EVEX_K; } multiclass avx512_icmp_packed_rmb opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> : avx512_icmp_packed { let mayLoad = 1 in { def rmb : AVX512BI, EVEX_4V, EVEX_B; def rmbk : AVX512BI, EVEX_4V, EVEX_K, EVEX_B; } } multiclass avx512_icmp_packed_vl opc, string OpcodeStr, SDNode OpNode, AVX512VLVectorVTInfo VTInfo, Predicate prd> { let Predicates = [prd] in defm Z : avx512_icmp_packed, EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : avx512_icmp_packed, EVEX_V256; defm Z128 : avx512_icmp_packed, EVEX_V128; } } multiclass avx512_icmp_packed_rmb_vl opc, string OpcodeStr, SDNode OpNode, AVX512VLVectorVTInfo VTInfo, Predicate prd> { let Predicates = [prd] in defm Z : avx512_icmp_packed_rmb, EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : avx512_icmp_packed_rmb, EVEX_V256; defm Z128 : avx512_icmp_packed_rmb, EVEX_V128; } } defm VPCMPEQB : avx512_icmp_packed_vl<0x74, "vpcmpeqb", X86pcmpeqm, avx512vl_i8_info, HasBWI>, EVEX_CD8<8, CD8VF>; defm VPCMPEQW : avx512_icmp_packed_vl<0x75, "vpcmpeqw", X86pcmpeqm, avx512vl_i16_info, HasBWI>, EVEX_CD8<16, CD8VF>; defm VPCMPEQD : avx512_icmp_packed_rmb_vl<0x76, "vpcmpeqd", X86pcmpeqm, avx512vl_i32_info, HasAVX512>, EVEX_CD8<32, CD8VF>; defm VPCMPEQQ : avx512_icmp_packed_rmb_vl<0x29, "vpcmpeqq", X86pcmpeqm, avx512vl_i64_info, HasAVX512>, T8PD, VEX_W, EVEX_CD8<64, CD8VF>; defm VPCMPGTB : avx512_icmp_packed_vl<0x64, "vpcmpgtb", X86pcmpgtm, avx512vl_i8_info, HasBWI>, EVEX_CD8<8, CD8VF>; defm VPCMPGTW : avx512_icmp_packed_vl<0x65, "vpcmpgtw", X86pcmpgtm, avx512vl_i16_info, HasBWI>, EVEX_CD8<16, CD8VF>; defm VPCMPGTD : avx512_icmp_packed_rmb_vl<0x66, "vpcmpgtd", X86pcmpgtm, avx512vl_i32_info, HasAVX512>, EVEX_CD8<32, CD8VF>; defm VPCMPGTQ : avx512_icmp_packed_rmb_vl<0x37, "vpcmpgtq", X86pcmpgtm, avx512vl_i64_info, HasAVX512>, T8PD, VEX_W, EVEX_CD8<64, CD8VF>; def : Pat<(v8i1 (X86pcmpgtm (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))), (COPY_TO_REGCLASS (VPCMPGTDZrr (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)), (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm))), VK8)>; def : Pat<(v8i1 (X86pcmpeqm (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))), (COPY_TO_REGCLASS (VPCMPEQDZrr (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)), (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm))), VK8)>; multiclass avx512_icmp_cc opc, string Suffix, SDNode OpNode, X86VectorVTInfo _> { def rri : AVX512AIi8, EVEX_4V; let mayLoad = 1 in def rmi : AVX512AIi8, EVEX_4V; def rrik : AVX512AIi8, EVEX_4V, EVEX_K; let mayLoad = 1 in def rmik : AVX512AIi8, EVEX_4V, EVEX_K; // Accept explicit immediate argument form instead of comparison code. let isAsmParserOnly = 1, hasSideEffects = 0 in { def rri_alt : AVX512AIi8, EVEX_4V; let mayLoad = 1 in def rmi_alt : AVX512AIi8, EVEX_4V; def rrik_alt : AVX512AIi8, EVEX_4V, EVEX_K; let mayLoad = 1 in def rmik_alt : AVX512AIi8, EVEX_4V, EVEX_K; } } multiclass avx512_icmp_cc_rmb opc, string Suffix, SDNode OpNode, X86VectorVTInfo _> : avx512_icmp_cc { def rmib : AVX512AIi8, EVEX_4V, EVEX_B; def rmibk : AVX512AIi8, EVEX_4V, EVEX_K, EVEX_B; // Accept explicit immediate argument form instead of comparison code. let isAsmParserOnly = 1, hasSideEffects = 0, mayLoad = 1 in { def rmib_alt : AVX512AIi8, EVEX_4V, EVEX_B; def rmibk_alt : AVX512AIi8, EVEX_4V, EVEX_K, EVEX_B; } } multiclass avx512_icmp_cc_vl opc, string Suffix, SDNode OpNode, AVX512VLVectorVTInfo VTInfo, Predicate prd> { let Predicates = [prd] in defm Z : avx512_icmp_cc, EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : avx512_icmp_cc, EVEX_V256; defm Z128 : avx512_icmp_cc, EVEX_V128; } } multiclass avx512_icmp_cc_rmb_vl opc, string Suffix, SDNode OpNode, AVX512VLVectorVTInfo VTInfo, Predicate prd> { let Predicates = [prd] in defm Z : avx512_icmp_cc_rmb, EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : avx512_icmp_cc_rmb, EVEX_V256; defm Z128 : avx512_icmp_cc_rmb, EVEX_V128; } } defm VPCMPB : avx512_icmp_cc_vl<0x3F, "b", X86cmpm, avx512vl_i8_info, HasBWI>, EVEX_CD8<8, CD8VF>; defm VPCMPUB : avx512_icmp_cc_vl<0x3E, "ub", X86cmpmu, avx512vl_i8_info, HasBWI>, EVEX_CD8<8, CD8VF>; defm VPCMPW : avx512_icmp_cc_vl<0x3F, "w", X86cmpm, avx512vl_i16_info, HasBWI>, VEX_W, EVEX_CD8<16, CD8VF>; defm VPCMPUW : avx512_icmp_cc_vl<0x3E, "uw", X86cmpmu, avx512vl_i16_info, HasBWI>, VEX_W, EVEX_CD8<16, CD8VF>; defm VPCMPD : avx512_icmp_cc_rmb_vl<0x1F, "d", X86cmpm, avx512vl_i32_info, HasAVX512>, EVEX_CD8<32, CD8VF>; defm VPCMPUD : avx512_icmp_cc_rmb_vl<0x1E, "ud", X86cmpmu, avx512vl_i32_info, HasAVX512>, EVEX_CD8<32, CD8VF>; defm VPCMPQ : avx512_icmp_cc_rmb_vl<0x1F, "q", X86cmpm, avx512vl_i64_info, HasAVX512>, VEX_W, EVEX_CD8<64, CD8VF>; defm VPCMPUQ : avx512_icmp_cc_rmb_vl<0x1E, "uq", X86cmpmu, avx512vl_i64_info, HasAVX512>, VEX_W, EVEX_CD8<64, CD8VF>; multiclass avx512_vcmp_common { defm rri : AVX512_maskable_cmp<0xC2, MRMSrcReg, _, (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2,AVXCC:$cc), "vcmp${cc}"#_.Suffix, "$src2, $src1", "$src1, $src2", (X86cmpm (_.VT _.RC:$src1), (_.VT _.RC:$src2), imm:$cc)>; let mayLoad = 1 in { defm rmi : AVX512_maskable_cmp<0xC2, MRMSrcMem, _, (outs _.KRC:$dst),(ins _.RC:$src1, _.MemOp:$src2, AVXCC:$cc), "vcmp${cc}"#_.Suffix, "$src2, $src1", "$src1, $src2", (X86cmpm (_.VT _.RC:$src1), (_.VT (bitconvert (_.LdFrag addr:$src2))), imm:$cc)>; defm rmbi : AVX512_maskable_cmp<0xC2, MRMSrcMem, _, (outs _.KRC:$dst), (ins _.RC:$src1, _.ScalarMemOp:$src2, AVXCC:$cc), "vcmp${cc}"#_.Suffix, "${src2}"##_.BroadcastStr##", $src1", "$src1, ${src2}"##_.BroadcastStr, (X86cmpm (_.VT _.RC:$src1), (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))), imm:$cc)>,EVEX_B; } // Accept explicit immediate argument form instead of comparison code. let isAsmParserOnly = 1, hasSideEffects = 0 in { defm rri_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _, (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2, u8imm:$cc), "vcmp"#_.Suffix, "$cc, $src2, $src1", "$src1, $src2, $cc">; let mayLoad = 1 in { defm rmi_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcMem, _, (outs _.KRC:$dst), (ins _.RC:$src1, _.MemOp:$src2, u8imm:$cc), "vcmp"#_.Suffix, "$cc, $src2, $src1", "$src1, $src2, $cc">; defm rmbi_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcMem, _, (outs _.KRC:$dst), (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$cc), "vcmp"#_.Suffix, "$cc, ${src2}"##_.BroadcastStr##", $src1", "$src1, ${src2}"##_.BroadcastStr##", $cc">,EVEX_B; } } } multiclass avx512_vcmp_sae { // comparison code form (VCMP[EQ/LT/LE/...] defm rrib : AVX512_maskable_cmp<0xC2, MRMSrcReg, _, (outs _.KRC:$dst),(ins _.RC:$src1, _.RC:$src2, AVXCC:$cc), "vcmp${cc}"#_.Suffix, "{sae}, $src2, $src1", "$src1, $src2,{sae}", (X86cmpmRnd (_.VT _.RC:$src1), (_.VT _.RC:$src2), imm:$cc, (i32 FROUND_NO_EXC))>, EVEX_B; let isAsmParserOnly = 1, hasSideEffects = 0 in { defm rrib_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _, (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2, u8imm:$cc), "vcmp"#_.Suffix, "$cc,{sae}, $src2, $src1", "$src1, $src2,{sae}, $cc">, EVEX_B; } } multiclass avx512_vcmp { let Predicates = [HasAVX512] in { defm Z : avx512_vcmp_common<_.info512>, avx512_vcmp_sae<_.info512>, EVEX_V512; } let Predicates = [HasAVX512,HasVLX] in { defm Z128 : avx512_vcmp_common<_.info128>, EVEX_V128; defm Z256 : avx512_vcmp_common<_.info256>, EVEX_V256; } } defm VCMPPD : avx512_vcmp, AVX512PDIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W; defm VCMPPS : avx512_vcmp, AVX512PSIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>; def : Pat<(v8i1 (X86cmpm (v8f32 VR256X:$src1), (v8f32 VR256X:$src2), imm:$cc)), (COPY_TO_REGCLASS (VCMPPSZrri (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)), (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)), imm:$cc), VK8)>; def : Pat<(v8i1 (X86cmpm (v8i32 VR256X:$src1), (v8i32 VR256X:$src2), imm:$cc)), (COPY_TO_REGCLASS (VPCMPDZrri (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)), (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)), imm:$cc), VK8)>; def : Pat<(v8i1 (X86cmpmu (v8i32 VR256X:$src1), (v8i32 VR256X:$src2), imm:$cc)), (COPY_TO_REGCLASS (VPCMPUDZrri (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)), (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)), imm:$cc), VK8)>; // ---------------------------------------------------------------- // FPClass //handle fpclass instruction mask = op(reg_scalar,imm) // op(mem_scalar,imm) multiclass avx512_scalar_fpclass opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _, Predicate prd> { let Predicates = [prd] in { def rr : AVX512; def rrk : AVX512, EVEX_K; let mayLoad = 1, AddedComplexity = 20 in { def rm : AVX512; def rmk : AVX512, EVEX_K; } } } //handle fpclass instruction mask = fpclass(reg_vec, reg_vec, imm) // fpclass(reg_vec, mem_vec, imm) // fpclass(reg_vec, broadcast(eltVt), imm) multiclass avx512_vector_fpclass opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _, string mem, string broadcast>{ def rr : AVX512; def rrk : AVX512, EVEX_K; let mayLoad = 1 in { def rm : AVX512; def rmk : AVX512, EVEX_K; def rmb : AVX512,EVEX_B; def rmbk : AVX512, EVEX_B, EVEX_K; } } multiclass avx512_vector_fpclass_all opc, SDNode OpNode, Predicate prd, string broadcast>{ let Predicates = [prd] in { defm Z : avx512_vector_fpclass, EVEX_V512; } let Predicates = [prd, HasVLX] in { defm Z128 : avx512_vector_fpclass, EVEX_V128; defm Z256 : avx512_vector_fpclass, EVEX_V256; } } multiclass avx512_fp_fpclass_all opcVec, bits<8> opcScalar, SDNode VecOpNode, SDNode ScalarOpNode, Predicate prd>{ defm PS : avx512_vector_fpclass_all, EVEX_CD8<32, CD8VF>; defm PD : avx512_vector_fpclass_all,EVEX_CD8<64, CD8VF> , VEX_W; defm SS : avx512_scalar_fpclass, EVEX_CD8<32, CD8VT1>; defm SD : avx512_scalar_fpclass, EVEX_CD8<64, CD8VT1>, VEX_W; } defm VFPCLASS : avx512_fp_fpclass_all<"vfpclass", 0x66, 0x67, X86Vfpclass, X86Vfpclasss, HasDQI>, AVX512AIi8Base,EVEX; //----------------------------------------------------------------- // Mask register copy, including // - copy between mask registers // - load/store mask registers // - copy from GPR to mask register and vice versa // multiclass avx512_mask_mov opc_kk, bits<8> opc_km, bits<8> opc_mk, string OpcodeStr, RegisterClass KRC, ValueType vvt, X86MemOperand x86memop> { let hasSideEffects = 0 in { def kk : I; let mayLoad = 1 in def km : I; let mayStore = 1 in def mk : I; } } multiclass avx512_mask_mov_gpr opc_kr, bits<8> opc_rk, string OpcodeStr, RegisterClass KRC, RegisterClass GRC> { let hasSideEffects = 0 in { def kr : I; def rk : I; } } let Predicates = [HasDQI] in defm KMOVB : avx512_mask_mov<0x90, 0x90, 0x91, "kmovb", VK8, v8i1, i8mem>, avx512_mask_mov_gpr<0x92, 0x93, "kmovb", VK8, GR32>, VEX, PD; let Predicates = [HasAVX512] in defm KMOVW : avx512_mask_mov<0x90, 0x90, 0x91, "kmovw", VK16, v16i1, i16mem>, avx512_mask_mov_gpr<0x92, 0x93, "kmovw", VK16, GR32>, VEX, PS; let Predicates = [HasBWI] in { defm KMOVD : avx512_mask_mov<0x90, 0x90, 0x91, "kmovd", VK32, v32i1,i32mem>, VEX, PD, VEX_W; defm KMOVD : avx512_mask_mov_gpr<0x92, 0x93, "kmovd", VK32, GR32>, VEX, XD; } let Predicates = [HasBWI] in { defm KMOVQ : avx512_mask_mov<0x90, 0x90, 0x91, "kmovq", VK64, v64i1, i64mem>, VEX, PS, VEX_W; defm KMOVQ : avx512_mask_mov_gpr<0x92, 0x93, "kmovq", VK64, GR64>, VEX, XD, VEX_W; } // GR from/to mask register let Predicates = [HasDQI] in { def : Pat<(v8i1 (bitconvert (i8 GR8:$src))), (KMOVBkr (SUBREG_TO_REG (i32 0), GR8:$src, sub_8bit))>; def : Pat<(i8 (bitconvert (v8i1 VK8:$src))), (EXTRACT_SUBREG (KMOVBrk VK8:$src), sub_8bit)>; } let Predicates = [HasAVX512] in { def : Pat<(v16i1 (bitconvert (i16 GR16:$src))), (KMOVWkr (SUBREG_TO_REG (i32 0), GR16:$src, sub_16bit))>; def : Pat<(i16 (bitconvert (v16i1 VK16:$src))), (EXTRACT_SUBREG (KMOVWrk VK16:$src), sub_16bit)>; } let Predicates = [HasBWI] in { def : Pat<(v32i1 (bitconvert (i32 GR32:$src))), (KMOVDkr GR32:$src)>; def : Pat<(i32 (bitconvert (v32i1 VK32:$src))), (KMOVDrk VK32:$src)>; } let Predicates = [HasBWI] in { def : Pat<(v64i1 (bitconvert (i64 GR64:$src))), (KMOVQkr GR64:$src)>; def : Pat<(i64 (bitconvert (v64i1 VK64:$src))), (KMOVQrk VK64:$src)>; } // Load/store kreg let Predicates = [HasDQI] in { def : Pat<(store (i8 (bitconvert (v8i1 VK8:$src))), addr:$dst), (KMOVBmk addr:$dst, VK8:$src)>; def : Pat<(v8i1 (bitconvert (i8 (load addr:$src)))), (KMOVBkm addr:$src)>; def : Pat<(store VK4:$src, addr:$dst), (KMOVBmk addr:$dst, (COPY_TO_REGCLASS VK4:$src, VK8))>; def : Pat<(store VK2:$src, addr:$dst), (KMOVBmk addr:$dst, (COPY_TO_REGCLASS VK2:$src, VK8))>; } let Predicates = [HasAVX512, NoDQI] in { def : Pat<(store (i8 (bitconvert (v8i1 VK8:$src))), addr:$dst), (KMOVWmk addr:$dst, (COPY_TO_REGCLASS VK8:$src, VK16))>; def : Pat<(v8i1 (bitconvert (i8 (load addr:$src)))), (COPY_TO_REGCLASS (KMOVWkm addr:$src), VK8)>; } let Predicates = [HasAVX512] in { def : Pat<(store (i16 (bitconvert (v16i1 VK16:$src))), addr:$dst), (KMOVWmk addr:$dst, VK16:$src)>; def : Pat<(i1 (load addr:$src)), (COPY_TO_REGCLASS (AND16ri (i16 (SUBREG_TO_REG (i32 0), (MOV8rm addr:$src), sub_8bit)), (i16 1)), VK1)>; def : Pat<(v16i1 (bitconvert (i16 (load addr:$src)))), (KMOVWkm addr:$src)>; } let Predicates = [HasBWI] in { def : Pat<(store (i32 (bitconvert (v32i1 VK32:$src))), addr:$dst), (KMOVDmk addr:$dst, VK32:$src)>; def : Pat<(v32i1 (bitconvert (i32 (load addr:$src)))), (KMOVDkm addr:$src)>; } let Predicates = [HasBWI] in { def : Pat<(store (i64 (bitconvert (v64i1 VK64:$src))), addr:$dst), (KMOVQmk addr:$dst, VK64:$src)>; def : Pat<(v64i1 (bitconvert (i64 (load addr:$src)))), (KMOVQkm addr:$src)>; } let Predicates = [HasAVX512] in { def : Pat<(i1 (trunc (i64 GR64:$src))), (COPY_TO_REGCLASS (KMOVWkr (AND32ri (EXTRACT_SUBREG $src, sub_32bit), (i32 1))), VK1)>; def : Pat<(i1 (trunc (i32 GR32:$src))), (COPY_TO_REGCLASS (KMOVWkr (AND32ri $src, (i32 1))), VK1)>; def : Pat<(i1 (trunc (i8 GR8:$src))), (COPY_TO_REGCLASS (KMOVWkr (AND32ri (SUBREG_TO_REG (i32 0), GR8:$src, sub_8bit), (i32 1))), VK1)>; def : Pat<(i1 (trunc (i16 GR16:$src))), (COPY_TO_REGCLASS (KMOVWkr (AND32ri (SUBREG_TO_REG (i32 0), $src, sub_16bit), (i32 1))), VK1)>; def : Pat<(i32 (zext VK1:$src)), (AND32ri (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)), (i32 1))>; def : Pat<(i32 (anyext VK1:$src)), (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16))>; def : Pat<(i8 (zext VK1:$src)), (EXTRACT_SUBREG (AND32ri (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)), (i32 1)), sub_8bit)>; def : Pat<(i8 (anyext VK1:$src)), (EXTRACT_SUBREG (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)), sub_8bit)>; def : Pat<(i64 (zext VK1:$src)), (AND64ri8 (SUBREG_TO_REG (i64 0), (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)), sub_32bit), (i64 1))>; def : Pat<(i16 (zext VK1:$src)), (EXTRACT_SUBREG (AND32ri (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)), (i32 1)), sub_16bit)>; } def : Pat<(v16i1 (scalar_to_vector VK1:$src)), (COPY_TO_REGCLASS VK1:$src, VK16)>; def : Pat<(v8i1 (scalar_to_vector VK1:$src)), (COPY_TO_REGCLASS VK1:$src, VK8)>; def : Pat<(v4i1 (scalar_to_vector VK1:$src)), (COPY_TO_REGCLASS VK1:$src, VK4)>; def : Pat<(v2i1 (scalar_to_vector VK1:$src)), (COPY_TO_REGCLASS VK1:$src, VK2)>; def : Pat<(v32i1 (scalar_to_vector VK1:$src)), (COPY_TO_REGCLASS VK1:$src, VK32)>; def : Pat<(v64i1 (scalar_to_vector VK1:$src)), (COPY_TO_REGCLASS VK1:$src, VK64)>; // With AVX-512 only, 8-bit mask is promoted to 16-bit mask. let Predicates = [HasAVX512, NoDQI] in { // GR from/to 8-bit mask without native support def : Pat<(v8i1 (bitconvert (i8 GR8:$src))), (COPY_TO_REGCLASS (KMOVWkr (MOVZX32rr8 GR8 :$src)), VK8)>; def : Pat<(i8 (bitconvert (v8i1 VK8:$src))), (EXTRACT_SUBREG (KMOVWrk (COPY_TO_REGCLASS VK8:$src, VK16)), sub_8bit)>; } let Predicates = [HasAVX512] in { def : Pat<(i1 (X86Vextract VK16:$src, (iPTR 0))), (COPY_TO_REGCLASS VK16:$src, VK1)>; def : Pat<(i1 (X86Vextract VK8:$src, (iPTR 0))), (COPY_TO_REGCLASS VK8:$src, VK1)>; } let Predicates = [HasBWI] in { def : Pat<(i1 (X86Vextract VK32:$src, (iPTR 0))), (COPY_TO_REGCLASS VK32:$src, VK1)>; def : Pat<(i1 (X86Vextract VK64:$src, (iPTR 0))), (COPY_TO_REGCLASS VK64:$src, VK1)>; } // Mask unary operation // - KNOT multiclass avx512_mask_unop opc, string OpcodeStr, RegisterClass KRC, SDPatternOperator OpNode, Predicate prd> { let Predicates = [prd] in def rr : I; } multiclass avx512_mask_unop_all opc, string OpcodeStr, SDPatternOperator OpNode> { defm B : avx512_mask_unop, VEX, PD; defm W : avx512_mask_unop, VEX, PS; defm D : avx512_mask_unop, VEX, PD, VEX_W; defm Q : avx512_mask_unop, VEX, PS, VEX_W; } defm KNOT : avx512_mask_unop_all<0x44, "knot", not>; multiclass avx512_mask_unop_int { let Predicates = [HasAVX512] in def : Pat<(!cast("int_x86_avx512_"##IntName##"_w") (i16 GR16:$src)), (COPY_TO_REGCLASS (!cast(InstName##"Wrr") (v16i1 (COPY_TO_REGCLASS GR16:$src, VK16))), GR16)>; } defm : avx512_mask_unop_int<"knot", "KNOT">; let Predicates = [HasDQI] in def : Pat<(xor VK8:$src1, (v8i1 immAllOnesV)), (KNOTBrr VK8:$src1)>; let Predicates = [HasAVX512] in def : Pat<(xor VK16:$src1, (v16i1 immAllOnesV)), (KNOTWrr VK16:$src1)>; let Predicates = [HasBWI] in def : Pat<(xor VK32:$src1, (v32i1 immAllOnesV)), (KNOTDrr VK32:$src1)>; let Predicates = [HasBWI] in def : Pat<(xor VK64:$src1, (v64i1 immAllOnesV)), (KNOTQrr VK64:$src1)>; // KNL does not support KMOVB, 8-bit mask is promoted to 16-bit let Predicates = [HasAVX512, NoDQI] in { def : Pat<(xor VK8:$src1, (v8i1 immAllOnesV)), (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK8:$src1, VK16)), VK8)>; def : Pat<(not VK8:$src), (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK8:$src, VK16)), VK8)>; } def : Pat<(xor VK4:$src1, (v4i1 immAllOnesV)), (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK4:$src1, VK16)), VK4)>; def : Pat<(xor VK2:$src1, (v2i1 immAllOnesV)), (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK2:$src1, VK16)), VK2)>; // Mask binary operation // - KAND, KANDN, KOR, KXNOR, KXOR multiclass avx512_mask_binop opc, string OpcodeStr, RegisterClass KRC, SDPatternOperator OpNode, Predicate prd, bit IsCommutable> { let Predicates = [prd], isCommutable = IsCommutable in def rr : I; } multiclass avx512_mask_binop_all opc, string OpcodeStr, SDPatternOperator OpNode, bit IsCommutable, Predicate prdW = HasAVX512> { defm B : avx512_mask_binop, VEX_4V, VEX_L, PD; defm W : avx512_mask_binop, VEX_4V, VEX_L, PS; defm D : avx512_mask_binop, VEX_4V, VEX_L, VEX_W, PD; defm Q : avx512_mask_binop, VEX_4V, VEX_L, VEX_W, PS; } def andn : PatFrag<(ops node:$i0, node:$i1), (and (not node:$i0), node:$i1)>; def xnor : PatFrag<(ops node:$i0, node:$i1), (not (xor node:$i0, node:$i1))>; defm KAND : avx512_mask_binop_all<0x41, "kand", and, 1>; defm KOR : avx512_mask_binop_all<0x45, "kor", or, 1>; defm KXNOR : avx512_mask_binop_all<0x46, "kxnor", xnor, 1>; defm KXOR : avx512_mask_binop_all<0x47, "kxor", xor, 1>; defm KANDN : avx512_mask_binop_all<0x42, "kandn", andn, 0>; defm KADD : avx512_mask_binop_all<0x4A, "kadd", add, 1, HasDQI>; multiclass avx512_mask_binop_int { let Predicates = [HasAVX512] in def : Pat<(!cast("int_x86_avx512_"##IntName##"_w") (i16 GR16:$src1), (i16 GR16:$src2)), (COPY_TO_REGCLASS (!cast(InstName##"Wrr") (v16i1 (COPY_TO_REGCLASS GR16:$src1, VK16)), (v16i1 (COPY_TO_REGCLASS GR16:$src2, VK16))), GR16)>; } defm : avx512_mask_binop_int<"kand", "KAND">; defm : avx512_mask_binop_int<"kandn", "KANDN">; defm : avx512_mask_binop_int<"kor", "KOR">; defm : avx512_mask_binop_int<"kxnor", "KXNOR">; defm : avx512_mask_binop_int<"kxor", "KXOR">; multiclass avx512_binop_pat { // With AVX512F, 8-bit mask is promoted to 16-bit mask, // for the DQI set, this type is legal and KxxxB instruction is used let Predicates = [NoDQI] in def : Pat<(OpNode VK8:$src1, VK8:$src2), (COPY_TO_REGCLASS (Inst (COPY_TO_REGCLASS VK8:$src1, VK16), (COPY_TO_REGCLASS VK8:$src2, VK16)), VK8)>; // All types smaller than 8 bits require conversion anyway def : Pat<(OpNode VK1:$src1, VK1:$src2), (COPY_TO_REGCLASS (Inst (COPY_TO_REGCLASS VK1:$src1, VK16), (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>; def : Pat<(OpNode VK2:$src1, VK2:$src2), (COPY_TO_REGCLASS (Inst (COPY_TO_REGCLASS VK2:$src1, VK16), (COPY_TO_REGCLASS VK2:$src2, VK16)), VK1)>; def : Pat<(OpNode VK4:$src1, VK4:$src2), (COPY_TO_REGCLASS (Inst (COPY_TO_REGCLASS VK4:$src1, VK16), (COPY_TO_REGCLASS VK4:$src2, VK16)), VK1)>; } defm : avx512_binop_pat; defm : avx512_binop_pat; defm : avx512_binop_pat; defm : avx512_binop_pat; defm : avx512_binop_pat; def : Pat<(xor (xor VK16:$src1, VK16:$src2), (v16i1 immAllOnesV)), (KXNORWrr VK16:$src1, VK16:$src2)>; def : Pat<(xor (xor VK8:$src1, VK8:$src2), (v8i1 immAllOnesV)), (KXNORBrr VK8:$src1, VK8:$src2)>, Requires<[HasDQI]>; def : Pat<(xor (xor VK32:$src1, VK32:$src2), (v32i1 immAllOnesV)), (KXNORDrr VK32:$src1, VK32:$src2)>, Requires<[HasBWI]>; def : Pat<(xor (xor VK64:$src1, VK64:$src2), (v64i1 immAllOnesV)), (KXNORQrr VK64:$src1, VK64:$src2)>, Requires<[HasBWI]>; let Predicates = [NoDQI] in def : Pat<(xor (xor VK8:$src1, VK8:$src2), (v8i1 immAllOnesV)), (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK8:$src1, VK16), (COPY_TO_REGCLASS VK8:$src2, VK16)), VK8)>; def : Pat<(xor (xor VK4:$src1, VK4:$src2), (v4i1 immAllOnesV)), (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK4:$src1, VK16), (COPY_TO_REGCLASS VK4:$src2, VK16)), VK4)>; def : Pat<(xor (xor VK2:$src1, VK2:$src2), (v2i1 immAllOnesV)), (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK2:$src1, VK16), (COPY_TO_REGCLASS VK2:$src2, VK16)), VK2)>; def : Pat<(xor (xor VK1:$src1, VK1:$src2), (i1 1)), (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK1:$src1, VK16), (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>; // Mask unpacking multiclass avx512_mask_unpck { let Predicates = [prd] in { def rr : I<0x4b, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src1, KRC:$src2), "kunpck"#Suffix#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, VEX_4V, VEX_L; def : Pat<(VT (concat_vectors KRCSrc:$src1, KRCSrc:$src2)), (!cast(NAME##rr) (COPY_TO_REGCLASS KRCSrc:$src2, KRC), (COPY_TO_REGCLASS KRCSrc:$src1, KRC))>; } } defm KUNPCKBW : avx512_mask_unpck<"bw", VK16, v16i1, VK8, HasAVX512>, PD; defm KUNPCKWD : avx512_mask_unpck<"wd", VK32, v32i1, VK16, HasBWI>, PS; defm KUNPCKDQ : avx512_mask_unpck<"dq", VK64, v64i1, VK32, HasBWI>, PS, VEX_W; // Mask bit testing multiclass avx512_mask_testop opc, string OpcodeStr, RegisterClass KRC, SDNode OpNode, Predicate prd> { let Predicates = [prd], Defs = [EFLAGS] in def rr : I; } multiclass avx512_mask_testop_w opc, string OpcodeStr, SDNode OpNode, Predicate prdW = HasAVX512> { defm B : avx512_mask_testop, VEX, PD; defm W : avx512_mask_testop, VEX, PS; defm Q : avx512_mask_testop, VEX, PS, VEX_W; defm D : avx512_mask_testop, VEX, PD, VEX_W; } defm KORTEST : avx512_mask_testop_w<0x98, "kortest", X86kortest>; defm KTEST : avx512_mask_testop_w<0x99, "ktest", X86ktest, HasDQI>; // Mask shift multiclass avx512_mask_shiftop opc, string OpcodeStr, RegisterClass KRC, SDNode OpNode> { let Predicates = [HasAVX512] in def ri : Ii8; } multiclass avx512_mask_shiftop_w opc1, bits<8> opc2, string OpcodeStr, SDNode OpNode> { defm W : avx512_mask_shiftop, VEX, TAPD, VEX_W; let Predicates = [HasDQI] in defm B : avx512_mask_shiftop, VEX, TAPD; let Predicates = [HasBWI] in { defm Q : avx512_mask_shiftop, VEX, TAPD, VEX_W; let Predicates = [HasDQI] in defm D : avx512_mask_shiftop, VEX, TAPD; } } defm KSHIFTL : avx512_mask_shiftop_w<0x32, 0x33, "kshiftl", X86vshli>; defm KSHIFTR : avx512_mask_shiftop_w<0x30, 0x31, "kshiftr", X86vsrli>; // Mask setting all 0s or 1s multiclass avx512_mask_setop { let Predicates = [HasAVX512] in let isReMaterializable = 1, isAsCheapAsAMove = 1, isPseudo = 1 in def #NAME# : I<0, Pseudo, (outs KRC:$dst), (ins), "", [(set KRC:$dst, (VT Val))]>; } multiclass avx512_mask_setop_w { defm B : avx512_mask_setop; defm W : avx512_mask_setop; defm D : avx512_mask_setop; defm Q : avx512_mask_setop; } defm KSET0 : avx512_mask_setop_w; defm KSET1 : avx512_mask_setop_w; // With AVX-512 only, 8-bit mask is promoted to 16-bit mask. let Predicates = [HasAVX512] in { def : Pat<(v8i1 immAllZerosV), (COPY_TO_REGCLASS (KSET0W), VK8)>; def : Pat<(v8i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK8)>; def : Pat<(v4i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK4)>; def : Pat<(v2i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK2)>; def : Pat<(i1 0), (COPY_TO_REGCLASS (KSET0W), VK1)>; def : Pat<(i1 1), (COPY_TO_REGCLASS (KSHIFTRWri (KSET1W), (i8 15)), VK1)>; def : Pat<(i1 -1), (COPY_TO_REGCLASS (KSHIFTRWri (KSET1W), (i8 15)), VK1)>; } def : Pat<(v8i1 (extract_subvector (v16i1 VK16:$src), (iPTR 0))), (v8i1 (COPY_TO_REGCLASS VK16:$src, VK8))>; def : Pat<(v16i1 (insert_subvector undef, (v8i1 VK8:$src), (iPTR 0))), (v16i1 (COPY_TO_REGCLASS VK8:$src, VK16))>; def : Pat<(v8i1 (extract_subvector (v16i1 VK16:$src), (iPTR 8))), (v8i1 (COPY_TO_REGCLASS (KSHIFTRWri VK16:$src, (i8 8)), VK8))>; def : Pat<(v16i1 (extract_subvector (v32i1 VK32:$src), (iPTR 0))), (v16i1 (COPY_TO_REGCLASS VK32:$src, VK16))>; def : Pat<(v16i1 (extract_subvector (v32i1 VK32:$src), (iPTR 16))), (v16i1 (COPY_TO_REGCLASS (KSHIFTRDri VK32:$src, (i8 16)), VK16))>; def : Pat<(v32i1 (extract_subvector (v64i1 VK64:$src), (iPTR 0))), (v32i1 (COPY_TO_REGCLASS VK64:$src, VK32))>; def : Pat<(v32i1 (extract_subvector (v64i1 VK64:$src), (iPTR 32))), (v32i1 (COPY_TO_REGCLASS (KSHIFTRQri VK64:$src, (i8 32)), VK32))>; def : Pat<(v4i1 (extract_subvector (v8i1 VK8:$src), (iPTR 0))), (v4i1 (COPY_TO_REGCLASS VK8:$src, VK4))>; def : Pat<(v2i1 (extract_subvector (v8i1 VK8:$src), (iPTR 0))), (v2i1 (COPY_TO_REGCLASS VK8:$src, VK2))>; def : Pat<(v4i1 (insert_subvector undef, (v2i1 VK2:$src), (iPTR 0))), (v4i1 (COPY_TO_REGCLASS VK2:$src, VK4))>; def : Pat<(v8i1 (insert_subvector undef, (v4i1 VK4:$src), (iPTR 0))), (v8i1 (COPY_TO_REGCLASS VK4:$src, VK8))>; def : Pat<(v8i1 (insert_subvector undef, (v2i1 VK2:$src), (iPTR 0))), (v8i1 (COPY_TO_REGCLASS VK2:$src, VK8))>; def : Pat<(v32i1 (insert_subvector undef, VK2:$src, (iPTR 0))), (v32i1 (COPY_TO_REGCLASS VK2:$src, VK32))>; def : Pat<(v32i1 (insert_subvector undef, VK4:$src, (iPTR 0))), (v32i1 (COPY_TO_REGCLASS VK4:$src, VK32))>; def : Pat<(v32i1 (insert_subvector undef, VK8:$src, (iPTR 0))), (v32i1 (COPY_TO_REGCLASS VK8:$src, VK32))>; def : Pat<(v32i1 (insert_subvector undef, VK16:$src, (iPTR 0))), (v32i1 (COPY_TO_REGCLASS VK16:$src, VK32))>; def : Pat<(v64i1 (insert_subvector undef, VK2:$src, (iPTR 0))), (v64i1 (COPY_TO_REGCLASS VK2:$src, VK64))>; def : Pat<(v64i1 (insert_subvector undef, VK4:$src, (iPTR 0))), (v64i1 (COPY_TO_REGCLASS VK4:$src, VK64))>; def : Pat<(v64i1 (insert_subvector undef, VK8:$src, (iPTR 0))), (v64i1 (COPY_TO_REGCLASS VK8:$src, VK64))>; def : Pat<(v64i1 (insert_subvector undef, VK16:$src, (iPTR 0))), (v64i1 (COPY_TO_REGCLASS VK16:$src, VK64))>; def : Pat<(v64i1 (insert_subvector undef, VK32:$src, (iPTR 0))), (v64i1 (COPY_TO_REGCLASS VK32:$src, VK64))>; def : Pat<(v8i1 (X86vshli VK8:$src, (i8 imm:$imm))), (v8i1 (COPY_TO_REGCLASS (KSHIFTLWri (COPY_TO_REGCLASS VK8:$src, VK16), (I8Imm $imm)), VK8))>, Requires<[HasAVX512, NoDQI]>; def : Pat<(v8i1 (X86vsrli VK8:$src, (i8 imm:$imm))), (v8i1 (COPY_TO_REGCLASS (KSHIFTRWri (COPY_TO_REGCLASS VK8:$src, VK16), (I8Imm $imm)), VK8))>, Requires<[HasAVX512, NoDQI]>; def : Pat<(v4i1 (X86vshli VK4:$src, (i8 imm:$imm))), (v4i1 (COPY_TO_REGCLASS (KSHIFTLWri (COPY_TO_REGCLASS VK4:$src, VK16), (I8Imm $imm)), VK4))>, Requires<[HasAVX512]>; def : Pat<(v4i1 (X86vsrli VK4:$src, (i8 imm:$imm))), (v4i1 (COPY_TO_REGCLASS (KSHIFTRWri (COPY_TO_REGCLASS VK4:$src, VK16), (I8Imm $imm)), VK4))>, Requires<[HasAVX512]>; //===----------------------------------------------------------------------===// // AVX-512 - Aligned and unaligned load and store // multiclass avx512_load opc, string OpcodeStr, X86VectorVTInfo _, PatFrag ld_frag, PatFrag mload, bit IsReMaterializable = 1> { let hasSideEffects = 0 in { def rr : AVX512PI, EVEX; def rrkz : AVX512PI, EVEX, EVEX_KZ; let canFoldAsLoad = 1, isReMaterializable = IsReMaterializable, SchedRW = [WriteLoad] in def rm : AVX512PI, EVEX; let Constraints = "$src0 = $dst" in { def rrk : AVX512PI, EVEX, EVEX_K; let mayLoad = 1, SchedRW = [WriteLoad] in def rmk : AVX512PI, EVEX, EVEX_K; } let mayLoad = 1, SchedRW = [WriteLoad] in def rmkz : AVX512PI, EVEX, EVEX_KZ; } def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, undef)), (!cast(NAME#_.ZSuffix##rmkz) _.KRCWM:$mask, addr:$ptr)>; def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, _.ImmAllZerosV)), (!cast(NAME#_.ZSuffix##rmkz) _.KRCWM:$mask, addr:$ptr)>; def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, (_.VT _.RC:$src0))), (!cast(NAME#_.ZSuffix##rmk) _.RC:$src0, _.KRCWM:$mask, addr:$ptr)>; } multiclass avx512_alignedload_vl opc, string OpcodeStr, AVX512VLVectorVTInfo _, Predicate prd, bit IsReMaterializable = 1> { let Predicates = [prd] in defm Z : avx512_load, EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : avx512_load, EVEX_V256; defm Z128 : avx512_load, EVEX_V128; } } multiclass avx512_load_vl opc, string OpcodeStr, AVX512VLVectorVTInfo _, Predicate prd, bit IsReMaterializable = 1> { let Predicates = [prd] in defm Z : avx512_load, EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : avx512_load, EVEX_V256; defm Z128 : avx512_load, EVEX_V128; } } multiclass avx512_store opc, string OpcodeStr, X86VectorVTInfo _, PatFrag st_frag, PatFrag mstore> { def rr_REV : AVX512PI, EVEX; def rrk_REV : AVX512PI, EVEX, EVEX_K; def rrkz_REV : AVX512PI, EVEX, EVEX_KZ; let mayStore = 1 in { def mr : AVX512PI, EVEX; def mrk : AVX512PI, EVEX, EVEX_K; } def: Pat<(mstore addr:$ptr, _.KRCWM:$mask, (_.VT _.RC:$src)), (!cast(NAME#_.ZSuffix##mrk) addr:$ptr, _.KRCWM:$mask, _.RC:$src)>; } multiclass avx512_store_vl< bits<8> opc, string OpcodeStr, AVX512VLVectorVTInfo _, Predicate prd> { let Predicates = [prd] in defm Z : avx512_store, EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : avx512_store, EVEX_V256; defm Z128 : avx512_store, EVEX_V128; } } multiclass avx512_alignedstore_vl opc, string OpcodeStr, AVX512VLVectorVTInfo _, Predicate prd> { let Predicates = [prd] in defm Z : avx512_store, EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : avx512_store, EVEX_V256; defm Z128 : avx512_store, EVEX_V128; } } defm VMOVAPS : avx512_alignedload_vl<0x28, "vmovaps", avx512vl_f32_info, HasAVX512>, avx512_alignedstore_vl<0x29, "vmovaps", avx512vl_f32_info, HasAVX512>, PS, EVEX_CD8<32, CD8VF>; defm VMOVAPD : avx512_alignedload_vl<0x28, "vmovapd", avx512vl_f64_info, HasAVX512>, avx512_alignedstore_vl<0x29, "vmovapd", avx512vl_f64_info, HasAVX512>, PD, VEX_W, EVEX_CD8<64, CD8VF>; defm VMOVUPS : avx512_load_vl<0x10, "vmovups", avx512vl_f32_info, HasAVX512>, avx512_store_vl<0x11, "vmovups", avx512vl_f32_info, HasAVX512>, PS, EVEX_CD8<32, CD8VF>; defm VMOVUPD : avx512_load_vl<0x10, "vmovupd", avx512vl_f64_info, HasAVX512, 0>, avx512_store_vl<0x11, "vmovupd", avx512vl_f64_info, HasAVX512>, PD, VEX_W, EVEX_CD8<64, CD8VF>; def: Pat<(v8f64 (int_x86_avx512_mask_loadu_pd_512 addr:$ptr, (bc_v8f64 (v16i32 immAllZerosV)), GR8:$mask)), (VMOVUPDZrmkz (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), addr:$ptr)>; def: Pat<(v16f32 (int_x86_avx512_mask_loadu_ps_512 addr:$ptr, (bc_v16f32 (v16i32 immAllZerosV)), GR16:$mask)), (VMOVUPSZrmkz (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), addr:$ptr)>; def: Pat<(v8f64 (int_x86_avx512_mask_load_pd_512 addr:$ptr, (bc_v8f64 (v16i32 immAllZerosV)), GR8:$mask)), (VMOVAPDZrmkz (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), addr:$ptr)>; def: Pat<(v16f32 (int_x86_avx512_mask_load_ps_512 addr:$ptr, (bc_v16f32 (v16i32 immAllZerosV)), GR16:$mask)), (VMOVAPSZrmkz (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), addr:$ptr)>; def: Pat<(v8f64 (int_x86_avx512_mask_load_pd_512 addr:$ptr, (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1))), (VMOVAPDZrm addr:$ptr)>; def: Pat<(v16f32 (int_x86_avx512_mask_load_ps_512 addr:$ptr, (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1))), (VMOVAPSZrm addr:$ptr)>; def: Pat<(int_x86_avx512_mask_storeu_ps_512 addr:$ptr, (v16f32 VR512:$src), GR16:$mask), (VMOVUPSZmrk addr:$ptr, (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), VR512:$src)>; def: Pat<(int_x86_avx512_mask_storeu_pd_512 addr:$ptr, (v8f64 VR512:$src), GR8:$mask), (VMOVUPDZmrk addr:$ptr, (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), VR512:$src)>; def: Pat<(int_x86_avx512_mask_store_ps_512 addr:$ptr, (v16f32 VR512:$src), GR16:$mask), (VMOVAPSZmrk addr:$ptr, (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), VR512:$src)>; def: Pat<(int_x86_avx512_mask_store_pd_512 addr:$ptr, (v8f64 VR512:$src), GR8:$mask), (VMOVAPDZmrk addr:$ptr, (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), VR512:$src)>; defm VMOVDQA32 : avx512_alignedload_vl<0x6F, "vmovdqa32", avx512vl_i32_info, HasAVX512>, avx512_alignedstore_vl<0x7F, "vmovdqa32", avx512vl_i32_info, HasAVX512>, PD, EVEX_CD8<32, CD8VF>; defm VMOVDQA64 : avx512_alignedload_vl<0x6F, "vmovdqa64", avx512vl_i64_info, HasAVX512>, avx512_alignedstore_vl<0x7F, "vmovdqa64", avx512vl_i64_info, HasAVX512>, PD, VEX_W, EVEX_CD8<64, CD8VF>; defm VMOVDQU8 : avx512_load_vl<0x6F, "vmovdqu8", avx512vl_i8_info, HasBWI>, avx512_store_vl<0x7F, "vmovdqu8", avx512vl_i8_info, HasBWI>, XD, EVEX_CD8<8, CD8VF>; defm VMOVDQU16 : avx512_load_vl<0x6F, "vmovdqu16", avx512vl_i16_info, HasBWI>, avx512_store_vl<0x7F, "vmovdqu16", avx512vl_i16_info, HasBWI>, XD, VEX_W, EVEX_CD8<16, CD8VF>; defm VMOVDQU32 : avx512_load_vl<0x6F, "vmovdqu32", avx512vl_i32_info, HasAVX512>, avx512_store_vl<0x7F, "vmovdqu32", avx512vl_i32_info, HasAVX512>, XS, EVEX_CD8<32, CD8VF>; defm VMOVDQU64 : avx512_load_vl<0x6F, "vmovdqu64", avx512vl_i64_info, HasAVX512>, avx512_store_vl<0x7F, "vmovdqu64", avx512vl_i64_info, HasAVX512>, XS, VEX_W, EVEX_CD8<64, CD8VF>; def: Pat<(v16i32 (int_x86_avx512_mask_loadu_d_512 addr:$ptr, (v16i32 immAllZerosV), GR16:$mask)), (VMOVDQU32Zrmkz (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), addr:$ptr)>; def: Pat<(v8i64 (int_x86_avx512_mask_loadu_q_512 addr:$ptr, (bc_v8i64 (v16i32 immAllZerosV)), GR8:$mask)), (VMOVDQU64Zrmkz (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), addr:$ptr)>; def: Pat<(int_x86_avx512_mask_storeu_d_512 addr:$ptr, (v16i32 VR512:$src), GR16:$mask), (VMOVDQU32Zmrk addr:$ptr, (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), VR512:$src)>; def: Pat<(int_x86_avx512_mask_storeu_q_512 addr:$ptr, (v8i64 VR512:$src), GR8:$mask), (VMOVDQU64Zmrk addr:$ptr, (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), VR512:$src)>; let AddedComplexity = 20 in { def : Pat<(v8i64 (vselect VK8WM:$mask, (v8i64 VR512:$src), (bc_v8i64 (v16i32 immAllZerosV)))), (VMOVDQU64Zrrkz VK8WM:$mask, VR512:$src)>; def : Pat<(v8i64 (vselect VK8WM:$mask, (bc_v8i64 (v16i32 immAllZerosV)), (v8i64 VR512:$src))), (VMOVDQU64Zrrkz (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK8:$mask, VK16)), VK8), VR512:$src)>; def : Pat<(v16i32 (vselect VK16WM:$mask, (v16i32 VR512:$src), (v16i32 immAllZerosV))), (VMOVDQU32Zrrkz VK16WM:$mask, VR512:$src)>; def : Pat<(v16i32 (vselect VK16WM:$mask, (v16i32 immAllZerosV), (v16i32 VR512:$src))), (VMOVDQU32Zrrkz (KNOTWrr VK16WM:$mask), VR512:$src)>; } // Move Int Doubleword to Packed Double Int // def VMOVDI2PDIZrr : AVX512BI<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR32:$src), "vmovd\t{$src, $dst|$dst, $src}", [(set VR128X:$dst, (v4i32 (scalar_to_vector GR32:$src)))], IIC_SSE_MOVDQ>, EVEX; def VMOVDI2PDIZrm : AVX512BI<0x6E, MRMSrcMem, (outs VR128X:$dst), (ins i32mem:$src), "vmovd\t{$src, $dst|$dst, $src}", [(set VR128X:$dst, (v4i32 (scalar_to_vector (loadi32 addr:$src))))], IIC_SSE_MOVDQ>, EVEX, EVEX_CD8<32, CD8VT1>; def VMOV64toPQIZrr : AVX512BI<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR64:$src), "vmovq\t{$src, $dst|$dst, $src}", [(set VR128X:$dst, (v2i64 (scalar_to_vector GR64:$src)))], IIC_SSE_MOVDQ>, EVEX, VEX_W; let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayLoad = 1 in def VMOV64toPQIZrm : AVX512BI<0x6E, MRMSrcMem, (outs VR128X:$dst), (ins i64mem:$src), "vmovq\t{$src, $dst|$dst, $src}", []>, EVEX, VEX_W, EVEX_CD8<64, CD8VT1>; let isCodeGenOnly = 1 in { def VMOV64toSDZrr : AVX512BI<0x6E, MRMSrcReg, (outs FR64X:$dst), (ins GR64:$src), "vmovq\t{$src, $dst|$dst, $src}", [(set FR64X:$dst, (bitconvert GR64:$src))], IIC_SSE_MOVDQ>, EVEX, VEX_W, Sched<[WriteMove]>; def VMOVSDto64Zrr : AVX512BI<0x7E, MRMDestReg, (outs GR64:$dst), (ins FR64X:$src), "vmovq\t{$src, $dst|$dst, $src}", [(set GR64:$dst, (bitconvert FR64X:$src))], IIC_SSE_MOVDQ>, EVEX, VEX_W, Sched<[WriteMove]>; def VMOVSDto64Zmr : AVX512BI<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, FR64X:$src), "vmovq\t{$src, $dst|$dst, $src}", [(store (i64 (bitconvert FR64X:$src)), addr:$dst)], IIC_SSE_MOVDQ>, EVEX, VEX_W, Sched<[WriteStore]>, EVEX_CD8<64, CD8VT1>; } // Move Int Doubleword to Single Scalar // let isCodeGenOnly = 1 in { def VMOVDI2SSZrr : AVX512BI<0x6E, MRMSrcReg, (outs FR32X:$dst), (ins GR32:$src), "vmovd\t{$src, $dst|$dst, $src}", [(set FR32X:$dst, (bitconvert GR32:$src))], IIC_SSE_MOVDQ>, EVEX; def VMOVDI2SSZrm : AVX512BI<0x6E, MRMSrcMem, (outs FR32X:$dst), (ins i32mem:$src), "vmovd\t{$src, $dst|$dst, $src}", [(set FR32X:$dst, (bitconvert (loadi32 addr:$src)))], IIC_SSE_MOVDQ>, EVEX, EVEX_CD8<32, CD8VT1>; } // Move doubleword from xmm register to r/m32 // def VMOVPDI2DIZrr : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128X:$src), "vmovd\t{$src, $dst|$dst, $src}", [(set GR32:$dst, (extractelt (v4i32 VR128X:$src), (iPTR 0)))], IIC_SSE_MOVD_ToGP>, EVEX; def VMOVPDI2DIZmr : AVX512BI<0x7E, MRMDestMem, (outs), (ins i32mem:$dst, VR128X:$src), "vmovd\t{$src, $dst|$dst, $src}", [(store (i32 (extractelt (v4i32 VR128X:$src), (iPTR 0))), addr:$dst)], IIC_SSE_MOVDQ>, EVEX, EVEX_CD8<32, CD8VT1>; // Move quadword from xmm1 register to r/m64 // def VMOVPQIto64Zrr : I<0x7E, MRMDestReg, (outs GR64:$dst), (ins VR128X:$src), "vmovq\t{$src, $dst|$dst, $src}", [(set GR64:$dst, (extractelt (v2i64 VR128X:$src), (iPTR 0)))], IIC_SSE_MOVD_ToGP>, PD, EVEX, VEX_W, Requires<[HasAVX512, In64BitMode]>; let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayStore = 1 in def VMOVPQIto64Zmr : I<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, VR128X:$src), "vmovq\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVD_ToGP>, PD, EVEX, VEX_W, Requires<[HasAVX512, In64BitMode]>; def VMOVPQI2QIZmr : I<0xD6, MRMDestMem, (outs), (ins i64mem:$dst, VR128X:$src), "vmovq\t{$src, $dst|$dst, $src}", [(store (extractelt (v2i64 VR128X:$src), (iPTR 0)), addr:$dst)], IIC_SSE_MOVDQ>, EVEX, PD, VEX_W, EVEX_CD8<64, CD8VT1>, Sched<[WriteStore]>, Requires<[HasAVX512, In64BitMode]>; let hasSideEffects = 0 in def VMOVPQI2QIZrr : AVX512BI<0xD6, MRMDestReg, (outs VR128X:$dst), (ins VR128X:$src), "vmovq.s\t{$src, $dst|$dst, $src}",[]>, EVEX, VEX_W; // Move Scalar Single to Double Int // let isCodeGenOnly = 1 in { def VMOVSS2DIZrr : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst), (ins FR32X:$src), "vmovd\t{$src, $dst|$dst, $src}", [(set GR32:$dst, (bitconvert FR32X:$src))], IIC_SSE_MOVD_ToGP>, EVEX; def VMOVSS2DIZmr : AVX512BI<0x7E, MRMDestMem, (outs), (ins i32mem:$dst, FR32X:$src), "vmovd\t{$src, $dst|$dst, $src}", [(store (i32 (bitconvert FR32X:$src)), addr:$dst)], IIC_SSE_MOVDQ>, EVEX, EVEX_CD8<32, CD8VT1>; } // Move Quadword Int to Packed Quadword Int // def VMOVQI2PQIZrm : AVX512XSI<0x7E, MRMSrcMem, (outs VR128X:$dst), (ins i64mem:$src), "vmovq\t{$src, $dst|$dst, $src}", [(set VR128X:$dst, (v2i64 (scalar_to_vector (loadi64 addr:$src))))]>, EVEX, VEX_W, EVEX_CD8<8, CD8VT8>; //===----------------------------------------------------------------------===// // AVX-512 MOVSS, MOVSD //===----------------------------------------------------------------------===// multiclass avx512_move_scalar { defm rr_Int : AVX512_maskable_scalar<0x10, MRMSrcReg, _, (outs _.RC:$dst), (ins _.RC:$src1, _.RC:$src2), asm, "$src2, $src1","$src1, $src2", (_.VT (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))), IIC_SSE_MOV_S_RR>, EVEX_4V; let Constraints = "$src1 = $dst" , mayLoad = 1 in defm rm_Int : AVX512_maskable_3src_scalar<0x10, MRMSrcMem, _, (outs _.RC:$dst), (ins _.ScalarMemOp:$src), asm,"$src","$src", (_.VT (OpNode (_.VT _.RC:$src1), (_.VT (scalar_to_vector (_.ScalarLdFrag addr:$src)))))>, EVEX; let isCodeGenOnly = 1 in { def rr : AVX512PI<0x10, MRMSrcReg, (outs _.RC:$dst), (ins _.RC:$src1, _.FRC:$src2), !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), [(set _.RC:$dst, (_.VT (OpNode _.RC:$src1, (scalar_to_vector _.FRC:$src2))))], _.ExeDomain,IIC_SSE_MOV_S_RR>, EVEX_4V; let mayLoad = 1 in def rm : AVX512PI<0x10, MRMSrcMem, (outs _.FRC:$dst), (ins _.ScalarMemOp:$src), !strconcat(asm, "\t{$src, $dst|$dst, $src}"), [(set _.FRC:$dst, (_.ScalarLdFrag addr:$src))], _.ExeDomain, IIC_SSE_MOV_S_RM>, EVEX; } let mayStore = 1 in { def mr: AVX512PI<0x11, MRMDestMem, (outs), (ins _.ScalarMemOp:$dst, _.FRC:$src), !strconcat(asm, "\t{$src, $dst|$dst, $src}"), [(store _.FRC:$src, addr:$dst)], _.ExeDomain, IIC_SSE_MOV_S_MR>, EVEX; def mrk: AVX512PI<0x11, MRMDestMem, (outs), (ins _.ScalarMemOp:$dst, VK1WM:$mask, _.FRC:$src), !strconcat(asm, "\t{$src, $dst {${mask}}|$dst {${mask}}, $src}"), [], _.ExeDomain, IIC_SSE_MOV_S_MR>, EVEX, EVEX_K; } // mayStore } defm VMOVSSZ : avx512_move_scalar<"vmovss", X86Movss, f32x_info>, VEX_LIG, XS, EVEX_CD8<32, CD8VT1>; defm VMOVSDZ : avx512_move_scalar<"vmovsd", X86Movsd, f64x_info>, VEX_LIG, XD, VEX_W, EVEX_CD8<64, CD8VT1>; def : Pat<(f32 (X86select VK1WM:$mask, (f32 FR32X:$src1), (f32 FR32X:$src2))), (COPY_TO_REGCLASS (VMOVSSZrr_Intk (COPY_TO_REGCLASS FR32X:$src2, VR128X), VK1WM:$mask, (v4f32 (IMPLICIT_DEF)),(COPY_TO_REGCLASS FR32X:$src1, VR128X)), FR32X)>; def : Pat<(f64 (X86select VK1WM:$mask, (f64 FR64X:$src1), (f64 FR64X:$src2))), (COPY_TO_REGCLASS (VMOVSDZrr_Intk (COPY_TO_REGCLASS FR64X:$src2, VR128X), VK1WM:$mask, (v2f64 (IMPLICIT_DEF)), (COPY_TO_REGCLASS FR64X:$src1, VR128X)), FR64X)>; def : Pat<(int_x86_avx512_mask_store_ss addr:$dst, VR128X:$src, GR8:$mask), (VMOVSSZmrk addr:$dst, (i1 (COPY_TO_REGCLASS GR8:$mask, VK1WM)), (COPY_TO_REGCLASS VR128X:$src, FR32X))>; defm VMOVSSZrr_REV : AVX512_maskable_in_asm<0x11, MRMDestReg, f32x_info, (outs VR128X:$dst), (ins VR128X:$src1, VR128X:$src2), "vmovss.s", "$src2, $src1", "$src1, $src2", []>, XS, EVEX_4V, VEX_LIG; defm VMOVSSDrr_REV : AVX512_maskable_in_asm<0x11, MRMDestReg, f64x_info, (outs VR128X:$dst), (ins VR128X:$src1, VR128X:$src2), "vmovsd.s", "$src2, $src1", "$src1, $src2", []>, XD, EVEX_4V, VEX_LIG, VEX_W; let Predicates = [HasAVX512] in { let AddedComplexity = 15 in { // Move scalar to XMM zero-extended, zeroing a VR128X then do a // MOVS{S,D} to the lower bits. def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector FR32X:$src)))), (VMOVSSZrr (v4f32 (V_SET0)), FR32X:$src)>; def : Pat<(v4f32 (X86vzmovl (v4f32 VR128X:$src))), (VMOVSSZrr (v4f32 (V_SET0)), (COPY_TO_REGCLASS VR128X:$src, FR32X))>; def : Pat<(v4i32 (X86vzmovl (v4i32 VR128X:$src))), (VMOVSSZrr (v4i32 (V_SET0)), (COPY_TO_REGCLASS VR128X:$src, FR32X))>; def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector FR64X:$src)))), (VMOVSDZrr (v2f64 (V_SET0)), FR64X:$src)>; // Move low f32 and clear high bits. def : Pat<(v8f32 (X86vzmovl (v8f32 VR256X:$src))), (SUBREG_TO_REG (i32 0), (VMOVSSZrr (v4f32 (V_SET0)), (EXTRACT_SUBREG (v8f32 VR256X:$src), sub_xmm)), sub_xmm)>; def : Pat<(v8i32 (X86vzmovl (v8i32 VR256X:$src))), (SUBREG_TO_REG (i32 0), (VMOVSSZrr (v4i32 (V_SET0)), (EXTRACT_SUBREG (v8i32 VR256X:$src), sub_xmm)), sub_xmm)>; } let AddedComplexity = 20 in { // MOVSSrm zeros the high parts of the register; represent this // with SUBREG_TO_REG. The AVX versions also write: DST[255:128] <- 0 def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector (loadf32 addr:$src))))), (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>; def : Pat<(v4f32 (scalar_to_vector (loadf32 addr:$src))), (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>; def : Pat<(v4f32 (X86vzmovl (loadv4f32 addr:$src))), (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>; // MOVSDrm zeros the high parts of the register; represent this // with SUBREG_TO_REG. The AVX versions also write: DST[255:128] <- 0 def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector (loadf64 addr:$src))))), (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>; def : Pat<(v2f64 (scalar_to_vector (loadf64 addr:$src))), (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>; def : Pat<(v2f64 (X86vzmovl (loadv2f64 addr:$src))), (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>; def : Pat<(v2f64 (X86vzmovl (bc_v2f64 (loadv4f32 addr:$src)))), (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>; def : Pat<(v2f64 (X86vzload addr:$src)), (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>; // Represent the same patterns above but in the form they appear for // 256-bit types def : Pat<(v8i32 (X86vzmovl (insert_subvector undef, (v4i32 (scalar_to_vector (loadi32 addr:$src))), (iPTR 0)))), (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrm addr:$src), sub_xmm)>; def : Pat<(v8f32 (X86vzmovl (insert_subvector undef, (v4f32 (scalar_to_vector (loadf32 addr:$src))), (iPTR 0)))), (SUBREG_TO_REG (i32 0), (VMOVSSZrm addr:$src), sub_xmm)>; def : Pat<(v4f64 (X86vzmovl (insert_subvector undef, (v2f64 (scalar_to_vector (loadf64 addr:$src))), (iPTR 0)))), (SUBREG_TO_REG (i32 0), (VMOVSDZrm addr:$src), sub_xmm)>; } def : Pat<(v8f32 (X86vzmovl (insert_subvector undef, (v4f32 (scalar_to_vector FR32X:$src)), (iPTR 0)))), (SUBREG_TO_REG (i32 0), (v4f32 (VMOVSSZrr (v4f32 (V_SET0)), FR32X:$src)), sub_xmm)>; def : Pat<(v4f64 (X86vzmovl (insert_subvector undef, (v2f64 (scalar_to_vector FR64X:$src)), (iPTR 0)))), (SUBREG_TO_REG (i64 0), (v2f64 (VMOVSDZrr (v2f64 (V_SET0)), FR64X:$src)), sub_xmm)>; def : Pat<(v4i64 (X86vzmovl (insert_subvector undef, (v2i64 (scalar_to_vector (loadi64 addr:$src))), (iPTR 0)))), (SUBREG_TO_REG (i64 0), (VMOVQI2PQIZrm addr:$src), sub_xmm)>; // Move low f64 and clear high bits. def : Pat<(v4f64 (X86vzmovl (v4f64 VR256X:$src))), (SUBREG_TO_REG (i32 0), (VMOVSDZrr (v2f64 (V_SET0)), (EXTRACT_SUBREG (v4f64 VR256X:$src), sub_xmm)), sub_xmm)>; def : Pat<(v4i64 (X86vzmovl (v4i64 VR256X:$src))), (SUBREG_TO_REG (i32 0), (VMOVSDZrr (v2i64 (V_SET0)), (EXTRACT_SUBREG (v4i64 VR256X:$src), sub_xmm)), sub_xmm)>; // Extract and store. def : Pat<(store (f32 (extractelt (v4f32 VR128X:$src), (iPTR 0))), addr:$dst), (VMOVSSZmr addr:$dst, (COPY_TO_REGCLASS (v4f32 VR128X:$src), FR32X))>; def : Pat<(store (f64 (extractelt (v2f64 VR128X:$src), (iPTR 0))), addr:$dst), (VMOVSDZmr addr:$dst, (COPY_TO_REGCLASS (v2f64 VR128X:$src), FR64X))>; // Shuffle with VMOVSS def : Pat<(v4i32 (X86Movss VR128X:$src1, VR128X:$src2)), (VMOVSSZrr (v4i32 VR128X:$src1), (COPY_TO_REGCLASS (v4i32 VR128X:$src2), FR32X))>; def : Pat<(v4f32 (X86Movss VR128X:$src1, VR128X:$src2)), (VMOVSSZrr (v4f32 VR128X:$src1), (COPY_TO_REGCLASS (v4f32 VR128X:$src2), FR32X))>; // 256-bit variants def : Pat<(v8i32 (X86Movss VR256X:$src1, VR256X:$src2)), (SUBREG_TO_REG (i32 0), (VMOVSSZrr (EXTRACT_SUBREG (v8i32 VR256X:$src1), sub_xmm), (EXTRACT_SUBREG (v8i32 VR256X:$src2), sub_xmm)), sub_xmm)>; def : Pat<(v8f32 (X86Movss VR256X:$src1, VR256X:$src2)), (SUBREG_TO_REG (i32 0), (VMOVSSZrr (EXTRACT_SUBREG (v8f32 VR256X:$src1), sub_xmm), (EXTRACT_SUBREG (v8f32 VR256X:$src2), sub_xmm)), sub_xmm)>; // Shuffle with VMOVSD def : Pat<(v2i64 (X86Movsd VR128X:$src1, VR128X:$src2)), (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; def : Pat<(v2f64 (X86Movsd VR128X:$src1, VR128X:$src2)), (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; def : Pat<(v4f32 (X86Movsd VR128X:$src1, VR128X:$src2)), (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; def : Pat<(v4i32 (X86Movsd VR128X:$src1, VR128X:$src2)), (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; // 256-bit variants def : Pat<(v4i64 (X86Movsd VR256X:$src1, VR256X:$src2)), (SUBREG_TO_REG (i32 0), (VMOVSDZrr (EXTRACT_SUBREG (v4i64 VR256X:$src1), sub_xmm), (EXTRACT_SUBREG (v4i64 VR256X:$src2), sub_xmm)), sub_xmm)>; def : Pat<(v4f64 (X86Movsd VR256X:$src1, VR256X:$src2)), (SUBREG_TO_REG (i32 0), (VMOVSDZrr (EXTRACT_SUBREG (v4f64 VR256X:$src1), sub_xmm), (EXTRACT_SUBREG (v4f64 VR256X:$src2), sub_xmm)), sub_xmm)>; def : Pat<(v2f64 (X86Movlpd VR128X:$src1, VR128X:$src2)), (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; def : Pat<(v2i64 (X86Movlpd VR128X:$src1, VR128X:$src2)), (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; def : Pat<(v4f32 (X86Movlps VR128X:$src1, VR128X:$src2)), (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; def : Pat<(v4i32 (X86Movlps VR128X:$src1, VR128X:$src2)), (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; } let AddedComplexity = 15 in def VMOVZPQILo2PQIZrr : AVX512XSI<0x7E, MRMSrcReg, (outs VR128X:$dst), (ins VR128X:$src), "vmovq\t{$src, $dst|$dst, $src}", [(set VR128X:$dst, (v2i64 (X86vzmovl (v2i64 VR128X:$src))))], IIC_SSE_MOVQ_RR>, EVEX, VEX_W; let AddedComplexity = 20 , isCodeGenOnly = 1 in def VMOVZPQILo2PQIZrm : AVX512XSI<0x7E, MRMSrcMem, (outs VR128X:$dst), (ins i128mem:$src), "vmovq\t{$src, $dst|$dst, $src}", [(set VR128X:$dst, (v2i64 (X86vzmovl (loadv2i64 addr:$src))))], IIC_SSE_MOVDQ>, EVEX, VEX_W, EVEX_CD8<8, CD8VT8>; let Predicates = [HasAVX512] in { // AVX 128-bit movd/movq instruction write zeros in the high 128-bit part. let AddedComplexity = 20 in { def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector (loadi32 addr:$src))))), (VMOVDI2PDIZrm addr:$src)>; def : Pat<(v2i64 (X86vzmovl (v2i64 (scalar_to_vector GR64:$src)))), (VMOV64toPQIZrr GR64:$src)>; def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector GR32:$src)))), (VMOVDI2PDIZrr GR32:$src)>; def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv4f32 addr:$src)))), (VMOVDI2PDIZrm addr:$src)>; def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv2i64 addr:$src)))), (VMOVDI2PDIZrm addr:$src)>; def : Pat<(v2i64 (X86vzmovl (loadv2i64 addr:$src))), (VMOVZPQILo2PQIZrm addr:$src)>; def : Pat<(v2f64 (X86vzmovl (v2f64 VR128X:$src))), (VMOVZPQILo2PQIZrr VR128X:$src)>; def : Pat<(v2i64 (X86vzload addr:$src)), (VMOVZPQILo2PQIZrm addr:$src)>; } // Use regular 128-bit instructions to match 256-bit scalar_to_vec+zext. def : Pat<(v8i32 (X86vzmovl (insert_subvector undef, (v4i32 (scalar_to_vector GR32:$src)),(iPTR 0)))), (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrr GR32:$src), sub_xmm)>; def : Pat<(v4i64 (X86vzmovl (insert_subvector undef, (v2i64 (scalar_to_vector GR64:$src)),(iPTR 0)))), (SUBREG_TO_REG (i64 0), (VMOV64toPQIZrr GR64:$src), sub_xmm)>; } def : Pat<(v16i32 (X86Vinsert (v16i32 immAllZerosV), GR32:$src2, (iPTR 0))), (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrr GR32:$src2), sub_xmm)>; def : Pat<(v8i64 (X86Vinsert (bc_v8i64 (v16i32 immAllZerosV)), GR64:$src2, (iPTR 0))), (SUBREG_TO_REG (i32 0), (VMOV64toPQIZrr GR64:$src2), sub_xmm)>; def : Pat<(v16i32 (X86Vinsert undef, GR32:$src2, (iPTR 0))), (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrr GR32:$src2), sub_xmm)>; def : Pat<(v8i64 (X86Vinsert undef, GR64:$src2, (iPTR 0))), (SUBREG_TO_REG (i32 0), (VMOV64toPQIZrr GR64:$src2), sub_xmm)>; //===----------------------------------------------------------------------===// // AVX-512 - Non-temporals //===----------------------------------------------------------------------===// let SchedRW = [WriteLoad] in { def VMOVNTDQAZrm : AVX512PI<0x2A, MRMSrcMem, (outs VR512:$dst), (ins i512mem:$src), "vmovntdqa\t{$src, $dst|$dst, $src}", [(set VR512:$dst, (int_x86_avx512_movntdqa addr:$src))], SSEPackedInt>, EVEX, T8PD, EVEX_V512, EVEX_CD8<64, CD8VF>; let Predicates = [HasAVX512, HasVLX] in { def VMOVNTDQAZ256rm : AVX512PI<0x2A, MRMSrcMem, (outs VR256X:$dst), (ins i256mem:$src), "vmovntdqa\t{$src, $dst|$dst, $src}", [], SSEPackedInt>, EVEX, T8PD, EVEX_V256, EVEX_CD8<64, CD8VF>; def VMOVNTDQAZ128rm : AVX512PI<0x2A, MRMSrcMem, (outs VR128X:$dst), (ins i128mem:$src), "vmovntdqa\t{$src, $dst|$dst, $src}", [], SSEPackedInt>, EVEX, T8PD, EVEX_V128, EVEX_CD8<64, CD8VF>; } } multiclass avx512_movnt opc, string OpcodeStr, PatFrag st_frag, ValueType OpVT, RegisterClass RC, X86MemOperand memop, Domain d, InstrItinClass itin = IIC_SSE_MOVNT> { let SchedRW = [WriteStore], mayStore = 1, AddedComplexity = 400 in def mr : AVX512PI, EVEX; } multiclass avx512_movnt_vl opc, string OpcodeStr, PatFrag st_frag, string elty, string elsz, string vsz512, string vsz256, string vsz128, Domain d, Predicate prd, InstrItinClass itin = IIC_SSE_MOVNT> { let Predicates = [prd] in defm Z : avx512_movnt("v"##vsz512##elty##elsz), VR512, !cast(elty##"512mem"), d, itin>, EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : avx512_movnt("v"##vsz256##elty##elsz), VR256X, !cast(elty##"256mem"), d, itin>, EVEX_V256; defm Z128 : avx512_movnt("v"##vsz128##elty##elsz), VR128X, !cast(elty##"128mem"), d, itin>, EVEX_V128; } } defm VMOVNTDQ : avx512_movnt_vl<0xE7, "vmovntdq", alignednontemporalstore, "i", "64", "8", "4", "2", SSEPackedInt, HasAVX512>, PD, EVEX_CD8<64, CD8VF>; defm VMOVNTPD : avx512_movnt_vl<0x2B, "vmovntpd", alignednontemporalstore, "f", "64", "8", "4", "2", SSEPackedDouble, HasAVX512>, PD, VEX_W, EVEX_CD8<64, CD8VF>; defm VMOVNTPS : avx512_movnt_vl<0x2B, "vmovntps", alignednontemporalstore, "f", "32", "16", "8", "4", SSEPackedSingle, HasAVX512>, PS, EVEX_CD8<32, CD8VF>; //===----------------------------------------------------------------------===// // AVX-512 - Integer arithmetic // multiclass avx512_binop_rm opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _, OpndItins itins, bit IsCommutable = 0> { defm rr : AVX512_maskable, AVX512BIBase, EVEX_4V; let mayLoad = 1 in defm rm : AVX512_maskable, AVX512BIBase, EVEX_4V; } multiclass avx512_binop_rmb opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _, OpndItins itins, bit IsCommutable = 0> : avx512_binop_rm { let mayLoad = 1 in defm rmb : AVX512_maskable, AVX512BIBase, EVEX_4V, EVEX_B; } multiclass avx512_binop_rm_vl opc, string OpcodeStr, SDNode OpNode, AVX512VLVectorVTInfo VTInfo, OpndItins itins, Predicate prd, bit IsCommutable = 0> { let Predicates = [prd] in defm Z : avx512_binop_rm, EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : avx512_binop_rm, EVEX_V256; defm Z128 : avx512_binop_rm, EVEX_V128; } } multiclass avx512_binop_rmb_vl opc, string OpcodeStr, SDNode OpNode, AVX512VLVectorVTInfo VTInfo, OpndItins itins, Predicate prd, bit IsCommutable = 0> { let Predicates = [prd] in defm Z : avx512_binop_rmb, EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : avx512_binop_rmb, EVEX_V256; defm Z128 : avx512_binop_rmb, EVEX_V128; } } multiclass avx512_binop_rm_vl_q opc, string OpcodeStr, SDNode OpNode, OpndItins itins, Predicate prd, bit IsCommutable = 0> { defm NAME : avx512_binop_rmb_vl, VEX_W, EVEX_CD8<64, CD8VF>; } multiclass avx512_binop_rm_vl_d opc, string OpcodeStr, SDNode OpNode, OpndItins itins, Predicate prd, bit IsCommutable = 0> { defm NAME : avx512_binop_rmb_vl, EVEX_CD8<32, CD8VF>; } multiclass avx512_binop_rm_vl_w opc, string OpcodeStr, SDNode OpNode, OpndItins itins, Predicate prd, bit IsCommutable = 0> { defm NAME : avx512_binop_rm_vl, EVEX_CD8<16, CD8VF>; } multiclass avx512_binop_rm_vl_b opc, string OpcodeStr, SDNode OpNode, OpndItins itins, Predicate prd, bit IsCommutable = 0> { defm NAME : avx512_binop_rm_vl, EVEX_CD8<8, CD8VF>; } multiclass avx512_binop_rm_vl_dq opc_d, bits<8> opc_q, string OpcodeStr, SDNode OpNode, OpndItins itins, Predicate prd, bit IsCommutable = 0> { defm Q : avx512_binop_rm_vl_q; defm D : avx512_binop_rm_vl_d; } multiclass avx512_binop_rm_vl_bw opc_b, bits<8> opc_w, string OpcodeStr, SDNode OpNode, OpndItins itins, Predicate prd, bit IsCommutable = 0> { defm W : avx512_binop_rm_vl_w; defm B : avx512_binop_rm_vl_b; } multiclass avx512_binop_rm_vl_all opc_b, bits<8> opc_w, bits<8> opc_d, bits<8> opc_q, string OpcodeStr, SDNode OpNode, OpndItins itins, bit IsCommutable = 0> { defm NAME : avx512_binop_rm_vl_dq, avx512_binop_rm_vl_bw; } multiclass avx512_binop_rm2 opc, string OpcodeStr, OpndItins itins, SDNode OpNode,X86VectorVTInfo _Src, X86VectorVTInfo _Dst, bit IsCommutable = 0> { defm rr : AVX512_maskable, AVX512BIBase, EVEX_4V; let mayLoad = 1 in { defm rm : AVX512_maskable, AVX512BIBase, EVEX_4V; defm rmb : AVX512_maskable, AVX512BIBase, EVEX_4V, EVEX_B; } } defm VPADD : avx512_binop_rm_vl_all<0xFC, 0xFD, 0xFE, 0xD4, "vpadd", add, SSE_INTALU_ITINS_P, 1>; defm VPSUB : avx512_binop_rm_vl_all<0xF8, 0xF9, 0xFA, 0xFB, "vpsub", sub, SSE_INTALU_ITINS_P, 0>; defm VPADDS : avx512_binop_rm_vl_bw<0xEC, 0xED, "vpadds", X86adds, SSE_INTALU_ITINS_P, HasBWI, 1>; defm VPSUBS : avx512_binop_rm_vl_bw<0xE8, 0xE9, "vpsubs", X86subs, SSE_INTALU_ITINS_P, HasBWI, 0>; defm VPADDUS : avx512_binop_rm_vl_bw<0xDC, 0xDD, "vpaddus", X86addus, SSE_INTALU_ITINS_P, HasBWI, 1>; defm VPSUBUS : avx512_binop_rm_vl_bw<0xD8, 0xD9, "vpsubus", X86subus, SSE_INTALU_ITINS_P, HasBWI, 0>; defm VPMULLD : avx512_binop_rm_vl_d<0x40, "vpmulld", mul, SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD; defm VPMULLW : avx512_binop_rm_vl_w<0xD5, "vpmullw", mul, SSE_INTALU_ITINS_P, HasBWI, 1>; defm VPMULLQ : avx512_binop_rm_vl_q<0x40, "vpmullq", mul, SSE_INTALU_ITINS_P, HasDQI, 1>, T8PD; defm VPMULHW : avx512_binop_rm_vl_w<0xE5, "vpmulhw", mulhs, SSE_INTALU_ITINS_P, HasBWI, 1>; defm VPMULHUW : avx512_binop_rm_vl_w<0xE4, "vpmulhuw", mulhu, SSE_INTMUL_ITINS_P, HasBWI, 1>; defm VPMULHRSW : avx512_binop_rm_vl_w<0x0B, "vpmulhrsw", X86mulhrs, SSE_INTMUL_ITINS_P, HasBWI, 1>, T8PD; defm VPAVG : avx512_binop_rm_vl_bw<0xE0, 0xE3, "vpavg", X86avg, SSE_INTALU_ITINS_P, HasBWI, 1>; multiclass avx512_binop_all opc, string OpcodeStr, OpndItins itins, SDNode OpNode, bit IsCommutable = 0> { defm NAME#Z : avx512_binop_rm2, EVEX_V512, EVEX_CD8<64, CD8VF>, VEX_W; let Predicates = [HasVLX] in { defm NAME#Z256 : avx512_binop_rm2, EVEX_V256, EVEX_CD8<64, CD8VF>, VEX_W; defm NAME#Z128 : avx512_binop_rm2, EVEX_V128, EVEX_CD8<64, CD8VF>, VEX_W; } } defm VPMULDQ : avx512_binop_all<0x28, "vpmuldq", SSE_INTALU_ITINS_P, X86pmuldq, 1>,T8PD; defm VPMULUDQ : avx512_binop_all<0xF4, "vpmuludq", SSE_INTMUL_ITINS_P, X86pmuludq, 1>; multiclass avx512_packs_rmb opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _Src, X86VectorVTInfo _Dst> { let mayLoad = 1 in { defm rmb : AVX512_maskable, EVEX_4V, EVEX_B, EVEX_CD8<_Src.EltSize, CD8VF>; } } multiclass avx512_packs_rm opc, string OpcodeStr, SDNode OpNode,X86VectorVTInfo _Src, X86VectorVTInfo _Dst> { defm rr : AVX512_maskable, EVEX_CD8<_Src.EltSize, CD8VF>, EVEX_4V; let mayLoad = 1 in { defm rm : AVX512_maskable, EVEX_4V, EVEX_CD8<_Src.EltSize, CD8VF>; } } multiclass avx512_packs_all_i32_i16 opc, string OpcodeStr, SDNode OpNode> { defm NAME#Z : avx512_packs_rm, avx512_packs_rmb, EVEX_V512; let Predicates = [HasVLX] in { defm NAME#Z256 : avx512_packs_rm, avx512_packs_rmb, EVEX_V256; defm NAME#Z128 : avx512_packs_rm, avx512_packs_rmb, EVEX_V128; } } multiclass avx512_packs_all_i16_i8 opc, string OpcodeStr, SDNode OpNode> { defm NAME#Z : avx512_packs_rm, EVEX_V512; let Predicates = [HasVLX] in { defm NAME#Z256 : avx512_packs_rm, EVEX_V256; defm NAME#Z128 : avx512_packs_rm, EVEX_V128; } } multiclass avx512_vpmadd opc, string OpcodeStr, SDNode OpNode, AVX512VLVectorVTInfo _Src, AVX512VLVectorVTInfo _Dst> { defm NAME#Z : avx512_packs_rm, EVEX_V512; let Predicates = [HasVLX] in { defm NAME#Z256 : avx512_packs_rm, EVEX_V256; defm NAME#Z128 : avx512_packs_rm, EVEX_V128; } } let Predicates = [HasBWI] in { defm VPACKSSDW : avx512_packs_all_i32_i16<0x6B, "vpackssdw", X86Packss>, PD; defm VPACKUSDW : avx512_packs_all_i32_i16<0x2b, "vpackusdw", X86Packus>, T8PD; defm VPACKSSWB : avx512_packs_all_i16_i8 <0x63, "vpacksswb", X86Packss>, AVX512BIBase, VEX_W; defm VPACKUSWB : avx512_packs_all_i16_i8 <0x67, "vpackuswb", X86Packus>, AVX512BIBase, VEX_W; defm VPMADDUBSW : avx512_vpmadd<0x04, "vpmaddubsw", X86vpmaddubsw, avx512vl_i8_info, avx512vl_i16_info>, AVX512BIBase, T8PD; defm VPMADDWD : avx512_vpmadd<0xF5, "vpmaddwd", X86vpmaddwd, avx512vl_i16_info, avx512vl_i32_info>, AVX512BIBase; } defm VPMAXSB : avx512_binop_rm_vl_b<0x3C, "vpmaxsb", smax, SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD; defm VPMAXSW : avx512_binop_rm_vl_w<0xEE, "vpmaxsw", smax, SSE_INTALU_ITINS_P, HasBWI, 1>; defm VPMAXS : avx512_binop_rm_vl_dq<0x3D, 0x3D, "vpmaxs", smax, SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD; defm VPMAXUB : avx512_binop_rm_vl_b<0xDE, "vpmaxub", umax, SSE_INTALU_ITINS_P, HasBWI, 1>; defm VPMAXUW : avx512_binop_rm_vl_w<0x3E, "vpmaxuw", umax, SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD; defm VPMAXU : avx512_binop_rm_vl_dq<0x3F, 0x3F, "vpmaxu", umax, SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD; defm VPMINSB : avx512_binop_rm_vl_b<0x38, "vpminsb", smin, SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD; defm VPMINSW : avx512_binop_rm_vl_w<0xEA, "vpminsw", smin, SSE_INTALU_ITINS_P, HasBWI, 1>; defm VPMINS : avx512_binop_rm_vl_dq<0x39, 0x39, "vpmins", smin, SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD; defm VPMINUB : avx512_binop_rm_vl_b<0xDA, "vpminub", umin, SSE_INTALU_ITINS_P, HasBWI, 1>; defm VPMINUW : avx512_binop_rm_vl_w<0x3A, "vpminuw", umin, SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD; defm VPMINU : avx512_binop_rm_vl_dq<0x3B, 0x3B, "vpminu", umin, SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD; //===----------------------------------------------------------------------===// // AVX-512 Logical Instructions //===----------------------------------------------------------------------===// defm VPAND : avx512_binop_rm_vl_dq<0xDB, 0xDB, "vpand", and, SSE_INTALU_ITINS_P, HasAVX512, 1>; defm VPOR : avx512_binop_rm_vl_dq<0xEB, 0xEB, "vpor", or, SSE_INTALU_ITINS_P, HasAVX512, 1>; defm VPXOR : avx512_binop_rm_vl_dq<0xEF, 0xEF, "vpxor", xor, SSE_INTALU_ITINS_P, HasAVX512, 1>; defm VPANDN : avx512_binop_rm_vl_dq<0xDF, 0xDF, "vpandn", X86andnp, SSE_INTALU_ITINS_P, HasAVX512, 0>; //===----------------------------------------------------------------------===// // AVX-512 FP arithmetic //===----------------------------------------------------------------------===// multiclass avx512_fp_scalar opc, string OpcodeStr,X86VectorVTInfo _, SDNode OpNode, SDNode VecNode, OpndItins itins, bit IsCommutable> { defm rr_Int : AVX512_maskable_scalar; defm rm_Int : AVX512_maskable_scalar; let isCodeGenOnly = 1, isCommutable = IsCommutable, Predicates = [HasAVX512] in { def rr : I< opc, MRMSrcReg, (outs _.FRC:$dst), (ins _.FRC:$src1, _.FRC:$src2), OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))], itins.rr>; def rm : I< opc, MRMSrcMem, (outs _.FRC:$dst), (ins _.FRC:$src1, _.ScalarMemOp:$src2), OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set _.FRC:$dst, (OpNode _.FRC:$src1, (_.ScalarLdFrag addr:$src2)))], itins.rr>; } } multiclass avx512_fp_scalar_round opc, string OpcodeStr,X86VectorVTInfo _, SDNode VecNode, OpndItins itins, bit IsCommutable = 0> { defm rrb : AVX512_maskable_scalar, EVEX_B, EVEX_RC; } multiclass avx512_fp_scalar_sae opc, string OpcodeStr,X86VectorVTInfo _, SDNode VecNode, OpndItins itins, bit IsCommutable> { defm rrb : AVX512_maskable_scalar, EVEX_B; } multiclass avx512_binop_s_round opc, string OpcodeStr, SDNode OpNode, SDNode VecNode, SizeItins itins, bit IsCommutable> { defm SSZ : avx512_fp_scalar, avx512_fp_scalar_round, XS, EVEX_4V, VEX_LIG, EVEX_CD8<32, CD8VT1>; defm SDZ : avx512_fp_scalar, avx512_fp_scalar_round, XD, VEX_W, EVEX_4V, VEX_LIG, EVEX_CD8<64, CD8VT1>; } multiclass avx512_binop_s_sae opc, string OpcodeStr, SDNode OpNode, SDNode VecNode, SizeItins itins, bit IsCommutable> { defm SSZ : avx512_fp_scalar, avx512_fp_scalar_sae, XS, EVEX_4V, VEX_LIG, EVEX_CD8<32, CD8VT1>; defm SDZ : avx512_fp_scalar, avx512_fp_scalar_sae, XD, VEX_W, EVEX_4V, VEX_LIG, EVEX_CD8<64, CD8VT1>; } defm VADD : avx512_binop_s_round<0x58, "vadd", fadd, X86faddRnd, SSE_ALU_ITINS_S, 1>; defm VMUL : avx512_binop_s_round<0x59, "vmul", fmul, X86fmulRnd, SSE_ALU_ITINS_S, 1>; defm VSUB : avx512_binop_s_round<0x5C, "vsub", fsub, X86fsubRnd, SSE_ALU_ITINS_S, 0>; defm VDIV : avx512_binop_s_round<0x5E, "vdiv", fdiv, X86fdivRnd, SSE_ALU_ITINS_S, 0>; defm VMIN : avx512_binop_s_sae <0x5D, "vmin", X86fmin, X86fminRnd, SSE_ALU_ITINS_S, 1>; defm VMAX : avx512_binop_s_sae <0x5F, "vmax", X86fmax, X86fmaxRnd, SSE_ALU_ITINS_S, 1>; multiclass avx512_fp_packed opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _, bit IsCommutable> { defm rr: AVX512_maskable, EVEX_4V; let mayLoad = 1 in { defm rm: AVX512_maskable, EVEX_4V; defm rmb: AVX512_maskable, EVEX_4V, EVEX_B; }//let mayLoad = 1 } multiclass avx512_fp_round_packed opc, string OpcodeStr, SDNode OpNodeRnd, X86VectorVTInfo _> { defm rb: AVX512_maskable, EVEX_4V, EVEX_B, EVEX_RC; } multiclass avx512_fp_sae_packed opc, string OpcodeStr, SDNode OpNodeRnd, X86VectorVTInfo _> { defm rb: AVX512_maskable, EVEX_4V, EVEX_B; } multiclass avx512_fp_binop_p opc, string OpcodeStr, SDNode OpNode, bit IsCommutable = 0> { defm PSZ : avx512_fp_packed, EVEX_V512, PS, EVEX_CD8<32, CD8VF>; defm PDZ : avx512_fp_packed, EVEX_V512, PD, VEX_W, EVEX_CD8<64, CD8VF>; // Define only if AVX512VL feature is present. let Predicates = [HasVLX] in { defm PSZ128 : avx512_fp_packed, EVEX_V128, PS, EVEX_CD8<32, CD8VF>; defm PSZ256 : avx512_fp_packed, EVEX_V256, PS, EVEX_CD8<32, CD8VF>; defm PDZ128 : avx512_fp_packed, EVEX_V128, PD, VEX_W, EVEX_CD8<64, CD8VF>; defm PDZ256 : avx512_fp_packed, EVEX_V256, PD, VEX_W, EVEX_CD8<64, CD8VF>; } } multiclass avx512_fp_binop_p_round opc, string OpcodeStr, SDNode OpNodeRnd> { defm PSZ : avx512_fp_round_packed, EVEX_V512, PS, EVEX_CD8<32, CD8VF>; defm PDZ : avx512_fp_round_packed, EVEX_V512, PD, VEX_W,EVEX_CD8<64, CD8VF>; } multiclass avx512_fp_binop_p_sae opc, string OpcodeStr, SDNode OpNodeRnd> { defm PSZ : avx512_fp_sae_packed, EVEX_V512, PS, EVEX_CD8<32, CD8VF>; defm PDZ : avx512_fp_sae_packed, EVEX_V512, PD, VEX_W,EVEX_CD8<64, CD8VF>; } defm VADD : avx512_fp_binop_p<0x58, "vadd", fadd, 1>, avx512_fp_binop_p_round<0x58, "vadd", X86faddRnd>; defm VMUL : avx512_fp_binop_p<0x59, "vmul", fmul, 1>, avx512_fp_binop_p_round<0x59, "vmul", X86fmulRnd>; defm VSUB : avx512_fp_binop_p<0x5C, "vsub", fsub>, avx512_fp_binop_p_round<0x5C, "vsub", X86fsubRnd>; defm VDIV : avx512_fp_binop_p<0x5E, "vdiv", fdiv>, avx512_fp_binop_p_round<0x5E, "vdiv", X86fdivRnd>; defm VMIN : avx512_fp_binop_p<0x5D, "vmin", X86fmin, 1>, avx512_fp_binop_p_sae<0x5D, "vmin", X86fminRnd>; defm VMAX : avx512_fp_binop_p<0x5F, "vmax", X86fmax, 1>, avx512_fp_binop_p_sae<0x5F, "vmax", X86fmaxRnd>; let Predicates = [HasDQI] in { defm VAND : avx512_fp_binop_p<0x54, "vand", X86fand, 1>; defm VANDN : avx512_fp_binop_p<0x55, "vandn", X86fandn, 0>; defm VOR : avx512_fp_binop_p<0x56, "vor", X86for, 1>; defm VXOR : avx512_fp_binop_p<0x57, "vxor", X86fxor, 1>; } multiclass avx512_fp_scalef_p opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { defm rr: AVX512_maskable, EVEX_4V; let mayLoad = 1 in { defm rm: AVX512_maskable, EVEX_4V; defm rmb: AVX512_maskable, EVEX_4V, EVEX_B; }//let mayLoad = 1 } multiclass avx512_fp_scalef_scalar opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { defm rr: AVX512_maskable_scalar; let mayLoad = 1 in { defm rm: AVX512_maskable_scalar; }//let mayLoad = 1 } multiclass avx512_fp_scalef_all opc, bits<8> opcScaler, string OpcodeStr, SDNode OpNode> { defm PSZ : avx512_fp_scalef_p, avx512_fp_round_packed, EVEX_V512, EVEX_CD8<32, CD8VF>; defm PDZ : avx512_fp_scalef_p, avx512_fp_round_packed, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; defm SSZ128 : avx512_fp_scalef_scalar, avx512_fp_scalar_round, EVEX_4V,EVEX_CD8<32, CD8VT1>; defm SDZ128 : avx512_fp_scalef_scalar, avx512_fp_scalar_round, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W; // Define only if AVX512VL feature is present. let Predicates = [HasVLX] in { defm PSZ128 : avx512_fp_scalef_p, EVEX_V128, EVEX_CD8<32, CD8VF>; defm PSZ256 : avx512_fp_scalef_p, EVEX_V256, EVEX_CD8<32, CD8VF>; defm PDZ128 : avx512_fp_scalef_p, EVEX_V128, VEX_W, EVEX_CD8<64, CD8VF>; defm PDZ256 : avx512_fp_scalef_p, EVEX_V256, VEX_W, EVEX_CD8<64, CD8VF>; } } defm VSCALEF : avx512_fp_scalef_all<0x2C, 0x2D, "vscalef", X86scalef>, T8PD; //===----------------------------------------------------------------------===// // AVX-512 VPTESTM instructions //===----------------------------------------------------------------------===// multiclass avx512_vptest opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { defm rr : AVX512_maskable_cmp, EVEX_4V; let mayLoad = 1 in defm rm : AVX512_maskable_cmp, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>; } multiclass avx512_vptest_mb opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { let mayLoad = 1 in defm rmb : AVX512_maskable_cmp, EVEX_B, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>; } multiclass avx512_vptest_dq_sizes opc, string OpcodeStr, SDNode OpNode, AVX512VLVectorVTInfo _> { let Predicates = [HasAVX512] in defm Z : avx512_vptest, avx512_vptest_mb, EVEX_V512; let Predicates = [HasAVX512, HasVLX] in { defm Z256 : avx512_vptest, avx512_vptest_mb, EVEX_V256; defm Z128 : avx512_vptest, avx512_vptest_mb, EVEX_V128; } } multiclass avx512_vptest_dq opc, string OpcodeStr, SDNode OpNode> { defm D : avx512_vptest_dq_sizes; defm Q : avx512_vptest_dq_sizes, VEX_W; } multiclass avx512_vptest_wb opc, string OpcodeStr, SDNode OpNode> { let Predicates = [HasBWI] in { defm WZ: avx512_vptest, EVEX_V512, VEX_W; defm BZ: avx512_vptest, EVEX_V512; } let Predicates = [HasVLX, HasBWI] in { defm WZ256: avx512_vptest, EVEX_V256, VEX_W; defm WZ128: avx512_vptest, EVEX_V128, VEX_W; defm BZ256: avx512_vptest, EVEX_V256; defm BZ128: avx512_vptest, EVEX_V128; } } multiclass avx512_vptest_all_forms opc_wb, bits<8> opc_dq, string OpcodeStr, SDNode OpNode> : avx512_vptest_wb , avx512_vptest_dq; defm VPTESTM : avx512_vptest_all_forms<0x26, 0x27, "vptestm", X86testm>, T8PD; defm VPTESTNM : avx512_vptest_all_forms<0x26, 0x27, "vptestnm", X86testnm>, T8XS; def : Pat <(i16 (int_x86_avx512_mask_ptestm_d_512 (v16i32 VR512:$src1), (v16i32 VR512:$src2), (i16 -1))), (COPY_TO_REGCLASS (VPTESTMDZrr VR512:$src1, VR512:$src2), GR16)>; def : Pat <(i8 (int_x86_avx512_mask_ptestm_q_512 (v8i64 VR512:$src1), (v8i64 VR512:$src2), (i8 -1))), (COPY_TO_REGCLASS (VPTESTMQZrr VR512:$src1, VR512:$src2), GR8)>; //===----------------------------------------------------------------------===// // AVX-512 Shift instructions //===----------------------------------------------------------------------===// multiclass avx512_shift_rmi opc, Format ImmFormR, Format ImmFormM, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { defm ri : AVX512_maskable; let mayLoad = 1 in defm mi : AVX512_maskable; } multiclass avx512_shift_rmbi opc, Format ImmFormM, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { let mayLoad = 1 in defm mbi : AVX512_maskable, EVEX_B; } multiclass avx512_shift_rrm opc, string OpcodeStr, SDNode OpNode, ValueType SrcVT, PatFrag bc_frag, X86VectorVTInfo _> { // src2 is always 128-bit defm rr : AVX512_maskable, AVX512BIBase, EVEX_4V; defm rm : AVX512_maskable, AVX512BIBase, EVEX_4V; } multiclass avx512_shift_sizes opc, string OpcodeStr, SDNode OpNode, ValueType SrcVT, PatFrag bc_frag, AVX512VLVectorVTInfo VTInfo, Predicate prd> { let Predicates = [prd] in defm Z : avx512_shift_rrm, EVEX_V512, EVEX_CD8 ; let Predicates = [prd, HasVLX] in { defm Z256 : avx512_shift_rrm, EVEX_V256, EVEX_CD8; defm Z128 : avx512_shift_rrm, EVEX_V128, EVEX_CD8; } } multiclass avx512_shift_types opcd, bits<8> opcq, bits<8> opcw, string OpcodeStr, SDNode OpNode> { defm D : avx512_shift_sizes; defm Q : avx512_shift_sizes, VEX_W; defm W : avx512_shift_sizes; } multiclass avx512_shift_rmi_sizes opc, Format ImmFormR, Format ImmFormM, string OpcodeStr, SDNode OpNode, AVX512VLVectorVTInfo VTInfo> { let Predicates = [HasAVX512] in defm Z: avx512_shift_rmi, avx512_shift_rmbi, EVEX_V512; let Predicates = [HasAVX512, HasVLX] in { defm Z256: avx512_shift_rmi, avx512_shift_rmbi, EVEX_V256; defm Z128: avx512_shift_rmi, avx512_shift_rmbi, EVEX_V128; } } multiclass avx512_shift_rmi_w opcw, Format ImmFormR, Format ImmFormM, string OpcodeStr, SDNode OpNode> { let Predicates = [HasBWI] in defm WZ: avx512_shift_rmi, EVEX_V512; let Predicates = [HasVLX, HasBWI] in { defm WZ256: avx512_shift_rmi, EVEX_V256; defm WZ128: avx512_shift_rmi, EVEX_V128; } } multiclass avx512_shift_rmi_dq opcd, bits<8> opcq, Format ImmFormR, Format ImmFormM, string OpcodeStr, SDNode OpNode> { defm D: avx512_shift_rmi_sizes, EVEX_CD8<32, CD8VF>; defm Q: avx512_shift_rmi_sizes, EVEX_CD8<64, CD8VF>, VEX_W; } defm VPSRL : avx512_shift_rmi_dq<0x72, 0x73, MRM2r, MRM2m, "vpsrl", X86vsrli>, avx512_shift_rmi_w<0x71, MRM2r, MRM2m, "vpsrlw", X86vsrli>, AVX512BIi8Base, EVEX_4V; defm VPSLL : avx512_shift_rmi_dq<0x72, 0x73, MRM6r, MRM6m, "vpsll", X86vshli>, avx512_shift_rmi_w<0x71, MRM6r, MRM6m, "vpsllw", X86vshli>, AVX512BIi8Base, EVEX_4V; defm VPSRA : avx512_shift_rmi_dq<0x72, 0x72, MRM4r, MRM4m, "vpsra", X86vsrai>, avx512_shift_rmi_w<0x71, MRM4r, MRM4m, "vpsraw", X86vsrai>, AVX512BIi8Base, EVEX_4V; defm VPROR : avx512_shift_rmi_dq<0x72, 0x72, MRM0r, MRM0m, "vpror", rotr>, AVX512BIi8Base, EVEX_4V; defm VPROL : avx512_shift_rmi_dq<0x72, 0x72, MRM1r, MRM1m, "vprol", rotl>, AVX512BIi8Base, EVEX_4V; defm VPSLL : avx512_shift_types<0xF2, 0xF3, 0xF1, "vpsll", X86vshl>; defm VPSRA : avx512_shift_types<0xE2, 0xE2, 0xE1, "vpsra", X86vsra>; defm VPSRL : avx512_shift_types<0xD2, 0xD3, 0xD1, "vpsrl", X86vsrl>; //===-------------------------------------------------------------------===// // Variable Bit Shifts //===-------------------------------------------------------------------===// multiclass avx512_var_shift opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { defm rr : AVX512_maskable, AVX5128IBase, EVEX_4V; let mayLoad = 1 in defm rm : AVX512_maskable, AVX5128IBase, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>; } multiclass avx512_var_shift_mb opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { let mayLoad = 1 in defm rmb : AVX512_maskable, AVX5128IBase, EVEX_B, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>; } multiclass avx512_var_shift_sizes opc, string OpcodeStr, SDNode OpNode, AVX512VLVectorVTInfo _> { let Predicates = [HasAVX512] in defm Z : avx512_var_shift, avx512_var_shift_mb, EVEX_V512; let Predicates = [HasAVX512, HasVLX] in { defm Z256 : avx512_var_shift, avx512_var_shift_mb, EVEX_V256; defm Z128 : avx512_var_shift, avx512_var_shift_mb, EVEX_V128; } } multiclass avx512_var_shift_types opc, string OpcodeStr, SDNode OpNode> { defm D : avx512_var_shift_sizes; defm Q : avx512_var_shift_sizes, VEX_W; } // Use 512bit version to implement 128/256 bit in case NoVLX. multiclass avx512_var_shift_w_lowering { let Predicates = [HasBWI, NoVLX] in { def : Pat<(_.info256.VT (OpNode (_.info256.VT _.info256.RC:$src1), (_.info256.VT _.info256.RC:$src2))), (EXTRACT_SUBREG (!cast(NAME#"WZrr") (INSERT_SUBREG (_.info512.VT (IMPLICIT_DEF)), VR256X:$src1, sub_ymm), (INSERT_SUBREG (_.info512.VT (IMPLICIT_DEF)), VR256X:$src2, sub_ymm)), sub_ymm)>; def : Pat<(_.info128.VT (OpNode (_.info128.VT _.info128.RC:$src1), (_.info128.VT _.info128.RC:$src2))), (EXTRACT_SUBREG (!cast(NAME#"WZrr") (INSERT_SUBREG (_.info512.VT (IMPLICIT_DEF)), VR128X:$src1, sub_xmm), (INSERT_SUBREG (_.info512.VT (IMPLICIT_DEF)), VR128X:$src2, sub_xmm)), sub_xmm)>; } } multiclass avx512_var_shift_w opc, string OpcodeStr, SDNode OpNode> { let Predicates = [HasBWI] in defm WZ: avx512_var_shift, EVEX_V512, VEX_W; let Predicates = [HasVLX, HasBWI] in { defm WZ256: avx512_var_shift, EVEX_V256, VEX_W; defm WZ128: avx512_var_shift, EVEX_V128, VEX_W; } } defm VPSLLV : avx512_var_shift_types<0x47, "vpsllv", shl>, avx512_var_shift_w<0x12, "vpsllvw", shl>, avx512_var_shift_w_lowering; defm VPSRAV : avx512_var_shift_types<0x46, "vpsrav", sra>, avx512_var_shift_w<0x11, "vpsravw", sra>, avx512_var_shift_w_lowering; defm VPSRLV : avx512_var_shift_types<0x45, "vpsrlv", srl>, avx512_var_shift_w<0x10, "vpsrlvw", srl>, avx512_var_shift_w_lowering; defm VPRORV : avx512_var_shift_types<0x14, "vprorv", rotr>; defm VPROLV : avx512_var_shift_types<0x15, "vprolv", rotl>; //===-------------------------------------------------------------------===// // 1-src variable permutation VPERMW/D/Q //===-------------------------------------------------------------------===// multiclass avx512_vperm_dq_sizes opc, string OpcodeStr, SDNode OpNode, AVX512VLVectorVTInfo _> { let Predicates = [HasAVX512] in defm Z : avx512_var_shift, avx512_var_shift_mb, EVEX_V512; let Predicates = [HasAVX512, HasVLX] in defm Z256 : avx512_var_shift, avx512_var_shift_mb, EVEX_V256; } multiclass avx512_vpermi_dq_sizes opc, Format ImmFormR, Format ImmFormM, string OpcodeStr, SDNode OpNode, AVX512VLVectorVTInfo VTInfo> { let Predicates = [HasAVX512] in defm Z: avx512_shift_rmi, avx512_shift_rmbi, EVEX_V512; let Predicates = [HasAVX512, HasVLX] in defm Z256: avx512_shift_rmi, avx512_shift_rmbi, EVEX_V256; } defm VPERM : avx512_var_shift_w<0x8D, "vpermw", X86VPermv>; defm VPERMD : avx512_vperm_dq_sizes<0x36, "vpermd", X86VPermv, avx512vl_i32_info>; defm VPERMQ : avx512_vperm_dq_sizes<0x36, "vpermq", X86VPermv, avx512vl_i64_info>, VEX_W; defm VPERMPS : avx512_vperm_dq_sizes<0x16, "vpermps", X86VPermv, avx512vl_f32_info>; defm VPERMPD : avx512_vperm_dq_sizes<0x16, "vpermpd", X86VPermv, avx512vl_f64_info>, VEX_W; defm VPERMQ : avx512_vpermi_dq_sizes<0x00, MRMSrcReg, MRMSrcMem, "vpermq", X86VPermi, avx512vl_i64_info>, EVEX, AVX512AIi8Base, EVEX_CD8<64, CD8VF>, VEX_W; defm VPERMPD : avx512_vpermi_dq_sizes<0x01, MRMSrcReg, MRMSrcMem, "vpermpd", X86VPermi, avx512vl_f64_info>, EVEX, AVX512AIi8Base, EVEX_CD8<64, CD8VF>, VEX_W; //===----------------------------------------------------------------------===// // AVX-512 - VPERMIL //===----------------------------------------------------------------------===// multiclass avx512_permil_vec OpcVar, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _, X86VectorVTInfo Ctrl> { defm rr: AVX512_maskable, T8PD, EVEX_4V; let mayLoad = 1 in { defm rm: AVX512_maskable, T8PD, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>; defm rmb: AVX512_maskable, T8PD, EVEX_4V, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>; }//let mayLoad = 1 } multiclass avx512_permil_vec_common OpcVar, AVX512VLVectorVTInfo _, AVX512VLVectorVTInfo Ctrl>{ let Predicates = [HasAVX512] in { defm Z : avx512_permil_vec, EVEX_V512; } let Predicates = [HasAVX512, HasVLX] in { defm Z128 : avx512_permil_vec, EVEX_V128; defm Z256 : avx512_permil_vec, EVEX_V256; } } multiclass avx512_permil OpcImm, bits<8> OpcVar, AVX512VLVectorVTInfo _, AVX512VLVectorVTInfo Ctrl>{ defm NAME: avx512_permil_vec_common; defm NAME: avx512_shift_rmi_sizes, EVEX, AVX512AIi8Base, EVEX_CD8<_.info128.EltSize, CD8VF>; } defm VPERMILPS : avx512_permil<"vpermilps", 0x04, 0x0C, avx512vl_f32_info, avx512vl_i32_info>; defm VPERMILPD : avx512_permil<"vpermilpd", 0x05, 0x0D, avx512vl_f64_info, avx512vl_i64_info>, VEX_W; //===----------------------------------------------------------------------===// // AVX-512 - VPSHUFD, VPSHUFLW, VPSHUFHW //===----------------------------------------------------------------------===// defm VPSHUFD : avx512_shift_rmi_sizes<0x70, MRMSrcReg, MRMSrcMem, "vpshufd", X86PShufd, avx512vl_i32_info>, EVEX, AVX512BIi8Base, EVEX_CD8<32, CD8VF>; defm VPSHUFH : avx512_shift_rmi_w<0x70, MRMSrcReg, MRMSrcMem, "vpshufhw", X86PShufhw>, EVEX, AVX512XSIi8Base; defm VPSHUFL : avx512_shift_rmi_w<0x70, MRMSrcReg, MRMSrcMem, "vpshuflw", X86PShuflw>, EVEX, AVX512XDIi8Base; multiclass avx512_pshufb_sizes opc, string OpcodeStr, SDNode OpNode> { let Predicates = [HasBWI] in defm Z: avx512_var_shift, EVEX_V512; let Predicates = [HasVLX, HasBWI] in { defm Z256: avx512_var_shift, EVEX_V256; defm Z128: avx512_var_shift, EVEX_V128; } } defm VPSHUFB: avx512_pshufb_sizes<0x00, "vpshufb", X86pshufb>; //===----------------------------------------------------------------------===// // Move Low to High and High to Low packed FP Instructions //===----------------------------------------------------------------------===// def VMOVLHPSZrr : AVX512PSI<0x16, MRMSrcReg, (outs VR128X:$dst), (ins VR128X:$src1, VR128X:$src2), "vmovlhps\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set VR128X:$dst, (v4f32 (X86Movlhps VR128X:$src1, VR128X:$src2)))], IIC_SSE_MOV_LH>, EVEX_4V; def VMOVHLPSZrr : AVX512PSI<0x12, MRMSrcReg, (outs VR128X:$dst), (ins VR128X:$src1, VR128X:$src2), "vmovhlps\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set VR128X:$dst, (v4f32 (X86Movhlps VR128X:$src1, VR128X:$src2)))], IIC_SSE_MOV_LH>, EVEX_4V; let Predicates = [HasAVX512] in { // MOVLHPS patterns def : Pat<(v4i32 (X86Movlhps VR128X:$src1, VR128X:$src2)), (VMOVLHPSZrr VR128X:$src1, VR128X:$src2)>; def : Pat<(v2i64 (X86Movlhps VR128X:$src1, VR128X:$src2)), (VMOVLHPSZrr (v2i64 VR128X:$src1), VR128X:$src2)>; // MOVHLPS patterns def : Pat<(v4i32 (X86Movhlps VR128X:$src1, VR128X:$src2)), (VMOVHLPSZrr VR128X:$src1, VR128X:$src2)>; } //===----------------------------------------------------------------------===// // VMOVHPS/PD VMOVLPS Instructions // All patterns was taken from SSS implementation. //===----------------------------------------------------------------------===// multiclass avx512_mov_hilo_packed opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { let mayLoad = 1 in def rm : AVX512, EVEX_4V; } defm VMOVHPSZ128 : avx512_mov_hilo_packed<0x16, "vmovhps", X86Movlhps, v4f32x_info>, EVEX_CD8<32, CD8VT2>, PS; defm VMOVHPDZ128 : avx512_mov_hilo_packed<0x16, "vmovhpd", X86Movlhpd, v2f64x_info>, EVEX_CD8<64, CD8VT1>, PD, VEX_W; defm VMOVLPSZ128 : avx512_mov_hilo_packed<0x12, "vmovlps", X86Movlps, v4f32x_info>, EVEX_CD8<32, CD8VT2>, PS; defm VMOVLPDZ128 : avx512_mov_hilo_packed<0x12, "vmovlpd", X86Movlpd, v2f64x_info>, EVEX_CD8<64, CD8VT1>, PD, VEX_W; let Predicates = [HasAVX512] in { // VMOVHPS patterns def : Pat<(X86Movlhps VR128X:$src1, (bc_v4f32 (v2i64 (scalar_to_vector (loadi64 addr:$src2))))), (VMOVHPSZ128rm VR128X:$src1, addr:$src2)>; def : Pat<(X86Movlhps VR128X:$src1, (bc_v4i32 (v2i64 (X86vzload addr:$src2)))), (VMOVHPSZ128rm VR128X:$src1, addr:$src2)>; // VMOVHPD patterns def : Pat<(v2f64 (X86Unpckl VR128X:$src1, (scalar_to_vector (loadf64 addr:$src2)))), (VMOVHPDZ128rm VR128X:$src1, addr:$src2)>; def : Pat<(v2f64 (X86Unpckl VR128X:$src1, (bc_v2f64 (v2i64 (scalar_to_vector (loadi64 addr:$src2)))))), (VMOVHPDZ128rm VR128X:$src1, addr:$src2)>; // VMOVLPS patterns def : Pat<(v4f32 (X86Movlps VR128X:$src1, (load addr:$src2))), (VMOVLPSZ128rm VR128X:$src1, addr:$src2)>; def : Pat<(v4i32 (X86Movlps VR128X:$src1, (load addr:$src2))), (VMOVLPSZ128rm VR128X:$src1, addr:$src2)>; // VMOVLPD patterns def : Pat<(v2f64 (X86Movlpd VR128X:$src1, (load addr:$src2))), (VMOVLPDZ128rm VR128X:$src1, addr:$src2)>; def : Pat<(v2i64 (X86Movlpd VR128X:$src1, (load addr:$src2))), (VMOVLPDZ128rm VR128X:$src1, addr:$src2)>; def : Pat<(v2f64 (X86Movsd VR128X:$src1, (v2f64 (scalar_to_vector (loadf64 addr:$src2))))), (VMOVLPDZ128rm VR128X:$src1, addr:$src2)>; } let mayStore = 1 in { def VMOVHPSZ128mr : AVX512PSI<0x17, MRMDestMem, (outs), (ins f64mem:$dst, VR128X:$src), "vmovhps\t{$src, $dst|$dst, $src}", [(store (f64 (vector_extract (X86Unpckh (bc_v2f64 (v4f32 VR128X:$src)), (bc_v2f64 (v4f32 VR128X:$src))), (iPTR 0))), addr:$dst)], IIC_SSE_MOV_LH>, EVEX, EVEX_CD8<32, CD8VT2>; def VMOVHPDZ128mr : AVX512PDI<0x17, MRMDestMem, (outs), (ins f64mem:$dst, VR128X:$src), "vmovhpd\t{$src, $dst|$dst, $src}", [(store (f64 (vector_extract (v2f64 (X86Unpckh VR128X:$src, VR128X:$src)), (iPTR 0))), addr:$dst)], IIC_SSE_MOV_LH>, EVEX, EVEX_CD8<64, CD8VT1>, VEX_W; def VMOVLPSZ128mr : AVX512PSI<0x13, MRMDestMem, (outs), (ins f64mem:$dst, VR128X:$src), "vmovlps\t{$src, $dst|$dst, $src}", [(store (f64 (vector_extract (bc_v2f64 (v4f32 VR128X:$src)), (iPTR 0))), addr:$dst)], IIC_SSE_MOV_LH>, EVEX, EVEX_CD8<32, CD8VT2>; def VMOVLPDZ128mr : AVX512PDI<0x13, MRMDestMem, (outs), (ins f64mem:$dst, VR128X:$src), "vmovlpd\t{$src, $dst|$dst, $src}", [(store (f64 (vector_extract (v2f64 VR128X:$src), (iPTR 0))), addr:$dst)], IIC_SSE_MOV_LH>, EVEX, EVEX_CD8<64, CD8VT1>, VEX_W; } let Predicates = [HasAVX512] in { // VMOVHPD patterns def : Pat<(store (f64 (vector_extract (v2f64 (X86VPermilpi VR128X:$src, (i8 1))), (iPTR 0))), addr:$dst), (VMOVHPDZ128mr addr:$dst, VR128X:$src)>; // VMOVLPS patterns def : Pat<(store (v4f32 (X86Movlps (load addr:$src1), VR128X:$src2)), addr:$src1), (VMOVLPSZ128mr addr:$src1, VR128X:$src2)>; def : Pat<(store (v4i32 (X86Movlps (bc_v4i32 (loadv2i64 addr:$src1)), VR128X:$src2)), addr:$src1), (VMOVLPSZ128mr addr:$src1, VR128X:$src2)>; // VMOVLPD patterns def : Pat<(store (v2f64 (X86Movlpd (load addr:$src1), VR128X:$src2)), addr:$src1), (VMOVLPDZ128mr addr:$src1, VR128X:$src2)>; def : Pat<(store (v2i64 (X86Movlpd (load addr:$src1), VR128X:$src2)), addr:$src1), (VMOVLPDZ128mr addr:$src1, VR128X:$src2)>; } //===----------------------------------------------------------------------===// // FMA - Fused Multiply Operations // let Constraints = "$src1 = $dst" in { multiclass avx512_fma3p_213_rm opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { defm r: AVX512_maskable_3src, AVX512FMA3Base; let mayLoad = 1 in { defm m: AVX512_maskable_3src, AVX512FMA3Base; defm mb: AVX512_maskable_3src, AVX512FMA3Base, EVEX_B; } } multiclass avx512_fma3_213_round opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { defm rb: AVX512_maskable_3src, AVX512FMA3Base, EVEX_B, EVEX_RC; } } // Constraints = "$src1 = $dst" multiclass avx512_fma3p_213_common opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd, AVX512VLVectorVTInfo _> { let Predicates = [HasAVX512] in { defm Z : avx512_fma3p_213_rm, avx512_fma3_213_round, EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>; } let Predicates = [HasVLX, HasAVX512] in { defm Z256 : avx512_fma3p_213_rm, EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>; defm Z128 : avx512_fma3p_213_rm, EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>; } } multiclass avx512_fma3p_213_f opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd > { defm PS : avx512_fma3p_213_common; defm PD : avx512_fma3p_213_common, VEX_W; } defm VFMADD213 : avx512_fma3p_213_f<0xA8, "vfmadd213", X86Fmadd, X86FmaddRnd>; defm VFMSUB213 : avx512_fma3p_213_f<0xAA, "vfmsub213", X86Fmsub, X86FmsubRnd>; defm VFMADDSUB213 : avx512_fma3p_213_f<0xA6, "vfmaddsub213", X86Fmaddsub, X86FmaddsubRnd>; defm VFMSUBADD213 : avx512_fma3p_213_f<0xA7, "vfmsubadd213", X86Fmsubadd, X86FmsubaddRnd>; defm VFNMADD213 : avx512_fma3p_213_f<0xAC, "vfnmadd213", X86Fnmadd, X86FnmaddRnd>; defm VFNMSUB213 : avx512_fma3p_213_f<0xAE, "vfnmsub213", X86Fnmsub, X86FnmsubRnd>; let Constraints = "$src1 = $dst" in { multiclass avx512_fma3p_231_rm opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { defm r: AVX512_maskable_3src, AVX512FMA3Base; let mayLoad = 1 in { defm m: AVX512_maskable_3src, AVX512FMA3Base; defm mb: AVX512_maskable_3src, AVX512FMA3Base, EVEX_B; } } multiclass avx512_fma3_231_round opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { defm rb: AVX512_maskable_3src, AVX512FMA3Base, EVEX_B, EVEX_RC; } } // Constraints = "$src1 = $dst" multiclass avx512_fma3p_231_common opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd, AVX512VLVectorVTInfo _> { let Predicates = [HasAVX512] in { defm Z : avx512_fma3p_231_rm, avx512_fma3_231_round, EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>; } let Predicates = [HasVLX, HasAVX512] in { defm Z256 : avx512_fma3p_231_rm, EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>; defm Z128 : avx512_fma3p_231_rm, EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>; } } multiclass avx512_fma3p_231_f opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd > { defm PS : avx512_fma3p_231_common; defm PD : avx512_fma3p_231_common, VEX_W; } defm VFMADD231 : avx512_fma3p_231_f<0xB8, "vfmadd231", X86Fmadd, X86FmaddRnd>; defm VFMSUB231 : avx512_fma3p_231_f<0xBA, "vfmsub231", X86Fmsub, X86FmsubRnd>; defm VFMADDSUB231 : avx512_fma3p_231_f<0xB6, "vfmaddsub231", X86Fmaddsub, X86FmaddsubRnd>; defm VFMSUBADD231 : avx512_fma3p_231_f<0xB7, "vfmsubadd231", X86Fmsubadd, X86FmsubaddRnd>; defm VFNMADD231 : avx512_fma3p_231_f<0xBC, "vfnmadd231", X86Fnmadd, X86FnmaddRnd>; defm VFNMSUB231 : avx512_fma3p_231_f<0xBE, "vfnmsub231", X86Fnmsub, X86FnmsubRnd>; let Constraints = "$src1 = $dst" in { multiclass avx512_fma3p_132_rm opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { defm r: AVX512_maskable_3src, AVX512FMA3Base; let mayLoad = 1 in { defm m: AVX512_maskable_3src, AVX512FMA3Base; defm mb: AVX512_maskable_3src, AVX512FMA3Base, EVEX_B; } } multiclass avx512_fma3_132_round opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { defm rb: AVX512_maskable_3src, AVX512FMA3Base, EVEX_B, EVEX_RC; } } // Constraints = "$src1 = $dst" multiclass avx512_fma3p_132_common opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd, AVX512VLVectorVTInfo _> { let Predicates = [HasAVX512] in { defm Z : avx512_fma3p_132_rm, avx512_fma3_132_round, EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>; } let Predicates = [HasVLX, HasAVX512] in { defm Z256 : avx512_fma3p_132_rm, EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>; defm Z128 : avx512_fma3p_132_rm, EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>; } } multiclass avx512_fma3p_132_f opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd > { defm PS : avx512_fma3p_132_common; defm PD : avx512_fma3p_132_common, VEX_W; } defm VFMADD132 : avx512_fma3p_132_f<0x98, "vfmadd132", X86Fmadd, X86FmaddRnd>; defm VFMSUB132 : avx512_fma3p_132_f<0x9A, "vfmsub132", X86Fmsub, X86FmsubRnd>; defm VFMADDSUB132 : avx512_fma3p_132_f<0x96, "vfmaddsub132", X86Fmaddsub, X86FmaddsubRnd>; defm VFMSUBADD132 : avx512_fma3p_132_f<0x97, "vfmsubadd132", X86Fmsubadd, X86FmsubaddRnd>; defm VFNMADD132 : avx512_fma3p_132_f<0x9C, "vfnmadd132", X86Fnmadd, X86FnmaddRnd>; defm VFNMSUB132 : avx512_fma3p_132_f<0x9E, "vfnmsub132", X86Fnmsub, X86FnmsubRnd>; // Scalar FMA let Constraints = "$src1 = $dst" in { multiclass avx512_fma3s_common opc, string OpcodeStr, X86VectorVTInfo _, dag RHS_VEC_r, dag RHS_VEC_m, dag RHS_VEC_rb, dag RHS_r, dag RHS_m > { defm r_Int: AVX512_maskable_3src_scalar, AVX512FMA3Base; let mayLoad = 1 in defm m_Int: AVX512_maskable_3src_scalar, AVX512FMA3Base; defm rb_Int: AVX512_maskable_3src_scalar, AVX512FMA3Base, EVEX_B, EVEX_RC; let isCodeGenOnly = 1 in { def r : AVX512FMA3; let mayLoad = 1 in def m : AVX512FMA3; }// isCodeGenOnly = 1 } }// Constraints = "$src1 = $dst" multiclass avx512_fma3s_all opc213, bits<8> opc231, bits<8> opc132, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd, X86VectorVTInfo _ , string SUFF> { defm NAME#213#SUFF: avx512_fma3s_common; defm NAME#231#SUFF: avx512_fma3s_common; defm NAME#132#SUFF: avx512_fma3s_common; } multiclass avx512_fma3s opc213, bits<8> opc231, bits<8> opc132, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd>{ let Predicates = [HasAVX512] in { defm NAME : avx512_fma3s_all, EVEX_CD8<32, CD8VT1>, VEX_LIG; defm NAME : avx512_fma3s_all, EVEX_CD8<64, CD8VT1>, VEX_LIG, VEX_W; } } defm VFMADD : avx512_fma3s<0xA9, 0xB9, 0x99, "vfmadd", X86Fmadd, X86FmaddRnd>; defm VFMSUB : avx512_fma3s<0xAB, 0xBB, 0x9B, "vfmsub", X86Fmsub, X86FmsubRnd>; defm VFNMADD : avx512_fma3s<0xAD, 0xBD, 0x9D, "vfnmadd", X86Fnmadd, X86FnmaddRnd>; defm VFNMSUB : avx512_fma3s<0xAF, 0xBF, 0x9F, "vfnmsub", X86Fnmsub, X86FnmsubRnd>; //===----------------------------------------------------------------------===// // AVX-512 Scalar convert from sign integer to float/double //===----------------------------------------------------------------------===// multiclass avx512_vcvtsi opc, SDNode OpNode, RegisterClass SrcRC, X86VectorVTInfo DstVT, X86MemOperand x86memop, PatFrag ld_frag, string asm> { let hasSideEffects = 0 in { def rr : SI, EVEX_4V; let mayLoad = 1 in def rm : SI, EVEX_4V; } // hasSideEffects = 0 let isCodeGenOnly = 1 in { def rr_Int : SI, EVEX_4V; def rm_Int : SI, EVEX_4V; }//isCodeGenOnly = 1 } multiclass avx512_vcvtsi_round opc, SDNode OpNode, RegisterClass SrcRC, X86VectorVTInfo DstVT, string asm> { def rrb_Int : SI, EVEX_4V, EVEX_B, EVEX_RC; } multiclass avx512_vcvtsi_common opc, SDNode OpNode, RegisterClass SrcRC, X86VectorVTInfo DstVT, X86MemOperand x86memop, PatFrag ld_frag, string asm> { defm NAME : avx512_vcvtsi_round, avx512_vcvtsi, VEX_LIG; } let Predicates = [HasAVX512] in { defm VCVTSI2SSZ : avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR32, v4f32x_info, i32mem, loadi32, "cvtsi2ss{l}">, XS, EVEX_CD8<32, CD8VT1>; defm VCVTSI642SSZ: avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR64, v4f32x_info, i64mem, loadi64, "cvtsi2ss{q}">, XS, VEX_W, EVEX_CD8<64, CD8VT1>; defm VCVTSI2SDZ : avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR32, v2f64x_info, i32mem, loadi32, "cvtsi2sd{l}">, XD, EVEX_CD8<32, CD8VT1>; defm VCVTSI642SDZ: avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR64, v2f64x_info, i64mem, loadi64, "cvtsi2sd{q}">, XD, VEX_W, EVEX_CD8<64, CD8VT1>; def : Pat<(f32 (sint_to_fp (loadi32 addr:$src))), (VCVTSI2SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>; def : Pat<(f32 (sint_to_fp (loadi64 addr:$src))), (VCVTSI642SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>; def : Pat<(f64 (sint_to_fp (loadi32 addr:$src))), (VCVTSI2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>; def : Pat<(f64 (sint_to_fp (loadi64 addr:$src))), (VCVTSI642SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>; def : Pat<(f32 (sint_to_fp GR32:$src)), (VCVTSI2SSZrr (f32 (IMPLICIT_DEF)), GR32:$src)>; def : Pat<(f32 (sint_to_fp GR64:$src)), (VCVTSI642SSZrr (f32 (IMPLICIT_DEF)), GR64:$src)>; def : Pat<(f64 (sint_to_fp GR32:$src)), (VCVTSI2SDZrr (f64 (IMPLICIT_DEF)), GR32:$src)>; def : Pat<(f64 (sint_to_fp GR64:$src)), (VCVTSI642SDZrr (f64 (IMPLICIT_DEF)), GR64:$src)>; defm VCVTUSI2SSZ : avx512_vcvtsi_common<0x7B, X86UintToFpRnd, GR32, v4f32x_info, i32mem, loadi32, "cvtusi2ss{l}">, XS, EVEX_CD8<32, CD8VT1>; defm VCVTUSI642SSZ : avx512_vcvtsi_common<0x7B, X86UintToFpRnd, GR64, v4f32x_info, i64mem, loadi64, "cvtusi2ss{q}">, XS, VEX_W, EVEX_CD8<64, CD8VT1>; defm VCVTUSI2SDZ : avx512_vcvtsi<0x7B, X86UintToFpRnd, GR32, v2f64x_info, i32mem, loadi32, "cvtusi2sd{l}">, XD, VEX_LIG, EVEX_CD8<32, CD8VT1>; defm VCVTUSI642SDZ : avx512_vcvtsi_common<0x7B, X86UintToFpRnd, GR64, v2f64x_info, i64mem, loadi64, "cvtusi2sd{q}">, XD, VEX_W, EVEX_CD8<64, CD8VT1>; def : Pat<(f32 (uint_to_fp (loadi32 addr:$src))), (VCVTUSI2SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>; def : Pat<(f32 (uint_to_fp (loadi64 addr:$src))), (VCVTUSI642SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>; def : Pat<(f64 (uint_to_fp (loadi32 addr:$src))), (VCVTUSI2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>; def : Pat<(f64 (uint_to_fp (loadi64 addr:$src))), (VCVTUSI642SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>; def : Pat<(f32 (uint_to_fp GR32:$src)), (VCVTUSI2SSZrr (f32 (IMPLICIT_DEF)), GR32:$src)>; def : Pat<(f32 (uint_to_fp GR64:$src)), (VCVTUSI642SSZrr (f32 (IMPLICIT_DEF)), GR64:$src)>; def : Pat<(f64 (uint_to_fp GR32:$src)), (VCVTUSI2SDZrr (f64 (IMPLICIT_DEF)), GR32:$src)>; def : Pat<(f64 (uint_to_fp GR64:$src)), (VCVTUSI642SDZrr (f64 (IMPLICIT_DEF)), GR64:$src)>; } //===----------------------------------------------------------------------===// // AVX-512 Scalar convert from float/double to integer //===----------------------------------------------------------------------===// multiclass avx512_cvt_s_int_round opc, RegisterClass SrcRC, RegisterClass DstRC, Intrinsic Int, Operand memop, ComplexPattern mem_cpat, string asm> { let hasSideEffects = 0, Predicates = [HasAVX512] in { def rr : SI, EVEX, VEX_LIG; def rb : SI, EVEX, VEX_LIG, EVEX_B, EVEX_RC; let mayLoad = 1 in def rm : SI, EVEX, VEX_LIG; } // hasSideEffects = 0, Predicates = [HasAVX512] } // Convert float/double to signed/unsigned int 32/64 defm VCVTSS2SIZ: avx512_cvt_s_int_round<0x2D, VR128X, GR32, int_x86_sse_cvtss2si, ssmem, sse_load_f32, "cvtss2si">, XS, EVEX_CD8<32, CD8VT1>; defm VCVTSS2SI64Z: avx512_cvt_s_int_round<0x2D, VR128X, GR64, int_x86_sse_cvtss2si64, ssmem, sse_load_f32, "cvtss2si">, XS, VEX_W, EVEX_CD8<32, CD8VT1>; defm VCVTSS2USIZ: avx512_cvt_s_int_round<0x79, VR128X, GR32, int_x86_avx512_cvtss2usi, ssmem, sse_load_f32, "cvtss2usi">, XS, EVEX_CD8<32, CD8VT1>; defm VCVTSS2USI64Z: avx512_cvt_s_int_round<0x79, VR128X, GR64, int_x86_avx512_cvtss2usi64, ssmem, sse_load_f32, "cvtss2usi">, XS, VEX_W, EVEX_CD8<32, CD8VT1>; defm VCVTSD2SIZ: avx512_cvt_s_int_round<0x2D, VR128X, GR32, int_x86_sse2_cvtsd2si, sdmem, sse_load_f64, "cvtsd2si">, XD, EVEX_CD8<64, CD8VT1>; defm VCVTSD2SI64Z: avx512_cvt_s_int_round<0x2D, VR128X, GR64, int_x86_sse2_cvtsd2si64, sdmem, sse_load_f64, "cvtsd2si">, XD, VEX_W, EVEX_CD8<64, CD8VT1>; defm VCVTSD2USIZ: avx512_cvt_s_int_round<0x79, VR128X, GR32, int_x86_avx512_cvtsd2usi, sdmem, sse_load_f64, "cvtsd2usi">, XD, EVEX_CD8<64, CD8VT1>; defm VCVTSD2USI64Z: avx512_cvt_s_int_round<0x79, VR128X, GR64, int_x86_avx512_cvtsd2usi64, sdmem, sse_load_f64, "cvtsd2usi">, XD, VEX_W, EVEX_CD8<64, CD8VT1>; let isCodeGenOnly = 1 , Predicates = [HasAVX512] in { defm Int_VCVTSI2SSZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X, int_x86_sse_cvtsi2ss, i32mem, loadi32, "cvtsi2ss{l}", SSE_CVT_Scalar, 0>, XS, EVEX_4V; defm Int_VCVTSI2SS64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X, int_x86_sse_cvtsi642ss, i64mem, loadi64, "cvtsi2ss{q}", SSE_CVT_Scalar, 0>, XS, EVEX_4V, VEX_W; defm Int_VCVTSI2SDZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X, int_x86_sse2_cvtsi2sd, i32mem, loadi32, "cvtsi2sd{l}", SSE_CVT_Scalar, 0>, XD, EVEX_4V; defm Int_VCVTSI2SD64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X, int_x86_sse2_cvtsi642sd, i64mem, loadi64, "cvtsi2sd{q}", SSE_CVT_Scalar, 0>, XD, EVEX_4V, VEX_W; defm Int_VCVTUSI2SDZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X, int_x86_avx512_cvtusi2sd, i32mem, loadi32, "cvtusi2sd{l}", SSE_CVT_Scalar, 0>, XD, EVEX_4V; } // isCodeGenOnly = 1, Predicates = [HasAVX512] // Convert float/double to signed/unsigned int 32/64 with truncation multiclass avx512_cvt_s_all opc, string asm, X86VectorVTInfo _SrcRC, X86VectorVTInfo _DstRC, SDNode OpNode, SDNode OpNodeRnd>{ let Predicates = [HasAVX512] in { def rr : SI, EVEX; def rb : SI, EVEX, EVEX_B; def rm : SI, EVEX; let isCodeGenOnly = 1,hasSideEffects = 0 in { def rr_Int : SI, EVEX, VEX_LIG; def rb_Int : SI, EVEX,VEX_LIG , EVEX_B; let mayLoad = 1 in def rm_Int : SI, EVEX, VEX_LIG; } // isCodeGenOnly = 1, hasSideEffects = 0 } //HasAVX512 } defm VCVTTSS2SIZ: avx512_cvt_s_all<0x2C, "cvttss2si", f32x_info, i32x_info, fp_to_sint,X86cvttss2IntRnd>, XS, EVEX_CD8<32, CD8VT1>; defm VCVTTSS2SI64Z: avx512_cvt_s_all<0x2C, "cvttss2si", f32x_info, i64x_info, fp_to_sint,X86cvttss2IntRnd>, VEX_W, XS, EVEX_CD8<32, CD8VT1>; defm VCVTTSD2SIZ: avx512_cvt_s_all<0x2C, "cvttsd2si", f64x_info, i32x_info, fp_to_sint,X86cvttsd2IntRnd>, XD, EVEX_CD8<64, CD8VT1>; defm VCVTTSD2SI64Z: avx512_cvt_s_all<0x2C, "cvttsd2si", f64x_info, i64x_info, fp_to_sint,X86cvttsd2IntRnd>, VEX_W, XD, EVEX_CD8<64, CD8VT1>; defm VCVTTSS2USIZ: avx512_cvt_s_all<0x78, "cvttss2usi", f32x_info, i32x_info, fp_to_uint,X86cvttss2UIntRnd>, XS, EVEX_CD8<32, CD8VT1>; defm VCVTTSS2USI64Z: avx512_cvt_s_all<0x78, "cvttss2usi", f32x_info, i64x_info, fp_to_uint,X86cvttss2UIntRnd>, XS,VEX_W, EVEX_CD8<32, CD8VT1>; defm VCVTTSD2USIZ: avx512_cvt_s_all<0x78, "cvttsd2usi", f64x_info, i32x_info, fp_to_uint,X86cvttsd2UIntRnd>, XD, EVEX_CD8<64, CD8VT1>; defm VCVTTSD2USI64Z: avx512_cvt_s_all<0x78, "cvttsd2usi", f64x_info, i64x_info, fp_to_uint,X86cvttsd2UIntRnd>, XD, VEX_W, EVEX_CD8<64, CD8VT1>; let Predicates = [HasAVX512] in { def : Pat<(i32 (int_x86_sse_cvttss2si (v4f32 VR128X:$src))), (VCVTTSS2SIZrr_Int (COPY_TO_REGCLASS VR128X:$src, FR32X))>; def : Pat<(i64 (int_x86_sse_cvttss2si64 (v4f32 VR128X:$src))), (VCVTTSS2SI64Zrr_Int (COPY_TO_REGCLASS VR128X:$src, FR32X))>; def : Pat<(i32 (int_x86_sse2_cvttsd2si (v2f64 VR128X:$src))), (VCVTTSD2SIZrr_Int (COPY_TO_REGCLASS VR128X:$src, FR64X))>; def : Pat<(i64 (int_x86_sse2_cvttsd2si64 (v2f64 VR128X:$src))), (VCVTTSD2SI64Zrr_Int (COPY_TO_REGCLASS VR128X:$src, FR64X))>; } // HasAVX512 //===----------------------------------------------------------------------===// // AVX-512 Convert form float to double and back //===----------------------------------------------------------------------===// multiclass avx512_cvt_fp_scalar opc, string OpcodeStr, X86VectorVTInfo _, X86VectorVTInfo _Src, SDNode OpNode> { defm rr : AVX512_maskable_scalar, EVEX_4V, VEX_LIG, Sched<[WriteCvtF2F]>; defm rm : AVX512_maskable_scalar, EVEX_4V, VEX_LIG, Sched<[WriteCvtF2FLd, ReadAfterLd]>; } // Scalar Coversion with SAE - suppress all exceptions multiclass avx512_cvt_fp_sae_scalar opc, string OpcodeStr, X86VectorVTInfo _, X86VectorVTInfo _Src, SDNode OpNodeRnd> { defm rrb : AVX512_maskable_scalar, EVEX_4V, VEX_LIG, EVEX_B; } // Scalar Conversion with rounding control (RC) multiclass avx512_cvt_fp_rc_scalar opc, string OpcodeStr, X86VectorVTInfo _, X86VectorVTInfo _Src, SDNode OpNodeRnd> { defm rrb : AVX512_maskable_scalar, EVEX_4V, VEX_LIG, Sched<[WriteCvtF2FLd, ReadAfterLd]>, EVEX_B, EVEX_RC; } multiclass avx512_cvt_fp_scalar_sd2ss opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd, X86VectorVTInfo _src, X86VectorVTInfo _dst> { let Predicates = [HasAVX512] in { defm Z : avx512_cvt_fp_scalar, avx512_cvt_fp_rc_scalar, VEX_W, EVEX_CD8<64, CD8VT1>, EVEX_V512, XD; } } multiclass avx512_cvt_fp_scalar_ss2sd opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd, X86VectorVTInfo _src, X86VectorVTInfo _dst> { let Predicates = [HasAVX512] in { defm Z : avx512_cvt_fp_scalar, avx512_cvt_fp_sae_scalar, EVEX_CD8<32, CD8VT1>, XS, EVEX_V512; } } defm VCVTSD2SS : avx512_cvt_fp_scalar_sd2ss<0x5A, "vcvtsd2ss", X86fround, X86froundRnd, f64x_info, f32x_info>; defm VCVTSS2SD : avx512_cvt_fp_scalar_ss2sd<0x5A, "vcvtss2sd", X86fpext, X86fpextRnd,f32x_info, f64x_info >; def : Pat<(f64 (fextend FR32X:$src)), (COPY_TO_REGCLASS (VCVTSS2SDZrr (COPY_TO_REGCLASS FR32X:$src, VR128X), (COPY_TO_REGCLASS FR32X:$src, VR128X)), VR128X)>, Requires<[HasAVX512]>; def : Pat<(f64 (fextend (loadf32 addr:$src))), (COPY_TO_REGCLASS (VCVTSS2SDZrm (v4f32 (IMPLICIT_DEF)), addr:$src), VR128X)>, Requires<[HasAVX512]>; def : Pat<(f64 (extloadf32 addr:$src)), (COPY_TO_REGCLASS (VCVTSS2SDZrm (v4f32 (IMPLICIT_DEF)), addr:$src), VR128X)>, Requires<[HasAVX512, OptForSize]>; def : Pat<(f64 (extloadf32 addr:$src)), (COPY_TO_REGCLASS (VCVTSS2SDZrr (v4f32 (IMPLICIT_DEF)), (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)), VR128X)>, Requires<[HasAVX512, OptForSpeed]>; def : Pat<(f32 (fround FR64X:$src)), (COPY_TO_REGCLASS (VCVTSD2SSZrr (COPY_TO_REGCLASS FR64X:$src, VR128X), (COPY_TO_REGCLASS FR64X:$src, VR128X)), VR128X)>, Requires<[HasAVX512]>; //===----------------------------------------------------------------------===// // AVX-512 Vector convert from signed/unsigned integer to float/double // and from float/double to signed/unsigned integer //===----------------------------------------------------------------------===// multiclass avx512_vcvt_fp opc, string OpcodeStr, X86VectorVTInfo _, X86VectorVTInfo _Src, SDNode OpNode, string Broadcast = _.BroadcastStr, string Alias = ""> { defm rr : AVX512_maskable, EVEX; defm rm : AVX512_maskable, EVEX; defm rmb : AVX512_maskable, EVEX, EVEX_B; } // Coversion with SAE - suppress all exceptions multiclass avx512_vcvt_fp_sae opc, string OpcodeStr, X86VectorVTInfo _, X86VectorVTInfo _Src, SDNode OpNodeRnd> { defm rrb : AVX512_maskable, EVEX, EVEX_B; } // Conversion with rounding control (RC) multiclass avx512_vcvt_fp_rc opc, string OpcodeStr, X86VectorVTInfo _, X86VectorVTInfo _Src, SDNode OpNodeRnd> { defm rrb : AVX512_maskable, EVEX, EVEX_B, EVEX_RC; } // Extend Float to Double multiclass avx512_cvtps2pd opc, string OpcodeStr> { let Predicates = [HasAVX512] in { defm Z : avx512_vcvt_fp, avx512_vcvt_fp_sae, EVEX_V512; } let Predicates = [HasVLX] in { defm Z128 : avx512_vcvt_fp, EVEX_V128; defm Z256 : avx512_vcvt_fp, EVEX_V256; } } // Truncate Double to Float multiclass avx512_cvtpd2ps opc, string OpcodeStr> { let Predicates = [HasAVX512] in { defm Z : avx512_vcvt_fp, avx512_vcvt_fp_rc, EVEX_V512; } let Predicates = [HasVLX] in { defm Z128 : avx512_vcvt_fp, EVEX_V128; defm Z256 : avx512_vcvt_fp, EVEX_V256; } } defm VCVTPD2PS : avx512_cvtpd2ps<0x5A, "vcvtpd2ps">, VEX_W, PD, EVEX_CD8<64, CD8VF>; defm VCVTPS2PD : avx512_cvtps2pd<0x5A, "vcvtps2pd">, PS, EVEX_CD8<32, CD8VH>; def : Pat<(v8f64 (extloadv8f32 addr:$src)), (VCVTPS2PDZrm addr:$src)>; let Predicates = [HasVLX] in { def : Pat<(v4f64 (extloadv4f32 addr:$src)), (VCVTPS2PDZ256rm addr:$src)>; } // Convert Signed/Unsigned Doubleword to Double multiclass avx512_cvtdq2pd opc, string OpcodeStr, SDNode OpNode, SDNode OpNode128> { // No rounding in this op let Predicates = [HasAVX512] in defm Z : avx512_vcvt_fp, EVEX_V512; let Predicates = [HasVLX] in { defm Z128 : avx512_vcvt_fp, EVEX_V128; defm Z256 : avx512_vcvt_fp, EVEX_V256; } } // Convert Signed/Unsigned Doubleword to Float multiclass avx512_cvtdq2ps opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd> { let Predicates = [HasAVX512] in defm Z : avx512_vcvt_fp, avx512_vcvt_fp_rc, EVEX_V512; let Predicates = [HasVLX] in { defm Z128 : avx512_vcvt_fp, EVEX_V128; defm Z256 : avx512_vcvt_fp, EVEX_V256; } } // Convert Float to Signed/Unsigned Doubleword with truncation multiclass avx512_cvttps2dq opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd> { let Predicates = [HasAVX512] in { defm Z : avx512_vcvt_fp, avx512_vcvt_fp_sae, EVEX_V512; } let Predicates = [HasVLX] in { defm Z128 : avx512_vcvt_fp, EVEX_V128; defm Z256 : avx512_vcvt_fp, EVEX_V256; } } // Convert Float to Signed/Unsigned Doubleword multiclass avx512_cvtps2dq opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd> { let Predicates = [HasAVX512] in { defm Z : avx512_vcvt_fp, avx512_vcvt_fp_rc, EVEX_V512; } let Predicates = [HasVLX] in { defm Z128 : avx512_vcvt_fp, EVEX_V128; defm Z256 : avx512_vcvt_fp, EVEX_V256; } } // Convert Double to Signed/Unsigned Doubleword with truncation multiclass avx512_cvttpd2dq opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd> { let Predicates = [HasAVX512] in { defm Z : avx512_vcvt_fp, avx512_vcvt_fp_sae, EVEX_V512; } let Predicates = [HasVLX] in { // we need "x"/"y" suffixes in order to distinguish between 128 and 256 // memory forms of these instructions in Asm Parcer. They have the same // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly // due to the same reason. defm Z128 : avx512_vcvt_fp, EVEX_V128; defm Z256 : avx512_vcvt_fp, EVEX_V256; } } // Convert Double to Signed/Unsigned Doubleword multiclass avx512_cvtpd2dq opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd> { let Predicates = [HasAVX512] in { defm Z : avx512_vcvt_fp, avx512_vcvt_fp_rc, EVEX_V512; } let Predicates = [HasVLX] in { // we need "x"/"y" suffixes in order to distinguish between 128 and 256 // memory forms of these instructions in Asm Parcer. They have the same // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly // due to the same reason. defm Z128 : avx512_vcvt_fp, EVEX_V128; defm Z256 : avx512_vcvt_fp, EVEX_V256; } } // Convert Double to Signed/Unsigned Quardword multiclass avx512_cvtpd2qq opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd> { let Predicates = [HasDQI] in { defm Z : avx512_vcvt_fp, avx512_vcvt_fp_rc, EVEX_V512; } let Predicates = [HasDQI, HasVLX] in { defm Z128 : avx512_vcvt_fp, EVEX_V128; defm Z256 : avx512_vcvt_fp, EVEX_V256; } } // Convert Double to Signed/Unsigned Quardword with truncation multiclass avx512_cvttpd2qq opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd> { let Predicates = [HasDQI] in { defm Z : avx512_vcvt_fp, avx512_vcvt_fp_sae, EVEX_V512; } let Predicates = [HasDQI, HasVLX] in { defm Z128 : avx512_vcvt_fp, EVEX_V128; defm Z256 : avx512_vcvt_fp, EVEX_V256; } } // Convert Signed/Unsigned Quardword to Double multiclass avx512_cvtqq2pd opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd> { let Predicates = [HasDQI] in { defm Z : avx512_vcvt_fp, avx512_vcvt_fp_rc, EVEX_V512; } let Predicates = [HasDQI, HasVLX] in { defm Z128 : avx512_vcvt_fp, EVEX_V128; defm Z256 : avx512_vcvt_fp, EVEX_V256; } } // Convert Float to Signed/Unsigned Quardword multiclass avx512_cvtps2qq opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd> { let Predicates = [HasDQI] in { defm Z : avx512_vcvt_fp, avx512_vcvt_fp_rc, EVEX_V512; } let Predicates = [HasDQI, HasVLX] in { // Explicitly specified broadcast string, since we take only 2 elements // from v4f32x_info source defm Z128 : avx512_vcvt_fp, EVEX_V128; defm Z256 : avx512_vcvt_fp, EVEX_V256; } } // Convert Float to Signed/Unsigned Quardword with truncation multiclass avx512_cvttps2qq opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd> { let Predicates = [HasDQI] in { defm Z : avx512_vcvt_fp, avx512_vcvt_fp_sae, EVEX_V512; } let Predicates = [HasDQI, HasVLX] in { // Explicitly specified broadcast string, since we take only 2 elements // from v4f32x_info source defm Z128 : avx512_vcvt_fp, EVEX_V128; defm Z256 : avx512_vcvt_fp, EVEX_V256; } } // Convert Signed/Unsigned Quardword to Float multiclass avx512_cvtqq2ps opc, string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd> { let Predicates = [HasDQI] in { defm Z : avx512_vcvt_fp, avx512_vcvt_fp_rc, EVEX_V512; } let Predicates = [HasDQI, HasVLX] in { // we need "x"/"y" suffixes in order to distinguish between 128 and 256 // memory forms of these instructions in Asm Parcer. They have the same // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly // due to the same reason. defm Z128 : avx512_vcvt_fp, EVEX_V128; defm Z256 : avx512_vcvt_fp, EVEX_V256; } } defm VCVTDQ2PD : avx512_cvtdq2pd<0xE6, "vcvtdq2pd", sint_to_fp, X86cvtdq2pd>, XS, EVEX_CD8<32, CD8VH>; defm VCVTDQ2PS : avx512_cvtdq2ps<0x5B, "vcvtdq2ps", sint_to_fp, X86VSintToFpRnd>, PS, EVEX_CD8<32, CD8VF>; defm VCVTTPS2DQ : avx512_cvttps2dq<0x5B, "vcvttps2dq", fp_to_sint, X86VFpToSintRnd>, XS, EVEX_CD8<32, CD8VF>; defm VCVTTPD2DQ : avx512_cvttpd2dq<0xE6, "vcvttpd2dq", fp_to_sint, X86VFpToSintRnd>, PD, VEX_W, EVEX_CD8<64, CD8VF>; defm VCVTTPS2UDQ : avx512_cvttps2dq<0x78, "vcvttps2udq", fp_to_uint, X86VFpToUintRnd>, PS, EVEX_CD8<32, CD8VF>; defm VCVTTPD2UDQ : avx512_cvttpd2dq<0x78, "vcvttpd2udq", fp_to_uint, X86VFpToUintRnd>, PS, VEX_W, EVEX_CD8<64, CD8VF>; defm VCVTUDQ2PD : avx512_cvtdq2pd<0x7A, "vcvtudq2pd", uint_to_fp, X86cvtudq2pd>, XS, EVEX_CD8<32, CD8VH>; defm VCVTUDQ2PS : avx512_cvtdq2ps<0x7A, "vcvtudq2ps", uint_to_fp, X86VUintToFpRnd>, XD, EVEX_CD8<32, CD8VF>; defm VCVTPS2DQ : avx512_cvtps2dq<0x5B, "vcvtps2dq", X86cvtps2Int, X86cvtps2IntRnd>, PD, EVEX_CD8<32, CD8VF>; defm VCVTPD2DQ : avx512_cvtpd2dq<0xE6, "vcvtpd2dq", X86cvtpd2Int, X86cvtpd2IntRnd>, XD, VEX_W, EVEX_CD8<64, CD8VF>; defm VCVTPS2UDQ : avx512_cvtps2dq<0x79, "vcvtps2udq", X86cvtps2UInt, X86cvtps2UIntRnd>, PS, EVEX_CD8<32, CD8VF>; defm VCVTPD2UDQ : avx512_cvtpd2dq<0x79, "vcvtpd2udq", X86cvtpd2UInt, X86cvtpd2UIntRnd>, VEX_W, PS, EVEX_CD8<64, CD8VF>; defm VCVTPD2QQ : avx512_cvtpd2qq<0x7B, "vcvtpd2qq", X86cvtpd2Int, X86cvtpd2IntRnd>, VEX_W, PD, EVEX_CD8<64, CD8VF>; defm VCVTPS2QQ : avx512_cvtps2qq<0x7B, "vcvtps2qq", X86cvtps2Int, X86cvtps2IntRnd>, PD, EVEX_CD8<32, CD8VH>; defm VCVTPD2UQQ : avx512_cvtpd2qq<0x79, "vcvtpd2uqq", X86cvtpd2UInt, X86cvtpd2UIntRnd>, VEX_W, PD, EVEX_CD8<64, CD8VF>; defm VCVTPS2UQQ : avx512_cvtps2qq<0x79, "vcvtps2uqq", X86cvtps2UInt, X86cvtps2UIntRnd>, PD, EVEX_CD8<32, CD8VH>; defm VCVTTPD2QQ : avx512_cvttpd2qq<0x7A, "vcvttpd2qq", fp_to_sint, X86VFpToSlongRnd>, VEX_W, PD, EVEX_CD8<64, CD8VF>; defm VCVTTPS2QQ : avx512_cvttps2qq<0x7A, "vcvttps2qq", fp_to_sint, X86VFpToSlongRnd>, PD, EVEX_CD8<32, CD8VH>; defm VCVTTPD2UQQ : avx512_cvttpd2qq<0x78, "vcvttpd2uqq", fp_to_uint, X86VFpToUlongRnd>, VEX_W, PD, EVEX_CD8<64, CD8VF>; defm VCVTTPS2UQQ : avx512_cvttps2qq<0x78, "vcvttps2uqq", fp_to_uint, X86VFpToUlongRnd>, PD, EVEX_CD8<32, CD8VH>; defm VCVTQQ2PD : avx512_cvtqq2pd<0xE6, "vcvtqq2pd", sint_to_fp, X86VSlongToFpRnd>, VEX_W, XS, EVEX_CD8<64, CD8VF>; defm VCVTUQQ2PD : avx512_cvtqq2pd<0x7A, "vcvtuqq2pd", uint_to_fp, X86VUlongToFpRnd>, VEX_W, XS, EVEX_CD8<64, CD8VF>; defm VCVTQQ2PS : avx512_cvtqq2ps<0x5B, "vcvtqq2ps", sint_to_fp, X86VSlongToFpRnd>, VEX_W, PS, EVEX_CD8<64, CD8VF>; defm VCVTUQQ2PS : avx512_cvtqq2ps<0x7A, "vcvtuqq2ps", uint_to_fp, X86VUlongToFpRnd>, VEX_W, XD, EVEX_CD8<64, CD8VF>; let Predicates = [HasAVX512, NoVLX] in { def : Pat<(v8i32 (fp_to_uint (v8f32 VR256X:$src1))), (EXTRACT_SUBREG (v16i32 (VCVTTPS2UDQZrr (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>; def : Pat<(v4i32 (fp_to_uint (v4f32 VR128X:$src1))), (EXTRACT_SUBREG (v16i32 (VCVTTPS2UDQZrr (v16f32 (SUBREG_TO_REG (i32 0), VR128X:$src1, sub_xmm)))), sub_xmm)>; def : Pat<(v8f32 (uint_to_fp (v8i32 VR256X:$src1))), (EXTRACT_SUBREG (v16f32 (VCVTUDQ2PSZrr (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>; def : Pat<(v4f32 (uint_to_fp (v4i32 VR128X:$src1))), (EXTRACT_SUBREG (v16f32 (VCVTUDQ2PSZrr (v16i32 (SUBREG_TO_REG (i32 0), VR128X:$src1, sub_xmm)))), sub_xmm)>; def : Pat<(v4f64 (uint_to_fp (v4i32 VR128X:$src1))), (EXTRACT_SUBREG (v8f64 (VCVTUDQ2PDZrr (v8i32 (SUBREG_TO_REG (i32 0), VR128X:$src1, sub_xmm)))), sub_ymm)>; } let Predicates = [HasAVX512] in { def : Pat<(v8f32 (fround (loadv8f64 addr:$src))), (VCVTPD2PSZrm addr:$src)>; def : Pat<(v8f64 (extloadv8f32 addr:$src)), (VCVTPS2PDZrm addr:$src)>; } //===----------------------------------------------------------------------===// // Half precision conversion instructions //===----------------------------------------------------------------------===// multiclass avx512_cvtph2ps { defm rr : AVX512_maskable<0x13, MRMSrcReg, _dest ,(outs _dest.RC:$dst), (ins _src.RC:$src), "vcvtph2ps", "$src", "$src", (X86cvtph2ps (_src.VT _src.RC:$src), (i32 FROUND_CURRENT))>, T8PD; let hasSideEffects = 0, mayLoad = 1 in { defm rm : AVX512_maskable<0x13, MRMSrcMem, _dest, (outs _dest.RC:$dst), (ins x86memop:$src), "vcvtph2ps", "$src", "$src", (X86cvtph2ps (_src.VT (bitconvert (ld_frag addr:$src))), (i32 FROUND_CURRENT))>, T8PD; } } multiclass avx512_cvtph2ps_sae { defm rb : AVX512_maskable<0x13, MRMSrcReg, _dest ,(outs _dest.RC:$dst), (ins _src.RC:$src), "vcvtph2ps", "{sae}, $src", "$src, {sae}", (X86cvtph2ps (_src.VT _src.RC:$src), (i32 FROUND_NO_EXC))>, T8PD, EVEX_B; } let Predicates = [HasAVX512] in { defm VCVTPH2PSZ : avx512_cvtph2ps, avx512_cvtph2ps_sae, EVEX, EVEX_V512, EVEX_CD8<32, CD8VH>; let Predicates = [HasVLX] in { defm VCVTPH2PSZ256 : avx512_cvtph2ps,EVEX, EVEX_V256, EVEX_CD8<32, CD8VH>; defm VCVTPH2PSZ128 : avx512_cvtph2ps, EVEX, EVEX_V128, EVEX_CD8<32, CD8VH>; } } multiclass avx512_cvtps2ph { defm rr : AVX512_maskable<0x1D, MRMDestReg, _dest ,(outs _dest.RC:$dst), (ins _src.RC:$src1, i32u8imm:$src2), "vcvtps2ph", "$src2, $src1", "$src1, $src2", (X86cvtps2ph (_src.VT _src.RC:$src1), (i32 imm:$src2), (i32 FROUND_CURRENT))>, AVX512AIi8Base; let hasSideEffects = 0, mayStore = 1 in { def mr : AVX512AIi8<0x1D, MRMDestMem, (outs), (ins x86memop:$dst, _src.RC:$src1, i32u8imm:$src2), "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(store (_dest.VT (X86cvtps2ph (_src.VT _src.RC:$src1), (i32 imm:$src2), (i32 FROUND_CURRENT) )), addr:$dst)]>; def mrk : AVX512AIi8<0x1D, MRMDestMem, (outs), (ins x86memop:$dst, _dest.KRCWM:$mask, _src.RC:$src1, i32u8imm:$src2), "vcvtps2ph\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}", []>, EVEX_K; } } multiclass avx512_cvtps2ph_sae { defm rb : AVX512_maskable<0x1D, MRMDestReg, _dest ,(outs _dest.RC:$dst), (ins _src.RC:$src1, i32u8imm:$src2), "vcvtps2ph", "$src2, {sae}, $src1", "$src1, $src2, {sae}", (X86cvtps2ph (_src.VT _src.RC:$src1), (i32 imm:$src2), (i32 FROUND_NO_EXC))>, EVEX_B, AVX512AIi8Base; } let Predicates = [HasAVX512] in { defm VCVTPS2PHZ : avx512_cvtps2ph, avx512_cvtps2ph_sae, EVEX, EVEX_V512, EVEX_CD8<32, CD8VH>; let Predicates = [HasVLX] in { defm VCVTPS2PHZ256 : avx512_cvtps2ph, EVEX, EVEX_V256, EVEX_CD8<32, CD8VH>; defm VCVTPS2PHZ128 : avx512_cvtps2ph, EVEX, EVEX_V128, EVEX_CD8<32, CD8VH>; } } // Unordered/Ordered scalar fp compare with Sea and set EFLAGS multiclass avx512_ord_cmp_sae opc, X86VectorVTInfo _, SDNode OpNode, string OpcodeStr> { def rb: AVX512, EVEX, EVEX_B, VEX_LIG, EVEX_V128, Sched<[WriteFAdd]>; } let Defs = [EFLAGS], Predicates = [HasAVX512] in { defm VUCOMISSZ : avx512_ord_cmp_sae<0x2E, v4f32x_info, X86ucomiSae, "vucomiss">, AVX512PSIi8Base, EVEX_CD8<32, CD8VT1>; defm VUCOMISDZ : avx512_ord_cmp_sae<0x2E, v2f64x_info, X86ucomiSae, "vucomisd">, AVX512PDIi8Base, VEX_W, EVEX_CD8<64, CD8VT1>; defm VCOMISSZ : avx512_ord_cmp_sae<0x2F, v4f32x_info, X86comiSae, "vcomiss">, AVX512PSIi8Base, EVEX_CD8<32, CD8VT1>; defm VCOMISDZ : avx512_ord_cmp_sae<0x2F, v2f64x_info, X86comiSae, "vcomisd">, AVX512PDIi8Base, VEX_W, EVEX_CD8<64, CD8VT1>; } let Defs = [EFLAGS], Predicates = [HasAVX512] in { defm VUCOMISSZ : sse12_ord_cmp<0x2E, FR32X, X86cmp, f32, f32mem, loadf32, "ucomiss">, PS, EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>; defm VUCOMISDZ : sse12_ord_cmp<0x2E, FR64X, X86cmp, f64, f64mem, loadf64, "ucomisd">, PD, EVEX, VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>; let Pattern = [] in { defm VCOMISSZ : sse12_ord_cmp<0x2F, FR32X, undef, f32, f32mem, loadf32, "comiss">, PS, EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>; defm VCOMISDZ : sse12_ord_cmp<0x2F, FR64X, undef, f64, f64mem, loadf64, "comisd">, PD, EVEX, VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>; } let isCodeGenOnly = 1 in { defm Int_VUCOMISSZ : sse12_ord_cmp<0x2E, VR128X, X86ucomi, v4f32, f128mem, load, "ucomiss">, PS, EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>; defm Int_VUCOMISDZ : sse12_ord_cmp<0x2E, VR128X, X86ucomi, v2f64, f128mem, load, "ucomisd">, PD, EVEX, VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>; defm Int_VCOMISSZ : sse12_ord_cmp<0x2F, VR128X, X86comi, v4f32, f128mem, load, "comiss">, PS, EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>; defm Int_VCOMISDZ : sse12_ord_cmp<0x2F, VR128X, X86comi, v2f64, f128mem, load, "comisd">, PD, EVEX, VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>; } } /// avx512_fp14_s rcp14ss, rcp14sd, rsqrt14ss, rsqrt14sd multiclass avx512_fp14_s opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { let hasSideEffects = 0, AddedComplexity = 20 , Predicates = [HasAVX512] in { defm rr : AVX512_maskable_scalar, EVEX_4V; let mayLoad = 1 in { defm rm : AVX512_maskable_scalar, EVEX_4V; } } } defm VRCP14SS : avx512_fp14_s<0x4D, "vrcp14ss", X86frcp14s, f32x_info>, EVEX_CD8<32, CD8VT1>, T8PD; defm VRCP14SD : avx512_fp14_s<0x4D, "vrcp14sd", X86frcp14s, f64x_info>, VEX_W, EVEX_CD8<64, CD8VT1>, T8PD; defm VRSQRT14SS : avx512_fp14_s<0x4F, "vrsqrt14ss", X86frsqrt14s, f32x_info>, EVEX_CD8<32, CD8VT1>, T8PD; defm VRSQRT14SD : avx512_fp14_s<0x4F, "vrsqrt14sd", X86frsqrt14s, f64x_info>, VEX_W, EVEX_CD8<64, CD8VT1>, T8PD; /// avx512_fp14_p rcp14ps, rcp14pd, rsqrt14ps, rsqrt14pd multiclass avx512_fp14_p opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { defm r: AVX512_maskable, EVEX, T8PD; let mayLoad = 1 in { defm m: AVX512_maskable, EVEX, T8PD; defm mb: AVX512_maskable, EVEX, T8PD, EVEX_B; } } multiclass avx512_fp14_p_vl_all opc, string OpcodeStr, SDNode OpNode> { defm PSZ : avx512_fp14_p, EVEX_V512, EVEX_CD8<32, CD8VF>; defm PDZ : avx512_fp14_p, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; // Define only if AVX512VL feature is present. let Predicates = [HasVLX] in { defm PSZ128 : avx512_fp14_p, EVEX_V128, EVEX_CD8<32, CD8VF>; defm PSZ256 : avx512_fp14_p, EVEX_V256, EVEX_CD8<32, CD8VF>; defm PDZ128 : avx512_fp14_p, EVEX_V128, VEX_W, EVEX_CD8<64, CD8VF>; defm PDZ256 : avx512_fp14_p, EVEX_V256, VEX_W, EVEX_CD8<64, CD8VF>; } } defm VRSQRT14 : avx512_fp14_p_vl_all<0x4E, "vrsqrt14", X86frsqrt>; defm VRCP14 : avx512_fp14_p_vl_all<0x4C, "vrcp14", X86frcp>; /// avx512_fp28_s rcp28ss, rcp28sd, rsqrt28ss, rsqrt28sd multiclass avx512_fp28_s opc, string OpcodeStr,X86VectorVTInfo _, SDNode OpNode> { defm r : AVX512_maskable_scalar; defm rb : AVX512_maskable_scalar, EVEX_B; defm m : AVX512_maskable_scalar; } multiclass avx512_eri_s opc, string OpcodeStr, SDNode OpNode> { defm SS : avx512_fp28_s, EVEX_CD8<32, CD8VT1>; defm SD : avx512_fp28_s, EVEX_CD8<64, CD8VT1>, VEX_W; } let hasSideEffects = 0, Predicates = [HasERI] in { defm VRCP28 : avx512_eri_s<0xCB, "vrcp28", X86rcp28s>, T8PD, EVEX_4V; defm VRSQRT28 : avx512_eri_s<0xCD, "vrsqrt28", X86rsqrt28s>, T8PD, EVEX_4V; } defm VGETEXP : avx512_eri_s<0x43, "vgetexp", X86fgetexpRnds>, T8PD, EVEX_4V; /// avx512_fp28_p rcp28ps, rcp28pd, rsqrt28ps, rsqrt28pd multiclass avx512_fp28_p opc, string OpcodeStr, X86VectorVTInfo _, SDNode OpNode> { defm r : AVX512_maskable; defm m : AVX512_maskable; defm mb : AVX512_maskable, EVEX_B; } multiclass avx512_fp28_p_round opc, string OpcodeStr, X86VectorVTInfo _, SDNode OpNode> { defm rb : AVX512_maskable, EVEX_B; } multiclass avx512_eri opc, string OpcodeStr, SDNode OpNode> { defm PS : avx512_fp28_p, avx512_fp28_p_round, T8PD, EVEX_V512, EVEX_CD8<32, CD8VF>; defm PD : avx512_fp28_p, avx512_fp28_p_round, T8PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; } multiclass avx512_fp_unaryop_packed opc, string OpcodeStr, SDNode OpNode> { // Define only if AVX512VL feature is present. let Predicates = [HasVLX] in { defm PSZ128 : avx512_fp28_p, EVEX_V128, T8PD, EVEX_CD8<32, CD8VF>; defm PSZ256 : avx512_fp28_p, EVEX_V256, T8PD, EVEX_CD8<32, CD8VF>; defm PDZ128 : avx512_fp28_p, EVEX_V128, VEX_W, T8PD, EVEX_CD8<64, CD8VF>; defm PDZ256 : avx512_fp28_p, EVEX_V256, VEX_W, T8PD, EVEX_CD8<64, CD8VF>; } } let Predicates = [HasERI], hasSideEffects = 0 in { defm VRSQRT28 : avx512_eri<0xCC, "vrsqrt28", X86rsqrt28>, EVEX; defm VRCP28 : avx512_eri<0xCA, "vrcp28", X86rcp28>, EVEX; defm VEXP2 : avx512_eri<0xC8, "vexp2", X86exp2>, EVEX; } defm VGETEXP : avx512_eri<0x42, "vgetexp", X86fgetexpRnd>, avx512_fp_unaryop_packed<0x42, "vgetexp", X86fgetexpRnd> , EVEX; multiclass avx512_sqrt_packed_round opc, string OpcodeStr, SDNode OpNodeRnd, X86VectorVTInfo _>{ defm rb: AVX512_maskable, EVEX, EVEX_B, EVEX_RC; } multiclass avx512_sqrt_packed opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _>{ defm r: AVX512_maskable, EVEX; let mayLoad = 1 in { defm m: AVX512_maskable, EVEX; defm mb: AVX512_maskable, EVEX, EVEX_B; } } multiclass avx512_sqrt_packed_all opc, string OpcodeStr, SDNode OpNode> { defm PSZ : avx512_sqrt_packed, EVEX_V512, PS, EVEX_CD8<32, CD8VF>; defm PDZ : avx512_sqrt_packed, EVEX_V512, VEX_W, PD, EVEX_CD8<64, CD8VF>; // Define only if AVX512VL feature is present. let Predicates = [HasVLX] in { defm PSZ128 : avx512_sqrt_packed, EVEX_V128, PS, EVEX_CD8<32, CD8VF>; defm PSZ256 : avx512_sqrt_packed, EVEX_V256, PS, EVEX_CD8<32, CD8VF>; defm PDZ128 : avx512_sqrt_packed, EVEX_V128, VEX_W, PD, EVEX_CD8<64, CD8VF>; defm PDZ256 : avx512_sqrt_packed, EVEX_V256, VEX_W, PD, EVEX_CD8<64, CD8VF>; } } multiclass avx512_sqrt_packed_all_round opc, string OpcodeStr, SDNode OpNodeRnd> { defm PSZ : avx512_sqrt_packed_round, EVEX_V512, PS, EVEX_CD8<32, CD8VF>; defm PDZ : avx512_sqrt_packed_round, EVEX_V512, VEX_W, PD, EVEX_CD8<64, CD8VF>; } multiclass avx512_sqrt_scalar opc, string OpcodeStr,X86VectorVTInfo _, string SUFF, SDNode OpNode, SDNode OpNodeRnd> { defm r_Int : AVX512_maskable_scalar; let mayLoad = 1 in defm m_Int : AVX512_maskable_scalar; defm rb_Int : AVX512_maskable_scalar, EVEX_B, EVEX_RC; let isCodeGenOnly = 1 in { def r : I; let mayLoad = 1 in def m : I; } def : Pat<(_.EltVT (OpNode _.FRC:$src)), (!cast(NAME#SUFF#Zr) (_.EltVT (IMPLICIT_DEF)), _.FRC:$src)>; def : Pat<(_.EltVT (OpNode (load addr:$src))), (!cast(NAME#SUFF#Zm) (_.EltVT (IMPLICIT_DEF)), addr:$src)>, Requires<[OptForSize]>; } multiclass avx512_sqrt_scalar_all opc, string OpcodeStr> { defm SSZ : avx512_sqrt_scalar, EVEX_CD8<32, CD8VT1>, EVEX_4V, XS; defm SDZ : avx512_sqrt_scalar, EVEX_CD8<64, CD8VT1>, EVEX_4V, XD, VEX_W; } defm VSQRT : avx512_sqrt_packed_all<0x51, "vsqrt", fsqrt>, avx512_sqrt_packed_all_round<0x51, "vsqrt", X86fsqrtRnd>; defm VSQRT : avx512_sqrt_scalar_all<0x51, "vsqrt">, VEX_LIG; let Predicates = [HasAVX512] in { def : Pat<(f32 (X86frsqrt FR32X:$src)), (COPY_TO_REGCLASS (VRSQRT14SSrr (v4f32 (IMPLICIT_DEF)), (COPY_TO_REGCLASS FR32X:$src, VR128X)), VR128X)>; def : Pat<(f32 (X86frsqrt (load addr:$src))), (COPY_TO_REGCLASS (VRSQRT14SSrm (v4f32 (IMPLICIT_DEF)), addr:$src), VR128X)>, Requires<[OptForSize]>; def : Pat<(f32 (X86frcp FR32X:$src)), (COPY_TO_REGCLASS (VRCP14SSrr (v4f32 (IMPLICIT_DEF)), (COPY_TO_REGCLASS FR32X:$src, VR128X)), VR128X )>; def : Pat<(f32 (X86frcp (load addr:$src))), (COPY_TO_REGCLASS (VRCP14SSrm (v4f32 (IMPLICIT_DEF)), addr:$src), VR128X)>, Requires<[OptForSize]>; } multiclass avx512_rndscale_scalar opc, string OpcodeStr, X86VectorVTInfo _> { let ExeDomain = _.ExeDomain in { defm r : AVX512_maskable_scalar; defm rb : AVX512_maskable_scalar, EVEX_B; let mayLoad = 1 in defm m : AVX512_maskable_scalar; } let Predicates = [HasAVX512] in { def : Pat<(ffloor _.FRC:$src), (COPY_TO_REGCLASS (_.VT (!cast(NAME##r) (_.VT (IMPLICIT_DEF)), (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0x1))), _.FRC)>; def : Pat<(fceil _.FRC:$src), (COPY_TO_REGCLASS (_.VT (!cast(NAME##r) (_.VT (IMPLICIT_DEF)), (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0x2))), _.FRC)>; def : Pat<(ftrunc _.FRC:$src), (COPY_TO_REGCLASS (_.VT (!cast(NAME##r) (_.VT (IMPLICIT_DEF)), (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0x3))), _.FRC)>; def : Pat<(frint _.FRC:$src), (COPY_TO_REGCLASS (_.VT (!cast(NAME##r) (_.VT (IMPLICIT_DEF)), (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0x4))), _.FRC)>; def : Pat<(fnearbyint _.FRC:$src), (COPY_TO_REGCLASS (_.VT (!cast(NAME##r) (_.VT (IMPLICIT_DEF)), (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0xc))), _.FRC)>; def : Pat<(ffloor (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS (_.VT (!cast(NAME##m) (_.VT (IMPLICIT_DEF)), addr:$src, (i32 0x1))), _.FRC)>; def : Pat<(fceil (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS (_.VT (!cast(NAME##m) (_.VT (IMPLICIT_DEF)), addr:$src, (i32 0x2))), _.FRC)>; def : Pat<(ftrunc (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS (_.VT (!cast(NAME##m) (_.VT (IMPLICIT_DEF)), addr:$src, (i32 0x3))), _.FRC)>; def : Pat<(frint (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS (_.VT (!cast(NAME##m) (_.VT (IMPLICIT_DEF)), addr:$src, (i32 0x4))), _.FRC)>; def : Pat<(fnearbyint (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS (_.VT (!cast(NAME##m) (_.VT (IMPLICIT_DEF)), addr:$src, (i32 0xc))), _.FRC)>; } } defm VRNDSCALESS : avx512_rndscale_scalar<0x0A, "vrndscaless", f32x_info>, AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VT1>; defm VRNDSCALESD : avx512_rndscale_scalar<0x0B, "vrndscalesd", f64x_info>, VEX_W, AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VT1>; //------------------------------------------------- // Integer truncate and extend operations //------------------------------------------------- multiclass avx512_trunc_common opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo SrcInfo, X86VectorVTInfo DestInfo, X86MemOperand x86memop> { defm rr : AVX512_maskable, EVEX, T8XS; // for intrinsic patter match def : Pat<(DestInfo.VT (X86select DestInfo.KRCWM:$mask, (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1))), undef)), (!cast(NAME#SrcInfo.ZSuffix##rrkz) DestInfo.KRCWM:$mask , SrcInfo.RC:$src1)>; def : Pat<(DestInfo.VT (X86select DestInfo.KRCWM:$mask, (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1))), DestInfo.ImmAllZerosV)), (!cast(NAME#SrcInfo.ZSuffix##rrkz) DestInfo.KRCWM:$mask , SrcInfo.RC:$src1)>; def : Pat<(DestInfo.VT (X86select DestInfo.KRCWM:$mask, (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1))), DestInfo.RC:$src0)), (!cast(NAME#SrcInfo.ZSuffix##rrk) DestInfo.RC:$src0, DestInfo.KRCWM:$mask , SrcInfo.RC:$src1)>; let mayStore = 1 in { def mr : AVX512XS8I, EVEX; def mrk : AVX512XS8I, EVEX, EVEX_K; }//mayStore = 1 } multiclass avx512_trunc_mr_lowering { def : Pat<(truncFrag (SrcInfo.VT SrcInfo.RC:$src), addr:$dst), (!cast(NAME#SrcInfo.ZSuffix##mr) addr:$dst, SrcInfo.RC:$src)>; def : Pat<(mtruncFrag addr:$dst, SrcInfo.KRCWM:$mask, (SrcInfo.VT SrcInfo.RC:$src)), (!cast(NAME#SrcInfo.ZSuffix##mrk) addr:$dst, SrcInfo.KRCWM:$mask, SrcInfo.RC:$src)>; } multiclass avx512_trunc_sat_mr_lowering { def: Pat<(!cast("int_x86_avx512_mask_pmov"#sat#"_"#SrcInfo.Suffix# DestInfo.Suffix#"_mem_"#SrcInfo.Size) addr:$ptr, (SrcInfo.VT SrcInfo.RC:$src), SrcInfo.MRC:$mask), (!cast(NAME#SrcInfo.ZSuffix##mrk) addr:$ptr, (COPY_TO_REGCLASS SrcInfo.MRC:$mask, SrcInfo.KRCWM), (SrcInfo.VT SrcInfo.RC:$src))>; def: Pat<(!cast("int_x86_avx512_mask_pmov"#sat#"_"#SrcInfo.Suffix# DestInfo.Suffix#"_mem_"#SrcInfo.Size) addr:$ptr, (SrcInfo.VT SrcInfo.RC:$src), -1), (!cast(NAME#SrcInfo.ZSuffix##mr) addr:$ptr, (SrcInfo.VT SrcInfo.RC:$src))>; } multiclass avx512_trunc opc, string OpcodeStr, SDNode OpNode, AVX512VLVectorVTInfo VTSrcInfo, X86VectorVTInfo DestInfoZ128, X86VectorVTInfo DestInfoZ256, X86VectorVTInfo DestInfoZ, X86MemOperand x86memopZ128, X86MemOperand x86memopZ256, X86MemOperand x86memopZ, PatFrag truncFrag, PatFrag mtruncFrag, Predicate prd = HasAVX512>{ let Predicates = [HasVLX, prd] in { defm Z128: avx512_trunc_common, avx512_trunc_mr_lowering, EVEX_V128; defm Z256: avx512_trunc_common, avx512_trunc_mr_lowering, EVEX_V256; } let Predicates = [prd] in defm Z: avx512_trunc_common, avx512_trunc_mr_lowering, EVEX_V512; } multiclass avx512_trunc_sat opc, string OpcodeStr, SDNode OpNode, AVX512VLVectorVTInfo VTSrcInfo, X86VectorVTInfo DestInfoZ128, X86VectorVTInfo DestInfoZ256, X86VectorVTInfo DestInfoZ, X86MemOperand x86memopZ128, X86MemOperand x86memopZ256, X86MemOperand x86memopZ, string sat, Predicate prd = HasAVX512>{ let Predicates = [HasVLX, prd] in { defm Z128: avx512_trunc_common, avx512_trunc_sat_mr_lowering, EVEX_V128; defm Z256: avx512_trunc_common, avx512_trunc_sat_mr_lowering, EVEX_V256; } let Predicates = [prd] in defm Z: avx512_trunc_common, avx512_trunc_sat_mr_lowering, EVEX_V512; } multiclass avx512_trunc_qb opc, string OpcodeStr, SDNode OpNode> { defm NAME: avx512_trunc, EVEX_CD8<8, CD8VO>; } multiclass avx512_trunc_sat_qb opc, string sat, SDNode OpNode> { defm NAME: avx512_trunc_sat, EVEX_CD8<8, CD8VO>; } multiclass avx512_trunc_qw opc, string OpcodeStr, SDNode OpNode> { defm NAME: avx512_trunc, EVEX_CD8<16, CD8VQ>; } multiclass avx512_trunc_sat_qw opc, string sat, SDNode OpNode> { defm NAME: avx512_trunc_sat, EVEX_CD8<16, CD8VQ>; } multiclass avx512_trunc_qd opc, string OpcodeStr, SDNode OpNode> { defm NAME: avx512_trunc, EVEX_CD8<32, CD8VH>; } multiclass avx512_trunc_sat_qd opc, string sat, SDNode OpNode> { defm NAME: avx512_trunc_sat, EVEX_CD8<32, CD8VH>; } multiclass avx512_trunc_db opc, string OpcodeStr, SDNode OpNode> { defm NAME: avx512_trunc, EVEX_CD8<8, CD8VQ>; } multiclass avx512_trunc_sat_db opc, string sat, SDNode OpNode> { defm NAME: avx512_trunc_sat, EVEX_CD8<8, CD8VQ>; } multiclass avx512_trunc_dw opc, string OpcodeStr, SDNode OpNode> { defm NAME: avx512_trunc, EVEX_CD8<16, CD8VH>; } multiclass avx512_trunc_sat_dw opc, string sat, SDNode OpNode> { defm NAME: avx512_trunc_sat, EVEX_CD8<16, CD8VH>; } multiclass avx512_trunc_wb opc, string OpcodeStr, SDNode OpNode> { defm NAME: avx512_trunc, EVEX_CD8<16, CD8VH>; } multiclass avx512_trunc_sat_wb opc, string sat, SDNode OpNode> { defm NAME: avx512_trunc_sat, EVEX_CD8<16, CD8VH>; } defm VPMOVQB : avx512_trunc_qb<0x32, "vpmovqb", X86vtrunc>; defm VPMOVSQB : avx512_trunc_sat_qb<0x22, "s", X86vtruncs>; defm VPMOVUSQB : avx512_trunc_sat_qb<0x12, "us", X86vtruncus>; defm VPMOVQW : avx512_trunc_qw<0x34, "vpmovqw", X86vtrunc>; defm VPMOVSQW : avx512_trunc_sat_qw<0x24, "s", X86vtruncs>; defm VPMOVUSQW : avx512_trunc_sat_qw<0x14, "us", X86vtruncus>; defm VPMOVQD : avx512_trunc_qd<0x35, "vpmovqd", X86vtrunc>; defm VPMOVSQD : avx512_trunc_sat_qd<0x25, "s", X86vtruncs>; defm VPMOVUSQD : avx512_trunc_sat_qd<0x15, "us", X86vtruncus>; defm VPMOVDB : avx512_trunc_db<0x31, "vpmovdb", X86vtrunc>; defm VPMOVSDB : avx512_trunc_sat_db<0x21, "s", X86vtruncs>; defm VPMOVUSDB : avx512_trunc_sat_db<0x11, "us", X86vtruncus>; defm VPMOVDW : avx512_trunc_dw<0x33, "vpmovdw", X86vtrunc>; defm VPMOVSDW : avx512_trunc_sat_dw<0x23, "s", X86vtruncs>; defm VPMOVUSDW : avx512_trunc_sat_dw<0x13, "us", X86vtruncus>; defm VPMOVWB : avx512_trunc_wb<0x30, "vpmovwb", X86vtrunc>; defm VPMOVSWB : avx512_trunc_sat_wb<0x20, "s", X86vtruncs>; defm VPMOVUSWB : avx512_trunc_sat_wb<0x10, "us", X86vtruncus>; let Predicates = [HasAVX512, NoVLX] in { def: Pat<(v8i16 (X86vtrunc (v8i32 VR256X:$src))), (v8i16 (EXTRACT_SUBREG (v16i16 (VPMOVDWZrr (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src, sub_ymm)))), sub_xmm))>; def: Pat<(v4i32 (X86vtrunc (v4i64 VR256X:$src))), (v4i32 (EXTRACT_SUBREG (v8i32 (VPMOVQDZrr (v8i64 (SUBREG_TO_REG (i32 0), VR256X:$src, sub_ymm)))), sub_xmm))>; } let Predicates = [HasBWI, NoVLX] in { def: Pat<(v16i8 (X86vtrunc (v16i16 VR256X:$src))), (v16i8 (EXTRACT_SUBREG (VPMOVWBZrr (v32i16 (SUBREG_TO_REG (i32 0), VR256X:$src, sub_ymm))), sub_xmm))>; } multiclass avx512_extend_common opc, string OpcodeStr, X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo, X86MemOperand x86memop, PatFrag LdFrag, SDNode OpNode>{ defm rr : AVX512_maskable, EVEX; let mayLoad = 1 in { defm rm : AVX512_maskable, EVEX; } } multiclass avx512_extend_BW opc, string OpcodeStr, SDNode OpNode, string ExtTy,PatFrag LdFrag = !cast(ExtTy#"extloadvi8")> { let Predicates = [HasVLX, HasBWI] in { defm Z128: avx512_extend_common, EVEX_CD8<8, CD8VH>, T8PD, EVEX_V128; defm Z256: avx512_extend_common, EVEX_CD8<8, CD8VH>, T8PD, EVEX_V256; } let Predicates = [HasBWI] in { defm Z : avx512_extend_common, EVEX_CD8<8, CD8VH>, T8PD, EVEX_V512; } } multiclass avx512_extend_BD opc, string OpcodeStr, SDNode OpNode, string ExtTy,PatFrag LdFrag = !cast(ExtTy#"extloadvi8")> { let Predicates = [HasVLX, HasAVX512] in { defm Z128: avx512_extend_common, EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V128; defm Z256: avx512_extend_common, EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V256; } let Predicates = [HasAVX512] in { defm Z : avx512_extend_common, EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V512; } } multiclass avx512_extend_BQ opc, string OpcodeStr, SDNode OpNode, string ExtTy,PatFrag LdFrag = !cast(ExtTy#"extloadvi8")> { let Predicates = [HasVLX, HasAVX512] in { defm Z128: avx512_extend_common, EVEX_CD8<8, CD8VO>, T8PD, EVEX_V128; defm Z256: avx512_extend_common, EVEX_CD8<8, CD8VO>, T8PD, EVEX_V256; } let Predicates = [HasAVX512] in { defm Z : avx512_extend_common, EVEX_CD8<8, CD8VO>, T8PD, EVEX_V512; } } multiclass avx512_extend_WD opc, string OpcodeStr, SDNode OpNode, string ExtTy,PatFrag LdFrag = !cast(ExtTy#"extloadvi16")> { let Predicates = [HasVLX, HasAVX512] in { defm Z128: avx512_extend_common, EVEX_CD8<16, CD8VH>, T8PD, EVEX_V128; defm Z256: avx512_extend_common, EVEX_CD8<16, CD8VH>, T8PD, EVEX_V256; } let Predicates = [HasAVX512] in { defm Z : avx512_extend_common, EVEX_CD8<16, CD8VH>, T8PD, EVEX_V512; } } multiclass avx512_extend_WQ opc, string OpcodeStr, SDNode OpNode, string ExtTy,PatFrag LdFrag = !cast(ExtTy#"extloadvi16")> { let Predicates = [HasVLX, HasAVX512] in { defm Z128: avx512_extend_common, EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V128; defm Z256: avx512_extend_common, EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V256; } let Predicates = [HasAVX512] in { defm Z : avx512_extend_common, EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V512; } } multiclass avx512_extend_DQ opc, string OpcodeStr, SDNode OpNode, string ExtTy,PatFrag LdFrag = !cast(ExtTy#"extloadvi32")> { let Predicates = [HasVLX, HasAVX512] in { defm Z128: avx512_extend_common, EVEX_CD8<32, CD8VH>, T8PD, EVEX_V128; defm Z256: avx512_extend_common, EVEX_CD8<32, CD8VH>, T8PD, EVEX_V256; } let Predicates = [HasAVX512] in { defm Z : avx512_extend_common, EVEX_CD8<32, CD8VH>, T8PD, EVEX_V512; } } defm VPMOVZXBW : avx512_extend_BW<0x30, "vpmovzxbw", X86vzext, "z">; defm VPMOVZXBD : avx512_extend_BD<0x31, "vpmovzxbd", X86vzext, "z">; defm VPMOVZXBQ : avx512_extend_BQ<0x32, "vpmovzxbq", X86vzext, "z">; defm VPMOVZXWD : avx512_extend_WD<0x33, "vpmovzxwd", X86vzext, "z">; defm VPMOVZXWQ : avx512_extend_WQ<0x34, "vpmovzxwq", X86vzext, "z">; defm VPMOVZXDQ : avx512_extend_DQ<0x35, "vpmovzxdq", X86vzext, "z">; defm VPMOVSXBW: avx512_extend_BW<0x20, "vpmovsxbw", X86vsext, "s">; defm VPMOVSXBD: avx512_extend_BD<0x21, "vpmovsxbd", X86vsext, "s">; defm VPMOVSXBQ: avx512_extend_BQ<0x22, "vpmovsxbq", X86vsext, "s">; defm VPMOVSXWD: avx512_extend_WD<0x23, "vpmovsxwd", X86vsext, "s">; defm VPMOVSXWQ: avx512_extend_WQ<0x24, "vpmovsxwq", X86vsext, "s">; defm VPMOVSXDQ: avx512_extend_DQ<0x25, "vpmovsxdq", X86vsext, "s">; //===----------------------------------------------------------------------===// // GATHER - SCATTER Operations multiclass avx512_gather opc, string OpcodeStr, X86VectorVTInfo _, X86MemOperand memop, PatFrag GatherNode> { let Constraints = "@earlyclobber $dst, $src1 = $dst, $mask = $mask_wb", ExeDomain = _.ExeDomain in def rm : AVX5128I, EVEX, EVEX_K, EVEX_CD8<_.EltSize, CD8VT1>; } multiclass avx512_gather_q_pd dopc, bits<8> qopc, AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> { defm NAME##D##SUFF##Z: avx512_gather, EVEX_V512, VEX_W; defm NAME##Q##SUFF##Z: avx512_gather, EVEX_V512, VEX_W; let Predicates = [HasVLX] in { defm NAME##D##SUFF##Z256: avx512_gather, EVEX_V256, VEX_W; defm NAME##Q##SUFF##Z256: avx512_gather, EVEX_V256, VEX_W; defm NAME##D##SUFF##Z128: avx512_gather, EVEX_V128, VEX_W; defm NAME##Q##SUFF##Z128: avx512_gather, EVEX_V128, VEX_W; } } multiclass avx512_gather_d_ps dopc, bits<8> qopc, AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> { defm NAME##D##SUFF##Z: avx512_gather, EVEX_V512; defm NAME##Q##SUFF##Z: avx512_gather, EVEX_V512; let Predicates = [HasVLX] in { defm NAME##D##SUFF##Z256: avx512_gather, EVEX_V256; defm NAME##Q##SUFF##Z256: avx512_gather, EVEX_V256; defm NAME##D##SUFF##Z128: avx512_gather, EVEX_V128; defm NAME##Q##SUFF##Z128: avx512_gather, EVEX_V128; } } defm VGATHER : avx512_gather_q_pd<0x92, 0x93, avx512vl_f64_info, "vgather", "PD">, avx512_gather_d_ps<0x92, 0x93, avx512vl_f32_info, "vgather", "PS">; defm VPGATHER : avx512_gather_q_pd<0x90, 0x91, avx512vl_i64_info, "vpgather", "Q">, avx512_gather_d_ps<0x90, 0x91, avx512vl_i32_info, "vpgather", "D">; multiclass avx512_scatter opc, string OpcodeStr, X86VectorVTInfo _, X86MemOperand memop, PatFrag ScatterNode> { let mayStore = 1, Constraints = "$mask = $mask_wb", ExeDomain = _.ExeDomain in def mr : AVX5128I, EVEX, EVEX_K, EVEX_CD8<_.EltSize, CD8VT1>; } multiclass avx512_scatter_q_pd dopc, bits<8> qopc, AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> { defm NAME##D##SUFF##Z: avx512_scatter, EVEX_V512, VEX_W; defm NAME##Q##SUFF##Z: avx512_scatter, EVEX_V512, VEX_W; let Predicates = [HasVLX] in { defm NAME##D##SUFF##Z256: avx512_scatter, EVEX_V256, VEX_W; defm NAME##Q##SUFF##Z256: avx512_scatter, EVEX_V256, VEX_W; defm NAME##D##SUFF##Z128: avx512_scatter, EVEX_V128, VEX_W; defm NAME##Q##SUFF##Z128: avx512_scatter, EVEX_V128, VEX_W; } } multiclass avx512_scatter_d_ps dopc, bits<8> qopc, AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> { defm NAME##D##SUFF##Z: avx512_scatter, EVEX_V512; defm NAME##Q##SUFF##Z: avx512_scatter, EVEX_V512; let Predicates = [HasVLX] in { defm NAME##D##SUFF##Z256: avx512_scatter, EVEX_V256; defm NAME##Q##SUFF##Z256: avx512_scatter, EVEX_V256; defm NAME##D##SUFF##Z128: avx512_scatter, EVEX_V128; defm NAME##Q##SUFF##Z128: avx512_scatter, EVEX_V128; } } defm VSCATTER : avx512_scatter_q_pd<0xA2, 0xA3, avx512vl_f64_info, "vscatter", "PD">, avx512_scatter_d_ps<0xA2, 0xA3, avx512vl_f32_info, "vscatter", "PS">; defm VPSCATTER : avx512_scatter_q_pd<0xA0, 0xA1, avx512vl_i64_info, "vpscatter", "Q">, avx512_scatter_d_ps<0xA0, 0xA1, avx512vl_i32_info, "vpscatter", "D">; // prefetch multiclass avx512_gather_scatter_prefetch opc, Format F, string OpcodeStr, RegisterClass KRC, X86MemOperand memop> { let Predicates = [HasPFI], hasSideEffects = 1 in def m : AVX5128I, EVEX, EVEX_K; } defm VGATHERPF0DPS: avx512_gather_scatter_prefetch<0xC6, MRM1m, "vgatherpf0dps", VK16WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>; defm VGATHERPF0QPS: avx512_gather_scatter_prefetch<0xC7, MRM1m, "vgatherpf0qps", VK8WM, vz64mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>; defm VGATHERPF0DPD: avx512_gather_scatter_prefetch<0xC6, MRM1m, "vgatherpf0dpd", VK8WM, vy32mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>; defm VGATHERPF0QPD: avx512_gather_scatter_prefetch<0xC7, MRM1m, "vgatherpf0qpd", VK8WM, vz64mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>; defm VGATHERPF1DPS: avx512_gather_scatter_prefetch<0xC6, MRM2m, "vgatherpf1dps", VK16WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>; defm VGATHERPF1QPS: avx512_gather_scatter_prefetch<0xC7, MRM2m, "vgatherpf1qps", VK8WM, vz64mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>; defm VGATHERPF1DPD: avx512_gather_scatter_prefetch<0xC6, MRM2m, "vgatherpf1dpd", VK8WM, vy32mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>; defm VGATHERPF1QPD: avx512_gather_scatter_prefetch<0xC7, MRM2m, "vgatherpf1qpd", VK8WM, vz64mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>; defm VSCATTERPF0DPS: avx512_gather_scatter_prefetch<0xC6, MRM5m, "vscatterpf0dps", VK16WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>; defm VSCATTERPF0QPS: avx512_gather_scatter_prefetch<0xC7, MRM5m, "vscatterpf0qps", VK8WM, vz64mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>; defm VSCATTERPF0DPD: avx512_gather_scatter_prefetch<0xC6, MRM5m, "vscatterpf0dpd", VK8WM, vy32mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>; defm VSCATTERPF0QPD: avx512_gather_scatter_prefetch<0xC7, MRM5m, "vscatterpf0qpd", VK8WM, vz64mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>; defm VSCATTERPF1DPS: avx512_gather_scatter_prefetch<0xC6, MRM6m, "vscatterpf1dps", VK16WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>; defm VSCATTERPF1QPS: avx512_gather_scatter_prefetch<0xC7, MRM6m, "vscatterpf1qps", VK8WM, vz64mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>; defm VSCATTERPF1DPD: avx512_gather_scatter_prefetch<0xC6, MRM6m, "vscatterpf1dpd", VK8WM, vy32mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>; defm VSCATTERPF1QPD: avx512_gather_scatter_prefetch<0xC7, MRM6m, "vscatterpf1qpd", VK8WM, vz64mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>; // Helper fragments to match sext vXi1 to vXiY. def v16i1sextv16i32 : PatLeaf<(v16i32 (X86vsrai VR512:$src, (i8 31)))>; def v8i1sextv8i64 : PatLeaf<(v8i64 (X86vsrai VR512:$src, (i8 63)))>; def : Pat<(store (i1 -1), addr:$dst), (MOV8mi addr:$dst, (i8 1))>; def : Pat<(store (i1 1), addr:$dst), (MOV8mi addr:$dst, (i8 1))>; def : Pat<(store (i1 0), addr:$dst), (MOV8mi addr:$dst, (i8 0))>; def : Pat<(store VK1:$src, addr:$dst), (MOV8mr addr:$dst, (EXTRACT_SUBREG (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)), sub_8bit))>, Requires<[HasAVX512, NoDQI]>; def : Pat<(store VK8:$src, addr:$dst), (MOV8mr addr:$dst, (EXTRACT_SUBREG (KMOVWrk (COPY_TO_REGCLASS VK8:$src, VK16)), sub_8bit))>, Requires<[HasAVX512, NoDQI]>; def truncstorei1 : PatFrag<(ops node:$val, node:$ptr), (truncstore node:$val, node:$ptr), [{ return cast(N)->getMemoryVT() == MVT::i1; }]>; def : Pat<(truncstorei1 GR8:$src, addr:$dst), (MOV8mr addr:$dst, GR8:$src)>; multiclass cvt_by_vec_width opc, X86VectorVTInfo Vec, string OpcodeStr > { def rr : AVX512XS8I, EVEX; } multiclass cvt_mask_by_elt_width opc, AVX512VLVectorVTInfo VTInfo, string OpcodeStr, Predicate prd> { let Predicates = [prd] in defm Z : cvt_by_vec_width, EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : cvt_by_vec_width, EVEX_V256; defm Z128 : cvt_by_vec_width, EVEX_V128; } } multiclass avx512_convert_mask_to_vector { defm NAME##B : cvt_mask_by_elt_width<0x28, avx512vl_i8_info, OpcodeStr, HasBWI>; defm NAME##W : cvt_mask_by_elt_width<0x28, avx512vl_i16_info, OpcodeStr, HasBWI>, VEX_W; defm NAME##D : cvt_mask_by_elt_width<0x38, avx512vl_i32_info, OpcodeStr, HasDQI>; defm NAME##Q : cvt_mask_by_elt_width<0x38, avx512vl_i64_info, OpcodeStr, HasDQI>, VEX_W; } defm VPMOVM2 : avx512_convert_mask_to_vector<"vpmovm2">; multiclass convert_vector_to_mask_common opc, X86VectorVTInfo _, string OpcodeStr > { def rr : AVX512XS8I, EVEX; } multiclass avx512_convert_vector_to_mask opc, string OpcodeStr, AVX512VLVectorVTInfo VTInfo, Predicate prd> { let Predicates = [prd] in defm Z : convert_vector_to_mask_common , EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : convert_vector_to_mask_common, EVEX_V256; defm Z128 : convert_vector_to_mask_common, EVEX_V128; } } defm VPMOVB2M : avx512_convert_vector_to_mask<0x29, "vpmovb2m", avx512vl_i8_info, HasBWI>; defm VPMOVW2M : avx512_convert_vector_to_mask<0x29, "vpmovw2m", avx512vl_i16_info, HasBWI>, VEX_W; defm VPMOVD2M : avx512_convert_vector_to_mask<0x39, "vpmovd2m", avx512vl_i32_info, HasDQI>; defm VPMOVQ2M : avx512_convert_vector_to_mask<0x39, "vpmovq2m", avx512vl_i64_info, HasDQI>, VEX_W; //===----------------------------------------------------------------------===// // AVX-512 - COMPRESS and EXPAND // multiclass compress_by_vec_width opc, X86VectorVTInfo _, string OpcodeStr> { defm rr : AVX512_maskable, AVX5128IBase; let mayStore = 1 in { def mr : AVX5128I, EVEX_CD8<_.EltSize, CD8VT1>; def mrk : AVX5128I, EVEX_K, EVEX_CD8<_.EltSize, CD8VT1>; } } multiclass compress_by_elt_width opc, string OpcodeStr, AVX512VLVectorVTInfo VTInfo> { defm Z : compress_by_vec_width, EVEX_V512; let Predicates = [HasVLX] in { defm Z256 : compress_by_vec_width, EVEX_V256; defm Z128 : compress_by_vec_width, EVEX_V128; } } defm VPCOMPRESSD : compress_by_elt_width <0x8B, "vpcompressd", avx512vl_i32_info>, EVEX; defm VPCOMPRESSQ : compress_by_elt_width <0x8B, "vpcompressq", avx512vl_i64_info>, EVEX, VEX_W; defm VCOMPRESSPS : compress_by_elt_width <0x8A, "vcompressps", avx512vl_f32_info>, EVEX; defm VCOMPRESSPD : compress_by_elt_width <0x8A, "vcompresspd", avx512vl_f64_info>, EVEX, VEX_W; // expand multiclass expand_by_vec_width opc, X86VectorVTInfo _, string OpcodeStr> { defm rr : AVX512_maskable, AVX5128IBase; let mayLoad = 1 in defm rm : AVX512_maskable, AVX5128IBase, EVEX_CD8<_.EltSize, CD8VT1>; } multiclass expand_by_elt_width opc, string OpcodeStr, AVX512VLVectorVTInfo VTInfo> { defm Z : expand_by_vec_width, EVEX_V512; let Predicates = [HasVLX] in { defm Z256 : expand_by_vec_width, EVEX_V256; defm Z128 : expand_by_vec_width, EVEX_V128; } } defm VPEXPANDD : expand_by_elt_width <0x89, "vpexpandd", avx512vl_i32_info>, EVEX; defm VPEXPANDQ : expand_by_elt_width <0x89, "vpexpandq", avx512vl_i64_info>, EVEX, VEX_W; defm VEXPANDPS : expand_by_elt_width <0x88, "vexpandps", avx512vl_f32_info>, EVEX; defm VEXPANDPD : expand_by_elt_width <0x88, "vexpandpd", avx512vl_f64_info>, EVEX, VEX_W; //handle instruction reg_vec1 = op(reg_vec,imm) // op(mem_vec,imm) // op(broadcast(eltVt),imm) //all instruction created with FROUND_CURRENT multiclass avx512_unary_fp_packed_imm opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _>{ defm rri : AVX512_maskable; let mayLoad = 1 in { defm rmi : AVX512_maskable; defm rmbi : AVX512_maskable, EVEX_B; } } //handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae} multiclass avx512_unary_fp_sae_packed_imm opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _>{ defm rrib : AVX512_maskable, EVEX_B; } multiclass avx512_common_unary_fp_sae_packed_imm opc, SDNode OpNode, Predicate prd>{ let Predicates = [prd] in { defm Z : avx512_unary_fp_packed_imm, avx512_unary_fp_sae_packed_imm, EVEX_V512; } let Predicates = [prd, HasVLX] in { defm Z128 : avx512_unary_fp_packed_imm, EVEX_V128; defm Z256 : avx512_unary_fp_packed_imm, EVEX_V256; } } //handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm) // op(reg_vec2,mem_vec,imm) // op(reg_vec2,broadcast(eltVt),imm) //all instruction created with FROUND_CURRENT multiclass avx512_fp_packed_imm opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _>{ defm rri : AVX512_maskable; let mayLoad = 1 in { defm rmi : AVX512_maskable; defm rmbi : AVX512_maskable, EVEX_B; } } //handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm) // op(reg_vec2,mem_vec,imm) multiclass avx512_3Op_rm_imm8 opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo>{ defm rri : AVX512_maskable; let mayLoad = 1 in defm rmi : AVX512_maskable; } //handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm) // op(reg_vec2,mem_vec,imm) // op(reg_vec2,broadcast(eltVt),imm) multiclass avx512_3Op_imm8 opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _>: avx512_3Op_rm_imm8{ let mayLoad = 1 in defm rmbi : AVX512_maskable, EVEX_B; } //handle scalar instruction reg_vec1 = op(reg_vec2,reg_vec3,imm) // op(reg_vec2,mem_scalar,imm) //all instruction created with FROUND_CURRENT multiclass avx512_fp_scalar_imm opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { defm rri : AVX512_maskable_scalar; let mayLoad = 1 in { defm rmi : AVX512_maskable_scalar; let isAsmParserOnly = 1 in { defm rmi_alt :AVX512_maskable_in_asm; } } } //handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae} multiclass avx512_fp_sae_packed_imm opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _>{ defm rrib : AVX512_maskable, EVEX_B; } //handle scalar instruction reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae} multiclass avx512_fp_sae_scalar_imm opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { defm NAME#rrib : AVX512_maskable_scalar, EVEX_B; } multiclass avx512_common_fp_sae_packed_imm opc, SDNode OpNode, Predicate prd>{ let Predicates = [prd] in { defm Z : avx512_fp_packed_imm, avx512_fp_sae_packed_imm, EVEX_V512; } let Predicates = [prd, HasVLX] in { defm Z128 : avx512_fp_packed_imm, EVEX_V128; defm Z256 : avx512_fp_packed_imm, EVEX_V256; } } multiclass avx512_common_3Op_rm_imm8 opc, SDNode OpNode, string OpStr, AVX512VLVectorVTInfo DestInfo, AVX512VLVectorVTInfo SrcInfo>{ let Predicates = [HasBWI] in { defm Z : avx512_3Op_rm_imm8, EVEX_V512, AVX512AIi8Base, EVEX_4V; } let Predicates = [HasBWI, HasVLX] in { defm Z128 : avx512_3Op_rm_imm8, EVEX_V128, AVX512AIi8Base, EVEX_4V; defm Z256 : avx512_3Op_rm_imm8, EVEX_V256, AVX512AIi8Base, EVEX_4V; } } multiclass avx512_common_3Op_imm8 opc, SDNode OpNode>{ let Predicates = [HasAVX512] in { defm Z : avx512_3Op_imm8, EVEX_V512; } let Predicates = [HasAVX512, HasVLX] in { defm Z128 : avx512_3Op_imm8, EVEX_V128; defm Z256 : avx512_3Op_imm8, EVEX_V256; } } multiclass avx512_common_fp_sae_scalar_imm opc, SDNode OpNode, Predicate prd>{ let Predicates = [prd] in { defm Z128 : avx512_fp_scalar_imm, avx512_fp_sae_scalar_imm; } } multiclass avx512_common_unary_fp_sae_packed_imm_all opcPs, bits<8> opcPd, SDNode OpNode, Predicate prd>{ defm PS : avx512_common_unary_fp_sae_packed_imm, EVEX_CD8<32, CD8VF>; defm PD : avx512_common_unary_fp_sae_packed_imm, EVEX_CD8<64, CD8VF>, VEX_W; } defm VFIXUPIMMPD : avx512_common_fp_sae_packed_imm<"vfixupimmpd", avx512vl_f64_info, 0x54, X86VFixupimm, HasAVX512>, AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W; defm VFIXUPIMMPS : avx512_common_fp_sae_packed_imm<"vfixupimmps", avx512vl_f32_info, 0x54, X86VFixupimm, HasAVX512>, AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>; defm VFIXUPIMMSD: avx512_common_fp_sae_scalar_imm<"vfixupimmsd", f64x_info, 0x55, X86VFixupimm, HasAVX512>, AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W; defm VFIXUPIMMSS: avx512_common_fp_sae_scalar_imm<"vfixupimmss", f32x_info, 0x55, X86VFixupimm, HasAVX512>, AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>; defm VREDUCE : avx512_common_unary_fp_sae_packed_imm_all<"vreduce", 0x56, 0x56, X86VReduce, HasDQI>, AVX512AIi8Base, EVEX; defm VRNDSCALE : avx512_common_unary_fp_sae_packed_imm_all<"vrndscale", 0x08, 0x09, X86VRndScale, HasAVX512>, AVX512AIi8Base, EVEX; defm VGETMANT : avx512_common_unary_fp_sae_packed_imm_all<"vgetmant", 0x26, 0x26, X86VGetMant, HasAVX512>, AVX512AIi8Base, EVEX; defm VRANGEPD : avx512_common_fp_sae_packed_imm<"vrangepd", avx512vl_f64_info, 0x50, X86VRange, HasDQI>, AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W; defm VRANGEPS : avx512_common_fp_sae_packed_imm<"vrangeps", avx512vl_f32_info, 0x50, X86VRange, HasDQI>, AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>; defm VRANGESD: avx512_common_fp_sae_scalar_imm<"vrangesd", f64x_info, 0x51, X86VRange, HasDQI>, AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W; defm VRANGESS: avx512_common_fp_sae_scalar_imm<"vrangess", f32x_info, 0x51, X86VRange, HasDQI>, AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>; defm VREDUCESD: avx512_common_fp_sae_scalar_imm<"vreducesd", f64x_info, 0x57, X86Reduces, HasDQI>, AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W; defm VREDUCESS: avx512_common_fp_sae_scalar_imm<"vreducess", f32x_info, 0x57, X86Reduces, HasDQI>, AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>; defm VGETMANTSD: avx512_common_fp_sae_scalar_imm<"vgetmantsd", f64x_info, 0x27, X86GetMants, HasAVX512>, AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W; defm VGETMANTSS: avx512_common_fp_sae_scalar_imm<"vgetmantss", f32x_info, 0x27, X86GetMants, HasAVX512>, AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>; multiclass avx512_shuff_packed_128 opc, SDNode OpNode = X86Shuf128>{ let Predicates = [HasAVX512] in { defm Z : avx512_3Op_imm8, EVEX_V512; } let Predicates = [HasAVX512, HasVLX] in { defm Z256 : avx512_3Op_imm8, EVEX_V256; } } let Predicates = [HasAVX512] in { def : Pat<(v16f32 (ffloor VR512:$src)), (VRNDSCALEPSZrri VR512:$src, (i32 0x1))>; def : Pat<(v16f32 (fnearbyint VR512:$src)), (VRNDSCALEPSZrri VR512:$src, (i32 0xC))>; def : Pat<(v16f32 (fceil VR512:$src)), (VRNDSCALEPSZrri VR512:$src, (i32 0x2))>; def : Pat<(v16f32 (frint VR512:$src)), (VRNDSCALEPSZrri VR512:$src, (i32 0x4))>; def : Pat<(v16f32 (ftrunc VR512:$src)), (VRNDSCALEPSZrri VR512:$src, (i32 0x3))>; def : Pat<(v8f64 (ffloor VR512:$src)), (VRNDSCALEPDZrri VR512:$src, (i32 0x1))>; def : Pat<(v8f64 (fnearbyint VR512:$src)), (VRNDSCALEPDZrri VR512:$src, (i32 0xC))>; def : Pat<(v8f64 (fceil VR512:$src)), (VRNDSCALEPDZrri VR512:$src, (i32 0x2))>; def : Pat<(v8f64 (frint VR512:$src)), (VRNDSCALEPDZrri VR512:$src, (i32 0x4))>; def : Pat<(v8f64 (ftrunc VR512:$src)), (VRNDSCALEPDZrri VR512:$src, (i32 0x3))>; } defm VSHUFF32X4 : avx512_shuff_packed_128<"vshuff32x4",avx512vl_f32_info, 0x23>, AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>; defm VSHUFF64X2 : avx512_shuff_packed_128<"vshuff64x2",avx512vl_f64_info, 0x23>, AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W; defm VSHUFI32X4 : avx512_shuff_packed_128<"vshufi32x4",avx512vl_i32_info, 0x43>, AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>; defm VSHUFI64X2 : avx512_shuff_packed_128<"vshufi64x2",avx512vl_i64_info, 0x43>, AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W; multiclass avx512_valign { defm NAME: avx512_common_3Op_imm8, AVX512AIi8Base, EVEX_4V; } defm VALIGND: avx512_valign<"valignd", avx512vl_i32_info>, EVEX_CD8<32, CD8VF>; defm VALIGNQ: avx512_valign<"valignq", avx512vl_i64_info>, EVEX_CD8<64, CD8VF>, VEX_W; multiclass avx512_vpalign_lowering p>{ let Predicates = p in def NAME#_.VTName#rri: Pat<(_.VT (X86PAlignr _.RC:$src1, _.RC:$src2, (i8 imm:$imm))), (!cast(NAME#_.ZSuffix#rri) _.RC:$src1, _.RC:$src2, imm:$imm)>; } multiclass avx512_vpalign_lowering_common: avx512_vpalign_lowering<_.info512, [HasBWI]>, avx512_vpalign_lowering<_.info128, [HasBWI, HasVLX]>, avx512_vpalign_lowering<_.info256, [HasBWI, HasVLX]>; defm VPALIGN: avx512_common_3Op_rm_imm8<0x0F, X86PAlignr, "vpalignr" , avx512vl_i8_info, avx512vl_i8_info>, avx512_vpalign_lowering_common, avx512_vpalign_lowering_common, avx512_vpalign_lowering_common, avx512_vpalign_lowering_common, avx512_vpalign_lowering_common, EVEX_CD8<8, CD8VF>; defm VDBPSADBW: avx512_common_3Op_rm_imm8<0x42, X86dbpsadbw, "vdbpsadbw" , avx512vl_i16_info, avx512vl_i8_info>, EVEX_CD8<8, CD8VF>; multiclass avx512_unary_rm opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { defm rr : AVX512_maskable, EVEX, AVX5128IBase; let mayLoad = 1 in defm rm : AVX512_maskable, EVEX, AVX5128IBase, EVEX_CD8<_.EltSize, CD8VF>; } multiclass avx512_unary_rmb opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> : avx512_unary_rm { let mayLoad = 1 in defm rmb : AVX512_maskable, EVEX, AVX5128IBase, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>; } multiclass avx512_unary_rm_vl opc, string OpcodeStr, SDNode OpNode, AVX512VLVectorVTInfo VTInfo, Predicate prd> { let Predicates = [prd] in defm Z : avx512_unary_rm, EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : avx512_unary_rm, EVEX_V256; defm Z128 : avx512_unary_rm, EVEX_V128; } } multiclass avx512_unary_rmb_vl opc, string OpcodeStr, SDNode OpNode, AVX512VLVectorVTInfo VTInfo, Predicate prd> { let Predicates = [prd] in defm Z : avx512_unary_rmb, EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : avx512_unary_rmb, EVEX_V256; defm Z128 : avx512_unary_rmb, EVEX_V128; } } multiclass avx512_unary_rm_vl_dq opc_d, bits<8> opc_q, string OpcodeStr, SDNode OpNode, Predicate prd> { defm Q : avx512_unary_rmb_vl, VEX_W; defm D : avx512_unary_rmb_vl; } multiclass avx512_unary_rm_vl_bw opc_b, bits<8> opc_w, string OpcodeStr, SDNode OpNode, Predicate prd> { defm W : avx512_unary_rm_vl; defm B : avx512_unary_rm_vl; } multiclass avx512_unary_rm_vl_all opc_b, bits<8> opc_w, bits<8> opc_d, bits<8> opc_q, string OpcodeStr, SDNode OpNode> { defm NAME : avx512_unary_rm_vl_dq, avx512_unary_rm_vl_bw; } defm VPABS : avx512_unary_rm_vl_all<0x1C, 0x1D, 0x1E, 0x1F, "vpabs", X86Abs>; def : Pat<(xor (bc_v16i32 (v16i1sextv16i32)), (bc_v16i32 (add (v16i32 VR512:$src), (v16i1sextv16i32)))), (VPABSDZrr VR512:$src)>; def : Pat<(xor (bc_v8i64 (v8i1sextv8i64)), (bc_v8i64 (add (v8i64 VR512:$src), (v8i1sextv8i64)))), (VPABSQZrr VR512:$src)>; multiclass avx512_ctlz opc, string OpcodeStr, Predicate prd>{ defm NAME : avx512_unary_rm_vl_dq; } defm VPLZCNT : avx512_ctlz<0x44, "vplzcnt", HasCDI>; defm VPCONFLICT : avx512_unary_rm_vl_dq<0xC4, 0xC4, "vpconflict", X86Conflict, HasCDI>; //===---------------------------------------------------------------------===// // Replicate Single FP - MOVSHDUP and MOVSLDUP //===---------------------------------------------------------------------===// multiclass avx512_replicate opc, string OpcodeStr, SDNode OpNode>{ defm NAME: avx512_unary_rm_vl, XS; } defm VMOVSHDUP : avx512_replicate<0x16, "vmovshdup", X86Movshdup>; defm VMOVSLDUP : avx512_replicate<0x12, "vmovsldup", X86Movsldup>; //===----------------------------------------------------------------------===// // AVX-512 - MOVDDUP //===----------------------------------------------------------------------===// multiclass avx512_movddup_128 opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { defm rr : AVX512_maskable, EVEX; let mayLoad = 1 in defm rm : AVX512_maskable, EVEX, EVEX_CD8<_.EltSize, CD8VH>; } multiclass avx512_movddup_common opc, string OpcodeStr, SDNode OpNode, AVX512VLVectorVTInfo VTInfo> { defm Z : avx512_unary_rm, EVEX_V512; let Predicates = [HasAVX512, HasVLX] in { defm Z256 : avx512_unary_rm, EVEX_V256; defm Z128 : avx512_movddup_128, EVEX_V128; } } multiclass avx512_movddup opc, string OpcodeStr, SDNode OpNode>{ defm NAME: avx512_movddup_common, XD, VEX_W; } defm VMOVDDUP : avx512_movddup<0x12, "vmovddup", X86Movddup>; def : Pat<(X86Movddup (loadv2f64 addr:$src)), (VMOVDDUPZ128rm addr:$src)>, Requires<[HasAVX512, HasVLX]>; def : Pat<(v2f64 (X86VBroadcast (loadf64 addr:$src))), (VMOVDDUPZ128rm addr:$src)>, Requires<[HasAVX512, HasVLX]>; //===----------------------------------------------------------------------===// // AVX-512 - Unpack Instructions //===----------------------------------------------------------------------===// defm VUNPCKH : avx512_fp_binop_p<0x15, "vunpckh", X86Unpckh>; defm VUNPCKL : avx512_fp_binop_p<0x14, "vunpckl", X86Unpckl>; defm VPUNPCKLBW : avx512_binop_rm_vl_b<0x60, "vpunpcklbw", X86Unpckl, SSE_INTALU_ITINS_P, HasBWI>; defm VPUNPCKHBW : avx512_binop_rm_vl_b<0x68, "vpunpckhbw", X86Unpckh, SSE_INTALU_ITINS_P, HasBWI>; defm VPUNPCKLWD : avx512_binop_rm_vl_w<0x61, "vpunpcklwd", X86Unpckl, SSE_INTALU_ITINS_P, HasBWI>; defm VPUNPCKHWD : avx512_binop_rm_vl_w<0x69, "vpunpckhwd", X86Unpckh, SSE_INTALU_ITINS_P, HasBWI>; defm VPUNPCKLDQ : avx512_binop_rm_vl_d<0x62, "vpunpckldq", X86Unpckl, SSE_INTALU_ITINS_P, HasAVX512>; defm VPUNPCKHDQ : avx512_binop_rm_vl_d<0x6A, "vpunpckhdq", X86Unpckh, SSE_INTALU_ITINS_P, HasAVX512>; defm VPUNPCKLQDQ : avx512_binop_rm_vl_q<0x6C, "vpunpcklqdq", X86Unpckl, SSE_INTALU_ITINS_P, HasAVX512>; defm VPUNPCKHQDQ : avx512_binop_rm_vl_q<0x6D, "vpunpckhqdq", X86Unpckh, SSE_INTALU_ITINS_P, HasAVX512>; //===----------------------------------------------------------------------===// // AVX-512 - Extract & Insert Integer Instructions //===----------------------------------------------------------------------===// multiclass avx512_extract_elt_bw_m opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { let mayStore = 1 in def mr : AVX512Ii8, EVEX, EVEX_CD8<_.EltSize, CD8VT1>; } multiclass avx512_extract_elt_b { let Predicates = [HasBWI] in { def rr : AVX512Ii8<0x14, MRMDestReg, (outs GR32orGR64:$dst), (ins _.RC:$src1, u8imm:$src2), OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set GR32orGR64:$dst, (X86pextrb (_.VT _.RC:$src1), imm:$src2))]>, EVEX, TAPD; defm NAME : avx512_extract_elt_bw_m<0x14, OpcodeStr, X86pextrb, _>, TAPD; } } multiclass avx512_extract_elt_w { let Predicates = [HasBWI] in { def rr : AVX512Ii8<0xC5, MRMSrcReg, (outs GR32orGR64:$dst), (ins _.RC:$src1, u8imm:$src2), OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set GR32orGR64:$dst, (X86pextrw (_.VT _.RC:$src1), imm:$src2))]>, EVEX, PD; def rr_REV : AVX512Ii8<0x15, MRMDestReg, (outs GR32orGR64:$dst), (ins _.RC:$src1, u8imm:$src2), OpcodeStr#".s\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, EVEX, TAPD; defm NAME : avx512_extract_elt_bw_m<0x15, OpcodeStr, X86pextrw, _>, TAPD; } } multiclass avx512_extract_elt_dq { let Predicates = [HasDQI] in { def rr : AVX512Ii8<0x16, MRMDestReg, (outs GRC:$dst), (ins _.RC:$src1, u8imm:$src2), OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set GRC:$dst, (extractelt (_.VT _.RC:$src1), imm:$src2))]>, EVEX, TAPD; let mayStore = 1 in def mr : AVX512Ii8<0x16, MRMDestMem, (outs), (ins _.ScalarMemOp:$dst, _.RC:$src1, u8imm:$src2), OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(store (extractelt (_.VT _.RC:$src1), imm:$src2),addr:$dst)]>, EVEX, EVEX_CD8<_.EltSize, CD8VT1>, TAPD; } } defm VPEXTRBZ : avx512_extract_elt_b<"vpextrb", v16i8x_info>; defm VPEXTRWZ : avx512_extract_elt_w<"vpextrw", v8i16x_info>; defm VPEXTRDZ : avx512_extract_elt_dq<"vpextrd", v4i32x_info, GR32>; defm VPEXTRQZ : avx512_extract_elt_dq<"vpextrq", v2i64x_info, GR64>, VEX_W; multiclass avx512_insert_elt_m opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _, PatFrag LdFrag> { def rm : AVX512Ii8, EVEX_4V, EVEX_CD8<_.EltSize, CD8VT1>; } multiclass avx512_insert_elt_bw opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _, PatFrag LdFrag> { let Predicates = [HasBWI] in { def rr : AVX512Ii8, EVEX_4V; defm NAME : avx512_insert_elt_m; } } multiclass avx512_insert_elt_dq opc, string OpcodeStr, X86VectorVTInfo _, RegisterClass GRC> { let Predicates = [HasDQI] in { def rr : AVX512Ii8, EVEX_4V, TAPD; defm NAME : avx512_insert_elt_m, TAPD; } } defm VPINSRBZ : avx512_insert_elt_bw<0x20, "vpinsrb", X86pinsrb, v16i8x_info, extloadi8>, TAPD; defm VPINSRWZ : avx512_insert_elt_bw<0xC4, "vpinsrw", X86pinsrw, v8i16x_info, extloadi16>, PD; defm VPINSRDZ : avx512_insert_elt_dq<0x22, "vpinsrd", v4i32x_info, GR32>; defm VPINSRQZ : avx512_insert_elt_dq<0x22, "vpinsrq", v2i64x_info, GR64>, VEX_W; //===----------------------------------------------------------------------===// // VSHUFPS - VSHUFPD Operations //===----------------------------------------------------------------------===// multiclass avx512_shufp{ defm NAME: avx512_common_3Op_imm8, EVEX_CD8, AVX512AIi8Base, EVEX_4V; } defm VSHUFPS: avx512_shufp<"vshufps", avx512vl_i32_info, avx512vl_f32_info>, PS; defm VSHUFPD: avx512_shufp<"vshufpd", avx512vl_i64_info, avx512vl_f64_info>, PD, VEX_W; //===----------------------------------------------------------------------===// // AVX-512 - Byte shift Left/Right //===----------------------------------------------------------------------===// multiclass avx512_shift_packed opc, SDNode OpNode, Format MRMr, Format MRMm, string OpcodeStr, X86VectorVTInfo _>{ def rr : AVX512; let mayLoad = 1 in def rm : AVX512; } multiclass avx512_shift_packed_all opc, SDNode OpNode, Format MRMr, Format MRMm, string OpcodeStr, Predicate prd>{ let Predicates = [prd] in defm Z512 : avx512_shift_packed, EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : avx512_shift_packed, EVEX_V256; defm Z128 : avx512_shift_packed, EVEX_V128; } } defm VPSLLDQ : avx512_shift_packed_all<0x73, X86vshldq, MRM7r, MRM7m, "vpslldq", HasBWI>, AVX512PDIi8Base, EVEX_4V; defm VPSRLDQ : avx512_shift_packed_all<0x73, X86vshrdq, MRM3r, MRM3m, "vpsrldq", HasBWI>, AVX512PDIi8Base, EVEX_4V; multiclass avx512_psadbw_packed opc, SDNode OpNode, string OpcodeStr, X86VectorVTInfo _dst, X86VectorVTInfo _src>{ def rr : AVX512BI; let mayLoad = 1 in def rm : AVX512BI; } multiclass avx512_psadbw_packed_all opc, SDNode OpNode, string OpcodeStr, Predicate prd> { let Predicates = [prd] in defm Z512 : avx512_psadbw_packed, EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : avx512_psadbw_packed, EVEX_V256; defm Z128 : avx512_psadbw_packed, EVEX_V128; } } defm VPSADBW : avx512_psadbw_packed_all<0xf6, X86psadbw, "vpsadbw", HasBWI>, EVEX_4V; multiclass avx512_ternlog opc, string OpcodeStr, SDNode OpNode, X86VectorVTInfo _>{ let Constraints = "$src1 = $dst" in { defm rri : AVX512_maskable_3src, AVX512AIi8Base, EVEX_4V; let mayLoad = 1 in { defm rmi : AVX512_maskable_3src, AVX512AIi8Base, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>; defm rmbi : AVX512_maskable_3src, EVEX_B, AVX512AIi8Base, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>; } }// Constraints = "$src1 = $dst" } multiclass avx512_common_ternlog{ let Predicates = [HasAVX512] in defm Z : avx512_ternlog<0x25, OpcodeStr, X86vpternlog, _.info512>, EVEX_V512; let Predicates = [HasAVX512, HasVLX] in { defm Z128 : avx512_ternlog<0x25, OpcodeStr, X86vpternlog, _.info128>, EVEX_V128; defm Z256 : avx512_ternlog<0x25, OpcodeStr, X86vpternlog, _.info256>, EVEX_V256; } } defm VPTERNLOGD : avx512_common_ternlog<"vpternlogd", avx512vl_i32_info>; defm VPTERNLOGQ : avx512_common_ternlog<"vpternlogq", avx512vl_i64_info>, VEX_W;