X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=include%2Fllvm%2FCodeGen%2FSelectionDAGNodes.h;h=23816bde07c0de9e12a3e45b2ff0acace8f79560;hb=52707d211b073942a461af811dcc442456a3064b;hp=c16a48aea2a9620fb54f64fc7ad5039872966f61;hpb=be5b032e4b06ae001098856c2dc4ebca41528132;p=oota-llvm.git diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h index c16a48aea2a..23816bde07c 100644 --- a/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/include/llvm/CodeGen/SelectionDAGNodes.h @@ -19,17 +19,22 @@ #ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H #define LLVM_CODEGEN_SELECTIONDAGNODES_H -#include "llvm/Constants.h" +#include "llvm/ADT/BitVector.h" #include "llvm/ADT/FoldingSet.h" #include "llvm/ADT/GraphTraits.h" -#include "llvm/ADT/ilist_node.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/CodeGen/ValueTypes.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/ilist_node.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/MachineMemOperand.h" +#include "llvm/CodeGen/ValueTypes.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DebugLoc.h" +#include "llvm/IR/Instructions.h" +#include "llvm/Support/DataTypes.h" #include "llvm/Support/MathExtras.h" -#include "llvm/System/DataTypes.h" -#include "llvm/Support/DebugLoc.h" #include namespace llvm { @@ -39,15 +44,17 @@ class GlobalValue; class MachineBasicBlock; class MachineConstantPoolValue; class SDNode; +class BinaryWithFlagsSDNode; class Value; class MCSymbol; template struct DenseMapInfo; template struct simplify_type; template struct ilist_traits; -void checkForCycles(const SDNode *N); - -/// SDVTList - This represents a list of ValueType's that has been intern'd by +void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr, + bool force = false); + +/// This represents a list of ValueType's that has been intern'd by /// a SelectionDAG. Instances of this simple value class are returned by /// SelectionDAG::getVTList(...). /// @@ -56,753 +63,32 @@ struct SDVTList { unsigned int NumVTs; }; -/// ISD namespace - This namespace contains an enum which represents all of the -/// SelectionDAG node types and value types. -/// namespace ISD { - - //===--------------------------------------------------------------------===// - /// ISD::NodeType enum - This enum defines the target-independent operators - /// for a SelectionDAG. - /// - /// Targets may also define target-dependent operator codes for SDNodes. For - /// example, on x86, these are the enum values in the X86ISD namespace. - /// Targets should aim to use target-independent operators to model their - /// instruction sets as much as possible, and only use target-dependent - /// operators when they have special requirements. - /// - /// Finally, during and after selection proper, SNodes may use special - /// operator codes that correspond directly with MachineInstr opcodes. These - /// are used to represent selected instructions. See the isMachineOpcode() - /// and getMachineOpcode() member functions of SDNode. - /// - enum NodeType { - // DELETED_NODE - This is an illegal value that is used to catch - // errors. This opcode is not a legal opcode for any node. - DELETED_NODE, - - // EntryToken - This is the marker used to indicate the start of the region. - EntryToken, - - // TokenFactor - This node takes multiple tokens as input and produces a - // single token result. This is used to represent the fact that the operand - // operators are independent of each other. - TokenFactor, - - // AssertSext, AssertZext - These nodes record if a register contains a - // value that has already been zero or sign extended from a narrower type. - // These nodes take two operands. The first is the node that has already - // been extended, and the second is a value type node indicating the width - // of the extension - AssertSext, AssertZext, - - // Various leaf nodes. - BasicBlock, VALUETYPE, CONDCODE, Register, - Constant, ConstantFP, - GlobalAddress, GlobalTLSAddress, FrameIndex, - JumpTable, ConstantPool, ExternalSymbol, BlockAddress, - - // The address of the GOT - GLOBAL_OFFSET_TABLE, - - // FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and - // llvm.returnaddress on the DAG. These nodes take one operand, the index - // of the frame or return address to return. An index of zero corresponds - // to the current function's frame or return address, an index of one to the - // parent's frame or return address, and so on. - FRAMEADDR, RETURNADDR, - - // FRAME_TO_ARGS_OFFSET - This node represents offset from frame pointer to - // first (possible) on-stack argument. This is needed for correct stack - // adjustment during unwind. - FRAME_TO_ARGS_OFFSET, - - // RESULT, OUTCHAIN = EXCEPTIONADDR(INCHAIN) - This node represents the - // address of the exception block on entry to an landing pad block. - EXCEPTIONADDR, - - // RESULT, OUTCHAIN = LSDAADDR(INCHAIN) - This node represents the - // address of the Language Specific Data Area for the enclosing function. - LSDAADDR, - - // RESULT, OUTCHAIN = EHSELECTION(INCHAIN, EXCEPTION) - This node represents - // the selection index of the exception thrown. - EHSELECTION, - - // OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents - // 'eh_return' gcc dwarf builtin, which is used to return from - // exception. The general meaning is: adjust stack by OFFSET and pass - // execution to HANDLER. Many platform-related details also :) - EH_RETURN, - - // TargetConstant* - Like Constant*, but the DAG does not do any folding or - // simplification of the constant. - TargetConstant, - TargetConstantFP, - - // TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or - // anything else with this node, and this is valid in the target-specific - // dag, turning into a GlobalAddress operand. - TargetGlobalAddress, - TargetGlobalTLSAddress, - TargetFrameIndex, - TargetJumpTable, - TargetConstantPool, - TargetExternalSymbol, - TargetBlockAddress, - - /// RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) - /// This node represents a target intrinsic function with no side effects. - /// The first operand is the ID number of the intrinsic from the - /// llvm::Intrinsic namespace. The operands to the intrinsic follow. The - /// node has returns the result of the intrinsic. - INTRINSIC_WO_CHAIN, - - /// RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) - /// This node represents a target intrinsic function with side effects that - /// returns a result. The first operand is a chain pointer. The second is - /// the ID number of the intrinsic from the llvm::Intrinsic namespace. The - /// operands to the intrinsic follow. The node has two results, the result - /// of the intrinsic and an output chain. - INTRINSIC_W_CHAIN, - - /// OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) - /// This node represents a target intrinsic function with side effects that - /// does not return a result. The first operand is a chain pointer. The - /// second is the ID number of the intrinsic from the llvm::Intrinsic - /// namespace. The operands to the intrinsic follow. - INTRINSIC_VOID, - - // CopyToReg - This node has three operands: a chain, a register number to - // set to this value, and a value. - CopyToReg, - - // CopyFromReg - This node indicates that the input value is a virtual or - // physical register that is defined outside of the scope of this - // SelectionDAG. The register is available from the RegisterSDNode object. - CopyFromReg, - - // UNDEF - An undefined node - UNDEF, - - // EXTRACT_ELEMENT - This is used to get the lower or upper (determined by - // a Constant, which is required to be operand #1) half of the integer or - // float value specified as operand #0. This is only for use before - // legalization, for values that will be broken into multiple registers. - EXTRACT_ELEMENT, - - // BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways. Given - // two values of the same integer value type, this produces a value twice as - // big. Like EXTRACT_ELEMENT, this can only be used before legalization. - BUILD_PAIR, - - // MERGE_VALUES - This node takes multiple discrete operands and returns - // them all as its individual results. This nodes has exactly the same - // number of inputs and outputs. This node is useful for some pieces of the - // code generator that want to think about a single node with multiple - // results, not multiple nodes. - MERGE_VALUES, - - // Simple integer binary arithmetic operators. - ADD, SUB, MUL, SDIV, UDIV, SREM, UREM, - - // SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing - // a signed/unsigned value of type i[2*N], and return the full value as - // two results, each of type iN. - SMUL_LOHI, UMUL_LOHI, - - // SDIVREM/UDIVREM - Divide two integers and produce both a quotient and - // remainder result. - SDIVREM, UDIVREM, - - // CARRY_FALSE - This node is used when folding other nodes, - // like ADDC/SUBC, which indicate the carry result is always false. - CARRY_FALSE, - - // Carry-setting nodes for multiple precision addition and subtraction. - // These nodes take two operands of the same value type, and produce two - // results. The first result is the normal add or sub result, the second - // result is the carry flag result. - ADDC, SUBC, - - // Carry-using nodes for multiple precision addition and subtraction. These - // nodes take three operands: The first two are the normal lhs and rhs to - // the add or sub, and the third is the input carry flag. These nodes - // produce two results; the normal result of the add or sub, and the output - // carry flag. These nodes both read and write a carry flag to allow them - // to them to be chained together for add and sub of arbitrarily large - // values. - ADDE, SUBE, - - // RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition. - // These nodes take two operands: the normal LHS and RHS to the add. They - // produce two results: the normal result of the add, and a boolean that - // indicates if an overflow occured (*not* a flag, because it may be stored - // to memory, etc.). If the type of the boolean is not i1 then the high - // bits conform to getBooleanContents. - // These nodes are generated from the llvm.[su]add.with.overflow intrinsics. - SADDO, UADDO, - - // Same for subtraction - SSUBO, USUBO, - - // Same for multiplication - SMULO, UMULO, - - // Simple binary floating point operators. - FADD, FSUB, FMUL, FDIV, FREM, - - // FCOPYSIGN(X, Y) - Return the value of X with the sign of Y. NOTE: This - // DAG node does not require that X and Y have the same type, just that they - // are both floating point. X and the result must have the same type. - // FCOPYSIGN(f32, f64) is allowed. - FCOPYSIGN, - - // INT = FGETSIGN(FP) - Return the sign bit of the specified floating point - // value as an integer 0/1 value. - FGETSIGN, - - /// BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the - /// specified, possibly variable, elements. The number of elements is - /// required to be a power of two. The types of the operands must all be - /// the same and must match the vector element type, except that integer - /// types are allowed to be larger than the element type, in which case - /// the operands are implicitly truncated. - BUILD_VECTOR, - - /// INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element - /// at IDX replaced with VAL. If the type of VAL is larger than the vector - /// element type then VAL is truncated before replacement. - INSERT_VECTOR_ELT, - - /// EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR - /// identified by the (potentially variable) element number IDX. If the - /// return type is an integer type larger than the element type of the - /// vector, the result is extended to the width of the return type. - EXTRACT_VECTOR_ELT, - - /// CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of - /// vector type with the same length and element type, this produces a - /// concatenated vector result value, with length equal to the sum of the - /// lengths of the input vectors. - CONCAT_VECTORS, - - /// EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an - /// vector value) starting with the (potentially variable) element number - /// IDX, which must be a multiple of the result vector length. - EXTRACT_SUBVECTOR, - - /// VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as - /// VEC1/VEC2. A VECTOR_SHUFFLE node also contains an array of constant int - /// values that indicate which value (or undef) each result element will - /// get. These constant ints are accessible through the - /// ShuffleVectorSDNode class. This is quite similar to the Altivec - /// 'vperm' instruction, except that the indices must be constants and are - /// in terms of the element size of VEC1/VEC2, not in terms of bytes. - VECTOR_SHUFFLE, - - /// SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a - /// scalar value into element 0 of the resultant vector type. The top - /// elements 1 to N-1 of the N-element vector are undefined. The type - /// of the operand must match the vector element type, except when they - /// are integer types. In this case the operand is allowed to be wider - /// than the vector element type, and is implicitly truncated to it. - SCALAR_TO_VECTOR, - - // MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing - // an unsigned/signed value of type i[2*N], then return the top part. - MULHU, MULHS, - - // Bitwise operators - logical and, logical or, logical xor, shift left, - // shift right algebraic (shift in sign bits), shift right logical (shift in - // zeroes), rotate left, rotate right, and byteswap. - AND, OR, XOR, SHL, SRA, SRL, ROTL, ROTR, BSWAP, - - // Counting operators - CTTZ, CTLZ, CTPOP, - - // Select(COND, TRUEVAL, FALSEVAL). If the type of the boolean COND is not - // i1 then the high bits must conform to getBooleanContents. - SELECT, - - // Select with condition operator - This selects between a true value and - // a false value (ops #2 and #3) based on the boolean result of comparing - // the lhs and rhs (ops #0 and #1) of a conditional expression with the - // condition code in op #4, a CondCodeSDNode. - SELECT_CC, - - // SetCC operator - This evaluates to a true value iff the condition is - // true. If the result value type is not i1 then the high bits conform - // to getBooleanContents. The operands to this are the left and right - // operands to compare (ops #0, and #1) and the condition code to compare - // them with (op #2) as a CondCodeSDNode. - SETCC, - - // RESULT = VSETCC(LHS, RHS, COND) operator - This evaluates to a vector of - // integer elements with all bits of the result elements set to true if the - // comparison is true or all cleared if the comparison is false. The - // operands to this are the left and right operands to compare (LHS/RHS) and - // the condition code to compare them with (COND) as a CondCodeSDNode. - VSETCC, - - // SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded - // integer shift operations, just like ADD/SUB_PARTS. The operation - // ordering is: - // [Lo,Hi] = op [LoLHS,HiLHS], Amt - SHL_PARTS, SRA_PARTS, SRL_PARTS, - - // Conversion operators. These are all single input single output - // operations. For all of these, the result type must be strictly - // wider or narrower (depending on the operation) than the source - // type. - - // SIGN_EXTEND - Used for integer types, replicating the sign bit - // into new bits. - SIGN_EXTEND, - - // ZERO_EXTEND - Used for integer types, zeroing the new bits. - ZERO_EXTEND, - - // ANY_EXTEND - Used for integer types. The high bits are undefined. - ANY_EXTEND, - - // TRUNCATE - Completely drop the high bits. - TRUNCATE, - - // [SU]INT_TO_FP - These operators convert integers (whose interpreted sign - // depends on the first letter) to floating point. - SINT_TO_FP, - UINT_TO_FP, - - // SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to - // sign extend a small value in a large integer register (e.g. sign - // extending the low 8 bits of a 32-bit register to fill the top 24 bits - // with the 7th bit). The size of the smaller type is indicated by the 1th - // operand, a ValueType node. - SIGN_EXTEND_INREG, - - /// FP_TO_[US]INT - Convert a floating point value to a signed or unsigned - /// integer. - FP_TO_SINT, - FP_TO_UINT, - - /// X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type - /// down to the precision of the destination VT. TRUNC is a flag, which is - /// always an integer that is zero or one. If TRUNC is 0, this is a - /// normal rounding, if it is 1, this FP_ROUND is known to not change the - /// value of Y. - /// - /// The TRUNC = 1 case is used in cases where we know that the value will - /// not be modified by the node, because Y is not using any of the extra - /// precision of source type. This allows certain transformations like - /// FP_EXTEND(FP_ROUND(X,1)) -> X which are not safe for - /// FP_EXTEND(FP_ROUND(X,0)) because the extra bits aren't removed. - FP_ROUND, - - // FLT_ROUNDS_ - Returns current rounding mode: - // -1 Undefined - // 0 Round to 0 - // 1 Round to nearest - // 2 Round to +inf - // 3 Round to -inf - FLT_ROUNDS_, - - /// X = FP_ROUND_INREG(Y, VT) - This operator takes an FP register, and - /// rounds it to a floating point value. It then promotes it and returns it - /// in a register of the same size. This operation effectively just - /// discards excess precision. The type to round down to is specified by - /// the VT operand, a VTSDNode. - FP_ROUND_INREG, - - /// X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type. - FP_EXTEND, - - // BIT_CONVERT - This operator converts between integer, vector and FP - // values, as if the value was stored to memory with one type and loaded - // from the same address with the other type (or equivalently for vector - // format conversions, etc). The source and result are required to have - // the same bit size (e.g. f32 <-> i32). This can also be used for - // int-to-int or fp-to-fp conversions, but that is a noop, deleted by - // getNode(). - BIT_CONVERT, - - // CONVERT_RNDSAT - This operator is used to support various conversions - // between various types (float, signed, unsigned and vectors of those - // types) with rounding and saturation. NOTE: Avoid using this operator as - // most target don't support it and the operator might be removed in the - // future. It takes the following arguments: - // 0) value - // 1) dest type (type to convert to) - // 2) src type (type to convert from) - // 3) rounding imm - // 4) saturation imm - // 5) ISD::CvtCode indicating the type of conversion to do - CONVERT_RNDSAT, - - // FP16_TO_FP32, FP32_TO_FP16 - These operators are used to perform - // promotions and truncation for half-precision (16 bit) floating - // numbers. We need special nodes since FP16 is a storage-only type with - // special semantics of operations. - FP16_TO_FP32, FP32_TO_FP16, - - // FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, - // FLOG, FLOG2, FLOG10, FEXP, FEXP2, - // FCEIL, FTRUNC, FRINT, FNEARBYINT, FFLOOR - Perform various unary floating - // point operations. These are inspired by libm. - FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, - FLOG, FLOG2, FLOG10, FEXP, FEXP2, - FCEIL, FTRUNC, FRINT, FNEARBYINT, FFLOOR, - - // LOAD and STORE have token chains as their first operand, then the same - // operands as an LLVM load/store instruction, then an offset node that - // is added / subtracted from the base pointer to form the address (for - // indexed memory ops). - LOAD, STORE, - - // DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned - // to a specified boundary. This node always has two return values: a new - // stack pointer value and a chain. The first operand is the token chain, - // the second is the number of bytes to allocate, and the third is the - // alignment boundary. The size is guaranteed to be a multiple of the stack - // alignment, and the alignment is guaranteed to be bigger than the stack - // alignment (if required) or 0 to get standard stack alignment. - DYNAMIC_STACKALLOC, - - // Control flow instructions. These all have token chains. - - // BR - Unconditional branch. The first operand is the chain - // operand, the second is the MBB to branch to. - BR, - - // BRIND - Indirect branch. The first operand is the chain, the second - // is the value to branch to, which must be of the same type as the target's - // pointer type. - BRIND, - - // BR_JT - Jumptable branch. The first operand is the chain, the second - // is the jumptable index, the last one is the jumptable entry index. - BR_JT, - - // BRCOND - Conditional branch. The first operand is the chain, the - // second is the condition, the third is the block to branch to if the - // condition is true. If the type of the condition is not i1, then the - // high bits must conform to getBooleanContents. - BRCOND, - - // BR_CC - Conditional branch. The behavior is like that of SELECT_CC, in - // that the condition is represented as condition code, and two nodes to - // compare, rather than as a combined SetCC node. The operands in order are - // chain, cc, lhs, rhs, block to branch to if condition is true. - BR_CC, - - // INLINEASM - Represents an inline asm block. This node always has two - // return values: a chain and a flag result. The inputs are as follows: - // Operand #0 : Input chain. - // Operand #1 : a ExternalSymbolSDNode with a pointer to the asm string. - // Operand #2n+2: A RegisterNode. - // Operand #2n+3: A TargetConstant, indicating if the reg is a use/def - // Operand #last: Optional, an incoming flag. - INLINEASM, - - // EH_LABEL - Represents a label in mid basic block used to track - // locations needed for debug and exception handling tables. These nodes - // take a chain as input and return a chain. - EH_LABEL, - - // STACKSAVE - STACKSAVE has one operand, an input chain. It produces a - // value, the same type as the pointer type for the system, and an output - // chain. - STACKSAVE, - - // STACKRESTORE has two operands, an input chain and a pointer to restore to - // it returns an output chain. - STACKRESTORE, - - // CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of - // a call sequence, and carry arbitrary information that target might want - // to know. The first operand is a chain, the rest are specified by the - // target and not touched by the DAG optimizers. - // CALLSEQ_START..CALLSEQ_END pairs may not be nested. - CALLSEQ_START, // Beginning of a call sequence - CALLSEQ_END, // End of a call sequence - - // VAARG - VAARG has three operands: an input chain, a pointer, and a - // SRCVALUE. It returns a pair of values: the vaarg value and a new chain. - VAARG, - - // VACOPY - VACOPY has five operands: an input chain, a destination pointer, - // a source pointer, a SRCVALUE for the destination, and a SRCVALUE for the - // source. - VACOPY, - - // VAEND, VASTART - VAEND and VASTART have three operands: an input chain, a - // pointer, and a SRCVALUE. - VAEND, VASTART, - - // SRCVALUE - This is a node type that holds a Value* that is used to - // make reference to a value in the LLVM IR. - SRCVALUE, - - // PCMARKER - This corresponds to the pcmarker intrinsic. - PCMARKER, - - // READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic. - // The only operand is a chain and a value and a chain are produced. The - // value is the contents of the architecture specific cycle counter like - // register (or other high accuracy low latency clock source) - READCYCLECOUNTER, - - // HANDLENODE node - Used as a handle for various purposes. - HANDLENODE, - - // TRAMPOLINE - This corresponds to the init_trampoline intrinsic. - // It takes as input a token chain, the pointer to the trampoline, - // the pointer to the nested function, the pointer to pass for the - // 'nest' parameter, a SRCVALUE for the trampoline and another for - // the nested function (allowing targets to access the original - // Function*). It produces the result of the intrinsic and a token - // chain as output. - TRAMPOLINE, - - // TRAP - Trapping instruction - TRAP, - - // PREFETCH - This corresponds to a prefetch intrinsic. It takes chains are - // their first operand. The other operands are the address to prefetch, - // read / write specifier, and locality specifier. - PREFETCH, - - // OUTCHAIN = MEMBARRIER(INCHAIN, load-load, load-store, store-load, - // store-store, device) - // This corresponds to the memory.barrier intrinsic. - // it takes an input chain, 4 operands to specify the type of barrier, an - // operand specifying if the barrier applies to device and uncached memory - // and produces an output chain. - MEMBARRIER, - - // Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) - // this corresponds to the atomic.lcs intrinsic. - // cmp is compared to *ptr, and if equal, swap is stored in *ptr. - // the return is always the original value in *ptr - ATOMIC_CMP_SWAP, - - // Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) - // this corresponds to the atomic.swap intrinsic. - // amt is stored to *ptr atomically. - // the return is always the original value in *ptr - ATOMIC_SWAP, - - // Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt) - // this corresponds to the atomic.load.[OpName] intrinsic. - // op(*ptr, amt) is stored to *ptr atomically. - // the return is always the original value in *ptr - ATOMIC_LOAD_ADD, - ATOMIC_LOAD_SUB, - ATOMIC_LOAD_AND, - ATOMIC_LOAD_OR, - ATOMIC_LOAD_XOR, - ATOMIC_LOAD_NAND, - ATOMIC_LOAD_MIN, - ATOMIC_LOAD_MAX, - ATOMIC_LOAD_UMIN, - ATOMIC_LOAD_UMAX, - - /// BUILTIN_OP_END - This must be the last enum value in this list. - /// The target-specific pre-isel opcode values start here. - BUILTIN_OP_END - }; - - /// FIRST_TARGET_MEMORY_OPCODE - Target-specific pre-isel operations - /// which do not reference a specific memory location should be less than - /// this value. Those that do must not be less than this value, and can - /// be used with SelectionDAG::getMemIntrinsicNode. - static const int FIRST_TARGET_MEMORY_OPCODE = BUILTIN_OP_END+80; - /// Node predicates - /// isBuildVectorAllOnes - Return true if the specified node is a + /// Return true if the specified node is a /// BUILD_VECTOR where all of the elements are ~0 or undef. bool isBuildVectorAllOnes(const SDNode *N); - /// isBuildVectorAllZeros - Return true if the specified node is a + /// Return true if the specified node is a /// BUILD_VECTOR where all of the elements are 0 or undef. bool isBuildVectorAllZeros(const SDNode *N); - /// isScalarToVector - Return true if the specified node is a - /// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low - /// element is not an undef. - bool isScalarToVector(const SDNode *N); - - //===--------------------------------------------------------------------===// - /// MemIndexedMode enum - This enum defines the load / store indexed - /// addressing modes. - /// - /// UNINDEXED "Normal" load / store. The effective address is already - /// computed and is available in the base pointer. The offset - /// operand is always undefined. In addition to producing a - /// chain, an unindexed load produces one value (result of the - /// load); an unindexed store does not produce a value. - /// - /// PRE_INC Similar to the unindexed mode where the effective address is - /// PRE_DEC the value of the base pointer add / subtract the offset. - /// It considers the computation as being folded into the load / - /// store operation (i.e. the load / store does the address - /// computation as well as performing the memory transaction). - /// The base operand is always undefined. In addition to - /// producing a chain, pre-indexed load produces two values - /// (result of the load and the result of the address - /// computation); a pre-indexed store produces one value (result - /// of the address computation). - /// - /// POST_INC The effective address is the value of the base pointer. The - /// POST_DEC value of the offset operand is then added to / subtracted - /// from the base after memory transaction. In addition to - /// producing a chain, post-indexed load produces two values - /// (the result of the load and the result of the base +/- offset - /// computation); a post-indexed store produces one value (the - /// the result of the base +/- offset computation). - /// - enum MemIndexedMode { - UNINDEXED = 0, - PRE_INC, - PRE_DEC, - POST_INC, - POST_DEC, - LAST_INDEXED_MODE - }; - - //===--------------------------------------------------------------------===// - /// LoadExtType enum - This enum defines the three variants of LOADEXT - /// (load with extension). - /// - /// SEXTLOAD loads the integer operand and sign extends it to a larger - /// integer result type. - /// ZEXTLOAD loads the integer operand and zero extends it to a larger - /// integer result type. - /// EXTLOAD is used for three things: floating point extending loads, - /// integer extending loads [the top bits are undefined], and vector - /// extending loads [load into low elt]. - /// - enum LoadExtType { - NON_EXTLOAD = 0, - EXTLOAD, - SEXTLOAD, - ZEXTLOAD, - LAST_LOADEXT_TYPE - }; - - //===--------------------------------------------------------------------===// - /// ISD::CondCode enum - These are ordered carefully to make the bitfields - /// below work out, when considering SETFALSE (something that never exists - /// dynamically) as 0. "U" -> Unsigned (for integer operands) or Unordered - /// (for floating point), "L" -> Less than, "G" -> Greater than, "E" -> Equal - /// to. If the "N" column is 1, the result of the comparison is undefined if - /// the input is a NAN. - /// - /// All of these (except for the 'always folded ops') should be handled for - /// floating point. For integer, only the SETEQ,SETNE,SETLT,SETLE,SETGT, - /// SETGE,SETULT,SETULE,SETUGT, and SETUGE opcodes are used. - /// - /// Note that these are laid out in a specific order to allow bit-twiddling - /// to transform conditions. - enum CondCode { - // Opcode N U L G E Intuitive operation - SETFALSE, // 0 0 0 0 Always false (always folded) - SETOEQ, // 0 0 0 1 True if ordered and equal - SETOGT, // 0 0 1 0 True if ordered and greater than - SETOGE, // 0 0 1 1 True if ordered and greater than or equal - SETOLT, // 0 1 0 0 True if ordered and less than - SETOLE, // 0 1 0 1 True if ordered and less than or equal - SETONE, // 0 1 1 0 True if ordered and operands are unequal - SETO, // 0 1 1 1 True if ordered (no nans) - SETUO, // 1 0 0 0 True if unordered: isnan(X) | isnan(Y) - SETUEQ, // 1 0 0 1 True if unordered or equal - SETUGT, // 1 0 1 0 True if unordered or greater than - SETUGE, // 1 0 1 1 True if unordered, greater than, or equal - SETULT, // 1 1 0 0 True if unordered or less than - SETULE, // 1 1 0 1 True if unordered, less than, or equal - SETUNE, // 1 1 1 0 True if unordered or not equal - SETTRUE, // 1 1 1 1 Always true (always folded) - // Don't care operations: undefined if the input is a nan. - SETFALSE2, // 1 X 0 0 0 Always false (always folded) - SETEQ, // 1 X 0 0 1 True if equal - SETGT, // 1 X 0 1 0 True if greater than - SETGE, // 1 X 0 1 1 True if greater than or equal - SETLT, // 1 X 1 0 0 True if less than - SETLE, // 1 X 1 0 1 True if less than or equal - SETNE, // 1 X 1 1 0 True if not equal - SETTRUE2, // 1 X 1 1 1 Always true (always folded) - - SETCC_INVALID // Marker value. - }; - - /// isSignedIntSetCC - Return true if this is a setcc instruction that - /// performs a signed comparison when used with integer operands. - inline bool isSignedIntSetCC(CondCode Code) { - return Code == SETGT || Code == SETGE || Code == SETLT || Code == SETLE; - } - - /// isUnsignedIntSetCC - Return true if this is a setcc instruction that - /// performs an unsigned comparison when used with integer operands. - inline bool isUnsignedIntSetCC(CondCode Code) { - return Code == SETUGT || Code == SETUGE || Code == SETULT || Code == SETULE; - } - - /// isTrueWhenEqual - Return true if the specified condition returns true if - /// the two operands to the condition are equal. Note that if one of the two - /// operands is a NaN, this value is meaningless. - inline bool isTrueWhenEqual(CondCode Cond) { - return ((int)Cond & 1) != 0; - } - - /// getUnorderedFlavor - This function returns 0 if the condition is always - /// false if an operand is a NaN, 1 if the condition is always true if the - /// operand is a NaN, and 2 if the condition is undefined if the operand is a - /// NaN. - inline unsigned getUnorderedFlavor(CondCode Cond) { - return ((int)Cond >> 3) & 3; - } - - /// getSetCCInverse - Return the operation corresponding to !(X op Y), where - /// 'op' is a valid SetCC operation. - CondCode getSetCCInverse(CondCode Operation, bool isInteger); + /// \brief Return true if the specified node is a BUILD_VECTOR node of + /// all ConstantSDNode or undef. + bool isBuildVectorOfConstantSDNodes(const SDNode *N); - /// getSetCCSwappedOperands - Return the operation corresponding to (Y op X) - /// when given the operation for (X op Y). - CondCode getSetCCSwappedOperands(CondCode Operation); - - /// getSetCCOrOperation - Return the result of a logical OR between different - /// comparisons of identical values: ((X op1 Y) | (X op2 Y)). This - /// function returns SETCC_INVALID if it is not possible to represent the - /// resultant comparison. - CondCode getSetCCOrOperation(CondCode Op1, CondCode Op2, bool isInteger); - - /// getSetCCAndOperation - Return the result of a logical AND between - /// different comparisons of identical values: ((X op1 Y) & (X op2 Y)). This - /// function returns SETCC_INVALID if it is not possible to represent the - /// resultant comparison. - CondCode getSetCCAndOperation(CondCode Op1, CondCode Op2, bool isInteger); - - //===--------------------------------------------------------------------===// - /// CvtCode enum - This enum defines the various converts CONVERT_RNDSAT - /// supports. - enum CvtCode { - CVT_FF, // Float from Float - CVT_FS, // Float from Signed - CVT_FU, // Float from Unsigned - CVT_SF, // Signed from Float - CVT_UF, // Unsigned from Float - CVT_SS, // Signed from Signed - CVT_SU, // Signed from Unsigned - CVT_US, // Unsigned from Signed - CVT_UU, // Unsigned from Unsigned - CVT_INVALID // Marker - Invalid opcode - }; -} // end llvm::ISD namespace + /// \brief Return true if the specified node is a BUILD_VECTOR node of + /// all ConstantFPSDNode or undef. + bool isBuildVectorOfConstantFPSDNodes(const SDNode *N); + /// Return true if the node has at least one operand + /// and all operands of the specified node are ISD::UNDEF. + bool allOperandsUndef(const SDNode *N); +} // end llvm:ISD namespace //===----------------------------------------------------------------------===// -/// SDValue - Unlike LLVM values, Selection DAG nodes may return multiple +/// Unlike LLVM values, Selection DAG nodes may return multiple /// values as the result of a computation. Many nodes return multiple values, /// from loads (which define a token and a return value) to ADDC (which returns /// a result and a carry value), to calls (which may return an arbitrary number @@ -813,11 +99,13 @@ namespace ISD { /// of information is represented with the SDValue value type. /// class SDValue { + friend struct DenseMapInfo; + SDNode *Node; // The node defining the value we are using. unsigned ResNo; // Which return value of the node we are using. public: - SDValue() : Node(0), ResNo(0) {} - SDValue(SDNode *node, unsigned resno) : Node(node), ResNo(resno) {} + SDValue() : Node(nullptr), ResNo(0) {} + SDValue(SDNode *node, unsigned resno); /// get the index which selects a specific result in the SDNode unsigned getResNo() const { return ResNo; } @@ -829,7 +117,7 @@ public: void setNode(SDNode *N) { Node = N; } inline SDNode *operator->() const { return Node; } - + bool operator==(const SDValue &O) const { return Node == O.Node && ResNo == O.ResNo; } @@ -837,26 +125,36 @@ public: return !operator==(O); } bool operator<(const SDValue &O) const { - return Node < O.Node || (Node == O.Node && ResNo < O.ResNo); + return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo); + } + explicit operator bool() const { + return Node != nullptr; } SDValue getValue(unsigned R) const { return SDValue(Node, R); } - // isOperandOf - Return true if this node is an operand of N. - bool isOperandOf(SDNode *N) const; + /// Return true if this node is an operand of N. + bool isOperandOf(const SDNode *N) const; - /// getValueType - Return the ValueType of the referenced return value. - /// + /// Return the ValueType of the referenced return value. inline EVT getValueType() const; - /// getValueSizeInBits - Returns the size of the value in bits. - /// + /// Return the simple ValueType of the referenced return value. + MVT getSimpleValueType() const { + return getValueType().getSimpleVT(); + } + + /// Returns the size of the value in bits. unsigned getValueSizeInBits() const { return getValueType().getSizeInBits(); } + unsigned getScalarValueSizeInBits() const { + return getValueType().getScalarType().getSizeInBits(); + } + // Forwarding methods - These forward to the corresponding methods in SDNode. inline unsigned getOpcode() const; inline unsigned getNumOperands() const; @@ -865,36 +163,38 @@ public: inline bool isTargetMemoryOpcode() const; inline bool isTargetOpcode() const; inline bool isMachineOpcode() const; + inline bool isUndef() const; inline unsigned getMachineOpcode() const; - inline const DebugLoc getDebugLoc() const; - - - /// reachesChainWithoutSideEffects - Return true if this operand (which must - /// be a chain) reaches the specified operand without crossing any - /// side-effecting instructions. In practice, this looks through token - /// factors and non-volatile loads. In order to remain efficient, this only + inline const DebugLoc &getDebugLoc() const; + inline void dump() const; + inline void dumpr() const; + + /// Return true if this operand (which must be a chain) reaches the + /// specified operand without crossing any side-effecting instructions. + /// In practice, this looks through token factors and non-volatile loads. + /// In order to remain efficient, this only /// looks a couple of nodes in, it does not do an exhaustive search. bool reachesChainWithoutSideEffects(SDValue Dest, unsigned Depth = 2) const; - /// use_empty - Return true if there are no nodes using value ResNo - /// of Node. - /// + /// Return true if there are no nodes using value ResNo of Node. inline bool use_empty() const; - /// hasOneUse - Return true if there is exactly one node using value - /// ResNo of Node. - /// + /// Return true if there is exactly one node using value ResNo of Node. inline bool hasOneUse() const; }; template<> struct DenseMapInfo { static inline SDValue getEmptyKey() { - return SDValue((SDNode*)-1, -1U); + SDValue V; + V.ResNo = -1U; + return V; } static inline SDValue getTombstoneKey() { - return SDValue((SDNode*)-1, 0); + SDValue V; + V.ResNo = -2U; + return V; } static unsigned getHashValue(const SDValue &Val) { return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^ @@ -907,22 +207,22 @@ template<> struct DenseMapInfo { template <> struct isPodLike { static const bool value = true; }; -/// simplify_type specializations - Allow casting operators to work directly on +/// Allow casting operators to work directly on /// SDValues as if they were SDNode*'s. template<> struct simplify_type { typedef SDNode* SimpleType; - static SimpleType getSimplifiedValue(const SDValue &Val) { - return static_cast(Val.getNode()); + static SimpleType getSimplifiedValue(SDValue &Val) { + return Val.getNode(); } }; template<> struct simplify_type { - typedef SDNode* SimpleType; + typedef /*const*/ SDNode* SimpleType; static SimpleType getSimplifiedValue(const SDValue &Val) { - return static_cast(Val.getNode()); + return Val.getNode(); } }; -/// SDUse - Represents a use of a SDNode. This class holds an SDValue, +/// Represents a use of a SDNode. This class holds an SDValue, /// which records the SDNode being used and the result number, a /// pointer to the SDNode using the value, and Next and Prev pointers, /// which link together all the uses of an SDNode. @@ -936,11 +236,11 @@ class SDUse { /// this operand. SDUse **Prev, *Next; - SDUse(const SDUse &U); // Do not implement - void operator=(const SDUse &U); // Do not implement + SDUse(const SDUse &U) = delete; + void operator=(const SDUse &U) = delete; public: - SDUse() : Val(), User(NULL), Prev(NULL), Next(NULL) {} + SDUse() : Val(), User(nullptr), Prev(nullptr), Next(nullptr) {} /// Normally SDUse will just implicitly convert to an SDValue that it holds. operator const SDValue&() const { return Val; } @@ -949,30 +249,30 @@ public: /// the SDValue. const SDValue &get() const { return Val; } - /// getUser - This returns the SDNode that contains this Use. + /// This returns the SDNode that contains this Use. SDNode *getUser() { return User; } - /// getNext - Get the next SDUse in the use list. + /// Get the next SDUse in the use list. SDUse *getNext() const { return Next; } - /// getNode - Convenience function for get().getNode(). + /// Convenience function for get().getNode(). SDNode *getNode() const { return Val.getNode(); } - /// getResNo - Convenience function for get().getResNo(). + /// Convenience function for get().getResNo(). unsigned getResNo() const { return Val.getResNo(); } - /// getValueType - Convenience function for get().getValueType(). + /// Convenience function for get().getValueType(). EVT getValueType() const { return Val.getValueType(); } - /// operator== - Convenience function for get().operator== + /// Convenience function for get().operator== bool operator==(const SDValue &V) const { return Val == V; } - /// operator!= - Convenience function for get().operator!= + /// Convenience function for get().operator!= bool operator!=(const SDValue &V) const { return Val != V; } - /// operator< - Convenience function for get().operator< + /// Convenience function for get().operator< bool operator<(const SDValue &V) const { return Val < V; } @@ -983,13 +283,13 @@ private: void setUser(SDNode *p) { User = p; } - /// set - Remove this use from its existing use list, assign it the + /// Remove this use from its existing use list, assign it the /// given value, and add it to the new value's node's use list. inline void set(const SDValue &V); - /// setInitial - like set, but only supports initializing a newly-allocated + /// Like set, but only supports initializing a newly-allocated /// SDUse with a non-null value. inline void setInitial(const SDValue &V); - /// setNode - like set, but only sets the Node portion of the value, + /// Like set, but only sets the Node portion of the value, /// leaving the ResNo portion unmodified. inline void setNode(SDNode *N); @@ -1010,94 +310,165 @@ private: /// SDValues as if they were SDNode*'s. template<> struct simplify_type { typedef SDNode* SimpleType; - static SimpleType getSimplifiedValue(const SDUse &Val) { - return static_cast(Val.getNode()); + static SimpleType getSimplifiedValue(SDUse &Val) { + return Val.getNode(); } }; -template<> struct simplify_type { - typedef SDNode* SimpleType; - static SimpleType getSimplifiedValue(const SDUse &Val) { - return static_cast(Val.getNode()); + +/// These are IR-level optimization flags that may be propagated to SDNodes. +/// TODO: This data structure should be shared by the IR optimizer and the +/// the backend. +struct SDNodeFlags { +private: + bool NoUnsignedWrap : 1; + bool NoSignedWrap : 1; + bool Exact : 1; + bool UnsafeAlgebra : 1; + bool NoNaNs : 1; + bool NoInfs : 1; + bool NoSignedZeros : 1; + bool AllowReciprocal : 1; + +public: + /// Default constructor turns off all optimization flags. + SDNodeFlags() { + NoUnsignedWrap = false; + NoSignedWrap = false; + Exact = false; + UnsafeAlgebra = false; + NoNaNs = false; + NoInfs = false; + NoSignedZeros = false; + AllowReciprocal = false; + } + + // These are mutators for each flag. + void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; } + void setNoSignedWrap(bool b) { NoSignedWrap = b; } + void setExact(bool b) { Exact = b; } + void setUnsafeAlgebra(bool b) { UnsafeAlgebra = b; } + void setNoNaNs(bool b) { NoNaNs = b; } + void setNoInfs(bool b) { NoInfs = b; } + void setNoSignedZeros(bool b) { NoSignedZeros = b; } + void setAllowReciprocal(bool b) { AllowReciprocal = b; } + + // These are accessors for each flag. + bool hasNoUnsignedWrap() const { return NoUnsignedWrap; } + bool hasNoSignedWrap() const { return NoSignedWrap; } + bool hasExact() const { return Exact; } + bool hasUnsafeAlgebra() const { return UnsafeAlgebra; } + bool hasNoNaNs() const { return NoNaNs; } + bool hasNoInfs() const { return NoInfs; } + bool hasNoSignedZeros() const { return NoSignedZeros; } + bool hasAllowReciprocal() const { return AllowReciprocal; } + + /// Return a raw encoding of the flags. + /// This function should only be used to add data to the NodeID value. + unsigned getRawFlags() const { + return (NoUnsignedWrap << 0) | (NoSignedWrap << 1) | (Exact << 2) | + (UnsafeAlgebra << 3) | (NoNaNs << 4) | (NoInfs << 5) | + (NoSignedZeros << 6) | (AllowReciprocal << 7); } }; - -/// SDNode - Represents one node in the SelectionDAG. +/// Represents one node in the SelectionDAG. /// class SDNode : public FoldingSetNode, public ilist_node { private: - /// NodeType - The operation that this node performs. - /// + /// The operation that this node performs. int16_t NodeType; - /// OperandsNeedDelete - This is true if OperandList was new[]'d. If true, + /// This is true if OperandList was new[]'d. If true, /// then they will be delete[]'d when the node is destroyed. uint16_t OperandsNeedDelete : 1; - /// HasDebugValue - This tracks whether this node has one or more dbg_value + /// This tracks whether this node has one or more dbg_value /// nodes corresponding to it. uint16_t HasDebugValue : 1; protected: - /// SubclassData - This member is defined by this class, but is not used for + /// This member is defined by this class, but is not used for /// anything. Subclasses can use it to hold whatever state they find useful. /// This field is initialized to zero by the ctor. uint16_t SubclassData : 14; private: - /// NodeId - Unique id per SDNode in the DAG. + /// Unique id per SDNode in the DAG. int NodeId; - /// OperandList - The values that are used by this operation. - /// + /// The values that are used by this operation. SDUse *OperandList; - /// ValueList - The types of the values this node defines. SDNode's may + /// The types of the values this node defines. SDNode's may /// define multiple values simultaneously. const EVT *ValueList; - /// UseList - List of uses for this SDNode. + /// List of uses for this SDNode. SDUse *UseList; - /// NumOperands/NumValues - The number of entries in the Operand/Value list. + /// The number of entries in the Operand/Value list. unsigned short NumOperands, NumValues; - /// debugLoc - source line information. + // The ordering of the SDNodes. It roughly corresponds to the ordering of the + // original LLVM instructions. + // This is used for turning off scheduling, because we'll forgo + // the normal scheduling algorithms and output the instructions according to + // this ordering. + unsigned IROrder; + + /// Source line information. DebugLoc debugLoc; - /// getValueTypeList - Return a pointer to the specified value type. + /// Return a pointer to the specified value type. static const EVT *getValueTypeList(EVT VT); friend class SelectionDAG; friend struct ilist_traits; public: + /// Unique and persistent id per SDNode in the DAG. + /// Used for debug printing. + uint16_t PersistentId; + //===--------------------------------------------------------------------===// // Accessors // - /// getOpcode - Return the SelectionDAG opcode value for this node. For + /// Return the SelectionDAG opcode value for this node. For /// pre-isel nodes (those for which isMachineOpcode returns false), these /// are the opcode values in the ISD and ISD namespaces. For /// post-isel opcodes, see getMachineOpcode. unsigned getOpcode() const { return (unsigned short)NodeType; } - /// isTargetOpcode - Test if this node has a target-specific opcode (in the + /// Test if this node has a target-specific opcode (in the /// \ISD namespace). bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; } - /// isTargetMemoryOpcode - Test if this node has a target-specific + /// Test if this node has a target-specific /// memory-referencing opcode (in the \ISD namespace and /// greater than FIRST_TARGET_MEMORY_OPCODE). bool isTargetMemoryOpcode() const { return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE; } - /// isMachineOpcode - Test if this node has a post-isel opcode, directly + /// Return true if the type of the node type undefined. + bool isUndef() const { return NodeType == ISD::UNDEF; } + + /// Test if this node is a memory intrinsic (with valid pointer information). + /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for + /// non-memory intrinsics (with chains) that are not really instances of + /// MemSDNode. For such nodes, we need some extra state to determine the + /// proper classof relationship. + bool isMemIntrinsic() const { + return (NodeType == ISD::INTRINSIC_W_CHAIN || + NodeType == ISD::INTRINSIC_VOID) && ((SubclassData >> 13) & 1); + } + + /// Test if this node has a post-isel opcode, directly /// corresponding to a MachineInstr opcode. bool isMachineOpcode() const { return NodeType < 0; } - /// getMachineOpcode - This may only be called if isMachineOpcode returns + /// This may only be called if isMachineOpcode returns /// true. It returns the MachineInstr opcode value that the node's opcode /// corresponds to. unsigned getMachineOpcode() const { @@ -1105,42 +476,44 @@ public: return ~NodeType; } - /// getHasDebugValue - get this bit. + /// Get this bit. bool getHasDebugValue() const { return HasDebugValue; } - /// setHasDebugValue - set this bit. + /// Set this bit. void setHasDebugValue(bool b) { HasDebugValue = b; } - /// use_empty - Return true if there are no uses of this node. - /// - bool use_empty() const { return UseList == NULL; } + /// Return true if there are no uses of this node. + bool use_empty() const { return UseList == nullptr; } - /// hasOneUse - Return true if there is exactly one use of this node. - /// + /// Return true if there is exactly one use of this node. bool hasOneUse() const { - return !use_empty() && llvm::next(use_begin()) == use_end(); + return !use_empty() && std::next(use_begin()) == use_end(); } - /// use_size - Return the number of uses of this node. This method takes + /// Return the number of uses of this node. This method takes /// time proportional to the number of uses. - /// size_t use_size() const { return std::distance(use_begin(), use_end()); } - /// getNodeId - Return the unique node id. - /// + /// Return the unique node id. int getNodeId() const { return NodeId; } - /// setNodeId - Set unique node id. + /// Set unique node id. void setNodeId(int Id) { NodeId = Id; } - /// getDebugLoc - Return the source location info. - const DebugLoc getDebugLoc() const { return debugLoc; } + /// Return the node ordering. + unsigned getIROrder() const { return IROrder; } + + /// Set the node ordering. + void setIROrder(unsigned Order) { IROrder = Order; } + + /// Return the source location info. + const DebugLoc &getDebugLoc() const { return debugLoc; } - /// setDebugLoc - Set source location info. Try to avoid this, putting + /// Set source location info. Try to avoid this, putting /// it in the constructor is preferable. - void setDebugLoc(const DebugLoc dl) { debugLoc = dl; } + void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); } - /// use_iterator - This class provides iterator support for SDUse + /// This class provides iterator support for SDUse /// operands that use a specific SDNode. class use_iterator : public std::iterator { @@ -1155,7 +528,7 @@ public: SDUse, ptrdiff_t>::pointer pointer; use_iterator(const use_iterator &I) : Op(I.Op) {} - use_iterator() : Op(0) {} + use_iterator() : Op(nullptr) {} bool operator==(const use_iterator &x) const { return Op == x.Op; @@ -1164,8 +537,8 @@ public: return !operator==(x); } - /// atEnd - return true if this iterator is at the end of uses list. - bool atEnd() const { return Op == 0; } + /// Return true if this iterator is at the end of uses list. + bool atEnd() const { return Op == nullptr; } // Iterator traversal: forward iteration only. use_iterator &operator++() { // Preincrement @@ -1188,53 +561,71 @@ public: SDUse &getUse() const { return *Op; } - /// getOperandNo - Retrieve the operand # of this use in its user. - /// + /// Retrieve the operand # of this use in its user. unsigned getOperandNo() const { assert(Op && "Cannot dereference end iterator!"); return (unsigned)(Op - Op->getUser()->OperandList); } }; - /// use_begin/use_end - Provide iteration support to walk over all uses - /// of an SDNode. - + /// Provide iteration support to walk over all uses of an SDNode. use_iterator use_begin() const { return use_iterator(UseList); } - static use_iterator use_end() { return use_iterator(0); } + static use_iterator use_end() { return use_iterator(nullptr); } + inline iterator_range uses() { + return make_range(use_begin(), use_end()); + } + inline iterator_range uses() const { + return make_range(use_begin(), use_end()); + } - /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the - /// indicated value. This method ignores uses of other values defined by this - /// operation. + /// Return true if there are exactly NUSES uses of the indicated value. + /// This method ignores uses of other values defined by this operation. bool hasNUsesOfValue(unsigned NUses, unsigned Value) const; - /// hasAnyUseOfValue - Return true if there are any use of the indicated - /// value. This method ignores uses of other values defined by this operation. + /// Return true if there are any use of the indicated value. + /// This method ignores uses of other values defined by this operation. bool hasAnyUseOfValue(unsigned Value) const; - /// isOnlyUserOf - Return true if this node is the only use of N. - /// - bool isOnlyUserOf(SDNode *N) const; + /// Return true if this node is the only use of N. + bool isOnlyUserOf(const SDNode *N) const; - /// isOperandOf - Return true if this node is an operand of N. - /// - bool isOperandOf(SDNode *N) const; + /// Return true if this node is an operand of N. + bool isOperandOf(const SDNode *N) const; - /// isPredecessorOf - Return true if this node is a predecessor of N. This - /// node is either an operand of N or it can be reached by recursively + /// Return true if this node is a predecessor of N. + /// NOTE: Implemented on top of hasPredecessor and every bit as + /// expensive. Use carefully. + bool isPredecessorOf(const SDNode *N) const { + return N->hasPredecessor(this); + } + + /// Return true if N is a predecessor of this node. + /// N is either an operand of this node, or can be reached by recursively /// traversing up the operands. - /// NOTE: this is an expensive method. Use it carefully. - bool isPredecessorOf(SDNode *N) const; + /// NOTE: This is an expensive method. Use it carefully. + bool hasPredecessor(const SDNode *N) const; - /// getNumOperands - Return the number of values used by this operation. - /// + /// Return true if N is a predecessor of this node. + /// N is either an operand of this node, or can be reached by recursively + /// traversing up the operands. + /// In this helper the Visited and worklist sets are held externally to + /// cache predecessors over multiple invocations. If you want to test for + /// multiple predecessors this method is preferable to multiple calls to + /// hasPredecessor. Be sure to clear Visited and Worklist if the DAG + /// changes. + /// NOTE: This is still very expensive. Use carefully. + bool hasPredecessorHelper(const SDNode *N, + SmallPtrSetImpl &Visited, + SmallVectorImpl &Worklist) const; + + /// Return the number of values used by this operation. unsigned getNumOperands() const { return NumOperands; } - /// getConstantOperandVal - Helper method returns the integer value of a - /// ConstantSDNode operand. + /// Helper method returns the integer value of a ConstantSDNode operand. uint64_t getConstantOperandVal(unsigned Num) const; const SDValue &getOperand(unsigned Num) const { @@ -1245,51 +636,67 @@ public: typedef SDUse* op_iterator; op_iterator op_begin() const { return OperandList; } op_iterator op_end() const { return OperandList+NumOperands; } + ArrayRef ops() const { return makeArrayRef(op_begin(), op_end()); } + + /// Iterator for directly iterating over the operand SDValue's. + struct value_op_iterator + : iterator_adaptor_base { + explicit value_op_iterator(SDUse *U = nullptr) + : iterator_adaptor_base(U) {} + + const SDValue &operator*() const { return I->get(); } + }; + + iterator_range op_values() const { + return make_range(value_op_iterator(op_begin()), + value_op_iterator(op_end())); + } SDVTList getVTList() const { SDVTList X = { ValueList, NumValues }; return X; } - /// getFlaggedNode - If this node has a flag operand, return the node - /// to which the flag operand points. Otherwise return NULL. - SDNode *getFlaggedNode() const { + /// If this node has a glue operand, return the node + /// to which the glue operand points. Otherwise return NULL. + SDNode *getGluedNode() const { if (getNumOperands() != 0 && - getOperand(getNumOperands()-1).getValueType().getSimpleVT() == MVT::Flag) + getOperand(getNumOperands()-1).getValueType() == MVT::Glue) return getOperand(getNumOperands()-1).getNode(); - return 0; + return nullptr; } - // If this is a pseudo op, like copyfromreg, look to see if there is a - // real target node flagged to it. If so, return the target node. - const SDNode *getFlaggedMachineNode() const { - const SDNode *FoundNode = this; - - // Climb up flag edges until a machine-opcode node is found, or the - // end of the chain is reached. - while (!FoundNode->isMachineOpcode()) { - const SDNode *N = FoundNode->getFlaggedNode(); - if (!N) break; - FoundNode = N; - } - - return FoundNode; + /// If this node has a glue value with a user, return + /// the user (there is at most one). Otherwise return NULL. + SDNode *getGluedUser() const { + for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI) + if (UI.getUse().get().getValueType() == MVT::Glue) + return *UI; + return nullptr; } - /// getNumValues - Return the number of values defined/returned by this - /// operator. - /// + /// This could be defined as a virtual function and implemented more simply + /// and directly, but it is not to avoid creating a vtable for this class. + const SDNodeFlags *getFlags() const; + + /// Return the number of values defined/returned by this operator. unsigned getNumValues() const { return NumValues; } - /// getValueType - Return the type of a specified result. - /// + /// Return the type of a specified result. EVT getValueType(unsigned ResNo) const { assert(ResNo < NumValues && "Illegal result number!"); return ValueList[ResNo]; } - /// getValueSizeInBits - Returns MVT::getSizeInBits(getValueType(ResNo)). - /// + /// Return the type of a specified result as a simple type. + MVT getSimpleValueType(unsigned ResNo) const { + return getValueType(ResNo).getSimpleVT(); + } + + /// Returns MVT::getSizeInBits(getValueType(ResNo)). unsigned getValueSizeInBits(unsigned ResNo) const { return getValueType(ResNo).getSizeInBits(); } @@ -1298,72 +705,66 @@ public: value_iterator value_begin() const { return ValueList; } value_iterator value_end() const { return ValueList+NumValues; } - /// getOperationName - Return the opcode of this operation for printing. - /// - std::string getOperationName(const SelectionDAG *G = 0) const; + /// Return the opcode of this operation for printing. + std::string getOperationName(const SelectionDAG *G = nullptr) const; static const char* getIndexedModeName(ISD::MemIndexedMode AM); void print_types(raw_ostream &OS, const SelectionDAG *G) const; void print_details(raw_ostream &OS, const SelectionDAG *G) const; - void print(raw_ostream &OS, const SelectionDAG *G = 0) const; - void printr(raw_ostream &OS, const SelectionDAG *G = 0) const; + void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const; + void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const; - /// printrFull - Print a SelectionDAG node and all children down to + /// Print a SelectionDAG node and all children down to /// the leaves. The given SelectionDAG allows target-specific nodes /// to be printed in human-readable form. Unlike printr, this will /// print the whole DAG, including children that appear multiple /// times. /// - void printrFull(raw_ostream &O, const SelectionDAG *G = 0) const; + void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const; - /// printrWithDepth - Print a SelectionDAG node and children up to + /// Print a SelectionDAG node and children up to /// depth "depth." The given SelectionDAG allows target-specific /// nodes to be printed in human-readable form. Unlike printr, this /// will print children that appear multiple times wherever they are /// used. /// - void printrWithDepth(raw_ostream &O, const SelectionDAG *G = 0, + void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr, unsigned depth = 100) const; - /// dump - Dump this node, for debugging. + /// Dump this node, for debugging. void dump() const; - /// dumpr - Dump (recursively) this node and its use-def subgraph. + /// Dump (recursively) this node and its use-def subgraph. void dumpr() const; - /// dump - Dump this node, for debugging. + /// Dump this node, for debugging. /// The given SelectionDAG allows target-specific nodes to be printed /// in human-readable form. void dump(const SelectionDAG *G) const; - /// dumpr - Dump (recursively) this node and its use-def subgraph. + /// Dump (recursively) this node and its use-def subgraph. /// The given SelectionDAG allows target-specific nodes to be printed /// in human-readable form. void dumpr(const SelectionDAG *G) const; - /// dumprFull - printrFull to dbgs(). The given SelectionDAG allows + /// printrFull to dbgs(). The given SelectionDAG allows /// target-specific nodes to be printed in human-readable form. /// Unlike dumpr, this will print the whole DAG, including children /// that appear multiple times. - /// - void dumprFull(const SelectionDAG *G = 0) const; + void dumprFull(const SelectionDAG *G = nullptr) const; - /// dumprWithDepth - printrWithDepth to dbgs(). The given + /// printrWithDepth to dbgs(). The given /// SelectionDAG allows target-specific nodes to be printed in /// human-readable form. Unlike dumpr, this will print children /// that appear multiple times wherever they are used. /// - void dumprWithDepth(const SelectionDAG *G = 0, unsigned depth = 100) const; + void dumprWithDepth(const SelectionDAG *G = nullptr, + unsigned depth = 100) const; - - static bool classof(const SDNode *) { return true; } - - /// Profile - Gather unique data for the node. - /// + /// Gather unique data for the node. void Profile(FoldingSetNodeID &ID) const; - /// addUse - This method should only be used by the SDUse class. - /// + /// This method should only be used by the SDUse class. void addUse(SDUse &U) { U.addToList(&UseList); } protected: @@ -1372,15 +773,20 @@ protected: return Ret; } - SDNode(unsigned Opc, const DebugLoc dl, SDVTList VTs, const SDValue *Ops, - unsigned NumOps) - : NodeType(Opc), OperandsNeedDelete(true), HasDebugValue(false), - SubclassData(0), NodeId(-1), - OperandList(NumOps ? new SDUse[NumOps] : 0), - ValueList(VTs.VTs), UseList(NULL), - NumOperands(NumOps), NumValues(VTs.NumVTs), - debugLoc(dl) { - for (unsigned i = 0; i != NumOps; ++i) { + SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, + ArrayRef Ops) + : NodeType(Opc), OperandsNeedDelete(true), HasDebugValue(false), + SubclassData(0), NodeId(-1), + OperandList(Ops.size() ? new SDUse[Ops.size()] : nullptr), + ValueList(VTs.VTs), UseList(nullptr), NumOperands(Ops.size()), + NumValues(VTs.NumVTs), IROrder(Order), debugLoc(std::move(dl)) { + assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor"); + assert(NumOperands == Ops.size() && + "NumOperands wasn't wide enough for its operands!"); + assert(NumValues == VTs.NumVTs && + "NumValues wasn't wide enough for its operands!"); + for (unsigned i = 0; i != Ops.size(); ++i) { + assert(OperandList && "no operands available"); OperandList[i].setUser(this); OperandList[i].setInitial(Ops[i]); } @@ -1389,13 +795,17 @@ protected: /// This constructor adds no operands itself; operands can be /// set later with InitOperands. - SDNode(unsigned Opc, const DebugLoc dl, SDVTList VTs) - : NodeType(Opc), OperandsNeedDelete(false), HasDebugValue(false), - SubclassData(0), NodeId(-1), OperandList(0), ValueList(VTs.VTs), - UseList(NULL), NumOperands(0), NumValues(VTs.NumVTs), - debugLoc(dl) {} + SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs) + : NodeType(Opc), OperandsNeedDelete(false), HasDebugValue(false), + SubclassData(0), NodeId(-1), OperandList(nullptr), ValueList(VTs.VTs), + UseList(nullptr), NumOperands(0), NumValues(VTs.NumVTs), + IROrder(Order), debugLoc(std::move(dl)) { + assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor"); + assert(NumValues == VTs.NumVTs && + "NumValues wasn't wide enough for its operands!"); + } - /// InitOperands - Initialize the operands list of this with 1 operand. + /// Initialize the operands list of this with 1 operand. void InitOperands(SDUse *Ops, const SDValue &Op0) { Ops[0].setUser(this); Ops[0].setInitial(Op0); @@ -1404,7 +814,7 @@ protected: checkForCycles(this); } - /// InitOperands - Initialize the operands list of this with 2 operands. + /// Initialize the operands list of this with 2 operands. void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1) { Ops[0].setUser(this); Ops[0].setInitial(Op0); @@ -1415,7 +825,7 @@ protected: checkForCycles(this); } - /// InitOperands - Initialize the operands list of this with 3 operands. + /// Initialize the operands list of this with 3 operands. void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1, const SDValue &Op2) { Ops[0].setUser(this); @@ -1429,7 +839,7 @@ protected: checkForCycles(this); } - /// InitOperands - Initialize the operands list of this with 4 operands. + /// Initialize the operands list of this with 4 operands. void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1, const SDValue &Op2, const SDValue &Op3) { Ops[0].setUser(this); @@ -1445,25 +855,80 @@ protected: checkForCycles(this); } - /// InitOperands - Initialize the operands list of this with N operands. + /// Initialize the operands list of this with N operands. void InitOperands(SDUse *Ops, const SDValue *Vals, unsigned N) { for (unsigned i = 0; i != N; ++i) { Ops[i].setUser(this); Ops[i].setInitial(Vals[i]); } NumOperands = N; + assert(NumOperands == N && + "NumOperands wasn't wide enough for its operands!"); OperandList = Ops; checkForCycles(this); } - /// DropOperands - Release the operands and set this node to have - /// zero operands. + /// Release the operands and set this node to have zero operands. void DropOperands(); }; +/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed +/// into SDNode creation functions. +/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted +/// from the original Instruction, and IROrder is the ordinal position of +/// the instruction. +/// When an SDNode is created after the DAG is being built, both DebugLoc and +/// the IROrder are propagated from the original SDNode. +/// So SDLoc class provides two constructors besides the default one, one to +/// be used by the DAGBuilder, the other to be used by others. +class SDLoc { +private: + // Ptr could be used for either Instruction* or SDNode*. It is used for + // Instruction* if IROrder is not -1. + const void *Ptr; + int IROrder; + +public: + SDLoc() : Ptr(nullptr), IROrder(0) {} + SDLoc(const SDNode *N) : Ptr(N), IROrder(-1) { + assert(N && "null SDNode"); + } + SDLoc(const SDValue V) : Ptr(V.getNode()), IROrder(-1) { + assert(Ptr && "null SDNode"); + } + SDLoc(const Instruction *I, int Order) : Ptr(I), IROrder(Order) { + assert(Order >= 0 && "bad IROrder"); + } + unsigned getIROrder() { + if (IROrder >= 0 || Ptr == nullptr) { + return (unsigned)IROrder; + } + const SDNode *N = (const SDNode*)(Ptr); + return N->getIROrder(); + } + DebugLoc getDebugLoc() { + if (!Ptr) { + return DebugLoc(); + } + if (IROrder >= 0) { + const Instruction *I = (const Instruction*)(Ptr); + return I->getDebugLoc(); + } + const SDNode *N = (const SDNode*)(Ptr); + return N->getDebugLoc(); + } +}; + // Define inline functions from the SDValue class. +inline SDValue::SDValue(SDNode *node, unsigned resno) + : Node(node), ResNo(resno) { + assert((!Node || ResNo < Node->getNumValues()) && + "Invalid result number for the given node!"); + assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps."); +} + inline unsigned SDValue::getOpcode() const { return Node->getOpcode(); } @@ -1491,16 +956,24 @@ inline bool SDValue::isMachineOpcode() const { inline unsigned SDValue::getMachineOpcode() const { return Node->getMachineOpcode(); } +inline bool SDValue::isUndef() const { + return Node->isUndef(); +} inline bool SDValue::use_empty() const { return !Node->hasAnyUseOfValue(ResNo); } inline bool SDValue::hasOneUse() const { return Node->hasNUsesOfValue(1, ResNo); } -inline const DebugLoc SDValue::getDebugLoc() const { +inline const DebugLoc &SDValue::getDebugLoc() const { return Node->getDebugLoc(); } - +inline void SDValue::dump() const { + return Node->dump(); +} +inline void SDValue::dumpr() const { + return Node->dumpr(); +} // Define inline functions from the SDUse class. inline void SDUse::set(const SDValue &V) { @@ -1520,92 +993,142 @@ inline void SDUse::setNode(SDNode *N) { if (N) N->addUse(*this); } -/// UnarySDNode - This class is used for single-operand SDNodes. This is solely +/// This class is used for single-operand SDNodes. This is solely /// to allow co-allocation of node operands with the node itself. class UnarySDNode : public SDNode { SDUse Op; public: - UnarySDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, SDValue X) - : SDNode(Opc, dl, VTs) { + UnarySDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, + SDValue X) + : SDNode(Opc, Order, dl, VTs) { InitOperands(&Op, X); } }; -/// BinarySDNode - This class is used for two-operand SDNodes. This is solely +/// This class is used for two-operand SDNodes. This is solely /// to allow co-allocation of node operands with the node itself. class BinarySDNode : public SDNode { SDUse Ops[2]; public: - BinarySDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, SDValue X, SDValue Y) - : SDNode(Opc, dl, VTs) { + BinarySDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, + SDValue X, SDValue Y) + : SDNode(Opc, Order, dl, VTs) { InitOperands(Ops, X, Y); } }; -/// TernarySDNode - This class is used for three-operand SDNodes. This is solely +/// Returns true if the opcode is a binary operation with flags. +static bool isBinOpWithFlags(unsigned Opcode) { + switch (Opcode) { + case ISD::SDIV: + case ISD::UDIV: + case ISD::SRA: + case ISD::SRL: + case ISD::MUL: + case ISD::ADD: + case ISD::SUB: + case ISD::SHL: + case ISD::FADD: + case ISD::FDIV: + case ISD::FMUL: + case ISD::FREM: + case ISD::FSUB: + return true; + default: + return false; + } +} + +/// This class is an extension of BinarySDNode +/// used from those opcodes that have associated extra flags. +class BinaryWithFlagsSDNode : public BinarySDNode { +public: + SDNodeFlags Flags; + BinaryWithFlagsSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, + SDValue X, SDValue Y, const SDNodeFlags &NodeFlags) + : BinarySDNode(Opc, Order, dl, VTs, X, Y), Flags(NodeFlags) {} + static bool classof(const SDNode *N) { + return isBinOpWithFlags(N->getOpcode()); + } +}; + +/// This class is used for three-operand SDNodes. This is solely /// to allow co-allocation of node operands with the node itself. class TernarySDNode : public SDNode { SDUse Ops[3]; public: - TernarySDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, SDValue X, SDValue Y, - SDValue Z) - : SDNode(Opc, dl, VTs) { + TernarySDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, + SDValue X, SDValue Y, SDValue Z) + : SDNode(Opc, Order, dl, VTs) { InitOperands(Ops, X, Y, Z); } }; -/// HandleSDNode - This class is used to form a handle around another node that -/// is persistant and is updated across invocations of replaceAllUsesWith on its +/// This class is used to form a handle around another node that +/// is persistent and is updated across invocations of replaceAllUsesWith on its /// operand. This node should be directly created by end-users and not added to /// the AllNodes list. class HandleSDNode : public SDNode { SDUse Op; public: - // FIXME: Remove the "noinline" attribute once is - // fixed. -#ifdef __GNUC__ - explicit __attribute__((__noinline__)) HandleSDNode(SDValue X) -#else explicit HandleSDNode(SDValue X) -#endif - : SDNode(ISD::HANDLENODE, DebugLoc::getUnknownLoc(), - getSDVTList(MVT::Other)) { + : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) { + // HandleSDNodes are never inserted into the DAG, so they won't be + // auto-numbered. Use ID 65535 as a sentinel. + PersistentId = 0xffff; InitOperands(&Op, X); } ~HandleSDNode(); const SDValue &getValue() const { return Op; } }; -/// Abstact virtual class for operations for memory operations +class AddrSpaceCastSDNode : public UnarySDNode { +private: + unsigned SrcAddrSpace; + unsigned DestAddrSpace; + +public: + AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT, SDValue X, + unsigned SrcAS, unsigned DestAS); + + unsigned getSrcAddressSpace() const { return SrcAddrSpace; } + unsigned getDestAddressSpace() const { return DestAddrSpace; } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::ADDRSPACECAST; + } +}; + +/// This is an abstract virtual class for memory operations. class MemSDNode : public SDNode { private: - // MemoryVT - VT of in-memory value. + // VT of in-memory value. EVT MemoryVT; protected: - /// MMO - Memory reference information. + /// Memory reference information. MachineMemOperand *MMO; public: - MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, EVT MemoryVT, - MachineMemOperand *MMO); + MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, + EVT MemoryVT, MachineMemOperand *MMO); - MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, const SDValue *Ops, - unsigned NumOps, EVT MemoryVT, MachineMemOperand *MMO); + MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, + ArrayRef Ops, EVT MemoryVT, MachineMemOperand *MMO); bool readMem() const { return MMO->isLoad(); } bool writeMem() const { return MMO->isStore(); } /// Returns alignment and volatility of the memory access - unsigned getOriginalAlignment() const { + unsigned getOriginalAlignment() const { return MMO->getBaseAlignment(); } unsigned getAlignment() const { return MMO->getAlignment(); } - /// getRawSubclassData - Return the SubclassData value, which contains an + /// Return the SubclassData value, which contains an /// encoding of the volatile flag, as well as bits used by subclasses. This /// function should only be used to compute a FoldingSetNodeID value. unsigned getRawSubclassData() const { @@ -1616,19 +1139,41 @@ public: // with MachineMemOperand information. bool isVolatile() const { return (SubclassData >> 5) & 1; } bool isNonTemporal() const { return (SubclassData >> 6) & 1; } + bool isInvariant() const { return (SubclassData >> 7) & 1; } - /// Returns the SrcValue and offset that describes the location of the access - const Value *getSrcValue() const { return MMO->getValue(); } + AtomicOrdering getOrdering() const { + return AtomicOrdering((SubclassData >> 8) & 15); + } + SynchronizationScope getSynchScope() const { + return SynchronizationScope((SubclassData >> 12) & 1); + } + + // Returns the offset from the location of the access. int64_t getSrcValueOffset() const { return MMO->getOffset(); } - /// getMemoryVT - Return the type of the in-memory value. + /// Returns the AA info that describes the dereference. + AAMDNodes getAAInfo() const { return MMO->getAAInfo(); } + + /// Returns the Ranges that describes the dereference. + const MDNode *getRanges() const { return MMO->getRanges(); } + + /// Return the type of the in-memory value. EVT getMemoryVT() const { return MemoryVT; } - /// getMemOperand - Return a MachineMemOperand object describing the memory + /// Return a MachineMemOperand object describing the memory /// reference performed by operation. MachineMemOperand *getMemOperand() const { return MMO; } - /// refineAlignment - Update this MemSDNode's MachineMemOperand information + const MachinePointerInfo &getPointerInfo() const { + return MMO->getPointerInfo(); + } + + /// Return the address space for the associated pointer + unsigned getAddressSpace() const { + return getPointerInfo().getAddrSpace(); + } + + /// Update this MemSDNode's MachineMemOperand information /// to reflect the alignment of NewMMO, if it has a greater alignment. /// This must only be used when the new alignment applies to all users of /// this MachineMemOperand. @@ -1642,13 +1187,14 @@ public: } // Methods to support isa and dyn_cast - static bool classof(const MemSDNode *) { return true; } static bool classof(const SDNode *N) { // For some targets, we lower some target intrinsics to a MemIntrinsicNode // with either an intrinsic or a target opcode. return N->getOpcode() == ISD::LOAD || N->getOpcode() == ISD::STORE || + N->getOpcode() == ISD::PREFETCH || N->getOpcode() == ISD::ATOMIC_CMP_SWAP || + N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS || N->getOpcode() == ISD::ATOMIC_SWAP || N->getOpcode() == ISD::ATOMIC_LOAD_ADD || N->getOpcode() == ISD::ATOMIC_LOAD_SUB || @@ -1660,15 +1206,45 @@ public: N->getOpcode() == ISD::ATOMIC_LOAD_MAX || N->getOpcode() == ISD::ATOMIC_LOAD_UMIN || N->getOpcode() == ISD::ATOMIC_LOAD_UMAX || + N->getOpcode() == ISD::ATOMIC_LOAD || + N->getOpcode() == ISD::ATOMIC_STORE || + N->getOpcode() == ISD::MLOAD || + N->getOpcode() == ISD::MSTORE || + N->getOpcode() == ISD::MGATHER || + N->getOpcode() == ISD::MSCATTER || + N->isMemIntrinsic() || N->isTargetMemoryOpcode(); } }; -/// AtomicSDNode - A SDNode reprenting atomic operations. -/// +/// This is an SDNode representing atomic operations. class AtomicSDNode : public MemSDNode { SDUse Ops[4]; + /// For cmpxchg instructions, the ordering requirements when a store does not + /// occur. + AtomicOrdering FailureOrdering; + + void InitAtomic(AtomicOrdering SuccessOrdering, + AtomicOrdering FailureOrdering, + SynchronizationScope SynchScope) { + // This must match encodeMemSDNodeFlags() in SelectionDAG.cpp. + assert((SuccessOrdering & 15) == SuccessOrdering && + "Ordering may not require more than 4 bits!"); + assert((FailureOrdering & 15) == FailureOrdering && + "Ordering may not require more than 4 bits!"); + assert((SynchScope & 1) == SynchScope && + "SynchScope may not require more than 1 bit!"); + SubclassData |= SuccessOrdering << 8; + SubclassData |= SynchScope << 12; + this->FailureOrdering = FailureOrdering; + assert(getSuccessOrdering() == SuccessOrdering && + "Ordering encoding error!"); + assert(getFailureOrdering() == FailureOrdering && + "Ordering encoding error!"); + assert(getSynchScope() == SynchScope && "Synch-scope encoding error!"); + } + public: // Opc: opcode for atomic // VTL: value type list @@ -1678,35 +1254,66 @@ public: // Swp: swap value // SrcVal: address to update as a Value (used for MemOperand) // Align: alignment of memory - AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT, - SDValue Chain, SDValue Ptr, - SDValue Cmp, SDValue Swp, MachineMemOperand *MMO) - : MemSDNode(Opc, dl, VTL, MemVT, MMO) { - assert(readMem() && "Atomic MachineMemOperand is not a load!"); - assert(writeMem() && "Atomic MachineMemOperand is not a store!"); + AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL, + EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, + MachineMemOperand *MMO, AtomicOrdering Ordering, + SynchronizationScope SynchScope) + : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { + InitAtomic(Ordering, Ordering, SynchScope); InitOperands(Ops, Chain, Ptr, Cmp, Swp); } - AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT, + AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL, + EVT MemVT, SDValue Chain, SDValue Ptr, - SDValue Val, MachineMemOperand *MMO) - : MemSDNode(Opc, dl, VTL, MemVT, MMO) { - assert(readMem() && "Atomic MachineMemOperand is not a load!"); - assert(writeMem() && "Atomic MachineMemOperand is not a store!"); + SDValue Val, MachineMemOperand *MMO, + AtomicOrdering Ordering, SynchronizationScope SynchScope) + : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { + InitAtomic(Ordering, Ordering, SynchScope); InitOperands(Ops, Chain, Ptr, Val); } + AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL, + EVT MemVT, + SDValue Chain, SDValue Ptr, + MachineMemOperand *MMO, + AtomicOrdering Ordering, SynchronizationScope SynchScope) + : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { + InitAtomic(Ordering, Ordering, SynchScope); + InitOperands(Ops, Chain, Ptr); + } + AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL, EVT MemVT, + const SDValue* AllOps, SDUse *DynOps, unsigned NumOps, + MachineMemOperand *MMO, + AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, + SynchronizationScope SynchScope) + : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { + InitAtomic(SuccessOrdering, FailureOrdering, SynchScope); + assert((DynOps || NumOps <= array_lengthof(Ops)) && + "Too many ops for internal storage!"); + InitOperands(DynOps ? DynOps : Ops, AllOps, NumOps); + } const SDValue &getBasePtr() const { return getOperand(1); } const SDValue &getVal() const { return getOperand(2); } + AtomicOrdering getSuccessOrdering() const { + return getOrdering(); + } + + // Not quite enough room in SubclassData for everything, so failure gets its + // own field. + AtomicOrdering getFailureOrdering() const { + return FailureOrdering; + } + bool isCompareAndSwap() const { unsigned Op = getOpcode(); - return Op == ISD::ATOMIC_CMP_SWAP; + return Op == ISD::ATOMIC_CMP_SWAP || Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS; } // Methods to support isa and dyn_cast - static bool classof(const AtomicSDNode *) { return true; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::ATOMIC_CMP_SWAP || + N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS || N->getOpcode() == ISD::ATOMIC_SWAP || N->getOpcode() == ISD::ATOMIC_LOAD_ADD || N->getOpcode() == ISD::ATOMIC_LOAD_SUB || @@ -1717,34 +1324,36 @@ public: N->getOpcode() == ISD::ATOMIC_LOAD_MIN || N->getOpcode() == ISD::ATOMIC_LOAD_MAX || N->getOpcode() == ISD::ATOMIC_LOAD_UMIN || - N->getOpcode() == ISD::ATOMIC_LOAD_UMAX; + N->getOpcode() == ISD::ATOMIC_LOAD_UMAX || + N->getOpcode() == ISD::ATOMIC_LOAD || + N->getOpcode() == ISD::ATOMIC_STORE; } }; -/// MemIntrinsicSDNode - This SDNode is used for target intrinsics that touch +/// This SDNode is used for target intrinsics that touch /// memory and need an associated MachineMemOperand. Its opcode may be -/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, or a target-specific opcode with a -/// value not less than FIRST_TARGET_MEMORY_OPCODE. +/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode +/// with a value not less than FIRST_TARGET_MEMORY_OPCODE. class MemIntrinsicSDNode : public MemSDNode { public: - MemIntrinsicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, - const SDValue *Ops, unsigned NumOps, - EVT MemoryVT, MachineMemOperand *MMO) - : MemSDNode(Opc, dl, VTs, Ops, NumOps, MemoryVT, MMO) { + MemIntrinsicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, + ArrayRef Ops, EVT MemoryVT, + MachineMemOperand *MMO) + : MemSDNode(Opc, Order, dl, VTs, Ops, MemoryVT, MMO) { + SubclassData |= 1u << 13; } // Methods to support isa and dyn_cast - static bool classof(const MemIntrinsicSDNode *) { return true; } static bool classof(const SDNode *N) { // We lower some target intrinsics to their target opcode // early a node with a target opcode can be of this class - return N->getOpcode() == ISD::INTRINSIC_W_CHAIN || - N->getOpcode() == ISD::INTRINSIC_VOID || + return N->isMemIntrinsic() || + N->getOpcode() == ISD::PREFETCH || N->isTargetMemoryOpcode(); } }; -/// ShuffleVectorSDNode - This SDNode is used to implement the code generator +/// This SDNode is used to implement the code generator /// support for the llvm IR shufflevector instruction. It combines elements /// from two input vectors into a new input vector, with the selection and /// ordering of elements determined by an array of integers, referred to as @@ -1760,48 +1369,62 @@ class ShuffleVectorSDNode : public SDNode { const int *Mask; protected: friend class SelectionDAG; - ShuffleVectorSDNode(EVT VT, DebugLoc dl, SDValue N1, SDValue N2, - const int *M) - : SDNode(ISD::VECTOR_SHUFFLE, dl, getSDVTList(VT)), Mask(M) { + ShuffleVectorSDNode(EVT VT, unsigned Order, DebugLoc dl, SDValue N1, + SDValue N2, const int *M) + : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) { InitOperands(Ops, N1, N2); } public: - void getMask(SmallVectorImpl &M) const { + ArrayRef getMask() const { EVT VT = getValueType(0); - M.clear(); - for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) - M.push_back(Mask[i]); + return makeArrayRef(Mask, VT.getVectorNumElements()); } int getMaskElt(unsigned Idx) const { assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!"); return Mask[Idx]; } - + bool isSplat() const { return isSplatMask(Mask, getValueType(0)); } - int getSplatIndex() const { + int getSplatIndex() const { assert(isSplat() && "Cannot get splat index for non-splat!"); EVT VT = getValueType(0); for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) { - if (Mask[i] != -1) + if (Mask[i] >= 0) return Mask[i]; } - return -1; + llvm_unreachable("Splat with all undef indices?"); } static bool isSplatMask(const int *Mask, EVT VT); - static bool classof(const ShuffleVectorSDNode *) { return true; } + /// Change values in a shuffle permute mask assuming + /// the two vector operands have swapped position. + static void commuteMask(SmallVectorImpl &Mask) { + unsigned NumElems = Mask.size(); + for (unsigned i = 0; i != NumElems; ++i) { + int idx = Mask[i]; + if (idx < 0) + continue; + else if (idx < (int)NumElems) + Mask[i] = idx + NumElems; + else + Mask[i] = idx - NumElems; + } + } + static bool classof(const SDNode *N) { return N->getOpcode() == ISD::VECTOR_SHUFFLE; } }; - + class ConstantSDNode : public SDNode { const ConstantInt *Value; friend class SelectionDAG; - ConstantSDNode(bool isTarget, const ConstantInt *val, EVT VT) + ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, + DebugLoc DL, EVT VT) : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, - DebugLoc::getUnknownLoc(), getSDVTList(VT)), Value(val) { + 0, DL, getSDVTList(VT)), Value(val) { + SubclassData |= (uint16_t)isOpaque; } public: @@ -1810,10 +1433,12 @@ public: uint64_t getZExtValue() const { return Value->getZExtValue(); } int64_t getSExtValue() const { return Value->getSExtValue(); } + bool isOne() const { return Value->isOne(); } bool isNullValue() const { return Value->isNullValue(); } bool isAllOnesValue() const { return Value->isAllOnesValue(); } - static bool classof(const ConstantSDNode *) { return true; } + bool isOpaque() const { return SubclassData & 1; } + static bool classof(const SDNode *N) { return N->getOpcode() == ISD::Constant || N->getOpcode() == ISD::TargetConstant; @@ -1823,22 +1448,28 @@ public: class ConstantFPSDNode : public SDNode { const ConstantFP *Value; friend class SelectionDAG; - ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT) + ConstantFPSDNode(bool isTarget, const ConstantFP *val, DebugLoc DL, EVT VT) : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, - DebugLoc::getUnknownLoc(), getSDVTList(VT)), Value(val) { + 0, DL, getSDVTList(VT)), Value(val) { } public: const APFloat& getValueAPF() const { return Value->getValueAPF(); } const ConstantFP *getConstantFPValue() const { return Value; } - /// isZero - Return true if the value is positive or negative zero. + /// Return true if the value is positive or negative zero. bool isZero() const { return Value->isZero(); } - /// isNaN - Return true if the value is a NaN. + /// Return true if the value is a NaN. bool isNaN() const { return Value->isNaN(); } - /// isExactlyValue - We don't rely on operator== working on double values, as + /// Return true if the value is an infinity + bool isInfinity() const { return Value->isInfinity(); } + + /// Return true if the value is negative. + bool isNegative() const { return Value->isNegative(); } + + /// We don't rely on operator== working on double values, as /// it returns true for things that are clearly not equal, like -0.0 and 0.0. /// As such, this method can be used to do an exact bit-for-bit comparison of /// two floating point values. @@ -1848,9 +1479,6 @@ public: /// have to duplicate its logic everywhere it's called. bool isExactlyValue(double V) const { bool ignored; - // convert is not supported on this type - if (&Value->getValueAPF().getSemantics() == &APFloat::PPCDoubleDouble) - return false; APFloat Tmp(V); Tmp.convert(Value->getValueAPF().getSemantics(), APFloat::rmNearestTiesToEven, &ignored); @@ -1858,31 +1486,39 @@ public: } bool isExactlyValue(const APFloat& V) const; - bool isValueValidForType(EVT VT, const APFloat& Val); + static bool isValueValidForType(EVT VT, const APFloat& Val); - static bool classof(const ConstantFPSDNode *) { return true; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::ConstantFP || N->getOpcode() == ISD::TargetConstantFP; } }; +/// Returns true if \p V is a constant integer zero. +bool isNullConstant(SDValue V); +/// Returns true if \p V is an FP constant with a value of positive zero. +bool isNullFPConstant(SDValue V); +/// Returns true if \p V is an integer constant with all bits set. +bool isAllOnesConstant(SDValue V); +/// Returns true if \p V is a constant integer one. +bool isOneConstant(SDValue V); + class GlobalAddressSDNode : public SDNode { - GlobalValue *TheGlobal; + const GlobalValue *TheGlobal; int64_t Offset; unsigned char TargetFlags; friend class SelectionDAG; - GlobalAddressSDNode(unsigned Opc, const GlobalValue *GA, EVT VT, - int64_t o, unsigned char TargetFlags); + GlobalAddressSDNode(unsigned Opc, unsigned Order, DebugLoc DL, + const GlobalValue *GA, EVT VT, int64_t o, + unsigned char TargetFlags); public: - GlobalValue *getGlobal() const { return TheGlobal; } + const GlobalValue *getGlobal() const { return TheGlobal; } int64_t getOffset() const { return Offset; } unsigned char getTargetFlags() const { return TargetFlags; } // Return the address space this GlobalAddress belongs to. unsigned getAddressSpace() const; - static bool classof(const GlobalAddressSDNode *) { return true; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::GlobalAddress || N->getOpcode() == ISD::TargetGlobalAddress || @@ -1896,13 +1532,12 @@ class FrameIndexSDNode : public SDNode { friend class SelectionDAG; FrameIndexSDNode(int fi, EVT VT, bool isTarg) : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex, - DebugLoc::getUnknownLoc(), getSDVTList(VT)), FI(fi) { + 0, DebugLoc(), getSDVTList(VT)), FI(fi) { } public: int getIndex() const { return FI; } - static bool classof(const FrameIndexSDNode *) { return true; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::FrameIndex || N->getOpcode() == ISD::TargetFrameIndex; @@ -1915,14 +1550,13 @@ class JumpTableSDNode : public SDNode { friend class SelectionDAG; JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned char TF) : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable, - DebugLoc::getUnknownLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) { + 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) { } public: int getIndex() const { return JTI; } unsigned char getTargetFlags() const { return TargetFlags; } - static bool classof(const JumpTableSDNode *) { return true; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::JumpTable || N->getOpcode() == ISD::TargetJumpTable; @@ -1931,38 +1565,37 @@ public: class ConstantPoolSDNode : public SDNode { union { - Constant *ConstVal; + const Constant *ConstVal; MachineConstantPoolValue *MachineCPVal; } Val; int Offset; // It's a MachineConstantPoolValue if top bit is set. unsigned Alignment; // Minimum alignment requirement of CP (not log2 value). unsigned char TargetFlags; friend class SelectionDAG; - ConstantPoolSDNode(bool isTarget, Constant *c, EVT VT, int o, unsigned Align, - unsigned char TF) - : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, - DebugLoc::getUnknownLoc(), - getSDVTList(VT)), Offset(o), Alignment(Align), TargetFlags(TF) { - assert((int)Offset >= 0 && "Offset is too large"); + ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o, + unsigned Align, unsigned char TF) + : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0, + DebugLoc(), getSDVTList(VT)), Offset(o), Alignment(Align), + TargetFlags(TF) { + assert(Offset >= 0 && "Offset is too large"); Val.ConstVal = c; } ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o, unsigned Align, unsigned char TF) - : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, - DebugLoc::getUnknownLoc(), - getSDVTList(VT)), Offset(o), Alignment(Align), TargetFlags(TF) { - assert((int)Offset >= 0 && "Offset is too large"); + : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0, + DebugLoc(), getSDVTList(VT)), Offset(o), Alignment(Align), + TargetFlags(TF) { + assert(Offset >= 0 && "Offset is too large"); Val.MachineCPVal = v; Offset |= 1 << (sizeof(unsigned)*CHAR_BIT-1); } public: - bool isMachineConstantPoolEntry() const { - return (int)Offset < 0; + return Offset < 0; } - Constant *getConstVal() const { + const Constant *getConstVal() const { assert(!isMachineConstantPoolEntry() && "Wrong constantpool type"); return Val.ConstVal; } @@ -1981,15 +1614,36 @@ public: unsigned getAlignment() const { return Alignment; } unsigned char getTargetFlags() const { return TargetFlags; } - const Type *getType() const; + Type *getType() const; - static bool classof(const ConstantPoolSDNode *) { return true; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::ConstantPool || N->getOpcode() == ISD::TargetConstantPool; } }; +/// Completely target-dependent object reference. +class TargetIndexSDNode : public SDNode { + unsigned char TargetFlags; + int Index; + int64_t Offset; + friend class SelectionDAG; +public: + + TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned char TF) + : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)), + TargetFlags(TF), Index(Idx), Offset(Ofs) {} +public: + + unsigned char getTargetFlags() const { return TargetFlags; } + int getIndex() const { return Index; } + int64_t getOffset() const { return Offset; } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::TargetIndex; + } +}; + class BasicBlockSDNode : public SDNode { MachineBasicBlock *MBB; friend class SelectionDAG; @@ -1997,26 +1651,23 @@ class BasicBlockSDNode : public SDNode { /// blocks out of order when they're jumped to, which makes it a bit /// harder. Let's see if we need it first. explicit BasicBlockSDNode(MachineBasicBlock *mbb) - : SDNode(ISD::BasicBlock, DebugLoc::getUnknownLoc(), - getSDVTList(MVT::Other)), MBB(mbb) { - } + : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb) + {} public: MachineBasicBlock *getBasicBlock() const { return MBB; } - static bool classof(const BasicBlockSDNode *) { return true; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::BasicBlock; } }; -/// BuildVectorSDNode - A "pseudo-class" with methods for operating on -/// BUILD_VECTORs. +/// A "pseudo-class" with methods for operating on BUILD_VECTORs. class BuildVectorSDNode : public SDNode { // These are constructed as SDNodes and then cast to BuildVectorSDNodes. - explicit BuildVectorSDNode(); // Do not implement + explicit BuildVectorSDNode() = delete; public: - /// isConstantSplat - Check if this is a constant splat, and if so, find the + /// Check if this is a constant splat, and if so, find the /// smallest element size that splats the vector. If MinSplatBits is /// nonzero, the element size must be at least that large. Note that the /// splat element may be the entire vector (i.e., a one element vector). @@ -2027,15 +1678,47 @@ public: /// undefined. isBigEndian describes the endianness of the target. bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, - unsigned MinSplatBits = 0, bool isBigEndian = false); + unsigned MinSplatBits = 0, + bool isBigEndian = false) const; + + /// \brief Returns the splatted value or a null value if this is not a splat. + /// + /// If passed a non-null UndefElements bitvector, it will resize it to match + /// the vector width and set the bits where elements are undef. + SDValue getSplatValue(BitVector *UndefElements = nullptr) const; + + /// \brief Returns the splatted constant or null if this is not a constant + /// splat. + /// + /// If passed a non-null UndefElements bitvector, it will resize it to match + /// the vector width and set the bits where elements are undef. + ConstantSDNode * + getConstantSplatNode(BitVector *UndefElements = nullptr) const; + + /// \brief Returns the splatted constant FP or null if this is not a constant + /// FP splat. + /// + /// If passed a non-null UndefElements bitvector, it will resize it to match + /// the vector width and set the bits where elements are undef. + ConstantFPSDNode * + getConstantFPSplatNode(BitVector *UndefElements = nullptr) const; + + /// \brief If this is a constant FP splat and the splatted constant FP is an + /// exact power or 2, return the log base 2 integer value. Otherwise, + /// return -1. + /// + /// The BitWidth specifies the necessary bit precision. + int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, + uint32_t BitWidth) const; + + bool isConstant() const; - static inline bool classof(const BuildVectorSDNode *) { return true; } static inline bool classof(const SDNode *N) { return N->getOpcode() == ISD::BUILD_VECTOR; } }; -/// SrcValueSDNode - An SDNode that holds an arbitrary LLVM IR Value. This is +/// An SDNode that holds an arbitrary LLVM IR Value. This is /// used when the SelectionDAG needs to make a simple reference to something /// in the LLVM IR representation. /// @@ -2044,51 +1727,78 @@ class SrcValueSDNode : public SDNode { friend class SelectionDAG; /// Create a SrcValue for a general value. explicit SrcValueSDNode(const Value *v) - : SDNode(ISD::SRCVALUE, DebugLoc::getUnknownLoc(), - getSDVTList(MVT::Other)), V(v) {} + : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {} public: - /// getValue - return the contained Value. + /// Return the contained Value. const Value *getValue() const { return V; } - static bool classof(const SrcValueSDNode *) { return true; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::SRCVALUE; } }; +class MDNodeSDNode : public SDNode { + const MDNode *MD; + friend class SelectionDAG; + explicit MDNodeSDNode(const MDNode *md) + : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md) + {} +public: + + const MDNode *getMD() const { return MD; } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::MDNODE_SDNODE; + } +}; class RegisterSDNode : public SDNode { unsigned Reg; friend class SelectionDAG; RegisterSDNode(unsigned reg, EVT VT) - : SDNode(ISD::Register, DebugLoc::getUnknownLoc(), - getSDVTList(VT)), Reg(reg) { + : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) { } public: unsigned getReg() const { return Reg; } - static bool classof(const RegisterSDNode *) { return true; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::Register; } }; +class RegisterMaskSDNode : public SDNode { + // The memory for RegMask is not owned by the node. + const uint32_t *RegMask; + friend class SelectionDAG; + RegisterMaskSDNode(const uint32_t *mask) + : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)), + RegMask(mask) {} +public: + + const uint32_t *getRegMask() const { return RegMask; } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::RegisterMask; + } +}; + class BlockAddressSDNode : public SDNode { - BlockAddress *BA; + const BlockAddress *BA; + int64_t Offset; unsigned char TargetFlags; friend class SelectionDAG; - BlockAddressSDNode(unsigned NodeTy, EVT VT, BlockAddress *ba, - unsigned char Flags) - : SDNode(NodeTy, DebugLoc::getUnknownLoc(), getSDVTList(VT)), - BA(ba), TargetFlags(Flags) { + BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba, + int64_t o, unsigned char Flags) + : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)), + BA(ba), Offset(o), TargetFlags(Flags) { } public: - BlockAddress *getBlockAddress() const { return BA; } + const BlockAddress *getBlockAddress() const { return BA; } + int64_t getOffset() const { return Offset; } unsigned char getTargetFlags() const { return TargetFlags; } - static bool classof(const BlockAddressSDNode *) { return true; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::BlockAddress || N->getOpcode() == ISD::TargetBlockAddress; @@ -2099,14 +1809,13 @@ class EHLabelSDNode : public SDNode { SDUse Chain; MCSymbol *Label; friend class SelectionDAG; - EHLabelSDNode(DebugLoc dl, SDValue ch, MCSymbol *L) - : SDNode(ISD::EH_LABEL, dl, getSDVTList(MVT::Other)), Label(L) { + EHLabelSDNode(unsigned Order, DebugLoc dl, SDValue ch, MCSymbol *L) + : SDNode(ISD::EH_LABEL, Order, dl, getSDVTList(MVT::Other)), Label(L) { InitOperands(&Chain, ch); } public: MCSymbol *getLabel() const { return Label; } - static bool classof(const EHLabelSDNode *) { return true; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::EH_LABEL; } @@ -2115,202 +1824,92 @@ public: class ExternalSymbolSDNode : public SDNode { const char *Symbol; unsigned char TargetFlags; - + friend class SelectionDAG; ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned char TF, EVT VT) : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, - DebugLoc::getUnknownLoc(), - getSDVTList(VT)), Symbol(Sym), TargetFlags(TF) { + 0, DebugLoc(), getSDVTList(VT)), Symbol(Sym), TargetFlags(TF) { } public: const char *getSymbol() const { return Symbol; } unsigned char getTargetFlags() const { return TargetFlags; } - static bool classof(const ExternalSymbolSDNode *) { return true; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::ExternalSymbol || N->getOpcode() == ISD::TargetExternalSymbol; } }; +class MCSymbolSDNode : public SDNode { + MCSymbol *Symbol; + + friend class SelectionDAG; + MCSymbolSDNode(MCSymbol *Symbol, EVT VT) + : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {} + +public: + MCSymbol *getMCSymbol() const { return Symbol; } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::MCSymbol; + } +}; + class CondCodeSDNode : public SDNode { ISD::CondCode Condition; friend class SelectionDAG; explicit CondCodeSDNode(ISD::CondCode Cond) - : SDNode(ISD::CONDCODE, DebugLoc::getUnknownLoc(), - getSDVTList(MVT::Other)), Condition(Cond) { + : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)), + Condition(Cond) { } public: ISD::CondCode get() const { return Condition; } - static bool classof(const CondCodeSDNode *) { return true; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::CONDCODE; } }; - -/// CvtRndSatSDNode - NOTE: avoid using this node as this may disappear in the + +/// NOTE: avoid using this node as this may disappear in the /// future and most targets don't support it. class CvtRndSatSDNode : public SDNode { ISD::CvtCode CvtCode; friend class SelectionDAG; - explicit CvtRndSatSDNode(EVT VT, DebugLoc dl, const SDValue *Ops, - unsigned NumOps, ISD::CvtCode Code) - : SDNode(ISD::CONVERT_RNDSAT, dl, getSDVTList(VT), Ops, NumOps), + explicit CvtRndSatSDNode(EVT VT, unsigned Order, DebugLoc dl, + ArrayRef Ops, ISD::CvtCode Code) + : SDNode(ISD::CONVERT_RNDSAT, Order, dl, getSDVTList(VT), Ops), CvtCode(Code) { - assert(NumOps == 5 && "wrong number of operations"); + assert(Ops.size() == 5 && "wrong number of operations"); } public: ISD::CvtCode getCvtCode() const { return CvtCode; } - static bool classof(const CvtRndSatSDNode *) { return true; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::CONVERT_RNDSAT; } }; -namespace ISD { - struct ArgFlagsTy { - private: - static const uint64_t NoFlagSet = 0ULL; - static const uint64_t ZExt = 1ULL<<0; ///< Zero extended - static const uint64_t ZExtOffs = 0; - static const uint64_t SExt = 1ULL<<1; ///< Sign extended - static const uint64_t SExtOffs = 1; - static const uint64_t InReg = 1ULL<<2; ///< Passed in register - static const uint64_t InRegOffs = 2; - static const uint64_t SRet = 1ULL<<3; ///< Hidden struct-ret ptr - static const uint64_t SRetOffs = 3; - static const uint64_t ByVal = 1ULL<<4; ///< Struct passed by value - static const uint64_t ByValOffs = 4; - static const uint64_t Nest = 1ULL<<5; ///< Nested fn static chain - static const uint64_t NestOffs = 5; - static const uint64_t ByValAlign = 0xFULL << 6; //< Struct alignment - static const uint64_t ByValAlignOffs = 6; - static const uint64_t Split = 1ULL << 10; - static const uint64_t SplitOffs = 10; - static const uint64_t OrigAlign = 0x1FULL<<27; - static const uint64_t OrigAlignOffs = 27; - static const uint64_t ByValSize = 0xffffffffULL << 32; //< Struct size - static const uint64_t ByValSizeOffs = 32; - - static const uint64_t One = 1ULL; //< 1 of this type, for shifts - - uint64_t Flags; - public: - ArgFlagsTy() : Flags(0) { } - - bool isZExt() const { return Flags & ZExt; } - void setZExt() { Flags |= One << ZExtOffs; } - - bool isSExt() const { return Flags & SExt; } - void setSExt() { Flags |= One << SExtOffs; } - - bool isInReg() const { return Flags & InReg; } - void setInReg() { Flags |= One << InRegOffs; } - - bool isSRet() const { return Flags & SRet; } - void setSRet() { Flags |= One << SRetOffs; } - - bool isByVal() const { return Flags & ByVal; } - void setByVal() { Flags |= One << ByValOffs; } - - bool isNest() const { return Flags & Nest; } - void setNest() { Flags |= One << NestOffs; } - - unsigned getByValAlign() const { - return (unsigned) - ((One << ((Flags & ByValAlign) >> ByValAlignOffs)) / 2); - } - void setByValAlign(unsigned A) { - Flags = (Flags & ~ByValAlign) | - (uint64_t(Log2_32(A) + 1) << ByValAlignOffs); - } - - bool isSplit() const { return Flags & Split; } - void setSplit() { Flags |= One << SplitOffs; } - - unsigned getOrigAlign() const { - return (unsigned) - ((One << ((Flags & OrigAlign) >> OrigAlignOffs)) / 2); - } - void setOrigAlign(unsigned A) { - Flags = (Flags & ~OrigAlign) | - (uint64_t(Log2_32(A) + 1) << OrigAlignOffs); - } - - unsigned getByValSize() const { - return (unsigned)((Flags & ByValSize) >> ByValSizeOffs); - } - void setByValSize(unsigned S) { - Flags = (Flags & ~ByValSize) | (uint64_t(S) << ByValSizeOffs); - } - - /// getArgFlagsString - Returns the flags as a string, eg: "zext align:4". - std::string getArgFlagsString(); - - /// getRawBits - Represent the flags as a bunch of bits. - uint64_t getRawBits() const { return Flags; } - }; - - /// InputArg - This struct carries flags and type information about a - /// single incoming (formal) argument or incoming (from the perspective - /// of the caller) return value virtual register. - /// - struct InputArg { - ArgFlagsTy Flags; - EVT VT; - bool Used; - - InputArg() : VT(MVT::Other), Used(false) {} - InputArg(ISD::ArgFlagsTy flags, EVT vt, bool used) - : Flags(flags), VT(vt), Used(used) { - assert(VT.isSimple() && - "InputArg value type must be Simple!"); - } - }; - - /// OutputArg - This struct carries flags and a value for a - /// single outgoing (actual) argument or outgoing (from the perspective - /// of the caller) return value virtual register. - /// - struct OutputArg { - ArgFlagsTy Flags; - SDValue Val; - bool IsFixed; - - OutputArg() : IsFixed(false) {} - OutputArg(ISD::ArgFlagsTy flags, SDValue val, bool isfixed) - : Flags(flags), Val(val), IsFixed(isfixed) { - assert(Val.getValueType().isSimple() && - "OutputArg value type must be Simple!"); - } - }; -} - -/// VTSDNode - This class is used to represent EVT's, which are used +/// This class is used to represent EVT's, which are used /// to parameterize some operations. class VTSDNode : public SDNode { EVT ValueType; friend class SelectionDAG; explicit VTSDNode(EVT VT) - : SDNode(ISD::VALUETYPE, DebugLoc::getUnknownLoc(), - getSDVTList(MVT::Other)), ValueType(VT) { + : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)), + ValueType(VT) { } public: EVT getVT() const { return ValueType; } - static bool classof(const VTSDNode *) { return true; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::VALUETYPE; } }; -/// LSBaseSDNode - Base class for LoadSDNode and StoreSDNode -/// +/// Base class for LoadSDNode and StoreSDNode class LSBaseSDNode : public MemSDNode { //! Operand array for load and store /*! @@ -2320,10 +1919,11 @@ class LSBaseSDNode : public MemSDNode { */ SDUse Ops[4]; public: - LSBaseSDNode(ISD::NodeType NodeTy, DebugLoc dl, SDValue *Operands, - unsigned numOperands, SDVTList VTs, ISD::MemIndexedMode AM, - EVT MemVT, MachineMemOperand *MMO) - : MemSDNode(NodeTy, dl, VTs, MemVT, MMO) { + LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl, + SDValue *Operands, unsigned numOperands, + SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT, + MachineMemOperand *MMO) + : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { SubclassData |= AM << 2; assert(getAddressingMode() == AM && "MemIndexedMode encoding error!"); InitOperands(Ops, Operands, numOperands); @@ -2335,34 +1935,31 @@ public: return getOperand(getOpcode() == ISD::LOAD ? 2 : 3); } - /// getAddressingMode - Return the addressing mode for this load or store: + /// Return the addressing mode for this load or store: /// unindexed, pre-inc, pre-dec, post-inc, or post-dec. ISD::MemIndexedMode getAddressingMode() const { return ISD::MemIndexedMode((SubclassData >> 2) & 7); } - /// isIndexed - Return true if this is a pre/post inc/dec load/store. + /// Return true if this is a pre/post inc/dec load/store. bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; } - /// isUnindexed - Return true if this is NOT a pre/post inc/dec load/store. + /// Return true if this is NOT a pre/post inc/dec load/store. bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; } - static bool classof(const LSBaseSDNode *) { return true; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::LOAD || N->getOpcode() == ISD::STORE; } }; -/// LoadSDNode - This class is used to represent ISD::LOAD nodes. -/// +/// This class is used to represent ISD::LOAD nodes. class LoadSDNode : public LSBaseSDNode { friend class SelectionDAG; - LoadSDNode(SDValue *ChainPtrOff, DebugLoc dl, SDVTList VTs, + LoadSDNode(SDValue *ChainPtrOff, unsigned Order, DebugLoc dl, SDVTList VTs, ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT, MachineMemOperand *MMO) - : LSBaseSDNode(ISD::LOAD, dl, ChainPtrOff, 3, - VTs, AM, MemVT, MMO) { + : LSBaseSDNode(ISD::LOAD, Order, dl, ChainPtrOff, 3, VTs, AM, MemVT, MMO) { SubclassData |= (unsigned short)ETy; assert(getExtensionType() == ETy && "LoadExtType encoding error!"); assert(readMem() && "Load MachineMemOperand is not a load!"); @@ -2370,7 +1967,7 @@ class LoadSDNode : public LSBaseSDNode { } public: - /// getExtensionType - Return whether this is a plain node, + /// Return whether this is a plain node, /// or one of the varieties of value-extending loads. ISD::LoadExtType getExtensionType() const { return ISD::LoadExtType(SubclassData & 3); @@ -2379,20 +1976,18 @@ public: const SDValue &getBasePtr() const { return getOperand(1); } const SDValue &getOffset() const { return getOperand(2); } - static bool classof(const LoadSDNode *) { return true; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::LOAD; } }; -/// StoreSDNode - This class is used to represent ISD::STORE nodes. -/// +/// This class is used to represent ISD::STORE nodes. class StoreSDNode : public LSBaseSDNode { friend class SelectionDAG; - StoreSDNode(SDValue *ChainValuePtrOff, DebugLoc dl, SDVTList VTs, - ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT, + StoreSDNode(SDValue *ChainValuePtrOff, unsigned Order, DebugLoc dl, + SDVTList VTs, ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT, MachineMemOperand *MMO) - : LSBaseSDNode(ISD::STORE, dl, ChainValuePtrOff, 4, + : LSBaseSDNode(ISD::STORE, Order, dl, ChainValuePtrOff, 4, VTs, AM, MemVT, MMO) { SubclassData |= (unsigned short)isTrunc; assert(isTruncatingStore() == isTrunc && "isTrunc encoding error!"); @@ -2401,7 +1996,7 @@ class StoreSDNode : public LSBaseSDNode { } public: - /// isTruncatingStore - Return true if the op does a truncation before store. + /// Return true if the op does a truncation before store. /// For integers this is the same as doing a TRUNCATE and storing the result. /// For floats, it is the same as doing an FP_ROUND and storing the result. bool isTruncatingStore() const { return SubclassData & 1; } @@ -2410,30 +2005,177 @@ public: const SDValue &getBasePtr() const { return getOperand(2); } const SDValue &getOffset() const { return getOperand(3); } - static bool classof(const StoreSDNode *) { return true; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::STORE; } }; -/// MachineSDNode - An SDNode that represents everything that will be needed +/// This base class is used to represent MLOAD and MSTORE nodes +class MaskedLoadStoreSDNode : public MemSDNode { + // Operands + SDUse Ops[4]; +public: + friend class SelectionDAG; + MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl, + SDValue *Operands, unsigned numOperands, SDVTList VTs, + EVT MemVT, MachineMemOperand *MMO) + : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { + InitOperands(Ops, Operands, numOperands); + } + + // In the both nodes address is Op1, mask is Op2: + // MaskedLoadSDNode (Chain, ptr, mask, src0), src0 is a passthru value + // MaskedStoreSDNode (Chain, ptr, mask, data) + // Mask is a vector of i1 elements + const SDValue &getBasePtr() const { return getOperand(1); } + const SDValue &getMask() const { return getOperand(2); } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::MLOAD || + N->getOpcode() == ISD::MSTORE; + } +}; + +/// This class is used to represent an MLOAD node +class MaskedLoadSDNode : public MaskedLoadStoreSDNode { +public: + friend class SelectionDAG; + MaskedLoadSDNode(unsigned Order, DebugLoc dl, SDValue *Operands, + unsigned numOperands, SDVTList VTs, ISD::LoadExtType ETy, + EVT MemVT, MachineMemOperand *MMO) + : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, Operands, numOperands, + VTs, MemVT, MMO) { + SubclassData |= (unsigned short)ETy; + } + + ISD::LoadExtType getExtensionType() const { + return ISD::LoadExtType(SubclassData & 3); + } + const SDValue &getSrc0() const { return getOperand(3); } + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::MLOAD; + } +}; + +/// This class is used to represent an MSTORE node +class MaskedStoreSDNode : public MaskedLoadStoreSDNode { + +public: + friend class SelectionDAG; + MaskedStoreSDNode(unsigned Order, DebugLoc dl, SDValue *Operands, + unsigned numOperands, SDVTList VTs, bool isTrunc, EVT MemVT, + MachineMemOperand *MMO) + : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, Operands, numOperands, + VTs, MemVT, MMO) { + SubclassData |= (unsigned short)isTrunc; + } + /// Return true if the op does a truncation before store. + /// For integers this is the same as doing a TRUNCATE and storing the result. + /// For floats, it is the same as doing an FP_ROUND and storing the result. + bool isTruncatingStore() const { return SubclassData & 1; } + + const SDValue &getValue() const { return getOperand(3); } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::MSTORE; + } +}; + +/// This is a base class used to represent +/// MGATHER and MSCATTER nodes +/// +class MaskedGatherScatterSDNode : public MemSDNode { + // Operands + SDUse Ops[5]; +public: + friend class SelectionDAG; + MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl, + ArrayRef Operands, SDVTList VTs, EVT MemVT, + MachineMemOperand *MMO) + : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { + assert(Operands.size() == 5 && "Incompatible number of operands"); + InitOperands(Ops, Operands.data(), Operands.size()); + } + + // In the both nodes address is Op1, mask is Op2: + // MaskedGatherSDNode (Chain, src0, mask, base, index), src0 is a passthru value + // MaskedScatterSDNode (Chain, value, mask, base, index) + // Mask is a vector of i1 elements + const SDValue &getBasePtr() const { return getOperand(3); } + const SDValue &getIndex() const { return getOperand(4); } + const SDValue &getMask() const { return getOperand(2); } + const SDValue &getValue() const { return getOperand(1); } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::MGATHER || + N->getOpcode() == ISD::MSCATTER; + } +}; + +/// This class is used to represent an MGATHER node +/// +class MaskedGatherSDNode : public MaskedGatherScatterSDNode { +public: + friend class SelectionDAG; + MaskedGatherSDNode(unsigned Order, DebugLoc dl, ArrayRef Operands, + SDVTList VTs, EVT MemVT, MachineMemOperand *MMO) + : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, Operands, VTs, MemVT, + MMO) { + assert(getValue().getValueType() == getValueType(0) && + "Incompatible type of the PassThru value in MaskedGatherSDNode"); + assert(getMask().getValueType().getVectorNumElements() == + getValueType(0).getVectorNumElements() && + "Vector width mismatch between mask and data"); + assert(getIndex().getValueType().getVectorNumElements() == + getValueType(0).getVectorNumElements() && + "Vector width mismatch between index and data"); + } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::MGATHER; + } +}; + +/// This class is used to represent an MSCATTER node +/// +class MaskedScatterSDNode : public MaskedGatherScatterSDNode { + +public: + friend class SelectionDAG; + MaskedScatterSDNode(unsigned Order, DebugLoc dl,ArrayRef Operands, + SDVTList VTs, EVT MemVT, MachineMemOperand *MMO) + : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, Operands, VTs, MemVT, + MMO) { + assert(getMask().getValueType().getVectorNumElements() == + getValue().getValueType().getVectorNumElements() && + "Vector width mismatch between mask and data"); + assert(getIndex().getValueType().getVectorNumElements() == + getValue().getValueType().getVectorNumElements() && + "Vector width mismatch between index and data"); + } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::MSCATTER; + } +}; + +/// An SDNode that represents everything that will be needed /// to construct a MachineInstr. These nodes are created during the /// instruction selection proper phase. -/// class MachineSDNode : public SDNode { public: typedef MachineMemOperand **mmo_iterator; private: friend class SelectionDAG; - MachineSDNode(unsigned Opc, const DebugLoc DL, SDVTList VTs) - : SDNode(Opc, DL, VTs), MemRefs(0), MemRefsEnd(0) {} + MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc DL, SDVTList VTs) + : SDNode(Opc, Order, DL, VTs), MemRefs(nullptr), MemRefsEnd(nullptr) {} - /// LocalOperands - Operands for this instruction, if they fit here. If + /// Operands for this instruction, if they fit here. If /// they don't, this field is unused. SDUse LocalOperands[4]; - /// MemRefs - Memory reference descriptions for this instruction. + /// Memory reference descriptions for this instruction. mmo_iterator MemRefs; mmo_iterator MemRefsEnd; @@ -2442,14 +2184,15 @@ public: mmo_iterator memoperands_end() const { return MemRefsEnd; } bool memoperands_empty() const { return MemRefsEnd == MemRefs; } - /// setMemRefs - Assign this MachineSDNodes's memory reference descriptor + /// Assign this MachineSDNodes's memory reference descriptor /// list. This does not transfer ownership. void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd) { + for (mmo_iterator MMI = NewMemRefs, MME = NewMemRefsEnd; MMI != MME; ++MMI) + assert(*MMI && "Null mem ref detected!"); MemRefs = NewMemRefs; MemRefsEnd = NewMemRefsEnd; } - static bool classof(const MachineSDNode *) { return true; } static bool classof(const SDNode *N) { return N->isMachineOpcode(); } @@ -2457,22 +2200,16 @@ public: class SDNodeIterator : public std::iterator { - SDNode *Node; + const SDNode *Node; unsigned Operand; - SDNodeIterator(SDNode *N, unsigned Op) : Node(N), Operand(Op) {} + SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {} public: bool operator==(const SDNodeIterator& x) const { return Operand == x.Operand; } bool operator!=(const SDNodeIterator& x) const { return !operator==(x); } - const SDNodeIterator &operator=(const SDNodeIterator &I) { - assert(I.Node == Node && "Cannot assign iterators to two different nodes!"); - Operand = I.Operand; - return *this; - } - pointer operator*() const { return Node->getOperand(Operand).getNode(); } @@ -2491,8 +2228,8 @@ public: return Operand - Other.Operand; } - static SDNodeIterator begin(SDNode *N) { return SDNodeIterator(N, 0); } - static SDNodeIterator end (SDNode *N) { + static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); } + static SDNodeIterator end (const SDNode *N) { return SDNodeIterator(N, N->getNumOperands()); } @@ -2512,60 +2249,51 @@ template <> struct GraphTraits { } }; -/// LargestSDNode - The largest SDNode class. -/// -typedef LoadSDNode LargestSDNode; +/// The largest SDNode class. +typedef MaskedGatherScatterSDNode LargestSDNode; -/// MostAlignedSDNode - The SDNode class with the greatest alignment -/// requirement. -/// +/// The SDNode class with the greatest alignment requirement. typedef GlobalAddressSDNode MostAlignedSDNode; namespace ISD { - /// isNormalLoad - Returns true if the specified node is a non-extending - /// and unindexed load. + /// Returns true if the specified node is a non-extending and unindexed load. inline bool isNormalLoad(const SDNode *N) { const LoadSDNode *Ld = dyn_cast(N); return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD && Ld->getAddressingMode() == ISD::UNINDEXED; } - /// isNON_EXTLoad - Returns true if the specified node is a non-extending - /// load. + /// Returns true if the specified node is a non-extending load. inline bool isNON_EXTLoad(const SDNode *N) { return isa(N) && cast(N)->getExtensionType() == ISD::NON_EXTLOAD; } - /// isEXTLoad - Returns true if the specified node is a EXTLOAD. - /// + /// Returns true if the specified node is a EXTLOAD. inline bool isEXTLoad(const SDNode *N) { return isa(N) && cast(N)->getExtensionType() == ISD::EXTLOAD; } - /// isSEXTLoad - Returns true if the specified node is a SEXTLOAD. - /// + /// Returns true if the specified node is a SEXTLOAD. inline bool isSEXTLoad(const SDNode *N) { return isa(N) && cast(N)->getExtensionType() == ISD::SEXTLOAD; } - /// isZEXTLoad - Returns true if the specified node is a ZEXTLOAD. - /// + /// Returns true if the specified node is a ZEXTLOAD. inline bool isZEXTLoad(const SDNode *N) { return isa(N) && cast(N)->getExtensionType() == ISD::ZEXTLOAD; } - /// isUNINDEXEDLoad - Returns true if the specified node is an unindexed load. - /// + /// Returns true if the specified node is an unindexed load. inline bool isUNINDEXEDLoad(const SDNode *N) { return isa(N) && cast(N)->getAddressingMode() == ISD::UNINDEXED; } - /// isNormalStore - Returns true if the specified node is a non-truncating + /// Returns true if the specified node is a non-truncating /// and unindexed store. inline bool isNormalStore(const SDNode *N) { const StoreSDNode *St = dyn_cast(N); @@ -2573,27 +2301,23 @@ namespace ISD { St->getAddressingMode() == ISD::UNINDEXED; } - /// isNON_TRUNCStore - Returns true if the specified node is a non-truncating - /// store. + /// Returns true if the specified node is a non-truncating store. inline bool isNON_TRUNCStore(const SDNode *N) { return isa(N) && !cast(N)->isTruncatingStore(); } - /// isTRUNCStore - Returns true if the specified node is a truncating - /// store. + /// Returns true if the specified node is a truncating store. inline bool isTRUNCStore(const SDNode *N) { return isa(N) && cast(N)->isTruncatingStore(); } - /// isUNINDEXEDStore - Returns true if the specified node is an - /// unindexed store. + /// Returns true if the specified node is an unindexed store. inline bool isUNINDEXEDStore(const SDNode *N) { return isa(N) && cast(N)->getAddressingMode() == ISD::UNINDEXED; } } - } // end llvm namespace #endif