1 //===-- LegalizeDAGTypes.cpp - Implement SelectionDAG::LegalizeTypes ------===//
3 // The LLVM Compiler Infrastructure
5 // This file was developed by Chris Lattner and is distributed under
6 // the University of Illinois Open Source License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the SelectionDAG::LegalizeTypes method. It transforms
11 // an arbitrary well-formed SelectionDAG to only consist of legal types.
13 //===----------------------------------------------------------------------===//
15 #include "LegalizeTypes.h"
16 #include "llvm/Constants.h"
17 #include "llvm/DerivedTypes.h"
18 #include "llvm/Support/Debug.h"
19 #include "llvm/Support/MathExtras.h"
22 /// run - This is the main entry point for the type legalizer. This does a
23 /// top-down traversal of the dag, legalizing types as it goes.
24 void DAGTypeLegalizer::run() {
25 // Create a dummy node (which is not added to allnodes), that adds a reference
26 // to the root node, preventing it from being deleted, and tracking any
27 // changes of the root.
28 HandleSDNode Dummy(DAG.getRoot());
30 // The root of the dag may dangle to deleted nodes until the type legalizer is
31 // done. Set it to null to avoid confusion.
32 DAG.setRoot(SDOperand());
34 // Walk all nodes in the graph, assigning them a NodeID of 'ReadyToProcess'
35 // (and remembering them) if they are leaves and assigning 'NewNode' if
37 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
38 E = DAG.allnodes_end(); I != E; ++I) {
39 if (I->getNumOperands() == 0) {
40 I->setNodeId(ReadyToProcess);
41 Worklist.push_back(I);
43 I->setNodeId(NewNode);
47 // Now that we have a set of nodes to process, handle them all.
48 while (!Worklist.empty()) {
49 SDNode *N = Worklist.back();
51 assert(N->getNodeId() == ReadyToProcess &&
52 "Node should be ready if on worklist!");
54 // Scan the values produced by the node, checking to see if any result
57 unsigned NumResults = N->getNumValues();
59 MVT::ValueType ResultVT = N->getValueType(i);
60 LegalizeAction Action = getTypeAction(ResultVT);
61 if (Action == Promote) {
64 } else if (Action == Expand) {
65 // Expand can mean 1) split integer in half 2) scalarize single-element
66 // vector 3) split vector in half.
67 if (!MVT::isVector(ResultVT))
69 else if (MVT::getVectorNumElements(ResultVT) == 1)
70 ScalarizeResult(N, i); // Scalarize the single-element vector.
71 else // Split the vector in half.
72 assert(0 && "Vector splitting not implemented");
75 assert(Action == Legal && "Unknown action!");
77 } while (++i < NumResults);
79 // Scan the operand list for the node, handling any nodes with operands that
82 unsigned NumOperands = N->getNumOperands();
83 bool NeedsRevisit = false;
84 for (i = 0; i != NumOperands; ++i) {
85 MVT::ValueType OpVT = N->getOperand(i).getValueType();
86 LegalizeAction Action = getTypeAction(OpVT);
87 if (Action == Promote) {
88 NeedsRevisit = PromoteOperand(N, i);
90 } else if (Action == Expand) {
91 // Expand can mean 1) split integer in half 2) scalarize single-element
92 // vector 3) split vector in half.
93 if (!MVT::isVector(OpVT)) {
94 NeedsRevisit = ExpandOperand(N, i);
95 } else if (MVT::getVectorNumElements(OpVT) == 1) {
96 // Scalarize the single-element vector.
97 NeedsRevisit = ScalarizeOperand(N, i);
99 // Split the vector in half.
100 assert(0 && "Vector splitting not implemented");
104 assert(Action == Legal && "Unknown action!");
108 // If the node needs revisiting, don't add all users to the worklist etc.
112 if (i == NumOperands)
113 DEBUG(cerr << "Legally typed node: "; N->dump(&DAG); cerr << "\n");
117 // If we reach here, the node was processed, potentially creating new nodes.
118 // Mark it as processed and add its users to the worklist as appropriate.
119 N->setNodeId(Processed);
121 for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end();
124 int NodeID = User->getNodeId();
125 assert(NodeID != ReadyToProcess && NodeID != Processed &&
126 "Invalid node id for user of unprocessed node!");
128 // This node has two options: it can either be a new node or its Node ID
129 // may be a count of the number of operands it has that are not ready.
131 User->setNodeId(NodeID-1);
133 // If this was the last use it was waiting on, add it to the ready list.
134 if (NodeID-1 == ReadyToProcess)
135 Worklist.push_back(User);
139 // Otherwise, this node is new: this is the first operand of it that
140 // became ready. Its new NodeID is the number of operands it has minus 1
141 // (as this node is now processed).
142 assert(NodeID == NewNode && "Unknown node ID!");
143 User->setNodeId(User->getNumOperands()-1);
145 // If the node only has a single operand, it is now ready.
146 if (User->getNumOperands() == 1)
147 Worklist.push_back(User);
151 // If the root changed (e.g. it was a dead load, update the root).
152 DAG.setRoot(Dummy.getValue());
156 // Remove dead nodes. This is important to do for cleanliness but also before
157 // the checking loop below. Implicit folding by the DAG.getNode operators can
158 // cause unreachable nodes to be around with their flags set to new.
159 DAG.RemoveDeadNodes();
161 // In a debug build, scan all the nodes to make sure we found them all. This
162 // ensures that there are no cycles and that everything got processed.
164 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
165 E = DAG.allnodes_end(); I != E; ++I) {
166 if (I->getNodeId() == Processed)
168 cerr << "Unprocessed node: ";
169 I->dump(&DAG); cerr << "\n";
171 if (I->getNodeId() == NewNode)
172 cerr << "New node not 'noticed'?\n";
173 else if (I->getNodeId() > 0)
174 cerr << "Operand not processed?\n";
175 else if (I->getNodeId() == ReadyToProcess)
176 cerr << "Not added to worklist?\n";
182 /// MarkNewNodes - The specified node is the root of a subtree of potentially
183 /// new nodes. Add the correct NodeId to mark it.
184 void DAGTypeLegalizer::MarkNewNodes(SDNode *N) {
185 // If this was an existing node that is already done, we're done.
186 if (N->getNodeId() != NewNode)
189 // Okay, we know that this node is new. Recursively walk all of its operands
190 // to see if they are new also. The depth of this walk is bounded by the size
191 // of the new tree that was constructed (usually 2-3 nodes), so we don't worry
192 // about revisiting of nodes.
194 // As we walk the operands, keep track of the number of nodes that are
195 // processed. If non-zero, this will become the new nodeid of this node.
196 unsigned NumProcessed = 0;
197 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
198 int OpId = N->getOperand(i).Val->getNodeId();
200 MarkNewNodes(N->getOperand(i).Val);
201 else if (OpId == Processed)
205 N->setNodeId(N->getNumOperands()-NumProcessed);
206 if (N->getNodeId() == ReadyToProcess)
207 Worklist.push_back(N);
210 /// ReplaceValueWith - The specified value was legalized to the specified other
211 /// value. If they are different, update the DAG and NodeIDs replacing any uses
212 /// of From to use To instead.
213 void DAGTypeLegalizer::ReplaceValueWith(SDOperand From, SDOperand To) {
214 if (From == To) return;
216 // If expansion produced new nodes, make sure they are properly marked.
217 if (To.Val->getNodeId() == NewNode)
218 MarkNewNodes(To.Val);
220 // Anything that used the old node should now use the new one. Note that this
221 // can potentially cause recursive merging.
222 DAG.ReplaceAllUsesOfValueWith(From, To);
224 // The old node may still be present in ExpandedNodes or PromotedNodes.
225 // Inform them about the replacement.
226 ReplacedNodes[From] = To;
228 // Since we just made an unstructured update to the DAG, which could wreak
229 // general havoc on anything that once used From and now uses To, walk all
230 // users of the result, updating their flags.
231 for (SDNode::use_iterator I = To.Val->use_begin(), E = To.Val->use_end();
234 // If the node isn't already processed or in the worklist, mark it as new,
235 // then use MarkNewNodes to recompute its ID.
236 int NodeId = User->getNodeId();
237 if (NodeId != ReadyToProcess && NodeId != Processed) {
238 User->setNodeId(NewNode);
244 /// ReplaceNodeWith - Replace uses of the 'from' node's results with the 'to'
245 /// node's results. The from and to node must define identical result types.
246 void DAGTypeLegalizer::ReplaceNodeWith(SDNode *From, SDNode *To) {
247 if (From == To) return;
248 assert(From->getNumValues() == To->getNumValues() &&
249 "Node results don't match");
251 // If expansion produced new nodes, make sure they are properly marked.
252 if (To->getNodeId() == NewNode)
255 // Anything that used the old node should now use the new one. Note that this
256 // can potentially cause recursive merging.
257 DAG.ReplaceAllUsesWith(From, To);
259 // The old node may still be present in ExpandedNodes or PromotedNodes.
260 // Inform them about the replacement.
261 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {
262 assert(From->getValueType(i) == To->getValueType(i) &&
263 "Node results don't match");
264 ReplacedNodes[SDOperand(From, i)] = SDOperand(To, i);
267 // Since we just made an unstructured update to the DAG, which could wreak
268 // general havoc on anything that once used From and now uses To, walk all
269 // users of the result, updating their flags.
270 for (SDNode::use_iterator I = To->use_begin(), E = To->use_end();I != E; ++I){
272 // If the node isn't already processed or in the worklist, mark it as new,
273 // then use MarkNewNodes to recompute its ID.
274 int NodeId = User->getNodeId();
275 if (NodeId != ReadyToProcess && NodeId != Processed) {
276 User->setNodeId(NewNode);
283 /// RemapNode - If the specified value was already legalized to another value,
284 /// replace it by that value.
285 void DAGTypeLegalizer::RemapNode(SDOperand &N) {
286 DenseMap<SDOperand, SDOperand>::iterator I = ReplacedNodes.find(N);
287 if (I != ReplacedNodes.end()) {
288 // Use path compression to speed up future lookups if values get multiply
289 // replaced with other values.
290 RemapNode(I->second);
295 void DAGTypeLegalizer::SetPromotedOp(SDOperand Op, SDOperand Result) {
296 if (Result.Val->getNodeId() == NewNode)
297 MarkNewNodes(Result.Val);
299 SDOperand &OpEntry = PromotedNodes[Op];
300 assert(OpEntry.Val == 0 && "Node is already promoted!");
304 void DAGTypeLegalizer::SetScalarizedOp(SDOperand Op, SDOperand Result) {
305 if (Result.Val->getNodeId() == NewNode)
306 MarkNewNodes(Result.Val);
308 SDOperand &OpEntry = ScalarizedNodes[Op];
309 assert(OpEntry.Val == 0 && "Node is already scalarized!");
314 void DAGTypeLegalizer::GetExpandedOp(SDOperand Op, SDOperand &Lo,
316 std::pair<SDOperand, SDOperand> &Entry = ExpandedNodes[Op];
317 RemapNode(Entry.first);
318 RemapNode(Entry.second);
319 assert(Entry.first.Val && "Operand isn't expanded");
324 void DAGTypeLegalizer::SetExpandedOp(SDOperand Op, SDOperand Lo,
326 // Remember that this is the result of the node.
327 std::pair<SDOperand, SDOperand> &Entry = ExpandedNodes[Op];
328 assert(Entry.first.Val == 0 && "Node already expanded");
332 // Lo/Hi may have been newly allocated, if so, add nodeid's as relevant.
333 if (Lo.Val->getNodeId() == NewNode)
334 MarkNewNodes(Lo.Val);
335 if (Hi.Val->getNodeId() == NewNode)
336 MarkNewNodes(Hi.Val);
339 SDOperand DAGTypeLegalizer::CreateStackStoreLoad(SDOperand Op,
340 MVT::ValueType DestVT) {
341 // Create the stack frame object.
342 SDOperand FIPtr = DAG.CreateStackTemporary(DestVT);
344 // Emit a store to the stack slot.
345 SDOperand Store = DAG.getStore(DAG.getEntryNode(), Op, FIPtr, NULL, 0);
346 // Result is a load from the stack slot.
347 return DAG.getLoad(DestVT, Store, FIPtr, NULL, 0);
350 /// HandleMemIntrinsic - This handles memcpy/memset/memmove with invalid
351 /// operands. This promotes or expands the operands as required.
352 SDOperand DAGTypeLegalizer::HandleMemIntrinsic(SDNode *N) {
353 // The chain and pointer [operands #0 and #1] are always valid types.
354 SDOperand Chain = N->getOperand(0);
355 SDOperand Ptr = N->getOperand(1);
356 SDOperand Op2 = N->getOperand(2);
358 // Op #2 is either a value (memset) or a pointer. Promote it if required.
359 switch (getTypeAction(Op2.getValueType())) {
360 default: assert(0 && "Unknown action for pointer/value operand");
362 case Promote: Op2 = GetPromotedOp(Op2); break;
365 // The length could have any action required.
366 SDOperand Length = N->getOperand(3);
367 switch (getTypeAction(Length.getValueType())) {
368 default: assert(0 && "Unknown action for memop operand");
370 case Promote: Length = GetPromotedZExtOp(Length); break;
372 SDOperand Dummy; // discard the high part.
373 GetExpandedOp(Length, Length, Dummy);
377 SDOperand Align = N->getOperand(4);
378 switch (getTypeAction(Align.getValueType())) {
379 default: assert(0 && "Unknown action for memop operand");
381 case Promote: Align = GetPromotedZExtOp(Align); break;
384 SDOperand AlwaysInline = N->getOperand(5);
385 switch (getTypeAction(AlwaysInline.getValueType())) {
386 default: assert(0 && "Unknown action for memop operand");
388 case Promote: AlwaysInline = GetPromotedZExtOp(AlwaysInline); break;
391 SDOperand Ops[] = { Chain, Ptr, Op2, Length, Align, AlwaysInline };
392 return DAG.UpdateNodeOperands(SDOperand(N, 0), Ops, 6);
395 /// SplitOp - Return the lower and upper halves of Op's bits in a value type
396 /// half the size of Op's.
397 void DAGTypeLegalizer::SplitOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi) {
398 unsigned NVTBits = MVT::getSizeInBits(Op.getValueType())/2;
399 assert(MVT::getSizeInBits(Op.getValueType()) == 2*NVTBits &&
400 "Cannot split odd sized integer type");
401 MVT::ValueType NVT = MVT::getIntegerType(NVTBits);
402 Lo = DAG.getNode(ISD::TRUNCATE, NVT, Op);
403 Hi = DAG.getNode(ISD::SRL, Op.getValueType(), Op,
404 DAG.getConstant(NVTBits, TLI.getShiftAmountTy()));
405 Hi = DAG.getNode(ISD::TRUNCATE, NVT, Hi);
409 //===----------------------------------------------------------------------===//
411 //===----------------------------------------------------------------------===//
413 /// PromoteResult - This method is called when a result of a node is found to be
414 /// in need of promotion to a larger type. At this point, the node may also
415 /// have invalid operands or may have other results that need expansion, we just
416 /// know that (at least) one result needs promotion.
417 void DAGTypeLegalizer::PromoteResult(SDNode *N, unsigned ResNo) {
418 DEBUG(cerr << "Promote node result: "; N->dump(&DAG); cerr << "\n");
419 SDOperand Result = SDOperand();
421 switch (N->getOpcode()) {
424 cerr << "PromoteResult #" << ResNo << ": ";
425 N->dump(&DAG); cerr << "\n";
427 assert(0 && "Do not know how to promote this operator!");
429 case ISD::UNDEF: Result = PromoteResult_UNDEF(N); break;
430 case ISD::Constant: Result = PromoteResult_Constant(N); break;
432 case ISD::TRUNCATE: Result = PromoteResult_TRUNCATE(N); break;
433 case ISD::SIGN_EXTEND:
434 case ISD::ZERO_EXTEND:
435 case ISD::ANY_EXTEND: Result = PromoteResult_INT_EXTEND(N); break;
436 case ISD::FP_ROUND: Result = PromoteResult_FP_ROUND(N); break;
437 case ISD::FP_TO_SINT:
438 case ISD::FP_TO_UINT: Result = PromoteResult_FP_TO_XINT(N); break;
439 case ISD::SETCC: Result = PromoteResult_SETCC(N); break;
440 case ISD::LOAD: Result = PromoteResult_LOAD(cast<LoadSDNode>(N)); break;
447 case ISD::MUL: Result = PromoteResult_SimpleIntBinOp(N); break;
450 case ISD::SREM: Result = PromoteResult_SDIV(N); break;
453 case ISD::UREM: Result = PromoteResult_UDIV(N); break;
455 case ISD::SHL: Result = PromoteResult_SHL(N); break;
456 case ISD::SRA: Result = PromoteResult_SRA(N); break;
457 case ISD::SRL: Result = PromoteResult_SRL(N); break;
459 case ISD::SELECT: Result = PromoteResult_SELECT(N); break;
460 case ISD::SELECT_CC: Result = PromoteResult_SELECT_CC(N); break;
464 // If Result is null, the sub-method took care of registering the result.
466 SetPromotedOp(SDOperand(N, ResNo), Result);
469 SDOperand DAGTypeLegalizer::PromoteResult_UNDEF(SDNode *N) {
470 return DAG.getNode(ISD::UNDEF, TLI.getTypeToTransformTo(N->getValueType(0)));
473 SDOperand DAGTypeLegalizer::PromoteResult_Constant(SDNode *N) {
474 MVT::ValueType VT = N->getValueType(0);
475 // Zero extend things like i1, sign extend everything else. It shouldn't
476 // matter in theory which one we pick, but this tends to give better code?
477 unsigned Opc = VT != MVT::i1 ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
478 SDOperand Result = DAG.getNode(Opc, TLI.getTypeToTransformTo(VT),
480 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
484 SDOperand DAGTypeLegalizer::PromoteResult_TRUNCATE(SDNode *N) {
487 switch (getTypeAction(N->getOperand(0).getValueType())) {
488 default: assert(0 && "Unknown type action!");
491 Res = N->getOperand(0);
494 Res = GetPromotedOp(N->getOperand(0));
498 MVT::ValueType NVT = TLI.getTypeToTransformTo(N->getValueType(0));
499 assert(MVT::getSizeInBits(Res.getValueType()) >= MVT::getSizeInBits(NVT) &&
500 "Truncation doesn't make sense!");
501 if (Res.getValueType() == NVT)
504 // Truncate to NVT instead of VT
505 return DAG.getNode(ISD::TRUNCATE, NVT, Res);
508 SDOperand DAGTypeLegalizer::PromoteResult_INT_EXTEND(SDNode *N) {
509 MVT::ValueType NVT = TLI.getTypeToTransformTo(N->getValueType(0));
511 if (getTypeAction(N->getOperand(0).getValueType()) == Promote) {
512 SDOperand Res = GetPromotedOp(N->getOperand(0));
513 assert(MVT::getSizeInBits(Res.getValueType()) <= MVT::getSizeInBits(NVT) &&
514 "Extension doesn't make sense!");
516 // If the result and operand types are the same after promotion, simplify
517 // to an in-register extension.
518 if (NVT == Res.getValueType()) {
519 // The high bits are not guaranteed to be anything. Insert an extend.
520 if (N->getOpcode() == ISD::SIGN_EXTEND)
521 return DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Res,
522 DAG.getValueType(N->getOperand(0).getValueType()));
523 if (N->getOpcode() == ISD::ZERO_EXTEND)
524 return DAG.getZeroExtendInReg(Res, N->getOperand(0).getValueType());
525 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
530 // Otherwise, just extend the original operand all the way to the larger type.
531 return DAG.getNode(N->getOpcode(), NVT, N->getOperand(0));
534 SDOperand DAGTypeLegalizer::PromoteResult_FP_ROUND(SDNode *N) {
535 // NOTE: Assumes input is legal.
536 return DAG.getNode(ISD::FP_ROUND_INREG, N->getOperand(0).getValueType(),
537 N->getOperand(0), DAG.getValueType(N->getValueType(0)));
540 SDOperand DAGTypeLegalizer::PromoteResult_FP_TO_XINT(SDNode *N) {
541 SDOperand Op = N->getOperand(0);
542 // If the operand needed to be promoted, do so now.
543 if (getTypeAction(Op.getValueType()) == Promote)
544 // The input result is prerounded, so we don't have to do anything special.
545 Op = GetPromotedOp(Op);
547 unsigned NewOpc = N->getOpcode();
548 MVT::ValueType NVT = TLI.getTypeToTransformTo(N->getValueType(0));
550 // If we're promoting a UINT to a larger size, check to see if the new node
551 // will be legal. If it isn't, check to see if FP_TO_SINT is legal, since
552 // we can use that instead. This allows us to generate better code for
553 // FP_TO_UINT for small destination sizes on targets where FP_TO_UINT is not
554 // legal, such as PowerPC.
555 if (N->getOpcode() == ISD::FP_TO_UINT) {
556 if (!TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
557 (TLI.isOperationLegal(ISD::FP_TO_SINT, NVT) ||
558 TLI.getOperationAction(ISD::FP_TO_SINT, NVT)==TargetLowering::Custom))
559 NewOpc = ISD::FP_TO_SINT;
562 return DAG.getNode(NewOpc, NVT, Op);
565 SDOperand DAGTypeLegalizer::PromoteResult_SETCC(SDNode *N) {
566 assert(isTypeLegal(TLI.getSetCCResultTy()) && "SetCC type is not legal??");
567 return DAG.getNode(ISD::SETCC, TLI.getSetCCResultTy(), N->getOperand(0),
568 N->getOperand(1), N->getOperand(2));
571 SDOperand DAGTypeLegalizer::PromoteResult_LOAD(LoadSDNode *N) {
572 MVT::ValueType NVT = TLI.getTypeToTransformTo(N->getValueType(0));
573 ISD::LoadExtType ExtType =
574 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
575 SDOperand Res = DAG.getExtLoad(ExtType, NVT, N->getChain(), N->getBasePtr(),
576 N->getSrcValue(), N->getSrcValueOffset(),
577 N->getLoadedVT(), N->isVolatile(),
580 // Legalized the chain result - switch anything that used the old chain to
582 ReplaceValueWith(SDOperand(N, 1), Res.getValue(1));
586 SDOperand DAGTypeLegalizer::PromoteResult_SimpleIntBinOp(SDNode *N) {
587 // The input may have strange things in the top bits of the registers, but
588 // these operations don't care. They may have weird bits going out, but
589 // that too is okay if they are integer operations.
590 SDOperand LHS = GetPromotedOp(N->getOperand(0));
591 SDOperand RHS = GetPromotedOp(N->getOperand(1));
592 return DAG.getNode(N->getOpcode(), LHS.getValueType(), LHS, RHS);
595 SDOperand DAGTypeLegalizer::PromoteResult_SDIV(SDNode *N) {
596 // Sign extend the input.
597 SDOperand LHS = GetPromotedOp(N->getOperand(0));
598 SDOperand RHS = GetPromotedOp(N->getOperand(1));
599 MVT::ValueType VT = N->getValueType(0);
600 LHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, LHS.getValueType(), LHS,
601 DAG.getValueType(VT));
602 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, RHS.getValueType(), RHS,
603 DAG.getValueType(VT));
605 return DAG.getNode(N->getOpcode(), LHS.getValueType(), LHS, RHS);
608 SDOperand DAGTypeLegalizer::PromoteResult_UDIV(SDNode *N) {
609 // Zero extend the input.
610 SDOperand LHS = GetPromotedOp(N->getOperand(0));
611 SDOperand RHS = GetPromotedOp(N->getOperand(1));
612 MVT::ValueType VT = N->getValueType(0);
613 LHS = DAG.getZeroExtendInReg(LHS, VT);
614 RHS = DAG.getZeroExtendInReg(RHS, VT);
616 return DAG.getNode(N->getOpcode(), LHS.getValueType(), LHS, RHS);
619 SDOperand DAGTypeLegalizer::PromoteResult_SHL(SDNode *N) {
620 return DAG.getNode(ISD::SHL, TLI.getTypeToTransformTo(N->getValueType(0)),
621 GetPromotedOp(N->getOperand(0)), N->getOperand(1));
624 SDOperand DAGTypeLegalizer::PromoteResult_SRA(SDNode *N) {
625 // The input value must be properly sign extended.
626 MVT::ValueType VT = N->getValueType(0);
627 MVT::ValueType NVT = TLI.getTypeToTransformTo(VT);
628 SDOperand Res = GetPromotedOp(N->getOperand(0));
629 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Res, DAG.getValueType(VT));
630 return DAG.getNode(ISD::SRA, NVT, Res, N->getOperand(1));
633 SDOperand DAGTypeLegalizer::PromoteResult_SRL(SDNode *N) {
634 // The input value must be properly zero extended.
635 MVT::ValueType VT = N->getValueType(0);
636 MVT::ValueType NVT = TLI.getTypeToTransformTo(VT);
637 SDOperand Res = GetPromotedZExtOp(N->getOperand(0));
638 return DAG.getNode(ISD::SRL, NVT, Res, N->getOperand(1));
641 SDOperand DAGTypeLegalizer::PromoteResult_SELECT(SDNode *N) {
642 SDOperand LHS = GetPromotedOp(N->getOperand(1));
643 SDOperand RHS = GetPromotedOp(N->getOperand(2));
644 return DAG.getNode(ISD::SELECT, LHS.getValueType(), N->getOperand(0),LHS,RHS);
647 SDOperand DAGTypeLegalizer::PromoteResult_SELECT_CC(SDNode *N) {
648 SDOperand LHS = GetPromotedOp(N->getOperand(2));
649 SDOperand RHS = GetPromotedOp(N->getOperand(3));
650 return DAG.getNode(ISD::SELECT_CC, LHS.getValueType(), N->getOperand(0),
651 N->getOperand(1), LHS, RHS, N->getOperand(4));
655 //===----------------------------------------------------------------------===//
657 //===----------------------------------------------------------------------===//
659 /// ExpandResult - This method is called when the specified result of the
660 /// specified node is found to need expansion. At this point, the node may also
661 /// have invalid operands or may have other results that need promotion, we just
662 /// know that (at least) one result needs expansion.
663 void DAGTypeLegalizer::ExpandResult(SDNode *N, unsigned ResNo) {
664 DEBUG(cerr << "Expand node result: "; N->dump(&DAG); cerr << "\n");
666 Lo = Hi = SDOperand();
668 // See if the target wants to custom expand this node.
669 if (TLI.getOperationAction(N->getOpcode(), N->getValueType(0)) ==
670 TargetLowering::Custom) {
671 // If the target wants to, allow it to lower this itself.
672 if (SDNode *P = TLI.ExpandOperationResult(N, DAG)) {
673 // Everything that once used N now uses P. We are guaranteed that the
674 // result value types of N and the result value types of P match.
675 ReplaceNodeWith(N, P);
680 switch (N->getOpcode()) {
683 cerr << "ExpandResult #" << ResNo << ": ";
684 N->dump(&DAG); cerr << "\n";
686 assert(0 && "Do not know how to expand the result of this operator!");
689 case ISD::UNDEF: ExpandResult_UNDEF(N, Lo, Hi); break;
690 case ISD::Constant: ExpandResult_Constant(N, Lo, Hi); break;
691 case ISD::BUILD_PAIR: ExpandResult_BUILD_PAIR(N, Lo, Hi); break;
692 case ISD::MERGE_VALUES: ExpandResult_MERGE_VALUES(N, Lo, Hi); break;
693 case ISD::ANY_EXTEND: ExpandResult_ANY_EXTEND(N, Lo, Hi); break;
694 case ISD::ZERO_EXTEND: ExpandResult_ZERO_EXTEND(N, Lo, Hi); break;
695 case ISD::SIGN_EXTEND: ExpandResult_SIGN_EXTEND(N, Lo, Hi); break;
696 case ISD::BIT_CONVERT: ExpandResult_BIT_CONVERT(N, Lo, Hi); break;
697 case ISD::SIGN_EXTEND_INREG: ExpandResult_SIGN_EXTEND_INREG(N, Lo, Hi); break;
698 case ISD::LOAD: ExpandResult_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
702 case ISD::XOR: ExpandResult_Logical(N, Lo, Hi); break;
703 case ISD::BSWAP: ExpandResult_BSWAP(N, Lo, Hi); break;
705 case ISD::SUB: ExpandResult_ADDSUB(N, Lo, Hi); break;
707 case ISD::SUBC: ExpandResult_ADDSUBC(N, Lo, Hi); break;
709 case ISD::SUBE: ExpandResult_ADDSUBE(N, Lo, Hi); break;
710 case ISD::SELECT: ExpandResult_SELECT(N, Lo, Hi); break;
711 case ISD::SELECT_CC: ExpandResult_SELECT_CC(N, Lo, Hi); break;
712 case ISD::MUL: ExpandResult_MUL(N, Lo, Hi); break;
715 case ISD::SRL: ExpandResult_Shift(N, Lo, Hi); break;
718 // If Lo/Hi is null, the sub-method took care of registering results etc.
720 SetExpandedOp(SDOperand(N, ResNo), Lo, Hi);
723 void DAGTypeLegalizer::ExpandResult_UNDEF(SDNode *N,
724 SDOperand &Lo, SDOperand &Hi) {
725 MVT::ValueType NVT = TLI.getTypeToTransformTo(N->getValueType(0));
726 Lo = Hi = DAG.getNode(ISD::UNDEF, NVT);
729 void DAGTypeLegalizer::ExpandResult_Constant(SDNode *N,
730 SDOperand &Lo, SDOperand &Hi) {
731 MVT::ValueType NVT = TLI.getTypeToTransformTo(N->getValueType(0));
732 uint64_t Cst = cast<ConstantSDNode>(N)->getValue();
733 Lo = DAG.getConstant(Cst, NVT);
734 Hi = DAG.getConstant(Cst >> MVT::getSizeInBits(NVT), NVT);
737 void DAGTypeLegalizer::ExpandResult_BUILD_PAIR(SDNode *N,
738 SDOperand &Lo, SDOperand &Hi) {
739 // Return the operands.
740 Lo = N->getOperand(0);
741 Hi = N->getOperand(1);
744 void DAGTypeLegalizer::ExpandResult_MERGE_VALUES(SDNode *N,
745 SDOperand &Lo, SDOperand &Hi) {
746 // A MERGE_VALUES node can produce any number of values. We know that the
747 // first illegal one needs to be expanded into Lo/Hi.
750 // The string of legal results gets turns into the input operands, which have
752 for (i = 0; isTypeLegal(N->getValueType(i)); ++i)
753 ReplaceValueWith(SDOperand(N, i), SDOperand(N->getOperand(i)));
755 // The first illegal result must be the one that needs to be expanded.
756 GetExpandedOp(N->getOperand(i), Lo, Hi);
758 // Legalize the rest of the results into the input operands whether they are
760 unsigned e = N->getNumValues();
761 for (++i; i != e; ++i)
762 ReplaceValueWith(SDOperand(N, i), SDOperand(N->getOperand(i)));
765 void DAGTypeLegalizer::ExpandResult_ANY_EXTEND(SDNode *N,
766 SDOperand &Lo, SDOperand &Hi) {
767 MVT::ValueType NVT = TLI.getTypeToTransformTo(N->getValueType(0));
768 SDOperand Op = N->getOperand(0);
769 if (MVT::getSizeInBits(Op.getValueType()) <= MVT::getSizeInBits(NVT)) {
770 // The low part is any extension of the input (which degenerates to a copy).
771 Lo = DAG.getNode(ISD::ANY_EXTEND, NVT, Op);
772 Hi = DAG.getNode(ISD::UNDEF, NVT); // The high part is undefined.
774 // For example, extension of an i48 to an i64. The operand type necessarily
775 // promotes to the result type, so will end up being expanded too.
776 assert(getTypeAction(Op.getValueType()) == Promote &&
777 "Don't know how to expand this result!");
778 SDOperand Res = GetPromotedOp(Op);
779 assert(Res.getValueType() == N->getValueType(0) &&
780 "Operand over promoted?");
781 // Split the promoted operand. This will simplify when it is expanded.
782 SplitOp(Res, Lo, Hi);
786 void DAGTypeLegalizer::ExpandResult_ZERO_EXTEND(SDNode *N,
787 SDOperand &Lo, SDOperand &Hi) {
788 MVT::ValueType NVT = TLI.getTypeToTransformTo(N->getValueType(0));
789 SDOperand Op = N->getOperand(0);
790 if (MVT::getSizeInBits(Op.getValueType()) <= MVT::getSizeInBits(NVT)) {
791 // The low part is zero extension of the input (which degenerates to a copy).
792 Lo = DAG.getNode(ISD::ZERO_EXTEND, NVT, N->getOperand(0));
793 Hi = DAG.getConstant(0, NVT); // The high part is just a zero.
795 // For example, extension of an i48 to an i64. The operand type necessarily
796 // promotes to the result type, so will end up being expanded too.
797 assert(getTypeAction(Op.getValueType()) == Promote &&
798 "Don't know how to expand this result!");
799 SDOperand Res = GetPromotedOp(Op);
800 assert(Res.getValueType() == N->getValueType(0) &&
801 "Operand over promoted?");
802 // Split the promoted operand. This will simplify when it is expanded.
803 SplitOp(Res, Lo, Hi);
804 unsigned ExcessBits =
805 MVT::getSizeInBits(Op.getValueType()) - MVT::getSizeInBits(NVT);
806 Hi = DAG.getZeroExtendInReg(Hi, MVT::getIntegerType(ExcessBits));
810 void DAGTypeLegalizer::ExpandResult_SIGN_EXTEND(SDNode *N,
811 SDOperand &Lo, SDOperand &Hi) {
812 MVT::ValueType NVT = TLI.getTypeToTransformTo(N->getValueType(0));
813 SDOperand Op = N->getOperand(0);
814 if (MVT::getSizeInBits(Op.getValueType()) <= MVT::getSizeInBits(NVT)) {
815 // The low part is sign extension of the input (which degenerates to a copy).
816 Lo = DAG.getNode(ISD::SIGN_EXTEND, NVT, N->getOperand(0));
817 // The high part is obtained by SRA'ing all but one of the bits of low part.
818 unsigned LoSize = MVT::getSizeInBits(NVT);
819 Hi = DAG.getNode(ISD::SRA, NVT, Lo,
820 DAG.getConstant(LoSize-1, TLI.getShiftAmountTy()));
822 // For example, extension of an i48 to an i64. The operand type necessarily
823 // promotes to the result type, so will end up being expanded too.
824 assert(getTypeAction(Op.getValueType()) == Promote &&
825 "Don't know how to expand this result!");
826 SDOperand Res = GetPromotedOp(Op);
827 assert(Res.getValueType() == N->getValueType(0) &&
828 "Operand over promoted?");
829 // Split the promoted operand. This will simplify when it is expanded.
830 SplitOp(Res, Lo, Hi);
831 unsigned ExcessBits =
832 MVT::getSizeInBits(Op.getValueType()) - MVT::getSizeInBits(NVT);
833 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, Hi.getValueType(), Hi,
834 DAG.getValueType(MVT::getIntegerType(ExcessBits)));
838 void DAGTypeLegalizer::ExpandResult_BIT_CONVERT(SDNode *N,
839 SDOperand &Lo, SDOperand &Hi) {
840 // Lower the bit-convert to a store/load from the stack, then expand the load.
841 SDOperand Op = CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
842 ExpandResult_LOAD(cast<LoadSDNode>(Op.Val), Lo, Hi);
845 void DAGTypeLegalizer::
846 ExpandResult_SIGN_EXTEND_INREG(SDNode *N, SDOperand &Lo, SDOperand &Hi) {
847 GetExpandedOp(N->getOperand(0), Lo, Hi);
848 MVT::ValueType EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
850 if (MVT::getSizeInBits(EVT) <= MVT::getSizeInBits(Lo.getValueType())) {
851 // sext_inreg the low part if needed.
852 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, Lo.getValueType(), Lo,
855 // The high part gets the sign extension from the lo-part. This handles
856 // things like sextinreg V:i64 from i8.
857 Hi = DAG.getNode(ISD::SRA, Hi.getValueType(), Lo,
858 DAG.getConstant(MVT::getSizeInBits(Hi.getValueType())-1,
859 TLI.getShiftAmountTy()));
861 // For example, extension of an i48 to an i64. Leave the low part alone,
862 // sext_inreg the high part.
863 unsigned ExcessBits =
864 MVT::getSizeInBits(EVT) - MVT::getSizeInBits(Lo.getValueType());
865 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, Hi.getValueType(), Hi,
866 DAG.getValueType(MVT::getIntegerType(ExcessBits)));
870 void DAGTypeLegalizer::ExpandResult_LOAD(LoadSDNode *N,
871 SDOperand &Lo, SDOperand &Hi) {
872 MVT::ValueType VT = N->getValueType(0);
873 MVT::ValueType NVT = TLI.getTypeToTransformTo(VT);
874 SDOperand Ch = N->getChain(); // Legalize the chain.
875 SDOperand Ptr = N->getBasePtr(); // Legalize the pointer.
876 ISD::LoadExtType ExtType = N->getExtensionType();
877 int SVOffset = N->getSrcValueOffset();
878 unsigned Alignment = N->getAlignment();
879 bool isVolatile = N->isVolatile();
881 assert(!(MVT::getSizeInBits(NVT) & 7) && "Expanded type not byte sized!");
883 if (ExtType == ISD::NON_EXTLOAD) {
884 Lo = DAG.getLoad(NVT, Ch, Ptr, N->getSrcValue(), SVOffset,
885 isVolatile, Alignment);
886 // Increment the pointer to the other half.
887 unsigned IncrementSize = MVT::getSizeInBits(NVT)/8;
888 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
889 getIntPtrConstant(IncrementSize));
890 Hi = DAG.getLoad(NVT, Ch, Ptr, N->getSrcValue(), SVOffset+IncrementSize,
891 isVolatile, MinAlign(Alignment, IncrementSize));
893 // Build a factor node to remember that this load is independent of the
895 Ch = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1),
898 // Handle endianness of the load.
899 if (!TLI.isLittleEndian())
901 } else if (MVT::getSizeInBits(N->getLoadedVT()) <= MVT::getSizeInBits(NVT)) {
902 MVT::ValueType EVT = N->getLoadedVT();
904 Lo = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, N->getSrcValue(), SVOffset, EVT,
905 isVolatile, Alignment);
907 // Remember the chain.
910 if (ExtType == ISD::SEXTLOAD) {
911 // The high part is obtained by SRA'ing all but one of the bits of the
913 unsigned LoSize = MVT::getSizeInBits(Lo.getValueType());
914 Hi = DAG.getNode(ISD::SRA, NVT, Lo,
915 DAG.getConstant(LoSize-1, TLI.getShiftAmountTy()));
916 } else if (ExtType == ISD::ZEXTLOAD) {
917 // The high part is just a zero.
918 Hi = DAG.getConstant(0, NVT);
920 assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
921 // The high part is undefined.
922 Hi = DAG.getNode(ISD::UNDEF, NVT);
924 } else if (TLI.isLittleEndian()) {
925 // Little-endian - low bits are at low addresses.
926 Lo = DAG.getLoad(NVT, Ch, Ptr, N->getSrcValue(), SVOffset,
927 isVolatile, Alignment);
929 unsigned ExcessBits =
930 MVT::getSizeInBits(N->getLoadedVT()) - MVT::getSizeInBits(NVT);
931 MVT::ValueType NEVT = MVT::getIntegerType(ExcessBits);
933 // Increment the pointer to the other half.
934 unsigned IncrementSize = MVT::getSizeInBits(NVT)/8;
935 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
936 getIntPtrConstant(IncrementSize));
937 Hi = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, N->getSrcValue(),
938 SVOffset+IncrementSize, NEVT,
939 isVolatile, MinAlign(Alignment, IncrementSize));
941 // Build a factor node to remember that this load is independent of the
943 Ch = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1),
946 // Big-endian - high bits are at low addresses. Favor aligned loads at
947 // the cost of some bit-fiddling.
948 MVT::ValueType EVT = N->getLoadedVT();
949 unsigned EBytes = MVT::getStoreSizeInBits(EVT)/8;
950 unsigned IncrementSize = MVT::getSizeInBits(NVT)/8;
951 unsigned ExcessBits = (EBytes - IncrementSize)*8;
953 // Load both the high bits and maybe some of the low bits.
954 Hi = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, N->getSrcValue(), SVOffset,
955 MVT::getIntegerType(MVT::getSizeInBits(EVT)-ExcessBits),
956 isVolatile, Alignment);
958 // Increment the pointer to the other half.
959 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
960 getIntPtrConstant(IncrementSize));
961 // Load the rest of the low bits.
962 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, NVT, Ch, Ptr, N->getSrcValue(),
963 SVOffset+IncrementSize, MVT::getIntegerType(ExcessBits),
964 isVolatile, MinAlign(Alignment, IncrementSize));
966 // Build a factor node to remember that this load is independent of the
968 Ch = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1),
971 if (ExcessBits < MVT::getSizeInBits(NVT)) {
972 // Transfer low bits from the bottom of Hi to the top of Lo.
973 Lo = DAG.getNode(ISD::OR, NVT, Lo,
974 DAG.getNode(ISD::SHL, NVT, Hi,
975 DAG.getConstant(ExcessBits,
976 TLI.getShiftAmountTy())));
977 // Move high bits to the right position in Hi.
978 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, NVT, Hi,
979 DAG.getConstant(MVT::getSizeInBits(NVT) - ExcessBits,
980 TLI.getShiftAmountTy()));
984 // Legalized the chain result - switch anything that used the old chain to
986 ReplaceValueWith(SDOperand(N, 1), Ch);
989 void DAGTypeLegalizer::ExpandResult_Logical(SDNode *N,
990 SDOperand &Lo, SDOperand &Hi) {
991 SDOperand LL, LH, RL, RH;
992 GetExpandedOp(N->getOperand(0), LL, LH);
993 GetExpandedOp(N->getOperand(1), RL, RH);
994 Lo = DAG.getNode(N->getOpcode(), LL.getValueType(), LL, RL);
995 Hi = DAG.getNode(N->getOpcode(), LL.getValueType(), LH, RH);
998 void DAGTypeLegalizer::ExpandResult_BSWAP(SDNode *N,
999 SDOperand &Lo, SDOperand &Hi) {
1000 GetExpandedOp(N->getOperand(0), Hi, Lo); // Note swapped operands.
1001 Lo = DAG.getNode(ISD::BSWAP, Lo.getValueType(), Lo);
1002 Hi = DAG.getNode(ISD::BSWAP, Hi.getValueType(), Hi);
1005 void DAGTypeLegalizer::ExpandResult_SELECT(SDNode *N,
1006 SDOperand &Lo, SDOperand &Hi) {
1007 SDOperand LL, LH, RL, RH;
1008 GetExpandedOp(N->getOperand(1), LL, LH);
1009 GetExpandedOp(N->getOperand(2), RL, RH);
1010 Lo = DAG.getNode(ISD::SELECT, LL.getValueType(), N->getOperand(0), LL, RL);
1012 assert(N->getOperand(0).getValueType() != MVT::f32 &&
1013 "FIXME: softfp shouldn't use expand!");
1014 Hi = DAG.getNode(ISD::SELECT, LL.getValueType(), N->getOperand(0), LH, RH);
1017 void DAGTypeLegalizer::ExpandResult_SELECT_CC(SDNode *N,
1018 SDOperand &Lo, SDOperand &Hi) {
1019 SDOperand LL, LH, RL, RH;
1020 GetExpandedOp(N->getOperand(2), LL, LH);
1021 GetExpandedOp(N->getOperand(3), RL, RH);
1022 Lo = DAG.getNode(ISD::SELECT_CC, LL.getValueType(), N->getOperand(0),
1023 N->getOperand(1), LL, RL, N->getOperand(4));
1025 assert(N->getOperand(0).getValueType() != MVT::f32 &&
1026 "FIXME: softfp shouldn't use expand!");
1027 Hi = DAG.getNode(ISD::SELECT_CC, LL.getValueType(), N->getOperand(0),
1028 N->getOperand(1), LH, RH, N->getOperand(4));
1031 void DAGTypeLegalizer::ExpandResult_ADDSUB(SDNode *N,
1032 SDOperand &Lo, SDOperand &Hi) {
1033 // Expand the subcomponents.
1034 SDOperand LHSL, LHSH, RHSL, RHSH;
1035 GetExpandedOp(N->getOperand(0), LHSL, LHSH);
1036 GetExpandedOp(N->getOperand(1), RHSL, RHSH);
1037 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag);
1038 SDOperand LoOps[2] = { LHSL, RHSL };
1039 SDOperand HiOps[3] = { LHSH, RHSH };
1041 if (N->getOpcode() == ISD::ADD) {
1042 Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2);
1043 HiOps[2] = Lo.getValue(1);
1044 Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3);
1046 Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2);
1047 HiOps[2] = Lo.getValue(1);
1048 Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3);
1052 void DAGTypeLegalizer::ExpandResult_ADDSUBC(SDNode *N,
1053 SDOperand &Lo, SDOperand &Hi) {
1054 // Expand the subcomponents.
1055 SDOperand LHSL, LHSH, RHSL, RHSH;
1056 GetExpandedOp(N->getOperand(0), LHSL, LHSH);
1057 GetExpandedOp(N->getOperand(1), RHSL, RHSH);
1058 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag);
1059 SDOperand LoOps[2] = { LHSL, RHSL };
1060 SDOperand HiOps[3] = { LHSH, RHSH };
1062 if (N->getOpcode() == ISD::ADDC) {
1063 Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2);
1064 HiOps[2] = Lo.getValue(1);
1065 Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3);
1067 Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2);
1068 HiOps[2] = Lo.getValue(1);
1069 Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3);
1072 // Legalized the flag result - switch anything that used the old flag to
1074 ReplaceValueWith(SDOperand(N, 1), Hi.getValue(1));
1077 void DAGTypeLegalizer::ExpandResult_ADDSUBE(SDNode *N,
1078 SDOperand &Lo, SDOperand &Hi) {
1079 // Expand the subcomponents.
1080 SDOperand LHSL, LHSH, RHSL, RHSH;
1081 GetExpandedOp(N->getOperand(0), LHSL, LHSH);
1082 GetExpandedOp(N->getOperand(1), RHSL, RHSH);
1083 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag);
1084 SDOperand LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
1085 SDOperand HiOps[3] = { LHSH, RHSH };
1087 Lo = DAG.getNode(N->getOpcode(), VTList, LoOps, 3);
1088 HiOps[2] = Lo.getValue(1);
1089 Hi = DAG.getNode(N->getOpcode(), VTList, HiOps, 3);
1091 // Legalized the flag result - switch anything that used the old flag to
1093 ReplaceValueWith(SDOperand(N, 1), Hi.getValue(1));
1096 void DAGTypeLegalizer::ExpandResult_MUL(SDNode *N,
1097 SDOperand &Lo, SDOperand &Hi) {
1098 MVT::ValueType VT = N->getValueType(0);
1099 MVT::ValueType NVT = TLI.getTypeToTransformTo(VT);
1101 bool HasMULHS = TLI.isOperationLegal(ISD::MULHS, NVT);
1102 bool HasMULHU = TLI.isOperationLegal(ISD::MULHU, NVT);
1103 bool HasSMUL_LOHI = TLI.isOperationLegal(ISD::SMUL_LOHI, NVT);
1104 bool HasUMUL_LOHI = TLI.isOperationLegal(ISD::UMUL_LOHI, NVT);
1105 if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) {
1106 SDOperand LL, LH, RL, RH;
1107 GetExpandedOp(N->getOperand(0), LL, LH);
1108 GetExpandedOp(N->getOperand(1), RL, RH);
1109 unsigned BitSize = MVT::getSizeInBits(NVT);
1110 unsigned LHSSB = DAG.ComputeNumSignBits(N->getOperand(0));
1111 unsigned RHSSB = DAG.ComputeNumSignBits(N->getOperand(1));
1113 // FIXME: generalize this to handle other bit sizes
1114 if (LHSSB == 32 && RHSSB == 32 &&
1115 DAG.MaskedValueIsZero(N->getOperand(0), 0xFFFFFFFF00000000ULL) &&
1116 DAG.MaskedValueIsZero(N->getOperand(1), 0xFFFFFFFF00000000ULL)) {
1117 // The inputs are both zero-extended.
1119 // We can emit a umul_lohi.
1120 Lo = DAG.getNode(ISD::UMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL);
1121 Hi = SDOperand(Lo.Val, 1);
1125 // We can emit a mulhu+mul.
1126 Lo = DAG.getNode(ISD::MUL, NVT, LL, RL);
1127 Hi = DAG.getNode(ISD::MULHU, NVT, LL, RL);
1131 if (LHSSB > BitSize && RHSSB > BitSize) {
1132 // The input values are both sign-extended.
1134 // We can emit a smul_lohi.
1135 Lo = DAG.getNode(ISD::SMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL);
1136 Hi = SDOperand(Lo.Val, 1);
1140 // We can emit a mulhs+mul.
1141 Lo = DAG.getNode(ISD::MUL, NVT, LL, RL);
1142 Hi = DAG.getNode(ISD::MULHS, NVT, LL, RL);
1147 // Lo,Hi = umul LHS, RHS.
1148 SDOperand UMulLOHI = DAG.getNode(ISD::UMUL_LOHI,
1149 DAG.getVTList(NVT, NVT), LL, RL);
1151 Hi = UMulLOHI.getValue(1);
1152 RH = DAG.getNode(ISD::MUL, NVT, LL, RH);
1153 LH = DAG.getNode(ISD::MUL, NVT, LH, RL);
1154 Hi = DAG.getNode(ISD::ADD, NVT, Hi, RH);
1155 Hi = DAG.getNode(ISD::ADD, NVT, Hi, LH);
1162 // If nothing else, we can make a libcall.
1163 Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::MUL_I64), N,
1164 false/*sign irrelevant*/, Hi);
1169 void DAGTypeLegalizer::ExpandResult_Shift(SDNode *N,
1170 SDOperand &Lo, SDOperand &Hi) {
1171 MVT::ValueType VT = N->getValueType(0);
1173 // If we can emit an efficient shift operation, do so now. Check to see if
1174 // the RHS is a constant.
1175 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
1176 return ExpandShiftByConstant(N, CN->getValue(), Lo, Hi);
1178 // If we can determine that the high bit of the shift is zero or one, even if
1179 // the low bits are variable, emit this shift in an optimized form.
1180 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
1183 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
1185 if (N->getOpcode() == ISD::SHL)
1186 PartsOpc = ISD::SHL_PARTS;
1187 else if (N->getOpcode() == ISD::SRL)
1188 PartsOpc = ISD::SRL_PARTS;
1190 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
1191 PartsOpc = ISD::SRA_PARTS;
1194 // Next check to see if the target supports this SHL_PARTS operation or if it
1195 // will custom expand it.
1196 MVT::ValueType NVT = TLI.getTypeToTransformTo(VT);
1197 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
1198 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
1199 Action == TargetLowering::Custom) {
1200 // Expand the subcomponents.
1201 SDOperand LHSL, LHSH;
1202 GetExpandedOp(N->getOperand(0), LHSL, LHSH);
1204 SDOperand Ops[] = { LHSL, LHSH, N->getOperand(1) };
1205 MVT::ValueType VT = LHSL.getValueType();
1206 Lo = DAG.getNode(PartsOpc, DAG.getNodeValueTypes(VT, VT), 2, Ops, 3);
1207 Hi = Lo.getValue(1);
1213 // Otherwise, emit a libcall.
1214 unsigned RuntimeCode = ; // SRL -> SRL_I64 etc.
1216 Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::SRL_I64), N,
1217 false/*lshr is unsigned*/, Hi);
1222 /// ExpandShiftByConstant - N is a shift by a value that needs to be expanded,
1223 /// and the shift amount is a constant 'Amt'. Expand the operation.
1224 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, unsigned Amt,
1225 SDOperand &Lo, SDOperand &Hi) {
1226 // Expand the incoming operand to be shifted, so that we have its parts
1228 GetExpandedOp(N->getOperand(0), InL, InH);
1230 MVT::ValueType NVT = InL.getValueType();
1231 unsigned VTBits = MVT::getSizeInBits(N->getValueType(0));
1232 unsigned NVTBits = MVT::getSizeInBits(NVT);
1233 MVT::ValueType ShTy = N->getOperand(1).getValueType();
1235 if (N->getOpcode() == ISD::SHL) {
1237 Lo = Hi = DAG.getConstant(0, NVT);
1238 } else if (Amt > NVTBits) {
1239 Lo = DAG.getConstant(0, NVT);
1240 Hi = DAG.getNode(ISD::SHL, NVT, InL, DAG.getConstant(Amt-NVTBits,ShTy));
1241 } else if (Amt == NVTBits) {
1242 Lo = DAG.getConstant(0, NVT);
1245 Lo = DAG.getNode(ISD::SHL, NVT, InL, DAG.getConstant(Amt, ShTy));
1246 Hi = DAG.getNode(ISD::OR, NVT,
1247 DAG.getNode(ISD::SHL, NVT, InH,
1248 DAG.getConstant(Amt, ShTy)),
1249 DAG.getNode(ISD::SRL, NVT, InL,
1250 DAG.getConstant(NVTBits-Amt, ShTy)));
1255 if (N->getOpcode() == ISD::SRL) {
1257 Lo = DAG.getConstant(0, NVT);
1258 Hi = DAG.getConstant(0, NVT);
1259 } else if (Amt > NVTBits) {
1260 Lo = DAG.getNode(ISD::SRL, NVT, InH, DAG.getConstant(Amt-NVTBits,ShTy));
1261 Hi = DAG.getConstant(0, NVT);
1262 } else if (Amt == NVTBits) {
1264 Hi = DAG.getConstant(0, NVT);
1266 Lo = DAG.getNode(ISD::OR, NVT,
1267 DAG.getNode(ISD::SRL, NVT, InL,
1268 DAG.getConstant(Amt, ShTy)),
1269 DAG.getNode(ISD::SHL, NVT, InH,
1270 DAG.getConstant(NVTBits-Amt, ShTy)));
1271 Hi = DAG.getNode(ISD::SRL, NVT, InH, DAG.getConstant(Amt, ShTy));
1276 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
1278 Hi = Lo = DAG.getNode(ISD::SRA, NVT, InH,
1279 DAG.getConstant(NVTBits-1, ShTy));
1280 } else if (Amt > NVTBits) {
1281 Lo = DAG.getNode(ISD::SRA, NVT, InH,
1282 DAG.getConstant(Amt-NVTBits, ShTy));
1283 Hi = DAG.getNode(ISD::SRA, NVT, InH,
1284 DAG.getConstant(NVTBits-1, ShTy));
1285 } else if (Amt == NVTBits) {
1287 Hi = DAG.getNode(ISD::SRA, NVT, InH,
1288 DAG.getConstant(NVTBits-1, ShTy));
1290 Lo = DAG.getNode(ISD::OR, NVT,
1291 DAG.getNode(ISD::SRL, NVT, InL,
1292 DAG.getConstant(Amt, ShTy)),
1293 DAG.getNode(ISD::SHL, NVT, InH,
1294 DAG.getConstant(NVTBits-Amt, ShTy)));
1295 Hi = DAG.getNode(ISD::SRA, NVT, InH, DAG.getConstant(Amt, ShTy));
1299 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
1300 /// this shift based on knowledge of the high bit of the shift amount. If we
1301 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
1303 bool DAGTypeLegalizer::
1304 ExpandShiftWithKnownAmountBit(SDNode *N, SDOperand &Lo, SDOperand &Hi) {
1305 MVT::ValueType NVT = TLI.getTypeToTransformTo(N->getValueType(0));
1306 unsigned NVTBits = MVT::getSizeInBits(NVT);
1307 assert(!(NVTBits & (NVTBits - 1)) &&
1308 "Expanded integer type size not a power of two!");
1310 uint64_t HighBitMask = NVTBits, KnownZero, KnownOne;
1311 DAG.ComputeMaskedBits(N->getOperand(1), HighBitMask, KnownZero, KnownOne);
1313 // If we don't know anything about the high bit, exit.
1314 if (((KnownZero|KnownOne) & HighBitMask) == 0)
1317 // Get the incoming operand to be shifted.
1319 GetExpandedOp(N->getOperand(0), InL, InH);
1320 SDOperand Amt = N->getOperand(1);
1322 // If we know that the high bit of the shift amount is one, then we can do
1323 // this as a couple of simple shifts.
1324 if (KnownOne & HighBitMask) {
1325 // Mask out the high bit, which we know is set.
1326 Amt = DAG.getNode(ISD::AND, Amt.getValueType(), Amt,
1327 DAG.getConstant(NVTBits-1, Amt.getValueType()));
1329 switch (N->getOpcode()) {
1330 default: assert(0 && "Unknown shift");
1332 Lo = DAG.getConstant(0, NVT); // Low part is zero.
1333 Hi = DAG.getNode(ISD::SHL, NVT, InL, Amt); // High part from Lo part.
1336 Hi = DAG.getConstant(0, NVT); // Hi part is zero.
1337 Lo = DAG.getNode(ISD::SRL, NVT, InH, Amt); // Lo part from Hi part.
1340 Hi = DAG.getNode(ISD::SRA, NVT, InH, // Sign extend high part.
1341 DAG.getConstant(NVTBits-1, Amt.getValueType()));
1342 Lo = DAG.getNode(ISD::SRA, NVT, InH, Amt); // Lo part from Hi part.
1347 // If we know that the high bit of the shift amount is zero, then we can do
1348 // this as a couple of simple shifts.
1349 assert((KnownZero & HighBitMask) && "Bad mask computation above");
1352 SDOperand Amt2 = DAG.getNode(ISD::SUB, Amt.getValueType(),
1353 DAG.getConstant(NVTBits, Amt.getValueType()),
1356 switch (N->getOpcode()) {
1357 default: assert(0 && "Unknown shift");
1358 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
1360 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
1363 Lo = DAG.getNode(N->getOpcode(), NVT, InL, Amt);
1364 Hi = DAG.getNode(ISD::OR, NVT,
1365 DAG.getNode(Op1, NVT, InH, Amt),
1366 DAG.getNode(Op2, NVT, InL, Amt2));
1370 //===----------------------------------------------------------------------===//
1371 // Result Vector Scalarization: <1 x ty> -> ty.
1372 //===----------------------------------------------------------------------===//
1375 void DAGTypeLegalizer::ScalarizeResult(SDNode *N, unsigned ResNo) {
1376 DEBUG(cerr << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
1378 SDOperand R = SDOperand();
1380 // FIXME: Custom lowering for scalarization?
1382 // See if the target wants to custom expand this node.
1383 if (TLI.getOperationAction(N->getOpcode(), N->getValueType(0)) ==
1384 TargetLowering::Custom) {
1385 // If the target wants to, allow it to lower this itself.
1386 if (SDNode *P = TLI.ExpandOperationResult(N, DAG)) {
1387 // Everything that once used N now uses P. We are guaranteed that the
1388 // result value types of N and the result value types of P match.
1389 ReplaceNodeWith(N, P);
1395 switch (N->getOpcode()) {
1398 cerr << "ScalarizeResult #" << ResNo << ": ";
1399 N->dump(&DAG); cerr << "\n";
1401 assert(0 && "Do not know how to scalarize the result of this operator!");
1404 case ISD::UNDEF: R = ScalarizeRes_UNDEF(N); break;
1405 case ISD::LOAD: R = ScalarizeRes_LOAD(cast<LoadSDNode>(N)); break;
1421 case ISD::XOR: R = ScalarizeRes_BinOp(N); break;
1426 case ISD::FCOS: R = ScalarizeRes_UnaryOp(N); break;
1427 case ISD::FPOWI: R = ScalarizeRes_FPOWI(N); break;
1428 case ISD::BUILD_VECTOR: R = N->getOperand(0); break;
1429 case ISD::INSERT_VECTOR_ELT: R = N->getOperand(1); break;
1430 case ISD::VECTOR_SHUFFLE: R = ScalarizeRes_VECTOR_SHUFFLE(N); break;
1431 case ISD::BIT_CONVERT: R = ScalarizeRes_BIT_CONVERT(N); break;
1432 case ISD::SELECT: R = ScalarizeRes_SELECT(N); break;
1435 // If R is null, the sub-method took care of registering the resul.
1437 SetScalarizedOp(SDOperand(N, ResNo), R);
1440 SDOperand DAGTypeLegalizer::ScalarizeRes_UNDEF(SDNode *N) {
1441 return DAG.getNode(ISD::UNDEF, MVT::getVectorElementType(N->getValueType(0)));
1444 SDOperand DAGTypeLegalizer::ScalarizeRes_LOAD(LoadSDNode *N) {
1445 SDOperand Result = DAG.getLoad(MVT::getVectorElementType(N->getValueType(0)),
1446 N->getChain(), N->getBasePtr(),
1447 N->getSrcValue(), N->getSrcValueOffset(),
1448 N->isVolatile(), N->getAlignment());
1450 // Legalized the chain result - switch anything that used the old chain to
1452 ReplaceValueWith(SDOperand(N, 1), Result.getValue(1));
1456 SDOperand DAGTypeLegalizer::ScalarizeRes_BinOp(SDNode *N) {
1457 SDOperand LHS = GetScalarizedOp(N->getOperand(0));
1458 SDOperand RHS = GetScalarizedOp(N->getOperand(1));
1459 return DAG.getNode(N->getOpcode(), LHS.getValueType(), LHS, RHS);
1462 SDOperand DAGTypeLegalizer::ScalarizeRes_UnaryOp(SDNode *N) {
1463 SDOperand Op = GetScalarizedOp(N->getOperand(0));
1464 return DAG.getNode(N->getOpcode(), Op.getValueType(), Op);
1467 SDOperand DAGTypeLegalizer::ScalarizeRes_FPOWI(SDNode *N) {
1468 SDOperand Op = GetScalarizedOp(N->getOperand(0));
1469 return DAG.getNode(ISD::FPOWI, Op.getValueType(), Op, N->getOperand(1));
1472 SDOperand DAGTypeLegalizer::ScalarizeRes_VECTOR_SHUFFLE(SDNode *N) {
1473 // Figure out if the scalar is the LHS or RHS and return it.
1474 SDOperand EltNum = N->getOperand(2).getOperand(0);
1475 unsigned Op = cast<ConstantSDNode>(EltNum)->getValue() != 0;
1476 return GetScalarizedOp(N->getOperand(Op));
1479 SDOperand DAGTypeLegalizer::ScalarizeRes_BIT_CONVERT(SDNode *N) {
1480 MVT::ValueType NewVT = MVT::getVectorElementType(N->getValueType(0));
1481 return DAG.getNode(ISD::BIT_CONVERT, NewVT, N->getOperand(0));
1484 SDOperand DAGTypeLegalizer::ScalarizeRes_SELECT(SDNode *N) {
1485 SDOperand LHS = GetScalarizedOp(N->getOperand(1));
1486 return DAG.getNode(ISD::SELECT, LHS.getValueType(), N->getOperand(0), LHS,
1487 GetScalarizedOp(N->getOperand(2)));
1491 //===----------------------------------------------------------------------===//
1492 // Operand Promotion
1493 //===----------------------------------------------------------------------===//
1495 /// PromoteOperand - This method is called when the specified operand of the
1496 /// specified node is found to need promotion. At this point, all of the result
1497 /// types of the node are known to be legal, but other operands of the node may
1498 /// need promotion or expansion as well as the specified one.
1499 bool DAGTypeLegalizer::PromoteOperand(SDNode *N, unsigned OpNo) {
1500 DEBUG(cerr << "Promote node operand: "; N->dump(&DAG); cerr << "\n");
1502 switch (N->getOpcode()) {
1505 cerr << "PromoteOperand Op #" << OpNo << ": ";
1506 N->dump(&DAG); cerr << "\n";
1508 assert(0 && "Do not know how to promote this operator's operand!");
1511 case ISD::ANY_EXTEND: Res = PromoteOperand_ANY_EXTEND(N); break;
1512 case ISD::ZERO_EXTEND: Res = PromoteOperand_ZERO_EXTEND(N); break;
1513 case ISD::SIGN_EXTEND: Res = PromoteOperand_SIGN_EXTEND(N); break;
1514 case ISD::TRUNCATE: Res = PromoteOperand_TRUNCATE(N); break;
1515 case ISD::FP_EXTEND: Res = PromoteOperand_FP_EXTEND(N); break;
1516 case ISD::FP_ROUND: Res = PromoteOperand_FP_ROUND(N); break;
1517 case ISD::SINT_TO_FP:
1518 case ISD::UINT_TO_FP: Res = PromoteOperand_INT_TO_FP(N); break;
1520 case ISD::SELECT: Res = PromoteOperand_SELECT(N, OpNo); break;
1521 case ISD::BRCOND: Res = PromoteOperand_BRCOND(N, OpNo); break;
1522 case ISD::BR_CC: Res = PromoteOperand_BR_CC(N, OpNo); break;
1523 case ISD::SETCC: Res = PromoteOperand_SETCC(N, OpNo); break;
1525 case ISD::STORE: Res = PromoteOperand_STORE(cast<StoreSDNode>(N),
1529 case ISD::MEMMOVE: Res = HandleMemIntrinsic(N); break;
1532 // If the result is null, the sub-method took care of registering results etc.
1533 if (!Res.Val) return false;
1534 // If the result is N, the sub-method updated N in place.
1536 // Mark N as new and remark N and its operands. This allows us to correctly
1537 // revisit N if it needs another step of promotion and allows us to visit
1538 // any new operands to N.
1539 N->setNodeId(NewNode);
1544 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1545 "Invalid operand expansion");
1547 ReplaceValueWith(SDOperand(N, 0), Res);
1551 SDOperand DAGTypeLegalizer::PromoteOperand_ANY_EXTEND(SDNode *N) {
1552 SDOperand Op = GetPromotedOp(N->getOperand(0));
1553 return DAG.getNode(ISD::ANY_EXTEND, N->getValueType(0), Op);
1556 SDOperand DAGTypeLegalizer::PromoteOperand_ZERO_EXTEND(SDNode *N) {
1557 SDOperand Op = GetPromotedOp(N->getOperand(0));
1558 Op = DAG.getNode(ISD::ANY_EXTEND, N->getValueType(0), Op);
1559 return DAG.getZeroExtendInReg(Op, N->getOperand(0).getValueType());
1562 SDOperand DAGTypeLegalizer::PromoteOperand_SIGN_EXTEND(SDNode *N) {
1563 SDOperand Op = GetPromotedOp(N->getOperand(0));
1564 Op = DAG.getNode(ISD::ANY_EXTEND, N->getValueType(0), Op);
1565 return DAG.getNode(ISD::SIGN_EXTEND_INREG, Op.getValueType(),
1566 Op, DAG.getValueType(N->getOperand(0).getValueType()));
1569 SDOperand DAGTypeLegalizer::PromoteOperand_TRUNCATE(SDNode *N) {
1570 SDOperand Op = GetPromotedOp(N->getOperand(0));
1571 return DAG.getNode(ISD::TRUNCATE, N->getValueType(0), Op);
1574 SDOperand DAGTypeLegalizer::PromoteOperand_FP_EXTEND(SDNode *N) {
1575 SDOperand Op = GetPromotedOp(N->getOperand(0));
1576 return DAG.getNode(ISD::FP_EXTEND, N->getValueType(0), Op);
1579 SDOperand DAGTypeLegalizer::PromoteOperand_FP_ROUND(SDNode *N) {
1580 SDOperand Op = GetPromotedOp(N->getOperand(0));
1581 return DAG.getNode(ISD::FP_ROUND, N->getValueType(0), Op);
1584 SDOperand DAGTypeLegalizer::PromoteOperand_INT_TO_FP(SDNode *N) {
1585 SDOperand In = GetPromotedOp(N->getOperand(0));
1586 MVT::ValueType OpVT = N->getOperand(0).getValueType();
1587 if (N->getOpcode() == ISD::UINT_TO_FP)
1588 In = DAG.getZeroExtendInReg(In, OpVT);
1590 In = DAG.getNode(ISD::SIGN_EXTEND_INREG, In.getValueType(),
1591 In, DAG.getValueType(OpVT));
1593 return DAG.UpdateNodeOperands(SDOperand(N, 0), In);
1596 SDOperand DAGTypeLegalizer::PromoteOperand_SELECT(SDNode *N, unsigned OpNo) {
1597 assert(OpNo == 0 && "Only know how to promote condition");
1598 SDOperand Cond = GetPromotedOp(N->getOperand(0)); // Promote the condition.
1600 // The top bits of the promoted condition are not necessarily zero, ensure
1601 // that the value is properly zero extended.
1602 if (!DAG.MaskedValueIsZero(Cond,
1603 MVT::getIntVTBitMask(Cond.getValueType())^1)) {
1604 Cond = DAG.getZeroExtendInReg(Cond, MVT::i1);
1605 MarkNewNodes(Cond.Val);
1608 // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1609 return DAG.UpdateNodeOperands(SDOperand(N, 0), Cond, N->getOperand(1),
1613 SDOperand DAGTypeLegalizer::PromoteOperand_BRCOND(SDNode *N, unsigned OpNo) {
1614 assert(OpNo == 1 && "only know how to promote condition");
1615 SDOperand Cond = GetPromotedOp(N->getOperand(1)); // Promote the condition.
1617 // The top bits of the promoted condition are not necessarily zero, ensure
1618 // that the value is properly zero extended.
1619 if (!DAG.MaskedValueIsZero(Cond,
1620 MVT::getIntVTBitMask(Cond.getValueType())^1)) {
1621 Cond = DAG.getZeroExtendInReg(Cond, MVT::i1);
1622 MarkNewNodes(Cond.Val);
1625 // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1626 return DAG.UpdateNodeOperands(SDOperand(N, 0), N->getOperand(0), Cond,
1630 SDOperand DAGTypeLegalizer::PromoteOperand_BR_CC(SDNode *N, unsigned OpNo) {
1631 assert(OpNo == 2 && "Don't know how to promote this operand");
1633 SDOperand LHS = N->getOperand(2);
1634 SDOperand RHS = N->getOperand(3);
1635 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1637 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1639 return DAG.UpdateNodeOperands(SDOperand(N, 0), N->getOperand(0),
1640 N->getOperand(1), LHS, RHS, N->getOperand(4));
1643 SDOperand DAGTypeLegalizer::PromoteOperand_SETCC(SDNode *N, unsigned OpNo) {
1644 assert(OpNo == 0 && "Don't know how to promote this operand");
1646 SDOperand LHS = N->getOperand(0);
1647 SDOperand RHS = N->getOperand(1);
1648 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1650 // The CC (#2) is always legal.
1651 return DAG.UpdateNodeOperands(SDOperand(N, 0), LHS, RHS, N->getOperand(2));
1654 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
1655 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
1656 void DAGTypeLegalizer::PromoteSetCCOperands(SDOperand &NewLHS,SDOperand &NewRHS,
1657 ISD::CondCode CCCode) {
1658 MVT::ValueType VT = NewLHS.getValueType();
1660 // Get the promoted values.
1661 NewLHS = GetPromotedOp(NewLHS);
1662 NewRHS = GetPromotedOp(NewRHS);
1664 // If this is an FP compare, the operands have already been extended.
1665 if (!MVT::isInteger(NewLHS.getValueType()))
1668 // Otherwise, we have to insert explicit sign or zero extends. Note
1669 // that we could insert sign extends for ALL conditions, but zero extend
1670 // is cheaper on many machines (an AND instead of two shifts), so prefer
1673 default: assert(0 && "Unknown integer comparison!");
1680 // ALL of these operations will work if we either sign or zero extend
1681 // the operands (including the unsigned comparisons!). Zero extend is
1682 // usually a simpler/cheaper operation, so prefer it.
1683 NewLHS = DAG.getZeroExtendInReg(NewLHS, VT);
1684 NewRHS = DAG.getZeroExtendInReg(NewRHS, VT);
1690 NewLHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, NewLHS.getValueType(), NewLHS,
1691 DAG.getValueType(VT));
1692 NewRHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, NewRHS.getValueType(), NewRHS,
1693 DAG.getValueType(VT));
1698 SDOperand DAGTypeLegalizer::PromoteOperand_STORE(StoreSDNode *N, unsigned OpNo){
1699 SDOperand Ch = N->getChain(), Ptr = N->getBasePtr();
1700 int SVOffset = N->getSrcValueOffset();
1701 unsigned Alignment = N->getAlignment();
1702 bool isVolatile = N->isVolatile();
1704 SDOperand Val = GetPromotedOp(N->getValue()); // Get promoted value.
1706 assert(!N->isTruncatingStore() && "Cannot promote this store operand!");
1708 // Truncate the value and store the result.
1709 return DAG.getTruncStore(Ch, Val, Ptr, N->getSrcValue(),
1710 SVOffset, N->getStoredVT(),
1711 isVolatile, Alignment);
1715 //===----------------------------------------------------------------------===//
1716 // Operand Expansion
1717 //===----------------------------------------------------------------------===//
1719 /// ExpandOperand - This method is called when the specified operand of the
1720 /// specified node is found to need expansion. At this point, all of the result
1721 /// types of the node are known to be legal, but other operands of the node may
1722 /// need promotion or expansion as well as the specified one.
1723 bool DAGTypeLegalizer::ExpandOperand(SDNode *N, unsigned OpNo) {
1724 DEBUG(cerr << "Expand node operand: "; N->dump(&DAG); cerr << "\n");
1725 SDOperand Res(0, 0);
1727 if (TLI.getOperationAction(N->getOpcode(), N->getValueType(0)) ==
1728 TargetLowering::Custom)
1729 Res = TLI.LowerOperation(SDOperand(N, 0), DAG);
1732 switch (N->getOpcode()) {
1735 cerr << "ExpandOperand Op #" << OpNo << ": ";
1736 N->dump(&DAG); cerr << "\n";
1738 assert(0 && "Do not know how to expand this operator's operand!");
1741 case ISD::TRUNCATE: Res = ExpandOperand_TRUNCATE(N); break;
1742 case ISD::BIT_CONVERT: Res = ExpandOperand_BIT_CONVERT(N); break;
1744 case ISD::SINT_TO_FP:
1745 Res = ExpandOperand_SINT_TO_FP(N->getOperand(0), N->getValueType(0));
1747 case ISD::UINT_TO_FP:
1748 Res = ExpandOperand_UINT_TO_FP(N->getOperand(0), N->getValueType(0));
1750 case ISD::EXTRACT_ELEMENT: Res = ExpandOperand_EXTRACT_ELEMENT(N); break;
1751 case ISD::SETCC: Res = ExpandOperand_SETCC(N); break;
1754 Res = ExpandOperand_STORE(cast<StoreSDNode>(N), OpNo);
1758 case ISD::MEMMOVE: Res = HandleMemIntrinsic(N); break;
1762 // If the result is null, the sub-method took care of registering results etc.
1763 if (!Res.Val) return false;
1764 // If the result is N, the sub-method updated N in place. Check to see if any
1765 // operands are new, and if so, mark them.
1767 // Mark N as new and remark N and its operands. This allows us to correctly
1768 // revisit N if it needs another step of promotion and allows us to visit
1769 // any new operands to N.
1770 N->setNodeId(NewNode);
1775 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1776 "Invalid operand expansion");
1778 ReplaceValueWith(SDOperand(N, 0), Res);
1782 SDOperand DAGTypeLegalizer::ExpandOperand_TRUNCATE(SDNode *N) {
1784 GetExpandedOp(N->getOperand(0), InL, InH);
1785 // Just truncate the low part of the source.
1786 return DAG.getNode(ISD::TRUNCATE, N->getValueType(0), InL);
1789 SDOperand DAGTypeLegalizer::ExpandOperand_BIT_CONVERT(SDNode *N) {
1790 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1793 SDOperand DAGTypeLegalizer::ExpandOperand_SINT_TO_FP(SDOperand Source,
1794 MVT::ValueType DestTy) {
1795 // We know the destination is legal, but that the input needs to be expanded.
1796 assert(Source.getValueType() == MVT::i64 && "Only handle expand from i64!");
1798 // Check to see if the target has a custom way to lower this. If so, use it.
1799 switch (TLI.getOperationAction(ISD::SINT_TO_FP, Source.getValueType())) {
1800 default: assert(0 && "This action not implemented for this operation!");
1801 case TargetLowering::Legal:
1802 case TargetLowering::Expand:
1803 break; // This case is handled below.
1804 case TargetLowering::Custom:
1805 SDOperand NV = TLI.LowerOperation(DAG.getNode(ISD::SINT_TO_FP, DestTy,
1807 if (NV.Val) return NV;
1808 break; // The target lowered this.
1812 if (DestTy == MVT::f32)
1813 LC = RTLIB::SINTTOFP_I64_F32;
1815 assert(DestTy == MVT::f64 && "Unknown fp value type!");
1816 LC = RTLIB::SINTTOFP_I64_F64;
1819 assert(0 && "FIXME: no libcalls yet!");
1822 assert(TLI.getLibcallName(LC) && "Don't know how to expand this SINT_TO_FP!");
1823 Source = DAG.getNode(ISD::SINT_TO_FP, DestTy, Source);
1824 SDOperand UnusedHiPart;
1825 return ExpandLibCall(TLI.getLibcallName(LC), Source.Val, true, UnusedHiPart);
1829 SDOperand DAGTypeLegalizer::ExpandOperand_UINT_TO_FP(SDOperand Source,
1830 MVT::ValueType DestTy) {
1831 // We know the destination is legal, but that the input needs to be expanded.
1832 assert(getTypeAction(Source.getValueType()) == Expand &&
1833 "This is not an expansion!");
1834 assert(Source.getValueType() == MVT::i64 && "Only handle expand from i64!");
1836 // If this is unsigned, and not supported, first perform the conversion to
1837 // signed, then adjust the result if the sign bit is set.
1838 SDOperand SignedConv = ExpandOperand_SINT_TO_FP(Source, DestTy);
1840 // The 64-bit value loaded will be incorrectly if the 'sign bit' of the
1841 // incoming integer is set. To handle this, we dynamically test to see if
1842 // it is set, and, if so, add a fudge factor.
1844 GetExpandedOp(Source, Lo, Hi);
1846 SDOperand SignSet = DAG.getSetCC(TLI.getSetCCResultTy(), Hi,
1847 DAG.getConstant(0, Hi.getValueType()),
1849 SDOperand Zero = getIntPtrConstant(0), Four = getIntPtrConstant(4);
1850 SDOperand CstOffset = DAG.getNode(ISD::SELECT, Zero.getValueType(),
1851 SignSet, Four, Zero);
1852 uint64_t FF = 0x5f800000ULL;
1853 if (TLI.isLittleEndian()) FF <<= 32;
1854 Constant *FudgeFactor = ConstantInt::get(Type::Int64Ty, FF);
1856 SDOperand CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy());
1857 CPIdx = DAG.getNode(ISD::ADD, TLI.getPointerTy(), CPIdx, CstOffset);
1858 SDOperand FudgeInReg;
1859 if (DestTy == MVT::f32)
1860 FudgeInReg = DAG.getLoad(MVT::f32, DAG.getEntryNode(), CPIdx, NULL, 0);
1861 else if (MVT::getSizeInBits(DestTy) > MVT::getSizeInBits(MVT::f32))
1862 // FIXME: Avoid the extend by construction the right constantpool?
1863 FudgeInReg = DAG.getExtLoad(ISD::EXTLOAD, DestTy, DAG.getEntryNode(),
1864 CPIdx, NULL, 0, MVT::f32);
1866 assert(0 && "Unexpected conversion");
1868 return DAG.getNode(ISD::FADD, DestTy, SignedConv, FudgeInReg);
1871 SDOperand DAGTypeLegalizer::ExpandOperand_EXTRACT_ELEMENT(SDNode *N) {
1873 GetExpandedOp(N->getOperand(0), Lo, Hi);
1874 return cast<ConstantSDNode>(N->getOperand(1))->getValue() ? Hi : Lo;
1877 SDOperand DAGTypeLegalizer::ExpandOperand_SETCC(SDNode *N) {
1878 SDOperand NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1879 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
1880 ExpandSetCCOperands(NewLHS, NewRHS, CCCode);
1882 // If ExpandSetCCOperands returned a scalar, use it.
1883 if (NewRHS.Val == 0) return NewLHS;
1885 // Otherwise, update N to have the operands specified.
1886 return DAG.UpdateNodeOperands(SDOperand(N, 0), NewLHS, NewRHS,
1887 DAG.getCondCode(CCCode));
1890 /// ExpandSetCCOperands - Expand the operands of a comparison. This code is
1891 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
1892 void DAGTypeLegalizer::ExpandSetCCOperands(SDOperand &NewLHS, SDOperand &NewRHS,
1893 ISD::CondCode &CCCode) {
1894 SDOperand LHSLo, LHSHi, RHSLo, RHSHi;
1895 GetExpandedOp(NewLHS, LHSLo, LHSHi);
1896 GetExpandedOp(NewRHS, RHSLo, RHSHi);
1898 MVT::ValueType VT = NewLHS.getValueType();
1899 if (VT == MVT::f32 || VT == MVT::f64) {
1900 assert(0 && "FIXME: softfp not implemented yet! should be promote not exp");
1903 if (VT == MVT::ppcf128) {
1904 // FIXME: This generated code sucks. We want to generate
1905 // FCMP crN, hi1, hi2
1907 // FCMP crN, lo1, lo2
1908 // The following can be improved, but not that much.
1909 SDOperand Tmp1, Tmp2, Tmp3;
1910 Tmp1 = DAG.getSetCC(TLI.getSetCCResultTy(), LHSHi, RHSHi, ISD::SETEQ);
1911 Tmp2 = DAG.getSetCC(TLI.getSetCCResultTy(), LHSLo, RHSLo, CCCode);
1912 Tmp3 = DAG.getNode(ISD::AND, Tmp1.getValueType(), Tmp1, Tmp2);
1913 Tmp1 = DAG.getSetCC(TLI.getSetCCResultTy(), LHSHi, RHSHi, ISD::SETNE);
1914 Tmp2 = DAG.getSetCC(TLI.getSetCCResultTy(), LHSHi, RHSHi, CCCode);
1915 Tmp1 = DAG.getNode(ISD::AND, Tmp1.getValueType(), Tmp1, Tmp2);
1916 NewLHS = DAG.getNode(ISD::OR, Tmp1.getValueType(), Tmp1, Tmp3);
1917 NewRHS = SDOperand(); // LHS is the result, not a compare.
1922 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
1924 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo))
1925 if (RHSCST->isAllOnesValue()) {
1926 // Equality comparison to -1.
1927 NewLHS = DAG.getNode(ISD::AND, LHSLo.getValueType(), LHSLo, LHSHi);
1932 NewLHS = DAG.getNode(ISD::XOR, LHSLo.getValueType(), LHSLo, RHSLo);
1933 NewRHS = DAG.getNode(ISD::XOR, LHSLo.getValueType(), LHSHi, RHSHi);
1934 NewLHS = DAG.getNode(ISD::OR, NewLHS.getValueType(), NewLHS, NewRHS);
1935 NewRHS = DAG.getConstant(0, NewLHS.getValueType());
1939 // If this is a comparison of the sign bit, just look at the top part.
1941 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
1942 if ((CCCode == ISD::SETLT && CST->getValue() == 0) || // X < 0
1943 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
1949 // FIXME: This generated code sucks.
1950 ISD::CondCode LowCC;
1952 default: assert(0 && "Unknown integer setcc!");
1954 case ISD::SETULT: LowCC = ISD::SETULT; break;
1956 case ISD::SETUGT: LowCC = ISD::SETUGT; break;
1958 case ISD::SETULE: LowCC = ISD::SETULE; break;
1960 case ISD::SETUGE: LowCC = ISD::SETUGE; break;
1963 // Tmp1 = lo(op1) < lo(op2) // Always unsigned comparison
1964 // Tmp2 = hi(op1) < hi(op2) // Signedness depends on operands
1965 // dest = hi(op1) == hi(op2) ? Tmp1 : Tmp2;
1967 // NOTE: on targets without efficient SELECT of bools, we can always use
1968 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
1969 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, false, true, NULL);
1970 SDOperand Tmp1, Tmp2;
1971 Tmp1 = TLI.SimplifySetCC(TLI.getSetCCResultTy(), LHSLo, RHSLo, LowCC,
1972 false, DagCombineInfo);
1974 Tmp1 = DAG.getSetCC(TLI.getSetCCResultTy(), LHSLo, RHSLo, LowCC);
1975 Tmp2 = TLI.SimplifySetCC(TLI.getSetCCResultTy(), LHSHi, RHSHi,
1976 CCCode, false, DagCombineInfo);
1978 Tmp2 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultTy(), LHSHi, RHSHi,
1979 DAG.getCondCode(CCCode));
1981 ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.Val);
1982 ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.Val);
1983 if ((Tmp1C && Tmp1C->getValue() == 0) ||
1984 (Tmp2C && Tmp2C->getValue() == 0 &&
1985 (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
1986 CCCode == ISD::SETUGE || CCCode == ISD::SETULE)) ||
1987 (Tmp2C && Tmp2C->getValue() == 1 &&
1988 (CCCode == ISD::SETLT || CCCode == ISD::SETGT ||
1989 CCCode == ISD::SETUGT || CCCode == ISD::SETULT))) {
1990 // low part is known false, returns high part.
1991 // For LE / GE, if high part is known false, ignore the low part.
1992 // For LT / GT, if high part is known true, ignore the low part.
1994 NewRHS = SDOperand();
1998 NewLHS = TLI.SimplifySetCC(TLI.getSetCCResultTy(), LHSHi, RHSHi,
1999 ISD::SETEQ, false, DagCombineInfo);
2001 NewLHS = DAG.getSetCC(TLI.getSetCCResultTy(), LHSHi, RHSHi, ISD::SETEQ);
2002 NewLHS = DAG.getNode(ISD::SELECT, Tmp1.getValueType(),
2003 NewLHS, Tmp1, Tmp2);
2004 NewRHS = SDOperand();
2007 SDOperand DAGTypeLegalizer::ExpandOperand_STORE(StoreSDNode *N, unsigned OpNo) {
2008 assert(OpNo == 1 && "Can only expand the stored value so far");
2010 MVT::ValueType VT = N->getOperand(1).getValueType();
2011 MVT::ValueType NVT = TLI.getTypeToTransformTo(VT);
2012 SDOperand Ch = N->getChain();
2013 SDOperand Ptr = N->getBasePtr();
2014 int SVOffset = N->getSrcValueOffset();
2015 unsigned Alignment = N->getAlignment();
2016 bool isVolatile = N->isVolatile();
2019 assert(!(MVT::getSizeInBits(NVT) & 7) && "Expanded type not byte sized!");
2021 if (!N->isTruncatingStore()) {
2022 unsigned IncrementSize = 0;
2024 // If this is a vector type, then we have to calculate the increment as
2025 // the product of the element size in bytes, and the number of elements
2026 // in the high half of the vector.
2027 if (MVT::isVector(N->getValue().getValueType())) {
2028 assert(0 && "Vectors not supported yet");
2030 SDNode *InVal = ST->getValue().Val;
2031 unsigned NumElems = MVT::getVectorNumElements(InVal->getValueType(0));
2032 MVT::ValueType EVT = MVT::getVectorElementType(InVal->getValueType(0));
2034 // Figure out if there is a simple type corresponding to this Vector
2035 // type. If so, convert to the vector type.
2036 MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems);
2037 if (TLI.isTypeLegal(TVT)) {
2038 // Turn this into a normal store of the vector type.
2039 Tmp3 = LegalizeOp(Node->getOperand(1));
2040 Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
2041 SVOffset, isVolatile, Alignment);
2042 Result = LegalizeOp(Result);
2044 } else if (NumElems == 1) {
2045 // Turn this into a normal store of the scalar type.
2046 Tmp3 = ScalarizeVectorOp(Node->getOperand(1));
2047 Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
2048 SVOffset, isVolatile, Alignment);
2049 // The scalarized value type may not be legal, e.g. it might require
2050 // promotion or expansion. Relegalize the scalar store.
2051 return LegalizeOp(Result);
2053 SplitVectorOp(Node->getOperand(1), Lo, Hi);
2054 IncrementSize = NumElems/2 * MVT::getSizeInBits(EVT)/8;
2058 GetExpandedOp(N->getValue(), Lo, Hi);
2059 IncrementSize = Hi.Val ? MVT::getSizeInBits(Hi.getValueType())/8 : 0;
2061 if (!TLI.isLittleEndian())
2065 Lo = DAG.getStore(Ch, Lo, Ptr, N->getSrcValue(),
2066 SVOffset, isVolatile, Alignment);
2068 assert(Hi.Val && "FIXME: int <-> float should be handled with promote!");
2070 if (Hi.Val == NULL) {
2071 // Must be int <-> float one-to-one expansion.
2076 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
2077 getIntPtrConstant(IncrementSize));
2078 assert(isTypeLegal(Ptr.getValueType()) && "Pointers must be legal!");
2079 Hi = DAG.getStore(Ch, Hi, Ptr, N->getSrcValue(), SVOffset+IncrementSize,
2080 isVolatile, MinAlign(Alignment, IncrementSize));
2081 return DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi);
2082 } else if (MVT::getSizeInBits(N->getStoredVT()) <= MVT::getSizeInBits(NVT)) {
2083 GetExpandedOp(N->getValue(), Lo, Hi);
2084 return DAG.getTruncStore(Ch, Lo, Ptr, N->getSrcValue(), SVOffset,
2085 N->getStoredVT(), isVolatile, Alignment);
2086 } else if (TLI.isLittleEndian()) {
2087 // Little-endian - low bits are at low addresses.
2088 GetExpandedOp(N->getValue(), Lo, Hi);
2090 Lo = DAG.getStore(Ch, Lo, Ptr, N->getSrcValue(), SVOffset,
2091 isVolatile, Alignment);
2093 unsigned ExcessBits =
2094 MVT::getSizeInBits(N->getStoredVT()) - MVT::getSizeInBits(NVT);
2095 MVT::ValueType NEVT = MVT::getIntegerType(ExcessBits);
2097 // Increment the pointer to the other half.
2098 unsigned IncrementSize = MVT::getSizeInBits(NVT)/8;
2099 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
2100 getIntPtrConstant(IncrementSize));
2101 Hi = DAG.getTruncStore(Ch, Hi, Ptr, N->getSrcValue(),
2102 SVOffset+IncrementSize, NEVT,
2103 isVolatile, MinAlign(Alignment, IncrementSize));
2104 return DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi);
2106 // Big-endian - high bits are at low addresses. Favor aligned stores at
2107 // the cost of some bit-fiddling.
2108 GetExpandedOp(N->getValue(), Lo, Hi);
2110 MVT::ValueType EVT = N->getStoredVT();
2111 unsigned EBytes = MVT::getStoreSizeInBits(EVT)/8;
2112 unsigned IncrementSize = MVT::getSizeInBits(NVT)/8;
2113 unsigned ExcessBits = (EBytes - IncrementSize)*8;
2114 MVT::ValueType HiVT =
2115 MVT::getIntegerType(MVT::getSizeInBits(EVT)-ExcessBits);
2117 if (ExcessBits < MVT::getSizeInBits(NVT)) {
2118 // Transfer high bits from the top of Lo to the bottom of Hi.
2119 Hi = DAG.getNode(ISD::SHL, NVT, Hi,
2120 DAG.getConstant(MVT::getSizeInBits(NVT) - ExcessBits,
2121 TLI.getShiftAmountTy()));
2122 Hi = DAG.getNode(ISD::OR, NVT, Hi,
2123 DAG.getNode(ISD::SRL, NVT, Lo,
2124 DAG.getConstant(ExcessBits,
2125 TLI.getShiftAmountTy())));
2128 // Store both the high bits and maybe some of the low bits.
2129 Hi = DAG.getTruncStore(Ch, Hi, Ptr, N->getSrcValue(),
2130 SVOffset, HiVT, isVolatile, Alignment);
2132 // Increment the pointer to the other half.
2133 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
2134 getIntPtrConstant(IncrementSize));
2135 // Store the lowest ExcessBits bits in the second half.
2136 Lo = DAG.getTruncStore(Ch, Lo, Ptr, N->getSrcValue(),
2137 SVOffset+IncrementSize,
2138 MVT::getIntegerType(ExcessBits),
2139 isVolatile, MinAlign(Alignment, IncrementSize));
2140 return DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi);
2144 //===----------------------------------------------------------------------===//
2145 // Operand Vector Scalarization <1 x ty> -> ty.
2146 //===----------------------------------------------------------------------===//
2148 bool DAGTypeLegalizer::ScalarizeOperand(SDNode *N, unsigned OpNo) {
2149 DEBUG(cerr << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
2151 SDOperand Res(0, 0);
2153 // FIXME: Should we support custom lowering for scalarization?
2155 if (TLI.getOperationAction(N->getOpcode(), N->getValueType(0)) ==
2156 TargetLowering::Custom)
2157 Res = TLI.LowerOperation(SDOperand(N, 0), DAG);
2161 switch (N->getOpcode()) {
2164 cerr << "ScalarizeOperand Op #" << OpNo << ": ";
2165 N->dump(&DAG); cerr << "\n";
2167 assert(0 && "Do not know how to scalarize this operator's operand!");
2170 case ISD::EXTRACT_VECTOR_ELT:
2171 Res = ScalarizeOp_EXTRACT_VECTOR_ELT(N, OpNo);
2176 // If the result is null, the sub-method took care of registering results etc.
2177 if (!Res.Val) return false;
2179 // If the result is N, the sub-method updated N in place. Check to see if any
2180 // operands are new, and if so, mark them.
2182 // Mark N as new and remark N and its operands. This allows us to correctly
2183 // revisit N if it needs another step of promotion and allows us to visit
2184 // any new operands to N.
2185 N->setNodeId(NewNode);
2190 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2191 "Invalid operand expansion");
2193 ReplaceValueWith(SDOperand(N, 0), Res);
2197 /// ScalarizeOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to be
2198 /// scalarized, it must be <1 x ty>, just return the operand, ignoring the
2200 SDOperand DAGTypeLegalizer::ScalarizeOp_EXTRACT_VECTOR_ELT(SDNode *N,
2202 return GetScalarizedOp(N->getOperand(0));
2206 //===----------------------------------------------------------------------===//
2208 //===----------------------------------------------------------------------===//
2210 /// LegalizeTypes - This transforms the SelectionDAG into a SelectionDAG that
2211 /// only uses types natively supported by the target.
2213 /// Note that this is an involved process that may invalidate pointers into
2215 void SelectionDAG::LegalizeTypes() {
2216 DAGTypeLegalizer(*this).run();