1 //===-- LegalizeTypesExpand.cpp - Expansion for LegalizeTypes -------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements expansion support for LegalizeTypes. Expansion is the
11 // act of changing a computation in an invalid type to be a computation in
12 // multiple registers of a smaller type. For example, implementing i64
13 // arithmetic in two i32 registers (as is often needed on 32-bit targets, for
16 //===----------------------------------------------------------------------===//
18 #include "LegalizeTypes.h"
19 #include "llvm/Constants.h"
22 //===----------------------------------------------------------------------===//
24 //===----------------------------------------------------------------------===//
26 /// ExpandResult - This method is called when the specified result of the
27 /// specified node is found to need expansion. At this point, the node may also
28 /// have invalid operands or may have other results that need promotion, we just
29 /// know that (at least) one result needs expansion.
30 void DAGTypeLegalizer::ExpandResult(SDNode *N, unsigned ResNo) {
31 DEBUG(cerr << "Expand node result: "; N->dump(&DAG); cerr << "\n");
33 Lo = Hi = SDOperand();
35 // See if the target wants to custom expand this node.
36 if (TLI.getOperationAction(N->getOpcode(), N->getValueType(0)) ==
37 TargetLowering::Custom) {
38 // If the target wants to, allow it to lower this itself.
39 if (SDNode *P = TLI.ExpandOperationResult(N, DAG)) {
40 // Everything that once used N now uses P. We are guaranteed that the
41 // result value types of N and the result value types of P match.
42 ReplaceNodeWith(N, P);
47 switch (N->getOpcode()) {
50 cerr << "ExpandResult #" << ResNo << ": ";
51 N->dump(&DAG); cerr << "\n";
53 assert(0 && "Do not know how to expand the result of this operator!");
56 case ISD::UNDEF: ExpandResult_UNDEF(N, Lo, Hi); break;
57 case ISD::Constant: ExpandResult_Constant(N, Lo, Hi); break;
58 case ISD::BUILD_PAIR: ExpandResult_BUILD_PAIR(N, Lo, Hi); break;
59 case ISD::MERGE_VALUES: ExpandResult_MERGE_VALUES(N, Lo, Hi); break;
60 case ISD::ANY_EXTEND: ExpandResult_ANY_EXTEND(N, Lo, Hi); break;
61 case ISD::ZERO_EXTEND: ExpandResult_ZERO_EXTEND(N, Lo, Hi); break;
62 case ISD::SIGN_EXTEND: ExpandResult_SIGN_EXTEND(N, Lo, Hi); break;
63 case ISD::AssertZext: ExpandResult_AssertZext(N, Lo, Hi); break;
64 case ISD::TRUNCATE: ExpandResult_TRUNCATE(N, Lo, Hi); break;
65 case ISD::BIT_CONVERT: ExpandResult_BIT_CONVERT(N, Lo, Hi); break;
66 case ISD::SIGN_EXTEND_INREG: ExpandResult_SIGN_EXTEND_INREG(N, Lo, Hi); break;
67 case ISD::LOAD: ExpandResult_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
71 case ISD::XOR: ExpandResult_Logical(N, Lo, Hi); break;
72 case ISD::BSWAP: ExpandResult_BSWAP(N, Lo, Hi); break;
74 case ISD::SUB: ExpandResult_ADDSUB(N, Lo, Hi); break;
76 case ISD::SUBC: ExpandResult_ADDSUBC(N, Lo, Hi); break;
78 case ISD::SUBE: ExpandResult_ADDSUBE(N, Lo, Hi); break;
79 case ISD::SELECT: ExpandResult_SELECT(N, Lo, Hi); break;
80 case ISD::SELECT_CC: ExpandResult_SELECT_CC(N, Lo, Hi); break;
81 case ISD::MUL: ExpandResult_MUL(N, Lo, Hi); break;
84 case ISD::SRL: ExpandResult_Shift(N, Lo, Hi); break;
86 case ISD::CTLZ: ExpandResult_CTLZ(N, Lo, Hi); break;
87 case ISD::CTPOP: ExpandResult_CTPOP(N, Lo, Hi); break;
88 case ISD::CTTZ: ExpandResult_CTTZ(N, Lo, Hi); break;
90 case ISD::EXTRACT_VECTOR_ELT:
91 ExpandResult_EXTRACT_VECTOR_ELT(N, Lo, Hi);
95 // If Lo/Hi is null, the sub-method took care of registering results etc.
97 SetExpandedOp(SDOperand(N, ResNo), Lo, Hi);
100 void DAGTypeLegalizer::ExpandResult_UNDEF(SDNode *N,
101 SDOperand &Lo, SDOperand &Hi) {
102 MVT::ValueType NVT = TLI.getTypeToTransformTo(N->getValueType(0));
103 Lo = Hi = DAG.getNode(ISD::UNDEF, NVT);
106 void DAGTypeLegalizer::ExpandResult_Constant(SDNode *N,
107 SDOperand &Lo, SDOperand &Hi) {
108 MVT::ValueType NVT = TLI.getTypeToTransformTo(N->getValueType(0));
109 uint64_t Cst = cast<ConstantSDNode>(N)->getValue();
110 Lo = DAG.getConstant(Cst, NVT);
111 Hi = DAG.getConstant(Cst >> MVT::getSizeInBits(NVT), NVT);
114 void DAGTypeLegalizer::ExpandResult_BUILD_PAIR(SDNode *N,
115 SDOperand &Lo, SDOperand &Hi) {
116 // Return the operands.
117 Lo = N->getOperand(0);
118 Hi = N->getOperand(1);
121 void DAGTypeLegalizer::ExpandResult_MERGE_VALUES(SDNode *N,
122 SDOperand &Lo, SDOperand &Hi) {
123 // A MERGE_VALUES node can produce any number of values. We know that the
124 // first illegal one needs to be expanded into Lo/Hi.
127 // The string of legal results gets turns into the input operands, which have
129 for (i = 0; isTypeLegal(N->getValueType(i)); ++i)
130 ReplaceValueWith(SDOperand(N, i), SDOperand(N->getOperand(i)));
132 // The first illegal result must be the one that needs to be expanded.
133 GetExpandedOp(N->getOperand(i), Lo, Hi);
135 // Legalize the rest of the results into the input operands whether they are
137 unsigned e = N->getNumValues();
138 for (++i; i != e; ++i)
139 ReplaceValueWith(SDOperand(N, i), SDOperand(N->getOperand(i)));
142 void DAGTypeLegalizer::ExpandResult_ANY_EXTEND(SDNode *N,
143 SDOperand &Lo, SDOperand &Hi) {
144 MVT::ValueType NVT = TLI.getTypeToTransformTo(N->getValueType(0));
145 SDOperand Op = N->getOperand(0);
146 if (MVT::getSizeInBits(Op.getValueType()) <= MVT::getSizeInBits(NVT)) {
147 // The low part is any extension of the input (which degenerates to a copy).
148 Lo = DAG.getNode(ISD::ANY_EXTEND, NVT, Op);
149 Hi = DAG.getNode(ISD::UNDEF, NVT); // The high part is undefined.
151 // For example, extension of an i48 to an i64. The operand type necessarily
152 // promotes to the result type, so will end up being expanded too.
153 assert(getTypeAction(Op.getValueType()) == Promote &&
154 "Only know how to promote this result!");
155 SDOperand Res = GetPromotedOp(Op);
156 assert(Res.getValueType() == N->getValueType(0) &&
157 "Operand over promoted?");
158 // Split the promoted operand. This will simplify when it is expanded.
159 SplitOp(Res, Lo, Hi);
163 void DAGTypeLegalizer::ExpandResult_ZERO_EXTEND(SDNode *N,
164 SDOperand &Lo, SDOperand &Hi) {
165 MVT::ValueType NVT = TLI.getTypeToTransformTo(N->getValueType(0));
166 SDOperand Op = N->getOperand(0);
167 if (MVT::getSizeInBits(Op.getValueType()) <= MVT::getSizeInBits(NVT)) {
168 // The low part is zero extension of the input (which degenerates to a copy).
169 Lo = DAG.getNode(ISD::ZERO_EXTEND, NVT, N->getOperand(0));
170 Hi = DAG.getConstant(0, NVT); // The high part is just a zero.
172 // For example, extension of an i48 to an i64. The operand type necessarily
173 // promotes to the result type, so will end up being expanded too.
174 assert(getTypeAction(Op.getValueType()) == Promote &&
175 "Only know how to promote this result!");
176 SDOperand Res = GetPromotedOp(Op);
177 assert(Res.getValueType() == N->getValueType(0) &&
178 "Operand over promoted?");
179 // Split the promoted operand. This will simplify when it is expanded.
180 SplitOp(Res, Lo, Hi);
181 unsigned ExcessBits =
182 MVT::getSizeInBits(Op.getValueType()) - MVT::getSizeInBits(NVT);
183 Hi = DAG.getZeroExtendInReg(Hi, MVT::getIntegerType(ExcessBits));
187 void DAGTypeLegalizer::ExpandResult_SIGN_EXTEND(SDNode *N,
188 SDOperand &Lo, SDOperand &Hi) {
189 MVT::ValueType NVT = TLI.getTypeToTransformTo(N->getValueType(0));
190 SDOperand Op = N->getOperand(0);
191 if (MVT::getSizeInBits(Op.getValueType()) <= MVT::getSizeInBits(NVT)) {
192 // The low part is sign extension of the input (which degenerates to a copy).
193 Lo = DAG.getNode(ISD::SIGN_EXTEND, NVT, N->getOperand(0));
194 // The high part is obtained by SRA'ing all but one of the bits of low part.
195 unsigned LoSize = MVT::getSizeInBits(NVT);
196 Hi = DAG.getNode(ISD::SRA, NVT, Lo,
197 DAG.getConstant(LoSize-1, TLI.getShiftAmountTy()));
199 // For example, extension of an i48 to an i64. The operand type necessarily
200 // promotes to the result type, so will end up being expanded too.
201 assert(getTypeAction(Op.getValueType()) == Promote &&
202 "Only know how to promote this result!");
203 SDOperand Res = GetPromotedOp(Op);
204 assert(Res.getValueType() == N->getValueType(0) &&
205 "Operand over promoted?");
206 // Split the promoted operand. This will simplify when it is expanded.
207 SplitOp(Res, Lo, Hi);
208 unsigned ExcessBits =
209 MVT::getSizeInBits(Op.getValueType()) - MVT::getSizeInBits(NVT);
210 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, Hi.getValueType(), Hi,
211 DAG.getValueType(MVT::getIntegerType(ExcessBits)));
215 void DAGTypeLegalizer::ExpandResult_AssertZext(SDNode *N,
216 SDOperand &Lo, SDOperand &Hi) {
217 GetExpandedOp(N->getOperand(0), Lo, Hi);
218 MVT::ValueType NVT = Lo.getValueType();
219 MVT::ValueType EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
220 unsigned NVTBits = MVT::getSizeInBits(NVT);
221 unsigned EVTBits = MVT::getSizeInBits(EVT);
223 if (NVTBits < EVTBits) {
224 Hi = DAG.getNode(ISD::AssertZext, NVT, Hi,
225 DAG.getValueType(MVT::getIntegerType(EVTBits - NVTBits)));
227 Lo = DAG.getNode(ISD::AssertZext, NVT, Lo, DAG.getValueType(EVT));
228 // The high part must be zero, make it explicit.
229 Hi = DAG.getConstant(0, NVT);
233 void DAGTypeLegalizer::ExpandResult_TRUNCATE(SDNode *N,
234 SDOperand &Lo, SDOperand &Hi) {
235 MVT::ValueType NVT = TLI.getTypeToTransformTo(N->getValueType(0));
236 Lo = DAG.getNode(ISD::TRUNCATE, NVT, N->getOperand(0));
237 Hi = DAG.getNode(ISD::SRL, N->getOperand(0).getValueType(), N->getOperand(0),
238 DAG.getConstant(MVT::getSizeInBits(NVT),
239 TLI.getShiftAmountTy()));
240 Hi = DAG.getNode(ISD::TRUNCATE, NVT, Hi);
243 void DAGTypeLegalizer::ExpandResult_BIT_CONVERT(SDNode *N,
244 SDOperand &Lo, SDOperand &Hi) {
245 // Lower the bit-convert to a store/load from the stack, then expand the load.
246 // TODO: If the operand also needs expansion then this could be turned into
247 // conversion of the expanded pieces. But there needs to be a testcase first!
248 SDOperand Op = CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
249 ExpandResult_LOAD(cast<LoadSDNode>(Op.Val), Lo, Hi);
252 void DAGTypeLegalizer::
253 ExpandResult_SIGN_EXTEND_INREG(SDNode *N, SDOperand &Lo, SDOperand &Hi) {
254 GetExpandedOp(N->getOperand(0), Lo, Hi);
255 MVT::ValueType EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
257 if (MVT::getSizeInBits(EVT) <= MVT::getSizeInBits(Lo.getValueType())) {
258 // sext_inreg the low part if needed.
259 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, Lo.getValueType(), Lo,
262 // The high part gets the sign extension from the lo-part. This handles
263 // things like sextinreg V:i64 from i8.
264 Hi = DAG.getNode(ISD::SRA, Hi.getValueType(), Lo,
265 DAG.getConstant(MVT::getSizeInBits(Hi.getValueType())-1,
266 TLI.getShiftAmountTy()));
268 // For example, extension of an i48 to an i64. Leave the low part alone,
269 // sext_inreg the high part.
270 unsigned ExcessBits =
271 MVT::getSizeInBits(EVT) - MVT::getSizeInBits(Lo.getValueType());
272 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, Hi.getValueType(), Hi,
273 DAG.getValueType(MVT::getIntegerType(ExcessBits)));
277 void DAGTypeLegalizer::ExpandResult_LOAD(LoadSDNode *N,
278 SDOperand &Lo, SDOperand &Hi) {
279 MVT::ValueType VT = N->getValueType(0);
280 MVT::ValueType NVT = TLI.getTypeToTransformTo(VT);
281 SDOperand Ch = N->getChain(); // Legalize the chain.
282 SDOperand Ptr = N->getBasePtr(); // Legalize the pointer.
283 ISD::LoadExtType ExtType = N->getExtensionType();
284 int SVOffset = N->getSrcValueOffset();
285 unsigned Alignment = N->getAlignment();
286 bool isVolatile = N->isVolatile();
288 assert(!(MVT::getSizeInBits(NVT) & 7) && "Expanded type not byte sized!");
290 if (ExtType == ISD::NON_EXTLOAD) {
291 Lo = DAG.getLoad(NVT, Ch, Ptr, N->getSrcValue(), SVOffset,
292 isVolatile, Alignment);
293 // Increment the pointer to the other half.
294 unsigned IncrementSize = MVT::getSizeInBits(NVT)/8;
295 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
296 DAG.getIntPtrConstant(IncrementSize));
297 Hi = DAG.getLoad(NVT, Ch, Ptr, N->getSrcValue(), SVOffset+IncrementSize,
298 isVolatile, MinAlign(Alignment, IncrementSize));
300 // Build a factor node to remember that this load is independent of the
302 Ch = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1),
305 // Handle endianness of the load.
306 if (TLI.isBigEndian())
308 } else if (MVT::getSizeInBits(N->getMemoryVT()) <= MVT::getSizeInBits(NVT)) {
309 MVT::ValueType EVT = N->getMemoryVT();
311 Lo = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, N->getSrcValue(), SVOffset, EVT,
312 isVolatile, Alignment);
314 // Remember the chain.
317 if (ExtType == ISD::SEXTLOAD) {
318 // The high part is obtained by SRA'ing all but one of the bits of the
320 unsigned LoSize = MVT::getSizeInBits(Lo.getValueType());
321 Hi = DAG.getNode(ISD::SRA, NVT, Lo,
322 DAG.getConstant(LoSize-1, TLI.getShiftAmountTy()));
323 } else if (ExtType == ISD::ZEXTLOAD) {
324 // The high part is just a zero.
325 Hi = DAG.getConstant(0, NVT);
327 assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
328 // The high part is undefined.
329 Hi = DAG.getNode(ISD::UNDEF, NVT);
331 } else if (TLI.isLittleEndian()) {
332 // Little-endian - low bits are at low addresses.
333 Lo = DAG.getLoad(NVT, Ch, Ptr, N->getSrcValue(), SVOffset,
334 isVolatile, Alignment);
336 unsigned ExcessBits =
337 MVT::getSizeInBits(N->getMemoryVT()) - MVT::getSizeInBits(NVT);
338 MVT::ValueType NEVT = MVT::getIntegerType(ExcessBits);
340 // Increment the pointer to the other half.
341 unsigned IncrementSize = MVT::getSizeInBits(NVT)/8;
342 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
343 DAG.getIntPtrConstant(IncrementSize));
344 Hi = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, N->getSrcValue(),
345 SVOffset+IncrementSize, NEVT,
346 isVolatile, MinAlign(Alignment, IncrementSize));
348 // Build a factor node to remember that this load is independent of the
350 Ch = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1),
353 // Big-endian - high bits are at low addresses. Favor aligned loads at
354 // the cost of some bit-fiddling.
355 MVT::ValueType EVT = N->getMemoryVT();
356 unsigned EBytes = MVT::getStoreSizeInBits(EVT)/8;
357 unsigned IncrementSize = MVT::getSizeInBits(NVT)/8;
358 unsigned ExcessBits = (EBytes - IncrementSize)*8;
360 // Load both the high bits and maybe some of the low bits.
361 Hi = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, N->getSrcValue(), SVOffset,
362 MVT::getIntegerType(MVT::getSizeInBits(EVT)-ExcessBits),
363 isVolatile, Alignment);
365 // Increment the pointer to the other half.
366 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
367 DAG.getIntPtrConstant(IncrementSize));
368 // Load the rest of the low bits.
369 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, NVT, Ch, Ptr, N->getSrcValue(),
370 SVOffset+IncrementSize, MVT::getIntegerType(ExcessBits),
371 isVolatile, MinAlign(Alignment, IncrementSize));
373 // Build a factor node to remember that this load is independent of the
375 Ch = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1),
378 if (ExcessBits < MVT::getSizeInBits(NVT)) {
379 // Transfer low bits from the bottom of Hi to the top of Lo.
380 Lo = DAG.getNode(ISD::OR, NVT, Lo,
381 DAG.getNode(ISD::SHL, NVT, Hi,
382 DAG.getConstant(ExcessBits,
383 TLI.getShiftAmountTy())));
384 // Move high bits to the right position in Hi.
385 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, NVT, Hi,
386 DAG.getConstant(MVT::getSizeInBits(NVT) - ExcessBits,
387 TLI.getShiftAmountTy()));
391 // Legalized the chain result - switch anything that used the old chain to
393 ReplaceValueWith(SDOperand(N, 1), Ch);
396 void DAGTypeLegalizer::ExpandResult_Logical(SDNode *N,
397 SDOperand &Lo, SDOperand &Hi) {
398 SDOperand LL, LH, RL, RH;
399 GetExpandedOp(N->getOperand(0), LL, LH);
400 GetExpandedOp(N->getOperand(1), RL, RH);
401 Lo = DAG.getNode(N->getOpcode(), LL.getValueType(), LL, RL);
402 Hi = DAG.getNode(N->getOpcode(), LL.getValueType(), LH, RH);
405 void DAGTypeLegalizer::ExpandResult_BSWAP(SDNode *N,
406 SDOperand &Lo, SDOperand &Hi) {
407 GetExpandedOp(N->getOperand(0), Hi, Lo); // Note swapped operands.
408 Lo = DAG.getNode(ISD::BSWAP, Lo.getValueType(), Lo);
409 Hi = DAG.getNode(ISD::BSWAP, Hi.getValueType(), Hi);
412 void DAGTypeLegalizer::ExpandResult_SELECT(SDNode *N,
413 SDOperand &Lo, SDOperand &Hi) {
414 SDOperand LL, LH, RL, RH;
415 GetExpandedOp(N->getOperand(1), LL, LH);
416 GetExpandedOp(N->getOperand(2), RL, RH);
417 Lo = DAG.getNode(ISD::SELECT, LL.getValueType(), N->getOperand(0), LL, RL);
419 assert(N->getOperand(0).getValueType() != MVT::f32 &&
420 "FIXME: softfp shouldn't use expand!");
421 Hi = DAG.getNode(ISD::SELECT, LL.getValueType(), N->getOperand(0), LH, RH);
424 void DAGTypeLegalizer::ExpandResult_SELECT_CC(SDNode *N,
425 SDOperand &Lo, SDOperand &Hi) {
426 SDOperand LL, LH, RL, RH;
427 GetExpandedOp(N->getOperand(2), LL, LH);
428 GetExpandedOp(N->getOperand(3), RL, RH);
429 Lo = DAG.getNode(ISD::SELECT_CC, LL.getValueType(), N->getOperand(0),
430 N->getOperand(1), LL, RL, N->getOperand(4));
432 assert(N->getOperand(0).getValueType() != MVT::f32 &&
433 "FIXME: softfp shouldn't use expand!");
434 Hi = DAG.getNode(ISD::SELECT_CC, LL.getValueType(), N->getOperand(0),
435 N->getOperand(1), LH, RH, N->getOperand(4));
438 void DAGTypeLegalizer::ExpandResult_ADDSUB(SDNode *N,
439 SDOperand &Lo, SDOperand &Hi) {
440 // Expand the subcomponents.
441 SDOperand LHSL, LHSH, RHSL, RHSH;
442 GetExpandedOp(N->getOperand(0), LHSL, LHSH);
443 GetExpandedOp(N->getOperand(1), RHSL, RHSH);
444 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag);
445 SDOperand LoOps[2] = { LHSL, RHSL };
446 SDOperand HiOps[3] = { LHSH, RHSH };
448 if (N->getOpcode() == ISD::ADD) {
449 Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2);
450 HiOps[2] = Lo.getValue(1);
451 Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3);
453 Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2);
454 HiOps[2] = Lo.getValue(1);
455 Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3);
459 void DAGTypeLegalizer::ExpandResult_ADDSUBC(SDNode *N,
460 SDOperand &Lo, SDOperand &Hi) {
461 // Expand the subcomponents.
462 SDOperand LHSL, LHSH, RHSL, RHSH;
463 GetExpandedOp(N->getOperand(0), LHSL, LHSH);
464 GetExpandedOp(N->getOperand(1), RHSL, RHSH);
465 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag);
466 SDOperand LoOps[2] = { LHSL, RHSL };
467 SDOperand HiOps[3] = { LHSH, RHSH };
469 if (N->getOpcode() == ISD::ADDC) {
470 Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2);
471 HiOps[2] = Lo.getValue(1);
472 Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3);
474 Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2);
475 HiOps[2] = Lo.getValue(1);
476 Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3);
479 // Legalized the flag result - switch anything that used the old flag to
481 ReplaceValueWith(SDOperand(N, 1), Hi.getValue(1));
484 void DAGTypeLegalizer::ExpandResult_ADDSUBE(SDNode *N,
485 SDOperand &Lo, SDOperand &Hi) {
486 // Expand the subcomponents.
487 SDOperand LHSL, LHSH, RHSL, RHSH;
488 GetExpandedOp(N->getOperand(0), LHSL, LHSH);
489 GetExpandedOp(N->getOperand(1), RHSL, RHSH);
490 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag);
491 SDOperand LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
492 SDOperand HiOps[3] = { LHSH, RHSH };
494 Lo = DAG.getNode(N->getOpcode(), VTList, LoOps, 3);
495 HiOps[2] = Lo.getValue(1);
496 Hi = DAG.getNode(N->getOpcode(), VTList, HiOps, 3);
498 // Legalized the flag result - switch anything that used the old flag to
500 ReplaceValueWith(SDOperand(N, 1), Hi.getValue(1));
503 void DAGTypeLegalizer::ExpandResult_MUL(SDNode *N,
504 SDOperand &Lo, SDOperand &Hi) {
505 MVT::ValueType VT = N->getValueType(0);
506 MVT::ValueType NVT = TLI.getTypeToTransformTo(VT);
508 bool HasMULHS = TLI.isOperationLegal(ISD::MULHS, NVT);
509 bool HasMULHU = TLI.isOperationLegal(ISD::MULHU, NVT);
510 bool HasSMUL_LOHI = TLI.isOperationLegal(ISD::SMUL_LOHI, NVT);
511 bool HasUMUL_LOHI = TLI.isOperationLegal(ISD::UMUL_LOHI, NVT);
512 if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) {
513 SDOperand LL, LH, RL, RH;
514 GetExpandedOp(N->getOperand(0), LL, LH);
515 GetExpandedOp(N->getOperand(1), RL, RH);
516 unsigned OuterBitSize = MVT::getSizeInBits(VT);
517 unsigned BitSize = MVT::getSizeInBits(NVT);
518 unsigned LHSSB = DAG.ComputeNumSignBits(N->getOperand(0));
519 unsigned RHSSB = DAG.ComputeNumSignBits(N->getOperand(1));
521 if (DAG.MaskedValueIsZero(N->getOperand(0),
522 APInt::getHighBitsSet(OuterBitSize, LHSSB)) &&
523 DAG.MaskedValueIsZero(N->getOperand(1),
524 APInt::getHighBitsSet(OuterBitSize, RHSSB))) {
525 // The inputs are both zero-extended.
527 // We can emit a umul_lohi.
528 Lo = DAG.getNode(ISD::UMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL);
529 Hi = SDOperand(Lo.Val, 1);
533 // We can emit a mulhu+mul.
534 Lo = DAG.getNode(ISD::MUL, NVT, LL, RL);
535 Hi = DAG.getNode(ISD::MULHU, NVT, LL, RL);
539 if (LHSSB > BitSize && RHSSB > BitSize) {
540 // The input values are both sign-extended.
542 // We can emit a smul_lohi.
543 Lo = DAG.getNode(ISD::SMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL);
544 Hi = SDOperand(Lo.Val, 1);
548 // We can emit a mulhs+mul.
549 Lo = DAG.getNode(ISD::MUL, NVT, LL, RL);
550 Hi = DAG.getNode(ISD::MULHS, NVT, LL, RL);
555 // Lo,Hi = umul LHS, RHS.
556 SDOperand UMulLOHI = DAG.getNode(ISD::UMUL_LOHI,
557 DAG.getVTList(NVT, NVT), LL, RL);
559 Hi = UMulLOHI.getValue(1);
560 RH = DAG.getNode(ISD::MUL, NVT, LL, RH);
561 LH = DAG.getNode(ISD::MUL, NVT, LH, RL);
562 Hi = DAG.getNode(ISD::ADD, NVT, Hi, RH);
563 Hi = DAG.getNode(ISD::ADD, NVT, Hi, LH);
570 // If nothing else, we can make a libcall.
571 Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::MUL_I64), N,
572 false/*sign irrelevant*/, Hi);
577 void DAGTypeLegalizer::ExpandResult_Shift(SDNode *N,
578 SDOperand &Lo, SDOperand &Hi) {
579 MVT::ValueType VT = N->getValueType(0);
581 // If we can emit an efficient shift operation, do so now. Check to see if
582 // the RHS is a constant.
583 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
584 return ExpandShiftByConstant(N, CN->getValue(), Lo, Hi);
586 // If we can determine that the high bit of the shift is zero or one, even if
587 // the low bits are variable, emit this shift in an optimized form.
588 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
591 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
593 if (N->getOpcode() == ISD::SHL)
594 PartsOpc = ISD::SHL_PARTS;
595 else if (N->getOpcode() == ISD::SRL)
596 PartsOpc = ISD::SRL_PARTS;
598 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
599 PartsOpc = ISD::SRA_PARTS;
602 // Next check to see if the target supports this SHL_PARTS operation or if it
603 // will custom expand it.
604 MVT::ValueType NVT = TLI.getTypeToTransformTo(VT);
605 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
606 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
607 Action == TargetLowering::Custom) {
608 // Expand the subcomponents.
609 SDOperand LHSL, LHSH;
610 GetExpandedOp(N->getOperand(0), LHSL, LHSH);
612 SDOperand Ops[] = { LHSL, LHSH, N->getOperand(1) };
613 MVT::ValueType VT = LHSL.getValueType();
614 Lo = DAG.getNode(PartsOpc, DAG.getNodeValueTypes(VT, VT), 2, Ops, 3);
621 // Otherwise, emit a libcall.
622 unsigned RuntimeCode = ; // SRL -> SRL_I64 etc.
624 Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::SRL_I64), N,
625 false/*lshr is unsigned*/, Hi);
629 void DAGTypeLegalizer::ExpandResult_CTLZ(SDNode *N,
630 SDOperand &Lo, SDOperand &Hi) {
631 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
632 GetExpandedOp(N->getOperand(0), Lo, Hi);
633 MVT::ValueType NVT = Lo.getValueType();
635 SDOperand HiNotZero = DAG.getSetCC(TLI.getSetCCResultType(Hi), Hi,
636 DAG.getConstant(0, NVT), ISD::SETNE);
638 SDOperand LoLZ = DAG.getNode(ISD::CTLZ, NVT, Lo);
639 SDOperand HiLZ = DAG.getNode(ISD::CTLZ, NVT, Hi);
641 Lo = DAG.getNode(ISD::SELECT, NVT, HiNotZero, HiLZ,
642 DAG.getNode(ISD::ADD, NVT, LoLZ,
643 DAG.getConstant(MVT::getSizeInBits(NVT), NVT)));
644 Hi = DAG.getConstant(0, NVT);
647 void DAGTypeLegalizer::ExpandResult_CTPOP(SDNode *N,
648 SDOperand &Lo, SDOperand &Hi) {
649 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
650 GetExpandedOp(N->getOperand(0), Lo, Hi);
651 MVT::ValueType NVT = Lo.getValueType();
652 Lo = DAG.getNode(ISD::ADD, NVT, DAG.getNode(ISD::CTPOP, NVT, Lo),
653 DAG.getNode(ISD::CTPOP, NVT, Hi));
654 Hi = DAG.getConstant(0, NVT);
657 void DAGTypeLegalizer::ExpandResult_CTTZ(SDNode *N,
658 SDOperand &Lo, SDOperand &Hi) {
659 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
660 GetExpandedOp(N->getOperand(0), Lo, Hi);
661 MVT::ValueType NVT = Lo.getValueType();
663 SDOperand LoNotZero = DAG.getSetCC(TLI.getSetCCResultType(Lo), Lo,
664 DAG.getConstant(0, NVT), ISD::SETNE);
666 SDOperand LoLZ = DAG.getNode(ISD::CTTZ, NVT, Lo);
667 SDOperand HiLZ = DAG.getNode(ISD::CTTZ, NVT, Hi);
669 Lo = DAG.getNode(ISD::SELECT, NVT, LoNotZero, LoLZ,
670 DAG.getNode(ISD::ADD, NVT, HiLZ,
671 DAG.getConstant(MVT::getSizeInBits(NVT), NVT)));
672 Hi = DAG.getConstant(0, NVT);
675 void DAGTypeLegalizer::ExpandResult_EXTRACT_VECTOR_ELT(SDNode *N,
678 SDOperand OldVec = N->getOperand(0);
679 unsigned OldElts = MVT::getVectorNumElements(OldVec.getValueType());
681 // Convert to a vector of the expanded element type, for example
682 // <2 x i64> -> <4 x i32>.
683 MVT::ValueType OldVT = N->getValueType(0);
684 MVT::ValueType NewVT = TLI.getTypeToTransformTo(OldVT);
685 assert(MVT::getSizeInBits(OldVT) == 2 * MVT::getSizeInBits(NewVT) &&
686 "Do not know how to handle this expansion!");
688 SDOperand NewVec = DAG.getNode(ISD::BIT_CONVERT,
689 MVT::getVectorType(NewVT, 2 * OldElts),
692 // Extract the elements at 2 * Idx and 2 * Idx + 1 from the new vector.
693 SDOperand Idx = N->getOperand(1);
695 // Make sure the type of Idx is big enough to hold the new values.
696 if (MVT::getSizeInBits(Idx.getValueType()) < 32)
697 Idx = DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, Idx);
699 Idx = DAG.getNode(ISD::ADD, Idx.getValueType(), Idx, Idx);
700 Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, NewVT, NewVec, Idx);
702 Idx = DAG.getNode(ISD::ADD, Idx.getValueType(), Idx,
703 DAG.getConstant(1, Idx.getValueType()));
704 Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, NewVT, NewVec, Idx);
706 if (TLI.isBigEndian())
710 /// ExpandShiftByConstant - N is a shift by a value that needs to be expanded,
711 /// and the shift amount is a constant 'Amt'. Expand the operation.
712 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, unsigned Amt,
713 SDOperand &Lo, SDOperand &Hi) {
714 // Expand the incoming operand to be shifted, so that we have its parts
716 GetExpandedOp(N->getOperand(0), InL, InH);
718 MVT::ValueType NVT = InL.getValueType();
719 unsigned VTBits = MVT::getSizeInBits(N->getValueType(0));
720 unsigned NVTBits = MVT::getSizeInBits(NVT);
721 MVT::ValueType ShTy = N->getOperand(1).getValueType();
723 if (N->getOpcode() == ISD::SHL) {
725 Lo = Hi = DAG.getConstant(0, NVT);
726 } else if (Amt > NVTBits) {
727 Lo = DAG.getConstant(0, NVT);
728 Hi = DAG.getNode(ISD::SHL, NVT, InL, DAG.getConstant(Amt-NVTBits,ShTy));
729 } else if (Amt == NVTBits) {
730 Lo = DAG.getConstant(0, NVT);
733 Lo = DAG.getNode(ISD::SHL, NVT, InL, DAG.getConstant(Amt, ShTy));
734 Hi = DAG.getNode(ISD::OR, NVT,
735 DAG.getNode(ISD::SHL, NVT, InH,
736 DAG.getConstant(Amt, ShTy)),
737 DAG.getNode(ISD::SRL, NVT, InL,
738 DAG.getConstant(NVTBits-Amt, ShTy)));
743 if (N->getOpcode() == ISD::SRL) {
745 Lo = DAG.getConstant(0, NVT);
746 Hi = DAG.getConstant(0, NVT);
747 } else if (Amt > NVTBits) {
748 Lo = DAG.getNode(ISD::SRL, NVT, InH, DAG.getConstant(Amt-NVTBits,ShTy));
749 Hi = DAG.getConstant(0, NVT);
750 } else if (Amt == NVTBits) {
752 Hi = DAG.getConstant(0, NVT);
754 Lo = DAG.getNode(ISD::OR, NVT,
755 DAG.getNode(ISD::SRL, NVT, InL,
756 DAG.getConstant(Amt, ShTy)),
757 DAG.getNode(ISD::SHL, NVT, InH,
758 DAG.getConstant(NVTBits-Amt, ShTy)));
759 Hi = DAG.getNode(ISD::SRL, NVT, InH, DAG.getConstant(Amt, ShTy));
764 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
766 Hi = Lo = DAG.getNode(ISD::SRA, NVT, InH,
767 DAG.getConstant(NVTBits-1, ShTy));
768 } else if (Amt > NVTBits) {
769 Lo = DAG.getNode(ISD::SRA, NVT, InH,
770 DAG.getConstant(Amt-NVTBits, ShTy));
771 Hi = DAG.getNode(ISD::SRA, NVT, InH,
772 DAG.getConstant(NVTBits-1, ShTy));
773 } else if (Amt == NVTBits) {
775 Hi = DAG.getNode(ISD::SRA, NVT, InH,
776 DAG.getConstant(NVTBits-1, ShTy));
778 Lo = DAG.getNode(ISD::OR, NVT,
779 DAG.getNode(ISD::SRL, NVT, InL,
780 DAG.getConstant(Amt, ShTy)),
781 DAG.getNode(ISD::SHL, NVT, InH,
782 DAG.getConstant(NVTBits-Amt, ShTy)));
783 Hi = DAG.getNode(ISD::SRA, NVT, InH, DAG.getConstant(Amt, ShTy));
787 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
788 /// this shift based on knowledge of the high bit of the shift amount. If we
789 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
791 bool DAGTypeLegalizer::
792 ExpandShiftWithKnownAmountBit(SDNode *N, SDOperand &Lo, SDOperand &Hi) {
793 SDOperand Amt = N->getOperand(1);
794 MVT::ValueType NVT = TLI.getTypeToTransformTo(N->getValueType(0));
795 MVT::ValueType ShTy = Amt.getValueType();
796 MVT::ValueType ShBits = MVT::getSizeInBits(ShTy);
797 unsigned NVTBits = MVT::getSizeInBits(NVT);
798 assert(isPowerOf2_32(NVTBits) &&
799 "Expanded integer type size not a power of two!");
801 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
802 APInt KnownZero, KnownOne;
803 DAG.ComputeMaskedBits(N->getOperand(1), HighBitMask, KnownZero, KnownOne);
805 // If we don't know anything about the high bits, exit.
806 if (((KnownZero|KnownOne) & HighBitMask) == 0)
809 // Get the incoming operand to be shifted.
811 GetExpandedOp(N->getOperand(0), InL, InH);
813 // If we know that any of the high bits of the shift amount are one, then we
814 // can do this as a couple of simple shifts.
815 if (KnownOne.intersects(HighBitMask)) {
816 // Mask out the high bit, which we know is set.
817 Amt = DAG.getNode(ISD::AND, ShTy, Amt,
818 DAG.getConstant(~HighBitMask, ShTy));
820 switch (N->getOpcode()) {
821 default: assert(0 && "Unknown shift");
823 Lo = DAG.getConstant(0, NVT); // Low part is zero.
824 Hi = DAG.getNode(ISD::SHL, NVT, InL, Amt); // High part from Lo part.
827 Hi = DAG.getConstant(0, NVT); // Hi part is zero.
828 Lo = DAG.getNode(ISD::SRL, NVT, InH, Amt); // Lo part from Hi part.
831 Hi = DAG.getNode(ISD::SRA, NVT, InH, // Sign extend high part.
832 DAG.getConstant(NVTBits-1, ShTy));
833 Lo = DAG.getNode(ISD::SRA, NVT, InH, Amt); // Lo part from Hi part.
838 // If we know that all of the high bits of the shift amount are zero, then we
839 // can do this as a couple of simple shifts.
840 if ((KnownZero & HighBitMask) == HighBitMask) {
842 SDOperand Amt2 = DAG.getNode(ISD::SUB, ShTy,
843 DAG.getConstant(NVTBits, ShTy),
846 switch (N->getOpcode()) {
847 default: assert(0 && "Unknown shift");
848 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
850 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
853 Lo = DAG.getNode(N->getOpcode(), NVT, InL, Amt);
854 Hi = DAG.getNode(ISD::OR, NVT,
855 DAG.getNode(Op1, NVT, InH, Amt),
856 DAG.getNode(Op2, NVT, InL, Amt2));
864 //===----------------------------------------------------------------------===//
866 //===----------------------------------------------------------------------===//
868 /// ExpandOperand - This method is called when the specified operand of the
869 /// specified node is found to need expansion. At this point, all of the result
870 /// types of the node are known to be legal, but other operands of the node may
871 /// need promotion or expansion as well as the specified one.
872 bool DAGTypeLegalizer::ExpandOperand(SDNode *N, unsigned OpNo) {
873 DEBUG(cerr << "Expand node operand: "; N->dump(&DAG); cerr << "\n");
876 if (TLI.getOperationAction(N->getOpcode(), N->getOperand(OpNo).getValueType())
877 == TargetLowering::Custom)
878 Res = TLI.LowerOperation(SDOperand(N, 0), DAG);
881 switch (N->getOpcode()) {
884 cerr << "ExpandOperand Op #" << OpNo << ": ";
885 N->dump(&DAG); cerr << "\n";
887 assert(0 && "Do not know how to expand this operator's operand!");
890 case ISD::TRUNCATE: Res = ExpandOperand_TRUNCATE(N); break;
891 case ISD::BIT_CONVERT: Res = ExpandOperand_BIT_CONVERT(N); break;
893 case ISD::SINT_TO_FP:
894 Res = ExpandOperand_SINT_TO_FP(N->getOperand(0), N->getValueType(0));
896 case ISD::UINT_TO_FP:
897 Res = ExpandOperand_UINT_TO_FP(N->getOperand(0), N->getValueType(0));
899 case ISD::EXTRACT_ELEMENT: Res = ExpandOperand_EXTRACT_ELEMENT(N); break;
901 case ISD::BR_CC: Res = ExpandOperand_BR_CC(N); break;
902 case ISD::SETCC: Res = ExpandOperand_SETCC(N); break;
905 Res = ExpandOperand_STORE(cast<StoreSDNode>(N), OpNo);
909 case ISD::MEMMOVE: Res = HandleMemIntrinsic(N); break;
911 case ISD::BUILD_VECTOR: Res = ExpandOperand_BUILD_VECTOR(N); break;
915 // If the result is null, the sub-method took care of registering results etc.
916 if (!Res.Val) return false;
917 // If the result is N, the sub-method updated N in place. Check to see if any
918 // operands are new, and if so, mark them.
920 // Mark N as new and remark N and its operands. This allows us to correctly
921 // revisit N if it needs another step of promotion and allows us to visit
922 // any new operands to N.
927 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
928 "Invalid operand expansion");
930 ReplaceValueWith(SDOperand(N, 0), Res);
934 SDOperand DAGTypeLegalizer::ExpandOperand_TRUNCATE(SDNode *N) {
936 GetExpandedOp(N->getOperand(0), InL, InH);
937 // Just truncate the low part of the source.
938 return DAG.getNode(ISD::TRUNCATE, N->getValueType(0), InL);
941 SDOperand DAGTypeLegalizer::ExpandOperand_BIT_CONVERT(SDNode *N) {
942 if (MVT::isVector(N->getValueType(0))) {
943 // An illegal integer type is being converted to a legal vector type.
944 // Make a two element vector out of the expanded parts and convert that
945 // instead, but only if the new vector type is legal (otherwise there
946 // is no point, and it might create expansion loops). For example, on
947 // x86 this turns v1i64 = BIT_CONVERT i64 into v1i64 = BIT_CONVERT v2i32.
948 MVT::ValueType OVT = N->getOperand(0).getValueType();
949 MVT::ValueType NVT = MVT::getVectorType(TLI.getTypeToTransformTo(OVT), 2);
951 if (isTypeLegal(NVT)) {
953 GetExpandedOp(N->getOperand(0), Parts[0], Parts[1]);
955 if (TLI.isBigEndian())
956 std::swap(Parts[0], Parts[1]);
958 SDOperand Vec = DAG.getNode(ISD::BUILD_VECTOR, NVT, Parts, 2);
959 return DAG.getNode(ISD::BIT_CONVERT, N->getValueType(0), Vec);
963 // Otherwise, store to a temporary and load out again as the new type.
964 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
967 SDOperand DAGTypeLegalizer::ExpandOperand_SINT_TO_FP(SDOperand Source,
968 MVT::ValueType DestTy) {
969 // We know the destination is legal, but that the input needs to be expanded.
970 MVT::ValueType SourceVT = Source.getValueType();
972 // Check to see if the target has a custom way to lower this. If so, use it.
973 switch (TLI.getOperationAction(ISD::SINT_TO_FP, SourceVT)) {
974 default: assert(0 && "This action not implemented for this operation!");
975 case TargetLowering::Legal:
976 case TargetLowering::Expand:
977 break; // This case is handled below.
978 case TargetLowering::Custom:
979 SDOperand NV = TLI.LowerOperation(DAG.getNode(ISD::SINT_TO_FP, DestTy,
981 if (NV.Val) return NV;
982 break; // The target lowered this.
986 if (SourceVT == MVT::i64) {
987 if (DestTy == MVT::f32)
988 LC = RTLIB::SINTTOFP_I64_F32;
990 assert(DestTy == MVT::f64 && "Unknown fp value type!");
991 LC = RTLIB::SINTTOFP_I64_F64;
993 } else if (SourceVT == MVT::i128) {
994 if (DestTy == MVT::f32)
995 LC = RTLIB::SINTTOFP_I128_F32;
996 else if (DestTy == MVT::f64)
997 LC = RTLIB::SINTTOFP_I128_F64;
998 else if (DestTy == MVT::f80)
999 LC = RTLIB::SINTTOFP_I128_F80;
1001 assert(DestTy == MVT::ppcf128 && "Unknown fp value type!");
1002 LC = RTLIB::SINTTOFP_I128_PPCF128;
1005 assert(0 && "Unknown int value type!");
1008 assert(0 && "FIXME: no libcalls yet!");
1011 assert(TLI.getLibcallName(LC) && "Don't know how to expand this SINT_TO_FP!");
1012 Source = DAG.getNode(ISD::SINT_TO_FP, DestTy, Source);
1013 SDOperand UnusedHiPart;
1014 return ExpandLibCall(TLI.getLibcallName(LC), Source.Val, true, UnusedHiPart);
1018 SDOperand DAGTypeLegalizer::ExpandOperand_UINT_TO_FP(SDOperand Source,
1019 MVT::ValueType DestTy) {
1020 // We know the destination is legal, but that the input needs to be expanded.
1021 assert(getTypeAction(Source.getValueType()) == Expand &&
1022 "This is not an expansion!");
1024 // If this is unsigned, and not supported, first perform the conversion to
1025 // signed, then adjust the result if the sign bit is set.
1026 SDOperand SignedConv = ExpandOperand_SINT_TO_FP(Source, DestTy);
1028 // The 64-bit value loaded will be incorrectly if the 'sign bit' of the
1029 // incoming integer is set. To handle this, we dynamically test to see if
1030 // it is set, and, if so, add a fudge factor.
1032 GetExpandedOp(Source, Lo, Hi);
1034 SDOperand SignSet = DAG.getSetCC(TLI.getSetCCResultType(Hi), Hi,
1035 DAG.getConstant(0, Hi.getValueType()),
1037 SDOperand Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4);
1038 SDOperand CstOffset = DAG.getNode(ISD::SELECT, Zero.getValueType(),
1039 SignSet, Four, Zero);
1040 uint64_t FF = 0x5f800000ULL;
1041 if (TLI.isLittleEndian()) FF <<= 32;
1042 Constant *FudgeFactor = ConstantInt::get((Type*)Type::Int64Ty, FF);
1044 SDOperand CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy());
1045 CPIdx = DAG.getNode(ISD::ADD, TLI.getPointerTy(), CPIdx, CstOffset);
1046 SDOperand FudgeInReg;
1047 if (DestTy == MVT::f32)
1048 FudgeInReg = DAG.getLoad(MVT::f32, DAG.getEntryNode(), CPIdx, NULL, 0);
1049 else if (MVT::getSizeInBits(DestTy) > MVT::getSizeInBits(MVT::f32))
1050 // FIXME: Avoid the extend by construction the right constantpool?
1051 FudgeInReg = DAG.getExtLoad(ISD::EXTLOAD, DestTy, DAG.getEntryNode(),
1052 CPIdx, NULL, 0, MVT::f32);
1054 assert(0 && "Unexpected conversion");
1056 return DAG.getNode(ISD::FADD, DestTy, SignedConv, FudgeInReg);
1059 SDOperand DAGTypeLegalizer::ExpandOperand_EXTRACT_ELEMENT(SDNode *N) {
1061 GetExpandedOp(N->getOperand(0), Lo, Hi);
1062 return cast<ConstantSDNode>(N->getOperand(1))->getValue() ? Hi : Lo;
1065 SDOperand DAGTypeLegalizer::ExpandOperand_BR_CC(SDNode *N) {
1066 SDOperand NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1067 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1068 ExpandSetCCOperands(NewLHS, NewRHS, CCCode);
1070 // If ExpandSetCCOperands returned a scalar, we need to compare the result
1071 // against zero to select between true and false values.
1072 if (NewRHS.Val == 0) {
1073 NewRHS = DAG.getConstant(0, NewLHS.getValueType());
1074 CCCode = ISD::SETNE;
1077 // Update N to have the operands specified.
1078 return DAG.UpdateNodeOperands(SDOperand(N, 0), N->getOperand(0),
1079 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1083 SDOperand DAGTypeLegalizer::ExpandOperand_SETCC(SDNode *N) {
1084 SDOperand NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1085 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
1086 ExpandSetCCOperands(NewLHS, NewRHS, CCCode);
1088 // If ExpandSetCCOperands returned a scalar, use it.
1089 if (NewRHS.Val == 0) return NewLHS;
1091 // Otherwise, update N to have the operands specified.
1092 return DAG.UpdateNodeOperands(SDOperand(N, 0), NewLHS, NewRHS,
1093 DAG.getCondCode(CCCode));
1096 /// ExpandSetCCOperands - Expand the operands of a comparison. This code is
1097 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
1098 void DAGTypeLegalizer::ExpandSetCCOperands(SDOperand &NewLHS, SDOperand &NewRHS,
1099 ISD::CondCode &CCCode) {
1100 SDOperand LHSLo, LHSHi, RHSLo, RHSHi;
1101 GetExpandedOp(NewLHS, LHSLo, LHSHi);
1102 GetExpandedOp(NewRHS, RHSLo, RHSHi);
1104 MVT::ValueType VT = NewLHS.getValueType();
1105 if (VT == MVT::f32 || VT == MVT::f64) {
1106 assert(0 && "FIXME: softfp not implemented yet! should be promote not exp");
1109 if (VT == MVT::ppcf128) {
1110 // FIXME: This generated code sucks. We want to generate
1111 // FCMP crN, hi1, hi2
1113 // FCMP crN, lo1, lo2
1114 // The following can be improved, but not that much.
1115 SDOperand Tmp1, Tmp2, Tmp3;
1116 Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, ISD::SETEQ);
1117 Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(LHSLo), LHSLo, RHSLo, CCCode);
1118 Tmp3 = DAG.getNode(ISD::AND, Tmp1.getValueType(), Tmp1, Tmp2);
1119 Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, ISD::SETNE);
1120 Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, CCCode);
1121 Tmp1 = DAG.getNode(ISD::AND, Tmp1.getValueType(), Tmp1, Tmp2);
1122 NewLHS = DAG.getNode(ISD::OR, Tmp1.getValueType(), Tmp1, Tmp3);
1123 NewRHS = SDOperand(); // LHS is the result, not a compare.
1128 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
1130 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo))
1131 if (RHSCST->isAllOnesValue()) {
1132 // Equality comparison to -1.
1133 NewLHS = DAG.getNode(ISD::AND, LHSLo.getValueType(), LHSLo, LHSHi);
1138 NewLHS = DAG.getNode(ISD::XOR, LHSLo.getValueType(), LHSLo, RHSLo);
1139 NewRHS = DAG.getNode(ISD::XOR, LHSLo.getValueType(), LHSHi, RHSHi);
1140 NewLHS = DAG.getNode(ISD::OR, NewLHS.getValueType(), NewLHS, NewRHS);
1141 NewRHS = DAG.getConstant(0, NewLHS.getValueType());
1145 // If this is a comparison of the sign bit, just look at the top part.
1147 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
1148 if ((CCCode == ISD::SETLT && CST->getValue() == 0) || // X < 0
1149 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
1155 // FIXME: This generated code sucks.
1156 ISD::CondCode LowCC;
1158 default: assert(0 && "Unknown integer setcc!");
1160 case ISD::SETULT: LowCC = ISD::SETULT; break;
1162 case ISD::SETUGT: LowCC = ISD::SETUGT; break;
1164 case ISD::SETULE: LowCC = ISD::SETULE; break;
1166 case ISD::SETUGE: LowCC = ISD::SETUGE; break;
1169 // Tmp1 = lo(op1) < lo(op2) // Always unsigned comparison
1170 // Tmp2 = hi(op1) < hi(op2) // Signedness depends on operands
1171 // dest = hi(op1) == hi(op2) ? Tmp1 : Tmp2;
1173 // NOTE: on targets without efficient SELECT of bools, we can always use
1174 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
1175 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, false, true, NULL);
1176 SDOperand Tmp1, Tmp2;
1177 Tmp1 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSLo), LHSLo, RHSLo, LowCC,
1178 false, DagCombineInfo);
1180 Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSLo), LHSLo, RHSLo, LowCC);
1181 Tmp2 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi,
1182 CCCode, false, DagCombineInfo);
1184 Tmp2 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi,
1185 DAG.getCondCode(CCCode));
1187 ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.Val);
1188 ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.Val);
1189 if ((Tmp1C && Tmp1C->getValue() == 0) ||
1190 (Tmp2C && Tmp2C->getValue() == 0 &&
1191 (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
1192 CCCode == ISD::SETUGE || CCCode == ISD::SETULE)) ||
1193 (Tmp2C && Tmp2C->getValue() == 1 &&
1194 (CCCode == ISD::SETLT || CCCode == ISD::SETGT ||
1195 CCCode == ISD::SETUGT || CCCode == ISD::SETULT))) {
1196 // low part is known false, returns high part.
1197 // For LE / GE, if high part is known false, ignore the low part.
1198 // For LT / GT, if high part is known true, ignore the low part.
1200 NewRHS = SDOperand();
1204 NewLHS = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi,
1205 ISD::SETEQ, false, DagCombineInfo);
1207 NewLHS = DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi,
1209 NewLHS = DAG.getNode(ISD::SELECT, Tmp1.getValueType(),
1210 NewLHS, Tmp1, Tmp2);
1211 NewRHS = SDOperand();
1214 SDOperand DAGTypeLegalizer::ExpandOperand_STORE(StoreSDNode *N, unsigned OpNo) {
1215 assert(OpNo == 1 && "Can only expand the stored value so far");
1217 MVT::ValueType VT = N->getOperand(1).getValueType();
1218 MVT::ValueType NVT = TLI.getTypeToTransformTo(VT);
1219 SDOperand Ch = N->getChain();
1220 SDOperand Ptr = N->getBasePtr();
1221 int SVOffset = N->getSrcValueOffset();
1222 unsigned Alignment = N->getAlignment();
1223 bool isVolatile = N->isVolatile();
1226 assert(!(MVT::getSizeInBits(NVT) & 7) && "Expanded type not byte sized!");
1228 if (!N->isTruncatingStore()) {
1229 unsigned IncrementSize = 0;
1230 GetExpandedOp(N->getValue(), Lo, Hi);
1231 IncrementSize = MVT::getSizeInBits(Hi.getValueType())/8;
1233 if (TLI.isBigEndian())
1236 Lo = DAG.getStore(Ch, Lo, Ptr, N->getSrcValue(),
1237 SVOffset, isVolatile, Alignment);
1239 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
1240 DAG.getIntPtrConstant(IncrementSize));
1241 assert(isTypeLegal(Ptr.getValueType()) && "Pointers must be legal!");
1242 Hi = DAG.getStore(Ch, Hi, Ptr, N->getSrcValue(), SVOffset+IncrementSize,
1243 isVolatile, MinAlign(Alignment, IncrementSize));
1244 return DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi);
1245 } else if (MVT::getSizeInBits(N->getMemoryVT()) <= MVT::getSizeInBits(NVT)) {
1246 GetExpandedOp(N->getValue(), Lo, Hi);
1247 return DAG.getTruncStore(Ch, Lo, Ptr, N->getSrcValue(), SVOffset,
1248 N->getMemoryVT(), isVolatile, Alignment);
1249 } else if (TLI.isLittleEndian()) {
1250 // Little-endian - low bits are at low addresses.
1251 GetExpandedOp(N->getValue(), Lo, Hi);
1253 Lo = DAG.getStore(Ch, Lo, Ptr, N->getSrcValue(), SVOffset,
1254 isVolatile, Alignment);
1256 unsigned ExcessBits =
1257 MVT::getSizeInBits(N->getMemoryVT()) - MVT::getSizeInBits(NVT);
1258 MVT::ValueType NEVT = MVT::getIntegerType(ExcessBits);
1260 // Increment the pointer to the other half.
1261 unsigned IncrementSize = MVT::getSizeInBits(NVT)/8;
1262 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
1263 DAG.getIntPtrConstant(IncrementSize));
1264 Hi = DAG.getTruncStore(Ch, Hi, Ptr, N->getSrcValue(),
1265 SVOffset+IncrementSize, NEVT,
1266 isVolatile, MinAlign(Alignment, IncrementSize));
1267 return DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi);
1269 // Big-endian - high bits are at low addresses. Favor aligned stores at
1270 // the cost of some bit-fiddling.
1271 GetExpandedOp(N->getValue(), Lo, Hi);
1273 MVT::ValueType EVT = N->getMemoryVT();
1274 unsigned EBytes = MVT::getStoreSizeInBits(EVT)/8;
1275 unsigned IncrementSize = MVT::getSizeInBits(NVT)/8;
1276 unsigned ExcessBits = (EBytes - IncrementSize)*8;
1277 MVT::ValueType HiVT =
1278 MVT::getIntegerType(MVT::getSizeInBits(EVT)-ExcessBits);
1280 if (ExcessBits < MVT::getSizeInBits(NVT)) {
1281 // Transfer high bits from the top of Lo to the bottom of Hi.
1282 Hi = DAG.getNode(ISD::SHL, NVT, Hi,
1283 DAG.getConstant(MVT::getSizeInBits(NVT) - ExcessBits,
1284 TLI.getShiftAmountTy()));
1285 Hi = DAG.getNode(ISD::OR, NVT, Hi,
1286 DAG.getNode(ISD::SRL, NVT, Lo,
1287 DAG.getConstant(ExcessBits,
1288 TLI.getShiftAmountTy())));
1291 // Store both the high bits and maybe some of the low bits.
1292 Hi = DAG.getTruncStore(Ch, Hi, Ptr, N->getSrcValue(),
1293 SVOffset, HiVT, isVolatile, Alignment);
1295 // Increment the pointer to the other half.
1296 Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
1297 DAG.getIntPtrConstant(IncrementSize));
1298 // Store the lowest ExcessBits bits in the second half.
1299 Lo = DAG.getTruncStore(Ch, Lo, Ptr, N->getSrcValue(),
1300 SVOffset+IncrementSize,
1301 MVT::getIntegerType(ExcessBits),
1302 isVolatile, MinAlign(Alignment, IncrementSize));
1303 return DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi);
1307 SDOperand DAGTypeLegalizer::ExpandOperand_BUILD_VECTOR(SDNode *N) {
1308 // The vector type is legal but the element type needs expansion.
1309 MVT::ValueType VecVT = N->getValueType(0);
1310 unsigned NumElts = MVT::getVectorNumElements(VecVT);
1311 MVT::ValueType OldVT = N->getOperand(0).getValueType();
1312 MVT::ValueType NewVT = TLI.getTypeToTransformTo(OldVT);
1314 assert(MVT::getSizeInBits(OldVT) == 2 * MVT::getSizeInBits(NewVT) &&
1315 "Do not know how to expand this operand!");
1317 // Build a vector of twice the length out of the expanded elements.
1318 // For example <2 x i64> -> <4 x i32>.
1319 std::vector<SDOperand> NewElts;
1320 NewElts.reserve(NumElts*2);
1322 for (unsigned i = 0; i < NumElts; ++i) {
1324 GetExpandedOp(N->getOperand(i), Lo, Hi);
1325 if (TLI.isBigEndian())
1327 NewElts.push_back(Lo);
1328 NewElts.push_back(Hi);
1331 SDOperand NewVec = DAG.getNode(ISD::BUILD_VECTOR,
1332 MVT::getVectorType(NewVT, NewElts.size()),
1333 &NewElts[0], NewElts.size());
1335 // Convert the new vector to the old vector type.
1336 return DAG.getNode(ISD::BIT_CONVERT, VecVT, NewVec);