1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file performs vector type splitting and scalarization for LegalizeTypes.
11 // Scalarization is the act of changing a computation in an illegal one-element
12 // vector type to be a computation in its scalar element type. For example,
13 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
14 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
15 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
17 // Splitting is the act of changing a computation in an invalid vector type to
18 // be a computation in two vectors of half the size. For example, implementing
19 // <128 x f32> operations in terms of two <64 x f32> operations.
21 //===----------------------------------------------------------------------===//
23 #include "LegalizeTypes.h"
24 #include "llvm/IR/DataLayout.h"
25 #include "llvm/Support/ErrorHandling.h"
26 #include "llvm/Support/raw_ostream.h"
29 //===----------------------------------------------------------------------===//
30 // Result Vector Scalarization: <1 x ty> -> ty.
31 //===----------------------------------------------------------------------===//
33 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
34 DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
37 SDValue R = SDValue();
39 switch (N->getOpcode()) {
42 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
46 report_fatal_error("Do not know how to scalarize the result of this "
49 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
50 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
51 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
52 case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break;
53 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
54 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
55 case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break;
56 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
57 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
58 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
59 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
60 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
61 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
62 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
63 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
64 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
65 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
66 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
90 case ISD::SIGN_EXTEND:
94 case ISD::ZERO_EXTEND:
95 R = ScalarizeVecRes_UnaryOp(N);
118 R = ScalarizeVecRes_BinOp(N);
121 R = ScalarizeVecRes_TernaryOp(N);
125 // If R is null, the sub-method took care of registering the result.
127 SetScalarizedVector(SDValue(N, ResNo), R);
130 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
131 SDValue LHS = GetScalarizedVector(N->getOperand(0));
132 SDValue RHS = GetScalarizedVector(N->getOperand(1));
133 return DAG.getNode(N->getOpcode(), SDLoc(N),
134 LHS.getValueType(), LHS, RHS);
137 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
138 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
139 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
140 SDValue Op2 = GetScalarizedVector(N->getOperand(2));
141 return DAG.getNode(N->getOpcode(), SDLoc(N),
142 Op0.getValueType(), Op0, Op1, Op2);
145 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
147 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
148 return GetScalarizedVector(Op);
151 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
152 EVT NewVT = N->getValueType(0).getVectorElementType();
153 return DAG.getNode(ISD::BITCAST, SDLoc(N),
154 NewVT, N->getOperand(0));
157 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
158 EVT EltVT = N->getValueType(0).getVectorElementType();
159 SDValue InOp = N->getOperand(0);
160 // The BUILD_VECTOR operands may be of wider element types and
161 // we may need to truncate them back to the requested return type.
162 if (EltVT.isInteger())
163 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
167 SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) {
168 EVT NewVT = N->getValueType(0).getVectorElementType();
169 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
170 return DAG.getConvertRndSat(NewVT, SDLoc(N),
171 Op0, DAG.getValueType(NewVT),
172 DAG.getValueType(Op0.getValueType()),
175 cast<CvtRndSatSDNode>(N)->getCvtCode());
178 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
179 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
180 N->getValueType(0).getVectorElementType(),
181 N->getOperand(0), N->getOperand(1));
184 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
185 EVT NewVT = N->getValueType(0).getVectorElementType();
186 SDValue Op = GetScalarizedVector(N->getOperand(0));
187 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
188 NewVT, Op, N->getOperand(1));
191 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
192 SDValue Op = GetScalarizedVector(N->getOperand(0));
193 return DAG.getNode(ISD::FPOWI, SDLoc(N),
194 Op.getValueType(), Op, N->getOperand(1));
197 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
198 // The value to insert may have a wider type than the vector element type,
199 // so be sure to truncate it to the element type if necessary.
200 SDValue Op = N->getOperand(1);
201 EVT EltVT = N->getValueType(0).getVectorElementType();
202 if (Op.getValueType() != EltVT)
203 // FIXME: Can this happen for floating point types?
204 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
208 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
209 assert(N->isUnindexed() && "Indexed vector load?");
211 SDValue Result = DAG.getLoad(ISD::UNINDEXED,
212 N->getExtensionType(),
213 N->getValueType(0).getVectorElementType(),
215 N->getChain(), N->getBasePtr(),
216 DAG.getUNDEF(N->getBasePtr().getValueType()),
218 N->getMemoryVT().getVectorElementType(),
219 N->isVolatile(), N->isNonTemporal(),
220 N->isInvariant(), N->getOriginalAlignment());
222 // Legalized the chain result - switch anything that used the old chain to
224 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
228 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
229 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
230 EVT DestVT = N->getValueType(0).getVectorElementType();
231 SDValue Op = GetScalarizedVector(N->getOperand(0));
232 return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op);
235 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
236 EVT EltVT = N->getValueType(0).getVectorElementType();
237 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
238 SDValue LHS = GetScalarizedVector(N->getOperand(0));
239 return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
240 LHS, DAG.getValueType(ExtVT));
243 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
244 // If the operand is wider than the vector element type then it is implicitly
245 // truncated. Make that explicit here.
246 EVT EltVT = N->getValueType(0).getVectorElementType();
247 SDValue InOp = N->getOperand(0);
248 if (InOp.getValueType() != EltVT)
249 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
253 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
254 SDValue Cond = GetScalarizedVector(N->getOperand(0));
255 SDValue LHS = GetScalarizedVector(N->getOperand(1));
256 TargetLowering::BooleanContent ScalarBool = TLI.getBooleanContents(false);
257 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true);
258 if (ScalarBool != VecBool) {
259 EVT CondVT = Cond.getValueType();
260 switch (ScalarBool) {
261 case TargetLowering::UndefinedBooleanContent:
263 case TargetLowering::ZeroOrOneBooleanContent:
264 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
265 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
266 // Vector read from all ones, scalar expects a single 1 so mask.
267 Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
268 Cond, DAG.getConstant(1, CondVT));
270 case TargetLowering::ZeroOrNegativeOneBooleanContent:
271 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
272 VecBool == TargetLowering::ZeroOrOneBooleanContent);
273 // Vector reads from a one, scalar from all ones so sign extend.
274 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
275 Cond, DAG.getValueType(MVT::i1));
280 return DAG.getSelect(SDLoc(N),
281 LHS.getValueType(), Cond, LHS,
282 GetScalarizedVector(N->getOperand(2)));
285 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
286 SDValue LHS = GetScalarizedVector(N->getOperand(1));
287 return DAG.getSelect(SDLoc(N),
288 LHS.getValueType(), N->getOperand(0), LHS,
289 GetScalarizedVector(N->getOperand(2)));
292 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
293 SDValue LHS = GetScalarizedVector(N->getOperand(2));
294 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
295 N->getOperand(0), N->getOperand(1),
296 LHS, GetScalarizedVector(N->getOperand(3)),
300 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
301 assert(N->getValueType(0).isVector() ==
302 N->getOperand(0).getValueType().isVector() &&
303 "Scalar/Vector type mismatch");
305 if (N->getValueType(0).isVector()) return ScalarizeVecRes_VSETCC(N);
307 SDValue LHS = GetScalarizedVector(N->getOperand(0));
308 SDValue RHS = GetScalarizedVector(N->getOperand(1));
311 // Turn it into a scalar SETCC.
312 return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2));
315 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
316 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
319 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
320 // Figure out if the scalar is the LHS or RHS and return it.
321 SDValue Arg = N->getOperand(2).getOperand(0);
322 if (Arg.getOpcode() == ISD::UNDEF)
323 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
324 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
325 return GetScalarizedVector(N->getOperand(Op));
328 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) {
329 assert(N->getValueType(0).isVector() &&
330 N->getOperand(0).getValueType().isVector() &&
331 "Operand types must be vectors");
333 SDValue LHS = GetScalarizedVector(N->getOperand(0));
334 SDValue RHS = GetScalarizedVector(N->getOperand(1));
335 EVT NVT = N->getValueType(0).getVectorElementType();
338 // Turn it into a scalar SETCC.
339 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
341 // Vectors may have a different boolean contents to scalars. Promote the
342 // value appropriately.
343 ISD::NodeType ExtendCode =
344 TargetLowering::getExtendForContent(TLI.getBooleanContents(true));
345 return DAG.getNode(ExtendCode, DL, NVT, Res);
349 //===----------------------------------------------------------------------===//
350 // Operand Vector Scalarization <1 x ty> -> ty.
351 //===----------------------------------------------------------------------===//
353 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
354 DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
357 SDValue Res = SDValue();
359 if (Res.getNode() == 0) {
360 switch (N->getOpcode()) {
363 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
367 llvm_unreachable("Do not know how to scalarize this operator's operand!");
369 Res = ScalarizeVecOp_BITCAST(N);
371 case ISD::ANY_EXTEND:
372 case ISD::ZERO_EXTEND:
373 case ISD::SIGN_EXTEND:
374 Res = ScalarizeVecOp_EXTEND(N);
376 case ISD::CONCAT_VECTORS:
377 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
379 case ISD::EXTRACT_VECTOR_ELT:
380 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
383 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
388 // If the result is null, the sub-method took care of registering results etc.
389 if (!Res.getNode()) return false;
391 // If the result is N, the sub-method updated N in place. Tell the legalizer
393 if (Res.getNode() == N)
396 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
397 "Invalid operand expansion");
399 ReplaceValueWith(SDValue(N, 0), Res);
403 /// ScalarizeVecOp_BITCAST - If the value to convert is a vector that needs
404 /// to be scalarized, it must be <1 x ty>. Convert the element instead.
405 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
406 SDValue Elt = GetScalarizedVector(N->getOperand(0));
407 return DAG.getNode(ISD::BITCAST, SDLoc(N),
408 N->getValueType(0), Elt);
411 /// ScalarizeVecOp_EXTEND - If the value to extend is a vector that needs
412 /// to be scalarized, it must be <1 x ty>. Extend the element instead.
413 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTEND(SDNode *N) {
414 assert(N->getValueType(0).getVectorNumElements() == 1 &&
415 "Unexected vector type!");
416 SDValue Elt = GetScalarizedVector(N->getOperand(0));
417 SmallVector<SDValue, 1> Ops(1);
418 Ops[0] = DAG.getNode(N->getOpcode(), SDLoc(N),
419 N->getValueType(0).getScalarType(), Elt);
420 // Revectorize the result so the types line up with what the uses of this
421 // expression expect.
422 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), N->getValueType(0),
426 /// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one -
427 /// use a BUILD_VECTOR instead.
428 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
429 SmallVector<SDValue, 8> Ops(N->getNumOperands());
430 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
431 Ops[i] = GetScalarizedVector(N->getOperand(i));
432 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), N->getValueType(0),
433 &Ops[0], Ops.size());
436 /// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to
437 /// be scalarized, it must be <1 x ty>, so just return the element, ignoring the
439 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
440 SDValue Res = GetScalarizedVector(N->getOperand(0));
441 if (Res.getValueType() != N->getValueType(0))
442 Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0),
447 /// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be
448 /// scalarized, it must be <1 x ty>. Just store the element.
449 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
450 assert(N->isUnindexed() && "Indexed store of one-element vector?");
451 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
454 if (N->isTruncatingStore())
455 return DAG.getTruncStore(N->getChain(), dl,
456 GetScalarizedVector(N->getOperand(1)),
457 N->getBasePtr(), N->getPointerInfo(),
458 N->getMemoryVT().getVectorElementType(),
459 N->isVolatile(), N->isNonTemporal(),
462 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
463 N->getBasePtr(), N->getPointerInfo(),
464 N->isVolatile(), N->isNonTemporal(),
465 N->getOriginalAlignment());
469 //===----------------------------------------------------------------------===//
470 // Result Vector Splitting
471 //===----------------------------------------------------------------------===//
473 /// SplitVectorResult - This method is called when the specified result of the
474 /// specified node is found to need vector splitting. At this point, the node
475 /// may also have invalid operands or may have other results that need
476 /// legalization, we just know that (at least) one result needs vector
478 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
479 DEBUG(dbgs() << "Split node result: ";
484 // See if the target wants to custom expand this node.
485 if (CustomLowerNode(N, N->getValueType(ResNo), true))
488 switch (N->getOpcode()) {
491 dbgs() << "SplitVectorResult #" << ResNo << ": ";
495 report_fatal_error("Do not know how to split the result of this "
498 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
500 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
501 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
502 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
503 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
504 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
505 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
506 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
507 case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
508 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
509 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
510 case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break;
511 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
513 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
516 SplitVecRes_SETCC(N, Lo, Hi);
518 case ISD::VECTOR_SHUFFLE:
519 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
522 case ISD::ANY_EXTEND:
523 case ISD::CONVERT_RNDSAT:
526 case ISD::CTLZ_ZERO_UNDEF:
527 case ISD::CTTZ_ZERO_UNDEF:
538 case ISD::FNEARBYINT:
542 case ISD::FP_TO_SINT:
543 case ISD::FP_TO_UINT:
549 case ISD::SIGN_EXTEND:
550 case ISD::SINT_TO_FP:
552 case ISD::UINT_TO_FP:
553 case ISD::ZERO_EXTEND:
554 SplitVecRes_UnaryOp(N, Lo, Hi);
577 SplitVecRes_BinOp(N, Lo, Hi);
580 SplitVecRes_TernaryOp(N, Lo, Hi);
584 // If Lo/Hi is null, the sub-method took care of registering results etc.
586 SetSplitVector(SDValue(N, ResNo), Lo, Hi);
589 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
591 SDValue LHSLo, LHSHi;
592 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
593 SDValue RHSLo, RHSHi;
594 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
597 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, RHSLo);
598 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, RHSHi);
601 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
603 SDValue Op0Lo, Op0Hi;
604 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
605 SDValue Op1Lo, Op1Hi;
606 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
607 SDValue Op2Lo, Op2Hi;
608 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
611 Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(),
612 Op0Lo, Op1Lo, Op2Lo);
613 Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(),
614 Op0Hi, Op1Hi, Op2Hi);
617 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
619 // We know the result is a vector. The input may be either a vector or a
622 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
625 SDValue InOp = N->getOperand(0);
626 EVT InVT = InOp.getValueType();
628 // Handle some special cases efficiently.
629 switch (getTypeAction(InVT)) {
630 case TargetLowering::TypeLegal:
631 case TargetLowering::TypePromoteInteger:
632 case TargetLowering::TypeSoftenFloat:
633 case TargetLowering::TypeScalarizeVector:
634 case TargetLowering::TypeWidenVector:
636 case TargetLowering::TypeExpandInteger:
637 case TargetLowering::TypeExpandFloat:
638 // A scalar to vector conversion, where the scalar needs expansion.
639 // If the vector is being split in two then we can just convert the
642 GetExpandedOp(InOp, Lo, Hi);
643 if (TLI.isBigEndian())
645 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
646 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
650 case TargetLowering::TypeSplitVector:
651 // If the input is a vector that needs to be split, convert each split
652 // piece of the input now.
653 GetSplitVector(InOp, Lo, Hi);
654 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
655 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
659 // In the general case, convert the input to an integer and split it by hand.
660 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
661 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
662 if (TLI.isBigEndian())
663 std::swap(LoIntVT, HiIntVT);
665 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
667 if (TLI.isBigEndian())
669 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
670 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
673 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
677 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
678 unsigned LoNumElts = LoVT.getVectorNumElements();
679 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
680 Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, &LoOps[0], LoOps.size());
682 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
683 Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, &HiOps[0], HiOps.size());
686 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
688 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
690 unsigned NumSubvectors = N->getNumOperands() / 2;
691 if (NumSubvectors == 1) {
692 Lo = N->getOperand(0);
693 Hi = N->getOperand(1);
698 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
700 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
701 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, &LoOps[0], LoOps.size());
703 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
704 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, &HiOps[0], HiOps.size());
707 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
709 SDValue Vec = N->getOperand(0);
710 SDValue Idx = N->getOperand(1);
714 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
716 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
717 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
718 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
719 DAG.getConstant(IdxVal + LoVT.getVectorNumElements(),
720 TLI.getVectorIdxTy()));
723 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
726 GetSplitVector(N->getOperand(0), Lo, Hi);
727 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
728 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
731 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
733 SDValue LHSLo, LHSHi;
734 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
738 GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT(), LoVT, HiVT);
740 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
741 DAG.getValueType(LoVT));
742 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
743 DAG.getValueType(HiVT));
746 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
748 SDValue Vec = N->getOperand(0);
749 SDValue Elt = N->getOperand(1);
750 SDValue Idx = N->getOperand(2);
752 GetSplitVector(Vec, Lo, Hi);
754 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
755 unsigned IdxVal = CIdx->getZExtValue();
756 unsigned LoNumElts = Lo.getValueType().getVectorNumElements();
757 if (IdxVal < LoNumElts)
758 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
759 Lo.getValueType(), Lo, Elt, Idx);
761 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
762 DAG.getConstant(IdxVal - LoNumElts,
763 TLI.getVectorIdxTy()));
767 // Spill the vector to the stack.
768 EVT VecVT = Vec.getValueType();
769 EVT EltVT = VecVT.getVectorElementType();
770 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
771 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
772 MachinePointerInfo(), false, false, 0);
774 // Store the new element. This may be larger than the vector element type,
775 // so use a truncating store.
776 SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx);
777 Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
779 TLI.getDataLayout()->getPrefTypeAlignment(VecType);
780 Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, MachinePointerInfo(), EltVT,
783 // Load the Lo part from the stack slot.
784 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
785 false, false, false, 0);
787 // Increment the pointer to the other part.
788 unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8;
789 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
790 DAG.getIntPtrConstant(IncrementSize));
792 // Load the Hi part from the stack slot.
793 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
794 false, false, false, MinAlign(Alignment, IncrementSize));
797 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
801 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
802 Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0));
803 Hi = DAG.getUNDEF(HiVT);
806 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
808 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
811 GetSplitDestVTs(LD->getValueType(0), LoVT, HiVT);
813 ISD::LoadExtType ExtType = LD->getExtensionType();
814 SDValue Ch = LD->getChain();
815 SDValue Ptr = LD->getBasePtr();
816 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
817 EVT MemoryVT = LD->getMemoryVT();
818 unsigned Alignment = LD->getOriginalAlignment();
819 bool isVolatile = LD->isVolatile();
820 bool isNonTemporal = LD->isNonTemporal();
821 bool isInvariant = LD->isInvariant();
823 EVT LoMemVT, HiMemVT;
824 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT);
826 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
827 LD->getPointerInfo(), LoMemVT, isVolatile, isNonTemporal,
828 isInvariant, Alignment);
830 unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
831 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
832 DAG.getIntPtrConstant(IncrementSize));
833 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset,
834 LD->getPointerInfo().getWithOffset(IncrementSize),
835 HiMemVT, isVolatile, isNonTemporal, isInvariant, Alignment);
837 // Build a factor node to remember that this load is independent of the
839 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
842 // Legalized the chain result - switch anything that used the old chain to
844 ReplaceValueWith(SDValue(LD, 1), Ch);
847 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
848 assert(N->getValueType(0).isVector() &&
849 N->getOperand(0).getValueType().isVector() &&
850 "Operand types must be vectors");
854 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
857 EVT InVT = N->getOperand(0).getValueType();
858 SDValue LL, LH, RL, RH;
859 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(),
860 LoVT.getVectorNumElements());
861 LL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0),
862 DAG.getConstant(0, TLI.getVectorIdxTy()));
863 LH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0),
864 DAG.getConstant(InNVT.getVectorNumElements(),
865 TLI.getVectorIdxTy()));
867 RL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1),
868 DAG.getConstant(0, TLI.getVectorIdxTy()));
869 RH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1),
870 DAG.getConstant(InNVT.getVectorNumElements(),
871 TLI.getVectorIdxTy()));
873 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
874 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
877 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
879 // Get the dest types - they may not match the input types, e.g. int_to_fp.
882 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
884 // If the input also splits, handle it directly for a compile time speedup.
885 // Otherwise split it by hand.
886 EVT InVT = N->getOperand(0).getValueType();
887 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
888 GetSplitVector(N->getOperand(0), Lo, Hi);
890 EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(),
891 LoVT.getVectorNumElements());
892 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0),
893 DAG.getConstant(0, TLI.getVectorIdxTy()));
894 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0),
895 DAG.getConstant(InNVT.getVectorNumElements(),
896 TLI.getVectorIdxTy()));
899 if (N->getOpcode() == ISD::FP_ROUND) {
900 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1));
901 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1));
902 } else if (N->getOpcode() == ISD::CONVERT_RNDSAT) {
903 SDValue DTyOpLo = DAG.getValueType(LoVT);
904 SDValue DTyOpHi = DAG.getValueType(HiVT);
905 SDValue STyOpLo = DAG.getValueType(Lo.getValueType());
906 SDValue STyOpHi = DAG.getValueType(Hi.getValueType());
907 SDValue RndOp = N->getOperand(3);
908 SDValue SatOp = N->getOperand(4);
909 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode();
910 Lo = DAG.getConvertRndSat(LoVT, dl, Lo, DTyOpLo, STyOpLo, RndOp, SatOp,
912 Hi = DAG.getConvertRndSat(HiVT, dl, Hi, DTyOpHi, STyOpHi, RndOp, SatOp,
915 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
916 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
920 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
921 SDValue &Lo, SDValue &Hi) {
922 // The low and high parts of the original input give four input vectors.
925 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
926 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
927 EVT NewVT = Inputs[0].getValueType();
928 unsigned NewElts = NewVT.getVectorNumElements();
930 // If Lo or Hi uses elements from at most two of the four input vectors, then
931 // express it as a vector shuffle of those two inputs. Otherwise extract the
932 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
933 SmallVector<int, 16> Ops;
934 for (unsigned High = 0; High < 2; ++High) {
935 SDValue &Output = High ? Hi : Lo;
937 // Build a shuffle mask for the output, discovering on the fly which
938 // input vectors to use as shuffle operands (recorded in InputUsed).
939 // If building a suitable shuffle vector proves too hard, then bail
940 // out with useBuildVector set.
941 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
942 unsigned FirstMaskIdx = High * NewElts;
943 bool useBuildVector = false;
944 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
945 // The mask element. This indexes into the input.
946 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
948 // The input vector this mask element indexes into.
949 unsigned Input = (unsigned)Idx / NewElts;
951 if (Input >= array_lengthof(Inputs)) {
952 // The mask element does not index into any input vector.
957 // Turn the index into an offset from the start of the input vector.
958 Idx -= Input * NewElts;
960 // Find or create a shuffle vector operand to hold this input.
962 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
963 if (InputUsed[OpNo] == Input) {
964 // This input vector is already an operand.
966 } else if (InputUsed[OpNo] == -1U) {
967 // Create a new operand for this input vector.
968 InputUsed[OpNo] = Input;
973 if (OpNo >= array_lengthof(InputUsed)) {
974 // More than two input vectors used! Give up on trying to create a
975 // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
976 useBuildVector = true;
980 // Add the mask index for the new shuffle vector.
981 Ops.push_back(Idx + OpNo * NewElts);
984 if (useBuildVector) {
985 EVT EltVT = NewVT.getVectorElementType();
986 SmallVector<SDValue, 16> SVOps;
988 // Extract the input elements by hand.
989 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
990 // The mask element. This indexes into the input.
991 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
993 // The input vector this mask element indexes into.
994 unsigned Input = (unsigned)Idx / NewElts;
996 if (Input >= array_lengthof(Inputs)) {
997 // The mask element is "undef" or indexes off the end of the input.
998 SVOps.push_back(DAG.getUNDEF(EltVT));
1002 // Turn the index into an offset from the start of the input vector.
1003 Idx -= Input * NewElts;
1005 // Extract the vector element by hand.
1006 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
1007 Inputs[Input], DAG.getConstant(Idx,
1008 TLI.getVectorIdxTy())));
1011 // Construct the Lo/Hi output using a BUILD_VECTOR.
1012 Output = DAG.getNode(ISD::BUILD_VECTOR,dl,NewVT, &SVOps[0], SVOps.size());
1013 } else if (InputUsed[0] == -1U) {
1014 // No input vectors were used! The result is undefined.
1015 Output = DAG.getUNDEF(NewVT);
1017 SDValue Op0 = Inputs[InputUsed[0]];
1018 // If only one input was used, use an undefined vector for the other.
1019 SDValue Op1 = InputUsed[1] == -1U ?
1020 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
1021 // At least one input vector was used. Create a new shuffle vector.
1022 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]);
1030 //===----------------------------------------------------------------------===//
1031 // Operand Vector Splitting
1032 //===----------------------------------------------------------------------===//
1034 /// SplitVectorOperand - This method is called when the specified operand of the
1035 /// specified node is found to need vector splitting. At this point, all of the
1036 /// result types of the node are known to be legal, but other operands of the
1037 /// node may need legalization as well as the specified one.
1038 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
1039 DEBUG(dbgs() << "Split node operand: ";
1042 SDValue Res = SDValue();
1044 // See if the target wants to custom split this node.
1045 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1048 if (Res.getNode() == 0) {
1049 switch (N->getOpcode()) {
1052 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
1056 report_fatal_error("Do not know how to split this operator's "
1059 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
1060 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
1061 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
1062 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
1063 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
1064 case ISD::TRUNCATE: Res = SplitVecOp_TRUNCATE(N); break;
1065 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
1067 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
1070 Res = SplitVecOp_VSELECT(N, OpNo);
1075 case ISD::FP_EXTEND:
1076 case ISD::FP_TO_SINT:
1077 case ISD::FP_TO_UINT:
1078 case ISD::SINT_TO_FP:
1079 case ISD::UINT_TO_FP:
1081 case ISD::SIGN_EXTEND:
1082 case ISD::ZERO_EXTEND:
1083 case ISD::ANY_EXTEND:
1084 Res = SplitVecOp_UnaryOp(N);
1089 // If the result is null, the sub-method took care of registering results etc.
1090 if (!Res.getNode()) return false;
1092 // If the result is N, the sub-method updated N in place. Tell the legalizer
1094 if (Res.getNode() == N)
1097 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1098 "Invalid operand expansion");
1100 ReplaceValueWith(SDValue(N, 0), Res);
1104 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
1105 // The only possibility for an illegal operand is the mask, since result type
1106 // legalization would have handled this node already otherwise.
1107 assert(OpNo == 0 && "Illegal operand must be mask");
1109 SDValue Mask = N->getOperand(0);
1110 SDValue Src0 = N->getOperand(1);
1111 SDValue Src1 = N->getOperand(2);
1113 EVT MaskVT = Mask.getValueType();
1114 assert(MaskVT.isVector() && "VSELECT without a vector mask?");
1117 GetSplitVector(N->getOperand(0), Lo, Hi);
1118 assert(Lo.getValueType() == Hi.getValueType() &&
1119 "Lo and Hi have differing types");
1121 unsigned LoNumElts = Lo.getValueType().getVectorNumElements();
1122 unsigned HiNumElts = Hi.getValueType().getVectorNumElements();
1123 assert(LoNumElts == HiNumElts && "Asymmetric vector split?");
1125 LLVMContext &Ctx = *DAG.getContext();
1126 SDValue Zero = DAG.getConstant(0, TLI.getVectorIdxTy());
1127 SDValue LoElts = DAG.getConstant(LoNumElts, TLI.getVectorIdxTy());
1128 EVT Src0VT = Src0.getValueType();
1129 EVT Src0EltTy = Src0VT.getVectorElementType();
1130 EVT MaskEltTy = MaskVT.getVectorElementType();
1132 EVT LoOpVT = EVT::getVectorVT(Ctx, Src0EltTy, LoNumElts);
1133 EVT LoMaskVT = EVT::getVectorVT(Ctx, MaskEltTy, LoNumElts);
1134 EVT HiOpVT = EVT::getVectorVT(Ctx, Src0EltTy, HiNumElts);
1135 EVT HiMaskVT = EVT::getVectorVT(Ctx, MaskEltTy, HiNumElts);
1137 SDValue LoOp0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoOpVT, Src0, Zero);
1138 SDValue LoOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoOpVT, Src1, Zero);
1140 SDValue HiOp0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiOpVT, Src0, LoElts);
1141 SDValue HiOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiOpVT, Src1, LoElts);
1144 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoMaskVT, Mask, Zero);
1146 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiMaskVT, Mask, LoElts);
1149 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
1151 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
1153 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
1156 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
1157 // The result has a legal vector type, but the input needs splitting.
1158 EVT ResVT = N->getValueType(0);
1161 GetSplitVector(N->getOperand(0), Lo, Hi);
1162 EVT InVT = Lo.getValueType();
1164 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
1165 InVT.getVectorNumElements());
1167 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
1168 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
1170 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
1173 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
1174 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
1175 // end up being split all the way down to individual components. Convert the
1176 // split pieces into integers and reassemble.
1178 GetSplitVector(N->getOperand(0), Lo, Hi);
1179 Lo = BitConvertToInteger(Lo);
1180 Hi = BitConvertToInteger(Hi);
1182 if (TLI.isBigEndian())
1185 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
1186 JoinIntegers(Lo, Hi));
1189 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
1190 // We know that the extracted result type is legal.
1191 EVT SubVT = N->getValueType(0);
1192 SDValue Idx = N->getOperand(1);
1195 GetSplitVector(N->getOperand(0), Lo, Hi);
1197 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1198 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1200 if (IdxVal < LoElts) {
1201 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts &&
1202 "Extracted subvector crosses vector split!");
1203 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
1205 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
1206 DAG.getConstant(IdxVal - LoElts, Idx.getValueType()));
1210 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
1211 SDValue Vec = N->getOperand(0);
1212 SDValue Idx = N->getOperand(1);
1213 EVT VecVT = Vec.getValueType();
1215 if (isa<ConstantSDNode>(Idx)) {
1216 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1217 assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!");
1220 GetSplitVector(Vec, Lo, Hi);
1222 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1224 if (IdxVal < LoElts)
1225 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
1226 return SDValue(DAG.UpdateNodeOperands(N, Hi,
1227 DAG.getConstant(IdxVal - LoElts,
1228 Idx.getValueType())), 0);
1231 // Store the vector to the stack.
1232 EVT EltVT = VecVT.getVectorElementType();
1234 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1235 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1236 MachinePointerInfo(), false, false, 0);
1238 // Load back the required element.
1239 StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx);
1240 return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
1241 MachinePointerInfo(), EltVT, false, false, 0);
1244 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
1245 assert(N->isUnindexed() && "Indexed store of vector?");
1246 assert(OpNo == 1 && "Can only split the stored value");
1249 bool isTruncating = N->isTruncatingStore();
1250 SDValue Ch = N->getChain();
1251 SDValue Ptr = N->getBasePtr();
1252 EVT MemoryVT = N->getMemoryVT();
1253 unsigned Alignment = N->getOriginalAlignment();
1254 bool isVol = N->isVolatile();
1255 bool isNT = N->isNonTemporal();
1257 GetSplitVector(N->getOperand(1), Lo, Hi);
1259 EVT LoMemVT, HiMemVT;
1260 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT);
1262 unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
1265 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(),
1266 LoMemVT, isVol, isNT, Alignment);
1268 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(),
1269 isVol, isNT, Alignment);
1271 // Increment the pointer to the other half.
1272 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
1273 DAG.getConstant(IncrementSize, Ptr.getValueType()));
1276 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr,
1277 N->getPointerInfo().getWithOffset(IncrementSize),
1278 HiMemVT, isVol, isNT, Alignment);
1280 Hi = DAG.getStore(Ch, DL, Hi, Ptr,
1281 N->getPointerInfo().getWithOffset(IncrementSize),
1282 isVol, isNT, Alignment);
1284 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
1287 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
1290 // The input operands all must have the same type, and we know the result
1291 // type is valid. Convert this to a buildvector which extracts all the
1293 // TODO: If the input elements are power-two vectors, we could convert this to
1294 // a new CONCAT_VECTORS node with elements that are half-wide.
1295 SmallVector<SDValue, 32> Elts;
1296 EVT EltVT = N->getValueType(0).getVectorElementType();
1297 for (unsigned op = 0, e = N->getNumOperands(); op != e; ++op) {
1298 SDValue Op = N->getOperand(op);
1299 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
1301 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT,
1302 Op, DAG.getConstant(i, TLI.getVectorIdxTy())));
1307 return DAG.getNode(ISD::BUILD_VECTOR, DL, N->getValueType(0),
1308 &Elts[0], Elts.size());
1311 SDValue DAGTypeLegalizer::SplitVecOp_TRUNCATE(SDNode *N) {
1312 // The result type is legal, but the input type is illegal. If splitting
1313 // ends up with the result type of each half still being legal, just
1314 // do that. If, however, that would result in an illegal result type,
1315 // we can try to get more clever with power-two vectors. Specifically,
1316 // split the input type, but also widen the result element size, then
1317 // concatenate the halves and truncate again. For example, consider a target
1318 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
1319 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
1320 // %inlo = v4i32 extract_subvector %in, 0
1321 // %inhi = v4i32 extract_subvector %in, 4
1322 // %lo16 = v4i16 trunc v4i32 %inlo
1323 // %hi16 = v4i16 trunc v4i32 %inhi
1324 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
1325 // %res = v8i8 trunc v8i16 %in16
1327 // Without this transform, the original truncate would end up being
1328 // scalarized, which is pretty much always a last resort.
1329 SDValue InVec = N->getOperand(0);
1330 EVT InVT = InVec->getValueType(0);
1331 EVT OutVT = N->getValueType(0);
1332 unsigned NumElements = OutVT.getVectorNumElements();
1333 // Widening should have already made sure this is a power-two vector
1334 // if we're trying to split it at all. assert() that's true, just in case.
1335 assert(!(NumElements & 1) && "Splitting vector, but not in half!");
1337 unsigned InElementSize = InVT.getVectorElementType().getSizeInBits();
1338 unsigned OutElementSize = OutVT.getVectorElementType().getSizeInBits();
1340 // If the input elements are only 1/2 the width of the result elements,
1341 // just use the normal splitting. Our trick only work if there's room
1342 // to split more than once.
1343 if (InElementSize <= OutElementSize * 2)
1344 return SplitVecOp_UnaryOp(N);
1347 // Extract the halves of the input via extract_subvector.
1348 EVT SplitVT = EVT::getVectorVT(*DAG.getContext(),
1349 InVT.getVectorElementType(), NumElements/2);
1350 SDValue InLoVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, InVec,
1351 DAG.getConstant(0, TLI.getVectorIdxTy()));
1352 SDValue InHiVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, InVec,
1353 DAG.getConstant(NumElements/2,
1354 TLI.getVectorIdxTy()));
1355 // Truncate them to 1/2 the element size.
1356 EVT HalfElementVT = EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
1357 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
1359 SDValue HalfLo = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, InLoVec);
1360 SDValue HalfHi = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, InHiVec);
1361 // Concatenate them to get the full intermediate truncation result.
1362 EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
1363 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
1365 // Now finish up by truncating all the way down to the original result
1366 // type. This should normally be something that ends up being legal directly,
1367 // but in theory if a target has very wide vectors and an annoyingly
1368 // restricted set of legal types, this split can chain to build things up.
1369 return DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
1372 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
1373 assert(N->getValueType(0).isVector() &&
1374 N->getOperand(0).getValueType().isVector() &&
1375 "Operand types must be vectors");
1376 // The result has a legal vector type, but the input needs splitting.
1377 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
1379 GetSplitVector(N->getOperand(0), Lo0, Hi0);
1380 GetSplitVector(N->getOperand(1), Lo1, Hi1);
1381 unsigned PartElements = Lo0.getValueType().getVectorNumElements();
1382 EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements);
1383 EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements);
1385 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
1386 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
1387 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
1388 return PromoteTargetBoolean(Con, N->getValueType(0));
1392 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
1393 // The result has a legal vector type, but the input needs splitting.
1394 EVT ResVT = N->getValueType(0);
1397 GetSplitVector(N->getOperand(0), Lo, Hi);
1398 EVT InVT = Lo.getValueType();
1400 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
1401 InVT.getVectorNumElements());
1403 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
1404 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
1406 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
1411 //===----------------------------------------------------------------------===//
1412 // Result Vector Widening
1413 //===----------------------------------------------------------------------===//
1415 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
1416 DEBUG(dbgs() << "Widen node result " << ResNo << ": ";
1420 // See if the target wants to custom widen this node.
1421 if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
1424 SDValue Res = SDValue();
1425 switch (N->getOpcode()) {
1428 dbgs() << "WidenVectorResult #" << ResNo << ": ";
1432 llvm_unreachable("Do not know how to widen the result of this operator!");
1434 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
1435 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
1436 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
1437 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
1438 case ISD::CONVERT_RNDSAT: Res = WidenVecRes_CONVERT_RNDSAT(N); break;
1439 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
1440 case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break;
1441 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
1442 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
1443 case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break;
1444 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
1446 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break;
1447 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
1448 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
1449 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
1450 case ISD::VECTOR_SHUFFLE:
1451 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
1463 Res = WidenVecRes_Binary(N);
1467 case ISD::FCOPYSIGN:
1477 Res = WidenVecRes_BinaryCanTrap(N);
1481 Res = WidenVecRes_POWI(N);
1487 Res = WidenVecRes_Shift(N);
1490 case ISD::ANY_EXTEND:
1491 case ISD::FP_EXTEND:
1493 case ISD::FP_TO_SINT:
1494 case ISD::FP_TO_UINT:
1495 case ISD::SIGN_EXTEND:
1496 case ISD::SINT_TO_FP:
1498 case ISD::UINT_TO_FP:
1499 case ISD::ZERO_EXTEND:
1500 Res = WidenVecRes_Convert(N);
1515 case ISD::FNEARBYINT:
1522 Res = WidenVecRes_Unary(N);
1525 Res = WidenVecRes_Ternary(N);
1529 // If Res is null, the sub-method took care of registering the result.
1531 SetWidenedVector(SDValue(N, ResNo), Res);
1534 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
1535 // Ternary op widening.
1537 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1538 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
1539 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
1540 SDValue InOp3 = GetWidenedVector(N->getOperand(2));
1541 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
1544 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
1545 // Binary op widening.
1547 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1548 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
1549 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
1550 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2);
1553 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
1554 // Binary op widening for operations that can trap.
1555 unsigned Opcode = N->getOpcode();
1557 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1558 EVT WidenEltVT = WidenVT.getVectorElementType();
1560 unsigned NumElts = VT.getVectorNumElements();
1561 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
1562 NumElts = NumElts / 2;
1563 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
1566 if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
1567 // Operation doesn't trap so just widen as normal.
1568 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
1569 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
1570 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2);
1573 // No legal vector version so unroll the vector operation and then widen.
1575 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
1577 // Since the operation can trap, apply operation on the original vector.
1579 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
1580 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
1581 unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
1583 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
1584 unsigned ConcatEnd = 0; // Current ConcatOps index.
1585 int Idx = 0; // Current Idx into input vectors.
1587 // NumElts := greatest legal vector size (at most WidenVT)
1588 // while (orig. vector has unhandled elements) {
1589 // take munches of size NumElts from the beginning and add to ConcatOps
1590 // NumElts := next smaller supported vector size or 1
1592 while (CurNumElts != 0) {
1593 while (CurNumElts >= NumElts) {
1594 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
1595 DAG.getConstant(Idx, TLI.getVectorIdxTy()));
1596 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
1597 DAG.getConstant(Idx, TLI.getVectorIdxTy()));
1598 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2);
1600 CurNumElts -= NumElts;
1603 NumElts = NumElts / 2;
1604 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
1605 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
1608 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
1609 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
1610 InOp1, DAG.getConstant(Idx,
1611 TLI.getVectorIdxTy()));
1612 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
1613 InOp2, DAG.getConstant(Idx,
1614 TLI.getVectorIdxTy()));
1615 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
1622 // Check to see if we have a single operation with the widen type.
1623 if (ConcatEnd == 1) {
1624 VT = ConcatOps[0].getValueType();
1626 return ConcatOps[0];
1629 // while (Some element of ConcatOps is not of type MaxVT) {
1630 // From the end of ConcatOps, collect elements of the same type and put
1631 // them into an op of the next larger supported type
1633 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
1634 Idx = ConcatEnd - 1;
1635 VT = ConcatOps[Idx--].getValueType();
1636 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
1639 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
1643 NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
1644 } while (!TLI.isTypeLegal(NextVT));
1646 if (!VT.isVector()) {
1647 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
1648 SDValue VecOp = DAG.getUNDEF(NextVT);
1649 unsigned NumToInsert = ConcatEnd - Idx - 1;
1650 for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
1651 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp,
1652 ConcatOps[OpIdx], DAG.getConstant(i,
1653 TLI.getVectorIdxTy()));
1655 ConcatOps[Idx+1] = VecOp;
1656 ConcatEnd = Idx + 2;
1658 // Vector type, create a CONCAT_VECTORS of type NextVT
1659 SDValue undefVec = DAG.getUNDEF(VT);
1660 unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
1661 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
1662 unsigned RealVals = ConcatEnd - Idx - 1;
1663 unsigned SubConcatEnd = 0;
1664 unsigned SubConcatIdx = Idx + 1;
1665 while (SubConcatEnd < RealVals)
1666 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
1667 while (SubConcatEnd < OpsToConcat)
1668 SubConcatOps[SubConcatEnd++] = undefVec;
1669 ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
1670 NextVT, &SubConcatOps[0],
1672 ConcatEnd = SubConcatIdx + 1;
1676 // Check to see if we have a single operation with the widen type.
1677 if (ConcatEnd == 1) {
1678 VT = ConcatOps[0].getValueType();
1680 return ConcatOps[0];
1683 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
1684 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
1685 if (NumOps != ConcatEnd ) {
1686 SDValue UndefVal = DAG.getUNDEF(MaxVT);
1687 for (unsigned j = ConcatEnd; j < NumOps; ++j)
1688 ConcatOps[j] = UndefVal;
1690 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &ConcatOps[0], NumOps);
1693 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
1694 SDValue InOp = N->getOperand(0);
1697 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1698 unsigned WidenNumElts = WidenVT.getVectorNumElements();
1700 EVT InVT = InOp.getValueType();
1701 EVT InEltVT = InVT.getVectorElementType();
1702 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts);
1704 unsigned Opcode = N->getOpcode();
1705 unsigned InVTNumElts = InVT.getVectorNumElements();
1707 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
1708 InOp = GetWidenedVector(N->getOperand(0));
1709 InVT = InOp.getValueType();
1710 InVTNumElts = InVT.getVectorNumElements();
1711 if (InVTNumElts == WidenNumElts) {
1712 if (N->getNumOperands() == 1)
1713 return DAG.getNode(Opcode, DL, WidenVT, InOp);
1714 return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1));
1718 if (TLI.isTypeLegal(InWidenVT)) {
1719 // Because the result and the input are different vector types, widening
1720 // the result could create a legal type but widening the input might make
1721 // it an illegal type that might lead to repeatedly splitting the input
1722 // and then widening it. To avoid this, we widen the input only if
1723 // it results in a legal type.
1724 if (WidenNumElts % InVTNumElts == 0) {
1725 // Widen the input and call convert on the widened input vector.
1726 unsigned NumConcat = WidenNumElts/InVTNumElts;
1727 SmallVector<SDValue, 16> Ops(NumConcat);
1729 SDValue UndefVal = DAG.getUNDEF(InVT);
1730 for (unsigned i = 1; i != NumConcat; ++i)
1732 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT,
1733 &Ops[0], NumConcat);
1734 if (N->getNumOperands() == 1)
1735 return DAG.getNode(Opcode, DL, WidenVT, InVec);
1736 return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1));
1739 if (InVTNumElts % WidenNumElts == 0) {
1740 SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT,
1741 InOp, DAG.getConstant(0,
1742 TLI.getVectorIdxTy()));
1743 // Extract the input and convert the shorten input vector.
1744 if (N->getNumOperands() == 1)
1745 return DAG.getNode(Opcode, DL, WidenVT, InVal);
1746 return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1));
1750 // Otherwise unroll into some nasty scalar code and rebuild the vector.
1751 SmallVector<SDValue, 16> Ops(WidenNumElts);
1752 EVT EltVT = WidenVT.getVectorElementType();
1753 unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
1755 for (i=0; i < MinElts; ++i) {
1756 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
1757 DAG.getConstant(i, TLI.getVectorIdxTy()));
1758 if (N->getNumOperands() == 1)
1759 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
1761 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1));
1764 SDValue UndefVal = DAG.getUNDEF(EltVT);
1765 for (; i < WidenNumElts; ++i)
1768 return DAG.getNode(ISD::BUILD_VECTOR, DL, WidenVT, &Ops[0], WidenNumElts);
1771 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
1772 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1773 SDValue InOp = GetWidenedVector(N->getOperand(0));
1774 SDValue ShOp = N->getOperand(1);
1775 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
1778 SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
1779 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1780 SDValue InOp = GetWidenedVector(N->getOperand(0));
1781 SDValue ShOp = N->getOperand(1);
1783 EVT ShVT = ShOp.getValueType();
1784 if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) {
1785 ShOp = GetWidenedVector(ShOp);
1786 ShVT = ShOp.getValueType();
1788 EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(),
1789 ShVT.getVectorElementType(),
1790 WidenVT.getVectorNumElements());
1791 if (ShVT != ShWidenVT)
1792 ShOp = ModifyToType(ShOp, ShWidenVT);
1794 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
1797 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
1798 // Unary op widening.
1799 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1800 SDValue InOp = GetWidenedVector(N->getOperand(0));
1801 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
1804 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
1805 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1806 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
1807 cast<VTSDNode>(N->getOperand(1))->getVT()
1808 .getVectorElementType(),
1809 WidenVT.getVectorNumElements());
1810 SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
1811 return DAG.getNode(N->getOpcode(), SDLoc(N),
1812 WidenVT, WidenLHS, DAG.getValueType(ExtVT));
1815 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
1816 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
1817 return GetWidenedVector(WidenVec);
1820 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
1821 SDValue InOp = N->getOperand(0);
1822 EVT InVT = InOp.getValueType();
1823 EVT VT = N->getValueType(0);
1824 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1827 switch (getTypeAction(InVT)) {
1828 case TargetLowering::TypeLegal:
1830 case TargetLowering::TypePromoteInteger:
1831 // If the incoming type is a vector that is being promoted, then
1832 // we know that the elements are arranged differently and that we
1833 // must perform the conversion using a stack slot.
1834 if (InVT.isVector())
1837 // If the InOp is promoted to the same size, convert it. Otherwise,
1838 // fall out of the switch and widen the promoted input.
1839 InOp = GetPromotedInteger(InOp);
1840 InVT = InOp.getValueType();
1841 if (WidenVT.bitsEq(InVT))
1842 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
1844 case TargetLowering::TypeSoftenFloat:
1845 case TargetLowering::TypeExpandInteger:
1846 case TargetLowering::TypeExpandFloat:
1847 case TargetLowering::TypeScalarizeVector:
1848 case TargetLowering::TypeSplitVector:
1850 case TargetLowering::TypeWidenVector:
1851 // If the InOp is widened to the same size, convert it. Otherwise, fall
1852 // out of the switch and widen the widened input.
1853 InOp = GetWidenedVector(InOp);
1854 InVT = InOp.getValueType();
1855 if (WidenVT.bitsEq(InVT))
1856 // The input widens to the same size. Convert to the widen value.
1857 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
1861 unsigned WidenSize = WidenVT.getSizeInBits();
1862 unsigned InSize = InVT.getSizeInBits();
1863 // x86mmx is not an acceptable vector element type, so don't try.
1864 if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
1865 // Determine new input vector type. The new input vector type will use
1866 // the same element type (if its a vector) or use the input type as a
1867 // vector. It is the same size as the type to widen to.
1869 unsigned NewNumElts = WidenSize / InSize;
1870 if (InVT.isVector()) {
1871 EVT InEltVT = InVT.getVectorElementType();
1872 NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
1873 WidenSize / InEltVT.getSizeInBits());
1875 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
1878 if (TLI.isTypeLegal(NewInVT)) {
1879 // Because the result and the input are different vector types, widening
1880 // the result could create a legal type but widening the input might make
1881 // it an illegal type that might lead to repeatedly splitting the input
1882 // and then widening it. To avoid this, we widen the input only if
1883 // it results in a legal type.
1884 SmallVector<SDValue, 16> Ops(NewNumElts);
1885 SDValue UndefVal = DAG.getUNDEF(InVT);
1887 for (unsigned i = 1; i < NewNumElts; ++i)
1891 if (InVT.isVector())
1892 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl,
1893 NewInVT, &Ops[0], NewNumElts);
1895 NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl,
1896 NewInVT, &Ops[0], NewNumElts);
1897 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
1901 return CreateStackStoreLoad(InOp, WidenVT);
1904 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
1906 // Build a vector with undefined for the new nodes.
1907 EVT VT = N->getValueType(0);
1909 // Integer BUILD_VECTOR operands may be larger than the node's vector element
1910 // type. The UNDEFs need to have the same type as the existing operands.
1911 EVT EltVT = N->getOperand(0).getValueType();
1912 unsigned NumElts = VT.getVectorNumElements();
1914 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1915 unsigned WidenNumElts = WidenVT.getVectorNumElements();
1917 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
1918 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
1919 NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
1921 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &NewOps[0], NewOps.size());
1924 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
1925 EVT InVT = N->getOperand(0).getValueType();
1926 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1928 unsigned WidenNumElts = WidenVT.getVectorNumElements();
1929 unsigned NumInElts = InVT.getVectorNumElements();
1930 unsigned NumOperands = N->getNumOperands();
1932 bool InputWidened = false; // Indicates we need to widen the input.
1933 if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
1934 if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) {
1935 // Add undef vectors to widen to correct length.
1936 unsigned NumConcat = WidenVT.getVectorNumElements() /
1937 InVT.getVectorNumElements();
1938 SDValue UndefVal = DAG.getUNDEF(InVT);
1939 SmallVector<SDValue, 16> Ops(NumConcat);
1940 for (unsigned i=0; i < NumOperands; ++i)
1941 Ops[i] = N->getOperand(i);
1942 for (unsigned i = NumOperands; i != NumConcat; ++i)
1944 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &Ops[0], NumConcat);
1947 InputWidened = true;
1948 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
1949 // The inputs and the result are widen to the same value.
1951 for (i=1; i < NumOperands; ++i)
1952 if (N->getOperand(i).getOpcode() != ISD::UNDEF)
1955 if (i == NumOperands)
1956 // Everything but the first operand is an UNDEF so just return the
1957 // widened first operand.
1958 return GetWidenedVector(N->getOperand(0));
1960 if (NumOperands == 2) {
1961 // Replace concat of two operands with a shuffle.
1962 SmallVector<int, 16> MaskOps(WidenNumElts, -1);
1963 for (unsigned i = 0; i < NumInElts; ++i) {
1965 MaskOps[i + NumInElts] = i + WidenNumElts;
1967 return DAG.getVectorShuffle(WidenVT, dl,
1968 GetWidenedVector(N->getOperand(0)),
1969 GetWidenedVector(N->getOperand(1)),
1975 // Fall back to use extracts and build vector.
1976 EVT EltVT = WidenVT.getVectorElementType();
1977 SmallVector<SDValue, 16> Ops(WidenNumElts);
1979 for (unsigned i=0; i < NumOperands; ++i) {
1980 SDValue InOp = N->getOperand(i);
1982 InOp = GetWidenedVector(InOp);
1983 for (unsigned j=0; j < NumInElts; ++j)
1984 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
1985 DAG.getConstant(j, TLI.getVectorIdxTy()));
1987 SDValue UndefVal = DAG.getUNDEF(EltVT);
1988 for (; Idx < WidenNumElts; ++Idx)
1989 Ops[Idx] = UndefVal;
1990 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts);
1993 SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) {
1995 SDValue InOp = N->getOperand(0);
1996 SDValue RndOp = N->getOperand(3);
1997 SDValue SatOp = N->getOperand(4);
1999 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2000 unsigned WidenNumElts = WidenVT.getVectorNumElements();
2002 EVT InVT = InOp.getValueType();
2003 EVT InEltVT = InVT.getVectorElementType();
2004 EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts);
2006 SDValue DTyOp = DAG.getValueType(WidenVT);
2007 SDValue STyOp = DAG.getValueType(InWidenVT);
2008 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode();
2010 unsigned InVTNumElts = InVT.getVectorNumElements();
2011 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
2012 InOp = GetWidenedVector(InOp);
2013 InVT = InOp.getValueType();
2014 InVTNumElts = InVT.getVectorNumElements();
2015 if (InVTNumElts == WidenNumElts)
2016 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp,
2020 if (TLI.isTypeLegal(InWidenVT)) {
2021 // Because the result and the input are different vector types, widening
2022 // the result could create a legal type but widening the input might make
2023 // it an illegal type that might lead to repeatedly splitting the input
2024 // and then widening it. To avoid this, we widen the input only if
2025 // it results in a legal type.
2026 if (WidenNumElts % InVTNumElts == 0) {
2027 // Widen the input and call convert on the widened input vector.
2028 unsigned NumConcat = WidenNumElts/InVTNumElts;
2029 SmallVector<SDValue, 16> Ops(NumConcat);
2031 SDValue UndefVal = DAG.getUNDEF(InVT);
2032 for (unsigned i = 1; i != NumConcat; ++i)
2035 InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, &Ops[0],NumConcat);
2036 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp,
2040 if (InVTNumElts % WidenNumElts == 0) {
2041 // Extract the input and convert the shorten input vector.
2042 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp,
2043 DAG.getConstant(0, TLI.getVectorIdxTy()));
2044 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp,
2049 // Otherwise unroll into some nasty scalar code and rebuild the vector.
2050 SmallVector<SDValue, 16> Ops(WidenNumElts);
2051 EVT EltVT = WidenVT.getVectorElementType();
2052 DTyOp = DAG.getValueType(EltVT);
2053 STyOp = DAG.getValueType(InEltVT);
2055 unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
2057 for (i=0; i < MinElts; ++i) {
2058 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
2059 DAG.getConstant(i, TLI.getVectorIdxTy()));
2060 Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp,
2064 SDValue UndefVal = DAG.getUNDEF(EltVT);
2065 for (; i < WidenNumElts; ++i)
2068 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts);
2071 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
2072 EVT VT = N->getValueType(0);
2073 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2074 unsigned WidenNumElts = WidenVT.getVectorNumElements();
2075 SDValue InOp = N->getOperand(0);
2076 SDValue Idx = N->getOperand(1);
2079 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
2080 InOp = GetWidenedVector(InOp);
2082 EVT InVT = InOp.getValueType();
2084 // Check if we can just return the input vector after widening.
2085 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2086 if (IdxVal == 0 && InVT == WidenVT)
2089 // Check if we can extract from the vector.
2090 unsigned InNumElts = InVT.getVectorNumElements();
2091 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
2092 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
2094 // We could try widening the input to the right length but for now, extract
2095 // the original elements, fill the rest with undefs and build a vector.
2096 SmallVector<SDValue, 16> Ops(WidenNumElts);
2097 EVT EltVT = VT.getVectorElementType();
2098 unsigned NumElts = VT.getVectorNumElements();
2100 for (i=0; i < NumElts; ++i)
2101 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
2102 DAG.getConstant(IdxVal+i, TLI.getVectorIdxTy()));
2104 SDValue UndefVal = DAG.getUNDEF(EltVT);
2105 for (; i < WidenNumElts; ++i)
2107 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts);
2110 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
2111 SDValue InOp = GetWidenedVector(N->getOperand(0));
2112 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
2113 InOp.getValueType(), InOp,
2114 N->getOperand(1), N->getOperand(2));
2117 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
2118 LoadSDNode *LD = cast<LoadSDNode>(N);
2119 ISD::LoadExtType ExtType = LD->getExtensionType();
2122 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
2123 if (ExtType != ISD::NON_EXTLOAD)
2124 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
2126 Result = GenWidenVectorLoads(LdChain, LD);
2128 // If we generate a single load, we can use that for the chain. Otherwise,
2129 // build a factor node to remember the multiple loads are independent and
2132 if (LdChain.size() == 1)
2133 NewChain = LdChain[0];
2135 NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other,
2136 &LdChain[0], LdChain.size());
2138 // Modified the chain - switch anything that used the old chain to use
2140 ReplaceValueWith(SDValue(N, 1), NewChain);
2145 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
2146 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2147 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N),
2148 WidenVT, N->getOperand(0));
2151 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
2152 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2153 unsigned WidenNumElts = WidenVT.getVectorNumElements();
2155 SDValue Cond1 = N->getOperand(0);
2156 EVT CondVT = Cond1.getValueType();
2157 if (CondVT.isVector()) {
2158 EVT CondEltVT = CondVT.getVectorElementType();
2159 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(),
2160 CondEltVT, WidenNumElts);
2161 if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
2162 Cond1 = GetWidenedVector(Cond1);
2164 if (Cond1.getValueType() != CondWidenVT)
2165 Cond1 = ModifyToType(Cond1, CondWidenVT);
2168 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
2169 SDValue InOp2 = GetWidenedVector(N->getOperand(2));
2170 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
2171 return DAG.getNode(N->getOpcode(), SDLoc(N),
2172 WidenVT, Cond1, InOp1, InOp2);
2175 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
2176 SDValue InOp1 = GetWidenedVector(N->getOperand(2));
2177 SDValue InOp2 = GetWidenedVector(N->getOperand(3));
2178 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2179 InOp1.getValueType(), N->getOperand(0),
2180 N->getOperand(1), InOp1, InOp2, N->getOperand(4));
2183 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
2184 assert(N->getValueType(0).isVector() ==
2185 N->getOperand(0).getValueType().isVector() &&
2186 "Scalar/Vector type mismatch");
2187 if (N->getValueType(0).isVector()) return WidenVecRes_VSETCC(N);
2189 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2190 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2191 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2192 return DAG.getNode(ISD::SETCC, SDLoc(N), WidenVT,
2193 InOp1, InOp2, N->getOperand(2));
2196 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
2197 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2198 return DAG.getUNDEF(WidenVT);
2201 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
2202 EVT VT = N->getValueType(0);
2205 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2206 unsigned NumElts = VT.getVectorNumElements();
2207 unsigned WidenNumElts = WidenVT.getVectorNumElements();
2209 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2210 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2212 // Adjust mask based on new input vector length.
2213 SmallVector<int, 16> NewMask;
2214 for (unsigned i = 0; i != NumElts; ++i) {
2215 int Idx = N->getMaskElt(i);
2216 if (Idx < (int)NumElts)
2217 NewMask.push_back(Idx);
2219 NewMask.push_back(Idx - NumElts + WidenNumElts);
2221 for (unsigned i = NumElts; i != WidenNumElts; ++i)
2222 NewMask.push_back(-1);
2223 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]);
2226 SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) {
2227 assert(N->getValueType(0).isVector() &&
2228 N->getOperand(0).getValueType().isVector() &&
2229 "Operands must be vectors");
2230 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2231 unsigned WidenNumElts = WidenVT.getVectorNumElements();
2233 SDValue InOp1 = N->getOperand(0);
2234 EVT InVT = InOp1.getValueType();
2235 assert(InVT.isVector() && "can not widen non vector type");
2236 EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(),
2237 InVT.getVectorElementType(), WidenNumElts);
2238 InOp1 = GetWidenedVector(InOp1);
2239 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2241 // Assume that the input and output will be widen appropriately. If not,
2242 // we will have to unroll it at some point.
2243 assert(InOp1.getValueType() == WidenInVT &&
2244 InOp2.getValueType() == WidenInVT &&
2245 "Input not widened to expected type!");
2247 return DAG.getNode(ISD::SETCC, SDLoc(N),
2248 WidenVT, InOp1, InOp2, N->getOperand(2));
2252 //===----------------------------------------------------------------------===//
2253 // Widen Vector Operand
2254 //===----------------------------------------------------------------------===//
2255 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
2256 DEBUG(dbgs() << "Widen node operand " << OpNo << ": ";
2259 SDValue Res = SDValue();
2261 // See if the target wants to custom widen this node.
2262 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2265 switch (N->getOpcode()) {
2268 dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
2272 llvm_unreachable("Do not know how to widen this operator's operand!");
2274 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
2275 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
2276 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
2277 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
2278 case ISD::STORE: Res = WidenVecOp_STORE(N); break;
2279 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
2281 case ISD::FP_EXTEND:
2282 case ISD::FP_TO_SINT:
2283 case ISD::FP_TO_UINT:
2284 case ISD::SINT_TO_FP:
2285 case ISD::UINT_TO_FP:
2287 case ISD::SIGN_EXTEND:
2288 case ISD::ZERO_EXTEND:
2289 case ISD::ANY_EXTEND:
2290 Res = WidenVecOp_Convert(N);
2294 // If Res is null, the sub-method took care of registering the result.
2295 if (!Res.getNode()) return false;
2297 // If the result is N, the sub-method updated N in place. Tell the legalizer
2299 if (Res.getNode() == N)
2303 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2304 "Invalid operand expansion");
2306 ReplaceValueWith(SDValue(N, 0), Res);
2310 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
2311 // Since the result is legal and the input is illegal, it is unlikely
2312 // that we can fix the input to a legal type so unroll the convert
2313 // into some scalar code and create a nasty build vector.
2314 EVT VT = N->getValueType(0);
2315 EVT EltVT = VT.getVectorElementType();
2317 unsigned NumElts = VT.getVectorNumElements();
2318 SDValue InOp = N->getOperand(0);
2319 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
2320 InOp = GetWidenedVector(InOp);
2321 EVT InVT = InOp.getValueType();
2322 EVT InEltVT = InVT.getVectorElementType();
2324 unsigned Opcode = N->getOpcode();
2325 SmallVector<SDValue, 16> Ops(NumElts);
2326 for (unsigned i=0; i < NumElts; ++i)
2327 Ops[i] = DAG.getNode(Opcode, dl, EltVT,
2328 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
2329 DAG.getConstant(i, TLI.getVectorIdxTy())));
2331 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts);
2334 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
2335 EVT VT = N->getValueType(0);
2336 SDValue InOp = GetWidenedVector(N->getOperand(0));
2337 EVT InWidenVT = InOp.getValueType();
2340 // Check if we can convert between two legal vector types and extract.
2341 unsigned InWidenSize = InWidenVT.getSizeInBits();
2342 unsigned Size = VT.getSizeInBits();
2343 // x86mmx is not an acceptable vector element type, so don't try.
2344 if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) {
2345 unsigned NewNumElts = InWidenSize / Size;
2346 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
2347 if (TLI.isTypeLegal(NewVT)) {
2348 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
2349 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
2350 DAG.getConstant(0, TLI.getVectorIdxTy()));
2354 return CreateStackStoreLoad(InOp, VT);
2357 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
2358 // If the input vector is not legal, it is likely that we will not find a
2359 // legal vector of the same size. Replace the concatenate vector with a
2360 // nasty build vector.
2361 EVT VT = N->getValueType(0);
2362 EVT EltVT = VT.getVectorElementType();
2364 unsigned NumElts = VT.getVectorNumElements();
2365 SmallVector<SDValue, 16> Ops(NumElts);
2367 EVT InVT = N->getOperand(0).getValueType();
2368 unsigned NumInElts = InVT.getVectorNumElements();
2371 unsigned NumOperands = N->getNumOperands();
2372 for (unsigned i=0; i < NumOperands; ++i) {
2373 SDValue InOp = N->getOperand(i);
2374 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
2375 InOp = GetWidenedVector(InOp);
2376 for (unsigned j=0; j < NumInElts; ++j)
2377 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
2378 DAG.getConstant(j, TLI.getVectorIdxTy()));
2380 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts);
2383 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
2384 SDValue InOp = GetWidenedVector(N->getOperand(0));
2385 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
2386 N->getValueType(0), InOp, N->getOperand(1));
2389 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
2390 SDValue InOp = GetWidenedVector(N->getOperand(0));
2391 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
2392 N->getValueType(0), InOp, N->getOperand(1));
2395 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
2396 // We have to widen the value but we want only to store the original
2398 StoreSDNode *ST = cast<StoreSDNode>(N);
2400 SmallVector<SDValue, 16> StChain;
2401 if (ST->isTruncatingStore())
2402 GenWidenVectorTruncStores(StChain, ST);
2404 GenWidenVectorStores(StChain, ST);
2406 if (StChain.size() == 1)
2409 return DAG.getNode(ISD::TokenFactor, SDLoc(ST),
2410 MVT::Other,&StChain[0],StChain.size());
2413 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
2414 SDValue InOp0 = GetWidenedVector(N->getOperand(0));
2415 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
2418 // WARNING: In this code we widen the compare instruction with garbage.
2419 // This garbage may contain denormal floats which may be slow. Is this a real
2420 // concern ? Should we zero the unused lanes if this is a float compare ?
2422 // Get a new SETCC node to compare the newly widened operands.
2423 // Only some of the compared elements are legal.
2424 EVT SVT = TLI.getSetCCResultType(*DAG.getContext(), InOp0.getValueType());
2425 SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
2426 SVT, InOp0, InOp1, N->getOperand(2));
2428 // Extract the needed results from the result vector.
2429 EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
2430 SVT.getVectorElementType(),
2431 N->getValueType(0).getVectorNumElements());
2432 SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl,
2433 ResVT, WideSETCC, DAG.getConstant(0,
2434 TLI.getVectorIdxTy()));
2436 return PromoteTargetBoolean(CC, N->getValueType(0));
2440 //===----------------------------------------------------------------------===//
2441 // Vector Widening Utilities
2442 //===----------------------------------------------------------------------===//
2444 // Utility function to find the type to chop up a widen vector for load/store
2445 // TLI: Target lowering used to determine legal types.
2446 // Width: Width left need to load/store.
2447 // WidenVT: The widen vector type to load to/store from
2448 // Align: If 0, don't allow use of a wider type
2449 // WidenEx: If Align is not 0, the amount additional we can load/store from.
2451 static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI,
2452 unsigned Width, EVT WidenVT,
2453 unsigned Align = 0, unsigned WidenEx = 0) {
2454 EVT WidenEltVT = WidenVT.getVectorElementType();
2455 unsigned WidenWidth = WidenVT.getSizeInBits();
2456 unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
2457 unsigned AlignInBits = Align*8;
2459 // If we have one element to load/store, return it.
2460 EVT RetVT = WidenEltVT;
2461 if (Width == WidenEltWidth)
2464 // See if there is larger legal integer than the element type to load/store
2466 for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE;
2467 VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) {
2468 EVT MemVT((MVT::SimpleValueType) VT);
2469 unsigned MemVTWidth = MemVT.getSizeInBits();
2470 if (MemVT.getSizeInBits() <= WidenEltWidth)
2472 if (TLI.isTypeLegal(MemVT) && (WidenWidth % MemVTWidth) == 0 &&
2473 isPowerOf2_32(WidenWidth / MemVTWidth) &&
2474 (MemVTWidth <= Width ||
2475 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
2481 // See if there is a larger vector type to load/store that has the same vector
2482 // element type and is evenly divisible with the WidenVT.
2483 for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE;
2484 VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) {
2485 EVT MemVT = (MVT::SimpleValueType) VT;
2486 unsigned MemVTWidth = MemVT.getSizeInBits();
2487 if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() &&
2488 (WidenWidth % MemVTWidth) == 0 &&
2489 isPowerOf2_32(WidenWidth / MemVTWidth) &&
2490 (MemVTWidth <= Width ||
2491 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
2492 if (RetVT.getSizeInBits() < MemVTWidth || MemVT == WidenVT)
2500 // Builds a vector type from scalar loads
2501 // VecTy: Resulting Vector type
2502 // LDOps: Load operators to build a vector type
2503 // [Start,End) the list of loads to use.
2504 static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
2505 SmallVectorImpl<SDValue> &LdOps,
2506 unsigned Start, unsigned End) {
2507 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2508 SDLoc dl(LdOps[Start]);
2509 EVT LdTy = LdOps[Start].getValueType();
2510 unsigned Width = VecTy.getSizeInBits();
2511 unsigned NumElts = Width / LdTy.getSizeInBits();
2512 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
2515 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
2517 for (unsigned i = Start + 1; i != End; ++i) {
2518 EVT NewLdTy = LdOps[i].getValueType();
2519 if (NewLdTy != LdTy) {
2520 NumElts = Width / NewLdTy.getSizeInBits();
2521 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
2522 VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
2523 // Readjust position and vector position based on new load type
2524 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
2527 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
2528 DAG.getConstant(Idx++, TLI.getVectorIdxTy()));
2530 return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
2533 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
2535 // The strategy assumes that we can efficiently load powers of two widths.
2536 // The routines chops the vector into the largest vector loads with the same
2537 // element type or scalar loads and then recombines it to the widen vector
2539 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
2540 unsigned WidenWidth = WidenVT.getSizeInBits();
2541 EVT LdVT = LD->getMemoryVT();
2543 assert(LdVT.isVector() && WidenVT.isVector());
2544 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
2547 SDValue Chain = LD->getChain();
2548 SDValue BasePtr = LD->getBasePtr();
2549 unsigned Align = LD->getAlignment();
2550 bool isVolatile = LD->isVolatile();
2551 bool isNonTemporal = LD->isNonTemporal();
2552 bool isInvariant = LD->isInvariant();
2554 int LdWidth = LdVT.getSizeInBits();
2555 int WidthDiff = WidenWidth - LdWidth; // Difference
2556 unsigned LdAlign = (isVolatile) ? 0 : Align; // Allow wider loads
2558 // Find the vector type that can load from.
2559 EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
2560 int NewVTWidth = NewVT.getSizeInBits();
2561 SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
2562 isVolatile, isNonTemporal, isInvariant, Align);
2563 LdChain.push_back(LdOp.getValue(1));
2565 // Check if we can load the element with one instruction
2566 if (LdWidth <= NewVTWidth) {
2567 if (!NewVT.isVector()) {
2568 unsigned NumElts = WidenWidth / NewVTWidth;
2569 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
2570 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
2571 return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
2573 if (NewVT == WidenVT)
2576 assert(WidenWidth % NewVTWidth == 0);
2577 unsigned NumConcat = WidenWidth / NewVTWidth;
2578 SmallVector<SDValue, 16> ConcatOps(NumConcat);
2579 SDValue UndefVal = DAG.getUNDEF(NewVT);
2580 ConcatOps[0] = LdOp;
2581 for (unsigned i = 1; i != NumConcat; ++i)
2582 ConcatOps[i] = UndefVal;
2583 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &ConcatOps[0],
2587 // Load vector by using multiple loads from largest vector to scalar
2588 SmallVector<SDValue, 16> LdOps;
2589 LdOps.push_back(LdOp);
2591 LdWidth -= NewVTWidth;
2592 unsigned Offset = 0;
2594 while (LdWidth > 0) {
2595 unsigned Increment = NewVTWidth / 8;
2596 Offset += Increment;
2597 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
2598 DAG.getIntPtrConstant(Increment));
2601 if (LdWidth < NewVTWidth) {
2602 // Our current type we are using is too large, find a better size
2603 NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
2604 NewVTWidth = NewVT.getSizeInBits();
2605 L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
2606 LD->getPointerInfo().getWithOffset(Offset), isVolatile,
2607 isNonTemporal, isInvariant, MinAlign(Align, Increment));
2608 LdChain.push_back(L.getValue(1));
2609 if (L->getValueType(0).isVector()) {
2610 SmallVector<SDValue, 16> Loads;
2612 unsigned size = L->getValueSizeInBits(0);
2613 while (size < LdOp->getValueSizeInBits(0)) {
2614 Loads.push_back(DAG.getUNDEF(L->getValueType(0)));
2615 size += L->getValueSizeInBits(0);
2617 L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0),
2618 &Loads[0], Loads.size());
2621 L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
2622 LD->getPointerInfo().getWithOffset(Offset), isVolatile,
2623 isNonTemporal, isInvariant, MinAlign(Align, Increment));
2624 LdChain.push_back(L.getValue(1));
2630 LdWidth -= NewVTWidth;
2633 // Build the vector from the loads operations
2634 unsigned End = LdOps.size();
2635 if (!LdOps[0].getValueType().isVector())
2636 // All the loads are scalar loads.
2637 return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
2639 // If the load contains vectors, build the vector using concat vector.
2640 // All of the vectors used to loads are power of 2 and the scalars load
2641 // can be combined to make a power of 2 vector.
2642 SmallVector<SDValue, 16> ConcatOps(End);
2645 EVT LdTy = LdOps[i].getValueType();
2646 // First combine the scalar loads to a vector
2647 if (!LdTy.isVector()) {
2648 for (--i; i >= 0; --i) {
2649 LdTy = LdOps[i].getValueType();
2650 if (LdTy.isVector())
2653 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i+1, End);
2655 ConcatOps[--Idx] = LdOps[i];
2656 for (--i; i >= 0; --i) {
2657 EVT NewLdTy = LdOps[i].getValueType();
2658 if (NewLdTy != LdTy) {
2659 // Create a larger vector
2660 ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
2661 &ConcatOps[Idx], End - Idx);
2665 ConcatOps[--Idx] = LdOps[i];
2668 if (WidenWidth == LdTy.getSizeInBits()*(End - Idx))
2669 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
2670 &ConcatOps[Idx], End - Idx);
2672 // We need to fill the rest with undefs to build the vector
2673 unsigned NumOps = WidenWidth / LdTy.getSizeInBits();
2674 SmallVector<SDValue, 16> WidenOps(NumOps);
2675 SDValue UndefVal = DAG.getUNDEF(LdTy);
2678 for (; i != End-Idx; ++i)
2679 WidenOps[i] = ConcatOps[Idx+i];
2680 for (; i != NumOps; ++i)
2681 WidenOps[i] = UndefVal;
2683 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &WidenOps[0],NumOps);
2687 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
2689 ISD::LoadExtType ExtType) {
2690 // For extension loads, it may not be more efficient to chop up the vector
2691 // and then extended it. Instead, we unroll the load and build a new vector.
2692 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
2693 EVT LdVT = LD->getMemoryVT();
2695 assert(LdVT.isVector() && WidenVT.isVector());
2698 SDValue Chain = LD->getChain();
2699 SDValue BasePtr = LD->getBasePtr();
2700 unsigned Align = LD->getAlignment();
2701 bool isVolatile = LD->isVolatile();
2702 bool isNonTemporal = LD->isNonTemporal();
2704 EVT EltVT = WidenVT.getVectorElementType();
2705 EVT LdEltVT = LdVT.getVectorElementType();
2706 unsigned NumElts = LdVT.getVectorNumElements();
2708 // Load each element and widen
2709 unsigned WidenNumElts = WidenVT.getVectorNumElements();
2710 SmallVector<SDValue, 16> Ops(WidenNumElts);
2711 unsigned Increment = LdEltVT.getSizeInBits() / 8;
2712 Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr,
2713 LD->getPointerInfo(),
2714 LdEltVT, isVolatile, isNonTemporal, Align);
2715 LdChain.push_back(Ops[0].getValue(1));
2716 unsigned i = 0, Offset = Increment;
2717 for (i=1; i < NumElts; ++i, Offset += Increment) {
2718 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(),
2719 BasePtr, DAG.getIntPtrConstant(Offset));
2720 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
2721 LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
2722 isVolatile, isNonTemporal, Align);
2723 LdChain.push_back(Ops[i].getValue(1));
2726 // Fill the rest with undefs
2727 SDValue UndefVal = DAG.getUNDEF(EltVT);
2728 for (; i != WidenNumElts; ++i)
2731 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], Ops.size());
2735 void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
2737 // The strategy assumes that we can efficiently store powers of two widths.
2738 // The routines chops the vector into the largest vector stores with the same
2739 // element type or scalar stores.
2740 SDValue Chain = ST->getChain();
2741 SDValue BasePtr = ST->getBasePtr();
2742 unsigned Align = ST->getAlignment();
2743 bool isVolatile = ST->isVolatile();
2744 bool isNonTemporal = ST->isNonTemporal();
2745 SDValue ValOp = GetWidenedVector(ST->getValue());
2748 EVT StVT = ST->getMemoryVT();
2749 unsigned StWidth = StVT.getSizeInBits();
2750 EVT ValVT = ValOp.getValueType();
2751 unsigned ValWidth = ValVT.getSizeInBits();
2752 EVT ValEltVT = ValVT.getVectorElementType();
2753 unsigned ValEltWidth = ValEltVT.getSizeInBits();
2754 assert(StVT.getVectorElementType() == ValEltVT);
2756 int Idx = 0; // current index to store
2757 unsigned Offset = 0; // offset from base to store
2758 while (StWidth != 0) {
2759 // Find the largest vector type we can store with
2760 EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT);
2761 unsigned NewVTWidth = NewVT.getSizeInBits();
2762 unsigned Increment = NewVTWidth / 8;
2763 if (NewVT.isVector()) {
2764 unsigned NumVTElts = NewVT.getVectorNumElements();
2766 SDValue EOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
2767 DAG.getConstant(Idx, TLI.getVectorIdxTy()));
2768 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr,
2769 ST->getPointerInfo().getWithOffset(Offset),
2770 isVolatile, isNonTemporal,
2771 MinAlign(Align, Offset)));
2772 StWidth -= NewVTWidth;
2773 Offset += Increment;
2775 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
2776 DAG.getIntPtrConstant(Increment));
2777 } while (StWidth != 0 && StWidth >= NewVTWidth);
2779 // Cast the vector to the scalar type we can store
2780 unsigned NumElts = ValWidth / NewVTWidth;
2781 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
2782 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
2783 // Readjust index position based on new vector type
2784 Idx = Idx * ValEltWidth / NewVTWidth;
2786 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
2787 DAG.getConstant(Idx++, TLI.getVectorIdxTy()));
2788 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr,
2789 ST->getPointerInfo().getWithOffset(Offset),
2790 isVolatile, isNonTemporal,
2791 MinAlign(Align, Offset)));
2792 StWidth -= NewVTWidth;
2793 Offset += Increment;
2794 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
2795 DAG.getIntPtrConstant(Increment));
2796 } while (StWidth != 0 && StWidth >= NewVTWidth);
2797 // Restore index back to be relative to the original widen element type
2798 Idx = Idx * NewVTWidth / ValEltWidth;
2804 DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
2806 // For extension loads, it may not be more efficient to truncate the vector
2807 // and then store it. Instead, we extract each element and then store it.
2808 SDValue Chain = ST->getChain();
2809 SDValue BasePtr = ST->getBasePtr();
2810 unsigned Align = ST->getAlignment();
2811 bool isVolatile = ST->isVolatile();
2812 bool isNonTemporal = ST->isNonTemporal();
2813 SDValue ValOp = GetWidenedVector(ST->getValue());
2816 EVT StVT = ST->getMemoryVT();
2817 EVT ValVT = ValOp.getValueType();
2819 // It must be true that we the widen vector type is bigger than where
2820 // we need to store.
2821 assert(StVT.isVector() && ValOp.getValueType().isVector());
2822 assert(StVT.bitsLT(ValOp.getValueType()));
2824 // For truncating stores, we can not play the tricks of chopping legal
2825 // vector types and bit cast it to the right type. Instead, we unroll
2827 EVT StEltVT = StVT.getVectorElementType();
2828 EVT ValEltVT = ValVT.getVectorElementType();
2829 unsigned Increment = ValEltVT.getSizeInBits() / 8;
2830 unsigned NumElts = StVT.getVectorNumElements();
2831 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
2832 DAG.getConstant(0, TLI.getVectorIdxTy()));
2833 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr,
2834 ST->getPointerInfo(), StEltVT,
2835 isVolatile, isNonTemporal, Align));
2836 unsigned Offset = Increment;
2837 for (unsigned i=1; i < NumElts; ++i, Offset += Increment) {
2838 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(),
2839 BasePtr, DAG.getIntPtrConstant(Offset));
2840 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
2841 DAG.getConstant(0, TLI.getVectorIdxTy()));
2842 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr,
2843 ST->getPointerInfo().getWithOffset(Offset),
2844 StEltVT, isVolatile, isNonTemporal,
2845 MinAlign(Align, Offset)));
2849 /// Modifies a vector input (widen or narrows) to a vector of NVT. The
2850 /// input vector must have the same element type as NVT.
2851 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT) {
2852 // Note that InOp might have been widened so it might already have
2853 // the right width or it might need be narrowed.
2854 EVT InVT = InOp.getValueType();
2855 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
2856 "input and widen element type must match");
2859 // Check if InOp already has the right width.
2863 unsigned InNumElts = InVT.getVectorNumElements();
2864 unsigned WidenNumElts = NVT.getVectorNumElements();
2865 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
2866 unsigned NumConcat = WidenNumElts / InNumElts;
2867 SmallVector<SDValue, 16> Ops(NumConcat);
2868 SDValue UndefVal = DAG.getUNDEF(InVT);
2870 for (unsigned i = 1; i != NumConcat; ++i)
2873 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, &Ops[0], NumConcat);
2876 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
2877 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
2878 DAG.getConstant(0, TLI.getVectorIdxTy()));
2880 // Fall back to extract and build.
2881 SmallVector<SDValue, 16> Ops(WidenNumElts);
2882 EVT EltVT = NVT.getVectorElementType();
2883 unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
2885 for (Idx = 0; Idx < MinNumElts; ++Idx)
2886 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
2887 DAG.getConstant(Idx, TLI.getVectorIdxTy()));
2889 SDValue UndefVal = DAG.getUNDEF(EltVT);
2890 for ( ; Idx < WidenNumElts; ++Idx)
2891 Ops[Idx] = UndefVal;
2892 return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, &Ops[0], WidenNumElts);