2 //===-- SPUISelLowering.cpp - Cell SPU DAG Lowering Implementation --------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the SPUTargetLowering class.
12 //===----------------------------------------------------------------------===//
14 #include "SPURegisterNames.h"
15 #include "SPUISelLowering.h"
16 #include "SPUTargetMachine.h"
17 #include "SPUFrameInfo.h"
18 #include "llvm/ADT/APInt.h"
19 #include "llvm/ADT/VectorExtras.h"
20 #include "llvm/CallingConv.h"
21 #include "llvm/CodeGen/CallingConvLower.h"
22 #include "llvm/CodeGen/MachineFrameInfo.h"
23 #include "llvm/CodeGen/MachineFunction.h"
24 #include "llvm/CodeGen/MachineInstrBuilder.h"
25 #include "llvm/CodeGen/MachineRegisterInfo.h"
26 #include "llvm/CodeGen/SelectionDAG.h"
27 #include "llvm/Constants.h"
28 #include "llvm/Function.h"
29 #include "llvm/Intrinsics.h"
30 #include "llvm/Support/Debug.h"
31 #include "llvm/Support/ErrorHandling.h"
32 #include "llvm/Support/MathExtras.h"
33 #include "llvm/Support/raw_ostream.h"
34 #include "llvm/Target/TargetOptions.h"
40 // Used in getTargetNodeName() below
42 std::map<unsigned, const char *> node_names;
44 //! MVT mapping to useful data for Cell SPU
45 struct valtype_map_s {
47 const int prefslot_byte;
50 const valtype_map_s valtype_map[] = {
61 const size_t n_valtype_map = sizeof(valtype_map) / sizeof(valtype_map[0]);
63 const valtype_map_s *getValueTypeMapEntry(MVT VT) {
64 const valtype_map_s *retval = 0;
66 for (size_t i = 0; i < n_valtype_map; ++i) {
67 if (valtype_map[i].valtype == VT) {
68 retval = valtype_map + i;
76 raw_string_ostream Msg(msg);
77 Msg << "getValueTypeMapEntry returns NULL for "
79 llvm_report_error(Msg.str());
86 //! Expand a library call into an actual call DAG node
89 This code is taken from SelectionDAGLegalize, since it is not exposed as
90 part of the LLVM SelectionDAG API.
94 ExpandLibCall(RTLIB::Libcall LC, SDValue Op, SelectionDAG &DAG,
95 bool isSigned, SDValue &Hi, SPUTargetLowering &TLI) {
96 // The input chain to this libcall is the entry node of the function.
97 // Legalizing the call will automatically add the previous call to the
99 SDValue InChain = DAG.getEntryNode();
101 TargetLowering::ArgListTy Args;
102 TargetLowering::ArgListEntry Entry;
103 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
104 MVT ArgVT = Op.getOperand(i).getValueType();
105 const Type *ArgTy = ArgVT.getTypeForMVT(*DAG.getContext());
106 Entry.Node = Op.getOperand(i);
108 Entry.isSExt = isSigned;
109 Entry.isZExt = !isSigned;
110 Args.push_back(Entry);
112 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
115 // Splice the libcall in wherever FindInputOutputChains tells us to.
117 Op.getNode()->getValueType(0).getTypeForMVT(*DAG.getContext());
118 std::pair<SDValue, SDValue> CallInfo =
119 TLI.LowerCallTo(InChain, RetTy, isSigned, !isSigned, false, false,
120 0, CallingConv::C, false, Callee, Args, DAG,
123 return CallInfo.first;
127 SPUTargetLowering::SPUTargetLowering(SPUTargetMachine &TM)
128 : TargetLowering(TM),
131 // Fold away setcc operations if possible.
134 // Use _setjmp/_longjmp instead of setjmp/longjmp.
135 setUseUnderscoreSetJmp(true);
136 setUseUnderscoreLongJmp(true);
138 // Set RTLIB libcall names as used by SPU:
139 setLibcallName(RTLIB::DIV_F64, "__fast_divdf3");
141 // Set up the SPU's register classes:
142 addRegisterClass(MVT::i8, SPU::R8CRegisterClass);
143 addRegisterClass(MVT::i16, SPU::R16CRegisterClass);
144 addRegisterClass(MVT::i32, SPU::R32CRegisterClass);
145 addRegisterClass(MVT::i64, SPU::R64CRegisterClass);
146 addRegisterClass(MVT::f32, SPU::R32FPRegisterClass);
147 addRegisterClass(MVT::f64, SPU::R64FPRegisterClass);
148 addRegisterClass(MVT::i128, SPU::GPRCRegisterClass);
150 // SPU has no sign or zero extended loads for i1, i8, i16:
151 setLoadExtAction(ISD::EXTLOAD, MVT::i1, Promote);
152 setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote);
153 setLoadExtAction(ISD::ZEXTLOAD, MVT::i1, Promote);
155 setLoadExtAction(ISD::EXTLOAD, MVT::f32, Expand);
156 setLoadExtAction(ISD::EXTLOAD, MVT::f64, Expand);
158 // SPU constant load actions are custom lowered:
159 setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
160 setOperationAction(ISD::ConstantFP, MVT::f64, Custom);
162 // SPU's loads and stores have to be custom lowered:
163 for (unsigned sctype = (unsigned) MVT::i8; sctype < (unsigned) MVT::i128;
165 MVT VT = (MVT::SimpleValueType)sctype;
167 setOperationAction(ISD::LOAD, VT, Custom);
168 setOperationAction(ISD::STORE, VT, Custom);
169 setLoadExtAction(ISD::EXTLOAD, VT, Custom);
170 setLoadExtAction(ISD::ZEXTLOAD, VT, Custom);
171 setLoadExtAction(ISD::SEXTLOAD, VT, Custom);
173 for (unsigned stype = sctype - 1; stype >= (unsigned) MVT::i8; --stype) {
174 MVT StoreVT = (MVT::SimpleValueType) stype;
175 setTruncStoreAction(VT, StoreVT, Expand);
179 for (unsigned sctype = (unsigned) MVT::f32; sctype < (unsigned) MVT::f64;
181 MVT VT = (MVT::SimpleValueType) sctype;
183 setOperationAction(ISD::LOAD, VT, Custom);
184 setOperationAction(ISD::STORE, VT, Custom);
186 for (unsigned stype = sctype - 1; stype >= (unsigned) MVT::f32; --stype) {
187 MVT StoreVT = (MVT::SimpleValueType) stype;
188 setTruncStoreAction(VT, StoreVT, Expand);
192 // Expand the jumptable branches
193 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
194 setOperationAction(ISD::BR_CC, MVT::Other, Expand);
196 // Custom lower SELECT_CC for most cases, but expand by default
197 setOperationAction(ISD::SELECT_CC, MVT::Other, Expand);
198 setOperationAction(ISD::SELECT_CC, MVT::i8, Custom);
199 setOperationAction(ISD::SELECT_CC, MVT::i16, Custom);
200 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
201 setOperationAction(ISD::SELECT_CC, MVT::i64, Custom);
203 // SPU has no intrinsics for these particular operations:
204 setOperationAction(ISD::MEMBARRIER, MVT::Other, Expand);
206 // SPU has no SREM/UREM instructions
207 setOperationAction(ISD::SREM, MVT::i32, Expand);
208 setOperationAction(ISD::UREM, MVT::i32, Expand);
209 setOperationAction(ISD::SREM, MVT::i64, Expand);
210 setOperationAction(ISD::UREM, MVT::i64, Expand);
212 // We don't support sin/cos/sqrt/fmod
213 setOperationAction(ISD::FSIN , MVT::f64, Expand);
214 setOperationAction(ISD::FCOS , MVT::f64, Expand);
215 setOperationAction(ISD::FREM , MVT::f64, Expand);
216 setOperationAction(ISD::FSIN , MVT::f32, Expand);
217 setOperationAction(ISD::FCOS , MVT::f32, Expand);
218 setOperationAction(ISD::FREM , MVT::f32, Expand);
220 // Expand fsqrt to the appropriate libcall (NOTE: should use h/w fsqrt
222 setOperationAction(ISD::FSQRT, MVT::f64, Expand);
223 setOperationAction(ISD::FSQRT, MVT::f32, Expand);
225 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
226 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand);
228 // SPU can do rotate right and left, so legalize it... but customize for i8
229 // because instructions don't exist.
231 // FIXME: Change from "expand" to appropriate type once ROTR is supported in
233 setOperationAction(ISD::ROTR, MVT::i32, Expand /*Legal*/);
234 setOperationAction(ISD::ROTR, MVT::i16, Expand /*Legal*/);
235 setOperationAction(ISD::ROTR, MVT::i8, Expand /*Custom*/);
237 setOperationAction(ISD::ROTL, MVT::i32, Legal);
238 setOperationAction(ISD::ROTL, MVT::i16, Legal);
239 setOperationAction(ISD::ROTL, MVT::i8, Custom);
241 // SPU has no native version of shift left/right for i8
242 setOperationAction(ISD::SHL, MVT::i8, Custom);
243 setOperationAction(ISD::SRL, MVT::i8, Custom);
244 setOperationAction(ISD::SRA, MVT::i8, Custom);
246 // Make these operations legal and handle them during instruction selection:
247 setOperationAction(ISD::SHL, MVT::i64, Legal);
248 setOperationAction(ISD::SRL, MVT::i64, Legal);
249 setOperationAction(ISD::SRA, MVT::i64, Legal);
251 // Custom lower i8, i32 and i64 multiplications
252 setOperationAction(ISD::MUL, MVT::i8, Custom);
253 setOperationAction(ISD::MUL, MVT::i32, Legal);
254 setOperationAction(ISD::MUL, MVT::i64, Legal);
256 // Expand double-width multiplication
257 // FIXME: It would probably be reasonable to support some of these operations
258 setOperationAction(ISD::UMUL_LOHI, MVT::i8, Expand);
259 setOperationAction(ISD::SMUL_LOHI, MVT::i8, Expand);
260 setOperationAction(ISD::MULHU, MVT::i8, Expand);
261 setOperationAction(ISD::MULHS, MVT::i8, Expand);
262 setOperationAction(ISD::UMUL_LOHI, MVT::i16, Expand);
263 setOperationAction(ISD::SMUL_LOHI, MVT::i16, Expand);
264 setOperationAction(ISD::MULHU, MVT::i16, Expand);
265 setOperationAction(ISD::MULHS, MVT::i16, Expand);
266 setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand);
267 setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand);
268 setOperationAction(ISD::MULHU, MVT::i32, Expand);
269 setOperationAction(ISD::MULHS, MVT::i32, Expand);
270 setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
271 setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
272 setOperationAction(ISD::MULHU, MVT::i64, Expand);
273 setOperationAction(ISD::MULHS, MVT::i64, Expand);
275 // Need to custom handle (some) common i8, i64 math ops
276 setOperationAction(ISD::ADD, MVT::i8, Custom);
277 setOperationAction(ISD::ADD, MVT::i64, Legal);
278 setOperationAction(ISD::SUB, MVT::i8, Custom);
279 setOperationAction(ISD::SUB, MVT::i64, Legal);
281 // SPU does not have BSWAP. It does have i32 support CTLZ.
282 // CTPOP has to be custom lowered.
283 setOperationAction(ISD::BSWAP, MVT::i32, Expand);
284 setOperationAction(ISD::BSWAP, MVT::i64, Expand);
286 setOperationAction(ISD::CTPOP, MVT::i8, Custom);
287 setOperationAction(ISD::CTPOP, MVT::i16, Custom);
288 setOperationAction(ISD::CTPOP, MVT::i32, Custom);
289 setOperationAction(ISD::CTPOP, MVT::i64, Custom);
291 setOperationAction(ISD::CTTZ , MVT::i32, Expand);
292 setOperationAction(ISD::CTTZ , MVT::i64, Expand);
294 setOperationAction(ISD::CTLZ , MVT::i32, Legal);
296 // SPU has a version of select that implements (a&~c)|(b&c), just like
297 // select ought to work:
298 setOperationAction(ISD::SELECT, MVT::i8, Legal);
299 setOperationAction(ISD::SELECT, MVT::i16, Legal);
300 setOperationAction(ISD::SELECT, MVT::i32, Legal);
301 setOperationAction(ISD::SELECT, MVT::i64, Legal);
303 setOperationAction(ISD::SETCC, MVT::i8, Legal);
304 setOperationAction(ISD::SETCC, MVT::i16, Legal);
305 setOperationAction(ISD::SETCC, MVT::i32, Legal);
306 setOperationAction(ISD::SETCC, MVT::i64, Legal);
307 setOperationAction(ISD::SETCC, MVT::f64, Custom);
309 // Custom lower i128 -> i64 truncates
310 setOperationAction(ISD::TRUNCATE, MVT::i64, Custom);
312 // SPU has a legal FP -> signed INT instruction for f32, but for f64, need
313 // to expand to a libcall, hence the custom lowering:
314 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
315 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
317 // FDIV on SPU requires custom lowering
318 setOperationAction(ISD::FDIV, MVT::f64, Expand); // to libcall
320 // SPU has [U|S]INT_TO_FP for f32->i32, but not for f64->i32, f64->i64:
321 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
322 setOperationAction(ISD::SINT_TO_FP, MVT::i16, Promote);
323 setOperationAction(ISD::SINT_TO_FP, MVT::i8, Promote);
324 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
325 setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote);
326 setOperationAction(ISD::UINT_TO_FP, MVT::i8, Promote);
327 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
328 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
330 setOperationAction(ISD::BIT_CONVERT, MVT::i32, Legal);
331 setOperationAction(ISD::BIT_CONVERT, MVT::f32, Legal);
332 setOperationAction(ISD::BIT_CONVERT, MVT::i64, Legal);
333 setOperationAction(ISD::BIT_CONVERT, MVT::f64, Legal);
335 // We cannot sextinreg(i1). Expand to shifts.
336 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
338 // Support label based line numbers.
339 setOperationAction(ISD::DBG_STOPPOINT, MVT::Other, Expand);
340 setOperationAction(ISD::DEBUG_LOC, MVT::Other, Expand);
342 // We want to legalize GlobalAddress and ConstantPool nodes into the
343 // appropriate instructions to materialize the address.
344 for (unsigned sctype = (unsigned) MVT::i8; sctype < (unsigned) MVT::f128;
346 MVT VT = (MVT::SimpleValueType)sctype;
348 setOperationAction(ISD::GlobalAddress, VT, Custom);
349 setOperationAction(ISD::ConstantPool, VT, Custom);
350 setOperationAction(ISD::JumpTable, VT, Custom);
353 // RET must be custom lowered, to meet ABI requirements
354 setOperationAction(ISD::RET, MVT::Other, Custom);
356 // VASTART needs to be custom lowered to use the VarArgsFrameIndex
357 setOperationAction(ISD::VASTART , MVT::Other, Custom);
359 // Use the default implementation.
360 setOperationAction(ISD::VAARG , MVT::Other, Expand);
361 setOperationAction(ISD::VACOPY , MVT::Other, Expand);
362 setOperationAction(ISD::VAEND , MVT::Other, Expand);
363 setOperationAction(ISD::STACKSAVE , MVT::Other, Expand);
364 setOperationAction(ISD::STACKRESTORE , MVT::Other, Expand);
365 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32 , Expand);
366 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64 , Expand);
368 // Cell SPU has instructions for converting between i64 and fp.
369 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
370 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
372 // To take advantage of the above i64 FP_TO_SINT, promote i32 FP_TO_UINT
373 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Promote);
375 // BUILD_PAIR can't be handled natively, and should be expanded to shl/or
376 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
378 // First set operation action for all vector types to expand. Then we
379 // will selectively turn on ones that can be effectively codegen'd.
380 addRegisterClass(MVT::v16i8, SPU::VECREGRegisterClass);
381 addRegisterClass(MVT::v8i16, SPU::VECREGRegisterClass);
382 addRegisterClass(MVT::v4i32, SPU::VECREGRegisterClass);
383 addRegisterClass(MVT::v2i64, SPU::VECREGRegisterClass);
384 addRegisterClass(MVT::v4f32, SPU::VECREGRegisterClass);
385 addRegisterClass(MVT::v2f64, SPU::VECREGRegisterClass);
387 // "Odd size" vector classes that we're willing to support:
388 addRegisterClass(MVT::v2i32, SPU::VECREGRegisterClass);
390 for (unsigned i = (unsigned)MVT::FIRST_VECTOR_VALUETYPE;
391 i <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++i) {
392 MVT VT = (MVT::SimpleValueType)i;
394 // add/sub are legal for all supported vector VT's.
395 setOperationAction(ISD::ADD, VT, Legal);
396 setOperationAction(ISD::SUB, VT, Legal);
397 // mul has to be custom lowered.
398 setOperationAction(ISD::MUL, VT, Legal);
400 setOperationAction(ISD::AND, VT, Legal);
401 setOperationAction(ISD::OR, VT, Legal);
402 setOperationAction(ISD::XOR, VT, Legal);
403 setOperationAction(ISD::LOAD, VT, Legal);
404 setOperationAction(ISD::SELECT, VT, Legal);
405 setOperationAction(ISD::STORE, VT, Legal);
407 // These operations need to be expanded:
408 setOperationAction(ISD::SDIV, VT, Expand);
409 setOperationAction(ISD::SREM, VT, Expand);
410 setOperationAction(ISD::UDIV, VT, Expand);
411 setOperationAction(ISD::UREM, VT, Expand);
413 // Custom lower build_vector, constant pool spills, insert and
414 // extract vector elements:
415 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
416 setOperationAction(ISD::ConstantPool, VT, Custom);
417 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
418 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
419 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
420 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
423 setOperationAction(ISD::AND, MVT::v16i8, Custom);
424 setOperationAction(ISD::OR, MVT::v16i8, Custom);
425 setOperationAction(ISD::XOR, MVT::v16i8, Custom);
426 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Custom);
428 setOperationAction(ISD::FDIV, MVT::v4f32, Legal);
430 setShiftAmountType(MVT::i32);
431 setBooleanContents(ZeroOrNegativeOneBooleanContent);
433 setStackPointerRegisterToSaveRestore(SPU::R1);
435 // We have target-specific dag combine patterns for the following nodes:
436 setTargetDAGCombine(ISD::ADD);
437 setTargetDAGCombine(ISD::ZERO_EXTEND);
438 setTargetDAGCombine(ISD::SIGN_EXTEND);
439 setTargetDAGCombine(ISD::ANY_EXTEND);
441 computeRegisterProperties();
443 // Set pre-RA register scheduler default to BURR, which produces slightly
444 // better code than the default (could also be TDRR, but TargetLowering.h
445 // needs a mod to support that model):
446 setSchedulingPreference(SchedulingForRegPressure);
450 SPUTargetLowering::getTargetNodeName(unsigned Opcode) const
452 if (node_names.empty()) {
453 node_names[(unsigned) SPUISD::RET_FLAG] = "SPUISD::RET_FLAG";
454 node_names[(unsigned) SPUISD::Hi] = "SPUISD::Hi";
455 node_names[(unsigned) SPUISD::Lo] = "SPUISD::Lo";
456 node_names[(unsigned) SPUISD::PCRelAddr] = "SPUISD::PCRelAddr";
457 node_names[(unsigned) SPUISD::AFormAddr] = "SPUISD::AFormAddr";
458 node_names[(unsigned) SPUISD::IndirectAddr] = "SPUISD::IndirectAddr";
459 node_names[(unsigned) SPUISD::LDRESULT] = "SPUISD::LDRESULT";
460 node_names[(unsigned) SPUISD::CALL] = "SPUISD::CALL";
461 node_names[(unsigned) SPUISD::SHUFB] = "SPUISD::SHUFB";
462 node_names[(unsigned) SPUISD::SHUFFLE_MASK] = "SPUISD::SHUFFLE_MASK";
463 node_names[(unsigned) SPUISD::CNTB] = "SPUISD::CNTB";
464 node_names[(unsigned) SPUISD::PREFSLOT2VEC] = "SPUISD::PREFSLOT2VEC";
465 node_names[(unsigned) SPUISD::VEC2PREFSLOT] = "SPUISD::VEC2PREFSLOT";
466 node_names[(unsigned) SPUISD::SHLQUAD_L_BITS] = "SPUISD::SHLQUAD_L_BITS";
467 node_names[(unsigned) SPUISD::SHLQUAD_L_BYTES] = "SPUISD::SHLQUAD_L_BYTES";
468 node_names[(unsigned) SPUISD::VEC_SHL] = "SPUISD::VEC_SHL";
469 node_names[(unsigned) SPUISD::VEC_SRL] = "SPUISD::VEC_SRL";
470 node_names[(unsigned) SPUISD::VEC_SRA] = "SPUISD::VEC_SRA";
471 node_names[(unsigned) SPUISD::VEC_ROTL] = "SPUISD::VEC_ROTL";
472 node_names[(unsigned) SPUISD::VEC_ROTR] = "SPUISD::VEC_ROTR";
473 node_names[(unsigned) SPUISD::ROTBYTES_LEFT] = "SPUISD::ROTBYTES_LEFT";
474 node_names[(unsigned) SPUISD::ROTBYTES_LEFT_BITS] =
475 "SPUISD::ROTBYTES_LEFT_BITS";
476 node_names[(unsigned) SPUISD::SELECT_MASK] = "SPUISD::SELECT_MASK";
477 node_names[(unsigned) SPUISD::SELB] = "SPUISD::SELB";
478 node_names[(unsigned) SPUISD::ADD64_MARKER] = "SPUISD::ADD64_MARKER";
479 node_names[(unsigned) SPUISD::SUB64_MARKER] = "SPUISD::SUB64_MARKER";
480 node_names[(unsigned) SPUISD::MUL64_MARKER] = "SPUISD::MUL64_MARKER";
483 std::map<unsigned, const char *>::iterator i = node_names.find(Opcode);
485 return ((i != node_names.end()) ? i->second : 0);
488 /// getFunctionAlignment - Return the Log2 alignment of this function.
489 unsigned SPUTargetLowering::getFunctionAlignment(const Function *) const {
493 //===----------------------------------------------------------------------===//
494 // Return the Cell SPU's SETCC result type
495 //===----------------------------------------------------------------------===//
497 MVT SPUTargetLowering::getSetCCResultType(MVT VT) const {
498 // i16 and i32 are valid SETCC result types
499 return ((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) ? VT : MVT::i32);
502 //===----------------------------------------------------------------------===//
503 // Calling convention code:
504 //===----------------------------------------------------------------------===//
506 #include "SPUGenCallingConv.inc"
508 //===----------------------------------------------------------------------===//
509 // LowerOperation implementation
510 //===----------------------------------------------------------------------===//
512 /// Custom lower loads for CellSPU
514 All CellSPU loads and stores are aligned to 16-byte boundaries, so for elements
515 within a 16-byte block, we have to rotate to extract the requested element.
517 For extending loads, we also want to ensure that the following sequence is
518 emitted, e.g. for MVT::f32 extending load to MVT::f64:
522 %2 v16i8,ch = rotate %1
523 %3 v4f8, ch = bitconvert %2
524 %4 f32 = vec2perfslot %3
525 %5 f64 = fp_extend %4
529 LowerLOAD(SDValue Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
530 LoadSDNode *LN = cast<LoadSDNode>(Op);
531 SDValue the_chain = LN->getChain();
532 MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
533 MVT InVT = LN->getMemoryVT();
534 MVT OutVT = Op.getValueType();
535 ISD::LoadExtType ExtType = LN->getExtensionType();
536 unsigned alignment = LN->getAlignment();
537 const valtype_map_s *vtm = getValueTypeMapEntry(InVT);
538 DebugLoc dl = Op.getDebugLoc();
540 switch (LN->getAddressingMode()) {
541 case ISD::UNINDEXED: {
543 SDValue basePtr = LN->getBasePtr();
546 if (alignment == 16) {
549 // Special cases for a known aligned load to simplify the base pointer
550 // and the rotation amount:
551 if (basePtr.getOpcode() == ISD::ADD
552 && (CN = dyn_cast<ConstantSDNode > (basePtr.getOperand(1))) != 0) {
553 // Known offset into basePtr
554 int64_t offset = CN->getSExtValue();
555 int64_t rotamt = int64_t((offset & 0xf) - vtm->prefslot_byte);
560 rotate = DAG.getConstant(rotamt, MVT::i16);
562 // Simplify the base pointer for this case:
563 basePtr = basePtr.getOperand(0);
564 if ((offset & ~0xf) > 0) {
565 basePtr = DAG.getNode(SPUISD::IndirectAddr, dl, PtrVT,
567 DAG.getConstant((offset & ~0xf), PtrVT));
569 } else if ((basePtr.getOpcode() == SPUISD::AFormAddr)
570 || (basePtr.getOpcode() == SPUISD::IndirectAddr
571 && basePtr.getOperand(0).getOpcode() == SPUISD::Hi
572 && basePtr.getOperand(1).getOpcode() == SPUISD::Lo)) {
573 // Plain aligned a-form address: rotate into preferred slot
574 // Same for (SPUindirect (SPUhi ...), (SPUlo ...))
575 int64_t rotamt = -vtm->prefslot_byte;
578 rotate = DAG.getConstant(rotamt, MVT::i16);
580 // Offset the rotate amount by the basePtr and the preferred slot
582 int64_t rotamt = -vtm->prefslot_byte;
585 rotate = DAG.getNode(ISD::ADD, dl, PtrVT,
587 DAG.getConstant(rotamt, PtrVT));
590 // Unaligned load: must be more pessimistic about addressing modes:
591 if (basePtr.getOpcode() == ISD::ADD) {
592 MachineFunction &MF = DAG.getMachineFunction();
593 MachineRegisterInfo &RegInfo = MF.getRegInfo();
594 unsigned VReg = RegInfo.createVirtualRegister(&SPU::R32CRegClass);
597 SDValue Op0 = basePtr.getOperand(0);
598 SDValue Op1 = basePtr.getOperand(1);
600 if (isa<ConstantSDNode>(Op1)) {
601 // Convert the (add <ptr>, <const>) to an indirect address contained
602 // in a register. Note that this is done because we need to avoid
603 // creating a 0(reg) d-form address due to the SPU's block loads.
604 basePtr = DAG.getNode(SPUISD::IndirectAddr, dl, PtrVT, Op0, Op1);
605 the_chain = DAG.getCopyToReg(the_chain, dl, VReg, basePtr, Flag);
606 basePtr = DAG.getCopyFromReg(the_chain, dl, VReg, PtrVT);
608 // Convert the (add <arg1>, <arg2>) to an indirect address, which
609 // will likely be lowered as a reg(reg) x-form address.
610 basePtr = DAG.getNode(SPUISD::IndirectAddr, dl, PtrVT, Op0, Op1);
613 basePtr = DAG.getNode(SPUISD::IndirectAddr, dl, PtrVT,
615 DAG.getConstant(0, PtrVT));
618 // Offset the rotate amount by the basePtr and the preferred slot
620 rotate = DAG.getNode(ISD::ADD, dl, PtrVT,
622 DAG.getConstant(-vtm->prefslot_byte, PtrVT));
625 // Re-emit as a v16i8 vector load
626 result = DAG.getLoad(MVT::v16i8, dl, the_chain, basePtr,
627 LN->getSrcValue(), LN->getSrcValueOffset(),
628 LN->isVolatile(), 16);
631 the_chain = result.getValue(1);
633 // Rotate into the preferred slot:
634 result = DAG.getNode(SPUISD::ROTBYTES_LEFT, dl, MVT::v16i8,
635 result.getValue(0), rotate);
637 // Convert the loaded v16i8 vector to the appropriate vector type
638 // specified by the operand:
639 MVT vecVT = MVT::getVectorVT(InVT, (128 / InVT.getSizeInBits()));
640 result = DAG.getNode(SPUISD::VEC2PREFSLOT, dl, InVT,
641 DAG.getNode(ISD::BIT_CONVERT, dl, vecVT, result));
643 // Handle extending loads by extending the scalar result:
644 if (ExtType == ISD::SEXTLOAD) {
645 result = DAG.getNode(ISD::SIGN_EXTEND, dl, OutVT, result);
646 } else if (ExtType == ISD::ZEXTLOAD) {
647 result = DAG.getNode(ISD::ZERO_EXTEND, dl, OutVT, result);
648 } else if (ExtType == ISD::EXTLOAD) {
649 unsigned NewOpc = ISD::ANY_EXTEND;
651 if (OutVT.isFloatingPoint())
652 NewOpc = ISD::FP_EXTEND;
654 result = DAG.getNode(NewOpc, dl, OutVT, result);
657 SDVTList retvts = DAG.getVTList(OutVT, MVT::Other);
658 SDValue retops[2] = {
663 result = DAG.getNode(SPUISD::LDRESULT, dl, retvts,
664 retops, sizeof(retops) / sizeof(retops[0]));
671 case ISD::LAST_INDEXED_MODE:
674 raw_string_ostream Msg(msg);
675 Msg << "LowerLOAD: Got a LoadSDNode with an addr mode other than "
677 Msg << (unsigned) LN->getAddressingMode();
678 llvm_report_error(Msg.str());
686 /// Custom lower stores for CellSPU
688 All CellSPU stores are aligned to 16-byte boundaries, so for elements
689 within a 16-byte block, we have to generate a shuffle to insert the
690 requested element into its place, then store the resulting block.
693 LowerSTORE(SDValue Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
694 StoreSDNode *SN = cast<StoreSDNode>(Op);
695 SDValue Value = SN->getValue();
696 MVT VT = Value.getValueType();
697 MVT StVT = (!SN->isTruncatingStore() ? VT : SN->getMemoryVT());
698 MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
699 DebugLoc dl = Op.getDebugLoc();
700 unsigned alignment = SN->getAlignment();
702 switch (SN->getAddressingMode()) {
703 case ISD::UNINDEXED: {
704 // The vector type we really want to load from the 16-byte chunk.
705 MVT vecVT = MVT::getVectorVT(VT, (128 / VT.getSizeInBits())),
706 stVecVT = MVT::getVectorVT(StVT, (128 / StVT.getSizeInBits()));
708 SDValue alignLoadVec;
709 SDValue basePtr = SN->getBasePtr();
710 SDValue the_chain = SN->getChain();
711 SDValue insertEltOffs;
713 if (alignment == 16) {
716 // Special cases for a known aligned load to simplify the base pointer
717 // and insertion byte:
718 if (basePtr.getOpcode() == ISD::ADD
719 && (CN = dyn_cast<ConstantSDNode>(basePtr.getOperand(1))) != 0) {
720 // Known offset into basePtr
721 int64_t offset = CN->getSExtValue();
723 // Simplify the base pointer for this case:
724 basePtr = basePtr.getOperand(0);
725 insertEltOffs = DAG.getNode(SPUISD::IndirectAddr, dl, PtrVT,
727 DAG.getConstant((offset & 0xf), PtrVT));
729 if ((offset & ~0xf) > 0) {
730 basePtr = DAG.getNode(SPUISD::IndirectAddr, dl, PtrVT,
732 DAG.getConstant((offset & ~0xf), PtrVT));
735 // Otherwise, assume it's at byte 0 of basePtr
736 insertEltOffs = DAG.getNode(SPUISD::IndirectAddr, dl, PtrVT,
738 DAG.getConstant(0, PtrVT));
741 // Unaligned load: must be more pessimistic about addressing modes:
742 if (basePtr.getOpcode() == ISD::ADD) {
743 MachineFunction &MF = DAG.getMachineFunction();
744 MachineRegisterInfo &RegInfo = MF.getRegInfo();
745 unsigned VReg = RegInfo.createVirtualRegister(&SPU::R32CRegClass);
748 SDValue Op0 = basePtr.getOperand(0);
749 SDValue Op1 = basePtr.getOperand(1);
751 if (isa<ConstantSDNode>(Op1)) {
752 // Convert the (add <ptr>, <const>) to an indirect address contained
753 // in a register. Note that this is done because we need to avoid
754 // creating a 0(reg) d-form address due to the SPU's block loads.
755 basePtr = DAG.getNode(SPUISD::IndirectAddr, dl, PtrVT, Op0, Op1);
756 the_chain = DAG.getCopyToReg(the_chain, dl, VReg, basePtr, Flag);
757 basePtr = DAG.getCopyFromReg(the_chain, dl, VReg, PtrVT);
759 // Convert the (add <arg1>, <arg2>) to an indirect address, which
760 // will likely be lowered as a reg(reg) x-form address.
761 basePtr = DAG.getNode(SPUISD::IndirectAddr, dl, PtrVT, Op0, Op1);
764 basePtr = DAG.getNode(SPUISD::IndirectAddr, dl, PtrVT,
766 DAG.getConstant(0, PtrVT));
769 // Insertion point is solely determined by basePtr's contents
770 insertEltOffs = DAG.getNode(ISD::ADD, dl, PtrVT,
772 DAG.getConstant(0, PtrVT));
775 // Re-emit as a v16i8 vector load
776 alignLoadVec = DAG.getLoad(MVT::v16i8, dl, the_chain, basePtr,
777 SN->getSrcValue(), SN->getSrcValueOffset(),
778 SN->isVolatile(), 16);
781 the_chain = alignLoadVec.getValue(1);
783 LoadSDNode *LN = cast<LoadSDNode>(alignLoadVec);
784 SDValue theValue = SN->getValue();
788 && (theValue.getOpcode() == ISD::AssertZext
789 || theValue.getOpcode() == ISD::AssertSext)) {
790 // Drill down and get the value for zero- and sign-extended
792 theValue = theValue.getOperand(0);
795 // If the base pointer is already a D-form address, then just create
796 // a new D-form address with a slot offset and the orignal base pointer.
797 // Otherwise generate a D-form address with the slot offset relative
798 // to the stack pointer, which is always aligned.
800 if (DebugFlag && isCurrentDebugType(DEBUG_TYPE)) {
801 cerr << "CellSPU LowerSTORE: basePtr = ";
802 basePtr.getNode()->dump(&DAG);
807 SDValue insertEltOp =
808 DAG.getNode(SPUISD::SHUFFLE_MASK, dl, vecVT, insertEltOffs);
809 SDValue vectorizeOp =
810 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, vecVT, theValue);
812 result = DAG.getNode(SPUISD::SHUFB, dl, vecVT,
813 vectorizeOp, alignLoadVec,
814 DAG.getNode(ISD::BIT_CONVERT, dl,
815 MVT::v4i32, insertEltOp));
817 result = DAG.getStore(the_chain, dl, result, basePtr,
818 LN->getSrcValue(), LN->getSrcValueOffset(),
819 LN->isVolatile(), LN->getAlignment());
821 #if 0 && !defined(NDEBUG)
822 if (DebugFlag && isCurrentDebugType(DEBUG_TYPE)) {
823 const SDValue ¤tRoot = DAG.getRoot();
826 cerr << "------- CellSPU:LowerStore result:\n";
829 DAG.setRoot(currentRoot);
840 case ISD::LAST_INDEXED_MODE:
843 raw_string_ostream Msg(msg);
844 Msg << "LowerLOAD: Got a LoadSDNode with an addr mode other than "
846 Msg << (unsigned) SN->getAddressingMode();
847 llvm_report_error(Msg.str());
855 //! Generate the address of a constant pool entry.
857 LowerConstantPool(SDValue Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
858 MVT PtrVT = Op.getValueType();
859 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
860 Constant *C = CP->getConstVal();
861 SDValue CPI = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment());
862 SDValue Zero = DAG.getConstant(0, PtrVT);
863 const TargetMachine &TM = DAG.getTarget();
864 // FIXME there is no actual debug info here
865 DebugLoc dl = Op.getDebugLoc();
867 if (TM.getRelocationModel() == Reloc::Static) {
868 if (!ST->usingLargeMem()) {
869 // Just return the SDValue with the constant pool address in it.
870 return DAG.getNode(SPUISD::AFormAddr, dl, PtrVT, CPI, Zero);
872 SDValue Hi = DAG.getNode(SPUISD::Hi, dl, PtrVT, CPI, Zero);
873 SDValue Lo = DAG.getNode(SPUISD::Lo, dl, PtrVT, CPI, Zero);
874 return DAG.getNode(SPUISD::IndirectAddr, dl, PtrVT, Hi, Lo);
879 "LowerConstantPool: Relocation model other than static"
884 //! Alternate entry point for generating the address of a constant pool entry
886 SPU::LowerConstantPool(SDValue Op, SelectionDAG &DAG, const SPUTargetMachine &TM) {
887 return ::LowerConstantPool(Op, DAG, TM.getSubtargetImpl());
891 LowerJumpTable(SDValue Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
892 MVT PtrVT = Op.getValueType();
893 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
894 SDValue JTI = DAG.getTargetJumpTable(JT->getIndex(), PtrVT);
895 SDValue Zero = DAG.getConstant(0, PtrVT);
896 const TargetMachine &TM = DAG.getTarget();
897 // FIXME there is no actual debug info here
898 DebugLoc dl = Op.getDebugLoc();
900 if (TM.getRelocationModel() == Reloc::Static) {
901 if (!ST->usingLargeMem()) {
902 return DAG.getNode(SPUISD::AFormAddr, dl, PtrVT, JTI, Zero);
904 SDValue Hi = DAG.getNode(SPUISD::Hi, dl, PtrVT, JTI, Zero);
905 SDValue Lo = DAG.getNode(SPUISD::Lo, dl, PtrVT, JTI, Zero);
906 return DAG.getNode(SPUISD::IndirectAddr, dl, PtrVT, Hi, Lo);
911 "LowerJumpTable: Relocation model other than static not supported.");
916 LowerGlobalAddress(SDValue Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
917 MVT PtrVT = Op.getValueType();
918 GlobalAddressSDNode *GSDN = cast<GlobalAddressSDNode>(Op);
919 GlobalValue *GV = GSDN->getGlobal();
920 SDValue GA = DAG.getTargetGlobalAddress(GV, PtrVT, GSDN->getOffset());
921 const TargetMachine &TM = DAG.getTarget();
922 SDValue Zero = DAG.getConstant(0, PtrVT);
923 // FIXME there is no actual debug info here
924 DebugLoc dl = Op.getDebugLoc();
926 if (TM.getRelocationModel() == Reloc::Static) {
927 if (!ST->usingLargeMem()) {
928 return DAG.getNode(SPUISD::AFormAddr, dl, PtrVT, GA, Zero);
930 SDValue Hi = DAG.getNode(SPUISD::Hi, dl, PtrVT, GA, Zero);
931 SDValue Lo = DAG.getNode(SPUISD::Lo, dl, PtrVT, GA, Zero);
932 return DAG.getNode(SPUISD::IndirectAddr, dl, PtrVT, Hi, Lo);
935 llvm_report_error("LowerGlobalAddress: Relocation model other than static"
943 //! Custom lower double precision floating point constants
945 LowerConstantFP(SDValue Op, SelectionDAG &DAG) {
946 MVT VT = Op.getValueType();
947 // FIXME there is no actual debug info here
948 DebugLoc dl = Op.getDebugLoc();
950 if (VT == MVT::f64) {
951 ConstantFPSDNode *FP = cast<ConstantFPSDNode>(Op.getNode());
954 "LowerConstantFP: Node is not ConstantFPSDNode");
956 uint64_t dbits = DoubleToBits(FP->getValueAPF().convertToDouble());
957 SDValue T = DAG.getConstant(dbits, MVT::i64);
958 SDValue Tvec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2i64, T, T);
959 return DAG.getNode(SPUISD::VEC2PREFSLOT, dl, VT,
960 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2f64, Tvec));
967 LowerFORMAL_ARGUMENTS(SDValue Op, SelectionDAG &DAG, int &VarArgsFrameIndex)
969 MachineFunction &MF = DAG.getMachineFunction();
970 MachineFrameInfo *MFI = MF.getFrameInfo();
971 MachineRegisterInfo &RegInfo = MF.getRegInfo();
972 SmallVector<SDValue, 48> ArgValues;
973 SDValue Root = Op.getOperand(0);
974 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue() != 0;
975 DebugLoc dl = Op.getDebugLoc();
977 const unsigned *ArgRegs = SPURegisterInfo::getArgRegs();
978 const unsigned NumArgRegs = SPURegisterInfo::getNumArgRegs();
980 unsigned ArgOffset = SPUFrameInfo::minStackSize();
981 unsigned ArgRegIdx = 0;
982 unsigned StackSlotSize = SPUFrameInfo::stackSlotSize();
984 MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
986 // Add DAG nodes to load the arguments or copy them out of registers.
987 for (unsigned ArgNo = 0, e = Op.getNode()->getNumValues() - 1;
988 ArgNo != e; ++ArgNo) {
989 MVT ObjectVT = Op.getValue(ArgNo).getValueType();
990 unsigned ObjSize = ObjectVT.getSizeInBits()/8;
993 if (ArgRegIdx < NumArgRegs) {
994 const TargetRegisterClass *ArgRegClass;
996 switch (ObjectVT.getSimpleVT()) {
999 raw_string_ostream Msg(msg);
1000 Msg << "LowerFORMAL_ARGUMENTS Unhandled argument type: "
1001 << ObjectVT.getMVTString();
1002 llvm_report_error(Msg.str());
1005 ArgRegClass = &SPU::R8CRegClass;
1008 ArgRegClass = &SPU::R16CRegClass;
1011 ArgRegClass = &SPU::R32CRegClass;
1014 ArgRegClass = &SPU::R64CRegClass;
1017 ArgRegClass = &SPU::GPRCRegClass;
1020 ArgRegClass = &SPU::R32FPRegClass;
1023 ArgRegClass = &SPU::R64FPRegClass;
1031 ArgRegClass = &SPU::VECREGRegClass;
1035 unsigned VReg = RegInfo.createVirtualRegister(ArgRegClass);
1036 RegInfo.addLiveIn(ArgRegs[ArgRegIdx], VReg);
1037 ArgVal = DAG.getCopyFromReg(Root, dl, VReg, ObjectVT);
1040 // We need to load the argument to a virtual register if we determined
1041 // above that we ran out of physical registers of the appropriate type
1042 // or we're forced to do vararg
1043 int FI = MFI->CreateFixedObject(ObjSize, ArgOffset);
1044 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
1045 ArgVal = DAG.getLoad(ObjectVT, dl, Root, FIN, NULL, 0);
1046 ArgOffset += StackSlotSize;
1049 ArgValues.push_back(ArgVal);
1051 Root = ArgVal.getOperand(0);
1056 // unsigned int ptr_size = PtrVT.getSizeInBits() / 8;
1057 // We will spill (79-3)+1 registers to the stack
1058 SmallVector<SDValue, 79-3+1> MemOps;
1060 // Create the frame slot
1062 for (; ArgRegIdx != NumArgRegs; ++ArgRegIdx) {
1063 VarArgsFrameIndex = MFI->CreateFixedObject(StackSlotSize, ArgOffset);
1064 SDValue FIN = DAG.getFrameIndex(VarArgsFrameIndex, PtrVT);
1065 SDValue ArgVal = DAG.getRegister(ArgRegs[ArgRegIdx], MVT::v16i8);
1066 SDValue Store = DAG.getStore(Root, dl, ArgVal, FIN, NULL, 0);
1067 Root = Store.getOperand(0);
1068 MemOps.push_back(Store);
1070 // Increment address by stack slot size for the next stored argument
1071 ArgOffset += StackSlotSize;
1073 if (!MemOps.empty())
1074 Root = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1075 &MemOps[0], MemOps.size());
1078 ArgValues.push_back(Root);
1080 // Return the new list of results.
1081 return DAG.getNode(ISD::MERGE_VALUES, dl, Op.getNode()->getVTList(),
1082 &ArgValues[0], ArgValues.size());
1085 /// isLSAAddress - Return the immediate to use if the specified
1086 /// value is representable as a LSA address.
1087 static SDNode *isLSAAddress(SDValue Op, SelectionDAG &DAG) {
1088 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op);
1091 int Addr = C->getZExtValue();
1092 if ((Addr & 3) != 0 || // Low 2 bits are implicitly zero.
1093 (Addr << 14 >> 14) != Addr)
1094 return 0; // Top 14 bits have to be sext of immediate.
1096 return DAG.getConstant((int)C->getZExtValue() >> 2, MVT::i32).getNode();
1100 LowerCALL(SDValue Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
1101 CallSDNode *TheCall = cast<CallSDNode>(Op.getNode());
1102 SDValue Chain = TheCall->getChain();
1103 SDValue Callee = TheCall->getCallee();
1104 unsigned NumOps = TheCall->getNumArgs();
1105 unsigned StackSlotSize = SPUFrameInfo::stackSlotSize();
1106 const unsigned *ArgRegs = SPURegisterInfo::getArgRegs();
1107 const unsigned NumArgRegs = SPURegisterInfo::getNumArgRegs();
1108 DebugLoc dl = TheCall->getDebugLoc();
1110 // Handy pointer type
1111 MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
1113 // Accumulate how many bytes are to be pushed on the stack, including the
1114 // linkage area, and parameter passing area. According to the SPU ABI,
1115 // we minimally need space for [LR] and [SP]
1116 unsigned NumStackBytes = SPUFrameInfo::minStackSize();
1118 // Set up a copy of the stack pointer for use loading and storing any
1119 // arguments that may not fit in the registers available for argument
1121 SDValue StackPtr = DAG.getRegister(SPU::R1, MVT::i32);
1123 // Figure out which arguments are going to go in registers, and which in
1125 unsigned ArgOffset = SPUFrameInfo::minStackSize(); // Just below [LR]
1126 unsigned ArgRegIdx = 0;
1128 // Keep track of registers passing arguments
1129 std::vector<std::pair<unsigned, SDValue> > RegsToPass;
1130 // And the arguments passed on the stack
1131 SmallVector<SDValue, 8> MemOpChains;
1133 for (unsigned i = 0; i != NumOps; ++i) {
1134 SDValue Arg = TheCall->getArg(i);
1136 // PtrOff will be used to store the current argument to the stack if a
1137 // register cannot be found for it.
1138 SDValue PtrOff = DAG.getConstant(ArgOffset, StackPtr.getValueType());
1139 PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
1141 switch (Arg.getValueType().getSimpleVT()) {
1142 default: assert(0 && "Unexpected ValueType for argument!");
1148 if (ArgRegIdx != NumArgRegs) {
1149 RegsToPass.push_back(std::make_pair(ArgRegs[ArgRegIdx++], Arg));
1151 MemOpChains.push_back(DAG.getStore(Chain, dl, Arg, PtrOff, NULL, 0));
1152 ArgOffset += StackSlotSize;
1157 if (ArgRegIdx != NumArgRegs) {
1158 RegsToPass.push_back(std::make_pair(ArgRegs[ArgRegIdx++], Arg));
1160 MemOpChains.push_back(DAG.getStore(Chain, dl, Arg, PtrOff, NULL, 0));
1161 ArgOffset += StackSlotSize;
1170 if (ArgRegIdx != NumArgRegs) {
1171 RegsToPass.push_back(std::make_pair(ArgRegs[ArgRegIdx++], Arg));
1173 MemOpChains.push_back(DAG.getStore(Chain, dl, Arg, PtrOff, NULL, 0));
1174 ArgOffset += StackSlotSize;
1180 // Update number of stack bytes actually used, insert a call sequence start
1181 NumStackBytes = (ArgOffset - SPUFrameInfo::minStackSize());
1182 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumStackBytes,
1185 if (!MemOpChains.empty()) {
1186 // Adjust the stack pointer for the stack arguments.
1187 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1188 &MemOpChains[0], MemOpChains.size());
1191 // Build a sequence of copy-to-reg nodes chained together with token chain
1192 // and flag operands which copy the outgoing args into the appropriate regs.
1194 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
1195 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
1196 RegsToPass[i].second, InFlag);
1197 InFlag = Chain.getValue(1);
1200 SmallVector<SDValue, 8> Ops;
1201 unsigned CallOpc = SPUISD::CALL;
1203 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
1204 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
1205 // node so that legalize doesn't hack it.
1206 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
1207 GlobalValue *GV = G->getGlobal();
1208 MVT CalleeVT = Callee.getValueType();
1209 SDValue Zero = DAG.getConstant(0, PtrVT);
1210 SDValue GA = DAG.getTargetGlobalAddress(GV, CalleeVT);
1212 if (!ST->usingLargeMem()) {
1213 // Turn calls to targets that are defined (i.e., have bodies) into BRSL
1214 // style calls, otherwise, external symbols are BRASL calls. This assumes
1215 // that declared/defined symbols are in the same compilation unit and can
1216 // be reached through PC-relative jumps.
1219 // This may be an unsafe assumption for JIT and really large compilation
1221 if (GV->isDeclaration()) {
1222 Callee = DAG.getNode(SPUISD::AFormAddr, dl, CalleeVT, GA, Zero);
1224 Callee = DAG.getNode(SPUISD::PCRelAddr, dl, CalleeVT, GA, Zero);
1227 // "Large memory" mode: Turn all calls into indirect calls with a X-form
1229 Callee = DAG.getNode(SPUISD::IndirectAddr, dl, PtrVT, GA, Zero);
1231 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
1232 MVT CalleeVT = Callee.getValueType();
1233 SDValue Zero = DAG.getConstant(0, PtrVT);
1234 SDValue ExtSym = DAG.getTargetExternalSymbol(S->getSymbol(),
1235 Callee.getValueType());
1237 if (!ST->usingLargeMem()) {
1238 Callee = DAG.getNode(SPUISD::AFormAddr, dl, CalleeVT, ExtSym, Zero);
1240 Callee = DAG.getNode(SPUISD::IndirectAddr, dl, PtrVT, ExtSym, Zero);
1242 } else if (SDNode *Dest = isLSAAddress(Callee, DAG)) {
1243 // If this is an absolute destination address that appears to be a legal
1244 // local store address, use the munged value.
1245 Callee = SDValue(Dest, 0);
1248 Ops.push_back(Chain);
1249 Ops.push_back(Callee);
1251 // Add argument registers to the end of the list so that they are known live
1253 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
1254 Ops.push_back(DAG.getRegister(RegsToPass[i].first,
1255 RegsToPass[i].second.getValueType()));
1257 if (InFlag.getNode())
1258 Ops.push_back(InFlag);
1259 // Returns a chain and a flag for retval copy to use.
1260 Chain = DAG.getNode(CallOpc, dl, DAG.getVTList(MVT::Other, MVT::Flag),
1261 &Ops[0], Ops.size());
1262 InFlag = Chain.getValue(1);
1264 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumStackBytes, true),
1265 DAG.getIntPtrConstant(0, true), InFlag);
1266 if (TheCall->getValueType(0) != MVT::Other)
1267 InFlag = Chain.getValue(1);
1269 SDValue ResultVals[3];
1270 unsigned NumResults = 0;
1272 // If the call has results, copy the values out of the ret val registers.
1273 switch (TheCall->getValueType(0).getSimpleVT()) {
1274 default: assert(0 && "Unexpected ret value!");
1275 case MVT::Other: break;
1277 if (TheCall->getValueType(1) == MVT::i32) {
1278 Chain = DAG.getCopyFromReg(Chain, dl, SPU::R4,
1279 MVT::i32, InFlag).getValue(1);
1280 ResultVals[0] = Chain.getValue(0);
1281 Chain = DAG.getCopyFromReg(Chain, dl, SPU::R3, MVT::i32,
1282 Chain.getValue(2)).getValue(1);
1283 ResultVals[1] = Chain.getValue(0);
1286 Chain = DAG.getCopyFromReg(Chain, dl, SPU::R3, MVT::i32,
1287 InFlag).getValue(1);
1288 ResultVals[0] = Chain.getValue(0);
1293 Chain = DAG.getCopyFromReg(Chain, dl, SPU::R3, MVT::i64,
1294 InFlag).getValue(1);
1295 ResultVals[0] = Chain.getValue(0);
1299 Chain = DAG.getCopyFromReg(Chain, dl, SPU::R3, MVT::i128,
1300 InFlag).getValue(1);
1301 ResultVals[0] = Chain.getValue(0);
1306 Chain = DAG.getCopyFromReg(Chain, dl, SPU::R3, TheCall->getValueType(0),
1307 InFlag).getValue(1);
1308 ResultVals[0] = Chain.getValue(0);
1317 Chain = DAG.getCopyFromReg(Chain, dl, SPU::R3, TheCall->getValueType(0),
1318 InFlag).getValue(1);
1319 ResultVals[0] = Chain.getValue(0);
1324 // If the function returns void, just return the chain.
1325 if (NumResults == 0)
1328 // Otherwise, merge everything together with a MERGE_VALUES node.
1329 ResultVals[NumResults++] = Chain;
1330 SDValue Res = DAG.getMergeValues(ResultVals, NumResults, dl);
1331 return Res.getValue(Op.getResNo());
1335 LowerRET(SDValue Op, SelectionDAG &DAG, TargetMachine &TM) {
1336 SmallVector<CCValAssign, 16> RVLocs;
1337 unsigned CC = DAG.getMachineFunction().getFunction()->getCallingConv();
1338 bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg();
1339 DebugLoc dl = Op.getDebugLoc();
1340 CCState CCInfo(CC, isVarArg, TM, RVLocs, DAG.getContext());
1341 CCInfo.AnalyzeReturn(Op.getNode(), RetCC_SPU);
1343 // If this is the first return lowered for this function, add the regs to the
1344 // liveout set for the function.
1345 if (DAG.getMachineFunction().getRegInfo().liveout_empty()) {
1346 for (unsigned i = 0; i != RVLocs.size(); ++i)
1347 DAG.getMachineFunction().getRegInfo().addLiveOut(RVLocs[i].getLocReg());
1350 SDValue Chain = Op.getOperand(0);
1353 // Copy the result values into the output registers.
1354 for (unsigned i = 0; i != RVLocs.size(); ++i) {
1355 CCValAssign &VA = RVLocs[i];
1356 assert(VA.isRegLoc() && "Can only return in registers!");
1357 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
1358 Op.getOperand(i*2+1), Flag);
1359 Flag = Chain.getValue(1);
1363 return DAG.getNode(SPUISD::RET_FLAG, dl, MVT::Other, Chain, Flag);
1365 return DAG.getNode(SPUISD::RET_FLAG, dl, MVT::Other, Chain);
1369 //===----------------------------------------------------------------------===//
1370 // Vector related lowering:
1371 //===----------------------------------------------------------------------===//
1373 static ConstantSDNode *
1374 getVecImm(SDNode *N) {
1375 SDValue OpVal(0, 0);
1377 // Check to see if this buildvec has a single non-undef value in its elements.
1378 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
1379 if (N->getOperand(i).getOpcode() == ISD::UNDEF) continue;
1380 if (OpVal.getNode() == 0)
1381 OpVal = N->getOperand(i);
1382 else if (OpVal != N->getOperand(i))
1386 if (OpVal.getNode() != 0) {
1387 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) {
1395 /// get_vec_i18imm - Test if this vector is a vector filled with the same value
1396 /// and the value fits into an unsigned 18-bit constant, and if so, return the
1398 SDValue SPU::get_vec_u18imm(SDNode *N, SelectionDAG &DAG,
1400 if (ConstantSDNode *CN = getVecImm(N)) {
1401 uint64_t Value = CN->getZExtValue();
1402 if (ValueType == MVT::i64) {
1403 uint64_t UValue = CN->getZExtValue();
1404 uint32_t upper = uint32_t(UValue >> 32);
1405 uint32_t lower = uint32_t(UValue);
1408 Value = Value >> 32;
1410 if (Value <= 0x3ffff)
1411 return DAG.getTargetConstant(Value, ValueType);
1417 /// get_vec_i16imm - Test if this vector is a vector filled with the same value
1418 /// and the value fits into a signed 16-bit constant, and if so, return the
1420 SDValue SPU::get_vec_i16imm(SDNode *N, SelectionDAG &DAG,
1422 if (ConstantSDNode *CN = getVecImm(N)) {
1423 int64_t Value = CN->getSExtValue();
1424 if (ValueType == MVT::i64) {
1425 uint64_t UValue = CN->getZExtValue();
1426 uint32_t upper = uint32_t(UValue >> 32);
1427 uint32_t lower = uint32_t(UValue);
1430 Value = Value >> 32;
1432 if (Value >= -(1 << 15) && Value <= ((1 << 15) - 1)) {
1433 return DAG.getTargetConstant(Value, ValueType);
1440 /// get_vec_i10imm - Test if this vector is a vector filled with the same value
1441 /// and the value fits into a signed 10-bit constant, and if so, return the
1443 SDValue SPU::get_vec_i10imm(SDNode *N, SelectionDAG &DAG,
1445 if (ConstantSDNode *CN = getVecImm(N)) {
1446 int64_t Value = CN->getSExtValue();
1447 if (ValueType == MVT::i64) {
1448 uint64_t UValue = CN->getZExtValue();
1449 uint32_t upper = uint32_t(UValue >> 32);
1450 uint32_t lower = uint32_t(UValue);
1453 Value = Value >> 32;
1455 if (isS10Constant(Value))
1456 return DAG.getTargetConstant(Value, ValueType);
1462 /// get_vec_i8imm - Test if this vector is a vector filled with the same value
1463 /// and the value fits into a signed 8-bit constant, and if so, return the
1466 /// @note: The incoming vector is v16i8 because that's the only way we can load
1467 /// constant vectors. Thus, we test to see if the upper and lower bytes are the
1469 SDValue SPU::get_vec_i8imm(SDNode *N, SelectionDAG &DAG,
1471 if (ConstantSDNode *CN = getVecImm(N)) {
1472 int Value = (int) CN->getZExtValue();
1473 if (ValueType == MVT::i16
1474 && Value <= 0xffff /* truncated from uint64_t */
1475 && ((short) Value >> 8) == ((short) Value & 0xff))
1476 return DAG.getTargetConstant(Value & 0xff, ValueType);
1477 else if (ValueType == MVT::i8
1478 && (Value & 0xff) == Value)
1479 return DAG.getTargetConstant(Value, ValueType);
1485 /// get_ILHUvec_imm - Test if this vector is a vector filled with the same value
1486 /// and the value fits into a signed 16-bit constant, and if so, return the
1488 SDValue SPU::get_ILHUvec_imm(SDNode *N, SelectionDAG &DAG,
1490 if (ConstantSDNode *CN = getVecImm(N)) {
1491 uint64_t Value = CN->getZExtValue();
1492 if ((ValueType == MVT::i32
1493 && ((unsigned) Value & 0xffff0000) == (unsigned) Value)
1494 || (ValueType == MVT::i64 && (Value & 0xffff0000) == Value))
1495 return DAG.getTargetConstant(Value >> 16, ValueType);
1501 /// get_v4i32_imm - Catch-all for general 32-bit constant vectors
1502 SDValue SPU::get_v4i32_imm(SDNode *N, SelectionDAG &DAG) {
1503 if (ConstantSDNode *CN = getVecImm(N)) {
1504 return DAG.getTargetConstant((unsigned) CN->getZExtValue(), MVT::i32);
1510 /// get_v4i32_imm - Catch-all for general 64-bit constant vectors
1511 SDValue SPU::get_v2i64_imm(SDNode *N, SelectionDAG &DAG) {
1512 if (ConstantSDNode *CN = getVecImm(N)) {
1513 return DAG.getTargetConstant((unsigned) CN->getZExtValue(), MVT::i64);
1519 //! Lower a BUILD_VECTOR instruction creatively:
1521 LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) {
1522 MVT VT = Op.getValueType();
1523 MVT EltVT = VT.getVectorElementType();
1524 DebugLoc dl = Op.getDebugLoc();
1525 BuildVectorSDNode *BCN = dyn_cast<BuildVectorSDNode>(Op.getNode());
1526 assert(BCN != 0 && "Expected BuildVectorSDNode in SPU LowerBUILD_VECTOR");
1527 unsigned minSplatBits = EltVT.getSizeInBits();
1529 if (minSplatBits < 16)
1532 APInt APSplatBits, APSplatUndef;
1533 unsigned SplatBitSize;
1536 if (!BCN->isConstantSplat(APSplatBits, APSplatUndef, SplatBitSize,
1537 HasAnyUndefs, minSplatBits)
1538 || minSplatBits < SplatBitSize)
1539 return SDValue(); // Wasn't a constant vector or splat exceeded min
1541 uint64_t SplatBits = APSplatBits.getZExtValue();
1543 switch (VT.getSimpleVT()) {
1546 raw_string_ostream Msg(msg);
1547 Msg << "CellSPU: Unhandled VT in LowerBUILD_VECTOR, VT = "
1548 << VT.getMVTString();
1549 llvm_report_error(Msg.str());
1553 uint32_t Value32 = uint32_t(SplatBits);
1554 assert(SplatBitSize == 32
1555 && "LowerBUILD_VECTOR: Unexpected floating point vector element.");
1556 // NOTE: pretend the constant is an integer. LLVM won't load FP constants
1557 SDValue T = DAG.getConstant(Value32, MVT::i32);
1558 return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v4f32,
1559 DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, T,T,T,T));
1563 uint64_t f64val = uint64_t(SplatBits);
1564 assert(SplatBitSize == 64
1565 && "LowerBUILD_VECTOR: 64-bit float vector size > 8 bytes.");
1566 // NOTE: pretend the constant is an integer. LLVM won't load FP constants
1567 SDValue T = DAG.getConstant(f64val, MVT::i64);
1568 return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2f64,
1569 DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2i64, T, T));
1573 // 8-bit constants have to be expanded to 16-bits
1574 unsigned short Value16 = SplatBits /* | (SplatBits << 8) */;
1575 SmallVector<SDValue, 8> Ops;
1577 Ops.assign(8, DAG.getConstant(Value16, MVT::i16));
1578 return DAG.getNode(ISD::BIT_CONVERT, dl, VT,
1579 DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i16, &Ops[0], Ops.size()));
1582 unsigned short Value16 = SplatBits;
1583 SDValue T = DAG.getConstant(Value16, EltVT);
1584 SmallVector<SDValue, 8> Ops;
1587 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], Ops.size());
1590 SDValue T = DAG.getConstant(unsigned(SplatBits), VT.getVectorElementType());
1591 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, T, T, T, T);
1594 SDValue T = DAG.getConstant(unsigned(SplatBits), VT.getVectorElementType());
1595 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, T, T);
1598 return SPU::LowerV2I64Splat(VT, DAG, SplatBits, dl);
1608 SPU::LowerV2I64Splat(MVT OpVT, SelectionDAG& DAG, uint64_t SplatVal,
1610 uint32_t upper = uint32_t(SplatVal >> 32);
1611 uint32_t lower = uint32_t(SplatVal);
1613 if (upper == lower) {
1614 // Magic constant that can be matched by IL, ILA, et. al.
1615 SDValue Val = DAG.getTargetConstant(upper, MVT::i32);
1616 return DAG.getNode(ISD::BIT_CONVERT, dl, OpVT,
1617 DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32,
1618 Val, Val, Val, Val));
1620 bool upper_special, lower_special;
1622 // NOTE: This code creates common-case shuffle masks that can be easily
1623 // detected as common expressions. It is not attempting to create highly
1624 // specialized masks to replace any and all 0's, 0xff's and 0x80's.
1626 // Detect if the upper or lower half is a special shuffle mask pattern:
1627 upper_special = (upper == 0 || upper == 0xffffffff || upper == 0x80000000);
1628 lower_special = (lower == 0 || lower == 0xffffffff || lower == 0x80000000);
1630 // Both upper and lower are special, lower to a constant pool load:
1631 if (lower_special && upper_special) {
1632 SDValue SplatValCN = DAG.getConstant(SplatVal, MVT::i64);
1633 return DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2i64,
1634 SplatValCN, SplatValCN);
1639 SmallVector<SDValue, 16> ShufBytes;
1642 // Create lower vector if not a special pattern
1643 if (!lower_special) {
1644 SDValue LO32C = DAG.getConstant(lower, MVT::i32);
1645 LO32 = DAG.getNode(ISD::BIT_CONVERT, dl, OpVT,
1646 DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32,
1647 LO32C, LO32C, LO32C, LO32C));
1650 // Create upper vector if not a special pattern
1651 if (!upper_special) {
1652 SDValue HI32C = DAG.getConstant(upper, MVT::i32);
1653 HI32 = DAG.getNode(ISD::BIT_CONVERT, dl, OpVT,
1654 DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32,
1655 HI32C, HI32C, HI32C, HI32C));
1658 // If either upper or lower are special, then the two input operands are
1659 // the same (basically, one of them is a "don't care")
1665 for (int i = 0; i < 4; ++i) {
1667 for (int j = 0; j < 4; ++j) {
1669 bool process_upper, process_lower;
1671 process_upper = (upper_special && (i & 1) == 0);
1672 process_lower = (lower_special && (i & 1) == 1);
1674 if (process_upper || process_lower) {
1675 if ((process_upper && upper == 0)
1676 || (process_lower && lower == 0))
1678 else if ((process_upper && upper == 0xffffffff)
1679 || (process_lower && lower == 0xffffffff))
1681 else if ((process_upper && upper == 0x80000000)
1682 || (process_lower && lower == 0x80000000))
1683 val |= (j == 0 ? 0xe0 : 0x80);
1685 val |= i * 4 + j + ((i & 1) * 16);
1688 ShufBytes.push_back(DAG.getConstant(val, MVT::i32));
1691 return DAG.getNode(SPUISD::SHUFB, dl, OpVT, HI32, LO32,
1692 DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32,
1693 &ShufBytes[0], ShufBytes.size()));
1697 /// LowerVECTOR_SHUFFLE - Lower a vector shuffle (V1, V2, V3) to something on
1698 /// which the Cell can operate. The code inspects V3 to ascertain whether the
1699 /// permutation vector, V3, is monotonically increasing with one "exception"
1700 /// element, e.g., (0, 1, _, 3). If this is the case, then generate a
1701 /// SHUFFLE_MASK synthetic instruction. Otherwise, spill V3 to the constant pool.
1702 /// In either case, the net result is going to eventually invoke SHUFB to
1703 /// permute/shuffle the bytes from V1 and V2.
1705 /// SHUFFLE_MASK is eventually selected as one of the C*D instructions, generate
1706 /// control word for byte/halfword/word insertion. This takes care of a single
1707 /// element move from V2 into V1.
1709 /// SPUISD::SHUFB is eventually selected as Cell's <i>shufb</i> instructions.
1710 static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) {
1711 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
1712 SDValue V1 = Op.getOperand(0);
1713 SDValue V2 = Op.getOperand(1);
1714 DebugLoc dl = Op.getDebugLoc();
1716 if (V2.getOpcode() == ISD::UNDEF) V2 = V1;
1718 // If we have a single element being moved from V1 to V2, this can be handled
1719 // using the C*[DX] compute mask instructions, but the vector elements have
1720 // to be monotonically increasing with one exception element.
1721 MVT VecVT = V1.getValueType();
1722 MVT EltVT = VecVT.getVectorElementType();
1723 unsigned EltsFromV2 = 0;
1725 unsigned V2EltIdx0 = 0;
1726 unsigned CurrElt = 0;
1727 unsigned MaxElts = VecVT.getVectorNumElements();
1728 unsigned PrevElt = 0;
1730 bool monotonic = true;
1733 if (EltVT == MVT::i8) {
1735 } else if (EltVT == MVT::i16) {
1737 } else if (EltVT == MVT::i32 || EltVT == MVT::f32) {
1739 } else if (EltVT == MVT::i64 || EltVT == MVT::f64) {
1742 assert(0 && "Unhandled vector type in LowerVECTOR_SHUFFLE");
1744 for (unsigned i = 0; i != MaxElts; ++i) {
1745 if (SVN->getMaskElt(i) < 0)
1748 unsigned SrcElt = SVN->getMaskElt(i);
1751 if (SrcElt >= V2EltIdx0) {
1752 if (1 >= (++EltsFromV2)) {
1753 V2Elt = (V2EltIdx0 - SrcElt) << 2;
1755 } else if (CurrElt != SrcElt) {
1763 if (PrevElt > 0 && SrcElt < MaxElts) {
1764 if ((PrevElt == SrcElt - 1)
1765 || (PrevElt == MaxElts - 1 && SrcElt == 0)) {
1772 } else if (PrevElt == 0) {
1773 // First time through, need to keep track of previous element
1776 // This isn't a rotation, takes elements from vector 2
1782 if (EltsFromV2 == 1 && monotonic) {
1783 // Compute mask and shuffle
1784 MachineFunction &MF = DAG.getMachineFunction();
1785 MachineRegisterInfo &RegInfo = MF.getRegInfo();
1786 unsigned VReg = RegInfo.createVirtualRegister(&SPU::R32CRegClass);
1787 MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
1788 // Initialize temporary register to 0
1789 SDValue InitTempReg =
1790 DAG.getCopyToReg(DAG.getEntryNode(), dl, VReg, DAG.getConstant(0, PtrVT));
1791 // Copy register's contents as index in SHUFFLE_MASK:
1792 SDValue ShufMaskOp =
1793 DAG.getNode(SPUISD::SHUFFLE_MASK, dl, MVT::v4i32,
1794 DAG.getTargetConstant(V2Elt, MVT::i32),
1795 DAG.getCopyFromReg(InitTempReg, dl, VReg, PtrVT));
1796 // Use shuffle mask in SHUFB synthetic instruction:
1797 return DAG.getNode(SPUISD::SHUFB, dl, V1.getValueType(), V2, V1,
1799 } else if (rotate) {
1800 int rotamt = (MaxElts - V0Elt) * EltVT.getSizeInBits()/8;
1802 return DAG.getNode(SPUISD::ROTBYTES_LEFT, dl, V1.getValueType(),
1803 V1, DAG.getConstant(rotamt, MVT::i16));
1805 // Convert the SHUFFLE_VECTOR mask's input element units to the
1807 unsigned BytesPerElement = EltVT.getSizeInBits()/8;
1809 SmallVector<SDValue, 16> ResultMask;
1810 for (unsigned i = 0, e = MaxElts; i != e; ++i) {
1811 unsigned SrcElt = SVN->getMaskElt(i) < 0 ? 0 : SVN->getMaskElt(i);
1813 for (unsigned j = 0; j < BytesPerElement; ++j)
1814 ResultMask.push_back(DAG.getConstant(SrcElt*BytesPerElement+j,MVT::i8));
1817 SDValue VPermMask = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i8,
1818 &ResultMask[0], ResultMask.size());
1819 return DAG.getNode(SPUISD::SHUFB, dl, V1.getValueType(), V1, V2, VPermMask);
1823 static SDValue LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) {
1824 SDValue Op0 = Op.getOperand(0); // Op0 = the scalar
1825 DebugLoc dl = Op.getDebugLoc();
1827 if (Op0.getNode()->getOpcode() == ISD::Constant) {
1828 // For a constant, build the appropriate constant vector, which will
1829 // eventually simplify to a vector register load.
1831 ConstantSDNode *CN = cast<ConstantSDNode>(Op0.getNode());
1832 SmallVector<SDValue, 16> ConstVecValues;
1836 // Create a constant vector:
1837 switch (Op.getValueType().getSimpleVT()) {
1838 default: assert(0 && "Unexpected constant value type in "
1839 "LowerSCALAR_TO_VECTOR");
1840 case MVT::v16i8: n_copies = 16; VT = MVT::i8; break;
1841 case MVT::v8i16: n_copies = 8; VT = MVT::i16; break;
1842 case MVT::v4i32: n_copies = 4; VT = MVT::i32; break;
1843 case MVT::v4f32: n_copies = 4; VT = MVT::f32; break;
1844 case MVT::v2i64: n_copies = 2; VT = MVT::i64; break;
1845 case MVT::v2f64: n_copies = 2; VT = MVT::f64; break;
1848 SDValue CValue = DAG.getConstant(CN->getZExtValue(), VT);
1849 for (size_t j = 0; j < n_copies; ++j)
1850 ConstVecValues.push_back(CValue);
1852 return DAG.getNode(ISD::BUILD_VECTOR, dl, Op.getValueType(),
1853 &ConstVecValues[0], ConstVecValues.size());
1855 // Otherwise, copy the value from one register to another:
1856 switch (Op0.getValueType().getSimpleVT()) {
1857 default: assert(0 && "Unexpected value type in LowerSCALAR_TO_VECTOR");
1864 return DAG.getNode(SPUISD::PREFSLOT2VEC, dl, Op.getValueType(), Op0, Op0);
1871 static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) {
1872 MVT VT = Op.getValueType();
1873 SDValue N = Op.getOperand(0);
1874 SDValue Elt = Op.getOperand(1);
1875 DebugLoc dl = Op.getDebugLoc();
1878 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Elt)) {
1879 // Constant argument:
1880 int EltNo = (int) C->getZExtValue();
1883 if (VT == MVT::i8 && EltNo >= 16)
1884 assert(0 && "SPU LowerEXTRACT_VECTOR_ELT: i8 extraction slot > 15");
1885 else if (VT == MVT::i16 && EltNo >= 8)
1886 assert(0 && "SPU LowerEXTRACT_VECTOR_ELT: i16 extraction slot > 7");
1887 else if (VT == MVT::i32 && EltNo >= 4)
1888 assert(0 && "SPU LowerEXTRACT_VECTOR_ELT: i32 extraction slot > 4");
1889 else if (VT == MVT::i64 && EltNo >= 2)
1890 assert(0 && "SPU LowerEXTRACT_VECTOR_ELT: i64 extraction slot > 2");
1892 if (EltNo == 0 && (VT == MVT::i32 || VT == MVT::i64)) {
1893 // i32 and i64: Element 0 is the preferred slot
1894 return DAG.getNode(SPUISD::VEC2PREFSLOT, dl, VT, N);
1897 // Need to generate shuffle mask and extract:
1898 int prefslot_begin = -1, prefslot_end = -1;
1899 int elt_byte = EltNo * VT.getSizeInBits() / 8;
1901 switch (VT.getSimpleVT()) {
1903 assert(false && "Invalid value type!");
1905 prefslot_begin = prefslot_end = 3;
1909 prefslot_begin = 2; prefslot_end = 3;
1914 prefslot_begin = 0; prefslot_end = 3;
1919 prefslot_begin = 0; prefslot_end = 7;
1924 assert(prefslot_begin != -1 && prefslot_end != -1 &&
1925 "LowerEXTRACT_VECTOR_ELT: preferred slots uninitialized");
1927 unsigned int ShufBytes[16];
1928 for (int i = 0; i < 16; ++i) {
1929 // zero fill uppper part of preferred slot, don't care about the
1931 unsigned int mask_val;
1932 if (i <= prefslot_end) {
1934 ((i < prefslot_begin)
1936 : elt_byte + (i - prefslot_begin));
1938 ShufBytes[i] = mask_val;
1940 ShufBytes[i] = ShufBytes[i % (prefslot_end + 1)];
1943 SDValue ShufMask[4];
1944 for (unsigned i = 0; i < sizeof(ShufMask)/sizeof(ShufMask[0]); ++i) {
1945 unsigned bidx = i * 4;
1946 unsigned int bits = ((ShufBytes[bidx] << 24) |
1947 (ShufBytes[bidx+1] << 16) |
1948 (ShufBytes[bidx+2] << 8) |
1950 ShufMask[i] = DAG.getConstant(bits, MVT::i32);
1953 SDValue ShufMaskVec =
1954 DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32,
1955 &ShufMask[0], sizeof(ShufMask)/sizeof(ShufMask[0]));
1957 retval = DAG.getNode(SPUISD::VEC2PREFSLOT, dl, VT,
1958 DAG.getNode(SPUISD::SHUFB, dl, N.getValueType(),
1959 N, N, ShufMaskVec));
1961 // Variable index: Rotate the requested element into slot 0, then replicate
1962 // slot 0 across the vector
1963 MVT VecVT = N.getValueType();
1964 if (!VecVT.isSimple() || !VecVT.isVector() || !VecVT.is128BitVector()) {
1965 llvm_report_error("LowerEXTRACT_VECTOR_ELT: Must have a simple, 128-bit"
1969 // Make life easier by making sure the index is zero-extended to i32
1970 if (Elt.getValueType() != MVT::i32)
1971 Elt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Elt);
1973 // Scale the index to a bit/byte shift quantity
1975 APInt(32, uint64_t(16 / N.getValueType().getVectorNumElements()), false);
1976 unsigned scaleShift = scaleFactor.logBase2();
1979 if (scaleShift > 0) {
1980 // Scale the shift factor:
1981 Elt = DAG.getNode(ISD::SHL, dl, MVT::i32, Elt,
1982 DAG.getConstant(scaleShift, MVT::i32));
1985 vecShift = DAG.getNode(SPUISD::SHLQUAD_L_BYTES, dl, VecVT, N, Elt);
1987 // Replicate the bytes starting at byte 0 across the entire vector (for
1988 // consistency with the notion of a unified register set)
1991 switch (VT.getSimpleVT()) {
1993 llvm_report_error("LowerEXTRACT_VECTOR_ELT(varable): Unhandled vector"
1997 SDValue factor = DAG.getConstant(0x00000000, MVT::i32);
1998 replicate = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32,
1999 factor, factor, factor, factor);
2003 SDValue factor = DAG.getConstant(0x00010001, MVT::i32);
2004 replicate = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32,
2005 factor, factor, factor, factor);
2010 SDValue factor = DAG.getConstant(0x00010203, MVT::i32);
2011 replicate = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32,
2012 factor, factor, factor, factor);
2017 SDValue loFactor = DAG.getConstant(0x00010203, MVT::i32);
2018 SDValue hiFactor = DAG.getConstant(0x04050607, MVT::i32);
2019 replicate = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32,
2020 loFactor, hiFactor, loFactor, hiFactor);
2025 retval = DAG.getNode(SPUISD::VEC2PREFSLOT, dl, VT,
2026 DAG.getNode(SPUISD::SHUFB, dl, VecVT,
2027 vecShift, vecShift, replicate));
2033 static SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) {
2034 SDValue VecOp = Op.getOperand(0);
2035 SDValue ValOp = Op.getOperand(1);
2036 SDValue IdxOp = Op.getOperand(2);
2037 DebugLoc dl = Op.getDebugLoc();
2038 MVT VT = Op.getValueType();
2040 ConstantSDNode *CN = cast<ConstantSDNode>(IdxOp);
2041 assert(CN != 0 && "LowerINSERT_VECTOR_ELT: Index is not constant!");
2043 MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
2044 // Use $sp ($1) because it's always 16-byte aligned and it's available:
2045 SDValue Pointer = DAG.getNode(SPUISD::IndirectAddr, dl, PtrVT,
2046 DAG.getRegister(SPU::R1, PtrVT),
2047 DAG.getConstant(CN->getSExtValue(), PtrVT));
2048 SDValue ShufMask = DAG.getNode(SPUISD::SHUFFLE_MASK, dl, VT, Pointer);
2051 DAG.getNode(SPUISD::SHUFB, dl, VT,
2052 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, ValOp),
2054 DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v4i32, ShufMask));
2059 static SDValue LowerI8Math(SDValue Op, SelectionDAG &DAG, unsigned Opc,
2060 const TargetLowering &TLI)
2062 SDValue N0 = Op.getOperand(0); // Everything has at least one operand
2063 DebugLoc dl = Op.getDebugLoc();
2064 MVT ShiftVT = TLI.getShiftAmountTy();
2066 assert(Op.getValueType() == MVT::i8);
2069 assert(0 && "Unhandled i8 math operator");
2073 // 8-bit addition: Promote the arguments up to 16-bits and truncate
2075 SDValue N1 = Op.getOperand(1);
2076 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i16, N0);
2077 N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i16, N1);
2078 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8,
2079 DAG.getNode(Opc, dl, MVT::i16, N0, N1));
2084 // 8-bit subtraction: Promote the arguments up to 16-bits and truncate
2086 SDValue N1 = Op.getOperand(1);
2087 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i16, N0);
2088 N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i16, N1);
2089 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8,
2090 DAG.getNode(Opc, dl, MVT::i16, N0, N1));
2094 SDValue N1 = Op.getOperand(1);
2095 MVT N1VT = N1.getValueType();
2097 N0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, N0);
2098 if (!N1VT.bitsEq(ShiftVT)) {
2099 unsigned N1Opc = N1.getValueType().bitsLT(ShiftVT)
2102 N1 = DAG.getNode(N1Opc, dl, ShiftVT, N1);
2105 // Replicate lower 8-bits into upper 8:
2107 DAG.getNode(ISD::OR, dl, MVT::i16, N0,
2108 DAG.getNode(ISD::SHL, dl, MVT::i16,
2109 N0, DAG.getConstant(8, MVT::i32)));
2111 // Truncate back down to i8
2112 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8,
2113 DAG.getNode(Opc, dl, MVT::i16, ExpandArg, N1));
2117 SDValue N1 = Op.getOperand(1);
2118 MVT N1VT = N1.getValueType();
2120 N0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, N0);
2121 if (!N1VT.bitsEq(ShiftVT)) {
2122 unsigned N1Opc = ISD::ZERO_EXTEND;
2124 if (N1.getValueType().bitsGT(ShiftVT))
2125 N1Opc = ISD::TRUNCATE;
2127 N1 = DAG.getNode(N1Opc, dl, ShiftVT, N1);
2130 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8,
2131 DAG.getNode(Opc, dl, MVT::i16, N0, N1));
2134 SDValue N1 = Op.getOperand(1);
2135 MVT N1VT = N1.getValueType();
2137 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i16, N0);
2138 if (!N1VT.bitsEq(ShiftVT)) {
2139 unsigned N1Opc = ISD::SIGN_EXTEND;
2141 if (N1VT.bitsGT(ShiftVT))
2142 N1Opc = ISD::TRUNCATE;
2143 N1 = DAG.getNode(N1Opc, dl, ShiftVT, N1);
2146 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8,
2147 DAG.getNode(Opc, dl, MVT::i16, N0, N1));
2150 SDValue N1 = Op.getOperand(1);
2152 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i16, N0);
2153 N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i16, N1);
2154 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8,
2155 DAG.getNode(Opc, dl, MVT::i16, N0, N1));
2163 //! Lower byte immediate operations for v16i8 vectors:
2165 LowerByteImmed(SDValue Op, SelectionDAG &DAG) {
2168 MVT VT = Op.getValueType();
2169 DebugLoc dl = Op.getDebugLoc();
2171 ConstVec = Op.getOperand(0);
2172 Arg = Op.getOperand(1);
2173 if (ConstVec.getNode()->getOpcode() != ISD::BUILD_VECTOR) {
2174 if (ConstVec.getNode()->getOpcode() == ISD::BIT_CONVERT) {
2175 ConstVec = ConstVec.getOperand(0);
2177 ConstVec = Op.getOperand(1);
2178 Arg = Op.getOperand(0);
2179 if (ConstVec.getNode()->getOpcode() == ISD::BIT_CONVERT) {
2180 ConstVec = ConstVec.getOperand(0);
2185 if (ConstVec.getNode()->getOpcode() == ISD::BUILD_VECTOR) {
2186 BuildVectorSDNode *BCN = dyn_cast<BuildVectorSDNode>(ConstVec.getNode());
2187 assert(BCN != 0 && "Expected BuildVectorSDNode in SPU LowerByteImmed");
2189 APInt APSplatBits, APSplatUndef;
2190 unsigned SplatBitSize;
2192 unsigned minSplatBits = VT.getVectorElementType().getSizeInBits();
2194 if (BCN->isConstantSplat(APSplatBits, APSplatUndef, SplatBitSize,
2195 HasAnyUndefs, minSplatBits)
2196 && minSplatBits <= SplatBitSize) {
2197 uint64_t SplatBits = APSplatBits.getZExtValue();
2198 SDValue tc = DAG.getTargetConstant(SplatBits & 0xff, MVT::i8);
2200 SmallVector<SDValue, 16> tcVec;
2201 tcVec.assign(16, tc);
2202 return DAG.getNode(Op.getNode()->getOpcode(), dl, VT, Arg,
2203 DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &tcVec[0], tcVec.size()));
2207 // These operations (AND, OR, XOR) are legal, they just couldn't be custom
2208 // lowered. Return the operation, rather than a null SDValue.
2212 //! Custom lowering for CTPOP (count population)
2214 Custom lowering code that counts the number ones in the input
2215 operand. SPU has such an instruction, but it counts the number of
2216 ones per byte, which then have to be accumulated.
2218 static SDValue LowerCTPOP(SDValue Op, SelectionDAG &DAG) {
2219 MVT VT = Op.getValueType();
2220 MVT vecVT = MVT::getVectorVT(VT, (128 / VT.getSizeInBits()));
2221 DebugLoc dl = Op.getDebugLoc();
2223 switch (VT.getSimpleVT()) {
2225 assert(false && "Invalid value type!");
2227 SDValue N = Op.getOperand(0);
2228 SDValue Elt0 = DAG.getConstant(0, MVT::i32);
2230 SDValue Promote = DAG.getNode(SPUISD::PREFSLOT2VEC, dl, vecVT, N, N);
2231 SDValue CNTB = DAG.getNode(SPUISD::CNTB, dl, vecVT, Promote);
2233 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i8, CNTB, Elt0);
2237 MachineFunction &MF = DAG.getMachineFunction();
2238 MachineRegisterInfo &RegInfo = MF.getRegInfo();
2240 unsigned CNTB_reg = RegInfo.createVirtualRegister(&SPU::R16CRegClass);
2242 SDValue N = Op.getOperand(0);
2243 SDValue Elt0 = DAG.getConstant(0, MVT::i16);
2244 SDValue Mask0 = DAG.getConstant(0x0f, MVT::i16);
2245 SDValue Shift1 = DAG.getConstant(8, MVT::i32);
2247 SDValue Promote = DAG.getNode(SPUISD::PREFSLOT2VEC, dl, vecVT, N, N);
2248 SDValue CNTB = DAG.getNode(SPUISD::CNTB, dl, vecVT, Promote);
2250 // CNTB_result becomes the chain to which all of the virtual registers
2251 // CNTB_reg, SUM1_reg become associated:
2252 SDValue CNTB_result =
2253 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, CNTB, Elt0);
2255 SDValue CNTB_rescopy =
2256 DAG.getCopyToReg(CNTB_result, dl, CNTB_reg, CNTB_result);
2258 SDValue Tmp1 = DAG.getCopyFromReg(CNTB_rescopy, dl, CNTB_reg, MVT::i16);
2260 return DAG.getNode(ISD::AND, dl, MVT::i16,
2261 DAG.getNode(ISD::ADD, dl, MVT::i16,
2262 DAG.getNode(ISD::SRL, dl, MVT::i16,
2269 MachineFunction &MF = DAG.getMachineFunction();
2270 MachineRegisterInfo &RegInfo = MF.getRegInfo();
2272 unsigned CNTB_reg = RegInfo.createVirtualRegister(&SPU::R32CRegClass);
2273 unsigned SUM1_reg = RegInfo.createVirtualRegister(&SPU::R32CRegClass);
2275 SDValue N = Op.getOperand(0);
2276 SDValue Elt0 = DAG.getConstant(0, MVT::i32);
2277 SDValue Mask0 = DAG.getConstant(0xff, MVT::i32);
2278 SDValue Shift1 = DAG.getConstant(16, MVT::i32);
2279 SDValue Shift2 = DAG.getConstant(8, MVT::i32);
2281 SDValue Promote = DAG.getNode(SPUISD::PREFSLOT2VEC, dl, vecVT, N, N);
2282 SDValue CNTB = DAG.getNode(SPUISD::CNTB, dl, vecVT, Promote);
2284 // CNTB_result becomes the chain to which all of the virtual registers
2285 // CNTB_reg, SUM1_reg become associated:
2286 SDValue CNTB_result =
2287 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, CNTB, Elt0);
2289 SDValue CNTB_rescopy =
2290 DAG.getCopyToReg(CNTB_result, dl, CNTB_reg, CNTB_result);
2293 DAG.getNode(ISD::SRL, dl, MVT::i32,
2294 DAG.getCopyFromReg(CNTB_rescopy, dl, CNTB_reg, MVT::i32),
2298 DAG.getNode(ISD::ADD, dl, MVT::i32, Comp1,
2299 DAG.getCopyFromReg(CNTB_rescopy, dl, CNTB_reg, MVT::i32));
2301 SDValue Sum1_rescopy =
2302 DAG.getCopyToReg(CNTB_result, dl, SUM1_reg, Sum1);
2305 DAG.getNode(ISD::SRL, dl, MVT::i32,
2306 DAG.getCopyFromReg(Sum1_rescopy, dl, SUM1_reg, MVT::i32),
2309 DAG.getNode(ISD::ADD, dl, MVT::i32, Comp2,
2310 DAG.getCopyFromReg(Sum1_rescopy, dl, SUM1_reg, MVT::i32));
2312 return DAG.getNode(ISD::AND, dl, MVT::i32, Sum2, Mask0);
2322 //! Lower ISD::FP_TO_SINT, ISD::FP_TO_UINT for i32
2324 f32->i32 passes through unchanged, whereas f64->i32 expands to a libcall.
2325 All conversions to i64 are expanded to a libcall.
2327 static SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
2328 SPUTargetLowering &TLI) {
2329 MVT OpVT = Op.getValueType();
2330 SDValue Op0 = Op.getOperand(0);
2331 MVT Op0VT = Op0.getValueType();
2333 if ((OpVT == MVT::i32 && Op0VT == MVT::f64)
2334 || OpVT == MVT::i64) {
2335 // Convert f32 / f64 to i32 / i64 via libcall.
2337 (Op.getOpcode() == ISD::FP_TO_SINT)
2338 ? RTLIB::getFPTOSINT(Op0VT, OpVT)
2339 : RTLIB::getFPTOUINT(Op0VT, OpVT);
2340 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpectd fp-to-int conversion!");
2342 return ExpandLibCall(LC, Op, DAG, false, Dummy, TLI);
2348 //! Lower ISD::SINT_TO_FP, ISD::UINT_TO_FP for i32
2350 i32->f32 passes through unchanged, whereas i32->f64 is expanded to a libcall.
2351 All conversions from i64 are expanded to a libcall.
2353 static SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG,
2354 SPUTargetLowering &TLI) {
2355 MVT OpVT = Op.getValueType();
2356 SDValue Op0 = Op.getOperand(0);
2357 MVT Op0VT = Op0.getValueType();
2359 if ((OpVT == MVT::f64 && Op0VT == MVT::i32)
2360 || Op0VT == MVT::i64) {
2361 // Convert i32, i64 to f64 via libcall:
2363 (Op.getOpcode() == ISD::SINT_TO_FP)
2364 ? RTLIB::getSINTTOFP(Op0VT, OpVT)
2365 : RTLIB::getUINTTOFP(Op0VT, OpVT);
2366 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpectd int-to-fp conversion!");
2368 return ExpandLibCall(LC, Op, DAG, false, Dummy, TLI);
2374 //! Lower ISD::SETCC
2376 This handles MVT::f64 (double floating point) condition lowering
2378 static SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG,
2379 const TargetLowering &TLI) {
2380 CondCodeSDNode *CC = dyn_cast<CondCodeSDNode>(Op.getOperand(2));
2381 DebugLoc dl = Op.getDebugLoc();
2382 assert(CC != 0 && "LowerSETCC: CondCodeSDNode should not be null here!\n");
2384 SDValue lhs = Op.getOperand(0);
2385 SDValue rhs = Op.getOperand(1);
2386 MVT lhsVT = lhs.getValueType();
2387 assert(lhsVT == MVT::f64 && "LowerSETCC: type other than MVT::64\n");
2389 MVT ccResultVT = TLI.getSetCCResultType(lhs.getValueType());
2390 APInt ccResultOnes = APInt::getAllOnesValue(ccResultVT.getSizeInBits());
2391 MVT IntVT(MVT::i64);
2393 // Take advantage of the fact that (truncate (sra arg, 32)) is efficiently
2394 // selected to a NOP:
2395 SDValue i64lhs = DAG.getNode(ISD::BIT_CONVERT, dl, IntVT, lhs);
2397 DAG.getNode(ISD::TRUNCATE, dl, MVT::i32,
2398 DAG.getNode(ISD::SRL, dl, IntVT,
2399 i64lhs, DAG.getConstant(32, MVT::i32)));
2400 SDValue lhsHi32abs =
2401 DAG.getNode(ISD::AND, dl, MVT::i32,
2402 lhsHi32, DAG.getConstant(0x7fffffff, MVT::i32));
2404 DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, i64lhs);
2406 // SETO and SETUO only use the lhs operand:
2407 if (CC->get() == ISD::SETO) {
2408 // Evaluates to true if Op0 is not [SQ]NaN - lowers to the inverse of
2410 APInt ccResultAllOnes = APInt::getAllOnesValue(ccResultVT.getSizeInBits());
2411 return DAG.getNode(ISD::XOR, dl, ccResultVT,
2412 DAG.getSetCC(dl, ccResultVT,
2413 lhs, DAG.getConstantFP(0.0, lhsVT),
2415 DAG.getConstant(ccResultAllOnes, ccResultVT));
2416 } else if (CC->get() == ISD::SETUO) {
2417 // Evaluates to true if Op0 is [SQ]NaN
2418 return DAG.getNode(ISD::AND, dl, ccResultVT,
2419 DAG.getSetCC(dl, ccResultVT,
2421 DAG.getConstant(0x7ff00000, MVT::i32),
2423 DAG.getSetCC(dl, ccResultVT,
2425 DAG.getConstant(0, MVT::i32),
2429 SDValue i64rhs = DAG.getNode(ISD::BIT_CONVERT, dl, IntVT, rhs);
2431 DAG.getNode(ISD::TRUNCATE, dl, MVT::i32,
2432 DAG.getNode(ISD::SRL, dl, IntVT,
2433 i64rhs, DAG.getConstant(32, MVT::i32)));
2435 // If a value is negative, subtract from the sign magnitude constant:
2436 SDValue signMag2TC = DAG.getConstant(0x8000000000000000ULL, IntVT);
2438 // Convert the sign-magnitude representation into 2's complement:
2439 SDValue lhsSelectMask = DAG.getNode(ISD::SRA, dl, ccResultVT,
2440 lhsHi32, DAG.getConstant(31, MVT::i32));
2441 SDValue lhsSignMag2TC = DAG.getNode(ISD::SUB, dl, IntVT, signMag2TC, i64lhs);
2443 DAG.getNode(ISD::SELECT, dl, IntVT,
2444 lhsSelectMask, lhsSignMag2TC, i64lhs);
2446 SDValue rhsSelectMask = DAG.getNode(ISD::SRA, dl, ccResultVT,
2447 rhsHi32, DAG.getConstant(31, MVT::i32));
2448 SDValue rhsSignMag2TC = DAG.getNode(ISD::SUB, dl, IntVT, signMag2TC, i64rhs);
2450 DAG.getNode(ISD::SELECT, dl, IntVT,
2451 rhsSelectMask, rhsSignMag2TC, i64rhs);
2455 switch (CC->get()) {
2458 compareOp = ISD::SETEQ; break;
2461 compareOp = ISD::SETGT; break;
2464 compareOp = ISD::SETGE; break;
2467 compareOp = ISD::SETLT; break;
2470 compareOp = ISD::SETLE; break;
2473 compareOp = ISD::SETNE; break;
2475 llvm_report_error("CellSPU ISel Select: unimplemented f64 condition");
2479 DAG.getSetCC(dl, ccResultVT, lhsSelect, rhsSelect,
2480 (ISD::CondCode) compareOp);
2482 if ((CC->get() & 0x8) == 0) {
2483 // Ordered comparison:
2484 SDValue lhsNaN = DAG.getSetCC(dl, ccResultVT,
2485 lhs, DAG.getConstantFP(0.0, MVT::f64),
2487 SDValue rhsNaN = DAG.getSetCC(dl, ccResultVT,
2488 rhs, DAG.getConstantFP(0.0, MVT::f64),
2490 SDValue ordered = DAG.getNode(ISD::AND, dl, ccResultVT, lhsNaN, rhsNaN);
2492 result = DAG.getNode(ISD::AND, dl, ccResultVT, ordered, result);
2498 //! Lower ISD::SELECT_CC
2500 ISD::SELECT_CC can (generally) be implemented directly on the SPU using the
2503 \note Need to revisit this in the future: if the code path through the true
2504 and false value computations is longer than the latency of a branch (6
2505 cycles), then it would be more advantageous to branch and insert a new basic
2506 block and branch on the condition. However, this code does not make that
2507 assumption, given the simplisitc uses so far.
2510 static SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG,
2511 const TargetLowering &TLI) {
2512 MVT VT = Op.getValueType();
2513 SDValue lhs = Op.getOperand(0);
2514 SDValue rhs = Op.getOperand(1);
2515 SDValue trueval = Op.getOperand(2);
2516 SDValue falseval = Op.getOperand(3);
2517 SDValue condition = Op.getOperand(4);
2518 DebugLoc dl = Op.getDebugLoc();
2520 // NOTE: SELB's arguments: $rA, $rB, $mask
2522 // SELB selects bits from $rA where bits in $mask are 0, bits from $rB
2523 // where bits in $mask are 1. CCond will be inverted, having 1s where the
2524 // condition was true and 0s where the condition was false. Hence, the
2525 // arguments to SELB get reversed.
2527 // Note: Really should be ISD::SELECT instead of SPUISD::SELB, but LLVM's
2528 // legalizer insists on combining SETCC/SELECT into SELECT_CC, so we end up
2529 // with another "cannot select select_cc" assert:
2531 SDValue compare = DAG.getNode(ISD::SETCC, dl,
2532 TLI.getSetCCResultType(Op.getValueType()),
2533 lhs, rhs, condition);
2534 return DAG.getNode(SPUISD::SELB, dl, VT, falseval, trueval, compare);
2537 //! Custom lower ISD::TRUNCATE
2538 static SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG)
2540 // Type to truncate to
2541 MVT VT = Op.getValueType();
2542 MVT::SimpleValueType simpleVT = VT.getSimpleVT();
2543 MVT VecVT = MVT::getVectorVT(VT, (128 / VT.getSizeInBits()));
2544 DebugLoc dl = Op.getDebugLoc();
2546 // Type to truncate from
2547 SDValue Op0 = Op.getOperand(0);
2548 MVT Op0VT = Op0.getValueType();
2550 if (Op0VT.getSimpleVT() == MVT::i128 && simpleVT == MVT::i64) {
2551 // Create shuffle mask, least significant doubleword of quadword
2552 unsigned maskHigh = 0x08090a0b;
2553 unsigned maskLow = 0x0c0d0e0f;
2554 // Use a shuffle to perform the truncation
2555 SDValue shufMask = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32,
2556 DAG.getConstant(maskHigh, MVT::i32),
2557 DAG.getConstant(maskLow, MVT::i32),
2558 DAG.getConstant(maskHigh, MVT::i32),
2559 DAG.getConstant(maskLow, MVT::i32));
2561 SDValue truncShuffle = DAG.getNode(SPUISD::SHUFB, dl, VecVT,
2562 Op0, Op0, shufMask);
2564 return DAG.getNode(SPUISD::VEC2PREFSLOT, dl, VT, truncShuffle);
2567 return SDValue(); // Leave the truncate unmolested
2570 //! Custom (target-specific) lowering entry point
2572 This is where LLVM's DAG selection process calls to do target-specific
2576 SPUTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG)
2578 unsigned Opc = (unsigned) Op.getOpcode();
2579 MVT VT = Op.getValueType();
2584 cerr << "SPUTargetLowering::LowerOperation(): need to lower this!\n";
2585 cerr << "Op.getOpcode() = " << Opc << "\n";
2586 cerr << "*Op.getNode():\n";
2587 Op.getNode()->dump();
2595 return LowerLOAD(Op, DAG, SPUTM.getSubtargetImpl());
2597 return LowerSTORE(Op, DAG, SPUTM.getSubtargetImpl());
2598 case ISD::ConstantPool:
2599 return LowerConstantPool(Op, DAG, SPUTM.getSubtargetImpl());
2600 case ISD::GlobalAddress:
2601 return LowerGlobalAddress(Op, DAG, SPUTM.getSubtargetImpl());
2602 case ISD::JumpTable:
2603 return LowerJumpTable(Op, DAG, SPUTM.getSubtargetImpl());
2604 case ISD::ConstantFP:
2605 return LowerConstantFP(Op, DAG);
2606 case ISD::FORMAL_ARGUMENTS:
2607 return LowerFORMAL_ARGUMENTS(Op, DAG, VarArgsFrameIndex);
2609 return LowerCALL(Op, DAG, SPUTM.getSubtargetImpl());
2611 return LowerRET(Op, DAG, getTargetMachine());
2613 // i8, i64 math ops:
2622 return LowerI8Math(Op, DAG, Opc, *this);
2626 case ISD::FP_TO_SINT:
2627 case ISD::FP_TO_UINT:
2628 return LowerFP_TO_INT(Op, DAG, *this);
2630 case ISD::SINT_TO_FP:
2631 case ISD::UINT_TO_FP:
2632 return LowerINT_TO_FP(Op, DAG, *this);
2634 // Vector-related lowering.
2635 case ISD::BUILD_VECTOR:
2636 return LowerBUILD_VECTOR(Op, DAG);
2637 case ISD::SCALAR_TO_VECTOR:
2638 return LowerSCALAR_TO_VECTOR(Op, DAG);
2639 case ISD::VECTOR_SHUFFLE:
2640 return LowerVECTOR_SHUFFLE(Op, DAG);
2641 case ISD::EXTRACT_VECTOR_ELT:
2642 return LowerEXTRACT_VECTOR_ELT(Op, DAG);
2643 case ISD::INSERT_VECTOR_ELT:
2644 return LowerINSERT_VECTOR_ELT(Op, DAG);
2646 // Look for ANDBI, ORBI and XORBI opportunities and lower appropriately:
2650 return LowerByteImmed(Op, DAG);
2652 // Vector and i8 multiply:
2655 return LowerI8Math(Op, DAG, Opc, *this);
2658 return LowerCTPOP(Op, DAG);
2660 case ISD::SELECT_CC:
2661 return LowerSELECT_CC(Op, DAG, *this);
2664 return LowerSETCC(Op, DAG, *this);
2667 return LowerTRUNCATE(Op, DAG);
2673 void SPUTargetLowering::ReplaceNodeResults(SDNode *N,
2674 SmallVectorImpl<SDValue>&Results,
2678 unsigned Opc = (unsigned) N->getOpcode();
2679 MVT OpVT = N->getValueType(0);
2683 cerr << "SPUTargetLowering::ReplaceNodeResults(): need to fix this!\n";
2684 cerr << "Op.getOpcode() = " << Opc << "\n";
2685 cerr << "*Op.getNode():\n";
2693 /* Otherwise, return unchanged */
2696 //===----------------------------------------------------------------------===//
2697 // Target Optimization Hooks
2698 //===----------------------------------------------------------------------===//
2701 SPUTargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const
2704 TargetMachine &TM = getTargetMachine();
2706 const SPUSubtarget *ST = SPUTM.getSubtargetImpl();
2707 SelectionDAG &DAG = DCI.DAG;
2708 SDValue Op0 = N->getOperand(0); // everything has at least one operand
2709 MVT NodeVT = N->getValueType(0); // The node's value type
2710 MVT Op0VT = Op0.getValueType(); // The first operand's result
2711 SDValue Result; // Initially, empty result
2712 DebugLoc dl = N->getDebugLoc();
2714 switch (N->getOpcode()) {
2717 SDValue Op1 = N->getOperand(1);
2719 if (Op0.getOpcode() == SPUISD::IndirectAddr
2720 || Op1.getOpcode() == SPUISD::IndirectAddr) {
2721 // Normalize the operands to reduce repeated code
2722 SDValue IndirectArg = Op0, AddArg = Op1;
2724 if (Op1.getOpcode() == SPUISD::IndirectAddr) {
2729 if (isa<ConstantSDNode>(AddArg)) {
2730 ConstantSDNode *CN0 = cast<ConstantSDNode > (AddArg);
2731 SDValue IndOp1 = IndirectArg.getOperand(1);
2733 if (CN0->isNullValue()) {
2734 // (add (SPUindirect <arg>, <arg>), 0) ->
2735 // (SPUindirect <arg>, <arg>)
2737 #if !defined(NDEBUG)
2738 if (DebugFlag && isCurrentDebugType(DEBUG_TYPE)) {
2740 << "Replace: (add (SPUindirect <arg>, <arg>), 0)\n"
2741 << "With: (SPUindirect <arg>, <arg>)\n";
2746 } else if (isa<ConstantSDNode>(IndOp1)) {
2747 // (add (SPUindirect <arg>, <const>), <const>) ->
2748 // (SPUindirect <arg>, <const + const>)
2749 ConstantSDNode *CN1 = cast<ConstantSDNode > (IndOp1);
2750 int64_t combinedConst = CN0->getSExtValue() + CN1->getSExtValue();
2751 SDValue combinedValue = DAG.getConstant(combinedConst, Op0VT);
2753 #if !defined(NDEBUG)
2754 if (DebugFlag && isCurrentDebugType(DEBUG_TYPE)) {
2756 << "Replace: (add (SPUindirect <arg>, " << CN1->getSExtValue()
2757 << "), " << CN0->getSExtValue() << ")\n"
2758 << "With: (SPUindirect <arg>, "
2759 << combinedConst << ")\n";
2763 return DAG.getNode(SPUISD::IndirectAddr, dl, Op0VT,
2764 IndirectArg, combinedValue);
2770 case ISD::SIGN_EXTEND:
2771 case ISD::ZERO_EXTEND:
2772 case ISD::ANY_EXTEND: {
2773 if (Op0.getOpcode() == SPUISD::VEC2PREFSLOT && NodeVT == Op0VT) {
2774 // (any_extend (SPUextract_elt0 <arg>)) ->
2775 // (SPUextract_elt0 <arg>)
2776 // Types must match, however...
2777 #if !defined(NDEBUG)
2778 if (DebugFlag && isCurrentDebugType(DEBUG_TYPE)) {
2779 cerr << "\nReplace: ";
2782 Op0.getNode()->dump(&DAG);
2791 case SPUISD::IndirectAddr: {
2792 if (!ST->usingLargeMem() && Op0.getOpcode() == SPUISD::AFormAddr) {
2793 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1));
2794 if (CN != 0 && CN->getZExtValue() == 0) {
2795 // (SPUindirect (SPUaform <addr>, 0), 0) ->
2796 // (SPUaform <addr>, 0)
2798 DEBUG(cerr << "Replace: ");
2799 DEBUG(N->dump(&DAG));
2800 DEBUG(cerr << "\nWith: ");
2801 DEBUG(Op0.getNode()->dump(&DAG));
2802 DEBUG(cerr << "\n");
2806 } else if (Op0.getOpcode() == ISD::ADD) {
2807 SDValue Op1 = N->getOperand(1);
2808 if (ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(Op1)) {
2809 // (SPUindirect (add <arg>, <arg>), 0) ->
2810 // (SPUindirect <arg>, <arg>)
2811 if (CN1->isNullValue()) {
2813 #if !defined(NDEBUG)
2814 if (DebugFlag && isCurrentDebugType(DEBUG_TYPE)) {
2816 << "Replace: (SPUindirect (add <arg>, <arg>), 0)\n"
2817 << "With: (SPUindirect <arg>, <arg>)\n";
2821 return DAG.getNode(SPUISD::IndirectAddr, dl, Op0VT,
2822 Op0.getOperand(0), Op0.getOperand(1));
2828 case SPUISD::SHLQUAD_L_BITS:
2829 case SPUISD::SHLQUAD_L_BYTES:
2830 case SPUISD::VEC_SHL:
2831 case SPUISD::VEC_SRL:
2832 case SPUISD::VEC_SRA:
2833 case SPUISD::ROTBYTES_LEFT: {
2834 SDValue Op1 = N->getOperand(1);
2836 // Kill degenerate vector shifts:
2837 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op1)) {
2838 if (CN->isNullValue()) {
2844 case SPUISD::PREFSLOT2VEC: {
2845 switch (Op0.getOpcode()) {
2848 case ISD::ANY_EXTEND:
2849 case ISD::ZERO_EXTEND:
2850 case ISD::SIGN_EXTEND: {
2851 // (SPUprefslot2vec (any|zero|sign_extend (SPUvec2prefslot <arg>))) ->
2853 // but only if the SPUprefslot2vec and <arg> types match.
2854 SDValue Op00 = Op0.getOperand(0);
2855 if (Op00.getOpcode() == SPUISD::VEC2PREFSLOT) {
2856 SDValue Op000 = Op00.getOperand(0);
2857 if (Op000.getValueType() == NodeVT) {
2863 case SPUISD::VEC2PREFSLOT: {
2864 // (SPUprefslot2vec (SPUvec2prefslot <arg>)) ->
2866 Result = Op0.getOperand(0);
2874 // Otherwise, return unchanged.
2876 if (Result.getNode()) {
2877 DEBUG(cerr << "\nReplace.SPU: ");
2878 DEBUG(N->dump(&DAG));
2879 DEBUG(cerr << "\nWith: ");
2880 DEBUG(Result.getNode()->dump(&DAG));
2881 DEBUG(cerr << "\n");
2888 //===----------------------------------------------------------------------===//
2889 // Inline Assembly Support
2890 //===----------------------------------------------------------------------===//
2892 /// getConstraintType - Given a constraint letter, return the type of
2893 /// constraint it is for this target.
2894 SPUTargetLowering::ConstraintType
2895 SPUTargetLowering::getConstraintType(const std::string &ConstraintLetter) const {
2896 if (ConstraintLetter.size() == 1) {
2897 switch (ConstraintLetter[0]) {
2904 return C_RegisterClass;
2907 return TargetLowering::getConstraintType(ConstraintLetter);
2910 std::pair<unsigned, const TargetRegisterClass*>
2911 SPUTargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint,
2914 if (Constraint.size() == 1) {
2915 // GCC RS6000 Constraint Letters
2916 switch (Constraint[0]) {
2920 return std::make_pair(0U, SPU::R64CRegisterClass);
2921 return std::make_pair(0U, SPU::R32CRegisterClass);
2924 return std::make_pair(0U, SPU::R32FPRegisterClass);
2925 else if (VT == MVT::f64)
2926 return std::make_pair(0U, SPU::R64FPRegisterClass);
2929 return std::make_pair(0U, SPU::GPRCRegisterClass);
2933 return TargetLowering::getRegForInlineAsmConstraint(Constraint, VT);
2936 //! Compute used/known bits for a SPU operand
2938 SPUTargetLowering::computeMaskedBitsForTargetNode(const SDValue Op,
2942 const SelectionDAG &DAG,
2943 unsigned Depth ) const {
2945 const uint64_t uint64_sizebits = sizeof(uint64_t) * CHAR_BIT;
2947 switch (Op.getOpcode()) {
2949 // KnownZero = KnownOne = APInt(Mask.getBitWidth(), 0);
2955 case SPUISD::PREFSLOT2VEC:
2956 case SPUISD::LDRESULT:
2957 case SPUISD::VEC2PREFSLOT:
2958 case SPUISD::SHLQUAD_L_BITS:
2959 case SPUISD::SHLQUAD_L_BYTES:
2960 case SPUISD::VEC_SHL:
2961 case SPUISD::VEC_SRL:
2962 case SPUISD::VEC_SRA:
2963 case SPUISD::VEC_ROTL:
2964 case SPUISD::VEC_ROTR:
2965 case SPUISD::ROTBYTES_LEFT:
2966 case SPUISD::SELECT_MASK:
2973 SPUTargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,
2974 unsigned Depth) const {
2975 switch (Op.getOpcode()) {
2980 MVT VT = Op.getValueType();
2982 if (VT != MVT::i8 && VT != MVT::i16 && VT != MVT::i32) {
2985 return VT.getSizeInBits();
2990 // LowerAsmOperandForConstraint
2992 SPUTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
2993 char ConstraintLetter,
2995 std::vector<SDValue> &Ops,
2996 SelectionDAG &DAG) const {
2997 // Default, for the time being, to the base class handler
2998 TargetLowering::LowerAsmOperandForConstraint(Op, ConstraintLetter, hasMemory,
3002 /// isLegalAddressImmediate - Return true if the integer value can be used
3003 /// as the offset of the target addressing mode.
3004 bool SPUTargetLowering::isLegalAddressImmediate(int64_t V,
3005 const Type *Ty) const {
3006 // SPU's addresses are 256K:
3007 return (V > -(1 << 18) && V < (1 << 18) - 1);
3010 bool SPUTargetLowering::isLegalAddressImmediate(llvm::GlobalValue* GV) const {
3015 SPUTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
3016 // The SPU target isn't yet aware of offsets.