1 //===-- NEONPreAllocPass.cpp - Allocate adjacent NEON registers--*- C++ -*-===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 #define DEBUG_TYPE "neon-prealloc"
12 #include "ARMInstrInfo.h"
13 #include "llvm/CodeGen/MachineInstr.h"
14 #include "llvm/CodeGen/MachineInstrBuilder.h"
15 #include "llvm/CodeGen/MachineRegisterInfo.h"
16 #include "llvm/CodeGen/MachineFunctionPass.h"
20 class NEONPreAllocPass : public MachineFunctionPass {
21 const TargetInstrInfo *TII;
22 MachineRegisterInfo *MRI;
26 NEONPreAllocPass() : MachineFunctionPass(&ID) {}
28 virtual bool runOnMachineFunction(MachineFunction &MF);
30 virtual const char *getPassName() const {
31 return "NEON register pre-allocation pass";
35 bool FormsRegSequence(MachineInstr *MI,
36 unsigned FirstOpnd, unsigned NumRegs,
37 unsigned Offset, unsigned Stride) const;
38 bool PreAllocNEONRegisters(MachineBasicBlock &MBB);
41 char NEONPreAllocPass::ID = 0;
44 static bool isNEONMultiRegOp(int Opcode, unsigned &FirstOpnd, unsigned &NumRegs,
45 unsigned &Offset, unsigned &Stride) {
46 // Default to unit stride with no offset.
83 case ARM::VLD2LNq16odd:
84 case ARM::VLD2LNq32odd:
102 case ARM::VLD3q8_UPD:
103 case ARM::VLD3q16_UPD:
104 case ARM::VLD3q32_UPD:
111 case ARM::VLD3q8odd_UPD:
112 case ARM::VLD3q16odd_UPD:
113 case ARM::VLD3q32odd_UPD:
128 case ARM::VLD3LNq16odd:
129 case ARM::VLD3LNq32odd:
147 case ARM::VLD4q8_UPD:
148 case ARM::VLD4q16_UPD:
149 case ARM::VLD4q32_UPD:
156 case ARM::VLD4q8odd_UPD:
157 case ARM::VLD4q16odd_UPD:
158 case ARM::VLD4q32odd_UPD:
173 case ARM::VLD4LNq16odd:
174 case ARM::VLD4LNq32odd:
210 case ARM::VST2LNq16odd:
211 case ARM::VST2LNq32odd:
229 case ARM::VST3q8_UPD:
230 case ARM::VST3q16_UPD:
231 case ARM::VST3q32_UPD:
238 case ARM::VST3q8odd_UPD:
239 case ARM::VST3q16odd_UPD:
240 case ARM::VST3q32odd_UPD:
255 case ARM::VST3LNq16odd:
256 case ARM::VST3LNq32odd:
274 case ARM::VST4q8_UPD:
275 case ARM::VST4q16_UPD:
276 case ARM::VST4q32_UPD:
283 case ARM::VST4q8odd_UPD:
284 case ARM::VST4q16odd_UPD:
285 case ARM::VST4q32odd_UPD:
300 case ARM::VST4LNq16odd:
301 case ARM::VST4LNq32odd:
343 NEONPreAllocPass::FormsRegSequence(MachineInstr *MI,
344 unsigned FirstOpnd, unsigned NumRegs,
345 unsigned Offset, unsigned Stride) const {
346 MachineOperand &FMO = MI->getOperand(FirstOpnd);
347 assert(FMO.isReg() && FMO.getSubReg() == 0 && "unexpected operand");
348 unsigned VirtReg = FMO.getReg();
350 assert(TargetRegisterInfo::isVirtualRegister(VirtReg) &&
351 "expected a virtual register");
353 unsigned LastSubIdx = 0;
355 MachineInstr *RegSeq = 0;
356 for (unsigned R = 0; R < NumRegs; ++R) {
357 const MachineOperand &MO = MI->getOperand(FirstOpnd + R);
358 assert(MO.isReg() && MO.getSubReg() == 0 && "unexpected operand");
359 unsigned VirtReg = MO.getReg();
360 assert(TargetRegisterInfo::isVirtualRegister(VirtReg) &&
361 "expected a virtual register");
362 // Feeding into a REG_SEQUENCE.
363 if (!MRI->hasOneNonDBGUse(VirtReg))
365 MachineInstr *UseMI = &*MRI->use_nodbg_begin(VirtReg);
366 if (!UseMI->isRegSequence())
368 if (RegSeq && RegSeq != UseMI)
370 unsigned OpIdx = 1 + (Offset + R * Stride) * 2;
371 if (UseMI->getOperand(OpIdx).getReg() != VirtReg)
372 llvm_unreachable("Malformed REG_SEQUENCE instruction!");
373 unsigned SubIdx = UseMI->getOperand(OpIdx + 1).getImm();
375 if (LastSubIdx != SubIdx-Stride)
378 // Must start from dsub_0 or qsub_0.
379 if (SubIdx != (ARM::dsub_0+Offset) &&
380 SubIdx != (ARM::qsub_0+Offset))
387 // In the case of vld3, etc., make sure the trailing operand of
388 // REG_SEQUENCE is an undef.
390 unsigned OpIdx = 1 + (Offset + 3 * Stride) * 2;
391 const MachineOperand &MO = RegSeq->getOperand(OpIdx);
392 unsigned VirtReg = MO.getReg();
393 MachineInstr *DefMI = MRI->getVRegDef(VirtReg);
394 if (!DefMI || !DefMI->isImplicitDef())
400 unsigned LastSrcReg = 0;
401 SmallVector<unsigned, 4> SubIds;
402 for (unsigned R = 0; R < NumRegs; ++R) {
403 const MachineOperand &MO = MI->getOperand(FirstOpnd + R);
404 assert(MO.isReg() && MO.getSubReg() == 0 && "unexpected operand");
405 unsigned VirtReg = MO.getReg();
406 assert(TargetRegisterInfo::isVirtualRegister(VirtReg) &&
407 "expected a virtual register");
408 // Extracting from a Q or QQ register.
409 MachineInstr *DefMI = MRI->getVRegDef(VirtReg);
410 if (!DefMI || !DefMI->isExtractSubreg())
412 VirtReg = DefMI->getOperand(1).getReg();
413 if (LastSrcReg && LastSrcReg != VirtReg)
415 LastSrcReg = VirtReg;
416 const TargetRegisterClass *RC = MRI->getRegClass(VirtReg);
417 if (RC != ARM::QPRRegisterClass &&
418 RC != ARM::QQPRRegisterClass &&
419 RC != ARM::QQQQPRRegisterClass)
421 unsigned SubIdx = DefMI->getOperand(2).getImm();
423 if (LastSubIdx != SubIdx-Stride)
426 // Must start from dsub_0 or qsub_0.
427 if (SubIdx != (ARM::dsub_0+Offset) &&
428 SubIdx != (ARM::qsub_0+Offset))
431 SubIds.push_back(SubIdx);
435 // FIXME: Update the uses of EXTRACT_SUBREG from REG_SEQUENCE is
436 // currently required for correctness. e.g.
437 // %reg1041;<def> = REG_SEQUENCE %reg1040<kill>, 5, %reg1035<kill>, 6
438 // %reg1042<def> = EXTRACT_SUBREG %reg1041, 6
439 // %reg1043<def> = EXTRACT_SUBREG %reg1041, 5
440 // VST1q16 %reg1025<kill>, 0, %reg1043<kill>, %reg1042<kill>,
441 // reg1025 and reg1043 should be replaced with reg1041:6 and reg1041:5
443 // We need to change how we model uses of REG_SEQUENCE.
444 for (unsigned R = 0; R < NumRegs; ++R) {
445 MachineOperand &MO = MI->getOperand(FirstOpnd + R);
446 unsigned OldReg = MO.getReg();
447 MachineInstr *DefMI = MRI->getVRegDef(OldReg);
448 assert(DefMI->isExtractSubreg());
449 MO.setReg(LastSrcReg);
450 MO.setSubReg(SubIds[R]);
453 // Delete the EXTRACT_SUBREG if its result is now dead.
454 if (MRI->use_empty(OldReg))
455 DefMI->eraseFromParent();
461 bool NEONPreAllocPass::PreAllocNEONRegisters(MachineBasicBlock &MBB) {
462 bool Modified = false;
464 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
465 for (; MBBI != E; ++MBBI) {
466 MachineInstr *MI = &*MBBI;
467 unsigned FirstOpnd, NumRegs, Offset, Stride;
468 if (!isNEONMultiRegOp(MI->getOpcode(), FirstOpnd, NumRegs, Offset, Stride))
470 if (llvm::ModelWithRegSequence() &&
471 FormsRegSequence(MI, FirstOpnd, NumRegs, Offset, Stride))
474 MachineBasicBlock::iterator NextI = llvm::next(MBBI);
475 for (unsigned R = 0; R < NumRegs; ++R) {
476 MachineOperand &MO = MI->getOperand(FirstOpnd + R);
477 assert(MO.isReg() && MO.getSubReg() == 0 && "unexpected operand");
478 unsigned VirtReg = MO.getReg();
479 assert(TargetRegisterInfo::isVirtualRegister(VirtReg) &&
480 "expected a virtual register");
482 // For now, just assign a fixed set of adjacent registers.
483 // This leaves plenty of room for future improvements.
484 static const unsigned NEONDRegs[] = {
485 ARM::D0, ARM::D1, ARM::D2, ARM::D3,
486 ARM::D4, ARM::D5, ARM::D6, ARM::D7
488 MO.setReg(NEONDRegs[Offset + R * Stride]);
491 // Insert a copy from VirtReg.
492 TII->copyRegToReg(MBB, MBBI, MO.getReg(), VirtReg,
493 ARM::DPRRegisterClass, ARM::DPRRegisterClass,
496 MachineInstr *CopyMI = prior(MBBI);
497 CopyMI->findRegisterUseOperand(VirtReg)->setIsKill();
500 } else if (MO.isDef() && !MO.isDead()) {
501 // Add a copy to VirtReg.
502 TII->copyRegToReg(MBB, NextI, VirtReg, MO.getReg(),
503 ARM::DPRRegisterClass, ARM::DPRRegisterClass,
512 bool NEONPreAllocPass::runOnMachineFunction(MachineFunction &MF) {
513 TII = MF.getTarget().getInstrInfo();
514 MRI = &MF.getRegInfo();
516 bool Modified = false;
517 for (MachineFunction::iterator MFI = MF.begin(), E = MF.end(); MFI != E;
519 MachineBasicBlock &MBB = *MFI;
520 Modified |= PreAllocNEONRegisters(MBB);
526 /// createNEONPreAllocPass - returns an instance of the NEON register
527 /// pre-allocation pass.
528 FunctionPass *llvm::createNEONPreAllocPass() {
529 return new NEONPreAllocPass();