bool visitLoadInst(LoadInst &);
bool visitStoreInst(StoreInst &);
+ static void registerOptions() {
+ // This is disabled by default because having separate loads and stores
+ // makes it more likely that the -combiner-alias-analysis limits will be
+ // reached.
+ OptionRegistry::registerOption<bool, Scalarizer,
+ &Scalarizer::ScalarizeLoadStore>(
+ "scalarize-load-store",
+ "Allow the scalarizer pass to scalarize loads and store", false);
+ }
+
private:
Scatterer scatter(Instruction *, Value *);
void gather(Instruction *, const ValueVector &);
bool canTransferMetadata(unsigned Kind);
void transferMetadata(Instruction *, const ValueVector &);
- bool getVectorLayout(Type *, unsigned, VectorLayout &);
+ bool getVectorLayout(Type *, unsigned, VectorLayout &, const DataLayout &);
bool finish();
template<typename T> bool splitBinary(Instruction &, const T &);
ScatterMap Scattered;
GatherList Gathered;
unsigned ParallelLoopAccessMDKind;
- const DataLayout *DL;
+ bool ScalarizeLoadStore;
};
char Scalarizer::ID = 0;
} // end anonymous namespace
-// This is disabled by default because having separate loads and stores makes
-// it more likely that the -combiner-alias-analysis limits will be reached.
-static cl::opt<bool> ScalarizeLoadStore
- ("scalarize-load-store", cl::Hidden, cl::init(false),
- cl::desc("Allow the scalarizer pass to scalarize loads and store"));
-
-INITIALIZE_PASS(Scalarizer, "scalarizer", "Scalarize vector operations",
- false, false)
+INITIALIZE_PASS_WITH_OPTIONS(Scalarizer, "scalarizer",
+ "Scalarize vector operations", false, false)
Scatterer::Scatterer(BasicBlock *bb, BasicBlock::iterator bbi, Value *v,
ValueVector *cachePtr)
CV[0] = Builder.CreateBitCast(V, Ty, V->getName() + ".i0");
}
if (I != 0)
- CV[I] = Builder.CreateConstGEP1_32(CV[0], I,
+ CV[I] = Builder.CreateConstGEP1_32(nullptr, CV[0], I,
V->getName() + ".i" + Twine(I));
} else {
// Search through a chain of InsertElementInsts looking for element I.
if (!Idx)
break;
unsigned J = Idx->getZExtValue();
- CV[J] = Insert->getOperand(1);
V = Insert->getOperand(0);
- if (I == J)
+ if (I == J) {
+ CV[J] = Insert->getOperand(1);
return CV[J];
+ } else if (!CV[J]) {
+ // Only cache the first entry we find for each index we're not actively
+ // searching for. This prevents us from going too far up the chain and
+ // caching incorrect entries.
+ CV[J] = Insert->getOperand(1);
+ }
}
CV[I] = Builder.CreateExtractElement(V, Builder.getInt32(I),
V->getName() + ".i" + Twine(I));
bool Scalarizer::doInitialization(Module &M) {
ParallelLoopAccessMDKind =
- M.getContext().getMDKindID("llvm.mem.parallel_loop_access");
+ M.getContext().getMDKindID("llvm.mem.parallel_loop_access");
+ ScalarizeLoadStore =
+ M.getContext().getOption<bool, Scalarizer, &Scalarizer::ScalarizeLoadStore>();
return false;
}
bool Scalarizer::runOnFunction(Function &F) {
- DataLayoutPass *DLP = getAnalysisIfAvailable<DataLayoutPass>();
- DL = DLP ? &DLP->getDataLayout() : nullptr;
- for (Function::iterator BBI = F.begin(), BBE = F.end(); BBI != BBE; ++BBI) {
- BasicBlock *BB = BBI;
- for (BasicBlock::iterator II = BB->begin(), IE = BB->end(); II != IE;) {
- Instruction *I = II;
+ assert(Gathered.empty() && Scattered.empty());
+ for (BasicBlock &BB : F) {
+ for (BasicBlock::iterator II = BB.begin(), IE = BB.end(); II != IE;) {
+ Instruction *I = &*II;
bool Done = visit(I);
++II;
if (Done && I->getType()->isVoidTy())
}
// In the fallback case, just put the scattered before Point and
// keep the result local to Point.
- return Scatterer(Point->getParent(), Point, V);
+ return Scatterer(Point->getParent(), Point->getIterator(), V);
}
// Replace Op with the gathered form of the components in CV. Defer the
|| Tag == LLVMContext::MD_fpmath
|| Tag == LLVMContext::MD_tbaa_struct
|| Tag == LLVMContext::MD_invariant_load
+ || Tag == LLVMContext::MD_alias_scope
+ || Tag == LLVMContext::MD_noalias
|| Tag == ParallelLoopAccessMDKind);
}
Op->getAllMetadataOtherThanDebugLoc(MDs);
for (unsigned I = 0, E = CV.size(); I != E; ++I) {
if (Instruction *New = dyn_cast<Instruction>(CV[I])) {
- for (SmallVectorImpl<std::pair<unsigned, MDNode *> >::iterator
- MI = MDs.begin(), ME = MDs.end(); MI != ME; ++MI)
+ for (SmallVectorImpl<std::pair<unsigned, MDNode *>>::iterator
+ MI = MDs.begin(),
+ ME = MDs.end();
+ MI != ME; ++MI)
if (canTransferMetadata(MI->first))
New->setMetadata(MI->first, MI->second);
New->setDebugLoc(Op->getDebugLoc());
// Try to fill in Layout from Ty, returning true on success. Alignment is
// the alignment of the vector, or 0 if the ABI default should be used.
bool Scalarizer::getVectorLayout(Type *Ty, unsigned Alignment,
- VectorLayout &Layout) {
- if (!DL)
- return false;
-
+ VectorLayout &Layout, const DataLayout &DL) {
// Make sure we're dealing with a vector.
Layout.VecTy = dyn_cast<VectorType>(Ty);
if (!Layout.VecTy)
// Check that we're dealing with full-byte elements.
Layout.ElemTy = Layout.VecTy->getElementType();
- if (DL->getTypeSizeInBits(Layout.ElemTy) !=
- DL->getTypeStoreSizeInBits(Layout.ElemTy))
+ if (DL.getTypeSizeInBits(Layout.ElemTy) !=
+ DL.getTypeStoreSizeInBits(Layout.ElemTy))
return false;
if (Alignment)
Layout.VecAlign = Alignment;
else
- Layout.VecAlign = DL->getABITypeAlignment(Layout.VecTy);
- Layout.ElemSize = DL->getTypeStoreSize(Layout.ElemTy);
+ Layout.VecAlign = DL.getABITypeAlignment(Layout.VecTy);
+ Layout.ElemSize = DL.getTypeStoreSize(Layout.ElemTy);
return true;
}
return false;
unsigned NumElems = VT->getNumElements();
- IRBuilder<> Builder(I.getParent(), &I);
+ IRBuilder<> Builder(&I);
Scatterer Op0 = scatter(&I, I.getOperand(0));
Scatterer Op1 = scatter(&I, I.getOperand(1));
assert(Op0.size() == NumElems && "Mismatched binary operation");
return false;
unsigned NumElems = VT->getNumElements();
- IRBuilder<> Builder(SI.getParent(), &SI);
+ IRBuilder<> Builder(&SI);
Scatterer Op1 = scatter(&SI, SI.getOperand(1));
Scatterer Op2 = scatter(&SI, SI.getOperand(2));
assert(Op1.size() == NumElems && "Mismatched select");
if (!VT)
return false;
- IRBuilder<> Builder(GEPI.getParent(), &GEPI);
+ IRBuilder<> Builder(&GEPI);
unsigned NumElems = VT->getNumElements();
unsigned NumIndices = GEPI.getNumIndices();
Indices.resize(NumIndices);
for (unsigned J = 0; J < NumIndices; ++J)
Indices[J] = Ops[J][I];
- Res[I] = Builder.CreateGEP(Base[I], Indices,
+ Res[I] = Builder.CreateGEP(GEPI.getSourceElementType(), Base[I], Indices,
GEPI.getName() + ".i" + Twine(I));
if (GEPI.isInBounds())
if (GetElementPtrInst *NewGEPI = dyn_cast<GetElementPtrInst>(Res[I]))
return false;
unsigned NumElems = VT->getNumElements();
- IRBuilder<> Builder(CI.getParent(), &CI);
+ IRBuilder<> Builder(&CI);
Scatterer Op0 = scatter(&CI, CI.getOperand(0));
assert(Op0.size() == NumElems && "Mismatched cast");
ValueVector Res;
unsigned DstNumElems = DstVT->getNumElements();
unsigned SrcNumElems = SrcVT->getNumElements();
- IRBuilder<> Builder(BCI.getParent(), &BCI);
+ IRBuilder<> Builder(&BCI);
Scatterer Op0 = scatter(&BCI, BCI.getOperand(0));
ValueVector Res;
Res.resize(DstNumElems);
return false;
unsigned NumElems = VT->getNumElements();
- IRBuilder<> Builder(PHI.getParent(), &PHI);
+ IRBuilder<> Builder(&PHI);
ValueVector Res;
Res.resize(NumElems);
return false;
VectorLayout Layout;
- if (!getVectorLayout(LI.getType(), LI.getAlignment(), Layout))
+ if (!getVectorLayout(LI.getType(), LI.getAlignment(), Layout,
+ LI.getModule()->getDataLayout()))
return false;
unsigned NumElems = Layout.VecTy->getNumElements();
- IRBuilder<> Builder(LI.getParent(), &LI);
+ IRBuilder<> Builder(&LI);
Scatterer Ptr = scatter(&LI, LI.getPointerOperand());
ValueVector Res;
Res.resize(NumElems);
VectorLayout Layout;
Value *FullValue = SI.getValueOperand();
- if (!getVectorLayout(FullValue->getType(), SI.getAlignment(), Layout))
+ if (!getVectorLayout(FullValue->getType(), SI.getAlignment(), Layout,
+ SI.getModule()->getDataLayout()))
return false;
unsigned NumElems = Layout.VecTy->getNumElements();
- IRBuilder<> Builder(SI.getParent(), &SI);
+ IRBuilder<> Builder(&SI);
Scatterer Ptr = scatter(&SI, SI.getPointerOperand());
Scatterer Val = scatter(&SI, FullValue);
// Delete the instructions that we scalarized. If a full vector result
// is still needed, recreate it using InsertElements.
bool Scalarizer::finish() {
- if (Gathered.empty())
+ // The presence of data in Gathered or Scattered indicates changes
+ // made to the Function.
+ if (Gathered.empty() && Scattered.empty())
return false;
for (GatherList::iterator GMI = Gathered.begin(), GME = Gathered.end();
GMI != GME; ++GMI) {
Value *Res = UndefValue::get(Ty);
BasicBlock *BB = Op->getParent();
unsigned Count = Ty->getVectorNumElements();
- IRBuilder<> Builder(BB, Op);
+ IRBuilder<> Builder(Op);
if (isa<PHINode>(Op))
Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
for (unsigned I = 0; I < Count; ++I)
projects / oota-llvm.git / blobdiff