/// ...
static cl::opt<bool> EnableMemAccessVersioning(
"enable-mem-access-versioning", cl::init(true), cl::Hidden,
- cl::desc("Enable symblic stride memory access versioning"));
+ cl::desc("Enable symbolic stride memory access versioning"));
static cl::opt<bool> EnableInterleavedMemAccesses(
"enable-interleaved-mem-accesses", cl::init(false), cl::Hidden,
// can be validly truncated to. The cost model has assumed this truncation
// will happen when vectorizing.
void vectorize(LoopVectorizationLegality *L,
- DenseMap<Instruction*,uint64_t> MinimumBitWidths) {
+ MapVector<Instruction*,uint64_t> MinimumBitWidths) {
MinBWs = MinimumBitWidths;
Legal = L;
// Create a new empty loop. Unlink the old loop and connect the new one.
/// Map of scalar integer values to the smallest bitwidth they can be legally
/// represented as. The vector equivalents of these values should be truncated
/// to this type.
- DenseMap<Instruction*,uint64_t> MinBWs;
+ MapVector<Instruction*,uint64_t> MinBWs;
LoopVectorizationLegality *Legal;
// Record whether runtime check is added.
}
/// \brief Propagate known metadata from one instruction to a vector of others.
-static void propagateMetadata(SmallVectorImpl<Value *> &To, const Instruction *From) {
+static void propagateMetadata(SmallVectorImpl<Value *> &To,
+ const Instruction *From) {
for (Value *V : To)
if (Instruction *I = dyn_cast<Instruction>(V))
propagateMetadata(I, From);
/// Returns True if V is an induction variable in this loop.
bool isInductionVariable(const Value *V);
+ /// Returns True if PN is a reduction variable in this loop.
+ bool isReductionVariable(PHINode *PN) { return Reductions.count(PN); }
+
/// Return true if the block BB needs to be predicated in order for the loop
/// to be vectorized.
bool blockNeedsPredication(BasicBlock *BB);
/// Map of scalar integer values to the smallest bitwidth they can be legally
/// represented as. The vector equivalents of these values should be truncated
/// to this type.
- DenseMap<Instruction*,uint64_t> MinBWs;
+ MapVector<Instruction*,uint64_t> MinBWs;
/// The loop that we evaluate.
Loop *TheLoop;
// If this scalar is unknown, assume that it is a constant or that it is
// loop invariant. Broadcast V and save the value for future uses.
Value *B = getBroadcastInstrs(V);
-
return WidenMap.splat(V, B);
}
}
}
-void InnerLoopVectorizer::scalarizeInstruction(Instruction *Instr, bool IfPredicateStore) {
+void InnerLoopVectorizer::scalarizeInstruction(Instruction *Instr,
+ bool IfPredicateStore) {
assert(!Instr->getType()->isAggregateType() && "Can't handle vectors");
// Holds vector parameters or scalars, in case of uniform vals.
SmallVector<VectorParts, 4> Params;
Value *Cmp = nullptr;
if (IfPredicateStore) {
Cmp = Builder.CreateExtractElement(Cond[Part], Builder.getInt32(Width));
- Cmp = Builder.CreateICmp(ICmpInst::ICMP_EQ, Cmp, ConstantInt::get(Cmp->getType(), 1));
+ Cmp = Builder.CreateICmp(ICmpInst::ICMP_EQ, Cmp,
+ ConstantInt::get(Cmp->getType(), 1));
}
Instruction *Cloned = Instr->clone();
IRBuilder<> Builder(L->getLoopPreheader()->getTerminator());
// Find the loop boundaries.
const SCEV *BackedgeTakenCount = SE->getBackedgeTakenCount(OrigLoop);
- assert(BackedgeTakenCount != SE->getCouldNotCompute() && "Invalid loop count");
+ assert(BackedgeTakenCount != SE->getCouldNotCompute() &&
+ "Invalid loop count");
Type *IdxTy = Legal->getWidestInductionType();
assert(RdxPhi && "Unable to recover vectorized PHI");
// Find the reduction variable descriptor.
- assert(Legal->getReductionVars()->count(RdxPhi) &&
+ assert(Legal->isReductionVariable(RdxPhi) &&
"Unable to find the reduction variable");
RecurrenceDescriptor RdxDesc = (*Legal->getReductionVars())[RdxPhi];
return BlockMask;
}
-void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN,
- InnerLoopVectorizer::VectorParts &Entry,
- unsigned UF, unsigned VF, PhiVector *PV) {
+void InnerLoopVectorizer::widenPHIInstruction(
+ Instruction *PN, InnerLoopVectorizer::VectorParts &Entry, unsigned UF,
+ unsigned VF, PhiVector *PV) {
PHINode* P = cast<PHINode>(PN);
// Handle reduction variables:
- if (Legal->getReductionVars()->count(P)) {
+ if (Legal->isReductionVariable(P)) {
for (unsigned part = 0; part < UF; ++part) {
// This is phase one of vectorizing PHIs.
Type *VecTy = (VF == 1) ? PN->getType() :
case InductionDescriptor::IK_NoInduction:
llvm_unreachable("Unknown induction");
case InductionDescriptor::IK_IntInduction: {
- assert(P->getType() == II.getStartValue()->getType() && "Types must match");
+ assert(P->getType() == II.getStartValue()->getType() &&
+ "Types must match");
// Handle other induction variables that are now based on the
// canonical one.
Value *V = Induction;
Value *ScalarCast = Builder.CreateCast(CI->getOpcode(), Induction,
CI->getType());
Value *Broadcasted = getBroadcastInstrs(ScalarCast);
- InductionDescriptor II = Legal->getInductionVars()->lookup(OldInduction);
- Constant *Step =
- ConstantInt::getSigned(CI->getType(), II.getStepValue()->getSExtValue());
+ InductionDescriptor II =
+ Legal->getInductionVars()->lookup(OldInduction);
+ Constant *Step = ConstantInt::getSigned(
+ CI->getType(), II.getStepValue()->getSExtValue());
for (unsigned Part = 0; Part < UF; ++Part)
Entry[Part] = getStepVector(Broadcasted, VF * Part, Step);
propagateMetadata(Entry, &*it);
if (++NumPredStores > NumberOfStoresToPredicate || !isSafePtr ||
!isSinglePredecessor) {
- // Build a masked store if it is legal for the target, otherwise scalarize
- // the block.
+ // Build a masked store if it is legal for the target, otherwise
+ // scalarize the block.
bool isLegalMaskedOp =
isLegalMaskedStore(SI->getValueOperand()->getType(),
SI->getPointerOperand());
// Examine PHI nodes that are reduction variables. Update the type to
// account for the recurrence type.
if (PHINode *PN = dyn_cast<PHINode>(it)) {
- if (!Legal->getReductionVars()->count(PN))
+ if (!Legal->isReductionVariable(PN))
continue;
RecurrenceDescriptor RdxDesc = (*Legal->getReductionVars())[PN];
T = RdxDesc.getRecurrenceType();
}
// Interleave if this is a large loop (small loops are already dealt with by
- // this
- // point) that could benefit from interleaving.
+ // this point) that could benefit from interleaving.
bool HasReductions = (Legal->getReductionVars()->size() > 0);
if (TTI.enableAggressiveInterleaving(HasReductions)) {
DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
continue;
}
- // Count the number of live interals.
+ // Count the number of live intevals.
unsigned RegUsage = 0;
for (auto Inst : OpenIntervals)
RegUsage += GetRegUsage(Inst->getType(), VFs[j]);
case Instruction::ICmp:
case Instruction::FCmp: {
Type *ValTy = I->getOperand(0)->getType();
- if (VF > 1 && MinBWs.count(dyn_cast<Instruction>(I->getOperand(0))))
- ValTy = IntegerType::get(ValTy->getContext(), MinBWs[I]);
+ Instruction *Op0AsInstruction = dyn_cast<Instruction>(I->getOperand(0));
+ auto It = MinBWs.find(Op0AsInstruction);
+ if (VF > 1 && It != MinBWs.end())
+ ValTy = IntegerType::get(ValTy->getContext(), It->second);
VectorTy = ToVectorTy(ValTy, VF);
return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy);
}
projects / oota-llvm.git / blobdiff