1 //===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the auto-upgrade helper functions
12 //===----------------------------------------------------------------------===//
14 #include "llvm/AutoUpgrade.h"
15 #include "llvm/IR/Constants.h"
16 #include "llvm/IR/Function.h"
17 #include "llvm/IR/IRBuilder.h"
18 #include "llvm/IR/Instruction.h"
19 #include "llvm/IR/IntrinsicInst.h"
20 #include "llvm/IR/LLVMContext.h"
21 #include "llvm/IR/Module.h"
22 #include "llvm/Support/CFG.h"
23 #include "llvm/Support/CallSite.h"
24 #include "llvm/Support/ErrorHandling.h"
28 // Upgrade the declarations of the SSE4.1 functions whose arguments have
29 // changed their type from v4f32 to v2i64.
30 static bool UpgradeSSE41Function(Function* F, Intrinsic::ID IID,
32 // Check whether this is an old version of the function, which received
34 Type *Arg0Type = F->getFunctionType()->getParamType(0);
35 if (Arg0Type != VectorType::get(Type::getFloatTy(F->getContext()), 4))
38 // Yes, it's old, replace it with new version.
39 F->setName(F->getName() + ".old");
40 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
44 static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
45 assert(F && "Illegal to upgrade a non-existent Function.");
47 // Quickly eliminate it, if it's not a candidate.
48 StringRef Name = F->getName();
49 if (Name.size() <= 8 || !Name.startswith("llvm."))
51 Name = Name.substr(5); // Strip off "llvm."
56 if (Name.startswith("arm.neon.vclz")) {
58 F->arg_begin()->getType(),
59 Type::getInt1Ty(F->getContext())
61 // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to
62 // the end of the name. Change name from llvm.arm.neon.vclz.* to
64 FunctionType* fType = FunctionType::get(F->getReturnType(), args, false);
65 NewFn = Function::Create(fType, F->getLinkage(),
66 "llvm.ctlz." + Name.substr(14), F->getParent());
69 if (Name.startswith("arm.neon.vcnt")) {
70 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
71 F->arg_begin()->getType());
77 if (Name.startswith("ctlz.") && F->arg_size() == 1) {
78 F->setName(Name + ".old");
79 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
80 F->arg_begin()->getType());
83 if (Name.startswith("cttz.") && F->arg_size() == 1) {
84 F->setName(Name + ".old");
85 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz,
86 F->arg_begin()->getType());
92 // We only need to change the name to match the mangling including the
94 if (F->arg_size() == 2 && Name.startswith("objectsize.")) {
95 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
96 if (F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) {
97 F->setName(Name + ".old");
98 NewFn = Intrinsic::getDeclaration(F->getParent(),
99 Intrinsic::objectsize, Tys);
106 if (Name.startswith("x86.sse2.pcmpeq.") ||
107 Name.startswith("x86.sse2.pcmpgt.") ||
108 Name.startswith("x86.avx2.pcmpeq.") ||
109 Name.startswith("x86.avx2.pcmpgt.") ||
110 Name.startswith("x86.avx.vpermil.") ||
111 Name == "x86.avx.movnt.dq.256" ||
112 Name == "x86.avx.movnt.pd.256" ||
113 Name == "x86.avx.movnt.ps.256" ||
114 (Name.startswith("x86.xop.vpcom") && F->arg_size() == 2)) {
118 // SSE4.1 ptest functions may have an old signature.
119 if (Name.startswith("x86.sse41.ptest")) {
120 if (Name == "x86.sse41.ptestc")
121 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestc, NewFn);
122 if (Name == "x86.sse41.ptestz")
123 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestz, NewFn);
124 if (Name == "x86.sse41.ptestnzc")
125 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestnzc, NewFn);
127 // frcz.ss/sd may need to have an argument dropped
128 if (Name.startswith("x86.xop.vfrcz.ss") && F->arg_size() == 2) {
129 F->setName(Name + ".old");
130 NewFn = Intrinsic::getDeclaration(F->getParent(),
131 Intrinsic::x86_xop_vfrcz_ss);
134 if (Name.startswith("x86.xop.vfrcz.sd") && F->arg_size() == 2) {
135 F->setName(Name + ".old");
136 NewFn = Intrinsic::getDeclaration(F->getParent(),
137 Intrinsic::x86_xop_vfrcz_sd);
140 // Fix the FMA4 intrinsics to remove the 4
141 if (Name.startswith("x86.fma4.")) {
142 F->setName("llvm.x86.fma" + Name.substr(8));
150 // This may not belong here. This function is effectively being overloaded
151 // to both detect an intrinsic which needs upgrading, and to provide the
152 // upgraded form of the intrinsic. We should perhaps have two separate
153 // functions for this.
157 bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {
159 bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn);
161 // Upgrade intrinsic attributes. This does not change the function.
164 if (unsigned id = F->getIntrinsicID())
165 F->setAttributes(Intrinsic::getAttributes(F->getContext(),
170 bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
171 // Nothing to do yet.
175 // UpgradeIntrinsicCall - Upgrade a call to an old intrinsic to be a call the
176 // upgraded intrinsic. All argument and return casting must be provided in
177 // order to seamlessly integrate with existing context.
178 void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
179 Function *F = CI->getCalledFunction();
180 LLVMContext &C = CI->getContext();
181 IRBuilder<> Builder(C);
182 Builder.SetInsertPoint(CI->getParent(), CI);
184 assert(F && "Intrinsic call is not direct?");
187 // Get the Function's name.
188 StringRef Name = F->getName();
191 // Upgrade packed integer vector compares intrinsics to compare instructions
192 if (Name.startswith("llvm.x86.sse2.pcmpeq.") ||
193 Name.startswith("llvm.x86.avx2.pcmpeq.")) {
194 Rep = Builder.CreateICmpEQ(CI->getArgOperand(0), CI->getArgOperand(1),
196 // need to sign extend since icmp returns vector of i1
197 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
198 } else if (Name.startswith("llvm.x86.sse2.pcmpgt.") ||
199 Name.startswith("llvm.x86.avx2.pcmpgt.")) {
200 Rep = Builder.CreateICmpSGT(CI->getArgOperand(0), CI->getArgOperand(1),
202 // need to sign extend since icmp returns vector of i1
203 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
204 } else if (Name == "llvm.x86.avx.movnt.dq.256" ||
205 Name == "llvm.x86.avx.movnt.ps.256" ||
206 Name == "llvm.x86.avx.movnt.pd.256") {
207 IRBuilder<> Builder(C);
208 Builder.SetInsertPoint(CI->getParent(), CI);
210 Module *M = F->getParent();
211 SmallVector<Value *, 1> Elts;
212 Elts.push_back(ConstantInt::get(Type::getInt32Ty(C), 1));
213 MDNode *Node = MDNode::get(C, Elts);
215 Value *Arg0 = CI->getArgOperand(0);
216 Value *Arg1 = CI->getArgOperand(1);
218 // Convert the type of the pointer to a pointer to the stored type.
219 Value *BC = Builder.CreateBitCast(Arg0,
220 PointerType::getUnqual(Arg1->getType()),
222 StoreInst *SI = Builder.CreateStore(Arg1, BC);
223 SI->setMetadata(M->getMDKindID("nontemporal"), Node);
224 SI->setAlignment(16);
227 CI->eraseFromParent();
229 } else if (Name.startswith("llvm.x86.xop.vpcom")) {
231 if (Name.endswith("ub"))
232 intID = Intrinsic::x86_xop_vpcomub;
233 else if (Name.endswith("uw"))
234 intID = Intrinsic::x86_xop_vpcomuw;
235 else if (Name.endswith("ud"))
236 intID = Intrinsic::x86_xop_vpcomud;
237 else if (Name.endswith("uq"))
238 intID = Intrinsic::x86_xop_vpcomuq;
239 else if (Name.endswith("b"))
240 intID = Intrinsic::x86_xop_vpcomb;
241 else if (Name.endswith("w"))
242 intID = Intrinsic::x86_xop_vpcomw;
243 else if (Name.endswith("d"))
244 intID = Intrinsic::x86_xop_vpcomd;
245 else if (Name.endswith("q"))
246 intID = Intrinsic::x86_xop_vpcomq;
248 llvm_unreachable("Unknown suffix");
250 Name = Name.substr(18); // strip off "llvm.x86.xop.vpcom"
252 if (Name.startswith("lt"))
254 else if (Name.startswith("le"))
256 else if (Name.startswith("gt"))
258 else if (Name.startswith("ge"))
260 else if (Name.startswith("eq"))
262 else if (Name.startswith("ne"))
264 else if (Name.startswith("true"))
266 else if (Name.startswith("false"))
269 llvm_unreachable("Unknown condition");
271 Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID);
272 Rep = Builder.CreateCall3(VPCOM, CI->getArgOperand(0),
273 CI->getArgOperand(1), Builder.getInt8(Imm));
275 bool PD128 = false, PD256 = false, PS128 = false, PS256 = false;
276 if (Name == "llvm.x86.avx.vpermil.pd.256")
278 else if (Name == "llvm.x86.avx.vpermil.pd")
280 else if (Name == "llvm.x86.avx.vpermil.ps.256")
282 else if (Name == "llvm.x86.avx.vpermil.ps")
285 if (PD256 || PD128 || PS256 || PS128) {
286 Value *Op0 = CI->getArgOperand(0);
287 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
288 SmallVector<Constant*, 8> Idxs;
291 for (unsigned i = 0; i != 2; ++i)
292 Idxs.push_back(Builder.getInt32((Imm >> i) & 0x1));
294 for (unsigned l = 0; l != 4; l+=2)
295 for (unsigned i = 0; i != 2; ++i)
296 Idxs.push_back(Builder.getInt32(((Imm >> (l+i)) & 0x1) + l));
298 for (unsigned i = 0; i != 4; ++i)
299 Idxs.push_back(Builder.getInt32((Imm >> (2 * i)) & 0x3));
301 for (unsigned l = 0; l != 8; l+=4)
302 for (unsigned i = 0; i != 4; ++i)
303 Idxs.push_back(Builder.getInt32(((Imm >> (2 * i)) & 0x3) + l));
305 llvm_unreachable("Unexpected function");
307 Rep = Builder.CreateShuffleVector(Op0, Op0, ConstantVector::get(Idxs));
309 llvm_unreachable("Unknown function for CallInst upgrade.");
313 CI->replaceAllUsesWith(Rep);
314 CI->eraseFromParent();
318 std::string Name = CI->getName().str();
319 CI->setName(Name + ".old");
321 switch (NewFn->getIntrinsicID()) {
323 llvm_unreachable("Unknown function for CallInst upgrade.");
325 case Intrinsic::ctlz:
326 case Intrinsic::cttz:
327 assert(CI->getNumArgOperands() == 1 &&
328 "Mismatch between function args and call args");
329 CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0),
330 Builder.getFalse(), Name));
331 CI->eraseFromParent();
334 case Intrinsic::objectsize:
335 CI->replaceAllUsesWith(Builder.CreateCall2(NewFn,
336 CI->getArgOperand(0),
337 CI->getArgOperand(1),
339 CI->eraseFromParent();
342 case Intrinsic::arm_neon_vclz: {
343 // Change name from llvm.arm.neon.vclz.* to llvm.ctlz.*
344 CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0),
346 "llvm.ctlz." + Name.substr(14)));
347 CI->eraseFromParent();
350 case Intrinsic::ctpop: {
351 CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(0)));
352 CI->eraseFromParent();
356 case Intrinsic::x86_xop_vfrcz_ss:
357 case Intrinsic::x86_xop_vfrcz_sd:
358 CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(1),
360 CI->eraseFromParent();
363 case Intrinsic::x86_sse41_ptestc:
364 case Intrinsic::x86_sse41_ptestz:
365 case Intrinsic::x86_sse41_ptestnzc: {
366 // The arguments for these intrinsics used to be v4f32, and changed
367 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
368 // So, the only thing required is a bitcast for both arguments.
369 // First, check the arguments have the old type.
370 Value *Arg0 = CI->getArgOperand(0);
371 if (Arg0->getType() != VectorType::get(Type::getFloatTy(C), 4))
374 // Old intrinsic, add bitcasts
375 Value *Arg1 = CI->getArgOperand(1);
378 Builder.CreateBitCast(Arg0,
379 VectorType::get(Type::getInt64Ty(C), 2),
382 Builder.CreateBitCast(Arg1,
383 VectorType::get(Type::getInt64Ty(C), 2),
386 CallInst* NewCall = Builder.CreateCall2(NewFn, BC0, BC1, Name);
387 CI->replaceAllUsesWith(NewCall);
388 CI->eraseFromParent();
394 // This tests each Function to determine if it needs upgrading. When we find
395 // one we are interested in, we then upgrade all calls to reflect the new
397 void llvm::UpgradeCallsToIntrinsic(Function* F) {
398 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
400 // Upgrade the function and check if it is a totaly new function.
402 if (UpgradeIntrinsicFunction(F, NewFn)) {
404 // Replace all uses to the old function with the new one if necessary.
405 for (Value::use_iterator UI = F->use_begin(), UE = F->use_end();
407 if (CallInst *CI = dyn_cast<CallInst>(*UI++))
408 UpgradeIntrinsicCall(CI, NewFn);
410 // Remove old function, no longer used, from the module.
411 F->eraseFromParent();
416 void llvm::UpgradeInstWithTBAATag(Instruction *I) {
417 MDNode *MD = I->getMetadata(LLVMContext::MD_tbaa);
418 assert(MD && "UpgradeInstWithTBAATag should have a TBAA tag");
419 // Check if the tag uses struct-path aware TBAA format.
420 if (isa<MDNode>(MD->getOperand(0)) && MD->getNumOperands() >= 3)
423 if (MD->getNumOperands() == 3) {
428 MDNode *ScalarType = MDNode::get(I->getContext(), Elts);
429 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
431 ScalarType, ScalarType,
432 Constant::getNullValue(Type::getInt64Ty(I->getContext())),
435 I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts2));
437 // Create a MDNode <MD, MD, offset 0>
438 Value *Elts[] = {MD, MD,
439 Constant::getNullValue(Type::getInt64Ty(I->getContext()))};
440 I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts));