1 //===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the auto-upgrade helper functions
12 //===----------------------------------------------------------------------===//
14 #include "llvm/IR/AutoUpgrade.h"
15 #include "llvm/IR/CFG.h"
16 #include "llvm/IR/CallSite.h"
17 #include "llvm/IR/Constants.h"
18 #include "llvm/IR/DIBuilder.h"
19 #include "llvm/IR/DebugInfo.h"
20 #include "llvm/IR/DiagnosticInfo.h"
21 #include "llvm/IR/Function.h"
22 #include "llvm/IR/IRBuilder.h"
23 #include "llvm/IR/Instruction.h"
24 #include "llvm/IR/IntrinsicInst.h"
25 #include "llvm/IR/LLVMContext.h"
26 #include "llvm/IR/Module.h"
27 #include "llvm/Support/ErrorHandling.h"
31 // Upgrade the declarations of the SSE4.1 functions whose arguments have
32 // changed their type from v4f32 to v2i64.
33 static bool UpgradeSSE41Function(Function* F, Intrinsic::ID IID,
35 // Check whether this is an old version of the function, which received
37 Type *Arg0Type = F->getFunctionType()->getParamType(0);
38 if (Arg0Type != VectorType::get(Type::getFloatTy(F->getContext()), 4))
41 // Yes, it's old, replace it with new version.
42 F->setName(F->getName() + ".old");
43 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
47 // Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
48 // arguments have changed their type from i32 to i8.
49 static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
51 // Check that the last argument is an i32.
52 Type *LastArgType = F->getFunctionType()->getParamType(
53 F->getFunctionType()->getNumParams() - 1);
54 if (!LastArgType->isIntegerTy(32))
57 // Move this function aside and map down.
58 F->setName(F->getName() + ".old");
59 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
63 // Upgrade the declarations of AVX-512 cmp intrinsic functions whose 8-bit
64 // immediates have changed their type from i32 to i8.
65 static bool UpgradeAVX512CmpIntrinsic(Function *F, Intrinsic::ID IID,
67 // Check that the last argument is an i32.
68 Type *LastArgType = F->getFunctionType()->getParamType(2);
69 if (!LastArgType->isIntegerTy(32))
72 // Move this function aside and map down.
73 F->setName(F->getName() + ".old");
74 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
78 static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
79 assert(F && "Illegal to upgrade a non-existent Function.");
81 // Quickly eliminate it, if it's not a candidate.
82 StringRef Name = F->getName();
83 if (Name.size() <= 8 || !Name.startswith("llvm."))
85 Name = Name.substr(5); // Strip off "llvm."
90 if (Name.startswith("arm.neon.vclz")) {
92 F->arg_begin()->getType(),
93 Type::getInt1Ty(F->getContext())
95 // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to
96 // the end of the name. Change name from llvm.arm.neon.vclz.* to
98 FunctionType* fType = FunctionType::get(F->getReturnType(), args, false);
99 NewFn = Function::Create(fType, F->getLinkage(),
100 "llvm.ctlz." + Name.substr(14), F->getParent());
103 if (Name.startswith("arm.neon.vcnt")) {
104 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
105 F->arg_begin()->getType());
111 if (Name.startswith("ctlz.") && F->arg_size() == 1) {
112 F->setName(Name + ".old");
113 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
114 F->arg_begin()->getType());
117 if (Name.startswith("cttz.") && F->arg_size() == 1) {
118 F->setName(Name + ".old");
119 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz,
120 F->arg_begin()->getType());
126 if (Name.startswith("dbg.declare") && F->arg_size() == 2) {
127 F->setName(Name + ".old");
128 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_declare);
131 if (Name.startswith("dbg.value") && F->arg_size() == 3) {
132 F->setName(Name + ".old");
133 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value);
140 // We only need to change the name to match the mangling including the
142 if (F->arg_size() == 2 && Name.startswith("objectsize.")) {
143 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
144 if (F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) {
145 F->setName(Name + ".old");
146 NewFn = Intrinsic::getDeclaration(F->getParent(),
147 Intrinsic::objectsize, Tys);
154 if (Name.startswith("x86.sse2.pcmpeq.") ||
155 Name.startswith("x86.sse2.pcmpgt.") ||
156 Name.startswith("x86.avx2.pcmpeq.") ||
157 Name.startswith("x86.avx2.pcmpgt.") ||
158 Name.startswith("x86.avx.vpermil.") ||
159 Name == "x86.avx.movnt.dq.256" ||
160 Name == "x86.avx.movnt.pd.256" ||
161 Name == "x86.avx.movnt.ps.256" ||
162 Name == "x86.sse42.crc32.64.8" ||
163 Name == "x86.avx.vbroadcast.ss" ||
164 Name == "x86.avx.vbroadcast.ss.256" ||
165 Name == "x86.avx.vbroadcast.sd.256" ||
166 Name == "x86.sse2.psll.dq.bs" ||
167 Name == "x86.sse2.psrl.dq.bs" ||
168 Name == "x86.avx2.psll.dq.bs" ||
169 Name == "x86.avx2.psrl.dq.bs" ||
170 (Name.startswith("x86.xop.vpcom") && F->arg_size() == 2)) {
174 // SSE4.1 ptest functions may have an old signature.
175 if (Name.startswith("x86.sse41.ptest")) {
176 if (Name == "x86.sse41.ptestc")
177 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestc, NewFn);
178 if (Name == "x86.sse41.ptestz")
179 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestz, NewFn);
180 if (Name == "x86.sse41.ptestnzc")
181 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestnzc, NewFn);
183 // Several blend and other instructions with maskes used the wrong number of
185 if (Name == "x86.sse41.pblendw")
186 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_pblendw,
188 if (Name == "x86.sse41.blendpd")
189 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_blendpd,
191 if (Name == "x86.sse41.blendps")
192 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_blendps,
194 if (Name == "x86.sse41.insertps")
195 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps,
197 if (Name == "x86.sse41.dppd")
198 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd,
200 if (Name == "x86.sse41.dpps")
201 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps,
203 if (Name == "x86.sse41.mpsadbw")
204 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw,
206 if (Name == "x86.avx.blend.pd.256")
207 return UpgradeX86IntrinsicsWith8BitMask(
208 F, Intrinsic::x86_avx_blend_pd_256, NewFn);
209 if (Name == "x86.avx.blend.ps.256")
210 return UpgradeX86IntrinsicsWith8BitMask(
211 F, Intrinsic::x86_avx_blend_ps_256, NewFn);
212 if (Name == "x86.avx.dp.ps.256")
213 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256,
215 if (Name == "x86.avx2.pblendw")
216 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_pblendw,
218 if (Name == "x86.avx2.pblendd.128")
219 return UpgradeX86IntrinsicsWith8BitMask(
220 F, Intrinsic::x86_avx2_pblendd_128, NewFn);
221 if (Name == "x86.avx2.pblendd.256")
222 return UpgradeX86IntrinsicsWith8BitMask(
223 F, Intrinsic::x86_avx2_pblendd_256, NewFn);
224 if (Name == "x86.avx2.mpsadbw")
225 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw,
228 if (Name == "x86.avx512.mask.cmp.ps.512")
229 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_ps_512,
231 if (Name == "x86.avx512.mask.cmp.pd.512")
232 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_pd_512,
235 if (Name == "x86.avx512.mask.cmp.b.512")
236 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_512,
238 if (Name == "x86.avx512.mask.cmp.w.512")
239 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_512,
241 if (Name == "x86.avx512.mask.cmp.d.512")
242 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_512,
244 if (Name == "x86.avx512.mask.cmp.q.512")
245 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_512,
247 if (Name == "x86.avx512.mask.ucmp.b.512")
248 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_512,
250 if (Name == "x86.avx512.mask.ucmp.w.512")
251 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_512,
253 if (Name == "x86.avx512.mask.ucmp.d.512")
254 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_512,
256 if (Name == "x86.avx512.mask.ucmp.q.512")
257 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_512,
260 if (Name == "x86.avx512.mask.cmp.b.256")
261 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_256,
263 if (Name == "x86.avx512.mask.cmp.w.256")
264 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_256,
266 if (Name == "x86.avx512.mask.cmp.d.256")
267 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_256,
269 if (Name == "x86.avx512.mask.cmp.q.256")
270 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_256,
272 if (Name == "x86.avx512.mask.ucmp.b.256")
273 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_256,
275 if (Name == "x86.avx512.mask.ucmp.w.256")
276 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_256,
278 if (Name == "x86.avx512.mask.ucmp.d.256")
279 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_256,
281 if (Name == "x86.avx512.mask.ucmp.q.256")
282 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_256,
285 if (Name == "x86.avx512.mask.cmp.b.128")
286 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_128,
288 if (Name == "x86.avx512.mask.cmp.w.128")
289 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_128,
291 if (Name == "x86.avx512.mask.cmp.d.128")
292 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_128,
294 if (Name == "x86.avx512.mask.cmp.q.128")
295 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_128,
297 if (Name == "x86.avx512.mask.ucmp.b.128")
298 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_128,
300 if (Name == "x86.avx512.mask.ucmp.w.128")
301 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_128,
303 if (Name == "x86.avx512.mask.ucmp.d.128")
304 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_128,
306 if (Name == "x86.avx512.mask.ucmp.q.128")
307 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_128,
310 // frcz.ss/sd may need to have an argument dropped
311 if (Name.startswith("x86.xop.vfrcz.ss") && F->arg_size() == 2) {
312 F->setName(Name + ".old");
313 NewFn = Intrinsic::getDeclaration(F->getParent(),
314 Intrinsic::x86_xop_vfrcz_ss);
317 if (Name.startswith("x86.xop.vfrcz.sd") && F->arg_size() == 2) {
318 F->setName(Name + ".old");
319 NewFn = Intrinsic::getDeclaration(F->getParent(),
320 Intrinsic::x86_xop_vfrcz_sd);
323 // Fix the FMA4 intrinsics to remove the 4
324 if (Name.startswith("x86.fma4.")) {
325 F->setName("llvm.x86.fma" + Name.substr(8));
333 // This may not belong here. This function is effectively being overloaded
334 // to both detect an intrinsic which needs upgrading, and to provide the
335 // upgraded form of the intrinsic. We should perhaps have two separate
336 // functions for this.
340 bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {
342 bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn);
344 // Upgrade intrinsic attributes. This does not change the function.
347 if (unsigned id = F->getIntrinsicID())
348 F->setAttributes(Intrinsic::getAttributes(F->getContext(),
353 bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
354 // Nothing to do yet.
358 static MDNode *getNodeField(const MDNode *DbgNode, unsigned Elt) {
359 if (!DbgNode || Elt >= DbgNode->getNumOperands())
361 return dyn_cast_or_null<MDNode>(DbgNode->getOperand(Elt));
364 static MetadataAsValue *getExpression(Value *VarOperand, Function *F) {
365 // Old-style DIVariables have an optional expression as the 8th element.
366 DIExpression Expr(getNodeField(
367 cast<MDNode>(cast<MetadataAsValue>(VarOperand)->getMetadata()), 8));
369 DIBuilder DIB(*F->getParent(), /*AllowUnresolved*/ false);
370 Expr = DIB.createExpression();
372 return MetadataAsValue::get(F->getContext(), Expr);
375 // UpgradeIntrinsicCall - Upgrade a call to an old intrinsic to be a call the
376 // upgraded intrinsic. All argument and return casting must be provided in
377 // order to seamlessly integrate with existing context.
378 void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
379 Function *F = CI->getCalledFunction();
380 LLVMContext &C = CI->getContext();
381 IRBuilder<> Builder(C);
382 Builder.SetInsertPoint(CI->getParent(), CI);
384 assert(F && "Intrinsic call is not direct?");
387 // Get the Function's name.
388 StringRef Name = F->getName();
391 // Upgrade packed integer vector compares intrinsics to compare instructions
392 if (Name.startswith("llvm.x86.sse2.pcmpeq.") ||
393 Name.startswith("llvm.x86.avx2.pcmpeq.")) {
394 Rep = Builder.CreateICmpEQ(CI->getArgOperand(0), CI->getArgOperand(1),
396 // need to sign extend since icmp returns vector of i1
397 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
398 } else if (Name.startswith("llvm.x86.sse2.pcmpgt.") ||
399 Name.startswith("llvm.x86.avx2.pcmpgt.")) {
400 Rep = Builder.CreateICmpSGT(CI->getArgOperand(0), CI->getArgOperand(1),
402 // need to sign extend since icmp returns vector of i1
403 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
404 } else if (Name == "llvm.x86.avx.movnt.dq.256" ||
405 Name == "llvm.x86.avx.movnt.ps.256" ||
406 Name == "llvm.x86.avx.movnt.pd.256") {
407 IRBuilder<> Builder(C);
408 Builder.SetInsertPoint(CI->getParent(), CI);
410 Module *M = F->getParent();
411 SmallVector<Metadata *, 1> Elts;
413 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
414 MDNode *Node = MDNode::get(C, Elts);
416 Value *Arg0 = CI->getArgOperand(0);
417 Value *Arg1 = CI->getArgOperand(1);
419 // Convert the type of the pointer to a pointer to the stored type.
420 Value *BC = Builder.CreateBitCast(Arg0,
421 PointerType::getUnqual(Arg1->getType()),
423 StoreInst *SI = Builder.CreateStore(Arg1, BC);
424 SI->setMetadata(M->getMDKindID("nontemporal"), Node);
425 SI->setAlignment(16);
428 CI->eraseFromParent();
430 } else if (Name.startswith("llvm.x86.xop.vpcom")) {
432 if (Name.endswith("ub"))
433 intID = Intrinsic::x86_xop_vpcomub;
434 else if (Name.endswith("uw"))
435 intID = Intrinsic::x86_xop_vpcomuw;
436 else if (Name.endswith("ud"))
437 intID = Intrinsic::x86_xop_vpcomud;
438 else if (Name.endswith("uq"))
439 intID = Intrinsic::x86_xop_vpcomuq;
440 else if (Name.endswith("b"))
441 intID = Intrinsic::x86_xop_vpcomb;
442 else if (Name.endswith("w"))
443 intID = Intrinsic::x86_xop_vpcomw;
444 else if (Name.endswith("d"))
445 intID = Intrinsic::x86_xop_vpcomd;
446 else if (Name.endswith("q"))
447 intID = Intrinsic::x86_xop_vpcomq;
449 llvm_unreachable("Unknown suffix");
451 Name = Name.substr(18); // strip off "llvm.x86.xop.vpcom"
453 if (Name.startswith("lt"))
455 else if (Name.startswith("le"))
457 else if (Name.startswith("gt"))
459 else if (Name.startswith("ge"))
461 else if (Name.startswith("eq"))
463 else if (Name.startswith("ne"))
465 else if (Name.startswith("false"))
467 else if (Name.startswith("true"))
470 llvm_unreachable("Unknown condition");
472 Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID);
473 Rep = Builder.CreateCall3(VPCOM, CI->getArgOperand(0),
474 CI->getArgOperand(1), Builder.getInt8(Imm));
475 } else if (Name == "llvm.x86.sse42.crc32.64.8") {
476 Function *CRC32 = Intrinsic::getDeclaration(F->getParent(),
477 Intrinsic::x86_sse42_crc32_32_8);
478 Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
479 Rep = Builder.CreateCall2(CRC32, Trunc0, CI->getArgOperand(1));
480 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
481 } else if (Name.startswith("llvm.x86.avx.vbroadcast")) {
482 // Replace broadcasts with a series of insertelements.
483 Type *VecTy = CI->getType();
484 Type *EltTy = VecTy->getVectorElementType();
485 unsigned EltNum = VecTy->getVectorNumElements();
486 Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),
487 EltTy->getPointerTo());
488 Value *Load = Builder.CreateLoad(Cast);
489 Type *I32Ty = Type::getInt32Ty(C);
490 Rep = UndefValue::get(VecTy);
491 for (unsigned I = 0; I < EltNum; ++I)
492 Rep = Builder.CreateInsertElement(Rep, Load,
493 ConstantInt::get(I32Ty, I));
494 } else if (Name == "llvm.x86.sse2.psll.dq.bs") {
495 Value *Op0 = ConstantVector::getSplat(16, Builder.getInt8(0));
496 Value *Op1 = Builder.CreateBitCast(CI->getArgOperand(0),
497 VectorType::get(Type::getInt8Ty(C),16),
500 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
503 SmallVector<Constant*, 16> Idxs;
504 for (unsigned i = 16; i != 32; ++i)
505 Idxs.push_back(Builder.getInt32(i - Shift));
507 Op0 = Builder.CreateShuffleVector(Op0, Op1, ConstantVector::get(Idxs));
510 Rep = Builder.CreateBitCast(Op0,
511 VectorType::get(Type::getInt64Ty(C), 2),
513 } else if (Name == "llvm.x86.sse2.psrl.dq.bs") {
514 Value *Op0 = Builder.CreateBitCast(CI->getArgOperand(0),
515 VectorType::get(Type::getInt8Ty(C),16),
517 Value *Op1 = ConstantVector::getSplat(16, Builder.getInt8(0));
519 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
522 SmallVector<Constant*, 16> Idxs;
523 for (unsigned i = 0; i != 16; ++i)
524 Idxs.push_back(Builder.getInt32(i + Shift));
526 Op1 = Builder.CreateShuffleVector(Op0, Op1, ConstantVector::get(Idxs));
528 Rep = Builder.CreateBitCast(Op1,
529 VectorType::get(Type::getInt64Ty(C), 2),
531 } else if (Name == "llvm.x86.avx2.psll.dq.bs") {
532 Value *Op0 = ConstantVector::getSplat(32, Builder.getInt8(0));
533 Value *Op1 = Builder.CreateBitCast(CI->getArgOperand(0),
534 VectorType::get(Type::getInt8Ty(C),32),
537 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
540 SmallVector<Constant*, 32> Idxs;
541 for (unsigned l = 0; l != 32; l += 16)
542 for (unsigned i = 0; i != 16; ++i) {
543 unsigned Idx = 32 + i - Shift;
544 if (Idx < 32) Idx -= 16; // end of lane, switch operand.
545 Idxs.push_back(Builder.getInt32(Idx + l));
548 Op1 = Builder.CreateShuffleVector(Op0, Op1, ConstantVector::get(Idxs));
551 Rep = Builder.CreateBitCast(Op1,
552 VectorType::get(Type::getInt64Ty(C), 4),
554 } else if (Name == "llvm.x86.avx2.psrl.dq.bs") {
555 Value *Op0 = Builder.CreateBitCast(CI->getArgOperand(0),
556 VectorType::get(Type::getInt8Ty(C),32),
558 Value *Op1 = ConstantVector::getSplat(32, Builder.getInt8(0));
560 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
563 SmallVector<Constant*, 32> Idxs;
564 for (unsigned l = 0; l != 32; l += 16)
565 for (unsigned i = 0; i != 16; ++i) {
566 unsigned Idx = i + Shift;
567 if (Idx >= 16) Idx += 16; // end of lane, switch operand.
568 Idxs.push_back(Builder.getInt32(Idx + l));
571 Op0 = Builder.CreateShuffleVector(Op0, Op1, ConstantVector::get(Idxs));
574 Rep = Builder.CreateBitCast(Op0,
575 VectorType::get(Type::getInt64Ty(C), 4),
578 bool PD128 = false, PD256 = false, PS128 = false, PS256 = false;
579 if (Name == "llvm.x86.avx.vpermil.pd.256")
581 else if (Name == "llvm.x86.avx.vpermil.pd")
583 else if (Name == "llvm.x86.avx.vpermil.ps.256")
585 else if (Name == "llvm.x86.avx.vpermil.ps")
588 if (PD256 || PD128 || PS256 || PS128) {
589 Value *Op0 = CI->getArgOperand(0);
590 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
591 SmallVector<Constant*, 8> Idxs;
594 for (unsigned i = 0; i != 2; ++i)
595 Idxs.push_back(Builder.getInt32((Imm >> i) & 0x1));
597 for (unsigned l = 0; l != 4; l+=2)
598 for (unsigned i = 0; i != 2; ++i)
599 Idxs.push_back(Builder.getInt32(((Imm >> (l+i)) & 0x1) + l));
601 for (unsigned i = 0; i != 4; ++i)
602 Idxs.push_back(Builder.getInt32((Imm >> (2 * i)) & 0x3));
604 for (unsigned l = 0; l != 8; l+=4)
605 for (unsigned i = 0; i != 4; ++i)
606 Idxs.push_back(Builder.getInt32(((Imm >> (2 * i)) & 0x3) + l));
608 llvm_unreachable("Unexpected function");
610 Rep = Builder.CreateShuffleVector(Op0, Op0, ConstantVector::get(Idxs));
612 llvm_unreachable("Unknown function for CallInst upgrade.");
616 CI->replaceAllUsesWith(Rep);
617 CI->eraseFromParent();
621 std::string Name = CI->getName().str();
623 CI->setName(Name + ".old");
625 switch (NewFn->getIntrinsicID()) {
627 llvm_unreachable("Unknown function for CallInst upgrade.");
629 // Upgrade debug intrinsics to use an additional DIExpression argument.
630 case Intrinsic::dbg_declare: {
632 Builder.CreateCall3(NewFn, CI->getArgOperand(0), CI->getArgOperand(1),
633 getExpression(CI->getArgOperand(1), F), Name);
634 NewCI->setDebugLoc(CI->getDebugLoc());
635 CI->replaceAllUsesWith(NewCI);
636 CI->eraseFromParent();
639 case Intrinsic::dbg_value: {
640 auto NewCI = Builder.CreateCall4(
641 NewFn, CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
642 getExpression(CI->getArgOperand(2), F), Name);
643 NewCI->setDebugLoc(CI->getDebugLoc());
644 CI->replaceAllUsesWith(NewCI);
645 CI->eraseFromParent();
648 case Intrinsic::ctlz:
649 case Intrinsic::cttz:
650 assert(CI->getNumArgOperands() == 1 &&
651 "Mismatch between function args and call args");
652 CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0),
653 Builder.getFalse(), Name));
654 CI->eraseFromParent();
657 case Intrinsic::objectsize:
658 CI->replaceAllUsesWith(Builder.CreateCall2(NewFn,
659 CI->getArgOperand(0),
660 CI->getArgOperand(1),
662 CI->eraseFromParent();
665 case Intrinsic::ctpop: {
666 CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(0)));
667 CI->eraseFromParent();
671 case Intrinsic::x86_xop_vfrcz_ss:
672 case Intrinsic::x86_xop_vfrcz_sd:
673 CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(1),
675 CI->eraseFromParent();
678 case Intrinsic::x86_sse41_ptestc:
679 case Intrinsic::x86_sse41_ptestz:
680 case Intrinsic::x86_sse41_ptestnzc: {
681 // The arguments for these intrinsics used to be v4f32, and changed
682 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
683 // So, the only thing required is a bitcast for both arguments.
684 // First, check the arguments have the old type.
685 Value *Arg0 = CI->getArgOperand(0);
686 if (Arg0->getType() != VectorType::get(Type::getFloatTy(C), 4))
689 // Old intrinsic, add bitcasts
690 Value *Arg1 = CI->getArgOperand(1);
693 Builder.CreateBitCast(Arg0,
694 VectorType::get(Type::getInt64Ty(C), 2),
697 Builder.CreateBitCast(Arg1,
698 VectorType::get(Type::getInt64Ty(C), 2),
701 CallInst* NewCall = Builder.CreateCall2(NewFn, BC0, BC1, Name);
702 CI->replaceAllUsesWith(NewCall);
703 CI->eraseFromParent();
707 case Intrinsic::x86_sse41_pblendw:
708 case Intrinsic::x86_sse41_blendpd:
709 case Intrinsic::x86_sse41_blendps:
710 case Intrinsic::x86_sse41_insertps:
711 case Intrinsic::x86_sse41_dppd:
712 case Intrinsic::x86_sse41_dpps:
713 case Intrinsic::x86_sse41_mpsadbw:
714 case Intrinsic::x86_avx_blend_pd_256:
715 case Intrinsic::x86_avx_blend_ps_256:
716 case Intrinsic::x86_avx_dp_ps_256:
717 case Intrinsic::x86_avx2_pblendw:
718 case Intrinsic::x86_avx2_pblendd_128:
719 case Intrinsic::x86_avx2_pblendd_256:
720 case Intrinsic::x86_avx2_mpsadbw: {
721 // Need to truncate the last argument from i32 to i8 -- this argument models
722 // an inherently 8-bit immediate operand to these x86 instructions.
723 SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
724 CI->arg_operands().end());
726 // Replace the last argument with a trunc.
727 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
729 CallInst *NewCall = Builder.CreateCall(NewFn, Args);
730 CI->replaceAllUsesWith(NewCall);
731 CI->eraseFromParent();
734 case Intrinsic::x86_avx512_mask_cmp_ps_512:
735 case Intrinsic::x86_avx512_mask_cmp_pd_512: {
736 // Need to truncate the last argument from i32 to i8 -- this argument models
737 // an inherently 8-bit immediate operand to these x86 instructions.
738 SmallVector<Value *, 5> Args(CI->arg_operands().begin(),
739 CI->arg_operands().end());
741 // Replace the last argument with a trunc.
742 Args[2] = Builder.CreateTrunc(Args[2], Type::getInt8Ty(C), "trunc");
744 CallInst *NewCall = Builder.CreateCall(NewFn, Args);
745 CI->replaceAllUsesWith(NewCall);
746 CI->eraseFromParent();
752 // This tests each Function to determine if it needs upgrading. When we find
753 // one we are interested in, we then upgrade all calls to reflect the new
755 void llvm::UpgradeCallsToIntrinsic(Function* F) {
756 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
758 // Upgrade the function and check if it is a totaly new function.
760 if (UpgradeIntrinsicFunction(F, NewFn)) {
762 // Replace all uses to the old function with the new one if necessary.
763 for (Value::user_iterator UI = F->user_begin(), UE = F->user_end();
765 if (CallInst *CI = dyn_cast<CallInst>(*UI++))
766 UpgradeIntrinsicCall(CI, NewFn);
768 // Remove old function, no longer used, from the module.
769 F->eraseFromParent();
774 void llvm::UpgradeInstWithTBAATag(Instruction *I) {
775 MDNode *MD = I->getMetadata(LLVMContext::MD_tbaa);
776 assert(MD && "UpgradeInstWithTBAATag should have a TBAA tag");
777 // Check if the tag uses struct-path aware TBAA format.
778 if (isa<MDNode>(MD->getOperand(0)) && MD->getNumOperands() >= 3)
781 if (MD->getNumOperands() == 3) {
782 Metadata *Elts[] = {MD->getOperand(0), MD->getOperand(1)};
783 MDNode *ScalarType = MDNode::get(I->getContext(), Elts);
784 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
785 Metadata *Elts2[] = {ScalarType, ScalarType,
786 ConstantAsMetadata::get(Constant::getNullValue(
787 Type::getInt64Ty(I->getContext()))),
789 I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts2));
791 // Create a MDNode <MD, MD, offset 0>
792 Metadata *Elts[] = {MD, MD, ConstantAsMetadata::get(Constant::getNullValue(
793 Type::getInt64Ty(I->getContext())))};
794 I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts));
798 Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
799 Instruction *&Temp) {
800 if (Opc != Instruction::BitCast)
804 Type *SrcTy = V->getType();
805 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
806 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
807 LLVMContext &Context = V->getContext();
809 // We have no information about target data layout, so we assume that
810 // the maximum pointer size is 64bit.
811 Type *MidTy = Type::getInt64Ty(Context);
812 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
814 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
820 Value *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
821 if (Opc != Instruction::BitCast)
824 Type *SrcTy = C->getType();
825 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
826 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
827 LLVMContext &Context = C->getContext();
829 // We have no information about target data layout, so we assume that
830 // the maximum pointer size is 64bit.
831 Type *MidTy = Type::getInt64Ty(Context);
833 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
840 /// Check the debug info version number, if it is out-dated, drop the debug
841 /// info. Return true if module is modified.
842 bool llvm::UpgradeDebugInfo(Module &M) {
843 unsigned Version = getDebugMetadataVersionFromModule(M);
844 if (Version == DEBUG_METADATA_VERSION)
847 bool RetCode = StripDebugInfo(M);
849 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
850 M.getContext().diagnose(DiagVersion);
855 void llvm::UpgradeMDStringConstant(std::string &String) {
856 const std::string OldPrefix = "llvm.vectorizer.";
857 if (String == "llvm.vectorizer.unroll") {
858 String = "llvm.loop.interleave.count";
859 } else if (String.find(OldPrefix) == 0) {
860 String.replace(0, OldPrefix.size(), "llvm.loop.vectorize.");