1 //===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the auto-upgrade helper functions
12 //===----------------------------------------------------------------------===//
14 #include "llvm/IR/AutoUpgrade.h"
15 #include "llvm/IR/CFG.h"
16 #include "llvm/IR/CallSite.h"
17 #include "llvm/IR/Constants.h"
18 #include "llvm/IR/DIBuilder.h"
19 #include "llvm/IR/DebugInfo.h"
20 #include "llvm/IR/DiagnosticInfo.h"
21 #include "llvm/IR/Function.h"
22 #include "llvm/IR/IRBuilder.h"
23 #include "llvm/IR/Instruction.h"
24 #include "llvm/IR/IntrinsicInst.h"
25 #include "llvm/IR/LLVMContext.h"
26 #include "llvm/IR/Module.h"
27 #include "llvm/Support/ErrorHandling.h"
31 // Upgrade the declarations of the SSE4.1 functions whose arguments have
32 // changed their type from v4f32 to v2i64.
33 static bool UpgradeSSE41Function(Function* F, Intrinsic::ID IID,
35 // Check whether this is an old version of the function, which received
37 Type *Arg0Type = F->getFunctionType()->getParamType(0);
38 if (Arg0Type != VectorType::get(Type::getFloatTy(F->getContext()), 4))
41 // Yes, it's old, replace it with new version.
42 F->setName(F->getName() + ".old");
43 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
47 // Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
48 // arguments have changed their type from i32 to i8.
49 static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
51 // Check that the last argument is an i32.
52 Type *LastArgType = F->getFunctionType()->getParamType(
53 F->getFunctionType()->getNumParams() - 1);
54 if (!LastArgType->isIntegerTy(32))
57 // Move this function aside and map down.
58 F->setName(F->getName() + ".old");
59 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
63 // Upgrade the declarations of AVX-512 cmp intrinsic functions whose 8-bit
64 // immediates have changed their type from i32 to i8.
65 static bool UpgradeAVX512CmpIntrinsic(Function *F, Intrinsic::ID IID,
67 // Check that the last argument is an i32.
68 Type *LastArgType = F->getFunctionType()->getParamType(2);
69 if (!LastArgType->isIntegerTy(32))
72 // Move this function aside and map down.
73 F->setName(F->getName() + ".old");
74 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
78 static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
79 assert(F && "Illegal to upgrade a non-existent Function.");
81 // Quickly eliminate it, if it's not a candidate.
82 StringRef Name = F->getName();
83 if (Name.size() <= 8 || !Name.startswith("llvm."))
85 Name = Name.substr(5); // Strip off "llvm."
90 if (Name.startswith("arm.neon.vclz")) {
92 F->arg_begin()->getType(),
93 Type::getInt1Ty(F->getContext())
95 // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to
96 // the end of the name. Change name from llvm.arm.neon.vclz.* to
98 FunctionType* fType = FunctionType::get(F->getReturnType(), args, false);
99 NewFn = Function::Create(fType, F->getLinkage(),
100 "llvm.ctlz." + Name.substr(14), F->getParent());
103 if (Name.startswith("arm.neon.vcnt")) {
104 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
105 F->arg_begin()->getType());
111 if (Name.startswith("ctlz.") && F->arg_size() == 1) {
112 F->setName(Name + ".old");
113 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
114 F->arg_begin()->getType());
117 if (Name.startswith("cttz.") && F->arg_size() == 1) {
118 F->setName(Name + ".old");
119 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz,
120 F->arg_begin()->getType());
126 if (Name.startswith("dbg.declare") && F->arg_size() == 2) {
127 F->setName(Name + ".old");
128 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_declare);
131 if (Name.startswith("dbg.value") && F->arg_size() == 3) {
132 F->setName(Name + ".old");
133 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value);
140 // We only need to change the name to match the mangling including the
142 if (F->arg_size() == 2 && Name.startswith("objectsize.")) {
143 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
144 if (F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) {
145 F->setName(Name + ".old");
146 NewFn = Intrinsic::getDeclaration(F->getParent(),
147 Intrinsic::objectsize, Tys);
154 if (Name.startswith("x86.sse2.pcmpeq.") ||
155 Name.startswith("x86.sse2.pcmpgt.") ||
156 Name.startswith("x86.avx2.pcmpeq.") ||
157 Name.startswith("x86.avx2.pcmpgt.") ||
158 Name.startswith("x86.avx.vpermil.") ||
159 Name == "x86.avx.movnt.dq.256" ||
160 Name == "x86.avx.movnt.pd.256" ||
161 Name == "x86.avx.movnt.ps.256" ||
162 Name == "x86.sse42.crc32.64.8" ||
163 Name == "x86.avx.vbroadcast.ss" ||
164 Name == "x86.avx.vbroadcast.ss.256" ||
165 Name == "x86.avx.vbroadcast.sd.256" ||
166 Name == "x86.sse2.psll.dq" ||
167 Name == "x86.sse2.psrl.dq" ||
168 Name == "x86.avx2.psll.dq" ||
169 Name == "x86.avx2.psrl.dq" ||
170 Name == "x86.sse2.psll.dq.bs" ||
171 Name == "x86.sse2.psrl.dq.bs" ||
172 Name == "x86.avx2.psll.dq.bs" ||
173 Name == "x86.avx2.psrl.dq.bs" ||
174 Name == "x86.sse41.pblendw" ||
175 Name == "x86.sse41.blendpd" ||
176 Name == "x86.sse41.blendps" ||
177 Name == "x86.avx.blend.pd.256" ||
178 Name == "x86.avx.blend.ps.256" ||
179 Name == "x86.avx2.pblendw" ||
180 Name == "x86.avx2.pblendd.128" ||
181 Name == "x86.avx2.pblendd.256" ||
182 Name == "x86.avx2.vbroadcasti128" ||
183 (Name.startswith("x86.xop.vpcom") && F->arg_size() == 2)) {
187 // SSE4.1 ptest functions may have an old signature.
188 if (Name.startswith("x86.sse41.ptest")) {
189 if (Name == "x86.sse41.ptestc")
190 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestc, NewFn);
191 if (Name == "x86.sse41.ptestz")
192 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestz, NewFn);
193 if (Name == "x86.sse41.ptestnzc")
194 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestnzc, NewFn);
196 // Several blend and other instructions with masks used the wrong number of
198 if (Name == "x86.sse41.insertps")
199 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps,
201 if (Name == "x86.sse41.dppd")
202 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd,
204 if (Name == "x86.sse41.dpps")
205 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps,
207 if (Name == "x86.sse41.mpsadbw")
208 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw,
210 if (Name == "x86.avx.dp.ps.256")
211 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256,
213 if (Name == "x86.avx2.mpsadbw")
214 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw,
217 if (Name == "x86.avx512.mask.cmp.ps.512")
218 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_ps_512,
220 if (Name == "x86.avx512.mask.cmp.pd.512")
221 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_pd_512,
224 if (Name == "x86.avx512.mask.cmp.b.512")
225 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_512,
227 if (Name == "x86.avx512.mask.cmp.w.512")
228 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_512,
230 if (Name == "x86.avx512.mask.cmp.d.512")
231 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_512,
233 if (Name == "x86.avx512.mask.cmp.q.512")
234 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_512,
236 if (Name == "x86.avx512.mask.ucmp.b.512")
237 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_512,
239 if (Name == "x86.avx512.mask.ucmp.w.512")
240 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_512,
242 if (Name == "x86.avx512.mask.ucmp.d.512")
243 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_512,
245 if (Name == "x86.avx512.mask.ucmp.q.512")
246 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_512,
249 if (Name == "x86.avx512.mask.cmp.b.256")
250 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_256,
252 if (Name == "x86.avx512.mask.cmp.w.256")
253 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_256,
255 if (Name == "x86.avx512.mask.cmp.d.256")
256 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_256,
258 if (Name == "x86.avx512.mask.cmp.q.256")
259 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_256,
261 if (Name == "x86.avx512.mask.ucmp.b.256")
262 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_256,
264 if (Name == "x86.avx512.mask.ucmp.w.256")
265 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_256,
267 if (Name == "x86.avx512.mask.ucmp.d.256")
268 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_256,
270 if (Name == "x86.avx512.mask.ucmp.q.256")
271 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_256,
274 if (Name == "x86.avx512.mask.cmp.b.128")
275 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_128,
277 if (Name == "x86.avx512.mask.cmp.w.128")
278 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_128,
280 if (Name == "x86.avx512.mask.cmp.d.128")
281 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_128,
283 if (Name == "x86.avx512.mask.cmp.q.128")
284 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_128,
286 if (Name == "x86.avx512.mask.ucmp.b.128")
287 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_128,
289 if (Name == "x86.avx512.mask.ucmp.w.128")
290 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_128,
292 if (Name == "x86.avx512.mask.ucmp.d.128")
293 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_128,
295 if (Name == "x86.avx512.mask.ucmp.q.128")
296 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_128,
299 // frcz.ss/sd may need to have an argument dropped
300 if (Name.startswith("x86.xop.vfrcz.ss") && F->arg_size() == 2) {
301 F->setName(Name + ".old");
302 NewFn = Intrinsic::getDeclaration(F->getParent(),
303 Intrinsic::x86_xop_vfrcz_ss);
306 if (Name.startswith("x86.xop.vfrcz.sd") && F->arg_size() == 2) {
307 F->setName(Name + ".old");
308 NewFn = Intrinsic::getDeclaration(F->getParent(),
309 Intrinsic::x86_xop_vfrcz_sd);
312 // Fix the FMA4 intrinsics to remove the 4
313 if (Name.startswith("x86.fma4.")) {
314 F->setName("llvm.x86.fma" + Name.substr(8));
322 // This may not belong here. This function is effectively being overloaded
323 // to both detect an intrinsic which needs upgrading, and to provide the
324 // upgraded form of the intrinsic. We should perhaps have two separate
325 // functions for this.
329 bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {
331 bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn);
333 // Upgrade intrinsic attributes. This does not change the function.
336 if (unsigned id = F->getIntrinsicID())
337 F->setAttributes(Intrinsic::getAttributes(F->getContext(),
342 bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
343 // Nothing to do yet.
347 static MDNode *getNodeField(const MDNode *DbgNode, unsigned Elt) {
348 if (!DbgNode || Elt >= DbgNode->getNumOperands())
350 return dyn_cast_or_null<MDNode>(DbgNode->getOperand(Elt));
353 static MetadataAsValue *getExpression(Value *VarOperand, Function *F) {
354 // Old-style DIVariables have an optional expression as the 8th element.
355 DIExpression Expr(getNodeField(
356 cast<MDNode>(cast<MetadataAsValue>(VarOperand)->getMetadata()), 8));
358 DIBuilder DIB(*F->getParent(), /*AllowUnresolved*/ false);
359 Expr = DIB.createExpression();
361 return MetadataAsValue::get(F->getContext(), Expr);
364 // Handles upgrading SSE2 and AVX2 PSLLDQ intrinsics by converting them
366 static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, LLVMContext &C,
367 Value *Op, unsigned NumLanes,
369 // Each lane is 16 bytes.
370 unsigned NumElts = NumLanes * 16;
372 // Bitcast from a 64-bit element type to a byte element type.
373 Op = Builder.CreateBitCast(Op,
374 VectorType::get(Type::getInt8Ty(C), NumElts),
376 // We'll be shuffling in zeroes.
377 Value *Res = ConstantVector::getSplat(NumElts, Builder.getInt8(0));
379 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
380 // we'll just return the zero vector.
382 SmallVector<Constant*, 32> Idxs;
383 // 256-bit version is split into two 16-byte lanes.
384 for (unsigned l = 0; l != NumElts; l += 16)
385 for (unsigned i = 0; i != 16; ++i) {
386 unsigned Idx = NumElts + i - Shift;
388 Idx -= NumElts - 16; // end of lane, switch operand.
389 Idxs.push_back(Builder.getInt32(Idx + l));
392 Res = Builder.CreateShuffleVector(Res, Op, ConstantVector::get(Idxs));
395 // Bitcast back to a 64-bit element type.
396 return Builder.CreateBitCast(Res,
397 VectorType::get(Type::getInt64Ty(C), 2*NumLanes),
401 // Handles upgrading SSE2 and AVX2 PSRLDQ intrinsics by converting them
403 static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, LLVMContext &C,
404 Value *Op, unsigned NumLanes,
406 // Each lane is 16 bytes.
407 unsigned NumElts = NumLanes * 16;
409 // Bitcast from a 64-bit element type to a byte element type.
410 Op = Builder.CreateBitCast(Op,
411 VectorType::get(Type::getInt8Ty(C), NumElts),
413 // We'll be shuffling in zeroes.
414 Value *Res = ConstantVector::getSplat(NumElts, Builder.getInt8(0));
416 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
417 // we'll just return the zero vector.
419 SmallVector<Constant*, 32> Idxs;
420 // 256-bit version is split into two 16-byte lanes.
421 for (unsigned l = 0; l != NumElts; l += 16)
422 for (unsigned i = 0; i != 16; ++i) {
423 unsigned Idx = i + Shift;
425 Idx += NumElts - 16; // end of lane, switch operand.
426 Idxs.push_back(Builder.getInt32(Idx + l));
429 Res = Builder.CreateShuffleVector(Op, Res, ConstantVector::get(Idxs));
432 // Bitcast back to a 64-bit element type.
433 return Builder.CreateBitCast(Res,
434 VectorType::get(Type::getInt64Ty(C), 2*NumLanes),
438 // UpgradeIntrinsicCall - Upgrade a call to an old intrinsic to be a call the
439 // upgraded intrinsic. All argument and return casting must be provided in
440 // order to seamlessly integrate with existing context.
441 void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
442 Function *F = CI->getCalledFunction();
443 LLVMContext &C = CI->getContext();
444 IRBuilder<> Builder(C);
445 Builder.SetInsertPoint(CI->getParent(), CI);
447 assert(F && "Intrinsic call is not direct?");
450 // Get the Function's name.
451 StringRef Name = F->getName();
454 // Upgrade packed integer vector compares intrinsics to compare instructions
455 if (Name.startswith("llvm.x86.sse2.pcmpeq.") ||
456 Name.startswith("llvm.x86.avx2.pcmpeq.")) {
457 Rep = Builder.CreateICmpEQ(CI->getArgOperand(0), CI->getArgOperand(1),
459 // need to sign extend since icmp returns vector of i1
460 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
461 } else if (Name.startswith("llvm.x86.sse2.pcmpgt.") ||
462 Name.startswith("llvm.x86.avx2.pcmpgt.")) {
463 Rep = Builder.CreateICmpSGT(CI->getArgOperand(0), CI->getArgOperand(1),
465 // need to sign extend since icmp returns vector of i1
466 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
467 } else if (Name == "llvm.x86.avx.movnt.dq.256" ||
468 Name == "llvm.x86.avx.movnt.ps.256" ||
469 Name == "llvm.x86.avx.movnt.pd.256") {
470 IRBuilder<> Builder(C);
471 Builder.SetInsertPoint(CI->getParent(), CI);
473 Module *M = F->getParent();
474 SmallVector<Metadata *, 1> Elts;
476 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
477 MDNode *Node = MDNode::get(C, Elts);
479 Value *Arg0 = CI->getArgOperand(0);
480 Value *Arg1 = CI->getArgOperand(1);
482 // Convert the type of the pointer to a pointer to the stored type.
483 Value *BC = Builder.CreateBitCast(Arg0,
484 PointerType::getUnqual(Arg1->getType()),
486 StoreInst *SI = Builder.CreateStore(Arg1, BC);
487 SI->setMetadata(M->getMDKindID("nontemporal"), Node);
488 SI->setAlignment(16);
491 CI->eraseFromParent();
493 } else if (Name.startswith("llvm.x86.xop.vpcom")) {
495 if (Name.endswith("ub"))
496 intID = Intrinsic::x86_xop_vpcomub;
497 else if (Name.endswith("uw"))
498 intID = Intrinsic::x86_xop_vpcomuw;
499 else if (Name.endswith("ud"))
500 intID = Intrinsic::x86_xop_vpcomud;
501 else if (Name.endswith("uq"))
502 intID = Intrinsic::x86_xop_vpcomuq;
503 else if (Name.endswith("b"))
504 intID = Intrinsic::x86_xop_vpcomb;
505 else if (Name.endswith("w"))
506 intID = Intrinsic::x86_xop_vpcomw;
507 else if (Name.endswith("d"))
508 intID = Intrinsic::x86_xop_vpcomd;
509 else if (Name.endswith("q"))
510 intID = Intrinsic::x86_xop_vpcomq;
512 llvm_unreachable("Unknown suffix");
514 Name = Name.substr(18); // strip off "llvm.x86.xop.vpcom"
516 if (Name.startswith("lt"))
518 else if (Name.startswith("le"))
520 else if (Name.startswith("gt"))
522 else if (Name.startswith("ge"))
524 else if (Name.startswith("eq"))
526 else if (Name.startswith("ne"))
528 else if (Name.startswith("false"))
530 else if (Name.startswith("true"))
533 llvm_unreachable("Unknown condition");
535 Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID);
536 Rep = Builder.CreateCall3(VPCOM, CI->getArgOperand(0),
537 CI->getArgOperand(1), Builder.getInt8(Imm));
538 } else if (Name == "llvm.x86.sse42.crc32.64.8") {
539 Function *CRC32 = Intrinsic::getDeclaration(F->getParent(),
540 Intrinsic::x86_sse42_crc32_32_8);
541 Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
542 Rep = Builder.CreateCall2(CRC32, Trunc0, CI->getArgOperand(1));
543 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
544 } else if (Name.startswith("llvm.x86.avx.vbroadcast")) {
545 // Replace broadcasts with a series of insertelements.
546 Type *VecTy = CI->getType();
547 Type *EltTy = VecTy->getVectorElementType();
548 unsigned EltNum = VecTy->getVectorNumElements();
549 Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),
550 EltTy->getPointerTo());
551 Value *Load = Builder.CreateLoad(Cast);
552 Type *I32Ty = Type::getInt32Ty(C);
553 Rep = UndefValue::get(VecTy);
554 for (unsigned I = 0; I < EltNum; ++I)
555 Rep = Builder.CreateInsertElement(Rep, Load,
556 ConstantInt::get(I32Ty, I));
557 } else if (Name == "llvm.x86.avx2.vbroadcasti128") {
558 // Replace vbroadcasts with a vector shuffle.
559 Value *Op = Builder.CreatePointerCast(
560 CI->getArgOperand(0),
561 PointerType::getUnqual(VectorType::get(Type::getInt64Ty(C), 2)));
562 Value *Load = Builder.CreateLoad(Op);
563 SmallVector<Constant *, 4> Idxs; // 0, 1, 0, 1.
564 for (unsigned i = 0; i != 4; ++i)
565 Idxs.push_back(Builder.getInt32(i & 1));
566 Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
567 ConstantVector::get(Idxs));
568 } else if (Name == "llvm.x86.sse2.psll.dq") {
569 // 128-bit shift left specified in bits.
570 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
571 Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1,
572 Shift / 8); // Shift is in bits.
573 } else if (Name == "llvm.x86.sse2.psrl.dq") {
574 // 128-bit shift right specified in bits.
575 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
576 Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1,
577 Shift / 8); // Shift is in bits.
578 } else if (Name == "llvm.x86.avx2.psll.dq") {
579 // 256-bit shift left specified in bits.
580 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
581 Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2,
582 Shift / 8); // Shift is in bits.
583 } else if (Name == "llvm.x86.avx2.psrl.dq") {
584 // 256-bit shift right specified in bits.
585 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
586 Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2,
587 Shift / 8); // Shift is in bits.
588 } else if (Name == "llvm.x86.sse2.psll.dq.bs") {
589 // 128-bit shift left specified in bytes.
590 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
591 Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1,
593 } else if (Name == "llvm.x86.sse2.psrl.dq.bs") {
594 // 128-bit shift right specified in bytes.
595 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
596 Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1,
598 } else if (Name == "llvm.x86.avx2.psll.dq.bs") {
599 // 256-bit shift left specified in bytes.
600 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
601 Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2,
603 } else if (Name == "llvm.x86.avx2.psrl.dq.bs") {
604 // 256-bit shift right specified in bytes.
605 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
606 Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2,
608 } else if (Name == "llvm.x86.sse41.pblendw" ||
609 Name == "llvm.x86.sse41.blendpd" ||
610 Name == "llvm.x86.sse41.blendps" ||
611 Name == "llvm.x86.avx.blend.pd.256" ||
612 Name == "llvm.x86.avx.blend.ps.256" ||
613 Name == "llvm.x86.avx2.pblendw" ||
614 Name == "llvm.x86.avx2.pblendd.128" ||
615 Name == "llvm.x86.avx2.pblendd.256") {
616 Value *Op0 = CI->getArgOperand(0);
617 Value *Op1 = CI->getArgOperand(1);
618 unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue();
619 VectorType *VecTy = cast<VectorType>(CI->getType());
620 unsigned NumElts = VecTy->getNumElements();
622 SmallVector<Constant*, 16> Idxs;
623 for (unsigned i = 0; i != NumElts; ++i) {
624 unsigned Idx = ((Imm >> (i%8)) & 1) ? i + NumElts : i;
625 Idxs.push_back(Builder.getInt32(Idx));
628 Rep = Builder.CreateShuffleVector(Op0, Op1, ConstantVector::get(Idxs));
630 bool PD128 = false, PD256 = false, PS128 = false, PS256 = false;
631 if (Name == "llvm.x86.avx.vpermil.pd.256")
633 else if (Name == "llvm.x86.avx.vpermil.pd")
635 else if (Name == "llvm.x86.avx.vpermil.ps.256")
637 else if (Name == "llvm.x86.avx.vpermil.ps")
640 if (PD256 || PD128 || PS256 || PS128) {
641 Value *Op0 = CI->getArgOperand(0);
642 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
643 SmallVector<Constant*, 8> Idxs;
646 for (unsigned i = 0; i != 2; ++i)
647 Idxs.push_back(Builder.getInt32((Imm >> i) & 0x1));
649 for (unsigned l = 0; l != 4; l+=2)
650 for (unsigned i = 0; i != 2; ++i)
651 Idxs.push_back(Builder.getInt32(((Imm >> (l+i)) & 0x1) + l));
653 for (unsigned i = 0; i != 4; ++i)
654 Idxs.push_back(Builder.getInt32((Imm >> (2 * i)) & 0x3));
656 for (unsigned l = 0; l != 8; l+=4)
657 for (unsigned i = 0; i != 4; ++i)
658 Idxs.push_back(Builder.getInt32(((Imm >> (2 * i)) & 0x3) + l));
660 llvm_unreachable("Unexpected function");
662 Rep = Builder.CreateShuffleVector(Op0, Op0, ConstantVector::get(Idxs));
664 llvm_unreachable("Unknown function for CallInst upgrade.");
668 CI->replaceAllUsesWith(Rep);
669 CI->eraseFromParent();
673 std::string Name = CI->getName().str();
675 CI->setName(Name + ".old");
677 switch (NewFn->getIntrinsicID()) {
679 llvm_unreachable("Unknown function for CallInst upgrade.");
681 // Upgrade debug intrinsics to use an additional DIExpression argument.
682 case Intrinsic::dbg_declare: {
684 Builder.CreateCall3(NewFn, CI->getArgOperand(0), CI->getArgOperand(1),
685 getExpression(CI->getArgOperand(1), F), Name);
686 NewCI->setDebugLoc(CI->getDebugLoc());
687 CI->replaceAllUsesWith(NewCI);
688 CI->eraseFromParent();
691 case Intrinsic::dbg_value: {
692 auto NewCI = Builder.CreateCall4(
693 NewFn, CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
694 getExpression(CI->getArgOperand(2), F), Name);
695 NewCI->setDebugLoc(CI->getDebugLoc());
696 CI->replaceAllUsesWith(NewCI);
697 CI->eraseFromParent();
700 case Intrinsic::ctlz:
701 case Intrinsic::cttz:
702 assert(CI->getNumArgOperands() == 1 &&
703 "Mismatch between function args and call args");
704 CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0),
705 Builder.getFalse(), Name));
706 CI->eraseFromParent();
709 case Intrinsic::objectsize:
710 CI->replaceAllUsesWith(Builder.CreateCall2(NewFn,
711 CI->getArgOperand(0),
712 CI->getArgOperand(1),
714 CI->eraseFromParent();
717 case Intrinsic::ctpop: {
718 CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(0)));
719 CI->eraseFromParent();
723 case Intrinsic::x86_xop_vfrcz_ss:
724 case Intrinsic::x86_xop_vfrcz_sd:
725 CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(1),
727 CI->eraseFromParent();
730 case Intrinsic::x86_sse41_ptestc:
731 case Intrinsic::x86_sse41_ptestz:
732 case Intrinsic::x86_sse41_ptestnzc: {
733 // The arguments for these intrinsics used to be v4f32, and changed
734 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
735 // So, the only thing required is a bitcast for both arguments.
736 // First, check the arguments have the old type.
737 Value *Arg0 = CI->getArgOperand(0);
738 if (Arg0->getType() != VectorType::get(Type::getFloatTy(C), 4))
741 // Old intrinsic, add bitcasts
742 Value *Arg1 = CI->getArgOperand(1);
745 Builder.CreateBitCast(Arg0,
746 VectorType::get(Type::getInt64Ty(C), 2),
749 Builder.CreateBitCast(Arg1,
750 VectorType::get(Type::getInt64Ty(C), 2),
753 CallInst* NewCall = Builder.CreateCall2(NewFn, BC0, BC1, Name);
754 CI->replaceAllUsesWith(NewCall);
755 CI->eraseFromParent();
759 case Intrinsic::x86_sse41_insertps:
760 case Intrinsic::x86_sse41_dppd:
761 case Intrinsic::x86_sse41_dpps:
762 case Intrinsic::x86_sse41_mpsadbw:
763 case Intrinsic::x86_avx_dp_ps_256:
764 case Intrinsic::x86_avx2_mpsadbw: {
765 // Need to truncate the last argument from i32 to i8 -- this argument models
766 // an inherently 8-bit immediate operand to these x86 instructions.
767 SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
768 CI->arg_operands().end());
770 // Replace the last argument with a trunc.
771 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
773 CallInst *NewCall = Builder.CreateCall(NewFn, Args);
774 CI->replaceAllUsesWith(NewCall);
775 CI->eraseFromParent();
778 case Intrinsic::x86_avx512_mask_cmp_ps_512:
779 case Intrinsic::x86_avx512_mask_cmp_pd_512: {
780 // Need to truncate the last argument from i32 to i8 -- this argument models
781 // an inherently 8-bit immediate operand to these x86 instructions.
782 SmallVector<Value *, 5> Args(CI->arg_operands().begin(),
783 CI->arg_operands().end());
785 // Replace the last argument with a trunc.
786 Args[2] = Builder.CreateTrunc(Args[2], Type::getInt8Ty(C), "trunc");
788 CallInst *NewCall = Builder.CreateCall(NewFn, Args);
789 CI->replaceAllUsesWith(NewCall);
790 CI->eraseFromParent();
796 // This tests each Function to determine if it needs upgrading. When we find
797 // one we are interested in, we then upgrade all calls to reflect the new
799 void llvm::UpgradeCallsToIntrinsic(Function* F) {
800 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
802 // Upgrade the function and check if it is a totaly new function.
804 if (UpgradeIntrinsicFunction(F, NewFn)) {
806 // Replace all uses to the old function with the new one if necessary.
807 for (Value::user_iterator UI = F->user_begin(), UE = F->user_end();
809 if (CallInst *CI = dyn_cast<CallInst>(*UI++))
810 UpgradeIntrinsicCall(CI, NewFn);
812 // Remove old function, no longer used, from the module.
813 F->eraseFromParent();
818 void llvm::UpgradeInstWithTBAATag(Instruction *I) {
819 MDNode *MD = I->getMetadata(LLVMContext::MD_tbaa);
820 assert(MD && "UpgradeInstWithTBAATag should have a TBAA tag");
821 // Check if the tag uses struct-path aware TBAA format.
822 if (isa<MDNode>(MD->getOperand(0)) && MD->getNumOperands() >= 3)
825 if (MD->getNumOperands() == 3) {
826 Metadata *Elts[] = {MD->getOperand(0), MD->getOperand(1)};
827 MDNode *ScalarType = MDNode::get(I->getContext(), Elts);
828 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
829 Metadata *Elts2[] = {ScalarType, ScalarType,
830 ConstantAsMetadata::get(Constant::getNullValue(
831 Type::getInt64Ty(I->getContext()))),
833 I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts2));
835 // Create a MDNode <MD, MD, offset 0>
836 Metadata *Elts[] = {MD, MD, ConstantAsMetadata::get(Constant::getNullValue(
837 Type::getInt64Ty(I->getContext())))};
838 I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts));
842 Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
843 Instruction *&Temp) {
844 if (Opc != Instruction::BitCast)
848 Type *SrcTy = V->getType();
849 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
850 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
851 LLVMContext &Context = V->getContext();
853 // We have no information about target data layout, so we assume that
854 // the maximum pointer size is 64bit.
855 Type *MidTy = Type::getInt64Ty(Context);
856 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
858 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
864 Value *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
865 if (Opc != Instruction::BitCast)
868 Type *SrcTy = C->getType();
869 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
870 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
871 LLVMContext &Context = C->getContext();
873 // We have no information about target data layout, so we assume that
874 // the maximum pointer size is 64bit.
875 Type *MidTy = Type::getInt64Ty(Context);
877 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
884 /// Check the debug info version number, if it is out-dated, drop the debug
885 /// info. Return true if module is modified.
886 bool llvm::UpgradeDebugInfo(Module &M) {
887 unsigned Version = getDebugMetadataVersionFromModule(M);
888 if (Version == DEBUG_METADATA_VERSION)
891 bool RetCode = StripDebugInfo(M);
893 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
894 M.getContext().diagnose(DiagVersion);
899 void llvm::UpgradeMDStringConstant(std::string &String) {
900 const std::string OldPrefix = "llvm.vectorizer.";
901 if (String == "llvm.vectorizer.unroll") {
902 String = "llvm.loop.interleave.count";
903 } else if (String.find(OldPrefix) == 0) {
904 String.replace(0, OldPrefix.size(), "llvm.loop.vectorize.");