1 //===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the auto-upgrade helper functions
12 //===----------------------------------------------------------------------===//
14 #include "llvm/IR/AutoUpgrade.h"
15 #include "llvm/IR/CFG.h"
16 #include "llvm/IR/CallSite.h"
17 #include "llvm/IR/Constants.h"
18 #include "llvm/IR/DIBuilder.h"
19 #include "llvm/IR/DebugInfo.h"
20 #include "llvm/IR/DiagnosticInfo.h"
21 #include "llvm/IR/Function.h"
22 #include "llvm/IR/IRBuilder.h"
23 #include "llvm/IR/Instruction.h"
24 #include "llvm/IR/IntrinsicInst.h"
25 #include "llvm/IR/LLVMContext.h"
26 #include "llvm/IR/Module.h"
27 #include "llvm/Support/ErrorHandling.h"
31 // Upgrade the declarations of the SSE4.1 functions whose arguments have
32 // changed their type from v4f32 to v2i64.
33 static bool UpgradeSSE41Function(Function* F, Intrinsic::ID IID,
35 // Check whether this is an old version of the function, which received
37 Type *Arg0Type = F->getFunctionType()->getParamType(0);
38 if (Arg0Type != VectorType::get(Type::getFloatTy(F->getContext()), 4))
41 // Yes, it's old, replace it with new version.
42 F->setName(F->getName() + ".old");
43 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
47 // Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
48 // arguments have changed their type from i32 to i8.
49 static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
51 // Check that the last argument is an i32.
52 Type *LastArgType = F->getFunctionType()->getParamType(
53 F->getFunctionType()->getNumParams() - 1);
54 if (!LastArgType->isIntegerTy(32))
57 // Move this function aside and map down.
58 F->setName(F->getName() + ".old");
59 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
63 // Upgrade the declarations of AVX-512 cmp intrinsic functions whose 8-bit
64 // immediates have changed their type from i32 to i8.
65 static bool UpgradeAVX512CmpIntrinsic(Function *F, Intrinsic::ID IID,
67 // Check that the last argument is an i32.
68 Type *LastArgType = F->getFunctionType()->getParamType(2);
69 if (!LastArgType->isIntegerTy(32))
72 // Move this function aside and map down.
73 F->setName(F->getName() + ".old");
74 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
78 static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
79 assert(F && "Illegal to upgrade a non-existent Function.");
81 // Quickly eliminate it, if it's not a candidate.
82 StringRef Name = F->getName();
83 if (Name.size() <= 8 || !Name.startswith("llvm."))
85 Name = Name.substr(5); // Strip off "llvm."
90 if (Name.startswith("arm.neon.vclz")) {
92 F->arg_begin()->getType(),
93 Type::getInt1Ty(F->getContext())
95 // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to
96 // the end of the name. Change name from llvm.arm.neon.vclz.* to
98 FunctionType* fType = FunctionType::get(F->getReturnType(), args, false);
99 NewFn = Function::Create(fType, F->getLinkage(),
100 "llvm.ctlz." + Name.substr(14), F->getParent());
103 if (Name.startswith("arm.neon.vcnt")) {
104 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
105 F->arg_begin()->getType());
111 if (Name.startswith("ctlz.") && F->arg_size() == 1) {
112 F->setName(Name + ".old");
113 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
114 F->arg_begin()->getType());
117 if (Name.startswith("cttz.") && F->arg_size() == 1) {
118 F->setName(Name + ".old");
119 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz,
120 F->arg_begin()->getType());
126 if (Name.startswith("dbg.declare") && F->arg_size() == 2) {
127 F->setName(Name + ".old");
128 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_declare);
131 if (Name.startswith("dbg.value") && F->arg_size() == 3) {
132 F->setName(Name + ".old");
133 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value);
140 // We only need to change the name to match the mangling including the
142 if (F->arg_size() == 2 && Name.startswith("objectsize.")) {
143 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
144 if (F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) {
145 F->setName(Name + ".old");
146 NewFn = Intrinsic::getDeclaration(F->getParent(),
147 Intrinsic::objectsize, Tys);
154 if (Name.startswith("x86.sse2.pcmpeq.") ||
155 Name.startswith("x86.sse2.pcmpgt.") ||
156 Name.startswith("x86.avx2.pcmpeq.") ||
157 Name.startswith("x86.avx2.pcmpgt.") ||
158 Name.startswith("x86.avx.vpermil.") ||
159 Name == "x86.avx.movnt.dq.256" ||
160 Name == "x86.avx.movnt.pd.256" ||
161 Name == "x86.avx.movnt.ps.256" ||
162 Name == "x86.sse42.crc32.64.8" ||
163 Name == "x86.avx.vbroadcast.ss" ||
164 Name == "x86.avx.vbroadcast.ss.256" ||
165 Name == "x86.avx.vbroadcast.sd.256" ||
166 Name == "x86.sse2.psll.dq" ||
167 Name == "x86.sse2.psrl.dq" ||
168 Name == "x86.avx2.psll.dq" ||
169 Name == "x86.avx2.psrl.dq" ||
170 Name == "x86.sse2.psll.dq.bs" ||
171 Name == "x86.sse2.psrl.dq.bs" ||
172 Name == "x86.avx2.psll.dq.bs" ||
173 Name == "x86.avx2.psrl.dq.bs" ||
174 (Name.startswith("x86.xop.vpcom") && F->arg_size() == 2)) {
178 // SSE4.1 ptest functions may have an old signature.
179 if (Name.startswith("x86.sse41.ptest")) {
180 if (Name == "x86.sse41.ptestc")
181 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestc, NewFn);
182 if (Name == "x86.sse41.ptestz")
183 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestz, NewFn);
184 if (Name == "x86.sse41.ptestnzc")
185 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestnzc, NewFn);
187 // Several blend and other instructions with maskes used the wrong number of
189 if (Name == "x86.sse41.pblendw")
190 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_pblendw,
192 if (Name == "x86.sse41.blendpd")
193 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_blendpd,
195 if (Name == "x86.sse41.blendps")
196 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_blendps,
198 if (Name == "x86.sse41.insertps")
199 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps,
201 if (Name == "x86.sse41.dppd")
202 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd,
204 if (Name == "x86.sse41.dpps")
205 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps,
207 if (Name == "x86.sse41.mpsadbw")
208 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw,
210 if (Name == "x86.avx.blend.pd.256")
211 return UpgradeX86IntrinsicsWith8BitMask(
212 F, Intrinsic::x86_avx_blend_pd_256, NewFn);
213 if (Name == "x86.avx.blend.ps.256")
214 return UpgradeX86IntrinsicsWith8BitMask(
215 F, Intrinsic::x86_avx_blend_ps_256, NewFn);
216 if (Name == "x86.avx.dp.ps.256")
217 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256,
219 if (Name == "x86.avx2.pblendw")
220 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_pblendw,
222 if (Name == "x86.avx2.pblendd.128")
223 return UpgradeX86IntrinsicsWith8BitMask(
224 F, Intrinsic::x86_avx2_pblendd_128, NewFn);
225 if (Name == "x86.avx2.pblendd.256")
226 return UpgradeX86IntrinsicsWith8BitMask(
227 F, Intrinsic::x86_avx2_pblendd_256, NewFn);
228 if (Name == "x86.avx2.mpsadbw")
229 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw,
232 if (Name == "x86.avx512.mask.cmp.ps.512")
233 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_ps_512,
235 if (Name == "x86.avx512.mask.cmp.pd.512")
236 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_pd_512,
239 if (Name == "x86.avx512.mask.cmp.b.512")
240 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_512,
242 if (Name == "x86.avx512.mask.cmp.w.512")
243 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_512,
245 if (Name == "x86.avx512.mask.cmp.d.512")
246 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_512,
248 if (Name == "x86.avx512.mask.cmp.q.512")
249 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_512,
251 if (Name == "x86.avx512.mask.ucmp.b.512")
252 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_512,
254 if (Name == "x86.avx512.mask.ucmp.w.512")
255 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_512,
257 if (Name == "x86.avx512.mask.ucmp.d.512")
258 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_512,
260 if (Name == "x86.avx512.mask.ucmp.q.512")
261 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_512,
264 if (Name == "x86.avx512.mask.cmp.b.256")
265 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_256,
267 if (Name == "x86.avx512.mask.cmp.w.256")
268 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_256,
270 if (Name == "x86.avx512.mask.cmp.d.256")
271 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_256,
273 if (Name == "x86.avx512.mask.cmp.q.256")
274 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_256,
276 if (Name == "x86.avx512.mask.ucmp.b.256")
277 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_256,
279 if (Name == "x86.avx512.mask.ucmp.w.256")
280 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_256,
282 if (Name == "x86.avx512.mask.ucmp.d.256")
283 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_256,
285 if (Name == "x86.avx512.mask.ucmp.q.256")
286 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_256,
289 if (Name == "x86.avx512.mask.cmp.b.128")
290 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_128,
292 if (Name == "x86.avx512.mask.cmp.w.128")
293 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_128,
295 if (Name == "x86.avx512.mask.cmp.d.128")
296 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_128,
298 if (Name == "x86.avx512.mask.cmp.q.128")
299 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_128,
301 if (Name == "x86.avx512.mask.ucmp.b.128")
302 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_128,
304 if (Name == "x86.avx512.mask.ucmp.w.128")
305 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_128,
307 if (Name == "x86.avx512.mask.ucmp.d.128")
308 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_128,
310 if (Name == "x86.avx512.mask.ucmp.q.128")
311 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_128,
314 // frcz.ss/sd may need to have an argument dropped
315 if (Name.startswith("x86.xop.vfrcz.ss") && F->arg_size() == 2) {
316 F->setName(Name + ".old");
317 NewFn = Intrinsic::getDeclaration(F->getParent(),
318 Intrinsic::x86_xop_vfrcz_ss);
321 if (Name.startswith("x86.xop.vfrcz.sd") && F->arg_size() == 2) {
322 F->setName(Name + ".old");
323 NewFn = Intrinsic::getDeclaration(F->getParent(),
324 Intrinsic::x86_xop_vfrcz_sd);
327 // Fix the FMA4 intrinsics to remove the 4
328 if (Name.startswith("x86.fma4.")) {
329 F->setName("llvm.x86.fma" + Name.substr(8));
337 // This may not belong here. This function is effectively being overloaded
338 // to both detect an intrinsic which needs upgrading, and to provide the
339 // upgraded form of the intrinsic. We should perhaps have two separate
340 // functions for this.
344 bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {
346 bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn);
348 // Upgrade intrinsic attributes. This does not change the function.
351 if (unsigned id = F->getIntrinsicID())
352 F->setAttributes(Intrinsic::getAttributes(F->getContext(),
357 bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
358 // Nothing to do yet.
362 static MDNode *getNodeField(const MDNode *DbgNode, unsigned Elt) {
363 if (!DbgNode || Elt >= DbgNode->getNumOperands())
365 return dyn_cast_or_null<MDNode>(DbgNode->getOperand(Elt));
368 static MetadataAsValue *getExpression(Value *VarOperand, Function *F) {
369 // Old-style DIVariables have an optional expression as the 8th element.
370 DIExpression Expr(getNodeField(
371 cast<MDNode>(cast<MetadataAsValue>(VarOperand)->getMetadata()), 8));
373 DIBuilder DIB(*F->getParent(), /*AllowUnresolved*/ false);
374 Expr = DIB.createExpression();
376 return MetadataAsValue::get(F->getContext(), Expr);
379 // Handles upgrading SSE2 and AVX2 PSLLDQ intrinsics by converting them
381 static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, LLVMContext &C,
382 Value *Op, unsigned NumLanes,
384 // Each lane is 16 bytes.
385 unsigned NumElts = NumLanes * 16;
387 // Bitcast from a 64-bit element type to a byte element type.
388 Op = Builder.CreateBitCast(Op,
389 VectorType::get(Type::getInt8Ty(C), NumElts),
391 // We'll be shuffling in zeroes.
392 Value *Res = ConstantVector::getSplat(NumElts, Builder.getInt8(0));
394 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
395 // we'll just return the zero vector.
397 SmallVector<Constant*, 32> Idxs;
398 // 256-bit version is split into two 16-byte lanes.
399 for (unsigned l = 0; l != NumElts; l += 16)
400 for (unsigned i = 0; i != 16; ++i) {
401 unsigned Idx = NumElts + i - Shift;
403 Idx -= NumElts - 16; // end of lane, switch operand.
404 Idxs.push_back(Builder.getInt32(Idx + l));
407 Res = Builder.CreateShuffleVector(Res, Op, ConstantVector::get(Idxs));
410 // Bitcast back to a 64-bit element type.
411 return Builder.CreateBitCast(Res,
412 VectorType::get(Type::getInt64Ty(C), 2*NumLanes),
416 // Handles upgrading SSE2 and AVX2 PSRLDQ intrinsics by converting them
418 static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, LLVMContext &C,
419 Value *Op, unsigned NumLanes,
421 // Each lane is 16 bytes.
422 unsigned NumElts = NumLanes * 16;
424 // Bitcast from a 64-bit element type to a byte element type.
425 Op = Builder.CreateBitCast(Op,
426 VectorType::get(Type::getInt8Ty(C), NumElts),
428 // We'll be shuffling in zeroes.
429 Value *Res = ConstantVector::getSplat(NumElts, Builder.getInt8(0));
431 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
432 // we'll just return the zero vector.
434 SmallVector<Constant*, 32> Idxs;
435 // 256-bit version is split into two 16-byte lanes.
436 for (unsigned l = 0; l != NumElts; l += 16)
437 for (unsigned i = 0; i != 16; ++i) {
438 unsigned Idx = i + Shift;
440 Idx += NumElts - 16; // end of lane, switch operand.
441 Idxs.push_back(Builder.getInt32(Idx + l));
444 Res = Builder.CreateShuffleVector(Op, Res, ConstantVector::get(Idxs));
447 // Bitcast back to a 64-bit element type.
448 return Builder.CreateBitCast(Res,
449 VectorType::get(Type::getInt64Ty(C), 2*NumLanes),
453 // UpgradeIntrinsicCall - Upgrade a call to an old intrinsic to be a call the
454 // upgraded intrinsic. All argument and return casting must be provided in
455 // order to seamlessly integrate with existing context.
456 void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
457 Function *F = CI->getCalledFunction();
458 LLVMContext &C = CI->getContext();
459 IRBuilder<> Builder(C);
460 Builder.SetInsertPoint(CI->getParent(), CI);
462 assert(F && "Intrinsic call is not direct?");
465 // Get the Function's name.
466 StringRef Name = F->getName();
469 // Upgrade packed integer vector compares intrinsics to compare instructions
470 if (Name.startswith("llvm.x86.sse2.pcmpeq.") ||
471 Name.startswith("llvm.x86.avx2.pcmpeq.")) {
472 Rep = Builder.CreateICmpEQ(CI->getArgOperand(0), CI->getArgOperand(1),
474 // need to sign extend since icmp returns vector of i1
475 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
476 } else if (Name.startswith("llvm.x86.sse2.pcmpgt.") ||
477 Name.startswith("llvm.x86.avx2.pcmpgt.")) {
478 Rep = Builder.CreateICmpSGT(CI->getArgOperand(0), CI->getArgOperand(1),
480 // need to sign extend since icmp returns vector of i1
481 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
482 } else if (Name == "llvm.x86.avx.movnt.dq.256" ||
483 Name == "llvm.x86.avx.movnt.ps.256" ||
484 Name == "llvm.x86.avx.movnt.pd.256") {
485 IRBuilder<> Builder(C);
486 Builder.SetInsertPoint(CI->getParent(), CI);
488 Module *M = F->getParent();
489 SmallVector<Metadata *, 1> Elts;
491 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
492 MDNode *Node = MDNode::get(C, Elts);
494 Value *Arg0 = CI->getArgOperand(0);
495 Value *Arg1 = CI->getArgOperand(1);
497 // Convert the type of the pointer to a pointer to the stored type.
498 Value *BC = Builder.CreateBitCast(Arg0,
499 PointerType::getUnqual(Arg1->getType()),
501 StoreInst *SI = Builder.CreateStore(Arg1, BC);
502 SI->setMetadata(M->getMDKindID("nontemporal"), Node);
503 SI->setAlignment(16);
506 CI->eraseFromParent();
508 } else if (Name.startswith("llvm.x86.xop.vpcom")) {
510 if (Name.endswith("ub"))
511 intID = Intrinsic::x86_xop_vpcomub;
512 else if (Name.endswith("uw"))
513 intID = Intrinsic::x86_xop_vpcomuw;
514 else if (Name.endswith("ud"))
515 intID = Intrinsic::x86_xop_vpcomud;
516 else if (Name.endswith("uq"))
517 intID = Intrinsic::x86_xop_vpcomuq;
518 else if (Name.endswith("b"))
519 intID = Intrinsic::x86_xop_vpcomb;
520 else if (Name.endswith("w"))
521 intID = Intrinsic::x86_xop_vpcomw;
522 else if (Name.endswith("d"))
523 intID = Intrinsic::x86_xop_vpcomd;
524 else if (Name.endswith("q"))
525 intID = Intrinsic::x86_xop_vpcomq;
527 llvm_unreachable("Unknown suffix");
529 Name = Name.substr(18); // strip off "llvm.x86.xop.vpcom"
531 if (Name.startswith("lt"))
533 else if (Name.startswith("le"))
535 else if (Name.startswith("gt"))
537 else if (Name.startswith("ge"))
539 else if (Name.startswith("eq"))
541 else if (Name.startswith("ne"))
543 else if (Name.startswith("false"))
545 else if (Name.startswith("true"))
548 llvm_unreachable("Unknown condition");
550 Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID);
551 Rep = Builder.CreateCall3(VPCOM, CI->getArgOperand(0),
552 CI->getArgOperand(1), Builder.getInt8(Imm));
553 } else if (Name == "llvm.x86.sse42.crc32.64.8") {
554 Function *CRC32 = Intrinsic::getDeclaration(F->getParent(),
555 Intrinsic::x86_sse42_crc32_32_8);
556 Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
557 Rep = Builder.CreateCall2(CRC32, Trunc0, CI->getArgOperand(1));
558 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
559 } else if (Name.startswith("llvm.x86.avx.vbroadcast")) {
560 // Replace broadcasts with a series of insertelements.
561 Type *VecTy = CI->getType();
562 Type *EltTy = VecTy->getVectorElementType();
563 unsigned EltNum = VecTy->getVectorNumElements();
564 Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),
565 EltTy->getPointerTo());
566 Value *Load = Builder.CreateLoad(Cast);
567 Type *I32Ty = Type::getInt32Ty(C);
568 Rep = UndefValue::get(VecTy);
569 for (unsigned I = 0; I < EltNum; ++I)
570 Rep = Builder.CreateInsertElement(Rep, Load,
571 ConstantInt::get(I32Ty, I));
572 } else if (Name == "llvm.x86.sse2.psll.dq") {
573 // 128-bit shift left specified in bits.
574 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
575 Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1,
576 Shift / 8); // Shift is in bits.
577 } else if (Name == "llvm.x86.sse2.psrl.dq") {
578 // 128-bit shift right specified in bits.
579 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
580 Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1,
581 Shift / 8); // Shift is in bits.
582 } else if (Name == "llvm.x86.avx2.psll.dq") {
583 // 256-bit shift left specified in bits.
584 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
585 Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2,
586 Shift / 8); // Shift is in bits.
587 } else if (Name == "llvm.x86.avx2.psrl.dq") {
588 // 256-bit shift right specified in bits.
589 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
590 Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2,
591 Shift / 8); // Shift is in bits.
592 } else if (Name == "llvm.x86.sse2.psll.dq.bs") {
593 // 128-bit shift left specified in bytes.
594 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
595 Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1,
597 } else if (Name == "llvm.x86.sse2.psrl.dq.bs") {
598 // 128-bit shift right specified in bytes.
599 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
600 Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1,
602 } else if (Name == "llvm.x86.avx2.psll.dq.bs") {
603 // 256-bit shift left specified in bytes.
604 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
605 Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2,
607 } else if (Name == "llvm.x86.avx2.psrl.dq.bs") {
608 // 256-bit shift right specified in bytes.
609 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
610 Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2,
613 bool PD128 = false, PD256 = false, PS128 = false, PS256 = false;
614 if (Name == "llvm.x86.avx.vpermil.pd.256")
616 else if (Name == "llvm.x86.avx.vpermil.pd")
618 else if (Name == "llvm.x86.avx.vpermil.ps.256")
620 else if (Name == "llvm.x86.avx.vpermil.ps")
623 if (PD256 || PD128 || PS256 || PS128) {
624 Value *Op0 = CI->getArgOperand(0);
625 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
626 SmallVector<Constant*, 8> Idxs;
629 for (unsigned i = 0; i != 2; ++i)
630 Idxs.push_back(Builder.getInt32((Imm >> i) & 0x1));
632 for (unsigned l = 0; l != 4; l+=2)
633 for (unsigned i = 0; i != 2; ++i)
634 Idxs.push_back(Builder.getInt32(((Imm >> (l+i)) & 0x1) + l));
636 for (unsigned i = 0; i != 4; ++i)
637 Idxs.push_back(Builder.getInt32((Imm >> (2 * i)) & 0x3));
639 for (unsigned l = 0; l != 8; l+=4)
640 for (unsigned i = 0; i != 4; ++i)
641 Idxs.push_back(Builder.getInt32(((Imm >> (2 * i)) & 0x3) + l));
643 llvm_unreachable("Unexpected function");
645 Rep = Builder.CreateShuffleVector(Op0, Op0, ConstantVector::get(Idxs));
647 llvm_unreachable("Unknown function for CallInst upgrade.");
651 CI->replaceAllUsesWith(Rep);
652 CI->eraseFromParent();
656 std::string Name = CI->getName().str();
658 CI->setName(Name + ".old");
660 switch (NewFn->getIntrinsicID()) {
662 llvm_unreachable("Unknown function for CallInst upgrade.");
664 // Upgrade debug intrinsics to use an additional DIExpression argument.
665 case Intrinsic::dbg_declare: {
667 Builder.CreateCall3(NewFn, CI->getArgOperand(0), CI->getArgOperand(1),
668 getExpression(CI->getArgOperand(1), F), Name);
669 NewCI->setDebugLoc(CI->getDebugLoc());
670 CI->replaceAllUsesWith(NewCI);
671 CI->eraseFromParent();
674 case Intrinsic::dbg_value: {
675 auto NewCI = Builder.CreateCall4(
676 NewFn, CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
677 getExpression(CI->getArgOperand(2), F), Name);
678 NewCI->setDebugLoc(CI->getDebugLoc());
679 CI->replaceAllUsesWith(NewCI);
680 CI->eraseFromParent();
683 case Intrinsic::ctlz:
684 case Intrinsic::cttz:
685 assert(CI->getNumArgOperands() == 1 &&
686 "Mismatch between function args and call args");
687 CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0),
688 Builder.getFalse(), Name));
689 CI->eraseFromParent();
692 case Intrinsic::objectsize:
693 CI->replaceAllUsesWith(Builder.CreateCall2(NewFn,
694 CI->getArgOperand(0),
695 CI->getArgOperand(1),
697 CI->eraseFromParent();
700 case Intrinsic::ctpop: {
701 CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(0)));
702 CI->eraseFromParent();
706 case Intrinsic::x86_xop_vfrcz_ss:
707 case Intrinsic::x86_xop_vfrcz_sd:
708 CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(1),
710 CI->eraseFromParent();
713 case Intrinsic::x86_sse41_ptestc:
714 case Intrinsic::x86_sse41_ptestz:
715 case Intrinsic::x86_sse41_ptestnzc: {
716 // The arguments for these intrinsics used to be v4f32, and changed
717 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
718 // So, the only thing required is a bitcast for both arguments.
719 // First, check the arguments have the old type.
720 Value *Arg0 = CI->getArgOperand(0);
721 if (Arg0->getType() != VectorType::get(Type::getFloatTy(C), 4))
724 // Old intrinsic, add bitcasts
725 Value *Arg1 = CI->getArgOperand(1);
728 Builder.CreateBitCast(Arg0,
729 VectorType::get(Type::getInt64Ty(C), 2),
732 Builder.CreateBitCast(Arg1,
733 VectorType::get(Type::getInt64Ty(C), 2),
736 CallInst* NewCall = Builder.CreateCall2(NewFn, BC0, BC1, Name);
737 CI->replaceAllUsesWith(NewCall);
738 CI->eraseFromParent();
742 case Intrinsic::x86_sse41_pblendw:
743 case Intrinsic::x86_sse41_blendpd:
744 case Intrinsic::x86_sse41_blendps:
745 case Intrinsic::x86_sse41_insertps:
746 case Intrinsic::x86_sse41_dppd:
747 case Intrinsic::x86_sse41_dpps:
748 case Intrinsic::x86_sse41_mpsadbw:
749 case Intrinsic::x86_avx_blend_pd_256:
750 case Intrinsic::x86_avx_blend_ps_256:
751 case Intrinsic::x86_avx_dp_ps_256:
752 case Intrinsic::x86_avx2_pblendw:
753 case Intrinsic::x86_avx2_pblendd_128:
754 case Intrinsic::x86_avx2_pblendd_256:
755 case Intrinsic::x86_avx2_mpsadbw: {
756 // Need to truncate the last argument from i32 to i8 -- this argument models
757 // an inherently 8-bit immediate operand to these x86 instructions.
758 SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
759 CI->arg_operands().end());
761 // Replace the last argument with a trunc.
762 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
764 CallInst *NewCall = Builder.CreateCall(NewFn, Args);
765 CI->replaceAllUsesWith(NewCall);
766 CI->eraseFromParent();
769 case Intrinsic::x86_avx512_mask_cmp_ps_512:
770 case Intrinsic::x86_avx512_mask_cmp_pd_512: {
771 // Need to truncate the last argument from i32 to i8 -- this argument models
772 // an inherently 8-bit immediate operand to these x86 instructions.
773 SmallVector<Value *, 5> Args(CI->arg_operands().begin(),
774 CI->arg_operands().end());
776 // Replace the last argument with a trunc.
777 Args[2] = Builder.CreateTrunc(Args[2], Type::getInt8Ty(C), "trunc");
779 CallInst *NewCall = Builder.CreateCall(NewFn, Args);
780 CI->replaceAllUsesWith(NewCall);
781 CI->eraseFromParent();
787 // This tests each Function to determine if it needs upgrading. When we find
788 // one we are interested in, we then upgrade all calls to reflect the new
790 void llvm::UpgradeCallsToIntrinsic(Function* F) {
791 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
793 // Upgrade the function and check if it is a totaly new function.
795 if (UpgradeIntrinsicFunction(F, NewFn)) {
797 // Replace all uses to the old function with the new one if necessary.
798 for (Value::user_iterator UI = F->user_begin(), UE = F->user_end();
800 if (CallInst *CI = dyn_cast<CallInst>(*UI++))
801 UpgradeIntrinsicCall(CI, NewFn);
803 // Remove old function, no longer used, from the module.
804 F->eraseFromParent();
809 void llvm::UpgradeInstWithTBAATag(Instruction *I) {
810 MDNode *MD = I->getMetadata(LLVMContext::MD_tbaa);
811 assert(MD && "UpgradeInstWithTBAATag should have a TBAA tag");
812 // Check if the tag uses struct-path aware TBAA format.
813 if (isa<MDNode>(MD->getOperand(0)) && MD->getNumOperands() >= 3)
816 if (MD->getNumOperands() == 3) {
817 Metadata *Elts[] = {MD->getOperand(0), MD->getOperand(1)};
818 MDNode *ScalarType = MDNode::get(I->getContext(), Elts);
819 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
820 Metadata *Elts2[] = {ScalarType, ScalarType,
821 ConstantAsMetadata::get(Constant::getNullValue(
822 Type::getInt64Ty(I->getContext()))),
824 I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts2));
826 // Create a MDNode <MD, MD, offset 0>
827 Metadata *Elts[] = {MD, MD, ConstantAsMetadata::get(Constant::getNullValue(
828 Type::getInt64Ty(I->getContext())))};
829 I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts));
833 Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
834 Instruction *&Temp) {
835 if (Opc != Instruction::BitCast)
839 Type *SrcTy = V->getType();
840 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
841 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
842 LLVMContext &Context = V->getContext();
844 // We have no information about target data layout, so we assume that
845 // the maximum pointer size is 64bit.
846 Type *MidTy = Type::getInt64Ty(Context);
847 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
849 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
855 Value *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
856 if (Opc != Instruction::BitCast)
859 Type *SrcTy = C->getType();
860 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
861 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
862 LLVMContext &Context = C->getContext();
864 // We have no information about target data layout, so we assume that
865 // the maximum pointer size is 64bit.
866 Type *MidTy = Type::getInt64Ty(Context);
868 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
875 /// Check the debug info version number, if it is out-dated, drop the debug
876 /// info. Return true if module is modified.
877 bool llvm::UpgradeDebugInfo(Module &M) {
878 unsigned Version = getDebugMetadataVersionFromModule(M);
879 if (Version == DEBUG_METADATA_VERSION)
882 bool RetCode = StripDebugInfo(M);
884 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
885 M.getContext().diagnose(DiagVersion);
890 void llvm::UpgradeMDStringConstant(std::string &String) {
891 const std::string OldPrefix = "llvm.vectorizer.";
892 if (String == "llvm.vectorizer.unroll") {
893 String = "llvm.loop.interleave.count";
894 } else if (String.find(OldPrefix) == 0) {
895 String.replace(0, OldPrefix.size(), "llvm.loop.vectorize.");