1 //===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the auto-upgrade helper functions.
11 // This is where deprecated IR intrinsics and other IR features are updated to
12 // current specifications.
14 //===----------------------------------------------------------------------===//
16 #include "llvm/IR/AutoUpgrade.h"
17 #include "llvm/IR/CFG.h"
18 #include "llvm/IR/CallSite.h"
19 #include "llvm/IR/Constants.h"
20 #include "llvm/IR/DIBuilder.h"
21 #include "llvm/IR/DebugInfo.h"
22 #include "llvm/IR/DiagnosticInfo.h"
23 #include "llvm/IR/Function.h"
24 #include "llvm/IR/IRBuilder.h"
25 #include "llvm/IR/Instruction.h"
26 #include "llvm/IR/IntrinsicInst.h"
27 #include "llvm/IR/LLVMContext.h"
28 #include "llvm/IR/Module.h"
29 #include "llvm/Support/ErrorHandling.h"
33 // Upgrade the declarations of the SSE4.1 functions whose arguments have
34 // changed their type from v4f32 to v2i64.
35 static bool UpgradeSSE41Function(Function* F, Intrinsic::ID IID,
37 // Check whether this is an old version of the function, which received
39 Type *Arg0Type = F->getFunctionType()->getParamType(0);
40 if (Arg0Type != VectorType::get(Type::getFloatTy(F->getContext()), 4))
43 // Yes, it's old, replace it with new version.
44 F->setName(F->getName() + ".old");
45 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
49 // Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
50 // arguments have changed their type from i32 to i8.
51 static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
53 // Check that the last argument is an i32.
54 Type *LastArgType = F->getFunctionType()->getParamType(
55 F->getFunctionType()->getNumParams() - 1);
56 if (!LastArgType->isIntegerTy(32))
59 // Move this function aside and map down.
60 F->setName(F->getName() + ".old");
61 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
65 // Upgrade the declarations of AVX-512 cmp intrinsic functions whose 8-bit
66 // immediates have changed their type from i32 to i8.
67 static bool UpgradeAVX512CmpIntrinsic(Function *F, Intrinsic::ID IID,
69 // Check that the last argument is an i32.
70 Type *LastArgType = F->getFunctionType()->getParamType(2);
71 if (!LastArgType->isIntegerTy(32))
74 // Move this function aside and map down.
75 F->setName(F->getName() + ".old");
76 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
80 static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
81 assert(F && "Illegal to upgrade a non-existent Function.");
83 // Quickly eliminate it, if it's not a candidate.
84 StringRef Name = F->getName();
85 if (Name.size() <= 8 || !Name.startswith("llvm."))
87 Name = Name.substr(5); // Strip off "llvm."
92 if (Name.startswith("arm.neon.vclz")) {
94 F->arg_begin()->getType(),
95 Type::getInt1Ty(F->getContext())
97 // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to
98 // the end of the name. Change name from llvm.arm.neon.vclz.* to
100 FunctionType* fType = FunctionType::get(F->getReturnType(), args, false);
101 NewFn = Function::Create(fType, F->getLinkage(),
102 "llvm.ctlz." + Name.substr(14), F->getParent());
105 if (Name.startswith("arm.neon.vcnt")) {
106 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
107 F->arg_begin()->getType());
113 if (Name.startswith("ctlz.") && F->arg_size() == 1) {
114 F->setName(Name + ".old");
115 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
116 F->arg_begin()->getType());
119 if (Name.startswith("cttz.") && F->arg_size() == 1) {
120 F->setName(Name + ".old");
121 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz,
122 F->arg_begin()->getType());
128 if (Name.startswith("dbg.declare") && F->arg_size() == 2) {
129 F->setName(Name + ".old");
130 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_declare);
133 if (Name.startswith("dbg.value") && F->arg_size() == 3) {
134 F->setName(Name + ".old");
135 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value);
142 // We only need to change the name to match the mangling including the
144 if (F->arg_size() == 2 && Name.startswith("objectsize.")) {
145 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
146 if (F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) {
147 F->setName(Name + ".old");
148 NewFn = Intrinsic::getDeclaration(F->getParent(),
149 Intrinsic::objectsize, Tys);
156 if (Name.startswith("x86.sse2.pcmpeq.") ||
157 Name.startswith("x86.sse2.pcmpgt.") ||
158 Name.startswith("x86.avx2.pcmpeq.") ||
159 Name.startswith("x86.avx2.pcmpgt.") ||
160 Name.startswith("x86.avx.vpermil.") ||
161 Name == "x86.avx.vinsertf128.pd.256" ||
162 Name == "x86.avx.vinsertf128.ps.256" ||
163 Name == "x86.avx.vinsertf128.si.256" ||
164 Name == "x86.avx.movnt.dq.256" ||
165 Name == "x86.avx.movnt.pd.256" ||
166 Name == "x86.avx.movnt.ps.256" ||
167 Name == "x86.sse42.crc32.64.8" ||
168 Name == "x86.avx.vbroadcast.ss" ||
169 Name == "x86.avx.vbroadcast.ss.256" ||
170 Name == "x86.avx.vbroadcast.sd.256" ||
171 Name == "x86.sse2.psll.dq" ||
172 Name == "x86.sse2.psrl.dq" ||
173 Name == "x86.avx2.psll.dq" ||
174 Name == "x86.avx2.psrl.dq" ||
175 Name == "x86.sse2.psll.dq.bs" ||
176 Name == "x86.sse2.psrl.dq.bs" ||
177 Name == "x86.avx2.psll.dq.bs" ||
178 Name == "x86.avx2.psrl.dq.bs" ||
179 Name == "x86.sse41.pblendw" ||
180 Name == "x86.sse41.blendpd" ||
181 Name == "x86.sse41.blendps" ||
182 Name == "x86.avx.blend.pd.256" ||
183 Name == "x86.avx.blend.ps.256" ||
184 Name == "x86.avx2.pblendw" ||
185 Name == "x86.avx2.pblendd.128" ||
186 Name == "x86.avx2.pblendd.256" ||
187 Name == "x86.avx2.vbroadcasti128" ||
188 (Name.startswith("x86.xop.vpcom") && F->arg_size() == 2)) {
192 // SSE4.1 ptest functions may have an old signature.
193 if (Name.startswith("x86.sse41.ptest")) {
194 if (Name == "x86.sse41.ptestc")
195 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestc, NewFn);
196 if (Name == "x86.sse41.ptestz")
197 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestz, NewFn);
198 if (Name == "x86.sse41.ptestnzc")
199 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestnzc, NewFn);
201 // Several blend and other instructions with masks used the wrong number of
203 if (Name == "x86.sse41.insertps")
204 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps,
206 if (Name == "x86.sse41.dppd")
207 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd,
209 if (Name == "x86.sse41.dpps")
210 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps,
212 if (Name == "x86.sse41.mpsadbw")
213 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw,
215 if (Name == "x86.avx.dp.ps.256")
216 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256,
218 if (Name == "x86.avx2.mpsadbw")
219 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw,
222 if (Name == "x86.avx512.mask.cmp.ps.512")
223 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_ps_512,
225 if (Name == "x86.avx512.mask.cmp.pd.512")
226 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_pd_512,
229 if (Name == "x86.avx512.mask.cmp.b.512")
230 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_512,
232 if (Name == "x86.avx512.mask.cmp.w.512")
233 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_512,
235 if (Name == "x86.avx512.mask.cmp.d.512")
236 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_512,
238 if (Name == "x86.avx512.mask.cmp.q.512")
239 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_512,
241 if (Name == "x86.avx512.mask.ucmp.b.512")
242 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_512,
244 if (Name == "x86.avx512.mask.ucmp.w.512")
245 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_512,
247 if (Name == "x86.avx512.mask.ucmp.d.512")
248 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_512,
250 if (Name == "x86.avx512.mask.ucmp.q.512")
251 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_512,
254 if (Name == "x86.avx512.mask.cmp.b.256")
255 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_256,
257 if (Name == "x86.avx512.mask.cmp.w.256")
258 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_256,
260 if (Name == "x86.avx512.mask.cmp.d.256")
261 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_256,
263 if (Name == "x86.avx512.mask.cmp.q.256")
264 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_256,
266 if (Name == "x86.avx512.mask.ucmp.b.256")
267 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_256,
269 if (Name == "x86.avx512.mask.ucmp.w.256")
270 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_256,
272 if (Name == "x86.avx512.mask.ucmp.d.256")
273 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_256,
275 if (Name == "x86.avx512.mask.ucmp.q.256")
276 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_256,
279 if (Name == "x86.avx512.mask.cmp.b.128")
280 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_128,
282 if (Name == "x86.avx512.mask.cmp.w.128")
283 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_128,
285 if (Name == "x86.avx512.mask.cmp.d.128")
286 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_128,
288 if (Name == "x86.avx512.mask.cmp.q.128")
289 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_128,
291 if (Name == "x86.avx512.mask.ucmp.b.128")
292 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_128,
294 if (Name == "x86.avx512.mask.ucmp.w.128")
295 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_128,
297 if (Name == "x86.avx512.mask.ucmp.d.128")
298 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_128,
300 if (Name == "x86.avx512.mask.ucmp.q.128")
301 return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_128,
304 // frcz.ss/sd may need to have an argument dropped
305 if (Name.startswith("x86.xop.vfrcz.ss") && F->arg_size() == 2) {
306 F->setName(Name + ".old");
307 NewFn = Intrinsic::getDeclaration(F->getParent(),
308 Intrinsic::x86_xop_vfrcz_ss);
311 if (Name.startswith("x86.xop.vfrcz.sd") && F->arg_size() == 2) {
312 F->setName(Name + ".old");
313 NewFn = Intrinsic::getDeclaration(F->getParent(),
314 Intrinsic::x86_xop_vfrcz_sd);
317 // Fix the FMA4 intrinsics to remove the 4
318 if (Name.startswith("x86.fma4.")) {
319 F->setName("llvm.x86.fma" + Name.substr(8));
327 // This may not belong here. This function is effectively being overloaded
328 // to both detect an intrinsic which needs upgrading, and to provide the
329 // upgraded form of the intrinsic. We should perhaps have two separate
330 // functions for this.
334 bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {
336 bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn);
338 // Upgrade intrinsic attributes. This does not change the function.
341 if (unsigned id = F->getIntrinsicID())
342 F->setAttributes(Intrinsic::getAttributes(F->getContext(),
347 bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
348 // Nothing to do yet.
352 static MDNode *getNodeField(const MDNode *DbgNode, unsigned Elt) {
353 if (!DbgNode || Elt >= DbgNode->getNumOperands())
355 return dyn_cast_or_null<MDNode>(DbgNode->getOperand(Elt));
358 static MetadataAsValue *getExpression(Value *VarOperand, Function *F) {
359 // Old-style DIVariables have an optional expression as the 8th element.
360 DIExpression Expr(getNodeField(
361 cast<MDNode>(cast<MetadataAsValue>(VarOperand)->getMetadata()), 8));
363 DIBuilder DIB(*F->getParent(), /*AllowUnresolved*/ false);
364 Expr = DIB.createExpression();
366 return MetadataAsValue::get(F->getContext(), Expr);
369 // Handles upgrading SSE2 and AVX2 PSLLDQ intrinsics by converting them
371 static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, LLVMContext &C,
372 Value *Op, unsigned NumLanes,
374 // Each lane is 16 bytes.
375 unsigned NumElts = NumLanes * 16;
377 // Bitcast from a 64-bit element type to a byte element type.
378 Op = Builder.CreateBitCast(Op,
379 VectorType::get(Type::getInt8Ty(C), NumElts),
381 // We'll be shuffling in zeroes.
382 Value *Res = ConstantVector::getSplat(NumElts, Builder.getInt8(0));
384 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
385 // we'll just return the zero vector.
387 SmallVector<Constant*, 32> Idxs;
388 // 256-bit version is split into two 16-byte lanes.
389 for (unsigned l = 0; l != NumElts; l += 16)
390 for (unsigned i = 0; i != 16; ++i) {
391 unsigned Idx = NumElts + i - Shift;
393 Idx -= NumElts - 16; // end of lane, switch operand.
394 Idxs.push_back(Builder.getInt32(Idx + l));
397 Res = Builder.CreateShuffleVector(Res, Op, ConstantVector::get(Idxs));
400 // Bitcast back to a 64-bit element type.
401 return Builder.CreateBitCast(Res,
402 VectorType::get(Type::getInt64Ty(C), 2*NumLanes),
406 // Handles upgrading SSE2 and AVX2 PSRLDQ intrinsics by converting them
408 static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, LLVMContext &C,
409 Value *Op, unsigned NumLanes,
411 // Each lane is 16 bytes.
412 unsigned NumElts = NumLanes * 16;
414 // Bitcast from a 64-bit element type to a byte element type.
415 Op = Builder.CreateBitCast(Op,
416 VectorType::get(Type::getInt8Ty(C), NumElts),
418 // We'll be shuffling in zeroes.
419 Value *Res = ConstantVector::getSplat(NumElts, Builder.getInt8(0));
421 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
422 // we'll just return the zero vector.
424 SmallVector<Constant*, 32> Idxs;
425 // 256-bit version is split into two 16-byte lanes.
426 for (unsigned l = 0; l != NumElts; l += 16)
427 for (unsigned i = 0; i != 16; ++i) {
428 unsigned Idx = i + Shift;
430 Idx += NumElts - 16; // end of lane, switch operand.
431 Idxs.push_back(Builder.getInt32(Idx + l));
434 Res = Builder.CreateShuffleVector(Op, Res, ConstantVector::get(Idxs));
437 // Bitcast back to a 64-bit element type.
438 return Builder.CreateBitCast(Res,
439 VectorType::get(Type::getInt64Ty(C), 2*NumLanes),
443 // UpgradeIntrinsicCall - Upgrade a call to an old intrinsic to be a call the
444 // upgraded intrinsic. All argument and return casting must be provided in
445 // order to seamlessly integrate with existing context.
446 void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
447 Function *F = CI->getCalledFunction();
448 LLVMContext &C = CI->getContext();
449 IRBuilder<> Builder(C);
450 Builder.SetInsertPoint(CI->getParent(), CI);
452 assert(F && "Intrinsic call is not direct?");
455 // Get the Function's name.
456 StringRef Name = F->getName();
459 // Upgrade packed integer vector compares intrinsics to compare instructions
460 if (Name.startswith("llvm.x86.sse2.pcmpeq.") ||
461 Name.startswith("llvm.x86.avx2.pcmpeq.")) {
462 Rep = Builder.CreateICmpEQ(CI->getArgOperand(0), CI->getArgOperand(1),
464 // need to sign extend since icmp returns vector of i1
465 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
466 } else if (Name.startswith("llvm.x86.sse2.pcmpgt.") ||
467 Name.startswith("llvm.x86.avx2.pcmpgt.")) {
468 Rep = Builder.CreateICmpSGT(CI->getArgOperand(0), CI->getArgOperand(1),
470 // need to sign extend since icmp returns vector of i1
471 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
472 } else if (Name == "llvm.x86.avx.movnt.dq.256" ||
473 Name == "llvm.x86.avx.movnt.ps.256" ||
474 Name == "llvm.x86.avx.movnt.pd.256") {
475 IRBuilder<> Builder(C);
476 Builder.SetInsertPoint(CI->getParent(), CI);
478 Module *M = F->getParent();
479 SmallVector<Metadata *, 1> Elts;
481 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
482 MDNode *Node = MDNode::get(C, Elts);
484 Value *Arg0 = CI->getArgOperand(0);
485 Value *Arg1 = CI->getArgOperand(1);
487 // Convert the type of the pointer to a pointer to the stored type.
488 Value *BC = Builder.CreateBitCast(Arg0,
489 PointerType::getUnqual(Arg1->getType()),
491 StoreInst *SI = Builder.CreateStore(Arg1, BC);
492 SI->setMetadata(M->getMDKindID("nontemporal"), Node);
493 SI->setAlignment(16);
496 CI->eraseFromParent();
498 } else if (Name.startswith("llvm.x86.xop.vpcom")) {
500 if (Name.endswith("ub"))
501 intID = Intrinsic::x86_xop_vpcomub;
502 else if (Name.endswith("uw"))
503 intID = Intrinsic::x86_xop_vpcomuw;
504 else if (Name.endswith("ud"))
505 intID = Intrinsic::x86_xop_vpcomud;
506 else if (Name.endswith("uq"))
507 intID = Intrinsic::x86_xop_vpcomuq;
508 else if (Name.endswith("b"))
509 intID = Intrinsic::x86_xop_vpcomb;
510 else if (Name.endswith("w"))
511 intID = Intrinsic::x86_xop_vpcomw;
512 else if (Name.endswith("d"))
513 intID = Intrinsic::x86_xop_vpcomd;
514 else if (Name.endswith("q"))
515 intID = Intrinsic::x86_xop_vpcomq;
517 llvm_unreachable("Unknown suffix");
519 Name = Name.substr(18); // strip off "llvm.x86.xop.vpcom"
521 if (Name.startswith("lt"))
523 else if (Name.startswith("le"))
525 else if (Name.startswith("gt"))
527 else if (Name.startswith("ge"))
529 else if (Name.startswith("eq"))
531 else if (Name.startswith("ne"))
533 else if (Name.startswith("false"))
535 else if (Name.startswith("true"))
538 llvm_unreachable("Unknown condition");
540 Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID);
541 Rep = Builder.CreateCall3(VPCOM, CI->getArgOperand(0),
542 CI->getArgOperand(1), Builder.getInt8(Imm));
543 } else if (Name == "llvm.x86.sse42.crc32.64.8") {
544 Function *CRC32 = Intrinsic::getDeclaration(F->getParent(),
545 Intrinsic::x86_sse42_crc32_32_8);
546 Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
547 Rep = Builder.CreateCall2(CRC32, Trunc0, CI->getArgOperand(1));
548 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
549 } else if (Name.startswith("llvm.x86.avx.vbroadcast")) {
550 // Replace broadcasts with a series of insertelements.
551 Type *VecTy = CI->getType();
552 Type *EltTy = VecTy->getVectorElementType();
553 unsigned EltNum = VecTy->getVectorNumElements();
554 Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),
555 EltTy->getPointerTo());
556 Value *Load = Builder.CreateLoad(Cast);
557 Type *I32Ty = Type::getInt32Ty(C);
558 Rep = UndefValue::get(VecTy);
559 for (unsigned I = 0; I < EltNum; ++I)
560 Rep = Builder.CreateInsertElement(Rep, Load,
561 ConstantInt::get(I32Ty, I));
562 } else if (Name == "llvm.x86.avx2.vbroadcasti128") {
563 // Replace vbroadcasts with a vector shuffle.
564 Value *Op = Builder.CreatePointerCast(
565 CI->getArgOperand(0),
566 PointerType::getUnqual(VectorType::get(Type::getInt64Ty(C), 2)));
567 Value *Load = Builder.CreateLoad(Op);
568 SmallVector<Constant *, 4> Idxs; // 0, 1, 0, 1.
569 for (unsigned i = 0; i != 4; ++i)
570 Idxs.push_back(Builder.getInt32(i & 1));
571 Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
572 ConstantVector::get(Idxs));
573 } else if (Name == "llvm.x86.sse2.psll.dq") {
574 // 128-bit shift left specified in bits.
575 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
576 Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1,
577 Shift / 8); // Shift is in bits.
578 } else if (Name == "llvm.x86.sse2.psrl.dq") {
579 // 128-bit shift right specified in bits.
580 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
581 Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1,
582 Shift / 8); // Shift is in bits.
583 } else if (Name == "llvm.x86.avx2.psll.dq") {
584 // 256-bit shift left specified in bits.
585 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
586 Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2,
587 Shift / 8); // Shift is in bits.
588 } else if (Name == "llvm.x86.avx2.psrl.dq") {
589 // 256-bit shift right specified in bits.
590 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
591 Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2,
592 Shift / 8); // Shift is in bits.
593 } else if (Name == "llvm.x86.sse2.psll.dq.bs") {
594 // 128-bit shift left specified in bytes.
595 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
596 Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1,
598 } else if (Name == "llvm.x86.sse2.psrl.dq.bs") {
599 // 128-bit shift right specified in bytes.
600 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
601 Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1,
603 } else if (Name == "llvm.x86.avx2.psll.dq.bs") {
604 // 256-bit shift left specified in bytes.
605 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
606 Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2,
608 } else if (Name == "llvm.x86.avx2.psrl.dq.bs") {
609 // 256-bit shift right specified in bytes.
610 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
611 Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2,
613 } else if (Name == "llvm.x86.sse41.pblendw" ||
614 Name == "llvm.x86.sse41.blendpd" ||
615 Name == "llvm.x86.sse41.blendps" ||
616 Name == "llvm.x86.avx.blend.pd.256" ||
617 Name == "llvm.x86.avx.blend.ps.256" ||
618 Name == "llvm.x86.avx2.pblendw" ||
619 Name == "llvm.x86.avx2.pblendd.128" ||
620 Name == "llvm.x86.avx2.pblendd.256") {
621 Value *Op0 = CI->getArgOperand(0);
622 Value *Op1 = CI->getArgOperand(1);
623 unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue();
624 VectorType *VecTy = cast<VectorType>(CI->getType());
625 unsigned NumElts = VecTy->getNumElements();
627 SmallVector<Constant*, 16> Idxs;
628 for (unsigned i = 0; i != NumElts; ++i) {
629 unsigned Idx = ((Imm >> (i%8)) & 1) ? i + NumElts : i;
630 Idxs.push_back(Builder.getInt32(Idx));
633 Rep = Builder.CreateShuffleVector(Op0, Op1, ConstantVector::get(Idxs));
634 } else if (Name == "llvm.x86.avx.vinsertf128.pd.256" ||
635 Name == "llvm.x86.avx.vinsertf128.ps.256" ||
636 Name == "llvm.x86.avx.vinsertf128.si.256") {
637 Value *Op0 = CI->getArgOperand(0);
638 Value *Op1 = CI->getArgOperand(1);
639 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
640 VectorType *VecTy = cast<VectorType>(CI->getType());
641 unsigned NumElts = VecTy->getNumElements();
643 // Mask off the high bits of the immediate value; hardware ignores those.
646 // Extend the second operand into a vector that is twice as big.
647 Value *UndefV = UndefValue::get(Op1->getType());
648 SmallVector<Constant*, 8> Idxs;
649 for (unsigned i = 0; i != NumElts; ++i) {
650 Idxs.push_back(Builder.getInt32(i));
652 Rep = Builder.CreateShuffleVector(Op1, UndefV, ConstantVector::get(Idxs));
654 // Insert the second operand into the first operand.
656 // Note that there is no guarantee that instruction lowering will actually
657 // produce a vinsertf128 instruction for the created shuffles. In
658 // particular, the 0 immediate case involves no lane changes, so it can
659 // be handled as a blend.
661 // Example of shuffle mask for 32-bit elements:
662 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
663 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
665 SmallVector<Constant*, 8> Idxs2;
666 // The low half of the result is either the low half of the 1st operand
667 // or the low half of the 2nd operand (the inserted vector).
668 for (unsigned i = 0; i != NumElts / 2; ++i) {
669 unsigned Idx = Imm ? i : (i + NumElts);
670 Idxs2.push_back(Builder.getInt32(Idx));
672 // The high half of the result is either the low half of the 2nd operand
673 // (the inserted vector) or the high half of the 1st operand.
674 for (unsigned i = NumElts / 2; i != NumElts; ++i) {
675 unsigned Idx = Imm ? (i + NumElts / 2) : i;
676 Idxs2.push_back(Builder.getInt32(Idx));
678 Rep = Builder.CreateShuffleVector(Op0, Rep, ConstantVector::get(Idxs2));
680 bool PD128 = false, PD256 = false, PS128 = false, PS256 = false;
681 if (Name == "llvm.x86.avx.vpermil.pd.256")
683 else if (Name == "llvm.x86.avx.vpermil.pd")
685 else if (Name == "llvm.x86.avx.vpermil.ps.256")
687 else if (Name == "llvm.x86.avx.vpermil.ps")
690 if (PD256 || PD128 || PS256 || PS128) {
691 Value *Op0 = CI->getArgOperand(0);
692 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
693 SmallVector<Constant*, 8> Idxs;
696 for (unsigned i = 0; i != 2; ++i)
697 Idxs.push_back(Builder.getInt32((Imm >> i) & 0x1));
699 for (unsigned l = 0; l != 4; l+=2)
700 for (unsigned i = 0; i != 2; ++i)
701 Idxs.push_back(Builder.getInt32(((Imm >> (l+i)) & 0x1) + l));
703 for (unsigned i = 0; i != 4; ++i)
704 Idxs.push_back(Builder.getInt32((Imm >> (2 * i)) & 0x3));
706 for (unsigned l = 0; l != 8; l+=4)
707 for (unsigned i = 0; i != 4; ++i)
708 Idxs.push_back(Builder.getInt32(((Imm >> (2 * i)) & 0x3) + l));
710 llvm_unreachable("Unexpected function");
712 Rep = Builder.CreateShuffleVector(Op0, Op0, ConstantVector::get(Idxs));
714 llvm_unreachable("Unknown function for CallInst upgrade.");
718 CI->replaceAllUsesWith(Rep);
719 CI->eraseFromParent();
723 std::string Name = CI->getName().str();
725 CI->setName(Name + ".old");
727 switch (NewFn->getIntrinsicID()) {
729 llvm_unreachable("Unknown function for CallInst upgrade.");
731 // Upgrade debug intrinsics to use an additional DIExpression argument.
732 case Intrinsic::dbg_declare: {
734 Builder.CreateCall3(NewFn, CI->getArgOperand(0), CI->getArgOperand(1),
735 getExpression(CI->getArgOperand(1), F), Name);
736 NewCI->setDebugLoc(CI->getDebugLoc());
737 CI->replaceAllUsesWith(NewCI);
738 CI->eraseFromParent();
741 case Intrinsic::dbg_value: {
742 auto NewCI = Builder.CreateCall4(
743 NewFn, CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
744 getExpression(CI->getArgOperand(2), F), Name);
745 NewCI->setDebugLoc(CI->getDebugLoc());
746 CI->replaceAllUsesWith(NewCI);
747 CI->eraseFromParent();
750 case Intrinsic::ctlz:
751 case Intrinsic::cttz:
752 assert(CI->getNumArgOperands() == 1 &&
753 "Mismatch between function args and call args");
754 CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0),
755 Builder.getFalse(), Name));
756 CI->eraseFromParent();
759 case Intrinsic::objectsize:
760 CI->replaceAllUsesWith(Builder.CreateCall2(NewFn,
761 CI->getArgOperand(0),
762 CI->getArgOperand(1),
764 CI->eraseFromParent();
767 case Intrinsic::ctpop: {
768 CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(0)));
769 CI->eraseFromParent();
773 case Intrinsic::x86_xop_vfrcz_ss:
774 case Intrinsic::x86_xop_vfrcz_sd:
775 CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(1),
777 CI->eraseFromParent();
780 case Intrinsic::x86_sse41_ptestc:
781 case Intrinsic::x86_sse41_ptestz:
782 case Intrinsic::x86_sse41_ptestnzc: {
783 // The arguments for these intrinsics used to be v4f32, and changed
784 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
785 // So, the only thing required is a bitcast for both arguments.
786 // First, check the arguments have the old type.
787 Value *Arg0 = CI->getArgOperand(0);
788 if (Arg0->getType() != VectorType::get(Type::getFloatTy(C), 4))
791 // Old intrinsic, add bitcasts
792 Value *Arg1 = CI->getArgOperand(1);
795 Builder.CreateBitCast(Arg0,
796 VectorType::get(Type::getInt64Ty(C), 2),
799 Builder.CreateBitCast(Arg1,
800 VectorType::get(Type::getInt64Ty(C), 2),
803 CallInst* NewCall = Builder.CreateCall2(NewFn, BC0, BC1, Name);
804 CI->replaceAllUsesWith(NewCall);
805 CI->eraseFromParent();
809 case Intrinsic::x86_sse41_insertps:
810 case Intrinsic::x86_sse41_dppd:
811 case Intrinsic::x86_sse41_dpps:
812 case Intrinsic::x86_sse41_mpsadbw:
813 case Intrinsic::x86_avx_dp_ps_256:
814 case Intrinsic::x86_avx2_mpsadbw: {
815 // Need to truncate the last argument from i32 to i8 -- this argument models
816 // an inherently 8-bit immediate operand to these x86 instructions.
817 SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
818 CI->arg_operands().end());
820 // Replace the last argument with a trunc.
821 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
823 CallInst *NewCall = Builder.CreateCall(NewFn, Args);
824 CI->replaceAllUsesWith(NewCall);
825 CI->eraseFromParent();
828 case Intrinsic::x86_avx512_mask_cmp_ps_512:
829 case Intrinsic::x86_avx512_mask_cmp_pd_512: {
830 // Need to truncate the last argument from i32 to i8 -- this argument models
831 // an inherently 8-bit immediate operand to these x86 instructions.
832 SmallVector<Value *, 5> Args(CI->arg_operands().begin(),
833 CI->arg_operands().end());
835 // Replace the last argument with a trunc.
836 Args[2] = Builder.CreateTrunc(Args[2], Type::getInt8Ty(C), "trunc");
838 CallInst *NewCall = Builder.CreateCall(NewFn, Args);
839 CI->replaceAllUsesWith(NewCall);
840 CI->eraseFromParent();
846 // This tests each Function to determine if it needs upgrading. When we find
847 // one we are interested in, we then upgrade all calls to reflect the new
849 void llvm::UpgradeCallsToIntrinsic(Function* F) {
850 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
852 // Upgrade the function and check if it is a totaly new function.
854 if (UpgradeIntrinsicFunction(F, NewFn)) {
856 // Replace all uses to the old function with the new one if necessary.
857 for (Value::user_iterator UI = F->user_begin(), UE = F->user_end();
859 if (CallInst *CI = dyn_cast<CallInst>(*UI++))
860 UpgradeIntrinsicCall(CI, NewFn);
862 // Remove old function, no longer used, from the module.
863 F->eraseFromParent();
868 void llvm::UpgradeInstWithTBAATag(Instruction *I) {
869 MDNode *MD = I->getMetadata(LLVMContext::MD_tbaa);
870 assert(MD && "UpgradeInstWithTBAATag should have a TBAA tag");
871 // Check if the tag uses struct-path aware TBAA format.
872 if (isa<MDNode>(MD->getOperand(0)) && MD->getNumOperands() >= 3)
875 if (MD->getNumOperands() == 3) {
876 Metadata *Elts[] = {MD->getOperand(0), MD->getOperand(1)};
877 MDNode *ScalarType = MDNode::get(I->getContext(), Elts);
878 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
879 Metadata *Elts2[] = {ScalarType, ScalarType,
880 ConstantAsMetadata::get(Constant::getNullValue(
881 Type::getInt64Ty(I->getContext()))),
883 I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts2));
885 // Create a MDNode <MD, MD, offset 0>
886 Metadata *Elts[] = {MD, MD, ConstantAsMetadata::get(Constant::getNullValue(
887 Type::getInt64Ty(I->getContext())))};
888 I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts));
892 Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
893 Instruction *&Temp) {
894 if (Opc != Instruction::BitCast)
898 Type *SrcTy = V->getType();
899 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
900 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
901 LLVMContext &Context = V->getContext();
903 // We have no information about target data layout, so we assume that
904 // the maximum pointer size is 64bit.
905 Type *MidTy = Type::getInt64Ty(Context);
906 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
908 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
914 Value *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
915 if (Opc != Instruction::BitCast)
918 Type *SrcTy = C->getType();
919 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
920 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
921 LLVMContext &Context = C->getContext();
923 // We have no information about target data layout, so we assume that
924 // the maximum pointer size is 64bit.
925 Type *MidTy = Type::getInt64Ty(Context);
927 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
934 /// Check the debug info version number, if it is out-dated, drop the debug
935 /// info. Return true if module is modified.
936 bool llvm::UpgradeDebugInfo(Module &M) {
937 unsigned Version = getDebugMetadataVersionFromModule(M);
938 if (Version == DEBUG_METADATA_VERSION)
941 bool RetCode = StripDebugInfo(M);
943 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
944 M.getContext().diagnose(DiagVersion);
949 void llvm::UpgradeMDStringConstant(std::string &String) {
950 const std::string OldPrefix = "llvm.vectorizer.";
951 if (String == "llvm.vectorizer.unroll") {
952 String = "llvm.loop.interleave.count";
953 } else if (String.find(OldPrefix) == 0) {
954 String.replace(0, OldPrefix.size(), "llvm.loop.vectorize.");