lib/VMCore/AutoUpgrade.cpp

   1 //===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This file implements the auto-upgrade helper functions
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/AutoUpgrade.h"
  15 #include "llvm/Constants.h"
  16 #include "llvm/Function.h"
  17 #include "llvm/LLVMContext.h"
  18 #include "llvm/Module.h"
  19 #include "llvm/IntrinsicInst.h"
  20 #include "llvm/ADT/SmallVector.h"
  21 #include "llvm/Support/CallSite.h"
  22 #include "llvm/Support/ErrorHandling.h"
  23 #include "llvm/Support/IRBuilder.h"
  24 #include <cstring>
  25 using namespace llvm;
  26
  27
  28 static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
  29   assert(F && "Illegal to upgrade a non-existent Function.");
  30
  31   // Get the Function's name.
  32   const std::string& Name = F->getName();
  33
  34   // Convenience
  35   const FunctionType *FTy = F->getFunctionType();
  36
  37   // Quickly eliminate it, if it's not a candidate.
  38   if (Name.length() <= 8 || Name[0] != 'l' || Name[1] != 'l' ||
  39       Name[2] != 'v' || Name[3] != 'm' || Name[4] != '.')
  40     return false;
  41
  42   Module *M = F->getParent();
  43   switch (Name[5]) {
  44   default: break;
  45   case 'a':
  46     // This upgrades the llvm.atomic.lcs, llvm.atomic.las, llvm.atomic.lss,
  47     // and atomics with default address spaces to their new names to their new
  48     // function name (e.g. llvm.atomic.add.i32 => llvm.atomic.add.i32.p0i32)
  49     if (Name.compare(5,7,"atomic.",7) == 0) {
  50       if (Name.compare(12,3,"lcs",3) == 0) {
  51         std::string::size_type delim = Name.find('.',12);
  52         F->setName("llvm.atomic.cmp.swap" + Name.substr(delim) +
  53                    ".p0" + Name.substr(delim+1));
  54         NewFn = F;
  55         return true;
  56       }
  57       else if (Name.compare(12,3,"las",3) == 0) {
  58         std::string::size_type delim = Name.find('.',12);
  59         F->setName("llvm.atomic.load.add"+Name.substr(delim)
  60                    + ".p0" + Name.substr(delim+1));
  61         NewFn = F;
  62         return true;
  63       }
  64       else if (Name.compare(12,3,"lss",3) == 0) {
  65         std::string::size_type delim = Name.find('.',12);
  66         F->setName("llvm.atomic.load.sub"+Name.substr(delim)
  67                    + ".p0" + Name.substr(delim+1));
  68         NewFn = F;
  69         return true;
  70       }
  71       else if (Name.rfind(".p") == std::string::npos) {
  72         // We don't have an address space qualifier so this has be upgraded
  73         // to the new name.  Copy the type name at the end of the intrinsic
  74         // and add to it
  75         std::string::size_type delim = Name.find_last_of('.');
  76         assert(delim != std::string::npos && "can not find type");
  77         F->setName(Name + ".p0" + Name.substr(delim+1));
  78         NewFn = F;
  79         return true;
  80       }
  81     } else if (Name.compare(5, 9, "arm.neon.", 9) == 0) {
  82       if (((Name.compare(14, 5, "vmovl", 5) == 0 ||
  83             Name.compare(14, 5, "vaddl", 5) == 0 ||
  84             Name.compare(14, 5, "vsubl", 5) == 0 ||
  85             Name.compare(14, 5, "vaddw", 5) == 0 ||
  86             Name.compare(14, 5, "vsubw", 5) == 0 ||
  87             Name.compare(14, 5, "vmlal", 5) == 0 ||
  88             Name.compare(14, 5, "vmlsl", 5) == 0 ||
  89             Name.compare(14, 5, "vabdl", 5) == 0 ||
  90             Name.compare(14, 5, "vabal", 5) == 0) &&
  91            (Name.compare(19, 2, "s.", 2) == 0 ||
  92             Name.compare(19, 2, "u.", 2) == 0)) ||
  93
  94           (Name.compare(14, 4, "vaba", 4) == 0 &&
  95            (Name.compare(18, 2, "s.", 2) == 0 ||
  96             Name.compare(18, 2, "u.", 2) == 0)) ||
  97
  98           (Name.compare(14, 6, "vmovn.", 6) == 0)) {
  99
 100         // Calls to these are transformed into IR without intrinsics.
 101         NewFn = 0;
 102         return true;
 103       }
 104       // Old versions of NEON ld/st intrinsics are missing alignment arguments.
 105       bool isVLd = (Name.compare(14, 3, "vld", 3) == 0);
 106       bool isVSt = (Name.compare(14, 3, "vst", 3) == 0);
 107       if (isVLd || isVSt) {
 108         unsigned NumVecs = Name.at(17) - '0';
 109         if (NumVecs == 0 || NumVecs > 4)
 110           return false;
 111         bool isLaneOp = (Name.compare(18, 5, "lane.", 5) == 0);
 112         if (!isLaneOp && Name.at(18) != '.')
 113           return false;
 114         unsigned ExpectedArgs = 2; // for the address and alignment
 115         if (isVSt || isLaneOp)
 116           ExpectedArgs += NumVecs;
 117         if (isLaneOp)
 118           ExpectedArgs += 1; // for the lane number
 119         unsigned NumP = FTy->getNumParams();
 120         if (NumP != ExpectedArgs - 1)
 121           return false;
 122
 123         // Change the name of the old (bad) intrinsic, because
 124         // its type is incorrect, but we cannot overload that name.
 125         F->setName("");
 126
 127         // One argument is missing: add the alignment argument.
 128         std::vector<const Type*> NewParams;
 129         for (unsigned p = 0; p < NumP; ++p)
 130           NewParams.push_back(FTy->getParamType(p));
 131         NewParams.push_back(Type::getInt32Ty(F->getContext()));
 132         FunctionType *NewFTy = FunctionType::get(FTy->getReturnType(),
 133                                                  NewParams, false);
 134         NewFn = cast<Function>(M->getOrInsertFunction(Name, NewFTy));
 135         return true;
 136       }
 137     }
 138     break;
 139   case 'b':
 140     //  This upgrades the name of the llvm.bswap intrinsic function to only use
 141     //  a single type name for overloading. We only care about the old format
 142     //  'llvm.bswap.i*.i*', so check for 'bswap.' and then for there being
 143     //  a '.' after 'bswap.'
 144     if (Name.compare(5,6,"bswap.",6) == 0) {
 145       std::string::size_type delim = Name.find('.',11);
 146
 147       if (delim != std::string::npos) {
 148         //  Construct the new name as 'llvm.bswap' + '.i*'
 149         F->setName(Name.substr(0,10)+Name.substr(delim));
 150         NewFn = F;
 151         return true;
 152       }
 153     }
 154     break;
 155
 156   case 'c':
 157     //  We only want to fix the 'llvm.ct*' intrinsics which do not have the
 158     //  correct return type, so we check for the name, and then check if the
 159     //  return type does not match the parameter type.
 160     if ( (Name.compare(5,5,"ctpop",5) == 0 ||
 161           Name.compare(5,4,"ctlz",4) == 0 ||
 162           Name.compare(5,4,"cttz",4) == 0) &&
 163         FTy->getReturnType() != FTy->getParamType(0)) {
 164       //  We first need to change the name of the old (bad) intrinsic, because
 165       //  its type is incorrect, but we cannot overload that name. We
 166       //  arbitrarily unique it here allowing us to construct a correctly named
 167       //  and typed function below.
 168       F->setName("");
 169
 170       //  Now construct the new intrinsic with the correct name and type. We
 171       //  leave the old function around in order to query its type, whatever it
 172       //  may be, and correctly convert up to the new type.
 173       NewFn = cast<Function>(M->getOrInsertFunction(Name,
 174                                                     FTy->getParamType(0),
 175                                                     FTy->getParamType(0),
 176                                                     (Type *)0));
 177       return true;
 178     }
 179     break;
 180
 181   case 'e':
 182     //  The old llvm.eh.selector.i32 is equivalent to the new llvm.eh.selector.
 183     if (Name.compare("llvm.eh.selector.i32") == 0) {
 184       F->setName("llvm.eh.selector");
 185       NewFn = F;
 186       return true;
 187     }
 188     //  The old llvm.eh.typeid.for.i32 is equivalent to llvm.eh.typeid.for.
 189     if (Name.compare("llvm.eh.typeid.for.i32") == 0) {
 190       F->setName("llvm.eh.typeid.for");
 191       NewFn = F;
 192       return true;
 193     }
 194     //  Convert the old llvm.eh.selector.i64 to a call to llvm.eh.selector.
 195     if (Name.compare("llvm.eh.selector.i64") == 0) {
 196       NewFn = Intrinsic::getDeclaration(M, Intrinsic::eh_selector);
 197       return true;
 198     }
 199     //  Convert the old llvm.eh.typeid.for.i64 to a call to llvm.eh.typeid.for.
 200     if (Name.compare("llvm.eh.typeid.for.i64") == 0) {
 201       NewFn = Intrinsic::getDeclaration(M, Intrinsic::eh_typeid_for);
 202       return true;
 203     }
 204     break;
 205
 206   case 'm': {
 207     // This upgrades the llvm.memcpy, llvm.memmove, and llvm.memset to the
 208     // new format that allows overloading the pointer for different address
 209     // space (e.g., llvm.memcpy.i16 => llvm.memcpy.p0i8.p0i8.i16)
 210     const char* NewFnName = NULL;
 211     if (Name.compare(5,8,"memcpy.i",8) == 0) {
 212       if (Name[13] == '8')
 213         NewFnName = "llvm.memcpy.p0i8.p0i8.i8";
 214       else if (Name.compare(13,2,"16") == 0)
 215         NewFnName = "llvm.memcpy.p0i8.p0i8.i16";
 216       else if (Name.compare(13,2,"32") == 0)
 217         NewFnName = "llvm.memcpy.p0i8.p0i8.i32";
 218       else if (Name.compare(13,2,"64") == 0)
 219         NewFnName = "llvm.memcpy.p0i8.p0i8.i64";
 220     } else if (Name.compare(5,9,"memmove.i",9) == 0) {
 221       if (Name[14] == '8')
 222         NewFnName = "llvm.memmove.p0i8.p0i8.i8";
 223       else if (Name.compare(14,2,"16") == 0)
 224         NewFnName = "llvm.memmove.p0i8.p0i8.i16";
 225       else if (Name.compare(14,2,"32") == 0)
 226         NewFnName = "llvm.memmove.p0i8.p0i8.i32";
 227       else if (Name.compare(14,2,"64") == 0)
 228         NewFnName = "llvm.memmove.p0i8.p0i8.i64";
 229     }
 230     else if (Name.compare(5,8,"memset.i",8) == 0) {
 231       if (Name[13] == '8')
 232         NewFnName = "llvm.memset.p0i8.i8";
 233       else if (Name.compare(13,2,"16") == 0)
 234         NewFnName = "llvm.memset.p0i8.i16";
 235       else if (Name.compare(13,2,"32") == 0)
 236         NewFnName = "llvm.memset.p0i8.i32";
 237       else if (Name.compare(13,2,"64") == 0)
 238         NewFnName = "llvm.memset.p0i8.i64";
 239     }
 240     if (NewFnName) {
 241       NewFn = cast<Function>(M->getOrInsertFunction(NewFnName,
 242                                             FTy->getReturnType(),
 243                                             FTy->getParamType(0),
 244                                             FTy->getParamType(1),
 245                                             FTy->getParamType(2),
 246                                             FTy->getParamType(3),
 247                                             Type::getInt1Ty(F->getContext()),
 248                                             (Type *)0));
 249       return true;
 250     }
 251     break;
 252   }
 253   case 'p':
 254     //  This upgrades the llvm.part.select overloaded intrinsic names to only
 255     //  use one type specifier in the name. We only care about the old format
 256     //  'llvm.part.select.i*.i*', and solve as above with bswap.
 257     if (Name.compare(5,12,"part.select.",12) == 0) {
 258       std::string::size_type delim = Name.find('.',17);
 259
 260       if (delim != std::string::npos) {
 261         //  Construct a new name as 'llvm.part.select' + '.i*'
 262         F->setName(Name.substr(0,16)+Name.substr(delim));
 263         NewFn = F;
 264         return true;
 265       }
 266       break;
 267     }
 268
 269     //  This upgrades the llvm.part.set intrinsics similarly as above, however
 270     //  we care about 'llvm.part.set.i*.i*.i*', but only the first two types
 271     //  must match. There is an additional type specifier after these two
 272     //  matching types that we must retain when upgrading.  Thus, we require
 273     //  finding 2 periods, not just one, after the intrinsic name.
 274     if (Name.compare(5,9,"part.set.",9) == 0) {
 275       std::string::size_type delim = Name.find('.',14);
 276
 277       if (delim != std::string::npos &&
 278           Name.find('.',delim+1) != std::string::npos) {
 279         //  Construct a new name as 'llvm.part.select' + '.i*.i*'
 280         F->setName(Name.substr(0,13)+Name.substr(delim));
 281         NewFn = F;
 282         return true;
 283       }
 284       break;
 285     }
 286
 287     break;
 288   case 'x':
 289     // This fixes all MMX shift intrinsic instructions to take a
 290     // x86_mmx instead of a v1i64, v2i32, v4i16, or v8i8.
 291     if (Name.compare(5, 8, "x86.mmx.", 8) == 0) {
 292       const Type *X86_MMXTy = VectorType::getX86_MMXTy(FTy->getContext());
 293
 294       if (Name.compare(13, 4, "padd", 4) == 0   ||
 295           Name.compare(13, 4, "psub", 4) == 0   ||
 296           Name.compare(13, 4, "pmul", 4) == 0   ||
 297           Name.compare(13, 5, "pmadd", 5) == 0  ||
 298           Name.compare(13, 4, "pand", 4) == 0   ||
 299           Name.compare(13, 3, "por", 3) == 0    ||
 300           Name.compare(13, 4, "pxor", 4) == 0   ||
 301           Name.compare(13, 4, "pavg", 4) == 0   ||
 302           Name.compare(13, 4, "pmax", 4) == 0   ||
 303           Name.compare(13, 4, "pmin", 4) == 0   ||
 304           Name.compare(13, 4, "psad", 4) == 0   ||
 305           Name.compare(13, 4, "psll", 4) == 0   ||
 306           Name.compare(13, 4, "psrl", 4) == 0   ||
 307           Name.compare(13, 4, "psra", 4) == 0   ||
 308           Name.compare(13, 4, "pack", 4) == 0   ||
 309           Name.compare(13, 6, "punpck", 6) == 0 ||
 310           Name.compare(13, 4, "pcmp", 4) == 0) {
 311         assert(FTy->getNumParams() == 2 && "MMX intrinsic takes 2 args!");
 312         const Type *SecondParamTy = X86_MMXTy;
 313
 314         if (Name.compare(13, 5, "pslli", 5) == 0 ||
 315             Name.compare(13, 5, "psrli", 5) == 0 ||
 316             Name.compare(13, 5, "psrai", 5) == 0)
 317           SecondParamTy = FTy->getParamType(1);
 318
 319         // Don't do anything if it has the correct types.
 320         if (FTy->getReturnType() == X86_MMXTy &&
 321             FTy->getParamType(0) == X86_MMXTy &&
 322             FTy->getParamType(1) == SecondParamTy)
 323           break;
 324
 325         // We first need to change the name of the old (bad) intrinsic, because
 326         // its type is incorrect, but we cannot overload that name. We
 327         // arbitrarily unique it here allowing us to construct a correctly named
 328         // and typed function below.
 329         F->setName("");
 330
 331         // Now construct the new intrinsic with the correct name and type. We
 332         // leave the old function around in order to query its type, whatever it
 333         // may be, and correctly convert up to the new type.
 334         NewFn = cast<Function>(M->getOrInsertFunction(Name,
 335                                                       X86_MMXTy, X86_MMXTy,
 336                                                       SecondParamTy, (Type*)0));
 337         return true;
 338       }
 339
 340       if (Name.compare(13, 8, "maskmovq", 8) == 0) {
 341         // Don't do anything if it has the correct types.
 342         if (FTy->getParamType(0) == X86_MMXTy &&
 343             FTy->getParamType(1) == X86_MMXTy)
 344           break;
 345
 346         F->setName("");
 347         NewFn = cast<Function>(M->getOrInsertFunction(Name,
 348                                                       FTy->getReturnType(),
 349                                                       X86_MMXTy,
 350                                                       X86_MMXTy,
 351                                                       FTy->getParamType(2),
 352                                                       (Type*)0));
 353         return true;
 354       }
 355
 356       if (Name.compare(13, 8, "pmovmskb", 8) == 0) {
 357         if (FTy->getParamType(0) == X86_MMXTy)
 358           break;
 359
 360         F->setName("");
 361         NewFn = cast<Function>(M->getOrInsertFunction(Name,
 362                                                       FTy->getReturnType(),
 363                                                       X86_MMXTy,
 364                                                       (Type*)0));
 365         return true;
 366       }
 367
 368       if (Name.compare(13, 5, "movnt", 5) == 0) {
 369         if (FTy->getParamType(1) == X86_MMXTy)
 370           break;
 371
 372         F->setName("");
 373         NewFn = cast<Function>(M->getOrInsertFunction(Name,
 374                                                       FTy->getReturnType(),
 375                                                       FTy->getParamType(0),
 376                                                       X86_MMXTy,
 377                                                       (Type*)0));
 378         return true;
 379       }
 380
 381       if (Name.compare(13, 7, "palignr", 7) == 0) {
 382         if (FTy->getReturnType() == X86_MMXTy &&
 383             FTy->getParamType(0) == X86_MMXTy &&
 384             FTy->getParamType(1) == X86_MMXTy)
 385           break;
 386
 387         F->setName("");
 388         NewFn = cast<Function>(M->getOrInsertFunction(Name,
 389                                                       X86_MMXTy,
 390                                                       X86_MMXTy,
 391                                                       X86_MMXTy,
 392                                                       FTy->getParamType(2),
 393                                                       (Type*)0));
 394         return true;
 395       }
 396
 397       if (Name.compare(13, 5, "pextr", 5) == 0) {
 398         if (FTy->getParamType(0) == X86_MMXTy)
 399           break;
 400
 401         F->setName("");
 402         NewFn = cast<Function>(M->getOrInsertFunction(Name,
 403                                                       FTy->getReturnType(),
 404                                                       X86_MMXTy,
 405                                                       FTy->getParamType(1),
 406                                                       (Type*)0));
 407         return true;
 408       }
 409
 410       if (Name.compare(13, 5, "pinsr", 5) == 0) {
 411         if (FTy->getReturnType() == X86_MMXTy &&
 412             FTy->getParamType(0) == X86_MMXTy)
 413           break;
 414
 415         F->setName("");
 416         NewFn = cast<Function>(M->getOrInsertFunction(Name,
 417                                                       X86_MMXTy,
 418                                                       X86_MMXTy,
 419                                                       FTy->getParamType(1),
 420                                                       FTy->getParamType(2),
 421                                                       (Type*)0));
 422         return true;
 423       }
 424
 425       if (Name.compare(13, 12, "cvtsi32.si64", 12) == 0) {
 426         if (FTy->getReturnType() == X86_MMXTy)
 427           break;
 428
 429         F->setName("");
 430         NewFn = cast<Function>(M->getOrInsertFunction(Name,
 431                                                       X86_MMXTy,
 432                                                       FTy->getParamType(0),
 433                                                       (Type*)0));
 434         return true;
 435       }
 436
 437       if (Name.compare(13, 12, "cvtsi64.si32", 12) == 0) {
 438         if (FTy->getParamType(0) == X86_MMXTy)
 439           break;
 440
 441         F->setName("");
 442         NewFn = cast<Function>(M->getOrInsertFunction(Name,
 443                                                       FTy->getReturnType(),
 444                                                       X86_MMXTy,
 445                                                       (Type*)0));
 446         return true;
 447       }
 448
 449       if (Name.compare(13, 8, "vec.init", 8) == 0) {
 450         if (FTy->getReturnType() == X86_MMXTy)
 451           break;
 452
 453         F->setName("");
 454
 455         if (Name.compare(21, 2, ".b", 2) == 0)
 456           NewFn = cast<Function>(M->getOrInsertFunction(Name,
 457                                                         X86_MMXTy,
 458                                                         FTy->getParamType(0),
 459                                                         FTy->getParamType(1),
 460                                                         FTy->getParamType(2),
 461                                                         FTy->getParamType(3),
 462                                                         FTy->getParamType(4),
 463                                                         FTy->getParamType(5),
 464                                                         FTy->getParamType(6),
 465                                                         FTy->getParamType(7),
 466                                                         (Type*)0));
 467         else if (Name.compare(21, 2, ".w", 2) == 0)
 468           NewFn = cast<Function>(M->getOrInsertFunction(Name,
 469                                                         X86_MMXTy,
 470                                                         FTy->getParamType(0),
 471                                                         FTy->getParamType(1),
 472                                                         FTy->getParamType(2),
 473                                                         FTy->getParamType(3),
 474                                                         (Type*)0));
 475         else if (Name.compare(21, 2, ".d", 2) == 0)
 476           NewFn = cast<Function>(M->getOrInsertFunction(Name,
 477                                                         X86_MMXTy,
 478                                                         FTy->getParamType(0),
 479                                                         FTy->getParamType(1),
 480                                                         (Type*)0));
 481         return true;
 482       }
 483
 484
 485       if (Name.compare(13, 9, "vec.ext.d", 9) == 0) {
 486         if (FTy->getReturnType() == X86_MMXTy &&
 487             FTy->getParamType(0) == X86_MMXTy)
 488           break;
 489
 490         F->setName("");
 491         NewFn = cast<Function>(M->getOrInsertFunction(Name,
 492                                                       X86_MMXTy,
 493                                                       X86_MMXTy,
 494                                                       FTy->getParamType(1),
 495                                                       (Type*)0));
 496         return true;
 497       }
 498
 499       if (Name.compare(13, 9, "emms", 4) == 0 ||
 500           Name.compare(13, 9, "femms", 5) == 0) {
 501         NewFn = 0;
 502         break;
 503       }
 504
 505       // We really shouldn't get here ever.
 506       assert(0 && "Invalid MMX intrinsic!");
 507       break;
 508     } else if (Name.compare(5,17,"x86.sse2.loadh.pd",17) == 0 ||
 509                Name.compare(5,17,"x86.sse2.loadl.pd",17) == 0 ||
 510                Name.compare(5,16,"x86.sse2.movl.dq",16) == 0 ||
 511                Name.compare(5,15,"x86.sse2.movs.d",15) == 0 ||
 512                Name.compare(5,16,"x86.sse2.shuf.pd",16) == 0 ||
 513                Name.compare(5,18,"x86.sse2.unpckh.pd",18) == 0 ||
 514                Name.compare(5,18,"x86.sse2.unpckl.pd",18) == 0 ||
 515                Name.compare(5,20,"x86.sse2.punpckh.qdq",20) == 0 ||
 516                Name.compare(5,20,"x86.sse2.punpckl.qdq",20) == 0) {
 517       // Calls to these intrinsics are transformed into ShuffleVector's.
 518       NewFn = 0;
 519       return true;
 520     } else if (Name.compare(5, 16, "x86.sse41.pmulld", 16) == 0) {
 521       // Calls to these intrinsics are transformed into vector multiplies.
 522       NewFn = 0;
 523       return true;
 524     } else if (Name.compare(5, 18, "x86.ssse3.palign.r", 18) == 0 ||
 525                Name.compare(5, 22, "x86.ssse3.palign.r.128", 22) == 0) {
 526       // Calls to these intrinsics are transformed into vector shuffles, shifts,
 527       // or 0.
 528       NewFn = 0;
 529       return true;
 530     } else if (Name.compare(5, 16, "x86.sse.loadu.ps", 16) == 0 ||
 531                Name.compare(5, 17, "x86.sse2.loadu.dq", 17) == 0 ||
 532                Name.compare(5, 17, "x86.sse2.loadu.pd", 17) == 0) {
 533       // Calls to these instructions are transformed into unaligned loads.
 534       NewFn = 0;
 535       return true;
 536     } else if (Name.compare(5, 16, "x86.sse.movnt.ps", 16) == 0 ||
 537                Name.compare(5, 17, "x86.sse2.movnt.dq", 17) == 0 ||
 538                Name.compare(5, 17, "x86.sse2.movnt.pd", 17) == 0 ||
 539                Name.compare(5, 17, "x86.sse2.movnt.i", 16) == 0) {
 540       // Calls to these instructions are transformed into nontemporal stores.
 541       NewFn = 0;
 542       return true;
 543     } else if (Name.compare(5, 17, "x86.ssse3.pshuf.w", 17) == 0) {
 544       // This is an SSE/MMX instruction.
 545       const Type *X86_MMXTy = VectorType::getX86_MMXTy(FTy->getContext());
 546       NewFn =
 547         cast<Function>(M->getOrInsertFunction("llvm.x86.sse.pshuf.w",
 548                                               X86_MMXTy,
 549                                               X86_MMXTy,
 550                                               Type::getInt8Ty(F->getContext()),
 551                                               (Type*)0));
 552       return true;
 553     }
 554
 555     break;
 556   }
 557
 558   //  This may not belong here. This function is effectively being overloaded
 559   //  to both detect an intrinsic which needs upgrading, and to provide the
 560   //  upgraded form of the intrinsic. We should perhaps have two separate
 561   //  functions for this.
 562   return false;
 563 }
 564
 565 bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {
 566   NewFn = 0;
 567   bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn);
 568
 569   // Upgrade intrinsic attributes.  This does not change the function.
 570   if (NewFn)
 571     F = NewFn;
 572   if (unsigned id = F->getIntrinsicID())
 573     F->setAttributes(Intrinsic::getAttributes((Intrinsic::ID)id));
 574   return Upgraded;
 575 }
 576
 577 bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
 578   StringRef Name(GV->getName());
 579
 580   // We are only upgrading one symbol here.
 581   if (Name == ".llvm.eh.catch.all.value") {
 582     GV->setName("llvm.eh.catch.all.value");
 583     return true;
 584   }
 585
 586   return false;
 587 }
 588
 589 /// ExtendNEONArgs - For NEON "long" and "wide" operations, where the results
 590 /// have vector elements twice as big as one or both source operands, do the
 591 /// sign- or zero-extension that used to be handled by intrinsics.  The
 592 /// extended values are returned via V0 and V1.
 593 static void ExtendNEONArgs(CallInst *CI, Value *Arg0, Value *Arg1,
 594                            Value *&V0, Value *&V1) {
 595   Function *F = CI->getCalledFunction();
 596   const std::string& Name = F->getName();
 597   bool isLong = (Name.at(18) == 'l');
 598   bool isSigned = (Name.at(19) == 's');
 599
 600   if (isSigned) {
 601     if (isLong)
 602       V0 = new SExtInst(Arg0, CI->getType(), "", CI);
 603     else
 604       V0 = Arg0;
 605     V1 = new SExtInst(Arg1, CI->getType(), "", CI);
 606   } else {
 607     if (isLong)
 608       V0 = new ZExtInst(Arg0, CI->getType(), "", CI);
 609     else
 610       V0 = Arg0;
 611     V1 = new ZExtInst(Arg1, CI->getType(), "", CI);
 612   }
 613 }
 614
 615 /// CallVABD - As part of expanding a call to one of the old NEON vabdl, vaba,
 616 /// or vabal intrinsics, construct a call to a vabd intrinsic.  Examine the
 617 /// name of the old intrinsic to determine whether to use a signed or unsigned
 618 /// vabd intrinsic.  Get the type from the old call instruction, adjusted for
 619 /// half-size vector elements if the old intrinsic was vabdl or vabal.
 620 static Instruction *CallVABD(CallInst *CI, Value *Arg0, Value *Arg1) {
 621   Function *F = CI->getCalledFunction();
 622   const std::string& Name = F->getName();
 623   bool isLong = (Name.at(18) == 'l');
 624   bool isSigned = (Name.at(isLong ? 19 : 18) == 's');
 625
 626   Intrinsic::ID intID;
 627   if (isSigned)
 628     intID = Intrinsic::arm_neon_vabds;
 629   else
 630     intID = Intrinsic::arm_neon_vabdu;
 631
 632   const Type *Ty = CI->getType();
 633   if (isLong)
 634     Ty = VectorType::getTruncatedElementVectorType(cast<const VectorType>(Ty));
 635
 636   Function *VABD = Intrinsic::getDeclaration(F->getParent(), intID, &Ty, 1);
 637   Value *Operands[2];
 638   Operands[0] = Arg0;
 639   Operands[1] = Arg1;
 640   return CallInst::Create(VABD, Operands, Operands+2,
 641                           "upgraded."+CI->getName(), CI);
 642 }
 643
 644 /// ConstructNewCallInst - Construct a new CallInst with the signature of NewFn.
 645 static void ConstructNewCallInst(Function *NewFn, CallInst *OldCI,
 646                                  Value **Operands, unsigned NumOps,
 647                                  bool AssignName = true) {
 648   // Construct a new CallInst.
 649   CallInst *NewCI =
 650     CallInst::Create(NewFn, Operands, Operands + NumOps,
 651                      AssignName ? "upgraded." + OldCI->getName() : "", OldCI);
 652
 653   NewCI->setTailCall(OldCI->isTailCall());
 654   NewCI->setCallingConv(OldCI->getCallingConv());
 655
 656   // Handle any uses of the old CallInst. If the type has changed, add a cast.
 657   if (!OldCI->use_empty()) {
 658     if (OldCI->getType() != NewCI->getType()) {
 659       Function *OldFn = OldCI->getCalledFunction();
 660       CastInst *RetCast =
 661         CastInst::Create(CastInst::getCastOpcode(NewCI, true,
 662                                                  OldFn->getReturnType(), true),
 663                          NewCI, OldFn->getReturnType(), NewCI->getName(),OldCI);
 664
 665       // Replace all uses of the old call with the new cast which has the
 666       // correct type.
 667       OldCI->replaceAllUsesWith(RetCast);
 668     } else {
 669       OldCI->replaceAllUsesWith(NewCI);
 670     }
 671   }
 672
 673   // Clean up the old call now that it has been completely upgraded.
 674   OldCI->eraseFromParent();
 675 }
 676
 677 // UpgradeIntrinsicCall - Upgrade a call to an old intrinsic to be a call the
 678 // upgraded intrinsic. All argument and return casting must be provided in
 679 // order to seamlessly integrate with existing context.
 680 void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
 681   Function *F = CI->getCalledFunction();
 682   LLVMContext &C = CI->getContext();
 683   ImmutableCallSite CS(CI);
 684
 685   assert(F && "CallInst has no function associated with it.");
 686
 687   if (!NewFn) {
 688     // Get the Function's name.
 689     const std::string& Name = F->getName();
 690
 691     // Upgrade ARM NEON intrinsics.
 692     if (Name.compare(5, 9, "arm.neon.", 9) == 0) {
 693       Instruction *NewI;
 694       Value *V0, *V1;
 695       if (Name.compare(14, 7, "vmovls.", 7) == 0) {
 696         NewI = new SExtInst(CI->getArgOperand(0), CI->getType(),
 697                             "upgraded." + CI->getName(), CI);
 698       } else if (Name.compare(14, 7, "vmovlu.", 7) == 0) {
 699         NewI = new ZExtInst(CI->getArgOperand(0), CI->getType(),
 700                             "upgraded." + CI->getName(), CI);
 701       } else if (Name.compare(14, 4, "vadd", 4) == 0) {
 702         ExtendNEONArgs(CI, CI->getArgOperand(0), CI->getArgOperand(1), V0, V1);
 703         NewI = BinaryOperator::CreateAdd(V0, V1, "upgraded."+CI->getName(), CI);
 704       } else if (Name.compare(14, 4, "vsub", 4) == 0) {
 705         ExtendNEONArgs(CI, CI->getArgOperand(0), CI->getArgOperand(1), V0, V1);
 706         NewI = BinaryOperator::CreateSub(V0, V1,"upgraded."+CI->getName(),CI);
 707       } else if (Name.compare(14, 4, "vmul", 4) == 0) {
 708         ExtendNEONArgs(CI, CI->getArgOperand(0), CI->getArgOperand(1), V0, V1);
 709         NewI = BinaryOperator::CreateMul(V0, V1,"upgraded."+CI->getName(),CI);
 710       } else if (Name.compare(14, 4, "vmla", 4) == 0) {
 711         ExtendNEONArgs(CI, CI->getArgOperand(1), CI->getArgOperand(2), V0, V1);
 712         Instruction *MulI = BinaryOperator::CreateMul(V0, V1, "", CI);
 713         NewI = BinaryOperator::CreateAdd(CI->getArgOperand(0), MulI,
 714                                          "upgraded."+CI->getName(), CI);
 715       } else if (Name.compare(14, 4, "vmls", 4) == 0) {
 716         ExtendNEONArgs(CI, CI->getArgOperand(1), CI->getArgOperand(2), V0, V1);
 717         Instruction *MulI = BinaryOperator::CreateMul(V0, V1, "", CI);
 718         NewI = BinaryOperator::CreateSub(CI->getArgOperand(0), MulI,
 719                                          "upgraded."+CI->getName(), CI);
 720       } else if (Name.compare(14, 4, "vabd", 4) == 0) {
 721         NewI = CallVABD(CI, CI->getArgOperand(0), CI->getArgOperand(1));
 722         NewI = new ZExtInst(NewI, CI->getType(), "upgraded."+CI->getName(), CI);
 723       } else if (Name.compare(14, 4, "vaba", 4) == 0) {
 724         NewI = CallVABD(CI, CI->getArgOperand(1), CI->getArgOperand(2));
 725         if (Name.at(18) == 'l')
 726           NewI = new ZExtInst(NewI, CI->getType(), "", CI);
 727         NewI = BinaryOperator::CreateAdd(CI->getArgOperand(0), NewI,
 728                                          "upgraded."+CI->getName(), CI);
 729       } else if (Name.compare(14, 6, "vmovn.", 6) == 0) {
 730         NewI = new TruncInst(CI->getArgOperand(0), CI->getType(),
 731                              "upgraded." + CI->getName(), CI);
 732       } else {
 733         llvm_unreachable("Unknown arm.neon function for CallInst upgrade.");
 734       }
 735       // Replace any uses of the old CallInst.
 736       if (!CI->use_empty())
 737         CI->replaceAllUsesWith(NewI);
 738       CI->eraseFromParent();
 739       return;
 740     }
 741
 742     bool isLoadH = false, isLoadL = false, isMovL = false;
 743     bool isMovSD = false, isShufPD = false;
 744     bool isUnpckhPD = false, isUnpcklPD = false;
 745     bool isPunpckhQPD = false, isPunpcklQPD = false;
 746     if (F->getName() == "llvm.x86.sse2.loadh.pd")
 747       isLoadH = true;
 748     else if (F->getName() == "llvm.x86.sse2.loadl.pd")
 749       isLoadL = true;
 750     else if (F->getName() == "llvm.x86.sse2.movl.dq")
 751       isMovL = true;
 752     else if (F->getName() == "llvm.x86.sse2.movs.d")
 753       isMovSD = true;
 754     else if (F->getName() == "llvm.x86.sse2.shuf.pd")
 755       isShufPD = true;
 756     else if (F->getName() == "llvm.x86.sse2.unpckh.pd")
 757       isUnpckhPD = true;
 758     else if (F->getName() == "llvm.x86.sse2.unpckl.pd")
 759       isUnpcklPD = true;
 760     else if (F->getName() ==  "llvm.x86.sse2.punpckh.qdq")
 761       isPunpckhQPD = true;
 762     else if (F->getName() ==  "llvm.x86.sse2.punpckl.qdq")
 763       isPunpcklQPD = true;
 764
 765     if (isLoadH || isLoadL || isMovL || isMovSD || isShufPD ||
 766         isUnpckhPD || isUnpcklPD || isPunpckhQPD || isPunpcklQPD) {
 767       std::vector<Constant*> Idxs;
 768       Value *Op0 = CI->getArgOperand(0);
 769       ShuffleVectorInst *SI = NULL;
 770       if (isLoadH || isLoadL) {
 771         Value *Op1 = UndefValue::get(Op0->getType());
 772         Value *Addr = new BitCastInst(CI->getArgOperand(1),
 773                                   Type::getDoublePtrTy(C),
 774                                       "upgraded.", CI);
 775         Value *Load = new LoadInst(Addr, "upgraded.", false, 8, CI);
 776         Value *Idx = ConstantInt::get(Type::getInt32Ty(C), 0);
 777         Op1 = InsertElementInst::Create(Op1, Load, Idx, "upgraded.", CI);
 778
 779         if (isLoadH) {
 780           Idxs.push_back(ConstantInt::get(Type::getInt32Ty(C), 0));
 781           Idxs.push_back(ConstantInt::get(Type::getInt32Ty(C), 2));
 782         } else {
 783           Idxs.push_back(ConstantInt::get(Type::getInt32Ty(C), 2));
 784           Idxs.push_back(ConstantInt::get(Type::getInt32Ty(C), 1));
 785         }
 786         Value *Mask = ConstantVector::get(Idxs);
 787         SI = new ShuffleVectorInst(Op0, Op1, Mask, "upgraded.", CI);
 788       } else if (isMovL) {
 789         Constant *Zero = ConstantInt::get(Type::getInt32Ty(C), 0);
 790         Idxs.push_back(Zero);
 791         Idxs.push_back(Zero);
 792         Idxs.push_back(Zero);
 793         Idxs.push_back(Zero);
 794         Value *ZeroV = ConstantVector::get(Idxs);
 795
 796         Idxs.clear();
 797         Idxs.push_back(ConstantInt::get(Type::getInt32Ty(C), 4));
 798         Idxs.push_back(ConstantInt::get(Type::getInt32Ty(C), 5));
 799         Idxs.push_back(ConstantInt::get(Type::getInt32Ty(C), 2));
 800         Idxs.push_back(ConstantInt::get(Type::getInt32Ty(C), 3));
 801         Value *Mask = ConstantVector::get(Idxs);
 802         SI = new ShuffleVectorInst(ZeroV, Op0, Mask, "upgraded.", CI);
 803       } else if (isMovSD ||
 804                  isUnpckhPD || isUnpcklPD || isPunpckhQPD || isPunpcklQPD) {
 805         Value *Op1 = CI->getArgOperand(1);
 806         if (isMovSD) {
 807           Idxs.push_back(ConstantInt::get(Type::getInt32Ty(C), 2));
 808           Idxs.push_back(ConstantInt::get(Type::getInt32Ty(C), 1));
 809         } else if (isUnpckhPD || isPunpckhQPD) {
 810           Idxs.push_back(ConstantInt::get(Type::getInt32Ty(C), 1));
 811           Idxs.push_back(ConstantInt::get(Type::getInt32Ty(C), 3));
 812         } else {
 813           Idxs.push_back(ConstantInt::get(Type::getInt32Ty(C), 0));
 814           Idxs.push_back(ConstantInt::get(Type::getInt32Ty(C), 2));
 815         }
 816         Value *Mask = ConstantVector::get(Idxs);
 817         SI = new ShuffleVectorInst(Op0, Op1, Mask, "upgraded.", CI);
 818       } else if (isShufPD) {
 819         Value *Op1 = CI->getArgOperand(1);
 820         unsigned MaskVal =
 821                         cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
 822         Idxs.push_back(ConstantInt::get(Type::getInt32Ty(C), MaskVal & 1));
 823         Idxs.push_back(ConstantInt::get(Type::getInt32Ty(C),
 824                                                ((MaskVal >> 1) & 1)+2));
 825         Value *Mask = ConstantVector::get(Idxs);
 826         SI = new ShuffleVectorInst(Op0, Op1, Mask, "upgraded.", CI);
 827       }
 828
 829       assert(SI && "Unexpected!");
 830
 831       // Handle any uses of the old CallInst.
 832       if (!CI->use_empty())
 833         //  Replace all uses of the old call with the new cast which has the
 834         //  correct type.
 835         CI->replaceAllUsesWith(SI);
 836
 837       //  Clean up the old call now that it has been completely upgraded.
 838       CI->eraseFromParent();
 839     } else if (F->getName() == "llvm.x86.sse41.pmulld") {
 840       // Upgrade this set of intrinsics into vector multiplies.
 841       Instruction *Mul = BinaryOperator::CreateMul(CI->getArgOperand(0),
 842                                                    CI->getArgOperand(1),
 843                                                    CI->getName(),
 844                                                    CI);
 845       // Fix up all the uses with our new multiply.
 846       if (!CI->use_empty())
 847         CI->replaceAllUsesWith(Mul);
 848
 849       // Remove upgraded multiply.
 850       CI->eraseFromParent();
 851     } else if (F->getName() == "llvm.x86.ssse3.palign.r") {
 852       Value *Op1 = CI->getArgOperand(0);
 853       Value *Op2 = CI->getArgOperand(1);
 854       Value *Op3 = CI->getArgOperand(2);
 855       unsigned shiftVal = cast<ConstantInt>(Op3)->getZExtValue();
 856       Value *Rep;
 857       IRBuilder<> Builder(C);
 858       Builder.SetInsertPoint(CI->getParent(), CI);
 859
 860       // If palignr is shifting the pair of input vectors less than 9 bytes,
 861       // emit a shuffle instruction.
 862       if (shiftVal <= 8) {
 863         const Type *IntTy = Type::getInt32Ty(C);
 864         const Type *EltTy = Type::getInt8Ty(C);
 865         const Type *VecTy = VectorType::get(EltTy, 8);
 866
 867         Op2 = Builder.CreateBitCast(Op2, VecTy);
 868         Op1 = Builder.CreateBitCast(Op1, VecTy);
 869
 870         llvm::SmallVector<llvm::Constant*, 8> Indices;
 871         for (unsigned i = 0; i != 8; ++i)
 872           Indices.push_back(ConstantInt::get(IntTy, shiftVal + i));
 873
 874         Value *SV = ConstantVector::get(Indices);
 875         Rep = Builder.CreateShuffleVector(Op2, Op1, SV, "palignr");
 876         Rep = Builder.CreateBitCast(Rep, F->getReturnType());
 877       }
 878
 879       // If palignr is shifting the pair of input vectors more than 8 but less
 880       // than 16 bytes, emit a logical right shift of the destination.
 881       else if (shiftVal < 16) {
 882         // MMX has these as 1 x i64 vectors for some odd optimization reasons.
 883         const Type *EltTy = Type::getInt64Ty(C);
 884         const Type *VecTy = VectorType::get(EltTy, 1);
 885
 886         Op1 = Builder.CreateBitCast(Op1, VecTy, "cast");
 887         Op2 = ConstantInt::get(VecTy, (shiftVal-8) * 8);
 888
 889         // create i32 constant
 890         Function *I =
 891           Intrinsic::getDeclaration(F->getParent(), Intrinsic::x86_mmx_psrl_q);
 892         Rep = Builder.CreateCall2(I, Op1, Op2, "palignr");
 893       }
 894
 895       // If palignr is shifting the pair of vectors more than 32 bytes, emit zero.
 896       else {
 897         Rep = Constant::getNullValue(F->getReturnType());
 898       }
 899
 900       // Replace any uses with our new instruction.
 901       if (!CI->use_empty())
 902         CI->replaceAllUsesWith(Rep);
 903
 904       // Remove upgraded instruction.
 905       CI->eraseFromParent();
 906
 907     } else if (F->getName() == "llvm.x86.ssse3.palign.r.128") {
 908       Value *Op1 = CI->getArgOperand(0);
 909       Value *Op2 = CI->getArgOperand(1);
 910       Value *Op3 = CI->getArgOperand(2);
 911       unsigned shiftVal = cast<ConstantInt>(Op3)->getZExtValue();
 912       Value *Rep;
 913       IRBuilder<> Builder(C);
 914       Builder.SetInsertPoint(CI->getParent(), CI);
 915
 916       // If palignr is shifting the pair of input vectors less than 17 bytes,
 917       // emit a shuffle instruction.
 918       if (shiftVal <= 16) {
 919         const Type *IntTy = Type::getInt32Ty(C);
 920         const Type *EltTy = Type::getInt8Ty(C);
 921         const Type *VecTy = VectorType::get(EltTy, 16);
 922
 923         Op2 = Builder.CreateBitCast(Op2, VecTy);
 924         Op1 = Builder.CreateBitCast(Op1, VecTy);
 925
 926         llvm::SmallVector<llvm::Constant*, 16> Indices;
 927         for (unsigned i = 0; i != 16; ++i)
 928           Indices.push_back(ConstantInt::get(IntTy, shiftVal + i));
 929
 930         Value *SV = ConstantVector::get(Indices);
 931         Rep = Builder.CreateShuffleVector(Op2, Op1, SV, "palignr");
 932         Rep = Builder.CreateBitCast(Rep, F->getReturnType());
 933       }
 934
 935       // If palignr is shifting the pair of input vectors more than 16 but less
 936       // than 32 bytes, emit a logical right shift of the destination.
 937       else if (shiftVal < 32) {
 938         const Type *EltTy = Type::getInt64Ty(C);
 939         const Type *VecTy = VectorType::get(EltTy, 2);
 940         const Type *IntTy = Type::getInt32Ty(C);
 941
 942         Op1 = Builder.CreateBitCast(Op1, VecTy, "cast");
 943         Op2 = ConstantInt::get(IntTy, (shiftVal-16) * 8);
 944
 945         // create i32 constant
 946         Function *I =
 947           Intrinsic::getDeclaration(F->getParent(), Intrinsic::x86_sse2_psrl_dq);
 948         Rep = Builder.CreateCall2(I, Op1, Op2, "palignr");
 949       }
 950
 951       // If palignr is shifting the pair of vectors more than 32 bytes, emit zero.
 952       else {
 953         Rep = Constant::getNullValue(F->getReturnType());
 954       }
 955
 956       // Replace any uses with our new instruction.
 957       if (!CI->use_empty())
 958         CI->replaceAllUsesWith(Rep);
 959
 960       // Remove upgraded instruction.
 961       CI->eraseFromParent();
 962
 963     } else if (F->getName() == "llvm.x86.sse.loadu.ps" ||
 964                F->getName() == "llvm.x86.sse2.loadu.dq" ||
 965                F->getName() == "llvm.x86.sse2.loadu.pd") {
 966       // Convert to a native, unaligned load.
 967       const Type *VecTy = CI->getType();
 968       const Type *IntTy = IntegerType::get(C, 128);
 969       IRBuilder<> Builder(C);
 970       Builder.SetInsertPoint(CI->getParent(), CI);
 971
 972       Value *BC = Builder.CreateBitCast(CI->getArgOperand(0),
 973                                         PointerType::getUnqual(IntTy),
 974                                         "cast");
 975       LoadInst *LI = Builder.CreateLoad(BC, CI->getName());
 976       LI->setAlignment(1);      // Unaligned load.
 977       BC = Builder.CreateBitCast(LI, VecTy, "new.cast");
 978
 979       // Fix up all the uses with our new load.
 980       if (!CI->use_empty())
 981         CI->replaceAllUsesWith(BC);
 982
 983       // Remove intrinsic.
 984       CI->eraseFromParent();
 985     } else if (F->getName() == "llvm.x86.sse.movnt.ps" ||
 986                F->getName() == "llvm.x86.sse2.movnt.dq" ||
 987                F->getName() == "llvm.x86.sse2.movnt.pd" ||
 988                F->getName() == "llvm.x86.sse2.movnt.i") {
 989       IRBuilder<> Builder(C);
 990       Builder.SetInsertPoint(CI->getParent(), CI);
 991
 992       Module *M = F->getParent();
 993       SmallVector<Value *, 1> Elts;
 994       Elts.push_back(ConstantInt::get(Type::getInt32Ty(C), 1));
 995       MDNode *Node = MDNode::get(C, Elts);
 996
 997       Value *Arg0 = CI->getArgOperand(0);
 998       Value *Arg1 = CI->getArgOperand(1);
 999
1000       // Convert the type of the pointer to a pointer to the stored type.
1001       Value *BC = Builder.CreateBitCast(Arg0,
1002                                         PointerType::getUnqual(Arg1->getType()),
1003                                         "cast");
1004       StoreInst *SI = Builder.CreateStore(Arg1, BC);
1005       SI->setMetadata(M->getMDKindID("nontemporal"), Node);
1006       SI->setAlignment(16);
1007
1008       // Remove intrinsic.
1009       CI->eraseFromParent();
1010     } else {
1011       llvm_unreachable("Unknown function for CallInst upgrade.");
1012     }
1013     return;
1014   }
1015
1016   switch (NewFn->getIntrinsicID()) {
1017   default: llvm_unreachable("Unknown function for CallInst upgrade.");
1018   case Intrinsic::arm_neon_vld1:
1019   case Intrinsic::arm_neon_vld2:
1020   case Intrinsic::arm_neon_vld3:
1021   case Intrinsic::arm_neon_vld4:
1022   case Intrinsic::arm_neon_vst1:
1023   case Intrinsic::arm_neon_vst2:
1024   case Intrinsic::arm_neon_vst3:
1025   case Intrinsic::arm_neon_vst4:
1026   case Intrinsic::arm_neon_vld2lane:
1027   case Intrinsic::arm_neon_vld3lane:
1028   case Intrinsic::arm_neon_vld4lane:
1029   case Intrinsic::arm_neon_vst2lane:
1030   case Intrinsic::arm_neon_vst3lane:
1031   case Intrinsic::arm_neon_vst4lane: {
1032     // Add a default alignment argument of 1.
1033     SmallVector<Value*, 8> Operands(CS.arg_begin(), CS.arg_end());
1034     Operands.push_back(ConstantInt::get(Type::getInt32Ty(C), 1));
1035     CallInst *NewCI = CallInst::Create(NewFn, Operands.begin(), Operands.end(),
1036                                        CI->getName(), CI);
1037     NewCI->setTailCall(CI->isTailCall());
1038     NewCI->setCallingConv(CI->getCallingConv());
1039
1040     //  Handle any uses of the old CallInst.
1041     if (!CI->use_empty())
1042       //  Replace all uses of the old call with the new cast which has the
1043       //  correct type.
1044       CI->replaceAllUsesWith(NewCI);
1045
1046     //  Clean up the old call now that it has been completely upgraded.
1047     CI->eraseFromParent();
1048     break;
1049   }
1050
1051   case Intrinsic::x86_mmx_padd_b:
1052   case Intrinsic::x86_mmx_padd_w:
1053   case Intrinsic::x86_mmx_padd_d:
1054   case Intrinsic::x86_mmx_padd_q:
1055   case Intrinsic::x86_mmx_padds_b:
1056   case Intrinsic::x86_mmx_padds_w:
1057   case Intrinsic::x86_mmx_paddus_b:
1058   case Intrinsic::x86_mmx_paddus_w:
1059   case Intrinsic::x86_mmx_psub_b:
1060   case Intrinsic::x86_mmx_psub_w:
1061   case Intrinsic::x86_mmx_psub_d:
1062   case Intrinsic::x86_mmx_psub_q:
1063   case Intrinsic::x86_mmx_psubs_b:
1064   case Intrinsic::x86_mmx_psubs_w:
1065   case Intrinsic::x86_mmx_psubus_b:
1066   case Intrinsic::x86_mmx_psubus_w:
1067   case Intrinsic::x86_mmx_pmulh_w:
1068   case Intrinsic::x86_mmx_pmull_w:
1069   case Intrinsic::x86_mmx_pmulhu_w:
1070   case Intrinsic::x86_mmx_pmulu_dq:
1071   case Intrinsic::x86_mmx_pmadd_wd:
1072   case Intrinsic::x86_mmx_pand:
1073   case Intrinsic::x86_mmx_pandn:
1074   case Intrinsic::x86_mmx_por:
1075   case Intrinsic::x86_mmx_pxor:
1076   case Intrinsic::x86_mmx_pavg_b:
1077   case Intrinsic::x86_mmx_pavg_w:
1078   case Intrinsic::x86_mmx_pmaxu_b:
1079   case Intrinsic::x86_mmx_pmaxs_w:
1080   case Intrinsic::x86_mmx_pminu_b:
1081   case Intrinsic::x86_mmx_pmins_w:
1082   case Intrinsic::x86_mmx_psad_bw:
1083   case Intrinsic::x86_mmx_psll_w:
1084   case Intrinsic::x86_mmx_psll_d:
1085   case Intrinsic::x86_mmx_psll_q:
1086   case Intrinsic::x86_mmx_pslli_w:
1087   case Intrinsic::x86_mmx_pslli_d:
1088   case Intrinsic::x86_mmx_pslli_q:
1089   case Intrinsic::x86_mmx_psrl_w:
1090   case Intrinsic::x86_mmx_psrl_d:
1091   case Intrinsic::x86_mmx_psrl_q:
1092   case Intrinsic::x86_mmx_psrli_w:
1093   case Intrinsic::x86_mmx_psrli_d:
1094   case Intrinsic::x86_mmx_psrli_q:
1095   case Intrinsic::x86_mmx_psra_w:
1096   case Intrinsic::x86_mmx_psra_d:
1097   case Intrinsic::x86_mmx_psrai_w:
1098   case Intrinsic::x86_mmx_psrai_d:
1099   case Intrinsic::x86_mmx_packsswb:
1100   case Intrinsic::x86_mmx_packssdw:
1101   case Intrinsic::x86_mmx_packuswb:
1102   case Intrinsic::x86_mmx_punpckhbw:
1103   case Intrinsic::x86_mmx_punpckhwd:
1104   case Intrinsic::x86_mmx_punpckhdq:
1105   case Intrinsic::x86_mmx_punpcklbw:
1106   case Intrinsic::x86_mmx_punpcklwd:
1107   case Intrinsic::x86_mmx_punpckldq:
1108   case Intrinsic::x86_mmx_pcmpeq_b:
1109   case Intrinsic::x86_mmx_pcmpeq_w:
1110   case Intrinsic::x86_mmx_pcmpeq_d:
1111   case Intrinsic::x86_mmx_pcmpgt_b:
1112   case Intrinsic::x86_mmx_pcmpgt_w:
1113   case Intrinsic::x86_mmx_pcmpgt_d: {
1114     Value *Operands[2];
1115
1116     // Cast the operand to the X86 MMX type.
1117     Operands[0] = new BitCastInst(CI->getArgOperand(0),
1118                                   NewFn->getFunctionType()->getParamType(0),
1119                                   "upgraded.", CI);
1120
1121     switch (NewFn->getIntrinsicID()) {
1122     default:
1123       // Cast to the X86 MMX type.
1124       Operands[1] = new BitCastInst(CI->getArgOperand(1),
1125                                     NewFn->getFunctionType()->getParamType(1),
1126                                     "upgraded.", CI);
1127       break;
1128     case Intrinsic::x86_mmx_pslli_w:
1129     case Intrinsic::x86_mmx_pslli_d:
1130     case Intrinsic::x86_mmx_pslli_q:
1131     case Intrinsic::x86_mmx_psrli_w:
1132     case Intrinsic::x86_mmx_psrli_d:
1133     case Intrinsic::x86_mmx_psrli_q:
1134     case Intrinsic::x86_mmx_psrai_w:
1135     case Intrinsic::x86_mmx_psrai_d:
1136       // These take an i32 as their second parameter.
1137       Operands[1] = CI->getArgOperand(1);
1138       break;
1139     }
1140
1141     ConstructNewCallInst(NewFn, CI, Operands, 2);
1142     break;
1143   }
1144   case Intrinsic::x86_mmx_maskmovq: {
1145     Value *Operands[3];
1146
1147     // Cast the operands to the X86 MMX type.
1148     Operands[0] = new BitCastInst(CI->getArgOperand(0),
1149                                   NewFn->getFunctionType()->getParamType(0),
1150                                   "upgraded.", CI);
1151     Operands[1] = new BitCastInst(CI->getArgOperand(1),
1152                                   NewFn->getFunctionType()->getParamType(1),
1153                                   "upgraded.", CI);
1154     Operands[2] = CI->getArgOperand(2);
1155
1156     ConstructNewCallInst(NewFn, CI, Operands, 3, false);
1157     break;
1158   }
1159   case Intrinsic::x86_mmx_pmovmskb: {
1160     Value *Operands[1];
1161
1162     // Cast the operand to the X86 MMX type.
1163     Operands[0] = new BitCastInst(CI->getArgOperand(0),
1164                                   NewFn->getFunctionType()->getParamType(0),
1165                                   "upgraded.", CI);
1166
1167     ConstructNewCallInst(NewFn, CI, Operands, 1);
1168     break;
1169   }
1170   case Intrinsic::x86_mmx_movnt_dq: {
1171     Value *Operands[2];
1172
1173     Operands[0] = CI->getArgOperand(0);
1174
1175     // Cast the operand to the X86 MMX type.
1176     Operands[1] = new BitCastInst(CI->getArgOperand(1),
1177                                   NewFn->getFunctionType()->getParamType(1),
1178                                   "upgraded.", CI);
1179
1180     ConstructNewCallInst(NewFn, CI, Operands, 2, false);
1181     break;
1182   }
1183   case Intrinsic::x86_mmx_palignr_b: {
1184     Value *Operands[3];
1185
1186     // Cast the operands to the X86 MMX type.
1187     Operands[0] = new BitCastInst(CI->getArgOperand(0),
1188                                   NewFn->getFunctionType()->getParamType(0),
1189                                   "upgraded.", CI);
1190     Operands[1] = new BitCastInst(CI->getArgOperand(1),
1191                                   NewFn->getFunctionType()->getParamType(1),
1192                                   "upgraded.", CI);
1193     Operands[2] = CI->getArgOperand(2);
1194
1195     ConstructNewCallInst(NewFn, CI, Operands, 3);
1196     break;
1197   }
1198   case Intrinsic::x86_mmx_pextr_w: {
1199     Value *Operands[2];
1200
1201     // Cast the operands to the X86 MMX type.
1202     Operands[0] = new BitCastInst(CI->getArgOperand(0),
1203                                   NewFn->getFunctionType()->getParamType(0),
1204                                   "upgraded.", CI);
1205     Operands[1] = CI->getArgOperand(1);
1206
1207     ConstructNewCallInst(NewFn, CI, Operands, 2);
1208     break;
1209   }
1210   case Intrinsic::x86_mmx_pinsr_w: {
1211     Value *Operands[3];
1212
1213     // Cast the operands to the X86 MMX type.
1214     Operands[0] = new BitCastInst(CI->getArgOperand(0),
1215                                   NewFn->getFunctionType()->getParamType(0),
1216                                   "upgraded.", CI);
1217     Operands[1] = CI->getArgOperand(1);
1218     Operands[2] = CI->getArgOperand(2);
1219
1220     ConstructNewCallInst(NewFn, CI, Operands, 3);
1221     break;
1222   }
1223   case Intrinsic::x86_sse_pshuf_w: {
1224     IRBuilder<> Builder(C);
1225     Builder.SetInsertPoint(CI->getParent(), CI);
1226
1227     // Cast the operand to the X86 MMX type.
1228     Value *Operands[2];
1229     Operands[0] =
1230       Builder.CreateBitCast(CI->getArgOperand(0),
1231                             NewFn->getFunctionType()->getParamType(0),
1232                             "upgraded.");
1233     Operands[1] =
1234       Builder.CreateTrunc(CI->getArgOperand(1),
1235                           Type::getInt8Ty(C),
1236                           "upgraded.");
1237
1238     ConstructNewCallInst(NewFn, CI, Operands, 2);
1239     break;
1240   }
1241
1242   case Intrinsic::ctlz:
1243   case Intrinsic::ctpop:
1244   case Intrinsic::cttz: {
1245     //  Build a small vector of the original arguments.
1246     SmallVector<Value*, 8> Operands(CS.arg_begin(), CS.arg_end());
1247
1248     //  Construct a new CallInst
1249     CallInst *NewCI = CallInst::Create(NewFn, Operands.begin(), Operands.end(),
1250                                        "upgraded."+CI->getName(), CI);
1251     NewCI->setTailCall(CI->isTailCall());
1252     NewCI->setCallingConv(CI->getCallingConv());
1253
1254     //  Handle any uses of the old CallInst.
1255     if (!CI->use_empty()) {
1256       //  Check for sign extend parameter attributes on the return values.
1257       bool SrcSExt = NewFn->getAttributes().paramHasAttr(0, Attribute::SExt);
1258       bool DestSExt = F->getAttributes().paramHasAttr(0, Attribute::SExt);
1259
1260       //  Construct an appropriate cast from the new return type to the old.
1261       CastInst *RetCast = CastInst::Create(
1262                             CastInst::getCastOpcode(NewCI, SrcSExt,
1263                                                     F->getReturnType(),
1264                                                     DestSExt),
1265                             NewCI, F->getReturnType(),
1266                             NewCI->getName(), CI);
1267       NewCI->moveBefore(RetCast);
1268
1269       //  Replace all uses of the old call with the new cast which has the
1270       //  correct type.
1271       CI->replaceAllUsesWith(RetCast);
1272     }
1273
1274     //  Clean up the old call now that it has been completely upgraded.
1275     CI->eraseFromParent();
1276   }
1277   break;
1278   case Intrinsic::eh_selector:
1279   case Intrinsic::eh_typeid_for: {
1280     // Only the return type changed.
1281     SmallVector<Value*, 8> Operands(CS.arg_begin(), CS.arg_end());
1282     CallInst *NewCI = CallInst::Create(NewFn, Operands.begin(), Operands.end(),
1283                                        "upgraded." + CI->getName(), CI);
1284     NewCI->setTailCall(CI->isTailCall());
1285     NewCI->setCallingConv(CI->getCallingConv());
1286
1287     //  Handle any uses of the old CallInst.
1288     if (!CI->use_empty()) {
1289       //  Construct an appropriate cast from the new return type to the old.
1290       CastInst *RetCast =
1291         CastInst::Create(CastInst::getCastOpcode(NewCI, true,
1292                                                  F->getReturnType(), true),
1293                          NewCI, F->getReturnType(), NewCI->getName(), CI);
1294       CI->replaceAllUsesWith(RetCast);
1295     }
1296     CI->eraseFromParent();
1297   }
1298   break;
1299   case Intrinsic::memcpy:
1300   case Intrinsic::memmove:
1301   case Intrinsic::memset: {
1302     // Add isVolatile
1303     const llvm::Type *I1Ty = llvm::Type::getInt1Ty(CI->getContext());
1304     Value *Operands[5] = { CI->getArgOperand(0), CI->getArgOperand(1),
1305                            CI->getArgOperand(2), CI->getArgOperand(3),
1306                            llvm::ConstantInt::get(I1Ty, 0) };
1307     CallInst *NewCI = CallInst::Create(NewFn, Operands, Operands+5,
1308                                        CI->getName(), CI);
1309     NewCI->setTailCall(CI->isTailCall());
1310     NewCI->setCallingConv(CI->getCallingConv());
1311     //  Handle any uses of the old CallInst.
1312     if (!CI->use_empty())
1313       //  Replace all uses of the old call with the new cast which has the
1314       //  correct type.
1315       CI->replaceAllUsesWith(NewCI);
1316
1317     //  Clean up the old call now that it has been completely upgraded.
1318     CI->eraseFromParent();
1319     break;
1320   }
1321   }
1322 }
1323
1324 // This tests each Function to determine if it needs upgrading. When we find
1325 // one we are interested in, we then upgrade all calls to reflect the new
1326 // function.
1327 void llvm::UpgradeCallsToIntrinsic(Function* F) {
1328   assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
1329
1330   // Upgrade the function and check if it is a totaly new function.
1331   Function* NewFn;
1332   if (UpgradeIntrinsicFunction(F, NewFn)) {
1333     if (NewFn != F) {
1334       // Replace all uses to the old function with the new one if necessary.
1335       for (Value::use_iterator UI = F->use_begin(), UE = F->use_end();
1336            UI != UE; ) {
1337         if (CallInst* CI = dyn_cast<CallInst>(*UI++))
1338           UpgradeIntrinsicCall(CI, NewFn);
1339       }
1340       // Remove old function, no longer used, from the module.
1341       F->eraseFromParent();
1342     }
1343   }
1344 }
1345
1346 /// This function strips all debug info intrinsics, except for llvm.dbg.declare.
1347 /// If an llvm.dbg.declare intrinsic is invalid, then this function simply
1348 /// strips that use.
1349 void llvm::CheckDebugInfoIntrinsics(Module *M) {
1350
1351
1352   if (Function *FuncStart = M->getFunction("llvm.dbg.func.start")) {
1353     while (!FuncStart->use_empty()) {
1354       CallInst *CI = cast<CallInst>(FuncStart->use_back());
1355       CI->eraseFromParent();
1356     }
1357     FuncStart->eraseFromParent();
1358   }
1359
1360   if (Function *StopPoint = M->getFunction("llvm.dbg.stoppoint")) {
1361     while (!StopPoint->use_empty()) {
1362       CallInst *CI = cast<CallInst>(StopPoint->use_back());
1363       CI->eraseFromParent();
1364     }
1365     StopPoint->eraseFromParent();
1366   }
1367
1368   if (Function *RegionStart = M->getFunction("llvm.dbg.region.start")) {
1369     while (!RegionStart->use_empty()) {
1370       CallInst *CI = cast<CallInst>(RegionStart->use_back());
1371       CI->eraseFromParent();
1372     }
1373     RegionStart->eraseFromParent();
1374   }
1375
1376   if (Function *RegionEnd = M->getFunction("llvm.dbg.region.end")) {
1377     while (!RegionEnd->use_empty()) {
1378       CallInst *CI = cast<CallInst>(RegionEnd->use_back());
1379       CI->eraseFromParent();
1380     }
1381     RegionEnd->eraseFromParent();
1382   }
1383
1384   if (Function *Declare = M->getFunction("llvm.dbg.declare")) {
1385     if (!Declare->use_empty()) {
1386       DbgDeclareInst *DDI = cast<DbgDeclareInst>(Declare->use_back());
1387       if (!isa<MDNode>(DDI->getArgOperand(0)) ||
1388           !isa<MDNode>(DDI->getArgOperand(1))) {
1389         while (!Declare->use_empty()) {
1390           CallInst *CI = cast<CallInst>(Declare->use_back());
1391           CI->eraseFromParent();
1392         }
1393         Declare->eraseFromParent();
1394       }
1395     }
1396   }
1397 }