From ee0d992b07d282ca217ba158a1cb5671ccdc0ced Mon Sep 17 00:00:00 2001 From: James Molloy Date: Fri, 10 Jul 2015 12:52:00 +0000 Subject: [PATCH] Add support for fast-math flags to the FCmp instruction. FCmp behaves a lot like a floating-point binary operator in many ways, and can benefit from fast-math information. Flags such as nsz and nnan can affect if this fcmp (in combination with a select) can be treated as a fminnum/fmaxnum operation. This adds backwards-compatible bitcode support, IR parsing and writing, LangRef changes and IRBuilder changes. I'll need to audit InstSimplify and InstCombine in a followup to find places where flags should be copied. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@241901 91177308-0d34-0410-b5e6-96231b3b80d8 --- docs/LangRef.rst | 15 ++++-- include/llvm/IR/IRBuilder.h | 75 +++++++++++++++++----------- include/llvm/IR/Operator.h | 3 +- lib/AsmParser/LLParser.cpp | 13 ++++- lib/Bitcode/Reader/BitcodeReader.cpp | 46 +++++++++++------ lib/Bitcode/Writer/BitcodeWriter.cpp | 6 ++- test/Bitcode/fcmp-fast.ll | 23 +++++++++ unittests/IR/IRBuilderTest.cpp | 22 +++++++- 8 files changed, 149 insertions(+), 54 deletions(-) create mode 100644 test/Bitcode/fcmp-fast.ll diff --git a/docs/LangRef.rst b/docs/LangRef.rst index ca5c16c0f49..478fe49af0f 100644 --- a/docs/LangRef.rst +++ b/docs/LangRef.rst @@ -1837,8 +1837,8 @@ Fast-Math Flags LLVM IR floating-point binary ops (:ref:`fadd `, :ref:`fsub `, :ref:`fmul `, :ref:`fdiv `, -:ref:`frem `) have the following flags that can be set to enable -otherwise unsafe floating point operations +:ref:`frem `, :ref:`fcmp `) have the following flags that can +be set to enable otherwise unsafe floating point operations ``nnan`` No NaNs - Allow optimizations to assume the arguments and result are not @@ -7573,7 +7573,7 @@ Syntax: :: - = fcmp , ; yields i1 or :result + = fcmp [fast-math flags]* , ; yields i1 or :result Overview: """"""""" @@ -7656,6 +7656,15 @@ always yields an :ref:`i1 ` result, as follows: #. ``uno``: yields ``true`` if either operand is a QNAN. #. ``true``: always yields ``true``, regardless of operands. +The ``fcmp`` instruction can also optionally take any number of +:ref:`fast-math flags `, which are optimization hints to enable +otherwise unsafe floating point optimizations. + +Any set of fast-math flags are legal on an ``fcmp`` instruction, but the +only flags that have any effect on its semantics are those that allow +assumptions to be made about the values of input arguments; namely +``nnan``, ``ninf``, and ``nsz``. See :ref:`fastmath` for more information. + Example: """""""" diff --git a/include/llvm/IR/IRBuilder.h b/include/llvm/IR/IRBuilder.h index e6b5393c339..4913e4c376d 100644 --- a/include/llvm/IR/IRBuilder.h +++ b/include/llvm/IR/IRBuilder.h @@ -1382,47 +1382,61 @@ public: return CreateICmp(ICmpInst::ICMP_SLE, LHS, RHS, Name); } - Value *CreateFCmpOEQ(Value *LHS, Value *RHS, const Twine &Name = "") { - return CreateFCmp(FCmpInst::FCMP_OEQ, LHS, RHS, Name); + Value *CreateFCmpOEQ(Value *LHS, Value *RHS, const Twine &Name = "", + MDNode *FPMathTag = nullptr) { + return CreateFCmp(FCmpInst::FCMP_OEQ, LHS, RHS, Name, FPMathTag); } - Value *CreateFCmpOGT(Value *LHS, Value *RHS, const Twine &Name = "") { - return CreateFCmp(FCmpInst::FCMP_OGT, LHS, RHS, Name); + Value *CreateFCmpOGT(Value *LHS, Value *RHS, const Twine &Name = "", + MDNode *FPMathTag = nullptr) { + return CreateFCmp(FCmpInst::FCMP_OGT, LHS, RHS, Name, FPMathTag); } - Value *CreateFCmpOGE(Value *LHS, Value *RHS, const Twine &Name = "") { - return CreateFCmp(FCmpInst::FCMP_OGE, LHS, RHS, Name); + Value *CreateFCmpOGE(Value *LHS, Value *RHS, const Twine &Name = "", + MDNode *FPMathTag = nullptr) { + return CreateFCmp(FCmpInst::FCMP_OGE, LHS, RHS, Name, FPMathTag); } - Value *CreateFCmpOLT(Value *LHS, Value *RHS, const Twine &Name = "") { - return CreateFCmp(FCmpInst::FCMP_OLT, LHS, RHS, Name); + Value *CreateFCmpOLT(Value *LHS, Value *RHS, const Twine &Name = "", + MDNode *FPMathTag = nullptr) { + return CreateFCmp(FCmpInst::FCMP_OLT, LHS, RHS, Name, FPMathTag); } - Value *CreateFCmpOLE(Value *LHS, Value *RHS, const Twine &Name = "") { - return CreateFCmp(FCmpInst::FCMP_OLE, LHS, RHS, Name); + Value *CreateFCmpOLE(Value *LHS, Value *RHS, const Twine &Name = "", + MDNode *FPMathTag = nullptr) { + return CreateFCmp(FCmpInst::FCMP_OLE, LHS, RHS, Name, FPMathTag); } - Value *CreateFCmpONE(Value *LHS, Value *RHS, const Twine &Name = "") { - return CreateFCmp(FCmpInst::FCMP_ONE, LHS, RHS, Name); + Value *CreateFCmpONE(Value *LHS, Value *RHS, const Twine &Name = "", + MDNode *FPMathTag = nullptr) { + return CreateFCmp(FCmpInst::FCMP_ONE, LHS, RHS, Name, FPMathTag); } - Value *CreateFCmpORD(Value *LHS, Value *RHS, const Twine &Name = "") { - return CreateFCmp(FCmpInst::FCMP_ORD, LHS, RHS, Name); + Value *CreateFCmpORD(Value *LHS, Value *RHS, const Twine &Name = "", + MDNode *FPMathTag = nullptr) { + return CreateFCmp(FCmpInst::FCMP_ORD, LHS, RHS, Name, FPMathTag); } - Value *CreateFCmpUNO(Value *LHS, Value *RHS, const Twine &Name = "") { - return CreateFCmp(FCmpInst::FCMP_UNO, LHS, RHS, Name); + Value *CreateFCmpUNO(Value *LHS, Value *RHS, const Twine &Name = "", + MDNode *FPMathTag = nullptr) { + return CreateFCmp(FCmpInst::FCMP_UNO, LHS, RHS, Name, FPMathTag); } - Value *CreateFCmpUEQ(Value *LHS, Value *RHS, const Twine &Name = "") { - return CreateFCmp(FCmpInst::FCMP_UEQ, LHS, RHS, Name); + Value *CreateFCmpUEQ(Value *LHS, Value *RHS, const Twine &Name = "", + MDNode *FPMathTag = nullptr) { + return CreateFCmp(FCmpInst::FCMP_UEQ, LHS, RHS, Name, FPMathTag); } - Value *CreateFCmpUGT(Value *LHS, Value *RHS, const Twine &Name = "") { - return CreateFCmp(FCmpInst::FCMP_UGT, LHS, RHS, Name); + Value *CreateFCmpUGT(Value *LHS, Value *RHS, const Twine &Name = "", + MDNode *FPMathTag = nullptr) { + return CreateFCmp(FCmpInst::FCMP_UGT, LHS, RHS, Name, FPMathTag); } - Value *CreateFCmpUGE(Value *LHS, Value *RHS, const Twine &Name = "") { - return CreateFCmp(FCmpInst::FCMP_UGE, LHS, RHS, Name); + Value *CreateFCmpUGE(Value *LHS, Value *RHS, const Twine &Name = "", + MDNode *FPMathTag = nullptr) { + return CreateFCmp(FCmpInst::FCMP_UGE, LHS, RHS, Name, FPMathTag); } - Value *CreateFCmpULT(Value *LHS, Value *RHS, const Twine &Name = "") { - return CreateFCmp(FCmpInst::FCMP_ULT, LHS, RHS, Name); + Value *CreateFCmpULT(Value *LHS, Value *RHS, const Twine &Name = "", + MDNode *FPMathTag = nullptr) { + return CreateFCmp(FCmpInst::FCMP_ULT, LHS, RHS, Name, FPMathTag); } - Value *CreateFCmpULE(Value *LHS, Value *RHS, const Twine &Name = "") { - return CreateFCmp(FCmpInst::FCMP_ULE, LHS, RHS, Name); + Value *CreateFCmpULE(Value *LHS, Value *RHS, const Twine &Name = "", + MDNode *FPMathTag = nullptr) { + return CreateFCmp(FCmpInst::FCMP_ULE, LHS, RHS, Name, FPMathTag); } - Value *CreateFCmpUNE(Value *LHS, Value *RHS, const Twine &Name = "") { - return CreateFCmp(FCmpInst::FCMP_UNE, LHS, RHS, Name); + Value *CreateFCmpUNE(Value *LHS, Value *RHS, const Twine &Name = "", + MDNode *FPMathTag = nullptr) { + return CreateFCmp(FCmpInst::FCMP_UNE, LHS, RHS, Name, FPMathTag); } Value *CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, @@ -1433,11 +1447,12 @@ public: return Insert(new ICmpInst(P, LHS, RHS), Name); } Value *CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS, - const Twine &Name = "") { + const Twine &Name = "", MDNode *FPMathTag = nullptr) { if (Constant *LC = dyn_cast(LHS)) if (Constant *RC = dyn_cast(RHS)) return Insert(Folder.CreateFCmp(P, LC, RC), Name); - return Insert(new FCmpInst(P, LHS, RHS), Name); + return Insert(AddFPMathAttributes(new FCmpInst(P, LHS, RHS), + FPMathTag, FMF), Name); } //===--------------------------------------------------------------------===// diff --git a/include/llvm/IR/Operator.h b/include/llvm/IR/Operator.h index 1b9102ecc7e..372b254ab18 100644 --- a/include/llvm/IR/Operator.h +++ b/include/llvm/IR/Operator.h @@ -305,7 +305,8 @@ public: float getFPAccuracy() const; static inline bool classof(const Instruction *I) { - return I->getType()->isFPOrFPVectorTy(); + return I->getType()->isFPOrFPVectorTy() || + I->getOpcode() == Instruction::FCmp; } static inline bool classof(const Value *V) { return isa(V) && classof(cast(V)); diff --git a/lib/AsmParser/LLParser.cpp b/lib/AsmParser/LLParser.cpp index 91a88bc91fd..aa96d60ca0d 100644 --- a/lib/AsmParser/LLParser.cpp +++ b/lib/AsmParser/LLParser.cpp @@ -4534,8 +4534,17 @@ int LLParser::ParseInstruction(Instruction *&Inst, BasicBlock *BB, case lltok::kw_and: case lltok::kw_or: case lltok::kw_xor: return ParseLogical(Inst, PFS, KeywordVal); - case lltok::kw_icmp: - case lltok::kw_fcmp: return ParseCompare(Inst, PFS, KeywordVal); + case lltok::kw_icmp: return ParseCompare(Inst, PFS, KeywordVal); + case lltok::kw_fcmp: { + FastMathFlags FMF = EatFastMathFlagsIfPresent(); + int Res = ParseCompare(Inst, PFS, KeywordVal); + if (Res != 0) + return Res; + if (FMF.any()) + Inst->setFastMathFlags(FMF); + return 0; + } + // Casts. case lltok::kw_trunc: case lltok::kw_zext: diff --git a/lib/Bitcode/Reader/BitcodeReader.cpp b/lib/Bitcode/Reader/BitcodeReader.cpp index 563e201336a..4695078303a 100644 --- a/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/lib/Bitcode/Reader/BitcodeReader.cpp @@ -697,6 +697,21 @@ static Comdat::SelectionKind getDecodedComdatSelectionKind(unsigned Val) { } } +static FastMathFlags getDecodedFastMathFlags(unsigned Val) { + FastMathFlags FMF; + if (0 != (Val & FastMathFlags::UnsafeAlgebra)) + FMF.setUnsafeAlgebra(); + if (0 != (Val & FastMathFlags::NoNaNs)) + FMF.setNoNaNs(); + if (0 != (Val & FastMathFlags::NoInfs)) + FMF.setNoInfs(); + if (0 != (Val & FastMathFlags::NoSignedZeros)) + FMF.setNoSignedZeros(); + if (0 != (Val & FastMathFlags::AllowReciprocal)) + FMF.setAllowReciprocal(); + return FMF; +} + static void upgradeDLLImportExportLinkage(llvm::GlobalValue *GV, unsigned Val) { switch (Val) { case 5: GV->setDLLStorageClass(GlobalValue::DLLImportStorageClass); break; @@ -3472,17 +3487,7 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { if (Record[OpNum] & (1 << bitc::PEO_EXACT)) cast(I)->setIsExact(true); } else if (isa(I)) { - FastMathFlags FMF; - if (0 != (Record[OpNum] & FastMathFlags::UnsafeAlgebra)) - FMF.setUnsafeAlgebra(); - if (0 != (Record[OpNum] & FastMathFlags::NoNaNs)) - FMF.setNoNaNs(); - if (0 != (Record[OpNum] & FastMathFlags::NoInfs)) - FMF.setNoInfs(); - if (0 != (Record[OpNum] & FastMathFlags::NoSignedZeros)) - FMF.setNoSignedZeros(); - if (0 != (Record[OpNum] & FastMathFlags::AllowReciprocal)) - FMF.setAllowReciprocal(); + FastMathFlags FMF = getDecodedFastMathFlags(Record[OpNum]); if (FMF.any()) I->setFastMathFlags(FMF); } @@ -3739,14 +3744,25 @@ std::error_code BitcodeReader::parseFunctionBody(Function *F) { unsigned OpNum = 0; Value *LHS, *RHS; if (getValueTypePair(Record, OpNum, NextValueNo, LHS) || - popValue(Record, OpNum, NextValueNo, LHS->getType(), RHS) || - OpNum+1 != Record.size()) + popValue(Record, OpNum, NextValueNo, LHS->getType(), RHS)) + return error("Invalid record"); + + unsigned PredVal = Record[OpNum]; + bool IsFP = LHS->getType()->isFPOrFPVectorTy(); + FastMathFlags FMF; + if (IsFP && Record.size() > OpNum+1) + FMF = getDecodedFastMathFlags(Record[++OpNum]); + + if (OpNum+1 != Record.size()) return error("Invalid record"); if (LHS->getType()->isFPOrFPVectorTy()) - I = new FCmpInst((FCmpInst::Predicate)Record[OpNum], LHS, RHS); + I = new FCmpInst((FCmpInst::Predicate)PredVal, LHS, RHS); else - I = new ICmpInst((ICmpInst::Predicate)Record[OpNum], LHS, RHS); + I = new ICmpInst((ICmpInst::Predicate)PredVal, LHS, RHS); + + if (FMF.any()) + I->setFastMathFlags(FMF); InstructionList.push_back(I); break; } diff --git a/lib/Bitcode/Writer/BitcodeWriter.cpp b/lib/Bitcode/Writer/BitcodeWriter.cpp index 622f7eaf078..8a3ed18ffeb 100644 --- a/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -1759,13 +1759,17 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, pushValue(I.getOperand(2), InstID, Vals, VE); break; case Instruction::ICmp: - case Instruction::FCmp: + case Instruction::FCmp: { // compare returning Int1Ty or vector of Int1Ty Code = bitc::FUNC_CODE_INST_CMP2; PushValueAndType(I.getOperand(0), InstID, Vals, VE); pushValue(I.getOperand(1), InstID, Vals, VE); Vals.push_back(cast(I).getPredicate()); + uint64_t Flags = GetOptimizationFlags(&I); + if (Flags != 0) + Vals.push_back(Flags); break; + } case Instruction::Ret: { diff --git a/test/Bitcode/fcmp-fast.ll b/test/Bitcode/fcmp-fast.ll new file mode 100644 index 00000000000..126e3652e3a --- /dev/null +++ b/test/Bitcode/fcmp-fast.ll @@ -0,0 +1,23 @@ +; RUN: llvm-as < %s | llvm-dis > %t0 +; RUN: opt -S < %s > %t1 +; RUN: diff %t0 %t1 +; RUN: FileCheck < %t1 %s + +; Make sure flags on fcmp instructions are serialized/deserialized properly. + +define i1 @foo(float %a, float %b, double %c, double %d) { + ; CHECK: %plain = fcmp ueq float %a, %b + %plain = fcmp ueq float %a, %b + ; CHECK: %fast = fcmp fast olt float %a, %b + %fast = fcmp fast olt float %a, %b + ; CHECK: %nsz = fcmp nsz uge float %a, %b + %nsz = fcmp nsz uge float %a, %b + ; CHECK: %nnan = fcmp nnan nsz oge double %c, %d + %nnan = fcmp nnan nsz oge double %c, %d + + %dce1 = or i1 %plain, %fast + %dce2 = or i1 %dce1, %nsz + %dce3 = or i1 %dce2, %nnan + + ret i1 %dce3 +} diff --git a/unittests/IR/IRBuilderTest.cpp b/unittests/IR/IRBuilderTest.cpp index f3db68feacc..093cbbfc779 100644 --- a/unittests/IR/IRBuilderTest.cpp +++ b/unittests/IR/IRBuilderTest.cpp @@ -130,8 +130,8 @@ TEST_F(IRBuilderTest, GetIntTy) { TEST_F(IRBuilderTest, FastMathFlags) { IRBuilder<> Builder(BB); - Value *F; - Instruction *FDiv, *FAdd; + Value *F, *FC; + Instruction *FDiv, *FAdd, *FCmp; F = Builder.CreateLoad(GV); F = Builder.CreateFAdd(F, F); @@ -190,6 +190,24 @@ TEST_F(IRBuilderTest, FastMathFlags) { Builder.clearFastMathFlags(); + FC = Builder.CreateFCmpOEQ(F, F); + ASSERT_TRUE(isa(FC)); + FCmp = cast(FC); + EXPECT_FALSE(FCmp->hasAllowReciprocal()); + + FMF.clear(); + FMF.setAllowReciprocal(); + Builder.SetFastMathFlags(FMF); + + FC = Builder.CreateFCmpOEQ(F, F); + EXPECT_TRUE(Builder.getFastMathFlags().any()); + EXPECT_TRUE(Builder.getFastMathFlags().AllowReciprocal); + ASSERT_TRUE(isa(FC)); + FCmp = cast(FC); + EXPECT_TRUE(FCmp->hasAllowReciprocal()); + + Builder.clearFastMathFlags(); + // To test a copy, make sure that a '0' and a '1' change state. F = Builder.CreateFDiv(F, F); ASSERT_TRUE(isa(F)); -- 2.34.1