IndVarSimplify.cpp [plain text]
#define DEBUG_TYPE "indvars"
#include "llvm/Transforms/Scalar.h"
#include "llvm/BasicBlock.h"
#include "llvm/Constants.h"
#include "llvm/Instructions.h"
#include "llvm/IntrinsicInst.h"
#include "llvm/LLVMContext.h"
#include "llvm/Type.h"
#include "llvm/Analysis/Dominators.h"
#include "llvm/Analysis/IVUsers.h"
#include "llvm/Analysis/ScalarEvolutionExpander.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopPass.h"
#include "llvm/Support/CFG.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/STLExtras.h"
using namespace llvm;
STATISTIC(NumRemoved , "Number of aux indvars removed");
STATISTIC(NumInserted, "Number of canonical indvars added");
STATISTIC(NumReplaced, "Number of exit values replaced");
STATISTIC(NumLFTR , "Number of loop exit tests replaced");
namespace {
class IndVarSimplify : public LoopPass {
IVUsers *IU;
LoopInfo *LI;
ScalarEvolution *SE;
DominatorTree *DT;
bool Changed;
public:
static char ID; IndVarSimplify() : LoopPass(&ID) {}
virtual bool runOnLoop(Loop *L, LPPassManager &LPM);
virtual void getAnalysisUsage(AnalysisUsage &AU) const {
AU.addRequired<DominatorTree>();
AU.addRequired<LoopInfo>();
AU.addRequired<ScalarEvolution>();
AU.addRequiredID(LoopSimplifyID);
AU.addRequiredID(LCSSAID);
AU.addRequired<IVUsers>();
AU.addPreserved<ScalarEvolution>();
AU.addPreservedID(LoopSimplifyID);
AU.addPreservedID(LCSSAID);
AU.addPreserved<IVUsers>();
AU.setPreservesCFG();
}
private:
void RewriteNonIntegerIVs(Loop *L);
ICmpInst *LinearFunctionTestReplace(Loop *L, const SCEV *BackedgeTakenCount,
Value *IndVar,
BasicBlock *ExitingBlock,
BranchInst *BI,
SCEVExpander &Rewriter);
void RewriteLoopExitValues(Loop *L, SCEVExpander &Rewriter);
void RewriteIVExpressions(Loop *L, SCEVExpander &Rewriter);
void SinkUnusedInvariants(Loop *L);
void HandleFloatingPointIV(Loop *L, PHINode *PH);
};
}
char IndVarSimplify::ID = 0;
static RegisterPass<IndVarSimplify>
X("indvars", "Canonicalize Induction Variables");
Pass *llvm::createIndVarSimplifyPass() {
return new IndVarSimplify();
}
ICmpInst *IndVarSimplify::LinearFunctionTestReplace(Loop *L,
const SCEV *BackedgeTakenCount,
Value *IndVar,
BasicBlock *ExitingBlock,
BranchInst *BI,
SCEVExpander &Rewriter) {
Value *CmpIndVar;
const SCEV *RHS = BackedgeTakenCount;
if (ExitingBlock == L->getLoopLatch()) {
const SCEV *Zero = SE->getIntegerSCEV(0, BackedgeTakenCount->getType());
const SCEV *N =
SE->getAddExpr(BackedgeTakenCount,
SE->getIntegerSCEV(1, BackedgeTakenCount->getType()));
if ((isa<SCEVConstant>(N) && !N->isZero()) ||
SE->isLoopGuardedByCond(L, ICmpInst::ICMP_NE, N, Zero)) {
RHS = SE->getTruncateOrZeroExtend(N, IndVar->getType());
} else {
RHS = SE->getTruncateOrZeroExtend(BackedgeTakenCount,
IndVar->getType());
RHS = SE->getAddExpr(RHS,
SE->getIntegerSCEV(1, IndVar->getType()));
}
CmpIndVar = L->getCanonicalInductionVariableIncrement();
} else {
RHS = SE->getTruncateOrZeroExtend(BackedgeTakenCount,
IndVar->getType());
CmpIndVar = IndVar;
}
assert(RHS->isLoopInvariant(L) &&
"Computed iteration count is not loop invariant!");
Value *ExitCnt = Rewriter.expandCodeFor(RHS, IndVar->getType(), BI);
ICmpInst::Predicate Opcode;
if (L->contains(BI->getSuccessor(0)))
Opcode = ICmpInst::ICMP_NE;
else
Opcode = ICmpInst::ICMP_EQ;
DEBUG(dbgs() << "INDVARS: Rewriting loop exit condition to:\n"
<< " LHS:" << *CmpIndVar << '\n'
<< " op:\t"
<< (Opcode == ICmpInst::ICMP_NE ? "!=" : "==") << "\n"
<< " RHS:\t" << *RHS << "\n");
ICmpInst *Cond = new ICmpInst(BI, Opcode, CmpIndVar, ExitCnt, "exitcond");
Value *OrigCond = BI->getCondition();
BI->setCondition(Cond);
RecursivelyDeleteTriviallyDeadInstructions(OrigCond);
++NumLFTR;
Changed = true;
return Cond;
}
void IndVarSimplify::RewriteLoopExitValues(Loop *L,
SCEVExpander &Rewriter) {
assert(L->isLCSSAForm(*DT));
SmallVector<BasicBlock*, 8> ExitBlocks;
L->getUniqueExitBlocks(ExitBlocks);
for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
BasicBlock *ExitBB = ExitBlocks[i];
PHINode *PN = dyn_cast<PHINode>(ExitBB->begin());
if (!PN) continue;
unsigned NumPreds = PN->getNumIncomingValues();
BasicBlock::iterator BBI = ExitBB->begin();
while ((PN = dyn_cast<PHINode>(BBI++))) {
if (PN->use_empty())
continue;
if (!PN->getType()->isIntegerTy() && !PN->getType()->isPointerTy())
continue;
SE->forgetValue(PN);
for (unsigned i = 0; i != NumPreds; ++i) {
Value *InVal = PN->getIncomingValue(i);
if (!isa<Instruction>(InVal))
continue;
if (LI->getLoopFor(PN->getIncomingBlock(i)) != L)
continue;
Instruction *Inst = cast<Instruction>(InVal);
if (!L->contains(Inst))
continue;
const SCEV *ExitValue = SE->getSCEVAtScope(Inst, L->getParentLoop());
if (!ExitValue->isLoopInvariant(L))
continue;
Changed = true;
++NumReplaced;
Value *ExitVal = Rewriter.expandCodeFor(ExitValue, PN->getType(), Inst);
DEBUG(dbgs() << "INDVARS: RLEV: AfterLoopVal = " << *ExitVal << '\n'
<< " LoopVal = " << *Inst << "\n");
PN->setIncomingValue(i, ExitVal);
RecursivelyDeleteTriviallyDeadInstructions(Inst);
if (NumPreds == 1) {
PN->replaceAllUsesWith(ExitVal);
RecursivelyDeleteTriviallyDeadInstructions(PN);
}
}
if (NumPreds != 1) {
PHINode *NewPN = cast<PHINode>(PN->clone());
NewPN->takeName(PN);
NewPN->insertBefore(PN);
PN->replaceAllUsesWith(NewPN);
PN->eraseFromParent();
}
}
}
}
void IndVarSimplify::RewriteNonIntegerIVs(Loop *L) {
BasicBlock *Header = L->getHeader();
SmallVector<WeakVH, 8> PHIs;
for (BasicBlock::iterator I = Header->begin();
PHINode *PN = dyn_cast<PHINode>(I); ++I)
PHIs.push_back(PN);
for (unsigned i = 0, e = PHIs.size(); i != e; ++i)
if (PHINode *PN = dyn_cast_or_null<PHINode>(PHIs[i]))
HandleFloatingPointIV(L, PN);
if (Changed)
SE->forgetLoop(L);
}
bool IndVarSimplify::runOnLoop(Loop *L, LPPassManager &LPM) {
IU = &getAnalysis<IVUsers>();
LI = &getAnalysis<LoopInfo>();
SE = &getAnalysis<ScalarEvolution>();
DT = &getAnalysis<DominatorTree>();
Changed = false;
RewriteNonIntegerIVs(L);
BasicBlock *ExitingBlock = L->getExitingBlock(); const SCEV *BackedgeTakenCount = SE->getBackedgeTakenCount(L);
SCEVExpander Rewriter(*SE);
if (!isa<SCEVCouldNotCompute>(BackedgeTakenCount))
RewriteLoopExitValues(L, Rewriter);
const Type *LargestType = 0;
bool NeedCannIV = false;
if (!isa<SCEVCouldNotCompute>(BackedgeTakenCount)) {
LargestType = BackedgeTakenCount->getType();
LargestType = SE->getEffectiveSCEVType(LargestType);
if (ExitingBlock)
NeedCannIV = true;
}
for (IVUsers::const_iterator I = IU->begin(), E = IU->end(); I != E; ++I) {
const Type *Ty =
SE->getEffectiveSCEVType(I->getOperandValToReplace()->getType());
if (!LargestType ||
SE->getTypeSizeInBits(Ty) >
SE->getTypeSizeInBits(LargestType))
LargestType = Ty;
NeedCannIV = true;
}
Value *IndVar = 0;
if (NeedCannIV) {
SmallVector<PHINode *, 2> OldCannIVs;
while (PHINode *OldCannIV = L->getCanonicalInductionVariable()) {
if (SE->getTypeSizeInBits(OldCannIV->getType()) >
SE->getTypeSizeInBits(LargestType))
OldCannIV->removeFromParent();
else
break;
OldCannIVs.push_back(OldCannIV);
}
IndVar = Rewriter.getOrInsertCanonicalInductionVariable(L, LargestType);
++NumInserted;
Changed = true;
DEBUG(dbgs() << "INDVARS: New CanIV: " << *IndVar << '\n');
while (!OldCannIVs.empty()) {
PHINode *OldCannIV = OldCannIVs.pop_back_val();
OldCannIV->insertBefore(L->getHeader()->getFirstNonPHI());
}
}
ICmpInst *NewICmp = 0;
if (!isa<SCEVCouldNotCompute>(BackedgeTakenCount) &&
!BackedgeTakenCount->isZero() &&
ExitingBlock) {
assert(NeedCannIV &&
"LinearFunctionTestReplace requires a canonical induction variable");
if (BranchInst *BI = dyn_cast<BranchInst>(ExitingBlock->getTerminator()))
NewICmp = LinearFunctionTestReplace(L, BackedgeTakenCount, IndVar,
ExitingBlock, BI, Rewriter);
}
RewriteIVExpressions(L, Rewriter);
SinkUnusedInvariants(L);
if (NewICmp)
IU->AddUsersIfInteresting(cast<Instruction>(NewICmp->getOperand(0)));
Changed |= DeleteDeadPHIs(L->getHeader());
assert(L->isLCSSAForm(*DT) && "Indvars did not leave the loop in lcssa form!");
return Changed;
}
static bool isSafe(const SCEV *S, const Loop *L) {
if (S->isLoopInvariant(L)) return true;
if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S))
return AR->isAffine();
if (const SCEVCommutativeExpr *Commutative = dyn_cast<SCEVCommutativeExpr>(S)) {
for (SCEVCommutativeExpr::op_iterator I = Commutative->op_begin(),
E = Commutative->op_end(); I != E; ++I)
if (!isSafe(*I, L)) return false;
return true;
}
if (const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(S))
return isSafe(C->getOperand(), L);
if (const SCEVUDivExpr *UD = dyn_cast<SCEVUDivExpr>(S))
return isSafe(UD->getLHS(), L) &&
isSafe(UD->getRHS(), L);
if (isa<SCEVUnknown>(S))
return true;
return false;
}
void IndVarSimplify::RewriteIVExpressions(Loop *L, SCEVExpander &Rewriter) {
SmallVector<WeakVH, 16> DeadInsts;
for (IVUsers::iterator UI = IU->begin(), E = IU->end(); UI != E; ++UI) {
Value *Op = UI->getOperandValToReplace();
const Type *UseTy = Op->getType();
Instruction *User = UI->getUser();
const SCEV *AR = IU->getReplacementExpr(*UI);
if (!L->contains(UI->getUser())) {
const SCEV *ExitVal = SE->getSCEVAtScope(AR, L->getParentLoop());
if (ExitVal->isLoopInvariant(L))
AR = ExitVal;
}
if (!isSafe(AR, L))
continue;
Instruction *InsertPt = User;
if (PHINode *PHI = dyn_cast<PHINode>(InsertPt))
for (unsigned i = 0, e = PHI->getNumIncomingValues(); i != e; ++i)
if (PHI->getIncomingValue(i) == Op) {
if (InsertPt == User)
InsertPt = PHI->getIncomingBlock(i)->getTerminator();
else
InsertPt =
DT->findNearestCommonDominator(InsertPt->getParent(),
PHI->getIncomingBlock(i))
->getTerminator();
}
Value *NewVal = Rewriter.expandCodeFor(AR, UseTy, InsertPt);
SE->forgetValue(User);
if (Op->hasName())
NewVal->takeName(Op);
User->replaceUsesOfWith(Op, NewVal);
UI->setOperandValToReplace(NewVal);
DEBUG(dbgs() << "INDVARS: Rewrote IV '" << *AR << "' " << *Op << '\n'
<< " into = " << *NewVal << "\n");
++NumRemoved;
Changed = true;
DeadInsts.push_back(Op);
}
Rewriter.clear();
while (!DeadInsts.empty())
if (Instruction *Inst =
dyn_cast_or_null<Instruction>(DeadInsts.pop_back_val()))
RecursivelyDeleteTriviallyDeadInstructions(Inst);
}
void IndVarSimplify::SinkUnusedInvariants(Loop *L) {
BasicBlock *ExitBlock = L->getExitBlock();
if (!ExitBlock) return;
BasicBlock *Preheader = L->getLoopPreheader();
if (!Preheader) return;
Instruction *InsertPt = ExitBlock->getFirstNonPHI();
BasicBlock::iterator I = Preheader->getTerminator();
while (I != Preheader->begin()) {
--I;
if (isa<PHINode>(I))
break;
if (I->mayHaveSideEffects() || I->mayReadFromMemory())
continue;
if (isa<DbgInfoIntrinsic>(I))
continue;
if (AllocaInst *AI = dyn_cast<AllocaInst>(I))
if (AI->isStaticAlloca())
continue;
bool UsedInLoop = false;
for (Value::use_iterator UI = I->use_begin(), UE = I->use_end();
UI != UE; ++UI) {
BasicBlock *UseBB = cast<Instruction>(UI)->getParent();
if (PHINode *P = dyn_cast<PHINode>(UI)) {
unsigned i =
PHINode::getIncomingValueNumForOperand(UI.getOperandNo());
UseBB = P->getIncomingBlock(i);
}
if (UseBB == Preheader || L->contains(UseBB)) {
UsedInLoop = true;
break;
}
}
if (UsedInLoop)
continue;
Instruction *ToMove = I;
bool Done = false;
if (I != Preheader->begin()) {
do {
--I;
} while (isa<DbgInfoIntrinsic>(I) && I != Preheader->begin());
if (isa<DbgInfoIntrinsic>(I) && I == Preheader->begin())
Done = true;
} else {
Done = true;
}
ToMove->moveBefore(InsertPt);
if (Done) break;
InsertPt = ToMove;
}
}
static bool ConvertToSInt(const APFloat &APF, int64_t &IntVal) {
bool isExact = false;
if (&APF.getSemantics() == &APFloat::PPCDoubleDouble)
return false;
uint64_t UIntVal;
if (APF.convertToInteger(&UIntVal, 64, true, APFloat::rmTowardZero,
&isExact) != APFloat::opOK || !isExact)
return false;
IntVal = UIntVal;
return true;
}
void IndVarSimplify::HandleFloatingPointIV(Loop *L, PHINode *PN) {
unsigned IncomingEdge = L->contains(PN->getIncomingBlock(0));
unsigned BackEdge = IncomingEdge^1;
ConstantFP *InitValueVal =
dyn_cast<ConstantFP>(PN->getIncomingValue(IncomingEdge));
int64_t InitValue;
if (!InitValueVal || !ConvertToSInt(InitValueVal->getValueAPF(), InitValue))
return;
BinaryOperator *Incr =
dyn_cast<BinaryOperator>(PN->getIncomingValue(BackEdge));
if (Incr == 0 || Incr->getOpcode() != Instruction::FAdd) return;
ConstantFP *IncValueVal = dyn_cast<ConstantFP>(Incr->getOperand(1));
int64_t IncValue;
if (IncValueVal == 0 || Incr->getOperand(0) != PN ||
!ConvertToSInt(IncValueVal->getValueAPF(), IncValue))
return;
Value::use_iterator IncrUse = Incr->use_begin();
Instruction *U1 = cast<Instruction>(IncrUse++);
if (IncrUse == Incr->use_end()) return;
Instruction *U2 = cast<Instruction>(IncrUse++);
if (IncrUse != Incr->use_end()) return;
FCmpInst *Compare = dyn_cast<FCmpInst>(U1);
if (!Compare)
Compare = dyn_cast<FCmpInst>(U2);
if (Compare == 0 || !Compare->hasOneUse() ||
!isa<BranchInst>(Compare->use_back()))
return;
BranchInst *TheBr = cast<BranchInst>(Compare->use_back());
assert(TheBr->isConditional() && "Can't use fcmp if not conditional");
if (!L->contains(TheBr->getParent()) ||
(L->contains(TheBr->getSuccessor(0)) &&
L->contains(TheBr->getSuccessor(1))))
return;
ConstantFP *ExitValueVal = dyn_cast<ConstantFP>(Compare->getOperand(1));
int64_t ExitValue;
if (ExitValueVal == 0 ||
!ConvertToSInt(ExitValueVal->getValueAPF(), ExitValue))
return;
CmpInst::Predicate NewPred = CmpInst::BAD_ICMP_PREDICATE;
switch (Compare->getPredicate()) {
default: return; case CmpInst::FCMP_OEQ:
case CmpInst::FCMP_UEQ: NewPred = CmpInst::ICMP_EQ; break;
case CmpInst::FCMP_ONE:
case CmpInst::FCMP_UNE: NewPred = CmpInst::ICMP_NE; break;
case CmpInst::FCMP_OGT:
case CmpInst::FCMP_UGT: NewPred = CmpInst::ICMP_SGT; break;
case CmpInst::FCMP_OGE:
case CmpInst::FCMP_UGE: NewPred = CmpInst::ICMP_SGE; break;
case CmpInst::FCMP_OLT:
case CmpInst::FCMP_ULT: NewPred = CmpInst::ICMP_SLT; break;
case CmpInst::FCMP_OLE:
case CmpInst::FCMP_ULE: NewPred = CmpInst::ICMP_SLE; break;
}
if (!isInt<32>(InitValue) || !isInt<32>(IncValue) || !isInt<32>(ExitValue))
return;
if (IncValue == 0)
return;
if (IncValue > 0) {
if (InitValue >= ExitValue ||
NewPred == CmpInst::ICMP_SGT || NewPred == CmpInst::ICMP_SGE)
return;
uint32_t Range = uint32_t(ExitValue-InitValue);
if (NewPred == CmpInst::ICMP_SLE) {
if (++Range == 0) return; }
unsigned Leftover = Range % uint32_t(IncValue);
if ((NewPred == CmpInst::ICMP_EQ || NewPred == CmpInst::ICMP_NE) &&
Leftover != 0)
return;
if (Leftover != 0 && int32_t(ExitValue+IncValue) < ExitValue)
return;
} else {
if (InitValue >= ExitValue ||
NewPred == CmpInst::ICMP_SLT || NewPred == CmpInst::ICMP_SLE)
return;
uint32_t Range = uint32_t(InitValue-ExitValue);
if (NewPred == CmpInst::ICMP_SGE) {
if (++Range == 0) return; }
unsigned Leftover = Range % uint32_t(-IncValue);
if ((NewPred == CmpInst::ICMP_EQ || NewPred == CmpInst::ICMP_NE) &&
Leftover != 0)
return;
if (Leftover != 0 && int32_t(ExitValue+IncValue) > ExitValue)
return;
}
const IntegerType *Int32Ty = Type::getInt32Ty(PN->getContext());
PHINode *NewPHI = PHINode::Create(Int32Ty, PN->getName()+".int", PN);
NewPHI->addIncoming(ConstantInt::get(Int32Ty, InitValue),
PN->getIncomingBlock(IncomingEdge));
Value *NewAdd =
BinaryOperator::CreateAdd(NewPHI, ConstantInt::get(Int32Ty, IncValue),
Incr->getName()+".int", Incr);
NewPHI->addIncoming(NewAdd, PN->getIncomingBlock(BackEdge));
ICmpInst *NewCompare = new ICmpInst(TheBr, NewPred, NewAdd,
ConstantInt::get(Int32Ty, ExitValue),
Compare->getName());
WeakVH WeakPH = PN;
NewCompare->takeName(Compare);
Compare->replaceAllUsesWith(NewCompare);
RecursivelyDeleteTriviallyDeadInstructions(Compare);
Incr->replaceAllUsesWith(UndefValue::get(Incr->getType()));
RecursivelyDeleteTriviallyDeadInstructions(Incr);
if (WeakPH) {
Value *Conv = new SIToFPInst(NewPHI, PN->getType(), "indvar.conv",
PN->getParent()->getFirstNonPHI());
PN->replaceAllUsesWith(Conv);
RecursivelyDeleteTriviallyDeadInstructions(PN);
}
IU->AddUsersIfInteresting(NewPHI);
}