DFGOSRExitCompiler64.cpp [plain text]
#include "config.h"
#include "DFGOSRExitCompiler.h"
#if ENABLE(DFG_JIT) && USE(JSVALUE64)
#include "DFGOperations.h"
namespace JSC { namespace DFG {
void OSRExitCompiler::compileExit(const OSRExit& exit, SpeculationRecovery* recovery)
{
#if DFG_ENABLE(DEBUG_VERBOSE)
dataLog("OSR exit for Node @%d (", (int)exit.m_nodeIndex);
for (CodeOrigin codeOrigin = exit.m_codeOrigin; ; codeOrigin = codeOrigin.inlineCallFrame->caller) {
dataLog("bc#%u", codeOrigin.bytecodeIndex);
if (!codeOrigin.inlineCallFrame)
break;
dataLog(" -> %p ", codeOrigin.inlineCallFrame->executable.get());
}
dataLog(") ");
exit.dump(WTF::dataFile());
#endif
#if DFG_ENABLE(VERBOSE_SPECULATION_FAILURE)
SpeculationFailureDebugInfo* debugInfo = new SpeculationFailureDebugInfo;
debugInfo->codeBlock = m_jit.codeBlock();
debugInfo->nodeIndex = exit.m_nodeIndex;
m_jit.debugCall(debugOperationPrintSpeculationFailure, debugInfo);
#endif
#if DFG_ENABLE(JIT_BREAK_ON_SPECULATION_FAILURE)
m_jit.breakpoint();
#endif
#if DFG_ENABLE(SUCCESS_STATS)
static SamplingCounter counter("SpeculationFailure");
m_jit.emitCount(counter);
#endif
GPRReg alreadyBoxed = InvalidGPRReg;
if (recovery) {
switch (recovery->type()) {
case SpeculativeAdd:
m_jit.sub32(recovery->src(), recovery->dest());
m_jit.orPtr(GPRInfo::tagTypeNumberRegister, recovery->dest());
alreadyBoxed = recovery->dest();
break;
case BooleanSpeculationCheck:
m_jit.xorPtr(AssemblyHelpers::TrustedImm32(static_cast<int32_t>(ValueFalse)), recovery->dest());
break;
default:
break;
}
}
if (!!exit.m_jsValueSource && !!exit.m_valueProfile) {
EncodedJSValue* bucket = exit.m_valueProfile.getSpecFailBucket(0);
#if DFG_ENABLE(VERBOSE_SPECULATION_FAILURE)
dataLog(" (have exit profile, bucket %p) ", bucket);
#endif
if (exit.m_jsValueSource.isAddress()) {
m_jit.loadPtr(AssemblyHelpers::Address(exit.m_jsValueSource.asAddress()), GPRInfo::tagTypeNumberRegister);
m_jit.storePtr(GPRInfo::tagTypeNumberRegister, bucket);
m_jit.move(AssemblyHelpers::TrustedImmPtr(bitwise_cast<void*>(TagTypeNumber)), GPRInfo::tagTypeNumberRegister);
} else
m_jit.storePtr(exit.m_jsValueSource.gpr(), bucket);
}
Vector<bool> poisonedVirtualRegisters(exit.m_variables.size());
for (unsigned i = 0; i < poisonedVirtualRegisters.size(); ++i)
poisonedVirtualRegisters[i] = false;
unsigned numberOfPoisonedVirtualRegisters = 0;
unsigned numberOfDisplacedVirtualRegisters = 0;
bool haveUnboxedInt32s = false;
bool haveUnboxedDoubles = false;
bool haveFPRs = false;
bool haveConstants = false;
bool haveUndefined = false;
bool haveUInt32s = false;
for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
const ValueRecovery& recovery = exit.valueRecovery(index);
switch (recovery.technique()) {
case Int32DisplacedInRegisterFile:
case DoubleDisplacedInRegisterFile:
case DisplacedInRegisterFile:
numberOfDisplacedVirtualRegisters++;
ASSERT((int)recovery.virtualRegister() >= 0);
if (recovery.virtualRegister() < (int)exit.m_variables.size()) {
switch (exit.m_variables[recovery.virtualRegister()].technique()) {
case InGPR:
case UnboxedInt32InGPR:
case UInt32InGPR:
case InFPR:
if (!poisonedVirtualRegisters[recovery.virtualRegister()]) {
poisonedVirtualRegisters[recovery.virtualRegister()] = true;
numberOfPoisonedVirtualRegisters++;
}
break;
default:
break;
}
}
break;
case UnboxedInt32InGPR:
case AlreadyInRegisterFileAsUnboxedInt32:
haveUnboxedInt32s = true;
break;
case AlreadyInRegisterFileAsUnboxedDouble:
haveUnboxedDoubles = true;
break;
case UInt32InGPR:
haveUInt32s = true;
break;
case InFPR:
haveFPRs = true;
break;
case Constant:
haveConstants = true;
if (recovery.constant().isUndefined())
haveUndefined = true;
break;
default:
break;
}
}
#if DFG_ENABLE(DEBUG_VERBOSE)
dataLog(" ");
if (numberOfPoisonedVirtualRegisters)
dataLog("Poisoned=%u ", numberOfPoisonedVirtualRegisters);
if (numberOfDisplacedVirtualRegisters)
dataLog("Displaced=%u ", numberOfDisplacedVirtualRegisters);
if (haveUnboxedInt32s)
dataLog("UnboxedInt32 ");
if (haveUnboxedDoubles)
dataLog("UnboxedDoubles ");
if (haveUInt32s)
dataLog("UInt32 ");
if (haveFPRs)
dataLog("FPR ");
if (haveConstants)
dataLog("Constants ");
if (haveUndefined)
dataLog("Undefined ");
dataLog(" ");
#endif
ScratchBuffer* scratchBuffer = m_jit.globalData()->scratchBufferForSize(sizeof(EncodedJSValue) * std::max(haveUInt32s ? 2u : 0u, numberOfPoisonedVirtualRegisters + (numberOfDisplacedVirtualRegisters <= GPRInfo::numberOfRegisters ? 0 : numberOfDisplacedVirtualRegisters)));
EncodedJSValue* scratchDataBuffer = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0;
if (haveUnboxedInt32s || haveUInt32s) {
for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
const ValueRecovery& recovery = exit.valueRecovery(index);
switch (recovery.technique()) {
case UnboxedInt32InGPR:
if (recovery.gpr() != alreadyBoxed)
m_jit.orPtr(GPRInfo::tagTypeNumberRegister, recovery.gpr());
break;
case AlreadyInRegisterFileAsUnboxedInt32:
m_jit.store32(AssemblyHelpers::TrustedImm32(static_cast<uint32_t>(TagTypeNumber >> 32)), AssemblyHelpers::tagFor(static_cast<VirtualRegister>(exit.operandForIndex(index))));
break;
case UInt32InGPR: {
GPRReg addressGPR = GPRInfo::regT0;
if (addressGPR == recovery.gpr())
addressGPR = GPRInfo::regT1;
m_jit.storePtr(addressGPR, scratchDataBuffer);
m_jit.move(AssemblyHelpers::TrustedImmPtr(scratchDataBuffer + 1), addressGPR);
m_jit.storeDouble(FPRInfo::fpRegT0, addressGPR);
AssemblyHelpers::Jump positive = m_jit.branch32(AssemblyHelpers::GreaterThanOrEqual, recovery.gpr(), AssemblyHelpers::TrustedImm32(0));
m_jit.convertInt32ToDouble(recovery.gpr(), FPRInfo::fpRegT0);
m_jit.addDouble(AssemblyHelpers::AbsoluteAddress(&AssemblyHelpers::twoToThe32), FPRInfo::fpRegT0);
m_jit.boxDouble(FPRInfo::fpRegT0, recovery.gpr());
AssemblyHelpers::Jump done = m_jit.jump();
positive.link(&m_jit);
m_jit.orPtr(GPRInfo::tagTypeNumberRegister, recovery.gpr());
done.link(&m_jit);
m_jit.loadDouble(addressGPR, FPRInfo::fpRegT0);
m_jit.loadPtr(scratchDataBuffer, addressGPR);
break;
}
default:
break;
}
}
}
initializePoisoned(exit.m_variables.size());
unsigned currentPoisonIndex = 0;
for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
const ValueRecovery& recovery = exit.valueRecovery(index);
int operand = exit.operandForIndex(index);
switch (recovery.technique()) {
case InGPR:
case UnboxedInt32InGPR:
case UInt32InGPR:
if (exit.isVariable(index) && poisonedVirtualRegisters[exit.variableForIndex(index)]) {
m_jit.storePtr(recovery.gpr(), scratchDataBuffer + currentPoisonIndex);
m_poisonScratchIndices[exit.variableForIndex(index)] = currentPoisonIndex;
currentPoisonIndex++;
} else
m_jit.storePtr(recovery.gpr(), AssemblyHelpers::addressFor((VirtualRegister)operand));
break;
default:
break;
}
}
if (haveFPRs) {
for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
const ValueRecovery& recovery = exit.valueRecovery(index);
if (recovery.technique() != InFPR)
continue;
FPRReg fpr = recovery.fpr();
GPRReg gpr = GPRInfo::toRegister(FPRInfo::toIndex(fpr));
m_jit.boxDouble(fpr, gpr);
}
for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
const ValueRecovery& recovery = exit.valueRecovery(index);
if (recovery.technique() != InFPR)
continue;
GPRReg gpr = GPRInfo::toRegister(FPRInfo::toIndex(recovery.fpr()));
if (exit.isVariable(index) && poisonedVirtualRegisters[exit.variableForIndex(index)]) {
m_jit.storePtr(gpr, scratchDataBuffer + currentPoisonIndex);
m_poisonScratchIndices[exit.variableForIndex(index)] = currentPoisonIndex;
currentPoisonIndex++;
} else
m_jit.storePtr(gpr, AssemblyHelpers::addressFor((VirtualRegister)exit.operandForIndex(index)));
}
}
if (haveUnboxedDoubles) {
for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
const ValueRecovery& recovery = exit.valueRecovery(index);
if (recovery.technique() != AlreadyInRegisterFileAsUnboxedDouble)
continue;
m_jit.loadDouble(AssemblyHelpers::addressFor((VirtualRegister)exit.operandForIndex(index)), FPRInfo::fpRegT0);
m_jit.boxDouble(FPRInfo::fpRegT0, GPRInfo::regT0);
m_jit.storePtr(GPRInfo::regT0, AssemblyHelpers::addressFor((VirtualRegister)exit.operandForIndex(index)));
}
}
ASSERT(currentPoisonIndex == numberOfPoisonedVirtualRegisters);
if (numberOfDisplacedVirtualRegisters) {
if (numberOfDisplacedVirtualRegisters <= GPRInfo::numberOfRegisters) {
unsigned displacementIndex = 0;
for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
const ValueRecovery& recovery = exit.valueRecovery(index);
switch (recovery.technique()) {
case DisplacedInRegisterFile:
m_jit.loadPtr(AssemblyHelpers::addressFor(recovery.virtualRegister()), GPRInfo::toRegister(displacementIndex++));
break;
case Int32DisplacedInRegisterFile: {
GPRReg gpr = GPRInfo::toRegister(displacementIndex++);
m_jit.load32(AssemblyHelpers::addressFor(recovery.virtualRegister()), gpr);
m_jit.orPtr(GPRInfo::tagTypeNumberRegister, gpr);
break;
}
case DoubleDisplacedInRegisterFile: {
GPRReg gpr = GPRInfo::toRegister(displacementIndex++);
m_jit.loadPtr(AssemblyHelpers::addressFor(recovery.virtualRegister()), gpr);
m_jit.subPtr(GPRInfo::tagTypeNumberRegister, gpr);
break;
}
default:
break;
}
}
displacementIndex = 0;
for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
const ValueRecovery& recovery = exit.valueRecovery(index);
switch (recovery.technique()) {
case DisplacedInRegisterFile:
case Int32DisplacedInRegisterFile:
case DoubleDisplacedInRegisterFile:
m_jit.storePtr(GPRInfo::toRegister(displacementIndex++), AssemblyHelpers::addressFor((VirtualRegister)exit.operandForIndex(index)));
break;
default:
break;
}
}
} else {
unsigned scratchIndex = numberOfPoisonedVirtualRegisters;
for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
const ValueRecovery& recovery = exit.valueRecovery(index);
switch (recovery.technique()) {
case DisplacedInRegisterFile:
m_jit.loadPtr(AssemblyHelpers::addressFor(recovery.virtualRegister()), GPRInfo::regT0);
m_jit.storePtr(GPRInfo::regT0, scratchDataBuffer + scratchIndex++);
break;
case Int32DisplacedInRegisterFile: {
m_jit.load32(AssemblyHelpers::addressFor(recovery.virtualRegister()), GPRInfo::regT0);
m_jit.orPtr(GPRInfo::tagTypeNumberRegister, GPRInfo::regT0);
m_jit.storePtr(GPRInfo::regT0, scratchDataBuffer + scratchIndex++);
break;
}
case DoubleDisplacedInRegisterFile: {
m_jit.loadPtr(AssemblyHelpers::addressFor(recovery.virtualRegister()), GPRInfo::regT0);
m_jit.subPtr(GPRInfo::tagTypeNumberRegister, GPRInfo::regT0);
m_jit.storePtr(GPRInfo::regT0, scratchDataBuffer + scratchIndex++);
break;
}
default:
break;
}
}
scratchIndex = numberOfPoisonedVirtualRegisters;
for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
const ValueRecovery& recovery = exit.valueRecovery(index);
switch (recovery.technique()) {
case DisplacedInRegisterFile:
case Int32DisplacedInRegisterFile:
case DoubleDisplacedInRegisterFile:
m_jit.loadPtr(scratchDataBuffer + scratchIndex++, GPRInfo::regT0);
m_jit.storePtr(GPRInfo::regT0, AssemblyHelpers::addressFor((VirtualRegister)exit.operandForIndex(index)));
break;
default:
break;
}
}
ASSERT(scratchIndex == numberOfPoisonedVirtualRegisters + numberOfDisplacedVirtualRegisters);
}
}
if (numberOfPoisonedVirtualRegisters) {
for (int virtualRegister = 0; virtualRegister < (int)exit.m_variables.size(); ++virtualRegister) {
if (!poisonedVirtualRegisters[virtualRegister])
continue;
const ValueRecovery& recovery = exit.m_variables[virtualRegister];
switch (recovery.technique()) {
case InGPR:
case UnboxedInt32InGPR:
case UInt32InGPR:
case InFPR:
m_jit.loadPtr(scratchDataBuffer + poisonIndex(virtualRegister), GPRInfo::regT0);
m_jit.storePtr(GPRInfo::regT0, AssemblyHelpers::addressFor((VirtualRegister)virtualRegister));
break;
default:
break;
}
}
}
if (haveConstants) {
if (haveUndefined)
m_jit.move(AssemblyHelpers::TrustedImmPtr(JSValue::encode(jsUndefined())), GPRInfo::regT0);
for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
const ValueRecovery& recovery = exit.valueRecovery(index);
if (recovery.technique() != Constant)
continue;
if (recovery.constant().isUndefined())
m_jit.storePtr(GPRInfo::regT0, AssemblyHelpers::addressFor((VirtualRegister)exit.operandForIndex(index)));
else
m_jit.storePtr(AssemblyHelpers::TrustedImmPtr(JSValue::encode(recovery.constant())), AssemblyHelpers::addressFor((VirtualRegister)exit.operandForIndex(index)));
}
}
handleExitCounts(exit);
if (exit.m_lastSetOperand != std::numeric_limits<int>::max())
m_jit.loadPtr(AssemblyHelpers::addressFor((VirtualRegister)exit.m_lastSetOperand), GPRInfo::cachedResultRegister);
ASSERT(m_jit.baselineCodeBlock()->getJITType() == JITCode::BaselineJIT);
m_jit.storePtr(AssemblyHelpers::TrustedImmPtr(m_jit.baselineCodeBlock()), AssemblyHelpers::addressFor((VirtualRegister)RegisterFile::CodeBlock));
for (CodeOrigin codeOrigin = exit.m_codeOrigin; codeOrigin.inlineCallFrame; codeOrigin = codeOrigin.inlineCallFrame->caller) {
InlineCallFrame* inlineCallFrame = codeOrigin.inlineCallFrame;
CodeBlock* baselineCodeBlock = m_jit.baselineCodeBlockFor(codeOrigin);
CodeBlock* baselineCodeBlockForCaller = m_jit.baselineCodeBlockFor(inlineCallFrame->caller);
Vector<BytecodeAndMachineOffset>& decodedCodeMap = m_jit.decodedCodeMapFor(baselineCodeBlockForCaller);
unsigned returnBytecodeIndex = inlineCallFrame->caller.bytecodeIndex + OPCODE_LENGTH(op_call);
BytecodeAndMachineOffset* mapping = binarySearch<BytecodeAndMachineOffset, unsigned, BytecodeAndMachineOffset::getBytecodeIndex>(decodedCodeMap.begin(), decodedCodeMap.size(), returnBytecodeIndex);
ASSERT(mapping);
ASSERT(mapping->m_bytecodeIndex == returnBytecodeIndex);
void* jumpTarget = baselineCodeBlockForCaller->getJITCode().executableAddressAtOffset(mapping->m_machineCodeOffset);
GPRReg callerFrameGPR;
if (inlineCallFrame->caller.inlineCallFrame) {
m_jit.addPtr(AssemblyHelpers::TrustedImm32(inlineCallFrame->caller.inlineCallFrame->stackOffset * sizeof(EncodedJSValue)), GPRInfo::callFrameRegister, GPRInfo::regT3);
callerFrameGPR = GPRInfo::regT3;
} else
callerFrameGPR = GPRInfo::callFrameRegister;
m_jit.storePtr(AssemblyHelpers::TrustedImmPtr(baselineCodeBlock), AssemblyHelpers::addressFor((VirtualRegister)(inlineCallFrame->stackOffset + RegisterFile::CodeBlock)));
m_jit.storePtr(AssemblyHelpers::TrustedImmPtr(inlineCallFrame->callee->scope()), AssemblyHelpers::addressFor((VirtualRegister)(inlineCallFrame->stackOffset + RegisterFile::ScopeChain)));
m_jit.storePtr(callerFrameGPR, AssemblyHelpers::addressFor((VirtualRegister)(inlineCallFrame->stackOffset + RegisterFile::CallerFrame)));
m_jit.storePtr(AssemblyHelpers::TrustedImmPtr(jumpTarget), AssemblyHelpers::addressFor((VirtualRegister)(inlineCallFrame->stackOffset + RegisterFile::ReturnPC)));
m_jit.store32(AssemblyHelpers::TrustedImm32(inlineCallFrame->arguments.size()), AssemblyHelpers::payloadFor((VirtualRegister)(inlineCallFrame->stackOffset + RegisterFile::ArgumentCount)));
m_jit.storePtr(AssemblyHelpers::TrustedImmPtr(inlineCallFrame->callee.get()), AssemblyHelpers::addressFor((VirtualRegister)(inlineCallFrame->stackOffset + RegisterFile::Callee)));
}
if (exit.m_codeOrigin.inlineCallFrame)
m_jit.addPtr(AssemblyHelpers::TrustedImm32(exit.m_codeOrigin.inlineCallFrame->stackOffset * sizeof(EncodedJSValue)), GPRInfo::callFrameRegister);
CodeBlock* baselineCodeBlock = m_jit.baselineCodeBlockFor(exit.m_codeOrigin);
Vector<BytecodeAndMachineOffset>& decodedCodeMap = m_jit.decodedCodeMapFor(baselineCodeBlock);
BytecodeAndMachineOffset* mapping = binarySearch<BytecodeAndMachineOffset, unsigned, BytecodeAndMachineOffset::getBytecodeIndex>(decodedCodeMap.begin(), decodedCodeMap.size(), exit.m_codeOrigin.bytecodeIndex);
ASSERT(mapping);
ASSERT(mapping->m_bytecodeIndex == exit.m_codeOrigin.bytecodeIndex);
void* jumpTarget = baselineCodeBlock->getJITCode().executableAddressAtOffset(mapping->m_machineCodeOffset);
ASSERT(GPRInfo::regT1 != GPRInfo::cachedResultRegister);
m_jit.move(AssemblyHelpers::TrustedImmPtr(jumpTarget), GPRInfo::regT1);
m_jit.jump(GPRInfo::regT1);
#if DFG_ENABLE(DEBUG_VERBOSE)
dataLog("-> %p\n", jumpTarget);
#endif
}
} }
#endif // ENABLE(DFG_JIT) && USE(JSVALUE64)