Attachment #459027 for bug #240205

View Details Raw Unified Return to Bug 240205

| Differences between



2022-05-08  Saam Barati  <sbarati@apple.com>

        Better handle clobbered registers in O0 register allocation
        https://bugs.webkit.org/show_bug.cgi?id=240205
        <rdar://87220688>

        Reviewed by NOBODY (OOPS!).

        This patch makes Air's O0 register allocator better handle clobbered
        registers. We now model both early and late clobber directly, and use
        this to perform a basic interference analysis when allocating a register
        to a Tmp. An early clobber interferes with any Use in an instruction, and
        any late Defs. A late clobber interferes with any Defs in an instruction,
        and any late Uses. What this enables is an early Use can be allocated
        to a register that is only late clobbered. And a result can be allocated 
        to a register that is only early clobbered.
        
        Prior to this, the algorithm had a bug where a Use may be allocated to
        a register that is early clobbered.

        * b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp:
        (JSC::B3::Air::GenerateAndAllocateRegisters::buildLiveRanges):
        (JSC::B3::Air::GenerateAndAllocateRegisters::alloc):
        (JSC::B3::Air::GenerateAndAllocateRegisters::freeDeadTmpsIfNeeded):
        (JSC::B3::Air::GenerateAndAllocateRegisters::assignTmp):
        (JSC::B3::Air::GenerateAndAllocateRegisters::prepareForGeneration):
        (JSC::B3::Air::GenerateAndAllocateRegisters::generate):
        * b3/air/AirAllocateRegistersAndStackAndGenerateCode.h:
        * b3/air/testair.cpp:
        * jit/RegisterSet.h:

2022-05-06  Ross Kirsling  <ross.kirsling@sony.com>

        Temporal.Duration#toString should never ignore fractionalSecondDigits



{
    m_liveRangeEnd = TmpMap<size_t>(m_code, 0);

    m_globalInstIndex = 1;
    for (BasicBlock* block : m_code) {
        for (Tmp tmp : liveness.liveAtHead(block)) {
            if (!tmp.isReg())

{
    ASSERT(reg);
    ASSERT(m_map[tmp].reg == reg);
    ASSERT(tmp.isReg() || m_liveRangeEnd[tmp] >= m_globalInstIndex);
    flush(tmp, reg);
    release(tmp, reg);
}

ALWAYS_INLINE void GenerateAndAllocateRegisters::alloc(Tmp tmp, Reg reg, Arg::Role role)
{
    if (Tmp occupyingTmp = m_currentAllocation->at(reg))
        spill(occupyingTmp, reg);

    m_availableRegs[tmp.bank()].clear(reg);
    m_currentAllocation->at(reg) = tmp;

    if (Arg::isAnyUse(role)) {
        intptr_t offset = m_map[tmp].spillSlot->offsetFromFP();
        if (tmp.bank() == GP)
            m_jit->load64(callFrameAddr(*m_jit, offset), reg.gpr());

    }
}

// freeDeadTmpsAtCurrentInst and freeDeadTmpsAtCurrentBlock are needed for correctness because 
// we reuse stack slots between Tmps that don't interfere. So we need to make sure we don't
// spill a dead Tmp to a live Tmp's slot.
// freeDeadTmpsAtCurrentInst is meant to be called as we walk through each instruction in a basic block
// because it doesn't consult the entire register file, and is faster than freeDeadTmpsAtCurrentBlock.
// However, it only prunes things that die at a particular inst index within a block, and doesn't prevent
// us from propagating a Tmp that is live in one block to the head of a block where it is dead. If
// something dies within a block, freeDeadTmpsAtCurrentInst will catch it. freeDeadTmpsAtCurrentBlock is 
// meant to ensure we prune away any Tmps that are dead at the head of a block.
ALWAYS_INLINE void GenerateAndAllocateRegisters::freeDeadTmpsAtCurrentInst()
{
    auto iter = m_tmpsToRelease.find(m_globalInstIndex);
    if (iter != m_tmpsToRelease.end()) {
        for (Tmp tmp : iter->value) {
            ASSERT(m_liveRangeEnd[tmp] < m_globalInstIndex);
            if (Reg reg = m_map[tmp].reg)
                release(tmp, reg);
        }
    }
}

ALWAYS_INLINE void GenerateAndAllocateRegisters::freeDeadTmpsAtCurrentBlock()
{
    for (size_t i = 0; i < m_currentAllocation->size(); ++i) {
        Tmp tmp = m_currentAllocation->at(i);
        if (!tmp)

    }
}

ALWAYS_INLINE bool GenerateAndAllocateRegisters::assignTmp(Tmp& tmp, Bank bank, Arg::Role role)
{
    ASSERT(!tmp.isReg());

    auto markRegisterAsUsed = [&] (Reg reg) {
        if (Arg::isAnyDef(role))
            m_clobberedToClear.clear(reg);
        // At this point, it doesn't matter if we add it to the m_namedUsedRegs or m_namedDefdRegs. 
        // We just need to mark that we can't use it again for another tmp.
        m_namedUsedRegs.set(reg);
    };

    bool mightInterfere = m_earlyClobber.numberOfSetRegisters() || m_lateClobber.numberOfSetRegisters();

    auto interferesWithClobber = [&] (Reg reg) {
        if (!mightInterfere)
            return false;
        if (Arg::isAnyUse(role) && m_earlyClobber.get(reg))
            return true;
        if (Arg::isAnyDef(role) && m_lateClobber.get(reg))
            return true;
        if (Arg::activeAt(role, Arg::Phase::Early) && m_earlyClobber.get(reg))
            return true;
        if (Arg::activeAt(role, Arg::Phase::Late) && m_lateClobber.get(reg))
            return true;
        return false;
    };

    if (Reg reg = m_map[tmp].reg) {
        if (!interferesWithClobber(reg)) {
            ASSERT(!m_namedDefdRegs.contains(reg));
            tmp = Tmp(reg);
            markRegisterAsUsed(reg);
            ASSERT(!m_availableRegs[bank].get(reg));
            return true;
        }
        // This is a rare case when we've already allocated a Tmp in some way, but another 
        // Role of the Tmp imposes some restriction on the register value. E.g, if
        // we have a program like:
        // Patch Use:tmp1, LateUse:tmp1, lateClobber:x0
        // The first use of tmp1 can be allocated to x0, but the second cannot.
        spill(tmp, reg);

    }
        freeDeadTmpsIfNeeded();

    if (m_availableRegs[bank].numberOfSetRegisters()) {
        // We first take an available register.
        for (Reg reg : m_registers[bank]) {
            if (interferesWithClobber(reg) || m_namedUsedRegs.contains(reg) || m_namedDefdRegs.contains(reg))
                continue;
            if (!m_availableRegs[bank].contains(reg))
                continue;

            markRegisterAsUsed(reg);
            alloc(tmp, reg, role);
            tmp = Tmp(reg);
            return true;
        }

        RELEASE_ASSERT_NOT_REACHED();
    }

    // Nothing was available, let's make some room.
    for (Reg reg : m_registers[bank]) {
        if (interferesWithClobber(reg) || m_namedUsedRegs.contains(reg) || m_namedDefdRegs.contains(reg))
            continue;

        markRegisterAsUsed(reg);
        alloc(tmp, reg, role);
        alloc(tmp, reg, isDef);
        tmp = Tmp(reg);
        return true;
    }


        Vector<StackSlot*, 16> freeSlots;
        Vector<StackSlot*, 4> toFree;
        m_globalInstIndex = 1;
        for (BasicBlock* block : m_code) {
            auto assignStackSlotToTmp = [&] (Tmp tmp) {
                if (tmp.isReg())


    buildLiveRanges(*m_liveness);

    m_code.forEachTmp([&] (Tmp tmp) {
        ASSERT(!tmp.isReg());
        if (size_t liveRangeEnd = m_liveRangeEnd[tmp])
            m_tmpsToRelease.add(liveRangeEnd + 1, Vector<Tmp, 2>()).iterator->value.append(tmp);
    });

    IndexMap<BasicBlock*, IndexMap<Reg, Tmp>> currentAllocationMap(m_code.size());
    {
        IndexMap<Reg, Tmp> defaultCurrentAllocation(Reg::maxIndex() + 1);


    Disassembler* disassembler = m_code.disassembler();

    m_globalInstIndex = 1;

    for (BasicBlock* block : m_code) {
        context.currentBlock = block;


        ++m_globalInstIndex;

        freeDeadTmpsAtCurrentBlock();

        bool isReplayingSameInst = false;
        for (size_t instIndex = 0; instIndex < block->size(); ++instIndex) {
            checkConsistency();


            Inst& inst = block->at(instIndex);

            m_didAlreadyFreeDeadSlots = false;

            m_namedUsedRegs = RegisterSet();
            m_namedDefdRegs = RegisterSet();
            m_earlyClobber = RegisterSet();
            m_lateClobber = RegisterSet();
            m_clobberedToClear = RegisterSet();

            bool needsToGenerate = ([&] () -> bool {
                // FIXME: We should consider trying to figure out if we can also elide Mov32s

                }
            });

            if (inst.kind.opcode == Patch) {
                m_earlyClobber.merge(inst.extraEarlyClobberedRegs());
                m_lateClobber.merge(inst.extraClobberedRegs());

                m_earlyClobber.filter(m_allowedRegisters);
                m_lateClobber.filter(m_allowedRegisters);

                m_clobberedToClear.merge(m_earlyClobber);
                m_clobberedToClear.merge(m_lateClobber);
                m_clobberedToClear.exclude(m_namedDefdRegs);
                    clobberedRegisters.filter(m_allowedRegisters);
                    clobberedRegisters.exclude(m_namedDefdRegs);
                    earlyNextClobberedRegisters.filter(m_allowedRegisters);

                    m_namedDefdRegs.merge(clobberedRegisters);
                }
            }

            auto allocNamed = [&] (const RegisterSet& named, Arg::Role role) {
                for (Reg reg : named) {
                    if (Tmp occupyingTmp = currentAllocation[reg]) {
                        if (occupyingTmp == Tmp(reg))
                            continue;
                    }

                    alloc(Tmp(reg), reg, role);
                    alloc(Tmp(reg), reg, isDef);
                }
            };

            auto spillIfNeeded = [&] (const RegisterSet& set) {
                for (Reg reg : set) {
                    if (Tmp tmp = m_currentAllocation->at(reg))
                        spill(tmp, reg);
                }
            };

            freeDeadTmpsAtCurrentInst();

            spillIfNeeded(m_earlyClobber);
            spillIfNeeded(m_lateClobber);

            allocNamed(m_namedUsedRegs, Arg::Role::Use); // Must come before the defd registers since we may use and def the same register.
            allocNamed(m_namedDefdRegs, Arg::Role::Def);

            if (needsToGenerate) {
                auto tryAllocate = [&] {
                    Vector<std::pair<Tmp*, Arg::Role>, 8> usesToAlloc;
                    Vector<std::pair<Tmp*, Arg::Role>, 8> defsToAlloc;

                    inst.forEachTmp([&] (Tmp& tmp, Arg::Role role, Bank, Width) {
                        if (tmp.isReg())


                        // We treat Use+Def as a use.
                        if (Arg::isAnyUse(role))
                            usesToAlloc.append(std::make_pair(&tmp, role));
                        else if (Arg::isAnyDef(role))
                            defsToAlloc.append(std::make_pair(&tmp, role));
                    });

                    auto tryAllocateTmps = [&] (auto& vector) {
                        bool success = true;
                        for (std::pair<Tmp*, Arg::Role> pair : vector) {
                            Tmp& tmp = *std::get<0>(pair);
                            Arg::Role role = std::get<1>(pair);
                            success &= assignTmp(tmp, tmp.bank(), role);
                        }
                        return success;
                    };


                    // some tmps may both be used and defd. So we handle uses first since forEachTmp could
                    // walk uses/defs in any order.
                    bool success = true;
                    success &= tryAllocateTmps(usesToAlloc);
                    success &= tryAllocateTmps(defsToAlloc);
                    return success;
                };


            }

            if (m_code.needsUsedRegisters() && inst.kind.opcode == Patch) {
                freeDeadTmpsIfNeeded();
                RegisterSet registerSet;
                for (size_t i = 0; i < currentAllocation.size(); ++i) {
                    if (currentAllocation[i])

                inst.reportUsedRegisters(registerSet);
            }

            auto handleClobber = [&] {
                for (Reg reg : m_clobberedToClear) {
                    if (Tmp tmp = m_currentAllocation->at(reg))
                        release(tmp, reg);
                    m_availableRegs[tmp.bank()].set(reg);
                    m_currentAllocation->at(reg) = Tmp();
                    m_map[tmp].reg = Reg();
                }
            };


                    jump = inst.generate(*m_jit, context);
                ASSERT_UNUSED(jump, !jump.isSet());

                handleClobber();
                clobberedRegisters.merge(earlyNextClobberedRegisters);
                clobber(clobberedRegisters);
            } else {
                ASSERT(needsToGenerate);

                handleClobber();
                ASSERT(earlyNextClobberedRegisters.isEmpty());

                if (block->numSuccessors()) {
                    // By default, we spill everything between block boundaries. However, we have a small



    void release(Tmp, Reg);
    void flush(Tmp, Reg);
    void spill(Tmp, Reg);
    void alloc(Tmp, Reg, Arg::Role);
    void freeDeadTmpsAtCurrentInst();
    void freeDeadTmpsAtCurrentBlock();
    bool assignTmp(Tmp&, Bank, Arg::Role);
    void buildLiveRanges(UnifiedTmpLiveness&);
    bool isDisallowedRegister(Reg);


    RegisterSet m_availableRegs[numBanks];
    size_t m_globalInstIndex;
    IndexMap<Reg, Tmp>* m_currentAllocation { nullptr };
    bool m_didAlreadyFreeDeadSlots;
    TmpMap<size_t> m_liveRangeEnd;
    HashMap<size_t, Vector<Tmp, 2>> m_tmpsToRelease;
    RegisterSet m_namedUsedRegs;
    RegisterSet m_namedDefdRegs;
    RegisterSet m_earlyClobber;
    RegisterSet m_lateClobber;
    RegisterSet m_clobberedToClear;
    RegisterSet m_allowedRegisters;
    std::unique_ptr<UnifiedTmpLiveness> m_liveness;




    CHECK(result == static_cast<int32_t>(numberOfSlots));
}

void testEarlyAndLateUseOfSameTmp()
{
    WeakRandom weakRandom;
    size_t numTmps = RegisterSet::allGPRs().numberOfSetRegisters();
    int64_t expectedResult = 0;
    for (size_t i = 0; i < numTmps; ++i)
        expectedResult += i;

    for (unsigned j = 0; j < 60; ++j) {
        B3::Procedure proc;
        Code& code = proc.code();

        B3::Air::Special* patchpointSpecial = code.addSpecial(makeUnique<B3::PatchpointSpecial>());

        BasicBlock* root = code.addBlock();
        Vector<Tmp> tmps;
        Tmp result = code.newTmp(B3::GP);
        root->append(Move, nullptr, Arg::imm(0), result);
        for (size_t i = 0; i < numTmps; ++i) {
            Tmp tmp = code.newTmp(B3::GP);
            tmps.append(tmp);
            root->append(Move, nullptr, Arg::imm(i), tmp);
        }

        {
            unsigned rand = weakRandom.getUint32(tmps.size());
            B3::Value* dummyValue = proc.addConstant(B3::Origin(), B3::Int64, 0);

            B3::PatchpointValue* patchpoint = proc.add<B3::PatchpointValue>(B3::Void, B3::Origin());
            patchpoint->append(dummyValue, B3::ValueRep::SomeRegister);
            patchpoint->append(dummyValue, B3::ValueRep::SomeLateRegister);
            patchpoint->clobberLate(RegisterSet::volatileRegistersForJSCall());
            patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
                RELEASE_ASSERT(!RegisterSet::volatileRegistersForJSCall().get(params[1].gpr()));

                auto good = jit.branch64(CCallHelpers::Equal, params[1].gpr(), CCallHelpers::TrustedImm32(rand));
                jit.breakpoint();
                good.link(&jit);

                auto good2 = jit.branch64(CCallHelpers::Equal, params[0].gpr(), CCallHelpers::TrustedImm32(rand));
                jit.breakpoint();
                good2.link(&jit);
            });

            Inst inst(Patch, patchpoint, Arg::special(patchpointSpecial));

            Tmp tmp = tmps[rand];
            inst.args.append(tmp);
            inst.args.append(tmp);
            root->append(inst);
        }

        for (Tmp tmp : tmps)
            root->append(Add64, nullptr, tmp, result);
        root->append(Move, nullptr, result, Tmp(GPRInfo::returnValueGPR));
        root->append(Ret64, nullptr, Tmp(GPRInfo::returnValueGPR));

        int64_t actualResult = compileAndRun<int64_t>(proc);
        CHECK(actualResult == expectedResult);
    }
}

void testEarlyClobberInterference()
{
    WeakRandom weakRandom;
    size_t numTmps = RegisterSet::allGPRs().numberOfSetRegisters();
    int64_t expectedResult = 0;
    for (size_t i = 0; i < numTmps; ++i)
        expectedResult += i;

    for (unsigned j = 0; j < 100; ++j) {
        B3::Procedure proc;
        Code& code = proc.code();

        B3::Air::Special* patchpointSpecial = code.addSpecial(makeUnique<B3::PatchpointSpecial>());

        BasicBlock* root = code.addBlock();
        Vector<Tmp> tmps;
        Tmp result = code.newTmp(B3::GP);
        root->append(Move, nullptr, Arg::imm(0), result);
        for (size_t i = 0; i < numTmps; ++i) {
            Tmp tmp = code.newTmp(B3::GP);
            tmps.append(tmp);
            root->append(Move, nullptr, Arg::imm(i), tmp);
        }

        {
            unsigned rand = weakRandom.getUint32(tmps.size());
            B3::Value* dummyValue = proc.addConstant(B3::Origin(), B3::Int64, 0);

            B3::PatchpointValue* patchpoint = proc.add<B3::PatchpointValue>(B3::Void, B3::Origin());
            patchpoint->append(dummyValue, B3::ValueRep::SomeRegister);
            patchpoint->clobberEarly(RegisterSet::volatileRegistersForJSCall());
            patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
                RELEASE_ASSERT(!RegisterSet::volatileRegistersForJSCall().get(params[0].gpr()));

                auto good = jit.branch64(CCallHelpers::Equal, params[0].gpr(), CCallHelpers::TrustedImm32(rand));
                jit.breakpoint();
                good.link(&jit);
            });

            Inst inst(Patch, patchpoint, Arg::special(patchpointSpecial));

            Tmp tmp = tmps[rand];
            inst.args.append(tmp);
            root->append(inst);
        }

        for (Tmp tmp : tmps)
            root->append(Add64, nullptr, tmp, result);
        root->append(Move, nullptr, result, Tmp(GPRInfo::returnValueGPR));
        root->append(Ret64, nullptr, Tmp(GPRInfo::returnValueGPR));

        int64_t actualResult = compileAndRun<int64_t>(proc);
        CHECK(actualResult == expectedResult);
    }
}

#define PREFIX "O", Options::defaultB3OptLevel(), ": "

#define RUN(test) do {                                 \


    RUN(testZDefOfSpillSlotWithOffsetNeedingToBeMaterializedInARegister());

    RUN(testEarlyAndLateUseOfSameTmp());
    RUN(testEarlyClobberInterference());

    if (tasks.isEmpty())
        usage();


Lines 60-66 public: Source/JavaScriptCore/jit/RegisterSet.h_sec1
- Source/JavaScriptCore/jit/RegisterSet.h -1 / +1 lines
60	#if ENABLE(WEBASSEMBLY)	60	#if ENABLE(WEBASSEMBLY)
61	static RegisterSet webAssemblyCalleeSaveRegisters(); // Registers saved and used by the WebAssembly JIT.	61	static RegisterSet webAssemblyCalleeSaveRegisters(); // Registers saved and used by the WebAssembly JIT.
62	#endif	62	#endif
63	static RegisterSet volatileRegistersForJSCall();	63	JS_EXPORT_PRIVATE static RegisterSet volatileRegistersForJSCall();
64	static RegisterSet stubUnavailableRegisters(); // The union of callee saves and special registers.	64	static RegisterSet stubUnavailableRegisters(); // The union of callee saves and special registers.
65	JS_EXPORT_PRIVATE static RegisterSet macroScratchRegisters();	65	JS_EXPORT_PRIVATE static RegisterSet macroScratchRegisters();
66	JS_EXPORT_PRIVATE static RegisterSet allGPRs();	66	JS_EXPORT_PRIVATE static RegisterSet allGPRs();

Return to Bug 240205

Lines 2392-2397 void testZDefOfSpillSlotWithOffsetNeedin Source/JavaScriptCore/b3/air/testair.cpp_sec1
- Source/JavaScriptCore/b3/air/testair.cpp +121 lines
2392	CHECK(result == static_cast<int32_t>(numberOfSlots));	2392	CHECK(result == static_cast<int32_t>(numberOfSlots));
2393	}	2393	}
2394		2394
		2395	void testEarlyAndLateUseOfSameTmp()
		2396	{
		2397	WeakRandom weakRandom;
		2398	size_t numTmps = RegisterSet::allGPRs().numberOfSetRegisters();
		2399	int64_t expectedResult = 0;
		2400	for (size_t i = 0; i < numTmps; ++i)
		2401	expectedResult += i;
		2402
		2403	for (unsigned j = 0; j < 60; ++j) {
		2404	B3::Procedure proc;
		2405	Code& code = proc.code();
		2406
		2407	B3::Air::Special* patchpointSpecial = code.addSpecial(makeUnique<B3::PatchpointSpecial>());
		2408
		2409	BasicBlock* root = code.addBlock();
		2410	Vector<Tmp> tmps;
		2411	Tmp result = code.newTmp(B3::GP);
		2412	root->append(Move, nullptr, Arg::imm(0), result);
		2413	for (size_t i = 0; i < numTmps; ++i) {
		2414	Tmp tmp = code.newTmp(B3::GP);
		2415	tmps.append(tmp);
		2416	root->append(Move, nullptr, Arg::imm(i), tmp);
		2417	}
		2418
		2419	{
		2420	unsigned rand = weakRandom.getUint32(tmps.size());
		2421	B3::Value* dummyValue = proc.addConstant(B3::Origin(), B3::Int64, 0);
		2422
		2423	B3::PatchpointValue* patchpoint = proc.add<B3::PatchpointValue>(B3::Void, B3::Origin());
		2424	patchpoint->append(dummyValue, B3::ValueRep::SomeRegister);
		2425	patchpoint->append(dummyValue, B3::ValueRep::SomeLateRegister);
		2426	patchpoint->clobberLate(RegisterSet::volatileRegistersForJSCall());
		2427	patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
		2428	RELEASE_ASSERT(!RegisterSet::volatileRegistersForJSCall().get(params[1].gpr()));
		2429
		2430	auto good = jit.branch64(CCallHelpers::Equal, params[1].gpr(), CCallHelpers::TrustedImm32(rand));
		2431	jit.breakpoint();
		2432	good.link(&jit);
		2433
		2434	auto good2 = jit.branch64(CCallHelpers::Equal, params[0].gpr(), CCallHelpers::TrustedImm32(rand));
		2435	jit.breakpoint();
		2436	good2.link(&jit);
		2437	});
		2438
		2439	Inst inst(Patch, patchpoint, Arg::special(patchpointSpecial));
		2440
		2441	Tmp tmp = tmps[rand];
		2442	inst.args.append(tmp);
		2443	inst.args.append(tmp);
		2444	root->append(inst);
		2445	}
		2446
		2447	for (Tmp tmp : tmps)
		2448	root->append(Add64, nullptr, tmp, result);
		2449	root->append(Move, nullptr, result, Tmp(GPRInfo::returnValueGPR));
		2450	root->append(Ret64, nullptr, Tmp(GPRInfo::returnValueGPR));
		2451
		2452	int64_t actualResult = compileAndRun<int64_t>(proc);
		2453	CHECK(actualResult == expectedResult);
		2454	}
		2455	}
		2456
		2457	void testEarlyClobberInterference()
		2458	{
2459	WeakRandom weakRandom;
2460	size_t numTmps = RegisterSet::allGPRs().numberOfSetRegisters();
2461	int64_t expectedResult = 0;
2462	for (size_t i = 0; i < numTmps; ++i)
2463	expectedResult += i;
2464
2465	for (unsigned j = 0; j < 100; ++j) {
2466	B3::Procedure proc;
2467	Code& code = proc.code();
2468
2469	B3::Air::Special* patchpointSpecial = code.addSpecial(makeUnique<B3::PatchpointSpecial>());
2470
2471	BasicBlock* root = code.addBlock();
2472	Vector<Tmp> tmps;
2473	Tmp result = code.newTmp(B3::GP);
2474	root->append(Move, nullptr, Arg::imm(0), result);
2475	for (size_t i = 0; i < numTmps; ++i) {
2476	Tmp tmp = code.newTmp(B3::GP);
2477	tmps.append(tmp);
2478	root->append(Move, nullptr, Arg::imm(i), tmp);
2479	}
2480
2481	{
2482	unsigned rand = weakRandom.getUint32(tmps.size());
2483	B3::Value* dummyValue = proc.addConstant(B3::Origin(), B3::Int64, 0);
2484
2485	B3::PatchpointValue* patchpoint = proc.add<B3::PatchpointValue>(B3::Void, B3::Origin());
2486	patchpoint->append(dummyValue, B3::ValueRep::SomeRegister);
2487	patchpoint->clobberEarly(RegisterSet::volatileRegistersForJSCall());
2488	patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2489	RELEASE_ASSERT(!RegisterSet::volatileRegistersForJSCall().get(params[0].gpr()));
2490
2491	auto good = jit.branch64(CCallHelpers::Equal, params[0].gpr(), CCallHelpers::TrustedImm32(rand));
2492	jit.breakpoint();
2493	good.link(&jit);
2494	});
2495
2496	Inst inst(Patch, patchpoint, Arg::special(patchpointSpecial));
2497
2498	Tmp tmp = tmps[rand];
2499	inst.args.append(tmp);
2500	root->append(inst);
2501	}
2502
2503	for (Tmp tmp : tmps)
2504	root->append(Add64, nullptr, tmp, result);
2505	root->append(Move, nullptr, result, Tmp(GPRInfo::returnValueGPR));
2506	root->append(Ret64, nullptr, Tmp(GPRInfo::returnValueGPR));
2507
2508	int64_t actualResult = compileAndRun<int64_t>(proc);
2509	CHECK(actualResult == expectedResult);
2510	}
2511	}
2512
2395	#define PREFIX "O", Options::defaultB3OptLevel(), ": "	2513	#define PREFIX "O", Options::defaultB3OptLevel(), ": "
2396		2514
2397	#define RUN(test) do { \	2515	#define RUN(test) do { \
Lines 2489-2494 void run(const char* filter) Source/JavaScriptCore/b3/air/testair.cpp_sec2
2489		2607
2490	RUN(testZDefOfSpillSlotWithOffsetNeedingToBeMaterializedInARegister());	2608	RUN(testZDefOfSpillSlotWithOffsetNeedingToBeMaterializedInARegister());
2491		2609
		2610	RUN(testEarlyAndLateUseOfSameTmp());
		2611	RUN(testEarlyClobberInterference());
		2612
2492	if (tasks.isEmpty())	2613	if (tasks.isEmpty())
2493	usage();	2614	usage();
2494		2615

Lines 1-3 Source/JavaScriptCore/ChangeLog_sec1
- Source/JavaScriptCore/ChangeLog +31 lines
		1	2022-05-08 Saam Barati <sbarati@apple.com>
		2
		3	Better handle clobbered registers in O0 register allocation
		4	https://bugs.webkit.org/show_bug.cgi?id=240205
		5	<rdar://87220688>
		6
		7	Reviewed by NOBODY (OOPS!).
		8
		9	This patch makes Air's O0 register allocator better handle clobbered
		10	registers. We now model both early and late clobber directly, and use
		11	this to perform a basic interference analysis when allocating a register
		12	to a Tmp. An early clobber interferes with any Use in an instruction, and
		13	any late Defs. A late clobber interferes with any Defs in an instruction,
		14	and any late Uses. What this enables is an early Use can be allocated
		15	to a register that is only late clobbered. And a result can be allocated
		16	to a register that is only early clobbered.
		17
		18	Prior to this, the algorithm had a bug where a Use may be allocated to
		19	a register that is early clobbered.
		20
		21	* b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp:
		22	(JSC::B3::Air::GenerateAndAllocateRegisters::buildLiveRanges):
		23	(JSC::B3::Air::GenerateAndAllocateRegisters::alloc):
		24	(JSC::B3::Air::GenerateAndAllocateRegisters::freeDeadTmpsIfNeeded):
		25	(JSC::B3::Air::GenerateAndAllocateRegisters::assignTmp):
		26	(JSC::B3::Air::GenerateAndAllocateRegisters::prepareForGeneration):
		27	(JSC::B3::Air::GenerateAndAllocateRegisters::generate):
		28	* b3/air/AirAllocateRegistersAndStackAndGenerateCode.h:
		29	* b3/air/testair.cpp:
		30	* jit/RegisterSet.h:
		31
1	2022-05-06 Ross Kirsling <ross.kirsling@sony.com>	32	2022-05-06 Ross Kirsling <ross.kirsling@sony.com>
2		33
3	Temporal.Duration#toString should never ignore fractionalSecondDigits	34	Temporal.Duration#toString should never ignore fractionalSecondDigits

Lines 78-84 void GenerateAndAllocateRegisters::build Source/JavaScriptCore/b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp_sec1
- Source/JavaScriptCore/b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp -73 / +130 lines
78	{	78	{
79	m_liveRangeEnd = TmpMap<size_t>(m_code, 0);	79	m_liveRangeEnd = TmpMap<size_t>(m_code, 0);
80		80
81	m_globalInstIndex = 0;	81	m_globalInstIndex = 1;
82	for (BasicBlock* block : m_code) {	82	for (BasicBlock* block : m_code) {
83	for (Tmp tmp : liveness.liveAtHead(block)) {	83	for (Tmp tmp : liveness.liveAtHead(block)) {
84	if (!tmp.isReg())	84	if (!tmp.isReg())
Lines 180-190 ALWAYS_INLINE void GenerateAndAllocateRe Source/JavaScriptCore/b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp_sec2
180	{	180	{
181	ASSERT(reg);	181	ASSERT(reg);
182	ASSERT(m_map[tmp].reg == reg);	182	ASSERT(m_map[tmp].reg == reg);
		183	ASSERT(tmp.isReg() \|\| m_liveRangeEnd[tmp] >= m_globalInstIndex);
183	flush(tmp, reg);	184	flush(tmp, reg);
184	release(tmp, reg);	185	release(tmp, reg);
185	}	186	}
186		187
187	ALWAYS_INLINE void GenerateAndAllocateRegisters::alloc(Tmp tmp, Reg reg, bool isDef)	188	ALWAYS_INLINE void GenerateAndAllocateRegisters::alloc(Tmp tmp, Reg reg, Arg::Role role)
188	{	189	{
189	if (Tmp occupyingTmp = m_currentAllocation->at(reg))	190	if (Tmp occupyingTmp = m_currentAllocation->at(reg))
190	spill(occupyingTmp, reg);	191	spill(occupyingTmp, reg);
Lines 197-203 ALWAYS_INLINE void GenerateAndAllocateRe Source/JavaScriptCore/b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp_sec3
197	m_availableRegs[tmp.bank()].clear(reg);	198	m_availableRegs[tmp.bank()].clear(reg);
198	m_currentAllocation->at(reg) = tmp;	199	m_currentAllocation->at(reg) = tmp;
199		200
200	if (!isDef) {	201	if (Arg::isAnyUse(role)) {
201	intptr_t offset = m_map[tmp].spillSlot->offsetFromFP();	202	intptr_t offset = m_map[tmp].spillSlot->offsetFromFP();
202	if (tmp.bank() == GP)	203	if (tmp.bank() == GP)
203	m_jit->load64(callFrameAddr(*m_jit, offset), reg.gpr());	204	m_jit->load64(callFrameAddr(*m_jit, offset), reg.gpr());
Lines 206-217 ALWAYS_INLINE void GenerateAndAllocateRe Source/JavaScriptCore/b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp_sec4
206	}	207	}
207	}	208	}
208		209
209	ALWAYS_INLINE void GenerateAndAllocateRegisters::freeDeadTmpsIfNeeded()	210	// freeDeadTmpsAtCurrentInst and freeDeadTmpsAtCurrentBlock are needed for correctness because
210	{	211	// we reuse stack slots between Tmps that don't interfere. So we need to make sure we don't
211	if (m_didAlreadyFreeDeadSlots)	212	// spill a dead Tmp to a live Tmp's slot.
212	return;	213	// freeDeadTmpsAtCurrentInst is meant to be called as we walk through each instruction in a basic block
		214	// because it doesn't consult the entire register file, and is faster than freeDeadTmpsAtCurrentBlock.
		215	// However, it only prunes things that die at a particular inst index within a block, and doesn't prevent
		216	// us from propagating a Tmp that is live in one block to the head of a block where it is dead. If
		217	// something dies within a block, freeDeadTmpsAtCurrentInst will catch it. freeDeadTmpsAtCurrentBlock is
		218	// meant to ensure we prune away any Tmps that are dead at the head of a block.
		219	ALWAYS_INLINE void GenerateAndAllocateRegisters::freeDeadTmpsAtCurrentInst()
		220	{
		221	auto iter = m_tmpsToRelease.find(m_globalInstIndex);
		222	if (iter != m_tmpsToRelease.end()) {
		223	for (Tmp tmp : iter->value) {
		224	ASSERT(m_liveRangeEnd[tmp] < m_globalInstIndex);
		225	if (Reg reg = m_map[tmp].reg)
		226	release(tmp, reg);
		227	}
		228	}
		229	}
213		230
214	m_didAlreadyFreeDeadSlots = true;	231	ALWAYS_INLINE void GenerateAndAllocateRegisters::freeDeadTmpsAtCurrentBlock()
		232	{
215	for (size_t i = 0; i < m_currentAllocation->size(); ++i) {	233	for (size_t i = 0; i < m_currentAllocation->size(); ++i) {
216	Tmp tmp = m_currentAllocation->at(i);	234	Tmp tmp = m_currentAllocation->at(i);
217	if (!tmp)	235	if (!tmp)
Lines 225-268 ALWAYS_INLINE void GenerateAndAllocateRe Source/JavaScriptCore/b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp_sec5
225	}	243	}
226	}	244	}
227		245
228	ALWAYS_INLINE bool GenerateAndAllocateRegisters::assignTmp(Tmp& tmp, Bank bank, bool isDef)	246	ALWAYS_INLINE bool GenerateAndAllocateRegisters::assignTmp(Tmp& tmp, Bank bank, Arg::Role role)
229	{	247	{
230	ASSERT(!tmp.isReg());	248	ASSERT(!tmp.isReg());
231	if (Reg reg = m_map[tmp].reg) {	249
232	ASSERT(!m_namedDefdRegs.contains(reg));	250	auto markRegisterAsUsed = [&] (Reg reg) {
233	tmp = Tmp(reg);	251	if (Arg::isAnyDef(role))
		252	m_clobberedToClear.clear(reg);
		253	// At this point, it doesn't matter if we add it to the m_namedUsedRegs or m_namedDefdRegs.
		254	// We just need to mark that we can't use it again for another tmp.
234	m_namedUsedRegs.set(reg);	255	m_namedUsedRegs.set(reg);
235	ASSERT(!m_availableRegs[bank].get(reg));	256	};
236	return true;	257
237	}	258	bool mightInterfere = m_earlyClobber.numberOfSetRegisters() \|\| m_lateClobber.numberOfSetRegisters();
		259
		260	auto interferesWithClobber = [&] (Reg reg) {
		261	if (!mightInterfere)
		262	return false;
		263	if (Arg::isAnyUse(role) && m_earlyClobber.get(reg))
		264	return true;
		265	if (Arg::isAnyDef(role) && m_lateClobber.get(reg))
		266	return true;
		267	if (Arg::activeAt(role, Arg::Phase::Early) && m_earlyClobber.get(reg))
		268	return true;
		269	if (Arg::activeAt(role, Arg::Phase::Late) && m_lateClobber.get(reg))
		270	return true;
		271	return false;
		272	};
		273
		274	if (Reg reg = m_map[tmp].reg) {
		275	if (!interferesWithClobber(reg)) {
		276	ASSERT(!m_namedDefdRegs.contains(reg));
		277	tmp = Tmp(reg);
		278	markRegisterAsUsed(reg);
		279	ASSERT(!m_availableRegs[bank].get(reg));
		280	return true;
		281	}
		282	// This is a rare case when we've already allocated a Tmp in some way, but another
		283	// Role of the Tmp imposes some restriction on the register value. E.g, if
		284	// we have a program like:
		285	// Patch Use:tmp1, LateUse:tmp1, lateClobber:x0
		286	// The first use of tmp1 can be allocated to x0, but the second cannot.
		287	spill(tmp, reg);
238		288
239	if (!m_availableRegs[bank].numberOfSetRegisters())	289	}
240	freeDeadTmpsIfNeeded();
241		290
242	if (m_availableRegs[bank].numberOfSetRegisters()) {	291	if (m_availableRegs[bank].numberOfSetRegisters()) {
243	// We first take an available register.	292	// We first take an available register.
244	for (Reg reg : m_registers[bank]) {	293	for (Reg reg : m_registers[bank]) {
245	if (m_namedUsedRegs.contains(reg) \|\| m_namedDefdRegs.contains(reg))	294	if (interferesWithClobber(reg) \|\| m_namedUsedRegs.contains(reg) \|\| m_namedDefdRegs.contains(reg))
246	continue;	295	continue;
247	if (!m_availableRegs[bank].contains(reg))	296	if (!m_availableRegs[bank].contains(reg))
248	continue;	297	continue;
249	m_namedUsedRegs.set(reg); // At this point, it doesn't matter if we add it to the m_namedUsedRegs or m_namedDefdRegs. We just need to mark that we can't use it again.	298
250	alloc(tmp, reg, isDef);	299	markRegisterAsUsed(reg);
		300	alloc(tmp, reg, role);
251	tmp = Tmp(reg);	301	tmp = Tmp(reg);
252	return true;	302	return true;
253	}	303	}
254
255	RELEASE_ASSERT_NOT_REACHED();
256	}	304	}
257		305
258	// Nothing was available, let's make some room.	306	// Nothing was available, let's make some room.
259	for (Reg reg : m_registers[bank]) {	307	for (Reg reg : m_registers[bank]) {
260	if (m_namedUsedRegs.contains(reg) \|\| m_namedDefdRegs.contains(reg))	308	if (interferesWithClobber(reg) \|\| m_namedUsedRegs.contains(reg) \|\| m_namedDefdRegs.contains(reg))
261	continue;	309	continue;
262		310
263	m_namedUsedRegs.set(reg);	311	markRegisterAsUsed(reg);
264		312	alloc(tmp, reg, role);
265	alloc(tmp, reg, isDef);
266	tmp = Tmp(reg);	313	tmp = Tmp(reg);
267	return true;	314	return true;
268	}	315	}
Lines 298-304 void GenerateAndAllocateRegisters::prepa Source/JavaScriptCore/b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp_sec6
298		345
299	Vector<StackSlot*, 16> freeSlots;	346	Vector<StackSlot*, 16> freeSlots;
300	Vector<StackSlot*, 4> toFree;	347	Vector<StackSlot*, 4> toFree;
301	m_globalInstIndex = 0;	348	m_globalInstIndex = 1;
302	for (BasicBlock* block : m_code) {	349	for (BasicBlock* block : m_code) {
303	auto assignStackSlotToTmp = [&] (Tmp tmp) {	350	auto assignStackSlotToTmp = [&] (Tmp tmp) {
304	if (tmp.isReg())	351	if (tmp.isReg())
Lines 424-429 void GenerateAndAllocateRegisters::gener Source/JavaScriptCore/b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp_sec7
424		471
425	buildLiveRanges(*m_liveness);	472	buildLiveRanges(*m_liveness);
426		473
		474	m_code.forEachTmp([&] (Tmp tmp) {
		475	ASSERT(!tmp.isReg());
		476	if (size_t liveRangeEnd = m_liveRangeEnd[tmp])
		477	m_tmpsToRelease.add(liveRangeEnd + 1, Vector<Tmp, 2>()).iterator->value.append(tmp);
		478	});
		479
427	IndexMap<BasicBlock*, IndexMap<Reg, Tmp>> currentAllocationMap(m_code.size());	480	IndexMap<BasicBlock*, IndexMap<Reg, Tmp>> currentAllocationMap(m_code.size());
428	{	481	{
429	IndexMap<Reg, Tmp> defaultCurrentAllocation(Reg::maxIndex() + 1);	482	IndexMap<Reg, Tmp> defaultCurrentAllocation(Reg::maxIndex() + 1);
Lines 461-467 void GenerateAndAllocateRegisters::gener Source/JavaScriptCore/b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp_sec8
461		514
462	Disassembler* disassembler = m_code.disassembler();	515	Disassembler* disassembler = m_code.disassembler();
463		516
464	m_globalInstIndex = 0;	517	m_globalInstIndex = 1;
465		518
466	for (BasicBlock* block : m_code) {	519	for (BasicBlock* block : m_code) {
467	context.currentBlock = block;	520	context.currentBlock = block;
Lines 525-530 void GenerateAndAllocateRegisters::gener Source/JavaScriptCore/b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp_sec9
525		578
526	++m_globalInstIndex;	579	++m_globalInstIndex;
527		580
		581	freeDeadTmpsAtCurrentBlock();
		582
528	bool isReplayingSameInst = false;	583	bool isReplayingSameInst = false;
529	for (size_t instIndex = 0; instIndex < block->size(); ++instIndex) {	584	for (size_t instIndex = 0; instIndex < block->size(); ++instIndex) {
530	checkConsistency();	585	checkConsistency();
Lines 536-545 void GenerateAndAllocateRegisters::gener Source/JavaScriptCore/b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp_sec10
536		591
537	Inst& inst = block->at(instIndex);	592	Inst& inst = block->at(instIndex);
538		593
539	m_didAlreadyFreeDeadSlots = false;
540
541	m_namedUsedRegs = RegisterSet();	594	m_namedUsedRegs = RegisterSet();
542	m_namedDefdRegs = RegisterSet();	595	m_namedDefdRegs = RegisterSet();
		596	m_earlyClobber = RegisterSet();
		597	m_lateClobber = RegisterSet();
		598	m_clobberedToClear = RegisterSet();
543		599
544	bool needsToGenerate = ([&] () -> bool {	600	bool needsToGenerate = ([&] () -> bool {
545	// FIXME: We should consider trying to figure out if we can also elide Mov32s	601	// FIXME: We should consider trying to figure out if we can also elide Mov32s
Lines 614-656 void GenerateAndAllocateRegisters::gener Source/JavaScriptCore/b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp_sec11
614	}	670	}
615	});	671	});
616		672
617	RegisterSet clobberedRegisters;	673	if (inst.kind.opcode == Patch) {
618	RegisterSet earlyNextClobberedRegisters;	674	m_earlyClobber.merge(inst.extraEarlyClobberedRegs());
619	{	675	m_lateClobber.merge(inst.extraClobberedRegs());
620	Inst* nextInst = block->get(instIndex + 1);	676
621	if (inst.kind.opcode == Patch \|\| (nextInst && nextInst->kind.opcode == Patch)) {	677	m_earlyClobber.filter(m_allowedRegisters);
622	if (inst.kind.opcode == Patch)	678	m_lateClobber.filter(m_allowedRegisters);
623	clobberedRegisters.merge(inst.extraClobberedRegs());	679
624	if (nextInst && nextInst->kind.opcode == Patch)	680	m_clobberedToClear.merge(m_earlyClobber);
625	earlyNextClobberedRegisters.merge(nextInst->extraEarlyClobberedRegs());	681	m_clobberedToClear.merge(m_lateClobber);
626		682	m_clobberedToClear.exclude(m_namedDefdRegs);
627	clobberedRegisters.filter(m_allowedRegisters);
628	clobberedRegisters.exclude(m_namedDefdRegs);
629	earlyNextClobberedRegisters.filter(m_allowedRegisters);
630
631	m_namedDefdRegs.merge(clobberedRegisters);
632	}
633	}	683	}
634		684
635	auto allocNamed = [&] (const RegisterSet& named, bool isDef) {	685	auto allocNamed = [&] (const RegisterSet& named, Arg::Role role) {
636	for (Reg reg : named) {	686	for (Reg reg : named) {
637	if (Tmp occupyingTmp = currentAllocation[reg]) {	687	if (Tmp occupyingTmp = currentAllocation[reg]) {
638	if (occupyingTmp == Tmp(reg))	688	if (occupyingTmp == Tmp(reg))
639	continue;	689	continue;
640	}	690	}
641		691
642	freeDeadTmpsIfNeeded(); // We don't want to spill a dead tmp.	692	alloc(Tmp(reg), reg, role);
643	alloc(Tmp(reg), reg, isDef);
644	}	693	}
645	};	694	};
646		695
647	allocNamed(m_namedUsedRegs, false); // Must come before the defd registers since we may use and def the same register.	696	auto spillIfNeeded = [&] (const RegisterSet& set) {
648	allocNamed(m_namedDefdRegs, true);	697	for (Reg reg : set) {
		698	if (Tmp tmp = m_currentAllocation->at(reg))
		699	spill(tmp, reg);
		700	}
		701	};
		702
		703	freeDeadTmpsAtCurrentInst();
		704
		705	spillIfNeeded(m_earlyClobber);
		706	spillIfNeeded(m_lateClobber);
		707
		708	allocNamed(m_namedUsedRegs, Arg::Role::Use); // Must come before the defd registers since we may use and def the same register.
		709	allocNamed(m_namedDefdRegs, Arg::Role::Def);
649		710
650	if (needsToGenerate) {	711	if (needsToGenerate) {
651	auto tryAllocate = [&] {	712	auto tryAllocate = [&] {
652	Vector<Tmp*, 8> usesToAlloc;	713	Vector<std::pair<Tmp*, Arg::Role>, 8> usesToAlloc;
653	Vector<Tmp*, 8> defsToAlloc;	714	Vector<std::pair<Tmp*, Arg::Role>, 8> defsToAlloc;
654		715
655	inst.forEachTmp([&] (Tmp& tmp, Arg::Role role, Bank, Width) {	716	inst.forEachTmp([&] (Tmp& tmp, Arg::Role role, Bank, Width) {
656	if (tmp.isReg())	717	if (tmp.isReg())
Lines 658-672 void GenerateAndAllocateRegisters::gener Source/JavaScriptCore/b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp_sec12
658		719
659	// We treat Use+Def as a use.	720	// We treat Use+Def as a use.
660	if (Arg::isAnyUse(role))	721	if (Arg::isAnyUse(role))
661	usesToAlloc.append(&tmp);	722	usesToAlloc.append(std::make_pair(&tmp, role));
662	else if (Arg::isAnyDef(role))	723	else if (Arg::isAnyDef(role))
663	defsToAlloc.append(&tmp);	724	defsToAlloc.append(std::make_pair(&tmp, role));
664	});	725	});
665		726
666	auto tryAllocateTmps = [&] (auto& vector, bool isDef) {	727	auto tryAllocateTmps = [&] (auto& vector) {
667	bool success = true;	728	bool success = true;
668	for (Tmp* tmp : vector)	729	for (std::pair<Tmp*, Arg::Role> pair : vector) {
669	success &= assignTmp(*tmp, tmp->bank(), isDef);	730	Tmp& tmp = *std::get<0>(pair);
		731	Arg::Role role = std::get<1>(pair);
		732	success &= assignTmp(tmp, tmp.bank(), role);
		733	}
670	return success;	734	return success;
671	};	735	};
672		736
Lines 676-683 void GenerateAndAllocateRegisters::gener Source/JavaScriptCore/b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp_sec13
676	// some tmps may both be used and defd. So we handle uses first since forEachTmp could	740	// some tmps may both be used and defd. So we handle uses first since forEachTmp could
677	// walk uses/defs in any order.	741	// walk uses/defs in any order.
678	bool success = true;	742	bool success = true;
679	success &= tryAllocateTmps(usesToAlloc, false);	743	success &= tryAllocateTmps(usesToAlloc);
680	success &= tryAllocateTmps(defsToAlloc, true);	744	success &= tryAllocateTmps(defsToAlloc);
681	return success;	745	return success;
682	};	746	};
683		747
Lines 729-735 void GenerateAndAllocateRegisters::gener Source/JavaScriptCore/b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp_sec14
729	}	793	}
730		794
731	if (m_code.needsUsedRegisters() && inst.kind.opcode == Patch) {	795	if (m_code.needsUsedRegisters() && inst.kind.opcode == Patch) {
732	freeDeadTmpsIfNeeded();
733	RegisterSet registerSet;	796	RegisterSet registerSet;
734	for (size_t i = 0; i < currentAllocation.size(); ++i) {	797	for (size_t i = 0; i < currentAllocation.size(); ++i) {
735	if (currentAllocation[i])	798	if (currentAllocation[i])
Lines 738-750 void GenerateAndAllocateRegisters::gener Source/JavaScriptCore/b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp_sec15
738	inst.reportUsedRegisters(registerSet);	801	inst.reportUsedRegisters(registerSet);
739	}	802	}
740		803
741	auto clobber = [&] (const RegisterSet& set) {	804	auto handleClobber = [&] {
742	for (Reg reg : set) {	805	for (Reg reg : m_clobberedToClear) {
743	Tmp tmp(reg);	806	if (Tmp tmp = m_currentAllocation->at(reg))
744	ASSERT(currentAllocation[reg] == tmp);	807	release(tmp, reg);
745	m_availableRegs[tmp.bank()].set(reg);
746	m_currentAllocation->at(reg) = Tmp();
747	m_map[tmp].reg = Reg();
748	}	808	}
749	};	809	};
750		810
Lines 754-767 void GenerateAndAllocateRegisters::gener Source/JavaScriptCore/b3/air/AirAllocateRegistersAndStackAndGenerateCode.cpp_sec16
754	jump = inst.generate(*m_jit, context);	814	jump = inst.generate(*m_jit, context);
755	ASSERT_UNUSED(jump, !jump.isSet());	815	ASSERT_UNUSED(jump, !jump.isSet());
756		816
757	allocNamed(earlyNextClobberedRegisters, true);	817	handleClobber();
758	clobberedRegisters.merge(earlyNextClobberedRegisters);
759	clobber(clobberedRegisters);
760	} else {	818	} else {
761	ASSERT(needsToGenerate);	819	ASSERT(needsToGenerate);
762		820
763	clobber(clobberedRegisters);	821	handleClobber();
764	ASSERT(earlyNextClobberedRegisters.isEmpty());
765		822
766	if (block->numSuccessors()) {	823	if (block->numSuccessors()) {
767	// By default, we spill everything between block boundaries. However, we have a small	824	// By default, we spill everything between block boundaries. However, we have a small

Lines 59-67 private: Source/JavaScriptCore/b3/air/AirAllocateRegistersAndStackAndGenerateCode.h_sec1
- Source/JavaScriptCore/b3/air/AirAllocateRegistersAndStackAndGenerateCode.h -4 / +8 lines
59	void release(Tmp, Reg);	59	void release(Tmp, Reg);
60	void flush(Tmp, Reg);	60	void flush(Tmp, Reg);
61	void spill(Tmp, Reg);	61	void spill(Tmp, Reg);
62	void alloc(Tmp, Reg, bool isDef);	62	void alloc(Tmp, Reg, Arg::Role);
63	void freeDeadTmpsIfNeeded();	63	void freeDeadTmpsAtCurrentInst();
64	bool assignTmp(Tmp&, Bank, bool isDef);	64	void freeDeadTmpsAtCurrentBlock();
		65	bool assignTmp(Tmp&, Bank, Arg::Role);
65	void buildLiveRanges(UnifiedTmpLiveness&);	66	void buildLiveRanges(UnifiedTmpLiveness&);
66	bool isDisallowedRegister(Reg);	67	bool isDisallowedRegister(Reg);
67		68
Lines 80-89 private: Source/JavaScriptCore/b3/air/AirAllocateRegistersAndStackAndGenerateCode.h_sec2
80	RegisterSet m_availableRegs[numBanks];	81	RegisterSet m_availableRegs[numBanks];
81	size_t m_globalInstIndex;	82	size_t m_globalInstIndex;
82	IndexMap<Reg, Tmp>* m_currentAllocation { nullptr };	83	IndexMap<Reg, Tmp>* m_currentAllocation { nullptr };
83	bool m_didAlreadyFreeDeadSlots;
84	TmpMap<size_t> m_liveRangeEnd;	84	TmpMap<size_t> m_liveRangeEnd;
		85	HashMap<size_t, Vector<Tmp, 2>> m_tmpsToRelease;
85	RegisterSet m_namedUsedRegs;	86	RegisterSet m_namedUsedRegs;
86	RegisterSet m_namedDefdRegs;	87	RegisterSet m_namedDefdRegs;
		88	RegisterSet m_earlyClobber;
		89	RegisterSet m_lateClobber;
		90	RegisterSet m_clobberedToClear;
87	RegisterSet m_allowedRegisters;	91	RegisterSet m_allowedRegisters;
88	std::unique_ptr<UnifiedTmpLiveness> m_liveness;	92	std::unique_ptr<UnifiedTmpLiveness> m_liveness;
89		93