Commit 0839f93

function shouldBe(actual, expected) {

if (actual !== expected)

throw new Error(`Expected ${expected} but got ${actual}`);

let chain = null;

let expectedLength = 0;

for (let i = 0; i < 300000; ++i) {

const garbage = { index: i, payload: [i, i + 1, i + 2] };

shouldBe(garbage.payload[2], i + 2);

if (!(i % 1000)) {

chain = { index: i, next: chain };

++expectedLength;

}

let length = 0;

for (let node = chain; node; node = node.next) {

++length;

shouldBe(node.index, (expectedLength - length) * 1000);

shouldBe(length, expectedLength);

#include "MarkedBlock.h"

#include "Options.h"

#include "VM.h"

#include <wtf/AutomaticThread.h>

#include <wtf/Box.h>

#include <wtf/Lock.h>

#include <wtf/NeverDestroyed.h>

#include <wtf/PageBlock.h>

#include <wtf/StdLibExtras.h>

#include <wtf/TZoneMallocInlines.h>

#include <wtf/Vector.h>

#if !ENABLE(MALLOC_HEAP_BREAKDOWN)

namespace {

std::atomic<bool> s_allocationFailsForTesting { false };

void* tryAllocateBlock()

{

if (s_allocationFailsForTesting.load(std::memory_order_relaxed)) [[unlikely]]

return nullptr;

return tryFastCompactAlignedMalloc(MarkedBlock::blockSize, MarkedBlock::blockSize);

}

class WarmUpBlockProvider {

using Phase = WarmUpMarkedBlockPhase;

WarmUpBlockProvider()

: m_lock(Box<Lock>::create())

, m_condition(AutomaticThreadCondition::create())

, m_thread(adoptRef(*new WarmUpThread(Locker { *m_lock }, *this)))

{

}

static bool isEnabled()

{

// Mini mode trades throughput for footprint, which is the opposite of the bargain here.

return Options::useWarmUpMarkedBlocks() && Options::warmUpMarkedBlockCount() && !VM::isInMiniMode();

}

static WarmUpBlockProvider& singleton()

{

static LazyNeverDestroyed<WarmUpBlockProvider> provider;

static std::once_flag flag;

std::call_once(flag, [] {

provider.construct();

});

return provider;

}

void* tryTake()

{

Locker locker { *m_lock };

void* result = m_blocks.isEmpty() ? nullptr : m_blocks.takeLast();

m_demandSinceRefill = true;

m_demandSinceIdleCheck = true;

// A miss means the mutator is about to take the very fault this exists to avoid, and it is

// also the only thing that brings the helper back once it has shut itself down. While it is

// standing down, a notify would only postpone the timeout that lifts the stand-down.

if ((!result || isRunningLow()) && m_phase != Phase::StandingDown)

m_condition->notifyOne(locker);

return result;

}

WarmUpMarkedBlockState stateForTesting()

{

Locker locker { *m_lock };

return { m_blocks.size(), m_phase };

}

// AutomaticThread has no voluntary temporary stop: PollResult::Stop is permanent, and start()

// release-asserts that the thread is still running. So giving up after an allocation failure has

// to be a wait that suppresses notifies, which is what leaves the idle timeout free to lift it.

class WarmUpThread final : public AutomaticThread {

WTF_MAKE_TZONE_ALLOCATED_INLINE(WarmUpThread);

WTF_OVERRIDE_DELETE_FOR_CHECKED_PTR(WarmUpThread);

public:

WarmUpThread(const AbstractLocker& locker, WarmUpBlockProvider& provider)

: AutomaticThread(locker, provider.m_lock, provider.m_condition.copyRef(), Seconds(Options::warmUpMarkedBlockIdleTimeout()))

, m_provider(provider)

{

}

ASCIILiteral name() const final { return "JSCWarmUp"_s; }

protected:

void threadDidStart() final

{

Locker locker { *m_provider.m_lock };

m_provider.m_phase = Phase::Armed;

}

PollResult poll(const AbstractLocker&) final

{

assertIsHeld(*m_provider.m_lock);

if (m_provider.m_phase != Phase::Armed)

return PollResult::Wait;

// Topping the supply back up while demand is still arriving keeps it near its full depth

// on a ramp, rather than letting it drain to the watermark first.

if (m_provider.m_demandSinceRefill || m_provider.isRunningLow())

return PollResult::Work;

return PollResult::Wait;

}

WorkResult work() final

{

m_provider.refill();

return WorkResult::Continue;

}

bool shouldSleep(const AbstractLocker&) final

{

assertIsHeld(*m_provider.m_lock);

if (!std::exchange(m_provider.m_demandSinceIdleCheck, false))

return true;

m_provider.m_phase = Phase::Armed;

return false;

}

void threadIsStopping(const AbstractLocker&) final

{

assertIsHeld(*m_provider.m_lock);

// fastFree does not reach back into this lock, so it is safe to call with it held.

m_provider.m_phase = Phase::Stopped;

for (void* block : std::exchange(m_provider.m_blocks, { }))

fastFree(block);

}

private:

WarmUpBlockProvider& m_provider;

};

size_t targetDepth() WTF_REQUIRES_LOCK(*m_lock) { return m_phase == Phase::Armed ? Options::warmUpMarkedBlockCount() : 0; }

bool isRunningLow() WTF_REQUIRES_LOCK(*m_lock) { return m_blocks.size() * 4 < targetDepth(); }

static void makeResident(void* block)

{

// One store per page is what takes the fault. Striding by the real page size rather than by

// the smallest a supported system could have avoids repeating the store within a page.

size_t pageSize = WTF::pageSize();

ASSERT(!(MarkedBlock::blockSize % pageSize));

auto bytes = unsafeMakeSpan(static_cast<volatile char*>(block), MarkedBlock::blockSize);

for (size_t offset = 0; offset < bytes.size(); offset += pageSize)

bytes[offset] = 0;

}

void refill()

{

size_t want;

{

Locker locker { *m_lock };

m_demandSinceRefill = false;

size_t target = targetDepth();

want = target > m_blocks.size() ? target - m_blocks.size() : 0;

}

Vector<void*, 32> staging;

bool exhausted = false;

for (size_t i = 0; i < want; ++i) {

void* block = tryAllocateBlock();

if (!block) {

exhausted = true;

break;

}

makeResident(block);

staging.append(block);

}

Locker locker { *m_lock };

m_blocks.appendVector(WTF::move(staging));

if (exhausted)

m_phase = Phase::StandingDown;

}

const Box<Lock> m_lock;

const Ref<AutomaticThreadCondition> m_condition;

const Ref<WarmUpThread> m_thread;

Vector<void*, 32> m_blocks WTF_GUARDED_BY_LOCK(*m_lock);

Phase m_phase WTF_GUARDED_BY_LOCK(*m_lock) { Phase::Stopped };

bool m_demandSinceRefill WTF_GUARDED_BY_LOCK(*m_lock) { false };

bool m_demandSinceIdleCheck WTF_GUARDED_BY_LOCK(*m_lock) { false };

};

} // anonymous namespace

WarmUpMarkedBlockState warmUpMarkedBlockStateForTesting()

{

return WarmUpBlockProvider::singleton().stateForTesting();

}

void setWarmUpMarkedBlockAllocationShouldFailForTesting(bool shouldFail)

{

s_allocationFailsForTesting.store(shouldFail, std::memory_order_relaxed);

}

#else // ENABLE(MALLOC_HEAP_BREAKDOWN)

WarmUpMarkedBlockState warmUpMarkedBlockStateForTesting() { return { }; }

void setWarmUpMarkedBlockAllocationShouldFailForTesting(bool) { }

#endif

// MarkedBlock::tryCreate is the only caller today and always asks for a block-shaped region.

// The guard keeps a future caller of some other size from being handed a block.

if (alignment == MarkedBlock::blockSize && size == MarkedBlock::blockSize && WarmUpBlockProvider::isEnabled()) {

if (void* block = WarmUpBlockProvider::singleton().tryTake())

return block;

}

enum class WarmUpMarkedBlockPhase : uint8_t { Stopped, Armed, StandingDown };

struct WarmUpMarkedBlockState {

size_t blockCount { 0 };

WarmUpMarkedBlockPhase phase { WarmUpMarkedBlockPhase::Stopped };

};

WarmUpMarkedBlockState warmUpMarkedBlockStateForTesting();

void setWarmUpMarkedBlockAllocationShouldFailForTesting(bool);

v(Bool, useWarmUpMarkedBlocks, true, Normal, "hand MarkedBlock allocation pages that a helper thread already made resident"_s) \

v(Unsigned, warmUpMarkedBlockCount, 32, Normal, "how many MarkedBlocks the helper thread keeps ready with their pages already resident; 0 turns it off"_s) \

v(Double, warmUpMarkedBlockIdleTimeout, 10, Normal, "seconds without a MarkedBlock request before the helper thread releases what it is holding and shuts down"_s) \