# Copyright (C) 2011-2025 Apple Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1. Redistributions of source code must retain the above copyright

# notice, this list of conditions and the following disclaimer.

# 2. Redistributions in binary form must reproduce the above copyright

# notice, this list of conditions and the following disclaimer in the

# documentation and/or other materials provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''

# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,

# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS

# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR

# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF

# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS

# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN

# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)

# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF

# THE POSSIBILITY OF SUCH DAMAGE.

# Utilities.

macro storePC()

storei PC, CallSiteIndex[cfr]

macro loadPC()

loadi CallSiteIndex[cfr], PC

macro getuOperandNarrow(opcodeStruct, fieldName, dst)

loadb constexpr %opcodeStruct%_%fieldName%_index + OpcodeIDNarrowSize[PB, PC, 1], dst

macro getOperandNarrow(opcodeStruct, fieldName, dst)

loadbsq constexpr %opcodeStruct%_%fieldName%_index + OpcodeIDNarrowSize[PB, PC, 1], dst

macro getuOperandWide16JS(opcodeStruct, fieldName, dst)

loadh constexpr %opcodeStruct%_%fieldName%_index * 2 + OpcodeIDWide16SizeJS[PB, PC, 1], dst

macro getOperandWide16JS(opcodeStruct, fieldName, dst)

loadhsq constexpr %opcodeStruct%_%fieldName%_index * 2 + OpcodeIDWide16SizeJS[PB, PC, 1], dst

macro getuOperandWide32JS(opcodeStruct, fieldName, dst)

loadi constexpr %opcodeStruct%_%fieldName%_index * 4 + OpcodeIDWide32SizeJS[PB, PC, 1], dst

macro getOperandWide32JS(opcodeStruct, fieldName, dst)

loadis constexpr %opcodeStruct%_%fieldName%_index * 4 + OpcodeIDWide32SizeJS[PB, PC, 1], dst

macro makeReturn(get, dispatch, fn)

fn(macro (value)

move value, t2

get(m_dst, t1)

storeq t2, [cfr, t1, 8]

dispatch()

end)

macro makeReturnProfiled(size, opcodeStruct, get, metadata, dispatch, fn)

fn(macro (value)

move value, t3

valueProfile(size, opcodeStruct, m_valueProfile, t3, t1)

get(m_dst, t1)

storeq t3, [cfr, t1, 8]

dispatch()

end)

macro valueProfile(size, opcodeStruct, profileName, value, scratch)

getu(size, opcodeStruct, profileName, scratch)

mulq constexpr (-sizeof(ValueProfile)), scratch

storeq value, constexpr (-sizeof(UnlinkedMetadataTable::LinkingData)) + ValueProfile::m_buckets[metadataTable, scratch, 1]

# After calling, calling bytecode is claiming input registers are not used.

macro dispatchAfterRegularCall(size, opcodeStruct, valueProfileName, dstVirtualRegister, dispatch)

loadPC()

if C_LOOP

# On non C_LOOP builds, CSR restore takes care of this.

loadp CodeBlock[cfr], PB

loadp CodeBlock::m_instructionsRawPointer[PB], PB

end

get(size, opcodeStruct, dstVirtualRegister, t1)

storeq r0, [cfr, t1, 8]

valueProfile(size, opcodeStruct, valueProfileName, r0, t2)

dispatch()

macro dispatchAfterTailCall(size, opcodeStruct, valueProfileName, dstVirtualRegister, dispatch)

loadPC()

if C_LOOP

# On non C_LOOP builds, CSR restore takes care of this.

loadp CodeBlock[cfr], PB

loadp CodeBlock::m_instructionsRawPointer[PB], PB

end

get(size, opcodeStruct, dstVirtualRegister, t1)

storeq r0, [cfr, t1, 8]

metadata(size, opcodeStruct, t2, t1)

dispatch()

macro dispatchAfterRegularCallIgnoreResult(size, opcodeStruct, valueProfileName, dstVirtualRegister, dispatch)

loadPC()

if C_LOOP

# On non C_LOOP builds, CSR restore takes care of this.

loadp CodeBlock[cfr], PB

loadp CodeBlock::m_instructionsRawPointer[PB], PB

end

dispatch()

macro cCall2(function)

if ARM64 or ARM64E

const scratch = t9

else

const scratch = t5

end

checkStackPointerAlignment(scratch, 0xbad0c002)

if C_LOOP

cloopCallSlowPath function, a0, a1

elsif X86_64 or ARM64 or ARM64E or RISCV64

call function

else

error

end

macro cCall2Void(function)

if C_LOOP

cloopCallSlowPathVoid function, a0, a1

else

cCall2(function)

end

macro cCall3(function)

if ARM64 or ARM64E

const scratch = t9

else

const scratch = t5

end

checkStackPointerAlignment(scratch, 0xbad0c003)

if C_LOOP

cloopCallSlowPath3 function, a0, a1, a2

elsif X86_64 or ARM64 or ARM64E or RISCV64

call function

else

error

end

# This barely works. arg3 and arg4 should probably be immediates.

macro cCall4(function)

if ARM64 or ARM64E

const scratch = t9

else

const scratch = t5

end

checkStackPointerAlignment(scratch, 0xbad0c004)

if C_LOOP

cloopCallSlowPath4 function, a0, a1, a2, a3

elsif X86_64 or ARM64 or ARM64E or RISCV64

call function

else

error

end

macro doVMEntry(makeCall)

functionPrologue()

const entry = a0

const vm = a1

const protoCallFrame = a2

vmEntryRecord(cfr, sp)

checkStackPointerAlignment(t4, 0xbad0dc01)

if (ARM64 or ARM64E) and ADDRESS64

loadp ProtoCallFrame::context[protoCallFrame], t3

storepairq vm, t3, VMEntryRecord::m_vm[sp]

loadpairq VM::topCallFrame[vm], t4, t3

storepairq t4, t3, VMEntryRecord::m_prevTopCallFrame[sp]

else

loadp ProtoCallFrame::context[protoCallFrame], t4

storep vm, VMEntryRecord::m_vm[sp]

storep t4, VMEntryRecord::m_context[sp]

loadp VM::topCallFrame[vm], t4

storep t4, VMEntryRecord::m_prevTopCallFrame[sp]

loadp VM::topEntryFrame[vm], t4

storep t4, VMEntryRecord::m_prevTopEntryFrame[sp]

end

loadi ProtoCallFrame::paddedArgCount[protoCallFrame], t4

addp CallFrameHeaderSlots, t4, t4

lshiftp 3, t4

subp sp, t4, t3

# Ensure that we have enough additional stack capacity for the incoming args,

# and the frame for the JS code we're executing. We need to do this check

# before we start copying the args from the protoCallFrame below.

if C_LOOP

bpaeq t3, VMCLoopStackLimitOffset[vm], .stackHeightOK

move entry, t4

move vm, t5

cloopCallSlowPath _llint_stack_check_at_vm_entry, vm, t3

bpeq t0, 0, .stackCheckFailed

move t4, entry

move t5, vm

jmp .stackHeightOK

move t4, entry

move t5, vm

jmp _llint_throw_stack_overflow_error_from_vm_entry

move t3, sp

else

bpbeq t3, VMSoftStackLimitOffset[vm], _llint_throw_stack_overflow_error_from_vm_entry

move t3, sp

end

# Copy the CodeBlock/Callee/ArgumentCountIncludingThis/|this| from protoCallFrame into the callee frame.

if ARM64 or ARM64E

loadpairq (8 * 0)[protoCallFrame], t5, t3

storepairq t5, t3, CodeBlock + (8 * 0)[sp]

loadpairq (8 * 2)[protoCallFrame], t5, t3

storepairq t5, t3, CodeBlock + (8 * 2)[sp]

else

loadq (8 * 0)[protoCallFrame], t5

storeq t5, CodeBlock + (8 * 0)[sp]

loadq (8 * 1)[protoCallFrame], t5

storeq t5, CodeBlock + (8 * 1)[sp]

loadq (8 * 2)[protoCallFrame], t5

storeq t5, CodeBlock + (8 * 2)[sp]

loadq (8 * 3)[protoCallFrame], t5

storeq t5, CodeBlock + (8 * 3)[sp]

end

loadi LowWordOffset + ProtoCallFrame::argCountAndCodeOriginValue[protoCallFrame], t4

subi 1, t4

loadi ProtoCallFrame::paddedArgCount[protoCallFrame], t5

subi 1, t5

bieq t4, t5, .copyArgs

move ValueUndefined, t3

subi 1, t5

storeq t3, ThisArgumentOffset + 8[sp, t5, 8]

bineq t4, t5, .fillExtraArgsLoop

loadp ProtoCallFrame::args[protoCallFrame], t3

btiz t4, .copyArgsDone

subi 1, t4

loadq [t3, t4, 8], t5

storeq t5, ThisArgumentOffset + 8[sp, t4, 8]

jmp .copyArgsLoop

if ARM64 or ARM64E

move sp, t4

if ADDRESS64

storepairq t4, cfr, VM::topCallFrame[vm]

else

storep t4, VM::topCallFrame[vm]

storep cfr, VM::topEntryFrame[vm]

end

else

storep sp, VM::topCallFrame[vm]

storep cfr, VM::topEntryFrame[vm]

end

checkStackPointerAlignment(t5, 0xbad0dc02)

makeCall(entry, protoCallFrame, t3, t4)

# We may have just made a call into a JS function, so we can't rely on sp

# for anything but the fact that our own locals (ie the VMEntryRecord) are

# not below it. It also still has to be aligned, though.

checkStackPointerAlignment(t2, 0xbad0dc03)

vmEntryRecord(cfr, t4)

loadp VMEntryRecord::m_vm[t4], vm

if (ARM64 or ARM64E) and ADDRESS64

loadpairq VMEntryRecord::m_prevTopCallFrame[t4], t4, t2

storepairq t4, t2, VM::topCallFrame[vm]

else

loadp VMEntryRecord::m_prevTopCallFrame[t4], t2

storep t2, VM::topCallFrame[vm]

loadp VMEntryRecord::m_prevTopEntryFrame[t4], t2

storep t2, VM::topEntryFrame[vm]

end

subp cfr, CalleeRegisterSaveSize, sp

functionEpilogue()

ret

const entry = a0

const vm = a1

const protoCallFrame = a2

move vm, a0

move protoCallFrame, a1

cCall2(_llint_throw_stack_overflow_error)

vmEntryRecord(cfr, t4)

loadp VMEntryRecord::m_vm[t4], vm

loadp VMEntryRecord::m_prevTopCallFrame[t4], t5

storep t5, VM::topCallFrame[vm]

loadp VMEntryRecord::m_prevTopEntryFrame[t4], t5

storep t5, VM::topEntryFrame[vm]

move ValueUndefined, r0

subp cfr, CalleeRegisterSaveSize, sp

functionEpilogue()

ret

# a0, a2, t3, t4

macro makeJavaScriptCall(entry, protoCallFrame, temp1, temp2)

addp CallerFrameAndPCSize, sp

if C_LOOP

cloopCallJSFunction entry

elsif ARM64E

move entry, t5

leap _g_config, a7

jmp JSCConfigGateMapOffset + (constexpr Gate::vmEntryToJavaScript) * PtrSize[a7], VMEntryToJITGatePtrTag # JSEntryPtrTag

global _vmEntryToJavaScriptTrampoline

_vmEntryToJavaScriptTrampoline:

call t5, JSEntryPtrTag

else

call entry, JSEntryPtrTag

end

if ARM64E

global _vmEntryToJavaScriptGateAfter

_vmEntryToJavaScriptGateAfter:

end

subp CallerFrameAndPCSize, sp

# a0, a2, t3, t4

macro makeHostFunctionCall(entry, protoCallFrame, temp1, temp2)

move entry, temp1

storep cfr, [sp]

loadp ProtoCallFrame::globalObject[protoCallFrame], a0

move sp, a1

if C_LOOP

storep lr, ReturnPC[sp]

cloopCallNative temp1

else

call temp1, HostFunctionPtrTag

end

op(llint_handle_uncaught_exception, macro ()

getVMFromCallFrame(t3, t0)

restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer(t3, t0)

storep 0, VM::callFrameForCatch[t3]

loadp VM::topEntryFrame[t3], cfr

vmEntryRecord(cfr, t2)

loadp VMEntryRecord::m_vm[t2], t3

loadp VMEntryRecord::m_prevTopCallFrame[t2], t5

storep t5, VM::topCallFrame[t3]

loadp VMEntryRecord::m_prevTopEntryFrame[t2], t5

storep t5, VM::topEntryFrame[t3]

move ValueUndefined, r0

subp cfr, CalleeRegisterSaveSize, sp

functionEpilogue()

ret

end)

op(llint_get_host_call_return_value, macro ()

functionPrologue()

loadp Callee[cfr], t0

convertJSCalleeToVM(t0)

loadq VM::encodedHostCallReturnValue[t0], t0

functionEpilogue()

ret

end)

macro prepareStateForCCall()

addp PB, PC

macro restoreStateAfterCCallWithoutExceptionCheck()

move r0, PC

subp PB, PC

macro restoreStateAfterCCall()

# Slow paths report pending exceptions by returning LLInt::exceptionSignal

# (all ones) in r1. In such cases, r0 contains a sentinel bytecode pointer

# that lives in a different allocation from this CodeBlock's instruction

# buffer. Therefore, PB and PC have different MTE tags that don't cancel

# when subtracted and we need to check for exceptionSignal here to jump

# straight to the throw trampoline.

bpeq r1, -1, _llint_throw_from_slow_path_trampoline

move r0, PC

subp PB, PC

macro callSlowPath(slowPath)

prepareStateForCCall()

move cfr, a0

move PC, a1

cCall2(slowPath)

restoreStateAfterCCall()

macro traceOperand(fromWhere, operand)

prepareStateForCCall()

move fromWhere, a2

move operand, a3

move cfr, a0

move PC, a1

cCall4(_llint_trace_operand)

restoreStateAfterCCall()

macro traceValue(fromWhere, operand)

prepareStateForCCall()

move fromWhere, a2

move operand, a3

move cfr, a0

move PC, a1

cCall4(_llint_trace_value)

restoreStateAfterCCall()

# Call a slow path for call opcodes.

macro callCallSlowPath(slowPath, action)

storePC()

prepareStateForCCall()

move cfr, a0

move PC, a1

cCall2(slowPath)

action(r0, r1)

macro callTrapHandler(throwHandler)

storePC()

prepareStateForCCall()

move cfr, a0

move PC, a1

cCall2(_llint_slow_path_handle_traps)

btpnz r0, throwHandler

loadi CallSiteIndex[cfr], PC

if JIT

macro checkSwitchToJITForLoop()

checkSwitchToJIT(

1,

macro()

storePC()

prepareStateForCCall()

move cfr, a0

move PC, a1

cCall2(_llint_loop_osr)

btpz r0, .recover

move r1, sp

loadBaselineJITConstantPool()

if ARM64E

leap _g_config, a2

jmp JSCConfigGateMapOffset + (constexpr Gate::loopOSREntry) * PtrSize[a2], NativeToJITGatePtrTag # JSEntryPtrTag

else

jmp r0, JSEntryPtrTag

end

.recover:

loadPC()

end)

end

macro checkSwitchToJITForLoop()

checkSwitchToJIT(

1,

macro()

storePC()

prepareStateForCCall()

move cfr, a0

move PC, a1

cCall2(_llint_loop_osr)

loadPC()

end)

end

macro cage(basePtr, mask, ptr, scratch)

if GIGACAGE_ENABLED and not C_LOOP

loadp basePtr, scratch

btpz scratch, .done

andp mask, ptr

addp scratch, ptr

.done:

end

macro cagePrimitive(basePtr, mask, ptr, scratch)

if GIGACAGE_ENABLED and not C_LOOP

loadb GigacageConfig + Gigacage::Config::disablingPrimitiveGigacageIsForbidden, scratch

btbnz scratch, .doCaging

loadb _disablePrimitiveGigacageRequested, scratch

btbnz scratch, .done

.doCaging:

cage(basePtr, mask, ptr, scratch)

.done:

end

macro cagedPrimitive(ptr, length, scratch, scratch2)

const source = ptr

if GIGACAGE_ENABLED

cagePrimitive(GigacageConfig + Gigacage::Config::basePtrs + GigacagePrimitiveBasePtrOffset, constexpr Gigacage::primitiveGigacageMask, source, scratch)

end

macro cagedPrimitiveMayBeNull(ptr, scratch)

# Note that we may produce non-nullptr for nullptr since we add Gigacage offset.

# This behavior is aligned to AssemblyHelpers::{cageConditionally,cage}, FTL implementation of caging etc.

if GIGACAGE_ENABLED

cagePrimitive(GigacageConfig + Gigacage::Config::basePtrs + GigacagePrimitiveBasePtrOffset, constexpr Gigacage::primitiveGigacageMask, ptr, scratch)

end

macro loadCagedJSValue(source, dest, scratchOrLength)

loadp source, dest

macro loadVariable(get, fieldName, valueReg)

get(fieldName, valueReg)

loadq [cfr, valueReg, 8], valueReg

macro storeVariable(get, fieldName, newValueReg, scratchReg)

get(fieldName, scratchReg)

storeq newValueReg, [cfr, scratchReg, 8]

# Index and value must be different registers. Index may be clobbered.

macro loadConstant(size, index, value)

macro loadNarrow()

loadp CodeBlock[cfr], value

loadp CodeBlock::m_constantRegisters + VectorBufferOffset[value], value

loadq -(FirstConstantRegisterIndexNarrow * 8)[value, index, 8], value

end

macro loadWide16()

loadp CodeBlock[cfr], value

loadp CodeBlock::m_constantRegisters + VectorBufferOffset[value], value

loadq -(FirstConstantRegisterIndexWide16 * 8)[value, index, 8], value

end

macro loadWide32()

loadp CodeBlock[cfr], value

loadp CodeBlock::m_constantRegisters + VectorBufferOffset[value], value

subp FirstConstantRegisterIndexWide32, index

loadq [value, index, 8], value

end

size(loadNarrow, loadWide16, loadWide32, macro (load) load() end)

# Index and value must be different registers. Index may be clobbered.

macro loadConstantOrVariable(size, index, value)

macro loadNarrow()

bpgteq index, FirstConstantRegisterIndexNarrow, .constant

loadq [cfr, index, 8], value

jmp .done

.constant:

loadConstant(size, index, value)

.done:

end

macro loadWide16()

bpgteq index, FirstConstantRegisterIndexWide16, .constant

loadq [cfr, index, 8], value

jmp .done

.constant:

loadConstant(size, index, value)

.done:

end

macro loadWide32()

bpgteq index, FirstConstantRegisterIndexWide32, .constant

loadq [cfr, index, 8], value

jmp .done

.constant:

loadConstant(size, index, value)

.done:

end

size(loadNarrow, loadWide16, loadWide32, macro (load) load() end)

macro loadConstantOrVariableInt32(size, index, value, slow)

loadConstantOrVariable(size, index, value)

bqb value, numberTag, slow

macro loadConstantOrVariableCell(size, index, value, slow)

loadConstantOrVariable(size, index, value)

btqnz value, notCellMask, slow

macro writeBarrierOnCellWithReload(cell, reloadAfterSlowPath)

skipIfIsRememberedOrInEden(

cell,

macro()

push PB, PC

move cell, a1 # cell can be a0

move cfr, a0

cCall2Void(_llint_write_barrier_slow)

pop PC, PB

reloadAfterSlowPath()

end)

macro writeBarrierOnCellAndValueWithReload(cell, value, reloadAfterSlowPath)

btqnz value, notCellMask, .writeBarrierDone

btqz value, .writeBarrierDone

writeBarrierOnCellWithReload(cell, reloadAfterSlowPath)

macro writeBarrierOnOperandWithReload(size, get, cellFieldName, reloadAfterSlowPath)

get(cellFieldName, t1)

loadConstantOrVariableCell(size, t1, t2, .writeBarrierDone)

writeBarrierOnCellWithReload(t2, reloadAfterSlowPath)

macro writeBarrierOnOperand(size, get, cellFieldName)

writeBarrierOnOperandWithReload(size, get, cellFieldName, macro () end)

macro writeBarrierOnOperands(size, get, cellFieldName, valueFieldName)

get(valueFieldName, t1)

loadConstantOrVariableCell(size, t1, t0, .writeBarrierDone)

btpz t0, .writeBarrierDone

writeBarrierOnOperand(size, get, cellFieldName)

macro writeBarrierOnGlobal(size, get, valueFieldName, loadMacro)

get(valueFieldName, t1)

loadConstantOrVariableCell(size, t1, t0, .writeBarrierDone)

btpz t0, .writeBarrierDone

loadMacro(t3)

writeBarrierOnCellWithReload(t3, macro() end)

macro writeBarrierOnGlobalObject(size, get, valueFieldName)

writeBarrierOnGlobal(size, get, valueFieldName,

macro(registerToStoreGlobal)

loadp CodeBlock[cfr], registerToStoreGlobal

loadp CodeBlock::m_globalObject[registerToStoreGlobal], registerToStoreGlobal

end)

macro writeBarrierOnGlobalLexicalEnvironment(size, get, valueFieldName)

writeBarrierOnGlobal(size, get, valueFieldName,

macro(registerToStoreGlobal)

loadp CodeBlock[cfr], registerToStoreGlobal

loadp CodeBlock::m_globalObject[registerToStoreGlobal], registerToStoreGlobal

loadp JSGlobalObject::m_globalLexicalEnvironment[registerToStoreGlobal], registerToStoreGlobal

end)

macro structureIDToStructureWithScratch(structureIDThenStructure, scratch)

if ADDRESS64

leap _g_config, scratch

loadp JSCConfigOffset + constexpr JSC::offsetOfJSCConfigStructureIDBase[scratch], scratch

addp scratch, structureIDThenStructure

end

macro loadStructureWithScratch(cell, structure, scratch)

loadi JSCell::m_structureID[cell], structure

structureIDToStructureWithScratch(structure, scratch)

# Entrypoints into the interpreter.

# Expects that CodeBlock is in t1, which is what prologue() leaves behind.

macro functionArityCheck(opcodeName, doneLabel)

loadi LowWordOffset + ArgumentCountIncludingThis[cfr], t0

loadi CodeBlock::m_numParameters[t1], t2

biaeq t0, t2, doneLabel

# t0 argumentCountIncludingThis

# t1 CodeBlock

# t2 numParameters

addi CallFrameHeaderSlots, t2, t3

btiz t3, 0x1, .arityCheck

addi 1, t2

subi t2, t0, t2

addi 1, t2, t3

andi ~1, t3

lshiftp 3, t3

subp sp, t3, t5

loadp CodeBlock::m_vm[t1], t0

if C_LOOP

bpbeq VMCLoopStackLimitOffset[t0], t5, .stackHeightOK

else

bpbeq VMSoftStackLimitOffset[t0], t5, .stackHeightOK

end

prepareStateForCCall()

move cfr, a0

move PC, a1

cCall2(_llint_slow_path_arityCheck) # This slowPath has the protocol: r0 = 0 => no error, r0 != 0 => error

btiz r0, .noError

# We're throwing before the frame is fully set up. This frame will be

# ignored by the unwinder. So, let's restore the callee saves before we

# start unwinding. We need to do this before we change the cfr.

restoreCalleeSavesUsedByLLInt()

move r1, cfr # r1 contains caller frame

jmp _llint_throw_from_slow_path_trampoline

move t2, r1

move r1, t1 # r1 contains slotsToAdd.

btiz t1, .continue

loadi LowWordOffset + ArgumentCountIncludingThis[cfr], t2

addi CallFrameHeaderSlots, t2

// Check if there are some unaligned slots we can use

move t1, t3

andi StackAlignmentSlots - 1, t3

btiz t3, .noExtraSlot

move ValueUndefined, t0

storeq t0, [cfr, t2, 8]

addi 1, t2

bsubinz 1, t3, .fillExtraSlots

andi ~(StackAlignmentSlots - 1), t1

btiz t1, .continue

if ARM64E

leap _g_config, t3

jmp JSCConfigGateMapOffset + (constexpr Gate::%opcodeName%Untag) * PtrSize[t3], NativeToJITGatePtrTag

_js_trampoline_%opcodeName%_untag:

loadp 8[cfr], lr

addp 16, cfr, t3

untagReturnAddress t3

_%opcodeName%UntagGateAfter:

else

_js_trampoline_%opcodeName%_untag:

end

// Move frame up t1 slots

negq t1

move cfr, t3

subp CalleeSaveSpaceAsVirtualRegisters * 8, t3

addi CalleeSaveSpaceAsVirtualRegisters, t2

move t1, t0

# Adds to sp are always 64-bit on arm64 so we need maintain t0's high bits.

lshiftq 3, t0

addp t0, cfr

addp t0, sp

loadq [t3], t0

storeq t0, [t3, t1, 8]

addp 8, t3

bsubinz 1, t2, .copyLoop

// Fill new slots with JSUndefined

move t1, t2

move ValueUndefined, t0

storeq t0, [t3, t1, 8]

addp 8, t3

baddinz 1, t2, .fillLoop

if ARM64E

leap _g_config, t3

jmp JSCConfigGateMapOffset + (constexpr Gate::%opcodeName%Tag) * PtrSize[t3], NativeToJITGatePtrTag

_js_trampoline_%opcodeName%_tag:

addp 16, cfr, t1

tagReturnAddress t1

storep lr, 8[cfr]

_%opcodeName%TagGateAfter:

else

_js_trampoline_%opcodeName%_tag:

end

# Reload CodeBlock and reset PC, since the slow_path clobbered them.

loadp CodeBlock[cfr], t1

loadp CodeBlock::m_instructionsRawPointer[t1], PB

move 0, PC

jmp doneLabel

# Instruction implementations

traceExecution()

checkStackPointerAlignment(t2, 0xdead00e1)

loadp CodeBlock[cfr], t2 // t2<CodeBlock> = cfr.CodeBlock

loadi CodeBlock::m_numVars[t2], t2 // t2<size_t> = t2<CodeBlock>.m_numVars

subq CalleeSaveSpaceAsVirtualRegisters, t2

move cfr, t1

subq CalleeSaveSpaceAsVirtualRegisters * 8, t1

btiz t2, .opEnterDone

move ValueUndefined, t0

negi t2

sxi2q t2, t2

storeq t0, [t1, t2, 8]

addq 1, t2

btqnz t2, .opEnterLoop

loadp CodeBlock[cfr], t2

loadi CodeBlock::m_numberOfArgumentsToSkipAndCouldBeTainted[t2], t1

btis t1, .opEnterSlow

loadp CodeBlock::m_vm[t2], t0

loadb JSCell::m_cellState[t2], t1

loadi (constexpr (VM::offsetOfHeapBarrierThreshold()))[t0], t0

bibeq t1, t0, .opEnterSlow

loadis CodeBlock::m_scopeRegister[t2], t1

loadp Callee[cfr], t0

loadp JSCallee::m_scope[t0], t0

storeq t0, [cfr, t1, 8]

checkTraps(macro()

dispatchOp(narrow, op_enter)

end)

callSlowPath(_slow_path_enter)

jmp .opEnterDispatch

llintOpWithProfile(op_get_argument, OpGetArgument, macro (size, get, dispatch, return)

get(m_index, t2)

loadi LowWordOffset + ArgumentCountIncludingThis[cfr], t0

bilteq t0, t2, .opGetArgumentOutOfBounds

loadq ThisArgumentOffset[cfr, t2, 8], t0

return(t0)

return(ValueUndefined)

end)

llintOpWithReturn(op_argument_count, OpArgumentCount, macro (size, get, dispatch, return)

loadi LowWordOffset + ArgumentCountIncludingThis[cfr], t0

subi 1, t0

orq TagNumber, t0

return(t0)

end)

llintOpWithReturn(op_get_scope, OpGetScope, macro (size, get, dispatch, return)

loadp Callee[cfr], t0

loadp JSCallee::m_scope[t0], t0

return(t0)

end)

llintOpWithMetadata(op_to_this, OpToThis, macro (size, get, dispatch, metadata, return)

get(m_srcDst, t0)

loadq [cfr, t0, 8], t0

btqnz t0, notCellMask, .opToThisSlow

bbneq JSCell::m_type[t0], FinalObjectType, .opToThisSlow

loadi JSCell::m_structureID[t0], t1

metadata(t2, t3)

loadi OpToThis::Metadata::m_cachedStructureID[t2], t2

bineq t1, t2, .opToThisSlow

dispatch()

callSlowPath(_slow_path_to_this)

dispatch()

end)

llintOp(op_check_tdz, OpCheckTdz, macro (size, get, dispatch)

get(m_targetVirtualRegister, t0)

loadConstantOrVariable(size, t0, t1)

bqneq t1, ValueEmpty, .opNotTDZ

callSlowPath(_slow_path_check_tdz)

dispatch()

end)

llintOpWithReturn(op_mov, OpMov, macro (size, get, dispatch, return)

get(m_src, t1)

loadConstantOrVariable(size, t1, t2)

return(t2)

end)

llintOpWithReturn(op_not, OpNot, macro (size, get, dispatch, return)

get(m_operand, t0)

loadConstantOrVariable(size, t0, t2)

xorq ValueFalse, t2

btqnz t2, ~1, .opNotSlow

xorq ValueTrue, t2

return(t2)

callSlowPath(_slow_path_not)

dispatch()

end)

macro equalityComparisonOp(opcodeName, opcodeStruct, integerComparison)

llintOpWithReturn(op_%opcodeName%, opcodeStruct, macro (size, get, dispatch, return)

get(m_rhs, t0)

get(m_lhs, t2)

loadConstantOrVariableInt32(size, t0, t1, .slow)

loadConstantOrVariableInt32(size, t2, t0, .slow)

integerComparison(t0, t1, t0)

orq ValueFalse, t0

return(t0)

.slow:

callSlowPath(_slow_path_%opcodeName%)

dispatch()

end)

macro equalNullComparisonOp(opcodeName, opcodeStruct, fn)

llintOpWithReturn(opcodeName, opcodeStruct, macro (size, get, dispatch, return)

get(m_operand, t1)

loadConstantOrVariable(size, t1, t0)

btqnz t0, notCellMask, .immediate

btbnz JSCell::m_flags[t0], MasqueradesAsUndefined, .masqueradesAsUndefined

move 0, t0

jmp .done

.masqueradesAsUndefined:

loadStructureWithScratch(t0, t2, t1)

loadp CodeBlock[cfr], t0

loadp CodeBlock::m_globalObject[t0], t0

cpeq Structure::m_realm[t2], t0, t0

jmp .done

.immediate:

andq ~TagUndefined, t0

cqeq t0, ValueNull, t0

.done:

fn(t0)

return(t0)

end)

equalNullComparisonOp(op_eq_null, OpEqNull,

macro (value) orq ValueFalse, value end)

equalNullComparisonOp(op_neq_null, OpNeqNull,

macro (value) xorq ValueTrue, value end)

llintOpWithReturn(op_is_undefined_or_null, OpIsUndefinedOrNull, macro (size, get, dispatch, return)

get(m_operand, t1)

loadConstantOrVariable(size, t1, t0)

andq ~TagUndefined, t0

cqeq t0, ValueNull, t0