// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// System calls and other sys.stuff for AMD64, Linux
//
#include "go_asm.h"
#include "go_tls.h"
#include "textflag.h"
#define AT_FDCWD -100
#define SYS_read 0
#define SYS_write 1
#define SYS_close 3
#define SYS_mmap 9
#define SYS_munmap 11
#define SYS_brk 12
#define SYS_rt_sigaction 13
#define SYS_rt_sigprocmask 14
#define SYS_rt_sigreturn 15
#define SYS_sched_yield 24
#define SYS_mincore 27
#define SYS_madvise 28
#define SYS_nanosleep 35
#define SYS_setittimer 38
#define SYS_getpid 39
#define SYS_socket 41
#define SYS_connect 42
#define SYS_clone 56
#define SYS_exit 60
#define SYS_kill 62
#define SYS_fcntl 72
#define SYS_sigaltstack 131
#define SYS_arch_prctl 158
#define SYS_gettid 186
#define SYS_futex 202
#define SYS_sched_getaffinity 204
#define SYS_epoll_create 213
#define SYS_exit_group 231
#define SYS_epoll_ctl 233
#define SYS_tgkill 234
#define SYS_openat 257
#define SYS_faccessat 269
#define SYS_epoll_pwait 281
#define SYS_epoll_create1 291
TEXT runtime·exit(SB),NOSPLIT,$0-4
MOVL code+0(FP), DI
MOVL $SYS_exit_group, AX
SYSCALL
RET
// func exitThread(wait *uint32)
TEXT runtime·exitThread(SB),NOSPLIT,$0-8
MOVQ wait+0(FP), AX
// We're done using the stack.
MOVL $0, (AX)
MOVL $0, DI // exit code
MOVL $SYS_exit, AX
SYSCALL
// We may not even have a stack any more.
INT $3
JMP 0(PC)
TEXT runtime·open(SB),NOSPLIT,$0-20
// This uses openat instead of open, because Android O blocks open.
MOVL $AT_FDCWD, DI // AT_FDCWD, so this acts like open
MOVQ name+0(FP), SI
MOVL mode+8(FP), DX
MOVL perm+12(FP), R10
MOVL $SYS_openat, AX
SYSCALL
CMPQ AX, $0xfffffffffffff001
JLS 2(PC)
MOVL $-1, AX
MOVL AX, ret+16(FP)
RET
TEXT runtime·closefd(SB),NOSPLIT,$0-12
MOVL fd+0(FP), DI
MOVL $SYS_close, AX
SYSCALL
CMPQ AX, $0xfffffffffffff001
JLS 2(PC)
MOVL $-1, AX
MOVL AX, ret+8(FP)
RET
TEXT runtime·write(SB),NOSPLIT,$0-28
MOVQ fd+0(FP), DI
MOVQ p+8(FP), SI
MOVL n+16(FP), DX
MOVL $SYS_write, AX
SYSCALL
CMPQ AX, $0xfffffffffffff001
JLS 2(PC)
MOVL $-1, AX
MOVL AX, ret+24(FP)
RET
TEXT runtime·read(SB),NOSPLIT,$0-28
MOVL fd+0(FP), DI
MOVQ p+8(FP), SI
MOVL n+16(FP), DX
MOVL $SYS_read, AX
SYSCALL
CMPQ AX, $0xfffffffffffff001
JLS 2(PC)
MOVL $-1, AX
MOVL AX, ret+24(FP)
RET
TEXT runtime·usleep(SB),NOSPLIT,$16
MOVL $0, DX
MOVL usec+0(FP), AX
MOVL $1000000, CX
DIVL CX
MOVQ AX, 0(SP)
MOVL $1000, AX // usec to nsec
MULL DX
MOVQ AX, 8(SP)
// nanosleep(&ts, 0)
MOVQ SP, DI
MOVL $0, SI
MOVL $SYS_nanosleep, AX
SYSCALL
RET
TEXT runtime·gettid(SB),NOSPLIT,$0-4
MOVL $SYS_gettid, AX
SYSCALL
MOVL AX, ret+0(FP)
RET
TEXT runtime·raise(SB),NOSPLIT,$0
MOVL $SYS_getpid, AX
SYSCALL
MOVL AX, R12
MOVL $SYS_gettid, AX
SYSCALL
MOVL AX, SI // arg 2 tid
MOVL R12, DI // arg 1 pid
MOVL sig+0(FP), DX // arg 3
MOVL $SYS_tgkill, AX
SYSCALL
RET
TEXT runtime·raiseproc(SB),NOSPLIT,$0
MOVL $SYS_getpid, AX
SYSCALL
MOVL AX, DI // arg 1 pid
MOVL sig+0(FP), SI // arg 2
MOVL $SYS_kill, AX
SYSCALL
RET
TEXT runtime·setitimer(SB),NOSPLIT,$0-24
MOVL mode+0(FP), DI
MOVQ new+8(FP), SI
MOVQ old+16(FP), DX
MOVL $SYS_setittimer, AX
SYSCALL
RET
TEXT runtime·mincore(SB),NOSPLIT,$0-28
MOVQ addr+0(FP), DI
MOVQ n+8(FP), SI
MOVQ dst+16(FP), DX
MOVL $SYS_mincore, AX
SYSCALL
MOVL AX, ret+24(FP)
RET
// func walltime() (sec int64, nsec int32)
TEXT runtime·walltime(SB),NOSPLIT,$0-12
// We don't know how much stack space the VDSO code will need,
// so switch to g0.
// In particular, a kernel configured with CONFIG_OPTIMIZE_INLINING=n
// and hardening can use a full page of stack space in gettime_sym
// due to stack probes inserted to avoid stack/heap collisions.
// See issue #20427.
MOVQ SP, BP // Save old SP; BP unchanged by C code.
get_tls(CX)
MOVQ g(CX), AX
MOVQ g_m(AX), BX // BX unchanged by C code.
// Set vdsoPC and vdsoSP for SIGPROF traceback.
MOVQ 0(SP), DX
MOVQ DX, m_vdsoPC(BX)
LEAQ sec+0(SP), DX
MOVQ DX, m_vdsoSP(BX)
CMPQ AX, m_curg(BX) // Only switch if on curg.
JNE noswitch
MOVQ m_g0(BX), DX
MOVQ (g_sched+gobuf_sp)(DX), SP // Set SP to g0 stack
noswitch:
SUBQ $16, SP // Space for results
ANDQ $~15, SP // Align for C code
MOVQ runtime·vdsoClockgettimeSym(SB), AX
CMPQ AX, $0
JEQ fallback
MOVL $0, DI // CLOCK_REALTIME
LEAQ 0(SP), SI
CALL AX
MOVQ 0(SP), AX // sec
MOVQ 8(SP), DX // nsec
MOVQ BP, SP // Restore real SP
MOVQ $0, m_vdsoSP(BX)
MOVQ AX, sec+0(FP)
MOVL DX, nsec+8(FP)
RET
fallback:
LEAQ 0(SP), DI
MOVQ $0, SI
MOVQ runtime·vdsoGettimeofdaySym(SB), AX
CALL AX
MOVQ 0(SP), AX // sec
MOVL 8(SP), DX // usec
IMULQ $1000, DX
MOVQ BP, SP // Restore real SP
MOVQ $0, m_vdsoSP(BX)
MOVQ AX, sec+0(FP)
MOVL DX, nsec+8(FP)
RET
TEXT runtime·nanotime(SB),NOSPLIT,$0-8
// Switch to g0 stack. See comment above in runtime·walltime.
MOVQ SP, BP // Save old SP; BP unchanged by C code.
get_tls(CX)
MOVQ g(CX), AX
MOVQ g_m(AX), BX // BX unchanged by C code.
// Set vdsoPC and vdsoSP for SIGPROF traceback.
MOVQ 0(SP), DX
MOVQ DX, m_vdsoPC(BX)
LEAQ ret+0(SP), DX
MOVQ DX, m_vdsoSP(BX)
CMPQ AX, m_curg(BX) // Only switch if on curg.
JNE noswitch
MOVQ m_g0(BX), DX
MOVQ (g_sched+gobuf_sp)(DX), SP // Set SP to g0 stack
noswitch:
SUBQ $16, SP // Space for results
ANDQ $~15, SP // Align for C code
MOVQ runtime·vdsoClockgettimeSym(SB), AX
CMPQ AX, $0
JEQ fallback
MOVL $1, DI // CLOCK_MONOTONIC
LEAQ 0(SP), SI
CALL AX
MOVQ 0(SP), AX // sec
MOVQ 8(SP), DX // nsec
MOVQ BP, SP // Restore real SP
MOVQ $0, m_vdsoSP(BX)
// sec is in AX, nsec in DX
// return nsec in AX
IMULQ $1000000000, AX
ADDQ DX, AX
MOVQ AX, ret+0(FP)
RET
fallback:
LEAQ 0(SP), DI
MOVQ $0, SI
MOVQ runtime·vdsoGettimeofdaySym(SB), AX
CALL AX
MOVQ 0(SP), AX // sec
MOVL 8(SP), DX // usec
MOVQ BP, SP // Restore real SP
MOVQ $0, m_vdsoSP(BX)
IMULQ $1000, DX
// sec is in AX, nsec in DX
// return nsec in AX
IMULQ $1000000000, AX
ADDQ DX, AX
MOVQ AX, ret+0(FP)
RET
TEXT runtime·rtsigprocmask(SB),NOSPLIT,$0-28
MOVL how+0(FP), DI
MOVQ new+8(FP), SI
MOVQ old+16(FP), DX
MOVL size+24(FP), R10
MOVL $SYS_rt_sigprocmask, AX
SYSCALL
CMPQ AX, $0xfffffffffffff001
JLS 2(PC)
MOVL $0xf1, 0xf1 // crash
RET
TEXT runtime·rt_sigaction(SB),NOSPLIT,$0-36
MOVQ sig+0(FP), DI
MOVQ new+8(FP), SI
MOVQ old+16(FP), DX
MOVQ size+24(FP), R10
MOVL $SYS_rt_sigaction, AX
SYSCALL
MOVL AX, ret+32(FP)
RET
// Call the function stored in _cgo_sigaction using the GCC calling convention.
TEXT runtime·callCgoSigaction(SB),NOSPLIT,$16
MOVQ sig+0(FP), DI
MOVQ new+8(FP), SI
MOVQ old+16(FP), DX
MOVQ _cgo_sigaction(SB), AX
MOVQ SP, BX // callee-saved
ANDQ $~15, SP // alignment as per amd64 psABI
CALL AX
MOVQ BX, SP
MOVL AX, ret+24(FP)
RET
TEXT runtime·sigfwd(SB),NOSPLIT,$0-32
MOVQ fn+0(FP), AX
MOVL sig+8(FP), DI
MOVQ info+16(FP), SI
MOVQ ctx+24(FP), DX
PUSHQ BP
MOVQ SP, BP
ANDQ $~15, SP // alignment for x86_64 ABI
CALL AX
MOVQ BP, SP
POPQ BP
RET
TEXT runtime·sigtramp(SB),NOSPLIT,$72
// Save callee-saved C registers, since the caller may be a C signal handler.
MOVQ BX, bx-8(SP)
MOVQ BP, bp-16(SP) // save in case GOEXPERIMENT=noframepointer is set
MOVQ R12, r12-24(SP)
MOVQ R13, r13-32(SP)
MOVQ R14, r14-40(SP)
MOVQ R15, r15-48(SP)
// We don't save mxcsr or the x87 control word because sigtrampgo doesn't
// modify them.
MOVQ DX, ctx-56(SP)
MOVQ SI, info-64(SP)
MOVQ DI, signum-72(SP)
MOVQ $runtime·sigtrampgo(SB), AX
CALL AX
MOVQ r15-48(SP), R15
MOVQ r14-40(SP), R14
MOVQ r13-32(SP), R13
MOVQ r12-24(SP), R12
MOVQ bp-16(SP), BP
MOVQ bx-8(SP), BX
RET
// Used instead of sigtramp in programs that use cgo.
// Arguments from kernel are in DI, SI, DX.
TEXT runtime·cgoSigtramp(SB),NOSPLIT,$0
// If no traceback function, do usual sigtramp.
MOVQ runtime·cgoTraceback(SB), AX
TESTQ AX, AX
JZ sigtramp
// If no traceback support function, which means that
// runtime/cgo was not linked in, do usual sigtramp.
MOVQ _cgo_callers(SB), AX
TESTQ AX, AX
JZ sigtramp
// Figure out if we are currently in a cgo call.
// If not, just do usual sigtramp.
get_tls(CX)
MOVQ g(CX),AX
TESTQ AX, AX
JZ sigtrampnog // g == nil
MOVQ g_m(AX), AX
TESTQ AX, AX
JZ sigtramp // g.m == nil
MOVL m_ncgo(AX), CX
TESTL CX, CX
JZ sigtramp // g.m.ncgo == 0
MOVQ m_curg(AX), CX
TESTQ CX, CX
JZ sigtramp // g.m.curg == nil
MOVQ g_syscallsp(CX), CX
TESTQ CX, CX
JZ sigtramp // g.m.curg.syscallsp == 0
MOVQ m_cgoCallers(AX), R8
TESTQ R8, R8
JZ sigtramp // g.m.cgoCallers == nil
MOVL m_cgoCallersUse(AX), CX
TESTL CX, CX
JNZ sigtramp // g.m.cgoCallersUse != 0
// Jump to a function in runtime/cgo.
// That function, written in C, will call the user's traceback
// function with proper unwind info, and will then call back here.
// The first three arguments, and the fifth, are already in registers.
// Set the two remaining arguments now.
MOVQ runtime·cgoTraceback(SB), CX
MOVQ $runtime·sigtramp(SB), R9
MOVQ _cgo_callers(SB), AX
JMP AX
sigtramp:
JMP runtime·sigtramp(SB)
sigtrampnog:
// Signal arrived on a non-Go thread. If this is SIGPROF, get a
// stack trace.
CMPL DI, $27 // 27 == SIGPROF
JNZ sigtramp
// Lock sigprofCallersUse.
MOVL $0, AX
MOVL $1, CX
MOVQ $runtime·sigprofCallersUse(SB), R11
LOCK
CMPXCHGL CX, 0(R11)
JNZ sigtramp // Skip stack trace if already locked.
// Jump to the traceback function in runtime/cgo.
// It will call back to sigprofNonGo, which will ignore the
// arguments passed in registers.
// First three arguments to traceback function are in registers already.
MOVQ runtime·cgoTraceback(SB), CX
MOVQ $runtime·sigprofCallers(SB), R8
MOVQ $runtime·sigprofNonGo(SB), R9
MOVQ _cgo_callers(SB), AX
JMP AX
// For cgo unwinding to work, this function must look precisely like
// the one in glibc. The glibc source code is:
// https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/unix/sysv/linux/x86_64/sigaction.c
// The code that cares about the precise instructions used is:
// https://gcc.gnu.org/viewcvs/gcc/trunk/libgcc/config/i386/linux-unwind.h?revision=219188&view=markup
TEXT runtime·sigreturn(SB),NOSPLIT,$0
MOVQ $SYS_rt_sigreturn, AX
SYSCALL
INT $3 // not reached
TEXT runtime·sysMmap(SB),NOSPLIT,$0
MOVQ addr+0(FP), DI
MOVQ n+8(FP), SI
MOVL prot+16(FP), DX
MOVL flags+20(FP), R10
MOVL fd+24(FP), R8
MOVL off+28(FP), R9
MOVL $SYS_mmap, AX
SYSCALL
CMPQ AX, $0xfffffffffffff001
JLS ok
NOTQ AX
INCQ AX
MOVQ $0, p+32(FP)
MOVQ AX, err+40(FP)
RET
ok:
MOVQ AX, p+32(FP)
MOVQ $0, err+40(FP)
RET
// Call the function stored in _cgo_mmap using the GCC calling convention.
// This must be called on the system stack.
TEXT runtime·callCgoMmap(SB),NOSPLIT,$16
MOVQ addr+0(FP), DI
MOVQ n+8(FP), SI
MOVL prot+16(FP), DX
MOVL flags+20(FP), CX
MOVL fd+24(FP), R8
MOVL off+28(FP), R9
MOVQ _cgo_mmap(SB), AX
MOVQ SP, BX
ANDQ $~15, SP // alignment as per amd64 psABI
MOVQ BX, 0(SP)
CALL AX
MOVQ 0(SP), SP
MOVQ AX, ret+32(FP)
RET
TEXT runtime·sysMunmap(SB),NOSPLIT,$0
MOVQ addr+0(FP), DI
MOVQ n+8(FP), SI
MOVQ $SYS_munmap, AX
SYSCALL
CMPQ AX, $0xfffffffffffff001
JLS 2(PC)
MOVL $0xf1, 0xf1 // crash
RET
// Call the function stored in _cgo_munmap using the GCC calling convention.
// This must be called on the system stack.
TEXT runtime·callCgoMunmap(SB),NOSPLIT,$16-16
MOVQ addr+0(FP), DI
MOVQ n+8(FP), SI
MOVQ _cgo_munmap(SB), AX
MOVQ SP, BX
ANDQ $~15, SP // alignment as per amd64 psABI
MOVQ BX, 0(SP)
CALL AX
MOVQ 0(SP), SP
RET
TEXT runtime·madvise(SB),NOSPLIT,$0
MOVQ addr+0(FP), DI
MOVQ n+8(FP), SI
MOVL flags+16(FP), DX
MOVQ $SYS_madvise, AX
SYSCALL
MOVL AX, ret+24(FP)
RET
// int64 futex(int32 *uaddr, int32 op, int32 val,
// struct timespec *timeout, int32 *uaddr2, int32 val2);
TEXT runtime·futex(SB),NOSPLIT,$0
MOVQ addr+0(FP), DI
MOVL op+8(FP), SI
MOVL val+12(FP), DX
MOVQ ts+16(FP), R10
MOVQ addr2+24(FP), R8
MOVL val3+32(FP), R9
MOVL $SYS_futex, AX
SYSCALL
MOVL AX, ret+40(FP)
RET
// int32 clone(int32 flags, void *stk, M *mp, G *gp, void (*fn)(void));
TEXT runtime·clone(SB),NOSPLIT,$0
MOVL flags+0(FP), DI
MOVQ stk+8(FP), SI
MOVQ $0, DX
MOVQ $0, R10
// Copy mp, gp, fn off parent stack for use by child.
// Careful: Linux system call clobbers CX and R11.
MOVQ mp+16(FP), R8
MOVQ gp+24(FP), R9
MOVQ fn+32(FP), R12
MOVL $SYS_clone, AX
SYSCALL
// In parent, return.
CMPQ AX, $0
JEQ 3(PC)
MOVL AX, ret+40(FP)
RET
// In child, on new stack.
MOVQ SI, SP
// If g or m are nil, skip Go-related setup.
CMPQ R8, $0 // m
JEQ nog
CMPQ R9, $0 // g
JEQ nog
// Initialize m->procid to Linux tid
MOVL $SYS_gettid, AX
SYSCALL
MOVQ AX, m_procid(R8)
// Set FS to point at m->tls.
LEAQ m_tls(R8), DI
CALL runtime·settls(SB)
// In child, set up new stack
get_tls(CX)
MOVQ R8, g_m(R9)
MOVQ R9, g(CX)
CALL runtime·stackcheck(SB)
nog:
// Call fn
CALL R12
// It shouldn't return. If it does, exit that thread.
MOVL $111, DI
MOVL $SYS_exit, AX
SYSCALL
JMP -3(PC) // keep exiting
TEXT runtime·sigaltstack(SB),NOSPLIT,$-8
MOVQ new+0(FP), DI
MOVQ old+8(FP), SI
MOVQ $SYS_sigaltstack, AX
SYSCALL
CMPQ AX, $0xfffffffffffff001
JLS 2(PC)
MOVL $0xf1, 0xf1 // crash
RET
// set tls base to DI
TEXT runtime·settls(SB),NOSPLIT,$32
#ifdef GOOS_android
// Android stores the TLS offset in runtime·tls_g.
SUBQ runtime·tls_g(SB), DI
#else
ADDQ $8, DI // ELF wants to use -8(FS)
#endif
MOVQ DI, SI
MOVQ $0x1002, DI // ARCH_SET_FS
MOVQ $SYS_arch_prctl, AX
SYSCALL
CMPQ AX, $0xfffffffffffff001
JLS 2(PC)
MOVL $0xf1, 0xf1 // crash
RET
TEXT runtime·osyield(SB),NOSPLIT,$0
MOVL $SYS_sched_yield, AX
SYSCALL
RET
TEXT runtime·sched_getaffinity(SB),NOSPLIT,$0
MOVQ pid+0(FP), DI
MOVQ len+8(FP), SI
MOVQ buf+16(FP), DX
MOVL $SYS_sched_getaffinity, AX
SYSCALL
MOVL AX, ret+24(FP)
RET
// int32 runtime·epollcreate(int32 size);
TEXT runtime·epollcreate(SB),NOSPLIT,$0
MOVL size+0(FP), DI
MOVL $SYS_epoll_create, AX
SYSCALL
MOVL AX, ret+8(FP)
RET
// int32 runtime·epollcreate1(int32 flags);
TEXT runtime·epollcreate1(SB),NOSPLIT,$0
MOVL flags+0(FP), DI
MOVL $SYS_epoll_create1, AX
SYSCALL
MOVL AX, ret+8(FP)
RET
// func epollctl(epfd, op, fd int32, ev *epollEvent) int
TEXT runtime·epollctl(SB),NOSPLIT,$0
MOVL epfd+0(FP), DI
MOVL op+4(FP), SI
MOVL fd+8(FP), DX
MOVQ ev+16(FP), R10
MOVL $SYS_epoll_ctl, AX
SYSCALL
MOVL AX, ret+24(FP)
RET
// int32 runtime·epollwait(int32 epfd, EpollEvent *ev, int32 nev, int32 timeout);
TEXT runtime·epollwait(SB),NOSPLIT,$0
// This uses pwait instead of wait, because Android O blocks wait.
MOVL epfd+0(FP), DI
MOVQ ev+8(FP), SI
MOVL nev+16(FP), DX
MOVL timeout+20(FP), R10
MOVQ $0, R8
MOVL $SYS_epoll_pwait, AX
SYSCALL
MOVL AX, ret+24(FP)
RET
// void runtime·closeonexec(int32 fd);
TEXT runtime·closeonexec(SB),NOSPLIT,$0
MOVL fd+0(FP), DI // fd
MOVQ $2, SI // F_SETFD
MOVQ $1, DX // FD_CLOEXEC
MOVL $SYS_fcntl, AX
SYSCALL
RET
// int access(const char *name, int mode)
TEXT runtime·access(SB),NOSPLIT,$0
// This uses faccessat instead of access, because Android O blocks access.
MOVL $AT_FDCWD, DI // AT_FDCWD, so this acts like access
MOVQ name+0(FP), SI
MOVL mode+8(FP), DX
MOVL $0, R10
MOVL $SYS_faccessat, AX
SYSCALL
MOVL AX, ret+16(FP)
RET
// int connect(int fd, const struct sockaddr *addr, socklen_t addrlen)
TEXT runtime·connect(SB),NOSPLIT,$0-28
MOVL fd+0(FP), DI
MOVQ addr+8(FP), SI
MOVL len+16(FP), DX
MOVL $SYS_connect, AX
SYSCALL
MOVL AX, ret+24(FP)
RET
// int socket(int domain, int type, int protocol)
TEXT runtime·socket(SB),NOSPLIT,$0-20
MOVL domain+0(FP), DI
MOVL typ+4(FP), SI
MOVL prot+8(FP), DX
MOVL $SYS_socket, AX
SYSCALL
MOVL AX, ret+16(FP)
RET
// func sbrk0() uintptr
TEXT runtime·sbrk0(SB),NOSPLIT,$0-8
// Implemented as brk(NULL).
MOVQ $0, DI
MOVL $SYS_brk, AX
SYSCALL
MOVQ AX, ret+0(FP)
RET
|