// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Cgo call and callback support.
//
// To call into the C function f from Go, the cgo-generated code calls
// runtime.cgocall(_cgo_Cfunc_f, frame), where _cgo_Cfunc_f is a
// gcc-compiled function written by cgo.
//
// runtime.cgocall (below) calls entersyscall so as not to block
// other goroutines or the garbage collector, and then calls
// runtime.asmcgocall(_cgo_Cfunc_f, frame).
//
// runtime.asmcgocall (in asm_$GOARCH.s) switches to the m->g0 stack
// (assumed to be an operating system-allocated stack, so safe to run
// gcc-compiled code on) and calls _cgo_Cfunc_f(frame).
//
// _cgo_Cfunc_f invokes the actual C function f with arguments
// taken from the frame structure, records the results in the frame,
// and returns to runtime.asmcgocall.
//
// After it regains control, runtime.asmcgocall switches back to the
// original g (m->curg)'s stack and returns to runtime.cgocall.
//
// After it regains control, runtime.cgocall calls exitsyscall, which blocks
// until this m can run Go code without violating the $GOMAXPROCS limit,
// and then unlocks g from m.
//
// The above description skipped over the possibility of the gcc-compiled
// function f calling back into Go. If that happens, we continue down
// the rabbit hole during the execution of f.
//
// To make it possible for gcc-compiled C code to call a Go function p.GoF,
// cgo writes a gcc-compiled function named GoF (not p.GoF, since gcc doesn't
// know about packages).  The gcc-compiled C function f calls GoF.
//
// GoF initializes "frame", a structure containing all of its
// arguments and slots for p.GoF's results. It calls
// crosscall2(_cgoexp_GoF, frame, framesize, ctxt) using the gcc ABI.
//
// crosscall2 (in cgo/asm_$GOARCH.s) is a four-argument adapter from
// the gcc function call ABI to the gc function call ABI. At this
// point we're in the Go runtime, but we're still running on m.g0's
// stack and outside the $GOMAXPROCS limit. crosscall2 calls
// runtime.cgocallback(_cgoexp_GoF, frame, ctxt) using the gc ABI.
// (crosscall2's framesize argument is no longer used, but there's one
// case where SWIG calls crosscall2 directly and expects to pass this
// argument. See _cgo_panic.)
//
// runtime.cgocallback (in asm_$GOARCH.s) switches from m.g0's stack
// to the original g (m.curg)'s stack, on which it calls
// runtime.cgocallbackg(_cgoexp_GoF, frame, ctxt). As part of the
// stack switch, runtime.cgocallback saves the current SP as
// m.g0.sched.sp, so that any use of m.g0's stack during the execution
// of the callback will be done below the existing stack frames.
// Before overwriting m.g0.sched.sp, it pushes the old value on the
// m.g0 stack, so that it can be restored later.
//
// runtime.cgocallbackg (below) is now running on a real goroutine
// stack (not an m.g0 stack).  First it calls runtime.exitsyscall, which will
// block until the $GOMAXPROCS limit allows running this goroutine.
// Once exitsyscall has returned, it is safe to do things like call the memory
// allocator or invoke the Go callback function.  runtime.cgocallbackg
// first defers a function to unwind m.g0.sched.sp, so that if p.GoF
// panics, m.g0.sched.sp will be restored to its old value: the m.g0 stack
// and the m.curg stack will be unwound in lock step.
// Then it calls _cgoexp_GoF(frame).
//
// _cgoexp_GoF, which was generated by cmd/cgo, unpacks the arguments
// from frame, calls p.GoF, writes the results back to frame, and
// returns. Now we start unwinding this whole process.
//
// runtime.cgocallbackg pops but does not execute the deferred
// function to unwind m.g0.sched.sp, calls runtime.entersyscall, and
// returns to runtime.cgocallback.
//
// After it regains control, runtime.cgocallback switches back to
// m.g0's stack (the pointer is still in m.g0.sched.sp), restores the old
// m.g0.sched.sp value from the stack, and returns to crosscall2.
//
// crosscall2 restores the callee-save registers for gcc and returns
// to GoF, which unpacks any result values and returns to f.

package runtime

import (
	
	
	
	
)

// Addresses collected in a cgo backtrace when crashing.
// Length must match arg.Max in x_cgo_callers in runtime/cgo/gcc_traceback.c.
type cgoCallers [32]uintptr

// argset matches runtime/cgo/linux_syscall.c:argset_t
type argset struct {
	args   unsafe.Pointer
	retval uintptr
}

// wrapper for syscall package to call cgocall for libc (cgo) calls.
//
//go:linkname syscall_cgocaller syscall.cgocaller
//go:nosplit
//go:uintptrescapes
func syscall_cgocaller( unsafe.Pointer,  ...uintptr) uintptr {
	 := argset{args: unsafe.Pointer(&[0])}
	cgocall(, unsafe.Pointer(&))
	return .retval
}

var ncgocall uint64 // number of cgo calls in total for dead m

// Call from Go to C.
//
// This must be nosplit because it's used for syscalls on some
// platforms. Syscalls may have untyped arguments on the stack, so
// it's not safe to grow or scan the stack.
//
//go:nosplit
func cgocall(,  unsafe.Pointer) int32 {
	if !iscgo && GOOS != "solaris" && GOOS != "illumos" && GOOS != "windows" {
		throw("cgocall unavailable")
	}

	if  == nil {
		throw("cgocall nil")
	}

	if raceenabled {
		racereleasemerge(unsafe.Pointer(&racecgosync))
	}

	 := getg().m
	.ncgocall++

	// Reset traceback.
	.cgoCallers[0] = 0

	// Announce we are entering a system call
	// so that the scheduler knows to create another
	// M to run goroutines while we are in the
	// foreign code.
	//
	// The call to asmcgocall is guaranteed not to
	// grow the stack and does not allocate memory,
	// so it is safe to call while "in a system call", outside
	// the $GOMAXPROCS accounting.
	//
	// fn may call back into Go code, in which case we'll exit the
	// "system call", run the Go code (which may grow the stack),
	// and then re-enter the "system call" reusing the PC and SP
	// saved by entersyscall here.
	entersyscall()

	// Tell asynchronous preemption that we're entering external
	// code. We do this after entersyscall because this may block
	// and cause an async preemption to fail, but at this point a
	// sync preemption will succeed (though this is not a matter
	// of correctness).
	osPreemptExtEnter()

	.incgo = true
	// We use ncgo as a check during execution tracing for whether there is
	// any C on the call stack, which there will be after this point. If
	// there isn't, we can use frame pointer unwinding to collect call
	// stacks efficiently. This will be the case for the first Go-to-C call
	// on a stack, so it's preferable to update it here, after we emit a
	// trace event in entersyscall above.
	.ncgo++

	 := asmcgocall(, )

	// Update accounting before exitsyscall because exitsyscall may
	// reschedule us on to a different M.
	.incgo = false
	.ncgo--

	osPreemptExtExit()

	exitsyscall()

	// Note that raceacquire must be called only after exitsyscall has
	// wired this M to a P.
	if raceenabled {
		raceacquire(unsafe.Pointer(&racecgosync))
	}

	// From the garbage collector's perspective, time can move
	// backwards in the sequence above. If there's a callback into
	// Go code, GC will see this function at the call to
	// asmcgocall. When the Go call later returns to C, the
	// syscall PC/SP is rolled back and the GC sees this function
	// back at the call to entersyscall. Normally, fn and arg
	// would be live at entersyscall and dead at asmcgocall, so if
	// time moved backwards, GC would see these arguments as dead
	// and then live. Prevent these undead arguments from crashing
	// GC by forcing them to stay live across this time warp.
	KeepAlive()
	KeepAlive()
	KeepAlive()

	return 
}

// Set or reset the system stack bounds for a callback on sp.
//
// Must be nosplit because it is called by needm prior to fully initializing
// the M.
//
//go:nosplit
func callbackUpdateSystemStack( *m,  uintptr,  bool) {
	 := .g0
	if  > .stack.lo &&  <= .stack.hi {
		// Stack already in bounds, nothing to do.
		return
	}

	if .ncgo > 0 {
		// ncgo > 0 indicates that this M was in Go further up the stack
		// (it called C and is now receiving a callback). It is not
		// safe for the C call to change the stack out from under us.

		// Note that this case isn't possible for signal == true, as
		// that is always passing a new M from needm.

		// Stack is bogus, but reset the bounds anyway so we can print.
		 := .stack.hi
		 := .stack.lo
		.stack.hi =  + 1024
		.stack.lo =  - 32*1024
		.stackguard0 = .stack.lo + stackGuard
		.stackguard1 = .stackguard0

		print("M ", .id, " procid ", .procid, " runtime: cgocallback with sp=", hex(), " out of bounds [", hex(), ", ", hex(), "]")
		print("\n")
		exit(2)
	}

	// This M does not have Go further up the stack. However, it may have
	// previously called into Go, initializing the stack bounds. Between
	// that call returning and now the stack may have changed (perhaps the
	// C thread is running a coroutine library). We need to update the
	// stack bounds for this case.
	//
	// Set the stack bounds to match the current stack. If we don't
	// actually know how big the stack is, like we don't know how big any
	// scheduling stack is, but we assume there's at least 32 kB. If we
	// can get a more accurate stack bound from pthread, use that, provided
	// it actually contains SP..
	.stack.hi =  + 1024
	.stack.lo =  - 32*1024
	if ! && _cgo_getstackbound != nil {
		// Don't adjust if called from the signal handler.
		// We are on the signal stack, not the pthread stack.
		// (We could get the stack bounds from sigaltstack, but
		// we're getting out of the signal handler very soon
		// anyway. Not worth it.)
		var  [2]uintptr
		asmcgocall(_cgo_getstackbound, unsafe.Pointer(&))
		// getstackbound is an unsupported no-op on Windows.
		//
		// Don't use these bounds if they don't contain SP. Perhaps we
		// were called by something not using the standard thread
		// stack.
		if [0] != 0 &&  > [0] &&  <= [1] {
			.stack.lo = [0]
			.stack.hi = [1]
		}
	}
	.stackguard0 = .stack.lo + stackGuard
	.stackguard1 = .stackguard0
}

// Call from C back to Go. fn must point to an ABIInternal Go entry-point.
//
//go:nosplit
func cgocallbackg(,  unsafe.Pointer,  uintptr) {
	 := getg()
	if  != .m.curg {
		println("runtime: bad g in cgocallback")
		exit(2)
	}

	 := .m.g0.sched.sp // system sp saved by cgocallback.
	callbackUpdateSystemStack(.m, , false)

	// The call from C is on gp.m's g0 stack, so we must ensure
	// that we stay on that M. We have to do this before calling
	// exitsyscall, since it would otherwise be free to move us to
	// a different M. The call to unlockOSThread is in this function
	// after cgocallbackg1, or in the case of panicking, in unwindm.
	lockOSThread()

	 := .m

	// Save current syscall parameters, so m.syscall can be
	// used again if callback decide to make syscall.
	 := .m.syscall

	// entersyscall saves the caller's SP to allow the GC to trace the Go
	// stack. However, since we're returning to an earlier stack frame and
	// need to pair with the entersyscall() call made by cgocall, we must
	// save syscall* and let reentersyscall restore them.
	 := unsafe.Pointer(.syscallsp)
	 := .syscallpc
	exitsyscall() // coming out of cgo call
	.m.incgo = false
	if .m.isextra {
		.m.isExtraInC = false
	}

	osPreemptExtExit(.m)

	if .nocgocallback {
		panic("runtime: function marked with #cgo nocallback called back into Go")
	}

	cgocallbackg1(, , )

	// At this point we're about to call unlockOSThread.
	// The following code must not change to a different m.
	// This is enforced by checking incgo in the schedule function.
	.m.incgo = true
	unlockOSThread()

	if .m.isextra {
		.m.isExtraInC = true
	}

	if .m !=  {
		throw("m changed unexpectedly in cgocallbackg")
	}

	osPreemptExtEnter(.m)

	// going back to cgo call
	reentersyscall(, uintptr())

	.m.syscall = 
}

func cgocallbackg1(,  unsafe.Pointer,  uintptr) {
	 := getg()

	if .m.needextram || extraMWaiters.Load() > 0 {
		.m.needextram = false
		systemstack(newextram)
	}

	if  != 0 {
		 := append(.cgoCtxt, )

		// Now we need to set gp.cgoCtxt = s, but we could get
		// a SIGPROF signal while manipulating the slice, and
		// the SIGPROF handler could pick up gp.cgoCtxt while
		// tracing up the stack.  We need to ensure that the
		// handler always sees a valid slice, so set the
		// values in an order such that it always does.
		 := (*slice)(unsafe.Pointer(&.cgoCtxt))
		atomicstorep(unsafe.Pointer(&.array), unsafe.Pointer(&[0]))
		.cap = cap()
		.len = len()

		defer func( *g) {
			// Decrease the length of the slice by one, safely.
			 := (*slice)(unsafe.Pointer(&.cgoCtxt))
			.len--
		}()
	}

	if .m.ncgo == 0 {
		// The C call to Go came from a thread not currently running
		// any Go. In the case of -buildmode=c-archive or c-shared,
		// this call may be coming in before package initialization
		// is complete. Wait until it is.
		<-main_init_done
	}

	// Check whether the profiler needs to be turned on or off; this route to
	// run Go code does not use runtime.execute, so bypasses the check there.
	 := sched.profilehz
	if .m.profilehz !=  {
		setThreadCPUProfiler()
	}

	// Add entry to defer stack in case of panic.
	 := true
	defer unwindm(&)

	if raceenabled {
		raceacquire(unsafe.Pointer(&racecgosync))
	}

	// Invoke callback. This function is generated by cmd/cgo and
	// will unpack the argument frame and call the Go function.
	var  func( unsafe.Pointer)
	 := funcval{uintptr()}
	*(*unsafe.Pointer)(unsafe.Pointer(&)) = noescape(unsafe.Pointer(&))
	()

	if raceenabled {
		racereleasemerge(unsafe.Pointer(&racecgosync))
	}

	// Do not unwind m->g0->sched.sp.
	// Our caller, cgocallback, will do that.
	 = false
}

func unwindm( *bool) {
	if * {
		// Restore sp saved by cgocallback during
		// unwind of g's stack (see comment at top of file).
		 := acquirem()
		 := &.g0.sched
		.sp = *(*uintptr)(unsafe.Pointer(.sp + alignUp(sys.MinFrameSize, sys.StackAlign)))

		// Do the accounting that cgocall will not have a chance to do
		// during an unwind.
		//
		// In the case where a Go call originates from C, ncgo is 0
		// and there is no matching cgocall to end.
		if .ncgo > 0 {
			.incgo = false
			.ncgo--
			osPreemptExtExit()
		}

		// Undo the call to lockOSThread in cgocallbackg, only on the
		// panicking path. In normal return case cgocallbackg will call
		// unlockOSThread, ensuring no preemption point after the unlock.
		// Here we don't need to worry about preemption, because we're
		// panicking out of the callback and unwinding the g0 stack,
		// instead of reentering cgo (which requires the same thread).
		unlockOSThread()

		releasem()
	}
}

// called from assembly.
func badcgocallback() {
	throw("misaligned stack in cgocallback")
}

// called from (incomplete) assembly.
func cgounimpl() {
	throw("cgo not implemented")
}

var racecgosync uint64 // represents possible synchronization in C code

// Pointer checking for cgo code.

// We want to detect all cases where a program that does not use
// unsafe makes a cgo call passing a Go pointer to memory that
// contains an unpinned Go pointer. Here a Go pointer is defined as a
// pointer to memory allocated by the Go runtime. Programs that use
// unsafe can evade this restriction easily, so we don't try to catch
// them. The cgo program will rewrite all possibly bad pointer
// arguments to call cgoCheckPointer, where we can catch cases of a Go
// pointer pointing to an unpinned Go pointer.

// Complicating matters, taking the address of a slice or array
// element permits the C program to access all elements of the slice
// or array. In that case we will see a pointer to a single element,
// but we need to check the entire data structure.

// The cgoCheckPointer call takes additional arguments indicating that
// it was called on an address expression. An additional argument of
// true means that it only needs to check a single element. An
// additional argument of a slice or array means that it needs to
// check the entire slice/array, but nothing else. Otherwise, the
// pointer could be anything, and we check the entire heap object,
// which is conservative but safe.

// When and if we implement a moving garbage collector,
// cgoCheckPointer will pin the pointer for the duration of the cgo
// call.  (This is necessary but not sufficient; the cgo program will
// also have to change to pin Go pointers that cannot point to Go
// pointers.)

// cgoCheckPointer checks if the argument contains a Go pointer that
// points to an unpinned Go pointer, and panics if it does.
func cgoCheckPointer( any,  any) {
	if !goexperiment.CgoCheck2 && debug.cgocheck == 0 {
		return
	}

	 := efaceOf(&)
	 := ._type

	 := true
	if  != nil && (.Kind_&kindMask == kindPtr || .Kind_&kindMask == kindUnsafePointer) {
		 := .data
		if .Kind_&kindDirectIface == 0 {
			 = *(*unsafe.Pointer)()
		}
		if  == nil || !cgoIsGoPointer() {
			return
		}
		 := efaceOf(&)
		switch ._type.Kind_ & kindMask {
		case kindBool:
			if .Kind_&kindMask == kindUnsafePointer {
				// We don't know the type of the element.
				break
			}
			 := (*ptrtype)(unsafe.Pointer())
			cgoCheckArg(.Elem, , true, false, cgoCheckPointerFail)
			return
		case kindSlice:
			// Check the slice rather than the pointer.
			 = 
			 = ._type
		case kindArray:
			// Check the array rather than the pointer.
			// Pass top as false since we have a pointer
			// to the array.
			 = 
			 = ._type
			 = false
		default:
			throw("can't happen")
		}
	}

	cgoCheckArg(, .data, .Kind_&kindDirectIface == 0, , cgoCheckPointerFail)
}

const cgoCheckPointerFail = "cgo argument has Go pointer to unpinned Go pointer"
const cgoResultFail = "cgo result is unpinned Go pointer or points to unpinned Go pointer"

// cgoCheckArg is the real work of cgoCheckPointer. The argument p
// is either a pointer to the value (of type t), or the value itself,
// depending on indir. The top parameter is whether we are at the top
// level, where Go pointers are allowed. Go pointers to pinned objects are
// allowed as long as they don't reference other unpinned pointers.
func cgoCheckArg( *_type,  unsafe.Pointer, ,  bool,  string) {
	if .PtrBytes == 0 ||  == nil {
		// If the type has no pointers there is nothing to do.
		return
	}

	switch .Kind_ & kindMask {
	default:
		throw("can't happen")
	case kindArray:
		 := (*arraytype)(unsafe.Pointer())
		if ! {
			if .Len != 1 {
				throw("can't happen")
			}
			(.Elem, , .Elem.Kind_&kindDirectIface == 0, , )
			return
		}
		for  := uintptr(0);  < .Len; ++ {
			(.Elem, , true, , )
			 = add(, .Elem.Size_)
		}
	case kindChan, kindMap:
		// These types contain internal pointers that will
		// always be allocated in the Go heap. It's never OK
		// to pass them to C.
		panic(errorString())
	case kindFunc:
		if  {
			 = *(*unsafe.Pointer)()
		}
		if !cgoIsGoPointer() {
			return
		}
		panic(errorString())
	case kindInterface:
		 := *(**_type)()
		if  == nil {
			return
		}
		// A type known at compile time is OK since it's
		// constant. A type not known at compile time will be
		// in the heap and will not be OK.
		if inheap(uintptr(unsafe.Pointer())) {
			panic(errorString())
		}
		 = *(*unsafe.Pointer)(add(, goarch.PtrSize))
		if !cgoIsGoPointer() {
			return
		}
		if ! && !isPinned() {
			panic(errorString())
		}
		(, , .Kind_&kindDirectIface == 0, false, )
	case kindSlice:
		 := (*slicetype)(unsafe.Pointer())
		 := (*slice)()
		 = .array
		if  == nil || !cgoIsGoPointer() {
			return
		}
		if ! && !isPinned() {
			panic(errorString())
		}
		if .Elem.PtrBytes == 0 {
			return
		}
		for  := 0;  < .cap; ++ {
			(.Elem, , true, false, )
			 = add(, .Elem.Size_)
		}
	case kindString:
		 := (*stringStruct)()
		if !cgoIsGoPointer(.str) {
			return
		}
		if ! && !isPinned(.str) {
			panic(errorString())
		}
	case kindStruct:
		 := (*structtype)(unsafe.Pointer())
		if ! {
			if len(.Fields) != 1 {
				throw("can't happen")
			}
			(.Fields[0].Typ, , .Fields[0].Typ.Kind_&kindDirectIface == 0, , )
			return
		}
		for ,  := range .Fields {
			if .Typ.PtrBytes == 0 {
				continue
			}
			(.Typ, add(, .Offset), true, , )
		}
	case kindPtr, kindUnsafePointer:
		if  {
			 = *(*unsafe.Pointer)()
			if  == nil {
				return
			}
		}

		if !cgoIsGoPointer() {
			return
		}
		if ! && !isPinned() {
			panic(errorString())
		}

		cgoCheckUnknownPointer(, )
	}
}

// cgoCheckUnknownPointer is called for an arbitrary pointer into Go
// memory. It checks whether that Go memory contains any other
// pointer into unpinned Go memory. If it does, we panic.
// The return values are unused but useful to see in panic tracebacks.
func cgoCheckUnknownPointer( unsafe.Pointer,  string) (,  uintptr) {
	if inheap(uintptr()) {
		, ,  := findObject(uintptr(), 0, 0)
		 = 
		if  == 0 {
			return
		}
		if goexperiment.AllocHeaders {
			 := .typePointersOfUnchecked()
			for {
				var  uintptr
				if ,  = .next( + .elemsize);  == 0 {
					break
				}
				 := *(*unsafe.Pointer)(unsafe.Pointer())
				if cgoIsGoPointer() && !isPinned() {
					panic(errorString())
				}
			}
		} else {
			 := .elemsize
			 := heapBitsForAddr(, )
			for {
				var  uintptr
				if ,  = .next();  == 0 {
					break
				}
				 := *(*unsafe.Pointer)(unsafe.Pointer())
				if cgoIsGoPointer() && !isPinned() {
					panic(errorString())
				}
			}
		}
		return
	}

	for ,  := range activeModules() {
		if cgoInRange(, .data, .edata) || cgoInRange(, .bss, .ebss) {
			// We have no way to know the size of the object.
			// We have to assume that it might contain a pointer.
			panic(errorString())
		}
		// In the text or noptr sections, we know that the
		// pointer does not point to a Go pointer.
	}

	return
}

// cgoIsGoPointer reports whether the pointer is a Go pointer--a
// pointer to Go memory. We only care about Go memory that might
// contain pointers.
//
//go:nosplit
//go:nowritebarrierrec
func cgoIsGoPointer( unsafe.Pointer) bool {
	if  == nil {
		return false
	}

	if inHeapOrStack(uintptr()) {
		return true
	}

	for ,  := range activeModules() {
		if cgoInRange(, .data, .edata) || cgoInRange(, .bss, .ebss) {
			return true
		}
	}

	return false
}

// cgoInRange reports whether p is between start and end.
//
//go:nosplit
//go:nowritebarrierrec
func cgoInRange( unsafe.Pointer, ,  uintptr) bool {
	return  <= uintptr() && uintptr() < 
}

// cgoCheckResult is called to check the result parameter of an
// exported Go function. It panics if the result is or contains any
// other pointer into unpinned Go memory.
func cgoCheckResult( any) {
	if !goexperiment.CgoCheck2 && debug.cgocheck == 0 {
		return
	}

	 := efaceOf(&)
	 := ._type
	cgoCheckArg(, .data, .Kind_&kindDirectIface == 0, false, cgoResultFail)
}