// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build aix || darwin || dragonfly || freebsd || (js && wasm) || linux || netbsd || openbsd || solaris || windows

package runtime

import (
	
	
)

// Integrated network poller (platform-independent part).
// A particular implementation (epoll/kqueue/port/AIX/Windows)
// must define the following functions:
//
// func netpollinit()
//     Initialize the poller. Only called once.
//
// func netpollopen(fd uintptr, pd *pollDesc) int32
//     Arm edge-triggered notifications for fd. The pd argument is to pass
//     back to netpollready when fd is ready. Return an errno value.
//
// func netpollclose(fd uintptr) int32
//     Disable notifications for fd. Return an errno value.
//
// func netpoll(delta int64) gList
//     Poll the network. If delta < 0, block indefinitely. If delta == 0,
//     poll without blocking. If delta > 0, block for up to delta nanoseconds.
//     Return a list of goroutines built by calling netpollready.
//
// func netpollBreak()
//     Wake up the network poller, assumed to be blocked in netpoll.
//
// func netpollIsPollDescriptor(fd uintptr) bool
//     Reports whether fd is a file descriptor used by the poller.

// Error codes returned by runtime_pollReset and runtime_pollWait.
// These must match the values in internal/poll/fd_poll_runtime.go.
const (
	pollNoError        = 0 // no error
	pollErrClosing     = 1 // descriptor is closed
	pollErrTimeout     = 2 // I/O timeout
	pollErrNotPollable = 3 // general error polling descriptor
)

// pollDesc contains 2 binary semaphores, rg and wg, to park reader and writer
// goroutines respectively. The semaphore can be in the following states:
// pdReady - io readiness notification is pending;
//           a goroutine consumes the notification by changing the state to nil.
// pdWait - a goroutine prepares to park on the semaphore, but not yet parked;
//          the goroutine commits to park by changing the state to G pointer,
//          or, alternatively, concurrent io notification changes the state to pdReady,
//          or, alternatively, concurrent timeout/close changes the state to nil.
// G pointer - the goroutine is blocked on the semaphore;
//             io notification or timeout/close changes the state to pdReady or nil respectively
//             and unparks the goroutine.
// nil - none of the above.
const (
	pdReady uintptr = 1
	pdWait  uintptr = 2
)

const pollBlockSize = 4 * 1024

// Network poller descriptor.
//
// No heap pointers.
//
//go:notinheap
type pollDesc struct {
	link *pollDesc // in pollcache, protected by pollcache.lock
	fd   uintptr   // constant for pollDesc usage lifetime

	// atomicInfo holds bits from closing, rd, and wd,
	// which are only ever written while holding the lock,
	// summarized for use by netpollcheckerr,
	// which cannot acquire the lock.
	// After writing these fields under lock in a way that
	// might change the summary, code must call publishInfo
	// before releasing the lock.
	// Code that changes fields and then calls netpollunblock
	// (while still holding the lock) must call publishInfo
	// before calling netpollunblock, because publishInfo is what
	// stops netpollblock from blocking anew
	// (by changing the result of netpollcheckerr).
	// atomicInfo also holds the eventErr bit,
	// recording whether a poll event on the fd got an error;
	// atomicInfo is the only source of truth for that bit.
	atomicInfo atomic.Uint32 // atomic pollInfo

	// rg, wg are accessed atomically and hold g pointers.
	// (Using atomic.Uintptr here is similar to using guintptr elsewhere.)
	rg atomic.Uintptr // pdReady, pdWait, G waiting for read or nil
	wg atomic.Uintptr // pdReady, pdWait, G waiting for write or nil

	lock    mutex // protects the following fields
	closing bool
	user    uint32    // user settable cookie
	rseq    uintptr   // protects from stale read timers
	rt      timer     // read deadline timer (set if rt.f != nil)
	rd      int64     // read deadline (a nanotime in the future, -1 when expired)
	wseq    uintptr   // protects from stale write timers
	wt      timer     // write deadline timer
	wd      int64     // write deadline (a nanotime in the future, -1 when expired)
	self    *pollDesc // storage for indirect interface. See (*pollDesc).makeArg.
}

// pollInfo is the bits needed by netpollcheckerr, stored atomically,
// mostly duplicating state that is manipulated under lock in pollDesc.
// The one exception is the pollEventErr bit, which is maintained only
// in the pollInfo.
type pollInfo uint32

const (
	pollClosing = 1 << iota
	pollEventErr
	pollExpiredReadDeadline
	pollExpiredWriteDeadline
)

func ( pollInfo) () bool              { return &pollClosing != 0 }
func ( pollInfo) () bool             { return &pollEventErr != 0 }
func ( pollInfo) () bool  { return &pollExpiredReadDeadline != 0 }
func ( pollInfo) () bool { return &pollExpiredWriteDeadline != 0 }

// info returns the pollInfo corresponding to pd.
func ( *pollDesc) () pollInfo {
	return pollInfo(.atomicInfo.Load())
}

// publishInfo updates pd.atomicInfo (returned by pd.info)
// using the other values in pd.
// It must be called while holding pd.lock,
// and it must be called after changing anything
// that might affect the info bits.
// In practice this means after changing closing
// or changing rd or wd from < 0 to >= 0.
func ( *pollDesc) () {
	var  uint32
	if .closing {
		 |= pollClosing
	}
	if .rd < 0 {
		 |= pollExpiredReadDeadline
	}
	if .wd < 0 {
		 |= pollExpiredWriteDeadline
	}

	// Set all of x except the pollEventErr bit.
	 := .atomicInfo.Load()
	for !.atomicInfo.CompareAndSwap(, (&pollEventErr)|) {
		 = .atomicInfo.Load()
	}
}

// setEventErr sets the result of pd.info().eventErr() to b.
func ( *pollDesc) ( bool) {
	 := .atomicInfo.Load()
	for (&pollEventErr != 0) !=  && !.atomicInfo.CompareAndSwap(, ^pollEventErr) {
		 = .atomicInfo.Load()
	}
}

type pollCache struct {
	lock  mutex
	first *pollDesc
	// PollDesc objects must be type-stable,
	// because we can get ready notification from epoll/kqueue
	// after the descriptor is closed/reused.
	// Stale notifications are detected using seq variable,
	// seq is incremented when deadlines are changed or descriptor is reused.
}

var (
	netpollInitLock mutex
	netpollInited   uint32

	pollcache      pollCache
	netpollWaiters uint32
)

//go:linkname poll_runtime_pollServerInit internal/poll.runtime_pollServerInit
func poll_runtime_pollServerInit() {
	netpollGenericInit()
}

func netpollGenericInit() {
	if atomic.Load(&netpollInited) == 0 {
		lockInit(&netpollInitLock, lockRankNetpollInit)
		lock(&netpollInitLock)
		if netpollInited == 0 {
			netpollinit()
			atomic.Store(&netpollInited, 1)
		}
		unlock(&netpollInitLock)
	}
}

func netpollinited() bool {
	return atomic.Load(&netpollInited) != 0
}

//go:linkname poll_runtime_isPollServerDescriptor internal/poll.runtime_isPollServerDescriptor

// poll_runtime_isPollServerDescriptor reports whether fd is a
// descriptor being used by netpoll.
func poll_runtime_isPollServerDescriptor( uintptr) bool {
	return netpollIsPollDescriptor()
}

//go:linkname poll_runtime_pollOpen internal/poll.runtime_pollOpen
func poll_runtime_pollOpen( uintptr) (*pollDesc, int) {
	 := pollcache.alloc()
	lock(&.lock)
	 := .wg.Load()
	if  != 0 &&  != pdReady {
		throw("runtime: blocked write on free polldesc")
	}
	 := .rg.Load()
	if  != 0 &&  != pdReady {
		throw("runtime: blocked read on free polldesc")
	}
	.fd = 
	.closing = false
	.setEventErr(false)
	.rseq++
	.rg.Store(0)
	.rd = 0
	.wseq++
	.wg.Store(0)
	.wd = 0
	.self = 
	.publishInfo()
	unlock(&.lock)

	 := netpollopen(, )
	if  != 0 {
		pollcache.free()
		return nil, int()
	}
	return , 0
}

//go:linkname poll_runtime_pollClose internal/poll.runtime_pollClose
func poll_runtime_pollClose( *pollDesc) {
	if !.closing {
		throw("runtime: close polldesc w/o unblock")
	}
	 := .wg.Load()
	if  != 0 &&  != pdReady {
		throw("runtime: blocked write on closing polldesc")
	}
	 := .rg.Load()
	if  != 0 &&  != pdReady {
		throw("runtime: blocked read on closing polldesc")
	}
	netpollclose(.fd)
	pollcache.free()
}

func ( *pollCache) ( *pollDesc) {
	lock(&.lock)
	.link = .first
	.first = 
	unlock(&.lock)
}

// poll_runtime_pollReset, which is internal/poll.runtime_pollReset,
// prepares a descriptor for polling in mode, which is 'r' or 'w'.
// This returns an error code; the codes are defined above.
//go:linkname poll_runtime_pollReset internal/poll.runtime_pollReset
func poll_runtime_pollReset( *pollDesc,  int) int {
	 := netpollcheckerr(, int32())
	if  != pollNoError {
		return 
	}
	if  == 'r' {
		.rg.Store(0)
	} else if  == 'w' {
		.wg.Store(0)
	}
	return pollNoError
}

// poll_runtime_pollWait, which is internal/poll.runtime_pollWait,
// waits for a descriptor to be ready for reading or writing,
// according to mode, which is 'r' or 'w'.
// This returns an error code; the codes are defined above.
//go:linkname poll_runtime_pollWait internal/poll.runtime_pollWait
func poll_runtime_pollWait( *pollDesc,  int) int {
	 := netpollcheckerr(, int32())
	if  != pollNoError {
		return 
	}
	// As for now only Solaris, illumos, and AIX use level-triggered IO.
	if GOOS == "solaris" || GOOS == "illumos" || GOOS == "aix" {
		netpollarm(, )
	}
	for !netpollblock(, int32(), false) {
		 = netpollcheckerr(, int32())
		if  != pollNoError {
			return 
		}
		// Can happen if timeout has fired and unblocked us,
		// but before we had a chance to run, timeout has been reset.
		// Pretend it has not happened and retry.
	}
	return pollNoError
}

//go:linkname poll_runtime_pollWaitCanceled internal/poll.runtime_pollWaitCanceled
func poll_runtime_pollWaitCanceled( *pollDesc,  int) {
	// This function is used only on windows after a failed attempt to cancel
	// a pending async IO operation. Wait for ioready, ignore closing or timeouts.
	for !netpollblock(, int32(), true) {
	}
}

//go:linkname poll_runtime_pollSetDeadline internal/poll.runtime_pollSetDeadline
func poll_runtime_pollSetDeadline( *pollDesc,  int64,  int) {
	lock(&.lock)
	if .closing {
		unlock(&.lock)
		return
	}
	,  := .rd, .wd
	 :=  > 0 &&  == 
	if  > 0 {
		 += nanotime()
		if  <= 0 {
			// If the user has a deadline in the future, but the delay calculation
			// overflows, then set the deadline to the maximum possible value.
			 = 1<<63 - 1
		}
	}
	if  == 'r' ||  == 'r'+'w' {
		.rd = 
	}
	if  == 'w' ||  == 'r'+'w' {
		.wd = 
	}
	.publishInfo()
	 := .rd > 0 && .rd == .wd
	 := netpollReadDeadline
	if  {
		 = netpollDeadline
	}
	if .rt.f == nil {
		if .rd > 0 {
			.rt.f = 
			// Copy current seq into the timer arg.
			// Timer func will check the seq against current descriptor seq,
			// if they differ the descriptor was reused or timers were reset.
			.rt.arg = .makeArg()
			.rt.seq = .rseq
			resettimer(&.rt, .rd)
		}
	} else if .rd !=  ||  !=  {
		.rseq++ // invalidate current timers
		if .rd > 0 {
			modtimer(&.rt, .rd, 0, , .makeArg(), .rseq)
		} else {
			deltimer(&.rt)
			.rt.f = nil
		}
	}
	if .wt.f == nil {
		if .wd > 0 && ! {
			.wt.f = netpollWriteDeadline
			.wt.arg = .makeArg()
			.wt.seq = .wseq
			resettimer(&.wt, .wd)
		}
	} else if .wd !=  ||  !=  {
		.wseq++ // invalidate current timers
		if .wd > 0 && ! {
			modtimer(&.wt, .wd, 0, netpollWriteDeadline, .makeArg(), .wseq)
		} else {
			deltimer(&.wt)
			.wt.f = nil
		}
	}
	// If we set the new deadline in the past, unblock currently pending IO if any.
	// Note that pd.publishInfo has already been called, above, immediately after modifying rd and wd.
	var ,  *g
	if .rd < 0 {
		 = netpollunblock(, 'r', false)
	}
	if .wd < 0 {
		 = netpollunblock(, 'w', false)
	}
	unlock(&.lock)
	if  != nil {
		netpollgoready(, 3)
	}
	if  != nil {
		netpollgoready(, 3)
	}
}

//go:linkname poll_runtime_pollUnblock internal/poll.runtime_pollUnblock
func poll_runtime_pollUnblock( *pollDesc) {
	lock(&.lock)
	if .closing {
		throw("runtime: unblock on closing polldesc")
	}
	.closing = true
	.rseq++
	.wseq++
	var ,  *g
	.publishInfo()
	 = netpollunblock(, 'r', false)
	 = netpollunblock(, 'w', false)
	if .rt.f != nil {
		deltimer(&.rt)
		.rt.f = nil
	}
	if .wt.f != nil {
		deltimer(&.wt)
		.wt.f = nil
	}
	unlock(&.lock)
	if  != nil {
		netpollgoready(, 3)
	}
	if  != nil {
		netpollgoready(, 3)
	}
}

// netpollready is called by the platform-specific netpoll function.
// It declares that the fd associated with pd is ready for I/O.
// The toRun argument is used to build a list of goroutines to return
// from netpoll. The mode argument is 'r', 'w', or 'r'+'w' to indicate
// whether the fd is ready for reading or writing or both.
//
// This may run while the world is stopped, so write barriers are not allowed.
//go:nowritebarrier
func netpollready( *gList,  *pollDesc,  int32) {
	var ,  *g
	if  == 'r' ||  == 'r'+'w' {
		 = netpollunblock(, 'r', true)
	}
	if  == 'w' ||  == 'r'+'w' {
		 = netpollunblock(, 'w', true)
	}
	if  != nil {
		.push()
	}
	if  != nil {
		.push()
	}
}

func netpollcheckerr( *pollDesc,  int32) int {
	 := .info()
	if .closing() {
		return pollErrClosing
	}
	if ( == 'r' && .expiredReadDeadline()) || ( == 'w' && .expiredWriteDeadline()) {
		return pollErrTimeout
	}
	// Report an event scanning error only on a read event.
	// An error on a write event will be captured in a subsequent
	// write call that is able to report a more specific error.
	if  == 'r' && .eventErr() {
		return pollErrNotPollable
	}
	return pollNoError
}

func netpollblockcommit( *g,  unsafe.Pointer) bool {
	 := atomic.Casuintptr((*uintptr)(), pdWait, uintptr(unsafe.Pointer()))
	if  {
		// Bump the count of goroutines waiting for the poller.
		// The scheduler uses this to decide whether to block
		// waiting for the poller if there is nothing else to do.
		atomic.Xadd(&netpollWaiters, 1)
	}
	return 
}

func netpollgoready( *g,  int) {
	atomic.Xadd(&netpollWaiters, -1)
	goready(, +1)
}

// returns true if IO is ready, or false if timedout or closed
// waitio - wait only for completed IO, ignore errors
// Concurrent calls to netpollblock in the same mode are forbidden, as pollDesc
// can hold only a single waiting goroutine for each mode.
func netpollblock( *pollDesc,  int32,  bool) bool {
	 := &.rg
	if  == 'w' {
		 = &.wg
	}

	// set the gpp semaphore to pdWait
	for {
		// Consume notification if already ready.
		if .CompareAndSwap(pdReady, 0) {
			return true
		}
		if .CompareAndSwap(0, pdWait) {
			break
		}

		// Double check that this isn't corrupt; otherwise we'd loop
		// forever.
		if  := .Load();  != pdReady &&  != 0 {
			throw("runtime: double wait")
		}
	}

	// need to recheck error states after setting gpp to pdWait
	// this is necessary because runtime_pollUnblock/runtime_pollSetDeadline/deadlineimpl
	// do the opposite: store to closing/rd/wd, publishInfo, load of rg/wg
	if  || netpollcheckerr(, ) == pollNoError {
		gopark(netpollblockcommit, unsafe.Pointer(), waitReasonIOWait, traceEvGoBlockNet, 5)
	}
	// be careful to not lose concurrent pdReady notification
	 := .Swap(0)
	if  > pdWait {
		throw("runtime: corrupted polldesc")
	}
	return  == pdReady
}

func netpollunblock( *pollDesc,  int32,  bool) *g {
	 := &.rg
	if  == 'w' {
		 = &.wg
	}

	for {
		 := .Load()
		if  == pdReady {
			return nil
		}
		if  == 0 && ! {
			// Only set pdReady for ioready. runtime_pollWait
			// will check for timeout/cancel before waiting.
			return nil
		}
		var  uintptr
		if  {
			 = pdReady
		}
		if .CompareAndSwap(, ) {
			if  == pdWait {
				 = 0
			}
			return (*g)(unsafe.Pointer())
		}
	}
}

func netpolldeadlineimpl( *pollDesc,  uintptr, ,  bool) {
	lock(&.lock)
	// Seq arg is seq when the timer was set.
	// If it's stale, ignore the timer event.
	 := .rseq
	if ! {
		 = .wseq
	}
	if  !=  {
		// The descriptor was reused or timers were reset.
		unlock(&.lock)
		return
	}
	var  *g
	if  {
		if .rd <= 0 || .rt.f == nil {
			throw("runtime: inconsistent read deadline")
		}
		.rd = -1
		.publishInfo()
		 = netpollunblock(, 'r', false)
	}
	var  *g
	if  {
		if .wd <= 0 || .wt.f == nil && ! {
			throw("runtime: inconsistent write deadline")
		}
		.wd = -1
		.publishInfo()
		 = netpollunblock(, 'w', false)
	}
	unlock(&.lock)
	if  != nil {
		netpollgoready(, 0)
	}
	if  != nil {
		netpollgoready(, 0)
	}
}

func netpollDeadline( any,  uintptr) {
	netpolldeadlineimpl(.(*pollDesc), , true, true)
}

func netpollReadDeadline( any,  uintptr) {
	netpolldeadlineimpl(.(*pollDesc), , true, false)
}

func netpollWriteDeadline( any,  uintptr) {
	netpolldeadlineimpl(.(*pollDesc), , false, true)
}

func ( *pollCache) () *pollDesc {
	lock(&.lock)
	if .first == nil {
		const  = unsafe.Sizeof(pollDesc{})
		 := pollBlockSize / 
		if  == 0 {
			 = 1
		}
		// Must be in non-GC memory because can be referenced
		// only from epoll/kqueue internals.
		 := persistentalloc(*, 0, &memstats.other_sys)
		for  := uintptr(0);  < ; ++ {
			 := (*pollDesc)(add(, *))
			.link = .first
			.first = 
		}
	}
	 := .first
	.first = .link
	lockInit(&.lock, lockRankPollDesc)
	unlock(&.lock)
	return 
}

// makeArg converts pd to an interface{}.
// makeArg does not do any allocation. Normally, such
// a conversion requires an allocation because pointers to
// go:notinheap types (which pollDesc is) must be stored
// in interfaces indirectly. See issue 42076.
func ( *pollDesc) () ( any) {
	 := (*eface)(unsafe.Pointer(&))
	._type = pdType
	.data = unsafe.Pointer(&.self)
	return
}

var (
	pdEface any    = (*pollDesc)(nil)
	pdType  *_type = efaceOf(&pdEface)._type
)