// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Go execution tracer.
// The tracer captures a wide range of execution events like goroutine
// creation/blocking/unblocking, syscall enter/exit/block, GC-related events,
// changes of heap size, processor start/stop, etc and writes them to a buffer
// in a compact form. A precise nanosecond-precision timestamp and a stack
// trace is captured for most events.
// See https://golang.org/s/go15trace for more info.

package runtime

import (
	
	
	
)

// Event types in the trace, args are given in square brackets.
const (
	traceEvNone              = 0  // unused
	traceEvBatch             = 1  // start of per-P batch of events [pid, timestamp]
	traceEvFrequency         = 2  // contains tracer timer frequency [frequency (ticks per second)]
	traceEvStack             = 3  // stack [stack id, number of PCs, array of {PC, func string ID, file string ID, line}]
	traceEvGomaxprocs        = 4  // current value of GOMAXPROCS [timestamp, GOMAXPROCS, stack id]
	traceEvProcStart         = 5  // start of P [timestamp, thread id]
	traceEvProcStop          = 6  // stop of P [timestamp]
	traceEvGCStart           = 7  // GC start [timestamp, seq, stack id]
	traceEvGCDone            = 8  // GC done [timestamp]
	traceEvGCSTWStart        = 9  // GC STW start [timestamp, kind]
	traceEvGCSTWDone         = 10 // GC STW done [timestamp]
	traceEvGCSweepStart      = 11 // GC sweep start [timestamp, stack id]
	traceEvGCSweepDone       = 12 // GC sweep done [timestamp, swept, reclaimed]
	traceEvGoCreate          = 13 // goroutine creation [timestamp, new goroutine id, new stack id, stack id]
	traceEvGoStart           = 14 // goroutine starts running [timestamp, goroutine id, seq]
	traceEvGoEnd             = 15 // goroutine ends [timestamp]
	traceEvGoStop            = 16 // goroutine stops (like in select{}) [timestamp, stack]
	traceEvGoSched           = 17 // goroutine calls Gosched [timestamp, stack]
	traceEvGoPreempt         = 18 // goroutine is preempted [timestamp, stack]
	traceEvGoSleep           = 19 // goroutine calls Sleep [timestamp, stack]
	traceEvGoBlock           = 20 // goroutine blocks [timestamp, stack]
	traceEvGoUnblock         = 21 // goroutine is unblocked [timestamp, goroutine id, seq, stack]
	traceEvGoBlockSend       = 22 // goroutine blocks on chan send [timestamp, stack]
	traceEvGoBlockRecv       = 23 // goroutine blocks on chan recv [timestamp, stack]
	traceEvGoBlockSelect     = 24 // goroutine blocks on select [timestamp, stack]
	traceEvGoBlockSync       = 25 // goroutine blocks on Mutex/RWMutex [timestamp, stack]
	traceEvGoBlockCond       = 26 // goroutine blocks on Cond [timestamp, stack]
	traceEvGoBlockNet        = 27 // goroutine blocks on network [timestamp, stack]
	traceEvGoSysCall         = 28 // syscall enter [timestamp, stack]
	traceEvGoSysExit         = 29 // syscall exit [timestamp, goroutine id, seq, real timestamp]
	traceEvGoSysBlock        = 30 // syscall blocks [timestamp]
	traceEvGoWaiting         = 31 // denotes that goroutine is blocked when tracing starts [timestamp, goroutine id]
	traceEvGoInSyscall       = 32 // denotes that goroutine is in syscall when tracing starts [timestamp, goroutine id]
	traceEvHeapAlloc         = 33 // memstats.heap_live change [timestamp, heap_alloc]
	traceEvNextGC            = 34 // memstats.next_gc change [timestamp, next_gc]
	traceEvTimerGoroutine    = 35 // not currently used; previously denoted timer goroutine [timer goroutine id]
	traceEvFutileWakeup      = 36 // denotes that the previous wakeup of this goroutine was futile [timestamp]
	traceEvString            = 37 // string dictionary entry [ID, length, string]
	traceEvGoStartLocal      = 38 // goroutine starts running on the same P as the last event [timestamp, goroutine id]
	traceEvGoUnblockLocal    = 39 // goroutine is unblocked on the same P as the last event [timestamp, goroutine id, stack]
	traceEvGoSysExitLocal    = 40 // syscall exit on the same P as the last event [timestamp, goroutine id, real timestamp]
	traceEvGoStartLabel      = 41 // goroutine starts running with label [timestamp, goroutine id, seq, label string id]
	traceEvGoBlockGC         = 42 // goroutine blocks on GC assist [timestamp, stack]
	traceEvGCMarkAssistStart = 43 // GC mark assist start [timestamp, stack]
	traceEvGCMarkAssistDone  = 44 // GC mark assist done [timestamp]
	traceEvUserTaskCreate    = 45 // trace.NewContext [timestamp, internal task id, internal parent task id, stack, name string]
	traceEvUserTaskEnd       = 46 // end of a task [timestamp, internal task id, stack]
	traceEvUserRegion        = 47 // trace.WithRegion [timestamp, internal task id, mode(0:start, 1:end), stack, name string]
	traceEvUserLog           = 48 // trace.Log [timestamp, internal task id, key string id, stack, value string]
	traceEvCount             = 49
	// Byte is used but only 6 bits are available for event type.
	// The remaining 2 bits are used to specify the number of arguments.
	// That means, the max event type value is 63.
)

const (
	// Timestamps in trace are cputicks/traceTickDiv.
	// This makes absolute values of timestamp diffs smaller,
	// and so they are encoded in less number of bytes.
	// 64 on x86 is somewhat arbitrary (one tick is ~20ns on a 3GHz machine).
	// The suggested increment frequency for PowerPC's time base register is
	// 512 MHz according to Power ISA v2.07 section 6.2, so we use 16 on ppc64
	// and ppc64le.
	// Tracing won't work reliably for architectures where cputicks is emulated
	// by nanotime, so the value doesn't matter for those architectures.
	traceTickDiv = 16 + 48*(sys.Goarch386|sys.GoarchAmd64)
	// Maximum number of PCs in a single stack trace.
	// Since events contain only stack id rather than whole stack trace,
	// we can allow quite large values here.
	traceStackSize = 128
	// Identifier of a fake P that is used when we trace without a real P.
	traceGlobProc = -1
	// Maximum number of bytes to encode uint64 in base-128.
	traceBytesPerNumber = 10
	// Shift of the number of arguments in the first event byte.
	traceArgCountShift = 6
	// Flag passed to traceGoPark to denote that the previous wakeup of this
	// goroutine was futile. For example, a goroutine was unblocked on a mutex,
	// but another goroutine got ahead and acquired the mutex before the first
	// goroutine is scheduled, so the first goroutine has to block again.
	// Such wakeups happen on buffered channels and sync.Mutex,
	// but are generally not interesting for end user.
	traceFutileWakeup byte = 128
)

// trace is global tracing context.
var trace struct {
	lock          mutex       // protects the following members
	lockOwner     *g          // to avoid deadlocks during recursive lock locks
	enabled       bool        // when set runtime traces events
	shutdown      bool        // set when we are waiting for trace reader to finish after setting enabled to false
	headerWritten bool        // whether ReadTrace has emitted trace header
	footerWritten bool        // whether ReadTrace has emitted trace footer
	shutdownSema  uint32      // used to wait for ReadTrace completion
	seqStart      uint64      // sequence number when tracing was started
	ticksStart    int64       // cputicks when tracing was started
	ticksEnd      int64       // cputicks when tracing was stopped
	timeStart     int64       // nanotime when tracing was started
	timeEnd       int64       // nanotime when tracing was stopped
	seqGC         uint64      // GC start/done sequencer
	reading       traceBufPtr // buffer currently handed off to user
	empty         traceBufPtr // stack of empty buffers
	fullHead      traceBufPtr // queue of full buffers
	fullTail      traceBufPtr
	reader        guintptr        // goroutine that called ReadTrace, or nil
	stackTab      traceStackTable // maps stack traces to unique ids

	// Dictionary for traceEvString.
	//
	// TODO: central lock to access the map is not ideal.
	//   option: pre-assign ids to all user annotation region names and tags
	//   option: per-P cache
	//   option: sync.Map like data structure
	stringsLock mutex
	strings     map[string]uint64
	stringSeq   uint64

	// markWorkerLabels maps gcMarkWorkerMode to string ID.
	markWorkerLabels [len(gcMarkWorkerModeStrings)]uint64

	bufLock mutex       // protects buf
	buf     traceBufPtr // global trace buffer, used when running without a p
}

// traceBufHeader is per-P tracing buffer.
type traceBufHeader struct {
	link      traceBufPtr             // in trace.empty/full
	lastTicks uint64                  // when we wrote the last event
	pos       int                     // next write offset in arr
	stk       [traceStackSize]uintptr // scratch buffer for traceback
}

// traceBuf is per-P tracing buffer.
//
//go:notinheap
type traceBuf struct {
	traceBufHeader
	arr [64<<10 - unsafe.Sizeof(traceBufHeader{})]byte // underlying buffer for traceBufHeader.buf
}

// traceBufPtr is a *traceBuf that is not traced by the garbage
// collector and doesn't have write barriers. traceBufs are not
// allocated from the GC'd heap, so this is safe, and are often
// manipulated in contexts where write barriers are not allowed, so
// this is necessary.
//
// TODO: Since traceBuf is now go:notinheap, this isn't necessary.
type traceBufPtr uintptr

func ( traceBufPtr) () *traceBuf   { return (*traceBuf)(unsafe.Pointer()) }
func ( *traceBufPtr) ( *traceBuf) { * = traceBufPtr(unsafe.Pointer()) }
func traceBufPtrOf( *traceBuf) traceBufPtr {
	return traceBufPtr(unsafe.Pointer())
}

// StartTrace enables tracing for the current process.
// While tracing, the data will be buffered and available via ReadTrace.
// StartTrace returns an error if tracing is already enabled.
// Most clients should use the runtime/trace package or the testing package's
// -test.trace flag instead of calling StartTrace directly.
func () error {
	// Stop the world so that we can take a consistent snapshot
	// of all goroutines at the beginning of the trace.
	// Do not stop the world during GC so we ensure we always see
	// a consistent view of GC-related events (e.g. a start is always
	// paired with an end).
	stopTheWorldGC("start tracing")

	// Prevent sysmon from running any code that could generate events.
	lock(&sched.sysmonlock)

	// We are in stop-the-world, but syscalls can finish and write to trace concurrently.
	// Exitsyscall could check trace.enabled long before and then suddenly wake up
	// and decide to write to trace at a random point in time.
	// However, such syscall will use the global trace.buf buffer, because we've
	// acquired all p's by doing stop-the-world. So this protects us from such races.
	lock(&trace.bufLock)

	if trace.enabled || trace.shutdown {
		unlock(&trace.bufLock)
		unlock(&sched.sysmonlock)
		startTheWorldGC()
		return errorString("tracing is already enabled")
	}

	// Can't set trace.enabled yet. While the world is stopped, exitsyscall could
	// already emit a delayed event (see exitTicks in exitsyscall) if we set trace.enabled here.
	// That would lead to an inconsistent trace:
	// - either GoSysExit appears before EvGoInSyscall,
	// - or GoSysExit appears for a goroutine for which we don't emit EvGoInSyscall below.
	// To instruct traceEvent that it must not ignore events below, we set startingtrace.
	// trace.enabled is set afterwards once we have emitted all preliminary events.
	 := getg()
	.m.startingtrace = true

	// Obtain current stack ID to use in all traceEvGoCreate events below.
	 := acquirem()
	 := make([]uintptr, traceStackSize)
	 := traceStackID(, , 2)
	releasem()

	for ,  := range allgs {
		 := readgstatus()
		if  != _Gdead {
			.traceseq = 0
			.tracelastp = getg().m.p
			// +PCQuantum because traceFrameForPC expects return PCs and subtracts PCQuantum.
			 := trace.stackTab.put([]uintptr{.startpc + sys.PCQuantum})
			traceEvent(traceEvGoCreate, -1, uint64(.goid), uint64(), )
		}
		if  == _Gwaiting {
			// traceEvGoWaiting is implied to have seq=1.
			.traceseq++
			traceEvent(traceEvGoWaiting, -1, uint64(.goid))
		}
		if  == _Gsyscall {
			.traceseq++
			traceEvent(traceEvGoInSyscall, -1, uint64(.goid))
		} else {
			.sysblocktraced = false
		}
	}
	traceProcStart()
	traceGoStart()
	// Note: ticksStart needs to be set after we emit traceEvGoInSyscall events.
	// If we do it the other way around, it is possible that exitsyscall will
	// query sysexitticks after ticksStart but before traceEvGoInSyscall timestamp.
	// It will lead to a false conclusion that cputicks is broken.
	trace.ticksStart = cputicks()
	trace.timeStart = nanotime()
	trace.headerWritten = false
	trace.footerWritten = false

	// string to id mapping
	//  0 : reserved for an empty string
	//  remaining: other strings registered by traceString
	trace.stringSeq = 0
	trace.strings = make(map[string]uint64)

	trace.seqGC = 0
	.m.startingtrace = false
	trace.enabled = true

	// Register runtime goroutine labels.
	, ,  := traceAcquireBuffer()
	for ,  := range gcMarkWorkerModeStrings[:] {
		trace.markWorkerLabels[],  = traceString(, , )
	}
	traceReleaseBuffer()

	unlock(&trace.bufLock)

	unlock(&sched.sysmonlock)

	startTheWorldGC()
	return nil
}

// StopTrace stops tracing, if it was previously enabled.
// StopTrace only returns after all the reads for the trace have completed.
func () {
	// Stop the world so that we can collect the trace buffers from all p's below,
	// and also to avoid races with traceEvent.
	stopTheWorldGC("stop tracing")

	// See the comment in StartTrace.
	lock(&sched.sysmonlock)

	// See the comment in StartTrace.
	lock(&trace.bufLock)

	if !trace.enabled {
		unlock(&trace.bufLock)
		unlock(&sched.sysmonlock)
		startTheWorldGC()
		return
	}

	traceGoSched()

	// Loop over all allocated Ps because dead Ps may still have
	// trace buffers.
	for ,  := range allp[:cap(allp)] {
		 := .tracebuf
		if  != 0 {
			traceFullQueue()
			.tracebuf = 0
		}
	}
	if trace.buf != 0 {
		 := trace.buf
		trace.buf = 0
		if .ptr().pos != 0 {
			traceFullQueue()
		}
	}

	for {
		trace.ticksEnd = cputicks()
		trace.timeEnd = nanotime()
		// Windows time can tick only every 15ms, wait for at least one tick.
		if trace.timeEnd != trace.timeStart {
			break
		}
		osyield()
	}

	trace.enabled = false
	trace.shutdown = true
	unlock(&trace.bufLock)

	unlock(&sched.sysmonlock)

	startTheWorldGC()

	// The world is started but we've set trace.shutdown, so new tracing can't start.
	// Wait for the trace reader to flush pending buffers and stop.
	semacquire(&trace.shutdownSema)
	if raceenabled {
		raceacquire(unsafe.Pointer(&trace.shutdownSema))
	}

	// The lock protects us from races with StartTrace/StopTrace because they do stop-the-world.
	lock(&trace.lock)
	for ,  := range allp[:cap(allp)] {
		if .tracebuf != 0 {
			throw("trace: non-empty trace buffer in proc")
		}
	}
	if trace.buf != 0 {
		throw("trace: non-empty global trace buffer")
	}
	if trace.fullHead != 0 || trace.fullTail != 0 {
		throw("trace: non-empty full trace buffer")
	}
	if trace.reading != 0 || trace.reader != 0 {
		throw("trace: reading after shutdown")
	}
	for trace.empty != 0 {
		 := trace.empty
		trace.empty = .ptr().link
		sysFree(unsafe.Pointer(), unsafe.Sizeof(*.ptr()), &memstats.other_sys)
	}
	trace.strings = nil
	trace.shutdown = false
	unlock(&trace.lock)
}

// ReadTrace returns the next chunk of binary tracing data, blocking until data
// is available. If tracing is turned off and all the data accumulated while it
// was on has been returned, ReadTrace returns nil. The caller must copy the
// returned data before calling ReadTrace again.
// ReadTrace must be called from one goroutine at a time.
func () []byte {
	// This function may need to lock trace.lock recursively
	// (goparkunlock -> traceGoPark -> traceEvent -> traceFlush).
	// To allow this we use trace.lockOwner.
	// Also this function must not allocate while holding trace.lock:
	// allocation can call heap allocate, which will try to emit a trace
	// event while holding heap lock.
	lock(&trace.lock)
	trace.lockOwner = getg()

	if trace.reader != 0 {
		// More than one goroutine reads trace. This is bad.
		// But we rather do not crash the program because of tracing,
		// because tracing can be enabled at runtime on prod servers.
		trace.lockOwner = nil
		unlock(&trace.lock)
		println("runtime: ReadTrace called from multiple goroutines simultaneously")
		return nil
	}
	// Recycle the old buffer.
	if  := trace.reading;  != 0 {
		.ptr().link = trace.empty
		trace.empty = 
		trace.reading = 0
	}
	// Write trace header.
	if !trace.headerWritten {
		trace.headerWritten = true
		trace.lockOwner = nil
		unlock(&trace.lock)
		return []byte("go 1.11 trace\x00\x00\x00")
	}
	// Wait for new data.
	if trace.fullHead == 0 && !trace.shutdown {
		trace.reader.set(getg())
		goparkunlock(&trace.lock, waitReasonTraceReaderBlocked, traceEvGoBlock, 2)
		lock(&trace.lock)
	}
	// Write a buffer.
	if trace.fullHead != 0 {
		 := traceFullDequeue()
		trace.reading = 
		trace.lockOwner = nil
		unlock(&trace.lock)
		return .ptr().arr[:.ptr().pos]
	}
	// Write footer with timer frequency.
	if !trace.footerWritten {
		trace.footerWritten = true
		// Use float64 because (trace.ticksEnd - trace.ticksStart) * 1e9 can overflow int64.
		 := float64(trace.ticksEnd-trace.ticksStart) * 1e9 / float64(trace.timeEnd-trace.timeStart) / traceTickDiv
		trace.lockOwner = nil
		unlock(&trace.lock)
		var  []byte
		 = append(, traceEvFrequency|0<<traceArgCountShift)
		 = traceAppend(, uint64())
		// This will emit a bunch of full buffers, we will pick them up
		// on the next iteration.
		trace.stackTab.dump()
		return 
	}
	// Done.
	if trace.shutdown {
		trace.lockOwner = nil
		unlock(&trace.lock)
		if raceenabled {
			// Model synchronization on trace.shutdownSema, which race
			// detector does not see. This is required to avoid false
			// race reports on writer passed to trace.Start.
			racerelease(unsafe.Pointer(&trace.shutdownSema))
		}
		// trace.enabled is already reset, so can call traceable functions.
		semrelease(&trace.shutdownSema)
		return nil
	}
	// Also bad, but see the comment above.
	trace.lockOwner = nil
	unlock(&trace.lock)
	println("runtime: spurious wakeup of trace reader")
	return nil
}

// traceReader returns the trace reader that should be woken up, if any.
func traceReader() *g {
	if trace.reader == 0 || (trace.fullHead == 0 && !trace.shutdown) {
		return nil
	}
	lock(&trace.lock)
	if trace.reader == 0 || (trace.fullHead == 0 && !trace.shutdown) {
		unlock(&trace.lock)
		return nil
	}
	 := trace.reader.ptr()
	trace.reader.set(nil)
	unlock(&trace.lock)
	return 
}

// traceProcFree frees trace buffer associated with pp.
func traceProcFree( *p) {
	 := .tracebuf
	.tracebuf = 0
	if  == 0 {
		return
	}
	lock(&trace.lock)
	traceFullQueue()
	unlock(&trace.lock)
}

// traceFullQueue queues buf into queue of full buffers.
func traceFullQueue( traceBufPtr) {
	.ptr().link = 0
	if trace.fullHead == 0 {
		trace.fullHead = 
	} else {
		trace.fullTail.ptr().link = 
	}
	trace.fullTail = 
}

// traceFullDequeue dequeues from queue of full buffers.
func traceFullDequeue() traceBufPtr {
	 := trace.fullHead
	if  == 0 {
		return 0
	}
	trace.fullHead = .ptr().link
	if trace.fullHead == 0 {
		trace.fullTail = 0
	}
	.ptr().link = 0
	return 
}

// traceEvent writes a single event to trace buffer, flushing the buffer if necessary.
// ev is event type.
// If skip > 0, write current stack id as the last argument (skipping skip top frames).
// If skip = 0, this event type should contain a stack, but we don't want
// to collect and remember it for this particular call.
func traceEvent( byte,  int,  ...uint64) {
	, ,  := traceAcquireBuffer()
	// Double-check trace.enabled now that we've done m.locks++ and acquired bufLock.
	// This protects from races between traceEvent and StartTrace/StopTrace.

	// The caller checked that trace.enabled == true, but trace.enabled might have been
	// turned off between the check and now. Check again. traceLockBuffer did mp.locks++,
	// StopTrace does stopTheWorld, and stopTheWorld waits for mp.locks to go back to zero,
	// so if we see trace.enabled == true now, we know it's true for the rest of the function.
	// Exitsyscall can run even during stopTheWorld. The race with StartTrace/StopTrace
	// during tracing in exitsyscall is resolved by locking trace.bufLock in traceLockBuffer.
	//
	// Note trace_userTaskCreate runs the same check.
	if !trace.enabled && !.startingtrace {
		traceReleaseBuffer()
		return
	}

	if  > 0 {
		if getg() == .curg {
			++ // +1 because stack is captured in traceEventLocked.
		}
	}
	traceEventLocked(0, , , , , , ...)
	traceReleaseBuffer()
}

func traceEventLocked( int,  *m,  int32,  *traceBufPtr,  byte,  int,  ...uint64) {
	 := .ptr()
	// TODO: test on non-zero extraBytes param.
	 := 2 + 5*traceBytesPerNumber +  // event type, length, sequence, timestamp, stack id and two add params
	if  == nil || len(.arr)-.pos <  {
		 = traceFlush(traceBufPtrOf(), ).ptr()
		.set()
	}

	 := uint64(cputicks()) / traceTickDiv
	 :=  - .lastTicks
	.lastTicks = 
	 := byte(len())
	if  >= 0 {
		++
	}
	// We have only 2 bits for number of arguments.
	// If number is >= 3, then the event type is followed by event length in bytes.
	if  > 3 {
		 = 3
	}
	 := .pos
	.byte( | <<traceArgCountShift)
	var  *byte
	if  == 3 {
		// Reserve the byte for length assuming that length < 128.
		.varint(0)
		 = &.arr[.pos-1]
	}
	.varint()
	for ,  := range  {
		.varint()
	}
	if  == 0 {
		.varint(0)
	} else if  > 0 {
		.varint(traceStackID(, .stk[:], ))
	}
	 := .pos - 
	if  >  {
		throw("invalid length of trace event")
	}
	if  != nil {
		// Fill in actual length.
		* = byte( - 2)
	}
}

func traceStackID( *m,  []uintptr,  int) uint64 {
	 := getg()
	 := .curg
	var  int
	if  ==  {
		 = callers(+1, )
	} else if  != nil {
		 = .curg
		 = gcallers(, , )
	}
	if  > 0 {
		-- // skip runtime.goexit
	}
	if  > 0 && .goid == 1 {
		-- // skip runtime.main
	}
	 := trace.stackTab.put([:])
	return uint64()
}

// traceAcquireBuffer returns trace buffer to use and, if necessary, locks it.
func traceAcquireBuffer() ( *m,  int32,  *traceBufPtr) {
	 = acquirem()
	if  := .p.ptr();  != nil {
		return , .id, &.tracebuf
	}
	lock(&trace.bufLock)
	return , traceGlobProc, &trace.buf
}

// traceReleaseBuffer releases a buffer previously acquired with traceAcquireBuffer.
func traceReleaseBuffer( int32) {
	if  == traceGlobProc {
		unlock(&trace.bufLock)
	}
	releasem(getg().m)
}

// traceFlush puts buf onto stack of full buffers and returns an empty buffer.
func traceFlush( traceBufPtr,  int32) traceBufPtr {
	 := trace.lockOwner
	 :=  == nil ||  != getg().m.curg
	if  {
		lock(&trace.lock)
	}
	if  != 0 {
		traceFullQueue()
	}
	if trace.empty != 0 {
		 = trace.empty
		trace.empty = .ptr().link
	} else {
		 = traceBufPtr(sysAlloc(unsafe.Sizeof(traceBuf{}), &memstats.other_sys))
		if  == 0 {
			throw("trace: out of memory")
		}
	}
	 := .ptr()
	.link.set(nil)
	.pos = 0

	// initialize the buffer for a new batch
	 := uint64(cputicks()) / traceTickDiv
	.lastTicks = 
	.byte(traceEvBatch | 1<<traceArgCountShift)
	.varint(uint64())
	.varint()

	if  {
		unlock(&trace.lock)
	}
	return 
}

// traceString adds a string to the trace.strings and returns the id.
func traceString( *traceBufPtr,  int32,  string) (uint64, *traceBufPtr) {
	if  == "" {
		return 0, 
	}

	lock(&trace.stringsLock)
	if raceenabled {
		// raceacquire is necessary because the map access
		// below is race annotated.
		raceacquire(unsafe.Pointer(&trace.stringsLock))
	}

	if ,  := trace.strings[];  {
		if raceenabled {
			racerelease(unsafe.Pointer(&trace.stringsLock))
		}
		unlock(&trace.stringsLock)

		return , 
	}

	trace.stringSeq++
	 := trace.stringSeq
	trace.strings[] = 

	if raceenabled {
		racerelease(unsafe.Pointer(&trace.stringsLock))
	}
	unlock(&trace.stringsLock)

	// memory allocation in above may trigger tracing and
	// cause *bufp changes. Following code now works with *bufp,
	// so there must be no memory allocation or any activities
	// that causes tracing after this point.

	 := .ptr()
	 := 1 + 2*traceBytesPerNumber + len()
	if  == nil || len(.arr)-.pos <  {
		 = traceFlush(traceBufPtrOf(), ).ptr()
		.set()
	}
	.byte(traceEvString)
	.varint()

	// double-check the string and the length can fit.
	// Otherwise, truncate the string.
	 := len()
	if  := len(.arr) - .pos;  < +traceBytesPerNumber {
		 = 
	}

	.varint(uint64())
	.pos += copy(.arr[.pos:], [:])

	.set()
	return , 
}

// traceAppend appends v to buf in little-endian-base-128 encoding.
func traceAppend( []byte,  uint64) []byte {
	for ;  >= 0x80;  >>= 7 {
		 = append(, 0x80|byte())
	}
	 = append(, byte())
	return 
}

// varint appends v to buf in little-endian-base-128 encoding.
func ( *traceBuf) ( uint64) {
	 := .pos
	for ;  >= 0x80;  >>= 7 {
		.arr[] = 0x80 | byte()
		++
	}
	.arr[] = byte()
	++
	.pos = 
}

// byte appends v to buf.
func ( *traceBuf) ( byte) {
	.arr[.pos] = 
	.pos++
}

// traceStackTable maps stack traces (arrays of PC's) to unique uint32 ids.
// It is lock-free for reading.
type traceStackTable struct {
	lock mutex
	seq  uint32
	mem  traceAlloc
	tab  [1 << 13]traceStackPtr
}

// traceStack is a single stack in traceStackTable.
type traceStack struct {
	link traceStackPtr
	hash uintptr
	id   uint32
	n    int
	stk  [0]uintptr // real type [n]uintptr
}

type traceStackPtr uintptr

func ( traceStackPtr) () *traceStack { return (*traceStack)(unsafe.Pointer()) }

// stack returns slice of PCs.
func ( *traceStack) () []uintptr {
	return (*[traceStackSize]uintptr)(unsafe.Pointer(&.stk))[:.n]
}

// put returns a unique id for the stack trace pcs and caches it in the table,
// if it sees the trace for the first time.
func ( *traceStackTable) ( []uintptr) uint32 {
	if len() == 0 {
		return 0
	}
	 := memhash(unsafe.Pointer(&[0]), 0, uintptr(len())*unsafe.Sizeof([0]))
	// First, search the hashtable w/o the mutex.
	if  := .find(, );  != 0 {
		return 
	}
	// Now, double check under the mutex.
	lock(&.lock)
	if  := .find(, );  != 0 {
		unlock(&.lock)
		return 
	}
	// Create new record.
	.seq++
	 := .newStack(len())
	.hash = 
	.id = .seq
	.n = len()
	 := .stack()
	for ,  := range  {
		[] = 
	}
	 := int( % uintptr(len(.tab)))
	.link = .tab[]
	atomicstorep(unsafe.Pointer(&.tab[]), unsafe.Pointer())
	unlock(&.lock)
	return .id
}

// find checks if the stack trace pcs is already present in the table.
func ( *traceStackTable) ( []uintptr,  uintptr) uint32 {
	 := int( % uintptr(len(.tab)))
:
	for  := .tab[].ptr();  != nil;  = .link.ptr() {
		if .hash ==  && .n == len() {
			for ,  := range .stack() {
				if  != [] {
					continue 
				}
			}
			return .id
		}
	}
	return 0
}

// newStack allocates a new stack of size n.
func ( *traceStackTable) ( int) *traceStack {
	return (*traceStack)(.mem.alloc(unsafe.Sizeof(traceStack{}) + uintptr()*sys.PtrSize))
}

// allFrames returns all of the Frames corresponding to pcs.
func allFrames( []uintptr) []Frame {
	 := make([]Frame, 0, len())
	 := CallersFrames()
	for {
		,  := .Next()
		 = append(, )
		if ! {
			return 
		}
	}
}

// dump writes all previously cached stacks to trace buffers,
// releases all memory and resets state.
func ( *traceStackTable) () {
	var  [(2 + 4*traceStackSize) * traceBytesPerNumber]byte
	 := traceFlush(0, 0)
	for ,  := range .tab {
		 := .ptr()
		for ;  != nil;  = .link.ptr() {
			 := [:0]
			 = traceAppend(, uint64(.id))
			 := allFrames(.stack())
			 = traceAppend(, uint64(len()))
			for ,  := range  {
				var  traceFrame
				,  = traceFrameForPC(, 0, )
				 = traceAppend(, uint64(.PC))
				 = traceAppend(, uint64(.funcID))
				 = traceAppend(, uint64(.fileID))
				 = traceAppend(, uint64(.line))
			}
			// Now copy to the buffer.
			 := 1 + traceBytesPerNumber + len()
			if  := .ptr(); len(.arr)-.pos <  {
				 = traceFlush(, 0)
			}
			 := .ptr()
			.byte(traceEvStack | 3<<traceArgCountShift)
			.varint(uint64(len()))
			.pos += copy(.arr[.pos:], )
		}
	}

	lock(&trace.lock)
	traceFullQueue()
	unlock(&trace.lock)

	.mem.drop()
	* = traceStackTable{}
	lockInit(&((*).lock), lockRankTraceStackTab)
}

type traceFrame struct {
	funcID uint64
	fileID uint64
	line   uint64
}

// traceFrameForPC records the frame information.
// It may allocate memory.
func traceFrameForPC( traceBufPtr,  int32,  Frame) (traceFrame, traceBufPtr) {
	 := &
	var  traceFrame

	 := .Function
	const  = 1 << 10
	if len() >  {
		 = [len()-:]
	}
	.funcID,  = traceString(, , )
	.line = uint64(.Line)
	 := .File
	if len() >  {
		 = [len()-:]
	}
	.fileID,  = traceString(, , )
	return , (*)
}

// traceAlloc is a non-thread-safe region allocator.
// It holds a linked list of traceAllocBlock.
type traceAlloc struct {
	head traceAllocBlockPtr
	off  uintptr
}

// traceAllocBlock is a block in traceAlloc.
//
// traceAllocBlock is allocated from non-GC'd memory, so it must not
// contain heap pointers. Writes to pointers to traceAllocBlocks do
// not need write barriers.
//
//go:notinheap
type traceAllocBlock struct {
	next traceAllocBlockPtr
	data [64<<10 - sys.PtrSize]byte
}

// TODO: Since traceAllocBlock is now go:notinheap, this isn't necessary.
type traceAllocBlockPtr uintptr

func ( traceAllocBlockPtr) () *traceAllocBlock   { return (*traceAllocBlock)(unsafe.Pointer()) }
func ( *traceAllocBlockPtr) ( *traceAllocBlock) { * = traceAllocBlockPtr(unsafe.Pointer()) }

// alloc allocates n-byte block.
func ( *traceAlloc) ( uintptr) unsafe.Pointer {
	 = alignUp(, sys.PtrSize)
	if .head == 0 || .off+ > uintptr(len(.head.ptr().data)) {
		if  > uintptr(len(.head.ptr().data)) {
			throw("trace: alloc too large")
		}
		 := (*traceAllocBlock)(sysAlloc(unsafe.Sizeof(traceAllocBlock{}), &memstats.other_sys))
		if  == nil {
			throw("trace: out of memory")
		}
		.next.set(.head.ptr())
		.head.set()
		.off = 0
	}
	 := &.head.ptr().data[.off]
	.off += 
	return unsafe.Pointer()
}

// drop frees all previously allocated memory and resets the allocator.
func ( *traceAlloc) () {
	for .head != 0 {
		 := .head.ptr()
		.head.set(.next.ptr())
		sysFree(unsafe.Pointer(), unsafe.Sizeof(traceAllocBlock{}), &memstats.other_sys)
	}
}

// The following functions write specific events to trace.

func traceGomaxprocs( int32) {
	traceEvent(traceEvGomaxprocs, 1, uint64())
}

func traceProcStart() {
	traceEvent(traceEvProcStart, -1, uint64(getg().m.id))
}

func traceProcStop( *p) {
	// Sysmon and stopTheWorld can stop Ps blocked in syscalls,
	// to handle this we temporary employ the P.
	 := acquirem()
	 := .p
	.p.set()
	traceEvent(traceEvProcStop, -1)
	.p = 
	releasem()
}

func traceGCStart() {
	traceEvent(traceEvGCStart, 3, trace.seqGC)
	trace.seqGC++
}

func traceGCDone() {
	traceEvent(traceEvGCDone, -1)
}

func traceGCSTWStart( int) {
	traceEvent(traceEvGCSTWStart, -1, uint64())
}

func traceGCSTWDone() {
	traceEvent(traceEvGCSTWDone, -1)
}

// traceGCSweepStart prepares to trace a sweep loop. This does not
// emit any events until traceGCSweepSpan is called.
//
// traceGCSweepStart must be paired with traceGCSweepDone and there
// must be no preemption points between these two calls.
func traceGCSweepStart() {
	// Delay the actual GCSweepStart event until the first span
	// sweep. If we don't sweep anything, don't emit any events.
	 := getg().m.p.ptr()
	if .traceSweep {
		throw("double traceGCSweepStart")
	}
	.traceSweep, .traceSwept, .traceReclaimed = true, 0, 0
}

// traceGCSweepSpan traces the sweep of a single page.
//
// This may be called outside a traceGCSweepStart/traceGCSweepDone
// pair; however, it will not emit any trace events in this case.
func traceGCSweepSpan( uintptr) {
	 := getg().m.p.ptr()
	if .traceSweep {
		if .traceSwept == 0 {
			traceEvent(traceEvGCSweepStart, 1)
		}
		.traceSwept += 
	}
}

func traceGCSweepDone() {
	 := getg().m.p.ptr()
	if !.traceSweep {
		throw("missing traceGCSweepStart")
	}
	if .traceSwept != 0 {
		traceEvent(traceEvGCSweepDone, -1, uint64(.traceSwept), uint64(.traceReclaimed))
	}
	.traceSweep = false
}

func traceGCMarkAssistStart() {
	traceEvent(traceEvGCMarkAssistStart, 1)
}

func traceGCMarkAssistDone() {
	traceEvent(traceEvGCMarkAssistDone, -1)
}

func traceGoCreate( *g,  uintptr) {
	.traceseq = 0
	.tracelastp = getg().m.p
	// +PCQuantum because traceFrameForPC expects return PCs and subtracts PCQuantum.
	 := trace.stackTab.put([]uintptr{ + sys.PCQuantum})
	traceEvent(traceEvGoCreate, 2, uint64(.goid), uint64())
}

func traceGoStart() {
	 := getg().m.curg
	 := .m.p
	.traceseq++
	if .ptr().gcMarkWorkerMode != gcMarkWorkerNotWorker {
		traceEvent(traceEvGoStartLabel, -1, uint64(.goid), .traceseq, trace.markWorkerLabels[.ptr().gcMarkWorkerMode])
	} else if .tracelastp ==  {
		traceEvent(traceEvGoStartLocal, -1, uint64(.goid))
	} else {
		.tracelastp = 
		traceEvent(traceEvGoStart, -1, uint64(.goid), .traceseq)
	}
}

func traceGoEnd() {
	traceEvent(traceEvGoEnd, -1)
}

func traceGoSched() {
	 := getg()
	.tracelastp = .m.p
	traceEvent(traceEvGoSched, 1)
}

func traceGoPreempt() {
	 := getg()
	.tracelastp = .m.p
	traceEvent(traceEvGoPreempt, 1)
}

func traceGoPark( byte,  int) {
	if &traceFutileWakeup != 0 {
		traceEvent(traceEvFutileWakeup, -1)
	}
	traceEvent( & ^traceFutileWakeup, )
}

func traceGoUnpark( *g,  int) {
	 := getg().m.p
	.traceseq++
	if .tracelastp ==  {
		traceEvent(traceEvGoUnblockLocal, , uint64(.goid))
	} else {
		.tracelastp = 
		traceEvent(traceEvGoUnblock, , uint64(.goid), .traceseq)
	}
}

func traceGoSysCall() {
	traceEvent(traceEvGoSysCall, 1)
}

func traceGoSysExit( int64) {
	if  != 0 &&  < trace.ticksStart {
		// There is a race between the code that initializes sysexitticks
		// (in exitsyscall, which runs without a P, and therefore is not
		// stopped with the rest of the world) and the code that initializes
		// a new trace. The recorded sysexitticks must therefore be treated
		// as "best effort". If they are valid for this trace, then great,
		// use them for greater accuracy. But if they're not valid for this
		// trace, assume that the trace was started after the actual syscall
		// exit (but before we actually managed to start the goroutine,
		// aka right now), and assign a fresh time stamp to keep the log consistent.
		 = 0
	}
	 := getg().m.curg
	.traceseq++
	.tracelastp = .m.p
	traceEvent(traceEvGoSysExit, -1, uint64(.goid), .traceseq, uint64()/traceTickDiv)
}

func traceGoSysBlock( *p) {
	// Sysmon and stopTheWorld can declare syscalls running on remote Ps as blocked,
	// to handle this we temporary employ the P.
	 := acquirem()
	 := .p
	.p.set()
	traceEvent(traceEvGoSysBlock, -1)
	.p = 
	releasem()
}

func traceHeapAlloc() {
	traceEvent(traceEvHeapAlloc, -1, memstats.heap_live)
}

func traceNextGC() {
	if  := atomic.Load64(&memstats.next_gc);  == ^uint64(0) {
		// Heap-based triggering is disabled.
		traceEvent(traceEvNextGC, -1, 0)
	} else {
		traceEvent(traceEvNextGC, -1, )
	}
}

// To access runtime functions from runtime/trace.
// See runtime/trace/annotation.go

//go:linkname trace_userTaskCreate runtime/trace.userTaskCreate
func trace_userTaskCreate(,  uint64,  string) {
	if !trace.enabled {
		return
	}

	// Same as in traceEvent.
	, ,  := traceAcquireBuffer()
	if !trace.enabled && !.startingtrace {
		traceReleaseBuffer()
		return
	}

	,  := traceString(, , )
	traceEventLocked(0, , , , traceEvUserTaskCreate, 3, , , )
	traceReleaseBuffer()
}

//go:linkname trace_userTaskEnd runtime/trace.userTaskEnd
func trace_userTaskEnd( uint64) {
	traceEvent(traceEvUserTaskEnd, 2, )
}

//go:linkname trace_userRegion runtime/trace.userRegion
func trace_userRegion(,  uint64,  string) {
	if !trace.enabled {
		return
	}

	, ,  := traceAcquireBuffer()
	if !trace.enabled && !.startingtrace {
		traceReleaseBuffer()
		return
	}

	,  := traceString(, , )
	traceEventLocked(0, , , , traceEvUserRegion, 3, , , )
	traceReleaseBuffer()
}

//go:linkname trace_userLog runtime/trace.userLog
func trace_userLog( uint64, ,  string) {
	if !trace.enabled {
		return
	}

	, ,  := traceAcquireBuffer()
	if !trace.enabled && !.startingtrace {
		traceReleaseBuffer()
		return
	}

	,  := traceString(, , )

	 := traceBytesPerNumber + len() // extraSpace for the value string
	traceEventLocked(, , , , traceEvUserLog, 3, , )
	// traceEventLocked reserved extra space for val and len(val)
	// in buf, so buf now has room for the following.
	 := .ptr()

	// double-check the message and its length can fit.
	// Otherwise, truncate the message.
	 := len()
	if  := len(.arr) - .pos;  < +traceBytesPerNumber {
		 = 
	}
	.varint(uint64())
	.pos += copy(.arr[.pos:], [:])

	traceReleaseBuffer()
}