package runtime
import (
"internal/abi"
"internal/runtime/atomic"
"unsafe"
)
type Pinner struct {
*pinner
}
func (p *Pinner ) Pin (pointer any ) {
if p .pinner == nil {
mp := acquirem ()
if pp := mp .p .ptr (); pp != nil {
p .pinner = pp .pinnerCache
pp .pinnerCache = nil
}
releasem (mp )
if p .pinner == nil {
p .pinner = new (pinner )
p .refs = p .refStore [:0 ]
SetFinalizer (p .pinner , func (i *pinner ) {
if len (i .refs ) != 0 {
i .unpin ()
pinnerLeakPanic ()
}
})
}
}
ptr := pinnerGetPtr (&pointer )
if setPinned (ptr , true ) {
p .refs = append (p .refs , ptr )
}
}
func (p *Pinner ) Unpin () {
p .pinner .unpin ()
mp := acquirem ()
if pp := mp .p .ptr (); pp != nil && pp .pinnerCache == nil {
pp .pinnerCache = p .pinner
p .pinner = nil
}
releasem (mp )
}
const (
pinnerSize = 64
pinnerRefStoreSize = (pinnerSize - unsafe .Sizeof ([]unsafe .Pointer {})) / unsafe .Sizeof (unsafe .Pointer (nil ))
)
type pinner struct {
refs []unsafe .Pointer
refStore [pinnerRefStoreSize ]unsafe .Pointer
}
func (p *pinner ) unpin () {
if p == nil || p .refs == nil {
return
}
for i := range p .refs {
setPinned (p .refs [i ], false )
}
p .refStore = [pinnerRefStoreSize ]unsafe .Pointer {}
p .refs = p .refStore [:0 ]
}
func pinnerGetPtr(i *any ) unsafe .Pointer {
e := efaceOf (i )
etyp := e ._type
if etyp == nil {
panic (errorString ("runtime.Pinner: argument is nil" ))
}
if kind := etyp .Kind_ & abi .KindMask ; kind != abi .Pointer && kind != abi .UnsafePointer {
panic (errorString ("runtime.Pinner: argument is not a pointer: " + toRType (etyp ).string ()))
}
if inUserArenaChunk (uintptr (e .data )) {
panic (errorString ("runtime.Pinner: object was allocated into an arena" ))
}
return e .data
}
func isPinned(ptr unsafe .Pointer ) bool {
span := spanOfHeap (uintptr (ptr ))
if span == nil {
return true
}
pinnerBits := span .getPinnerBits ()
if pinnerBits == nil {
return false
}
objIndex := span .objIndex (uintptr (ptr ))
pinState := pinnerBits .ofObject (objIndex )
KeepAlive (ptr )
return pinState .isPinned ()
}
func setPinned(ptr unsafe .Pointer , pin bool ) bool {
span := spanOfHeap (uintptr (ptr ))
if span == nil {
if !pin {
panic (errorString ("tried to unpin non-Go pointer" ))
}
return false
}
mp := acquirem ()
span .ensureSwept ()
KeepAlive (ptr )
objIndex := span .objIndex (uintptr (ptr ))
lock (&span .speciallock )
pinnerBits := span .getPinnerBits ()
if pinnerBits == nil {
pinnerBits = span .newPinnerBits ()
span .setPinnerBits (pinnerBits )
}
pinState := pinnerBits .ofObject (objIndex )
if pin {
if pinState .isPinned () {
pinState .setMultiPinned (true )
systemstack (func () {
offset := objIndex * span .elemsize
span .incPinCounter (offset )
})
} else {
pinState .setPinned (true )
}
} else {
if pinState .isPinned () {
if pinState .isMultiPinned () {
var exists bool
systemstack (func () {
offset := objIndex * span .elemsize
exists = span .decPinCounter (offset )
})
if !exists {
pinState .setMultiPinned (false )
}
} else {
pinState .setPinned (false )
}
} else {
throw ("runtime.Pinner: object already unpinned" )
}
}
unlock (&span .speciallock )
releasem (mp )
return true
}
type pinState struct {
bytep *uint8
byteVal uint8
mask uint8
}
func (v *pinState ) isPinned () bool {
return (v .byteVal & v .mask ) != 0
}
func (v *pinState ) isMultiPinned () bool {
return (v .byteVal & (v .mask << 1 )) != 0
}
func (v *pinState ) setPinned (val bool ) {
v .set (val , false )
}
func (v *pinState ) setMultiPinned (val bool ) {
v .set (val , true )
}
func (v *pinState ) set (val bool , multipin bool ) {
mask := v .mask
if multipin {
mask <<= 1
}
if val {
atomic .Or8 (v .bytep , mask )
} else {
atomic .And8 (v .bytep , ^mask )
}
}
type pinnerBits gcBits
func (p *pinnerBits ) ofObject (n uintptr ) pinState {
bytep , mask := (*gcBits )(p ).bitp (n * 2 )
byteVal := atomic .Load8 (bytep )
return pinState {bytep , byteVal , mask }
}
func (s *mspan ) pinnerBitSize () uintptr {
return divRoundUp (uintptr (s .nelems )*2 , 8 )
}
func (s *mspan ) newPinnerBits () *pinnerBits {
return (*pinnerBits )(newMarkBits (uintptr (s .nelems ) * 2 ))
}
func (s *mspan ) getPinnerBits () *pinnerBits {
return (*pinnerBits )(atomic .Loadp (unsafe .Pointer (&s .pinnerBits )))
}
func (s *mspan ) setPinnerBits (p *pinnerBits ) {
atomicstorep (unsafe .Pointer (&s .pinnerBits ), unsafe .Pointer (p ))
}
func (s *mspan ) refreshPinnerBits () {
p := s .getPinnerBits ()
if p == nil {
return
}
hasPins := false
bytes := alignUp (s .pinnerBitSize (), 8 )
for _ , x := range unsafe .Slice ((*uint64 )(unsafe .Pointer (&p .x )), bytes /8 ) {
if x != 0 {
hasPins = true
break
}
}
if hasPins {
newPinnerBits := s .newPinnerBits ()
memmove (unsafe .Pointer (&newPinnerBits .x ), unsafe .Pointer (&p .x ), bytes )
s .setPinnerBits (newPinnerBits )
} else {
s .setPinnerBits (nil )
}
}
func (span *mspan ) incPinCounter (offset uintptr ) {
var rec *specialPinCounter
ref , exists := span .specialFindSplicePoint (offset , _KindSpecialPinCounter )
if !exists {
lock (&mheap_ .speciallock )
rec = (*specialPinCounter )(mheap_ .specialPinCounterAlloc .alloc ())
unlock (&mheap_ .speciallock )
rec .special .offset = uint16 (offset )
rec .special .kind = _KindSpecialPinCounter
rec .special .next = *ref
*ref = (*special )(unsafe .Pointer (rec ))
spanHasSpecials (span )
} else {
rec = (*specialPinCounter )(unsafe .Pointer (*ref ))
}
rec .counter ++
}
func (span *mspan ) decPinCounter (offset uintptr ) bool {
ref , exists := span .specialFindSplicePoint (offset , _KindSpecialPinCounter )
if !exists {
throw ("runtime.Pinner: decreased non-existing pin counter" )
}
counter := (*specialPinCounter )(unsafe .Pointer (*ref ))
counter .counter --
if counter .counter == 0 {
*ref = counter .special .next
if span .specials == nil {
spanHasNoSpecials (span )
}
lock (&mheap_ .speciallock )
mheap_ .specialPinCounterAlloc .free (unsafe .Pointer (counter ))
unlock (&mheap_ .speciallock )
return false
}
return true
}
func pinnerGetPinCounter(addr unsafe .Pointer ) *uintptr {
_ , span , objIndex := findObject (uintptr (addr ), 0 , 0 )
offset := objIndex * span .elemsize
t , exists := span .specialFindSplicePoint (offset , _KindSpecialPinCounter )
if !exists {
return nil
}
counter := (*specialPinCounter )(unsafe .Pointer (*t ))
return &counter .counter
}
var pinnerLeakPanic = func () {
panic (errorString ("runtime.Pinner: found leaking pinned pointer; forgot to call Unpin()?" ))
}
The pages are generated with Golds v0.6.9-preview . (GOOS=linux GOARCH=amd64)
Golds is a Go 101 project developed by Tapir Liu .
PR and bug reports are welcome and can be submitted to the issue list .
Please follow @Go100and1 (reachable from the left QR code) to get the latest news of Golds .