languages/go
languages/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-10-14 20:05:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

runtime: make mheap statically allocated again

Browse source 
This depends on: 9791044: runtime: allocate page table lazily
Once page table is moved out of heap, the heap becomes small.
This removes unnecessary dereferences during heap access.
No logical changes.

R=golang-dev, khr
CC=golang-dev
https://golang.org/cl/9802043
This commit is contained in:
Dmitriy Vyukov 2013-05-28 22:14:47 +04:00
parent 671814b904
commit 8bbb08533d
8 changed files with 96 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

61
src/pkg/runtime/malloc.goc
View file

@ -14,7 +14,7 @@ package runtime
#include "typekind.h"
#include "race.h"
MHeap *runtime·mheap;
MHeap runtime·mheap;
int32 runtime·checking;
@ -81,7 +81,7 @@ runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
npages = size >> PageShift;
if((size & PageMask) != 0)
npages++;
s = runtime·MHeap_Alloc(runtime·mheap, npages, 0, 1, zeroed);
s = runtime·MHeap_Alloc(&runtime·mheap, npages, 0, 1, zeroed);
if(s == nil)
runtime·throw("out of memory");
size = npages<<PageShift;
@ -95,9 +95,9 @@ runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
if (sizeof(void*) == 4 && c->local_total_alloc >= (1<<30)) {
// purge cache stats to prevent overflow
runtime·lock(runtime·mheap);
runtime·lock(&runtime·mheap);
runtime·purgecachedstats(c);
runtime·unlock(runtime·mheap);
runtime·unlock(&runtime·mheap);
}
if(!(flag & FlagNoGC))
@ -181,7 +181,7 @@ runtime·free(void *v)
// they might coalesce v into other spans and change the bitmap further.
runtime·markfreed(v, size);
runtime·unmarkspan(v, 1<<PageShift);
runtime·MHeap_Free(runtime·mheap, s, 1);
runtime·MHeap_Free(&runtime·mheap, s, 1);
} else {
// Small object.
size = runtime·class_to_size[sizeclass];
@ -211,12 +211,12 @@ runtime·mlookup(void *v, byte **base, uintptr *size, MSpan **sp)
m->mcache->local_nlookup++;
if (sizeof(void*) == 4 && m->mcache->local_nlookup >= (1<<30)) {
// purge cache stats to prevent overflow
runtime·lock(runtime·mheap);
runtime·lock(&runtime·mheap);
runtime·purgecachedstats(m->mcache);
runtime·unlock(runtime·mheap);
runtime·unlock(&runtime·mheap);
}
s = runtime·MHeap_LookupMaybe(runtime·mheap, v);
s = runtime·MHeap_LookupMaybe(&runtime·mheap, v);
if(sp)
*sp = s;
if(s == nil) {
@ -260,11 +260,11 @@ runtime·allocmcache(void)
intgo rate;
MCache *c;
runtime·lock(runtime·mheap);
c = runtime·FixAlloc_Alloc(&runtime·mheap->cachealloc);
mstats.mcache_inuse = runtime·mheap->cachealloc.inuse;
mstats.mcache_sys = runtime·mheap->cachealloc.sys;
runtime·unlock(runtime·mheap);
runtime·lock(&runtime·mheap);
c = runtime·FixAlloc_Alloc(&runtime·mheap.cachealloc);
mstats.mcache_inuse = runtime·mheap.cachealloc.inuse;
mstats.mcache_sys = runtime·mheap.cachealloc.sys;
runtime·unlock(&runtime·mheap);
runtime·memclr((byte*)c, sizeof(*c));
// Set first allocation sample size.
@ -281,10 +281,10 @@ void
runtime·freemcache(MCache *c)
{
runtime·MCache_ReleaseAll(c);
runtime·lock(runtime·mheap);
runtime·lock(&runtime·mheap);
runtime·purgecachedstats(c);
runtime·FixAlloc_Free(&runtime·mheap->cachealloc, c);
runtime·unlock(runtime·mheap);
runtime·FixAlloc_Free(&runtime·mheap.cachealloc, c);
runtime·unlock(&runtime·mheap);
}
void
@ -339,9 +339,6 @@ runtime·mallocinit(void)
USED(bitmap_size);
USED(spans_size);
if((runtime·mheap = runtime·SysAlloc(sizeof(*runtime·mheap))) == nil)
runtime·throw("runtime: cannot allocate heap metadata");
runtime·InitSizes();
// limit = runtime·memlimit();
@ -377,7 +374,7 @@ runtime·mallocinit(void)
// If this fails we fall back to the 32 bit memory mechanism
arena_size = MaxMem;
bitmap_size = arena_size / (sizeof(void*)*8/4);
spans_size = arena_size / PageSize * sizeof(runtime·mheap->map[0]);
spans_size = arena_size / PageSize * sizeof(runtime·mheap.map[0]);
p = runtime·SysReserve((void*)(0x00c0ULL<<32), bitmap_size + spans_size + arena_size);
}
if (p == nil) {
@ -400,11 +397,11 @@ runtime·mallocinit(void)
// of address space, which is probably too much in a 32-bit world.
bitmap_size = MaxArena32 / (sizeof(void*)*8/4);
arena_size = 512<<20;
spans_size = MaxArena32 / PageSize * sizeof(runtime·mheap->map[0]);
spans_size = MaxArena32 / PageSize * sizeof(runtime·mheap.map[0]);
if(limit > 0 && arena_size+bitmap_size+spans_size > limit) {
bitmap_size = (limit / 9) & ~((1<<PageShift) - 1);
arena_size = bitmap_size * 8;
spans_size = arena_size / PageSize * sizeof(runtime·mheap->map[0]);
spans_size = arena_size / PageSize * sizeof(runtime·mheap.map[0]);
}
// SysReserve treats the address we ask for, end, as a hint,
@ -427,14 +424,14 @@ runtime·mallocinit(void)
if((uintptr)p & (((uintptr)1<<PageShift)-1))
runtime·throw("runtime: SysReserve returned unaligned address");
runtime·mheap->map = (MSpan**)p;
runtime·mheap->bitmap = p + spans_size;
runtime·mheap->arena_start = p + spans_size + bitmap_size;
runtime·mheap->arena_used = runtime·mheap->arena_start;
runtime·mheap->arena_end = runtime·mheap->arena_start + arena_size;
runtime·mheap.map = (MSpan**)p;
runtime·mheap.bitmap = p + spans_size;
runtime·mheap.arena_start = p + spans_size + bitmap_size;
runtime·mheap.arena_used = runtime·mheap.arena_start;
runtime·mheap.arena_end = runtime·mheap.arena_start + arena_size;
// Initialize the rest of the allocator.
runtime·MHeap_Init(runtime·mheap, runtime·SysAlloc);
runtime·MHeap_Init(&runtime·mheap, runtime·SysAlloc);
m->mcache = runtime·allocmcache();
// See if it works.
@ -534,8 +531,8 @@ runtime·settype_flush(M *mp, bool sysalloc)
// (Manually inlined copy of runtime·MHeap_Lookup)
p = (uintptr)v>>PageShift;
if(sizeof(void*) == 8)
p -= (uintptr)runtime·mheap->arena_start >> PageShift;
s = runtime·mheap->map[p];
p -= (uintptr)runtime·mheap.arena_start >> PageShift;
s = runtime·mheap.map[p];
if(s->sizeclass == 0) {
s->types.compression = MTypes_Single;
@ -652,7 +649,7 @@ runtime·settype(void *v, uintptr t)
}
if(DebugTypeAtBlockEnd) {
s = runtime·MHeap_Lookup(runtime·mheap, v);
s = runtime·MHeap_Lookup(&runtime·mheap, v);
*(uintptr*)((uintptr)v+s->elemsize-sizeof(uintptr)) = t;
}
}
@ -691,7 +688,7 @@ runtime·gettype(void *v)
uintptr t, ofs;
byte *data;
s = runtime·MHeap_LookupMaybe(runtime·mheap, v);
s = runtime·MHeap_LookupMaybe(&runtime·mheap, v);
if(s != nil) {
t = 0;
switch(s->types.compression) {

2
src/pkg/runtime/malloc.h
View file

@ -433,7 +433,7 @@ struct MHeap
FixAlloc spanalloc; // allocator for Span*
FixAlloc cachealloc; // allocator for MCache*
};
extern MHeap *runtime·mheap;
extern MHeap runtime·mheap;
void runtime·MHeap_Init(MHeap *h, void *(*allocator)(uintptr));
MSpan* runtime·MHeap_Alloc(MHeap *h, uintptr npage, int32 sizeclass, int32 acct, int32 zeroed);

6
src/pkg/runtime/mcache.c
View file

@ -19,7 +19,7 @@ runtime·MCache_Refill(MCache *c, int32 sizeclass)
l = &c->list[sizeclass];
if(l->list)
runtime·throw("MCache_Refill: the list is not empty");
l->nlist = runtime·MCentral_AllocList(&runtime·mheap->central[sizeclass], &l->list);
l->nlist = runtime·MCentral_AllocList(&runtime·mheap.central[sizeclass], &l->list);
if(l->list == nil)
runtime·throw("out of memory");
}
@ -41,7 +41,7 @@ ReleaseN(MCacheList *l, int32 n, int32 sizeclass)
l->nlist -= n;
// Return them to central free list.
runtime·MCentral_FreeList(&runtime·mheap->central[sizeclass], first);
runtime·MCentral_FreeList(&runtime·mheap.central[sizeclass], first);
}
void
@ -74,7 +74,7 @@ runtime·MCache_ReleaseAll(MCache *c)
for(i=0; i<NumSizeClasses; i++) {
l = &c->list[i];
if(l->list) {
runtime·MCentral_FreeList(&runtime·mheap->central[i], l->list);
runtime·MCentral_FreeList(&runtime·mheap.central[i], l->list);
l->list = nil;
l->nlist = 0;
}

8
src/pkg/runtime/mcentral.c
View file

@ -85,7 +85,7 @@ MCentral_Free(MCentral *c, void *v)
int32 size;
// Find span for v.
s = runtime·MHeap_Lookup(runtime·mheap, v);
s = runtime·MHeap_Lookup(&runtime·mheap, v);
if(s == nil || s->ref == 0)
runtime·throw("invalid free");
@ -110,7 +110,7 @@ MCentral_Free(MCentral *c, void *v)
s->freelist = nil;
c->nfree -= (s->npages << PageShift) / size;
runtime·unlock(c);
runtime·MHeap_Free(runtime·mheap, s, 0);
runtime·MHeap_Free(&runtime·mheap, s, 0);
runtime·lock(c);
}
}
@ -145,7 +145,7 @@ runtime·MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink *
c->nfree -= (s->npages << PageShift) / size;
runtime·unlock(c);
runtime·unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift);
runtime·MHeap_Free(runtime·mheap, s, 0);
runtime·MHeap_Free(&runtime·mheap, s, 0);
} else {
runtime·unlock(c);
}
@ -177,7 +177,7 @@ MCentral_Grow(MCentral *c)
runtime·unlock(c);
runtime·MGetSizeClassInfo(c->sizeclass, &size, &npages, &n);
s = runtime·MHeap_Alloc(runtime·mheap, npages, c->sizeclass, 0, 1);
s = runtime·MHeap_Alloc(&runtime·mheap, npages, c->sizeclass, 0, 1);
if(s == nil) {
// TODO(rsc): Log out of memory
runtime·lock(c);

106
src/pkg/runtime/mgc0.c
View file

@ -204,7 +204,7 @@ markonly(void *obj)
PageID k;
// Words outside the arena cannot be pointers.
if(obj < runtime·mheap->arena_start || obj >= runtime·mheap->arena_used)
if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used)
return false;
// obj may be a pointer to a live object.
@ -214,8 +214,8 @@ markonly(void *obj)
obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1));
// Find bits for this word.
off = (uintptr*)obj - (uintptr*)runtime·mheap->arena_start;
bitp = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1;
off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start;
bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
shift = off % wordsPerBitmapWord;
xbits = *bitp;
bits = xbits >> shift;
@ -229,8 +229,8 @@ markonly(void *obj)
k = (uintptr)obj>>PageShift;
x = k;
if(sizeof(void*) == 8)
x -= (uintptr)runtime·mheap->arena_start>>PageShift;
s = runtime·mheap->map[x];
x -= (uintptr)runtime·mheap.arena_start>>PageShift;
s = runtime·mheap.map[x];
if(s == nil || k < s->start || k - s->start >= s->npages || s->state != MSpanInUse)
return false;
p = (byte*)((uintptr)s->start<<PageShift);
@ -245,8 +245,8 @@ markonly(void *obj)
}
// Now that we know the object header, reload bits.
off = (uintptr*)obj - (uintptr*)runtime·mheap->arena_start;
bitp = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1;
off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start;
bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
shift = off % wordsPerBitmapWord;
xbits = *bitp;
bits = xbits >> shift;
@ -328,7 +328,7 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
Workbuf *wbuf;
PtrTarget *ptrbuf_end;
arena_start = runtime·mheap->arena_start;
arena_start = runtime·mheap.arena_start;
wp = *_wp;
wbuf = *_wbuf;
@ -367,7 +367,7 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
// obj belongs to interval [mheap.arena_start, mheap.arena_used).
if(Debug > 1) {
if(obj < runtime·mheap->arena_start || obj >= runtime·mheap->arena_used)
if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used)
runtime·throw("object is outside of mheap");
}
@ -410,7 +410,7 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
x = k;
if(sizeof(void*) == 8)
x -= (uintptr)arena_start>>PageShift;
s = runtime·mheap->map[x];
s = runtime·mheap.map[x];
if(s == nil || k < s->start || k - s->start >= s->npages || s->state != MSpanInUse)
continue;
p = (byte*)((uintptr)s->start<<PageShift);
@ -458,7 +458,7 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
x = (uintptr)obj >> PageShift;
if(sizeof(void*) == 8)
x -= (uintptr)arena_start>>PageShift;
s = runtime·mheap->map[x];
s = runtime·mheap.map[x];
PREFETCH(obj);
@ -566,7 +566,7 @@ checkptr(void *obj, uintptr objti)
if(!Debug)
runtime·throw("checkptr is debug only");
if(obj < runtime·mheap->arena_start || obj >= runtime·mheap->arena_used)
if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used)
return;
type = runtime·gettype(obj);
t = (Type*)(type & ~(uintptr)(PtrSize-1));
@ -574,8 +574,8 @@ checkptr(void *obj, uintptr objti)
return;
x = (uintptr)obj >> PageShift;
if(sizeof(void*) == 8)
x -= (uintptr)(runtime·mheap->arena_start)>>PageShift;
s = runtime·mheap->map[x];
x -= (uintptr)(runtime·mheap.arena_start)>>PageShift;
s = runtime·mheap.map[x];
objstart = (byte*)((uintptr)s->start<<PageShift);
if(s->sizeclass != 0) {
i = ((byte*)obj - objstart)/s->elemsize;
@ -645,8 +645,8 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
runtime·throw("scanblock: size of Workbuf is suboptimal");
// Memory arena parameters.
arena_start = runtime·mheap->arena_start;
arena_used = runtime·mheap->arena_used;
arena_start = runtime·mheap.arena_start;
arena_used = runtime·mheap.arena_used;
stack_ptr = stack+nelem(stack)-1;
@ -1157,14 +1157,14 @@ debug_scanblock(byte *b, uintptr n)
obj = (byte*)vp[i];
// Words outside the arena cannot be pointers.
if((byte*)obj < runtime·mheap->arena_start || (byte*)obj >= runtime·mheap->arena_used)
if((byte*)obj < runtime·mheap.arena_start || (byte*)obj >= runtime·mheap.arena_used)
continue;
// Round down to word boundary.
obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1));
// Consult span table to find beginning.
s = runtime·MHeap_LookupMaybe(runtime·mheap, obj);
s = runtime·MHeap_LookupMaybe(&runtime·mheap, obj);
if(s == nil)
continue;
@ -1180,8 +1180,8 @@ debug_scanblock(byte *b, uintptr n)
}
// Now that we know the object header, reload bits.
off = (uintptr*)obj - (uintptr*)runtime·mheap->arena_start;
bitp = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1;
off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start;
bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
shift = off % wordsPerBitmapWord;
xbits = *bitp;
bits = xbits >> shift;
@ -1521,8 +1521,8 @@ addroots(void)
addroot((Obj){bss, ebss - bss, (uintptr)gcbss});
// MSpan.types
allspans = runtime·mheap->allspans;
for(spanidx=0; spanidx<runtime·mheap->nspan; spanidx++) {
allspans = runtime·mheap.allspans;
for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) {
s = allspans[spanidx];
if(s->state == MSpanInUse) {
// The garbage collector ignores type pointers stored in MSpan.types:
@ -1624,10 +1624,10 @@ sweepspan(ParFor *desc, uint32 idx)
MSpan *s;
USED(&desc);
s = runtime·mheap->allspans[idx];
s = runtime·mheap.allspans[idx];
if(s->state != MSpanInUse)
return;
arena_start = runtime·mheap->arena_start;
arena_start = runtime·mheap.arena_start;
p = (byte*)(s->start << PageShift);
cl = s->sizeclass;
size = s->elemsize;
@ -1691,7 +1691,7 @@ sweepspan(ParFor *desc, uint32 idx)
// Free large span.
runtime·unmarkspan(p, 1<<PageShift);
*(uintptr*)p = (uintptr)0xdeaddeaddeaddeadll; // needs zeroing
runtime·MHeap_Free(runtime·mheap, s, 1);
runtime·MHeap_Free(&runtime·mheap, s, 1);
c->local_alloc -= size;
c->local_nfree++;
} else {
@ -1719,7 +1719,7 @@ sweepspan(ParFor *desc, uint32 idx)
c->local_nfree += nfree;
c->local_cachealloc -= nfree * size;
c->local_objects -= nfree;
runtime·MCentral_FreeSpan(&runtime·mheap->central[cl], s, nfree, head.next, end);
runtime·MCentral_FreeSpan(&runtime·mheap.central[cl], s, nfree, head.next, end);
}
}
@ -1733,10 +1733,10 @@ dumpspan(uint32 idx)
MSpan *s;
bool allocated, special;
s = runtime·mheap->allspans[idx];
s = runtime·mheap.allspans[idx];
if(s->state != MSpanInUse)
return;
arena_start = runtime·mheap->arena_start;
arena_start = runtime·mheap.arena_start;
p = (byte*)(s->start << PageShift);
sizeclass = s->sizeclass;
size = s->elemsize;
@ -1794,7 +1794,7 @@ runtime·memorydump(void)
{
uint32 spanidx;
for(spanidx=0; spanidx<runtime·mheap->nspan; spanidx++) {
for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) {
dumpspan(spanidx);
}
}
@ -1995,7 +1995,7 @@ gc(struct gc_args *args)
work.nproc = runtime·gcprocs();
addroots();
runtime·parforsetup(work.markfor, work.nproc, work.nroot, nil, false, markroot);
runtime·parforsetup(work.sweepfor, work.nproc, runtime·mheap->nspan, nil, true, sweepspan);
runtime·parforsetup(work.sweepfor, work.nproc, runtime·mheap.nspan, nil, true, sweepspan);
if(work.nproc > 1) {
runtime·noteclear(&work.alldone);
runtime·helpgc(work.nproc);
@ -2121,7 +2121,7 @@ runtime∕debug·readGCStats(Slice *pauses)
// Pass back: pauses, last gc (absolute time), number of gc, total pause ns.
p = (uint64*)pauses->array;
runtime·lock(runtime·mheap);
runtime·lock(&runtime·mheap);
n = mstats.numgc;
if(n > nelem(mstats.pause_ns))
n = nelem(mstats.pause_ns);
@ -2136,21 +2136,21 @@ runtime∕debug·readGCStats(Slice *pauses)
p[n] = mstats.last_gc;
p[n+1] = mstats.numgc;
p[n+2] = mstats.pause_total_ns;
runtime·unlock(runtime·mheap);
runtime·unlock(&runtime·mheap);
pauses->len = n+3;
}
void
runtime∕debug·setGCPercent(intgo in, intgo out)
{
runtime·lock(runtime·mheap);
runtime·lock(&runtime·mheap);
if(gcpercent == GcpercentUnknown)
gcpercent = readgogc();
out = gcpercent;
if(in < 0)
in = -1;
gcpercent = in;
runtime·unlock(runtime·mheap);
runtime·unlock(&runtime·mheap);
FLUSH(&out);
}
@ -2218,11 +2218,11 @@ runtime·markallocated(void *v, uintptr n, bool noptr)
if(0)
runtime·printf("markallocated %p+%p\n", v, n);
if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·mheap->arena_start)
if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
runtime·throw("markallocated: bad pointer");
off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; // word offset
b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1;
off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset
b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
shift = off % wordsPerBitmapWord;
for(;;) {
@ -2250,11 +2250,11 @@ runtime·markfreed(void *v, uintptr n)
if(0)
runtime·printf("markallocated %p+%p\n", v, n);
if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·mheap->arena_start)
if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
runtime·throw("markallocated: bad pointer");
off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; // word offset
b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1;
off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset
b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
shift = off % wordsPerBitmapWord;
for(;;) {
@ -2280,11 +2280,11 @@ runtime·checkfreed(void *v, uintptr n)
if(!runtime·checking)
return;
if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·mheap->arena_start)
if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
return; // not allocated, so okay
off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; // word offset
b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1;
off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset
b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
shift = off % wordsPerBitmapWord;
bits = *b>>shift;
@ -2303,7 +2303,7 @@ runtime·markspan(void *v, uintptr size, uintptr n, bool leftover)
uintptr *b, off, shift;
byte *p;
if((byte*)v+size*n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·mheap->arena_start)
if((byte*)v+size*n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
runtime·throw("markspan: bad pointer");
p = v;
@ -2314,8 +2314,8 @@ runtime·markspan(void *v, uintptr size, uintptr n, bool leftover)
// the entire span, and each bitmap word has bits for only
// one span, so no other goroutines are changing these
// bitmap words.
off = (uintptr*)p - (uintptr*)runtime·mheap->arena_start; // word offset
b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1;
off = (uintptr*)p - (uintptr*)runtime·mheap.arena_start; // word offset
b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
shift = off % wordsPerBitmapWord;
*b = (*b & ~(bitMask<<shift)) | (bitBlockBoundary<<shift);
}
@ -2327,14 +2327,14 @@ runtime·unmarkspan(void *v, uintptr n)
{
uintptr *p, *b, off;
if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·mheap->arena_start)
if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
runtime·throw("markspan: bad pointer");
p = v;
off = p - (uintptr*)runtime·mheap->arena_start; // word offset
off = p - (uintptr*)runtime·mheap.arena_start; // word offset
if(off % wordsPerBitmapWord != 0)
runtime·throw("markspan: unaligned pointer");
b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1;
b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
n /= PtrSize;
if(n%wordsPerBitmapWord != 0)
runtime·throw("unmarkspan: unaligned length");
@ -2355,8 +2355,8 @@ runtime·blockspecial(void *v)
if(DebugMark)
return true;
off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start;
b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1;
off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start;
b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
shift = off % wordsPerBitmapWord;
return (*b & (bitSpecial<<shift)) != 0;
@ -2370,8 +2370,8 @@ runtime·setblockspecial(void *v, bool s)
if(DebugMark)
return;
off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start;
b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1;
off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start;
b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
shift = off % wordsPerBitmapWord;
for(;;) {

8
src/pkg/runtime/mheap.c
View file

@ -424,7 +424,7 @@ scavenge(uint64 now, uint64 limit)
uintptr sumreleased;
MHeap *h;
h = runtime·mheap;
h = &runtime·mheap;
sumreleased = 0;
for(i=0; i < nelem(h->free); i++)
sumreleased += scavengelist(&h->free[i], now, limit);
@ -467,7 +467,7 @@ runtime·MHeap_Scavenger(void)
if(env != nil)
trace = runtime·atoi(env) > 0;
h = runtime·mheap;
h = &runtime·mheap;
for(k=0;; k++) {
runtime·noteclear(&note);
runtime·entersyscallblock();
@ -509,9 +509,9 @@ void
runtime∕debug·freeOSMemory(void)
{
runtime·gc(1);
runtime·lock(runtime·mheap);
runtime·lock(&runtime·mheap);
scavenge(~(uintptr)0, 0);
runtime·unlock(runtime·mheap);
runtime·unlock(&runtime·mheap);
}
// Initialize a new span with the given start and npages.

2
src/pkg/runtime/panic.c
View file

@ -384,7 +384,7 @@ nomatch:
void
runtime·startpanic(void)
{
if(runtime·mheap == 0 || runtime·mheap->cachealloc.size == 0) { // very early
if(runtime·mheap.cachealloc.size == 0) { // very early
runtime·printf("runtime: panic before malloc heap initialized\n");
m->mallocing = 1; // tell rest of panic not to try to malloc
} else if(m->mcache == nil) // can happen if called from signal handler or throw

2
src/pkg/runtime/race.c
View file

@ -351,7 +351,7 @@ onstack(uintptr argp)
// the layout is in ../../cmd/ld/data.c
if((byte*)argp >= noptrdata && (byte*)argp < enoptrbss)
return false;
if((byte*)argp >= runtime·mheap->arena_start && (byte*)argp < runtime·mheap->arena_used)
if((byte*)argp >= runtime·mheap.arena_start && (byte*)argp < runtime·mheap.arena_used)
return false;
return true;
}