123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652 |
- #include "u.h"
- #include "../port/lib.h"
- #include "mem.h"
- #include "dat.h"
- #include "fns.h"
- #include "../port/error.h"
- #define pghash(daddr) palloc.hash[(daddr>>PGSHIFT)&(PGHSIZE-1)]
- struct Palloc palloc;
- void
- pageinit(void)
- {
- int color, i, j;
- Page *p;
- Pallocmem *pm;
- ulong m, np, k, vkb, pkb;
- np = 0;
- for(i=0; i<nelem(palloc.mem); i++){
- pm = &palloc.mem[i];
- np += pm->npage;
- }
- palloc.pages = xalloc(np*sizeof(Page));
- if(palloc.pages == 0)
- panic("pageinit");
- color = 0;
- palloc.head = palloc.pages;
- p = palloc.head;
- for(i=0; i<nelem(palloc.mem); i++){
- pm = &palloc.mem[i];
- for(j=0; j<pm->npage; j++){
- p->prev = p-1;
- p->next = p+1;
- p->pa = pm->base+j*BY2PG;
- p->color = color;
- palloc.freecount++;
- color = (color+1)%NCOLOR;
- p++;
- }
- }
- palloc.tail = p - 1;
- palloc.head->prev = 0;
- palloc.tail->next = 0;
- palloc.user = p - palloc.pages;
- pkb = palloc.user*BY2PG/1024;
- vkb = pkb + (conf.nswap*BY2PG)/1024;
- /* Paging numbers */
- swapalloc.highwater = (palloc.user*5)/100;
- swapalloc.headroom = swapalloc.highwater + (swapalloc.highwater/4);
- m = 0;
- for(i=0; i<nelem(conf.mem); i++)
- if(conf.mem[i].npage)
- m += conf.mem[i].npage*BY2PG;
- k = PGROUND(end - (char*)KTZERO);
- print("%ldM memory: ", (m+k+1024*1024-1)/(1024*1024));
- print("%ldM kernel data, ", (m+k-pkb*1024+1024*1024-1)/(1024*1024));
- print("%ldM user, ", pkb/1024);
- print("%ldM swap\n", vkb/1024);
- }
- static void
- pageunchain(Page *p)
- {
- if(canlock(&palloc))
- panic("pageunchain (palloc %p)", &palloc);
- if(p->prev)
- p->prev->next = p->next;
- else
- palloc.head = p->next;
- if(p->next)
- p->next->prev = p->prev;
- else
- palloc.tail = p->prev;
- p->prev = p->next = nil;
- palloc.freecount--;
- }
- void
- pagechaintail(Page *p)
- {
- if(canlock(&palloc))
- panic("pagechaintail");
- if(palloc.tail) {
- p->prev = palloc.tail;
- palloc.tail->next = p;
- }
- else {
- palloc.head = p;
- p->prev = 0;
- }
- palloc.tail = p;
- p->next = 0;
- palloc.freecount++;
- }
- void
- pagechainhead(Page *p)
- {
- if(canlock(&palloc))
- panic("pagechainhead");
- if(palloc.head) {
- p->next = palloc.head;
- palloc.head->prev = p;
- }
- else {
- palloc.tail = p;
- p->next = 0;
- }
- palloc.head = p;
- p->prev = 0;
- palloc.freecount++;
- }
- Page*
- newpage(int clear, Segment **s, ulong va)
- {
- Page *p;
- KMap *k;
- uchar ct;
- int i, hw, dontalloc, color;
- lock(&palloc);
- color = getpgcolor(va);
- hw = swapalloc.highwater;
- for(;;) {
- if(palloc.freecount > hw)
- break;
- if(up->kp && palloc.freecount > 0)
- break;
- unlock(&palloc);
- dontalloc = 0;
- if(s && *s) {
- qunlock(&((*s)->lk));
- *s = 0;
- dontalloc = 1;
- }
- qlock(&palloc.pwait); /* Hold memory requesters here */
- while(waserror()) /* Ignore interrupts */
- ;
- kickpager();
- tsleep(&palloc.r, ispages, 0, 1000);
- poperror();
- qunlock(&palloc.pwait);
- /*
- * If called from fault and we lost the segment from
- * underneath don't waste time allocating and freeing
- * a page. Fault will call newpage again when it has
- * reacquired the segment locks
- */
- if(dontalloc)
- return 0;
- lock(&palloc);
- }
- /* First try for our colour */
- for(p = palloc.head; p; p = p->next)
- if(p->color == color)
- break;
- ct = PG_NOFLUSH;
- if(p == 0) {
- p = palloc.head;
- p->color = color;
- ct = PG_NEWCOL;
- }
- pageunchain(p);
- lock(p);
- if(p->ref != 0)
- panic("newpage");
- uncachepage(p);
- p->ref++;
- p->va = va;
- p->modref = 0;
- for(i = 0; i < MAXMACH; i++)
- p->cachectl[i] = ct;
- unlock(p);
- unlock(&palloc);
- if(clear) {
- k = kmap(p);
- memset((void*)VA(k), 0, BY2PG);
- kunmap(k);
- }
- return p;
- }
- int
- ispages(void*)
- {
- return palloc.freecount >= swapalloc.highwater;
- }
- void
- putpage(Page *p)
- {
- if(onswap(p)) {
- putswap(p);
- return;
- }
- lock(&palloc);
- lock(p);
- if(p->ref == 0)
- panic("putpage");
- if(--p->ref > 0) {
- unlock(p);
- unlock(&palloc);
- return;
- }
- if(p->image && p->image != &swapimage)
- pagechaintail(p);
- else
- pagechainhead(p);
- if(palloc.r.p != 0)
- wakeup(&palloc.r);
- unlock(p);
- unlock(&palloc);
- }
- Page*
- auxpage(void)
- {
- Page *p;
- lock(&palloc);
- p = palloc.head;
- if(palloc.freecount < swapalloc.highwater) {
- unlock(&palloc);
- return 0;
- }
- pageunchain(p);
- lock(p);
- if(p->ref != 0)
- panic("auxpage");
- p->ref++;
- uncachepage(p);
- unlock(p);
- unlock(&palloc);
- return p;
- }
- static int dupretries = 15000;
- int
- duppage(Page *p) /* Always call with p locked */
- {
- Page *np;
- int color;
- int retries;
- retries = 0;
- retry:
- if(retries++ > dupretries){
- print("duppage %d, up %p\n", retries, up);
- dupretries += 100;
- if(dupretries > 100000)
- panic("duppage\n");
- uncachepage(p);
- return 1;
- }
-
- /* don't dup pages with no image */
- if(p->ref == 0 || p->image == nil || p->image->notext)
- return 0;
- /*
- * normal lock ordering is to call
- * lock(&palloc) before lock(p).
- * To avoid deadlock, we have to drop
- * our locks and try again.
- */
- if(!canlock(&palloc)){
- unlock(p);
- if(up)
- sched();
- lock(p);
- goto retry;
- }
- /* No freelist cache when memory is very low */
- if(palloc.freecount < swapalloc.highwater) {
- unlock(&palloc);
- uncachepage(p);
- return 1;
- }
- color = getpgcolor(p->va);
- for(np = palloc.head; np; np = np->next)
- if(np->color == color)
- break;
- /* No page of the correct color */
- if(np == 0) {
- unlock(&palloc);
- uncachepage(p);
- return 1;
- }
- pageunchain(np);
- pagechaintail(np);
- /*
- * XXX - here's a bug? - np is on the freelist but it's not really free.
- * when we unlock palloc someone else can come in, decide to
- * use np, and then try to lock it. they succeed after we've
- * run copypage and cachepage and unlock(np). then what?
- * they call pageunchain before locking(np), so it's removed
- * from the freelist, but still in the cache because of
- * cachepage below. if someone else looks in the cache
- * before they remove it, the page will have a nonzero ref
- * once they finally lock(np).
- */
- lock(np);
- unlock(&palloc);
- /* Cache the new version */
- uncachepage(np);
- np->va = p->va;
- np->daddr = p->daddr;
- copypage(p, np);
- cachepage(np, p->image);
- unlock(np);
- uncachepage(p);
- return 0;
- }
- void
- copypage(Page *f, Page *t)
- {
- KMap *ks, *kd;
- ks = kmap(f);
- kd = kmap(t);
- memmove((void*)VA(kd), (void*)VA(ks), BY2PG);
- kunmap(ks);
- kunmap(kd);
- }
- void
- uncachepage(Page *p) /* Always called with a locked page */
- {
- Page **l, *f;
- if(p->image == 0)
- return;
- lock(&palloc.hashlock);
- l = &pghash(p->daddr);
- for(f = *l; f; f = f->hash) {
- if(f == p) {
- *l = p->hash;
- break;
- }
- l = &f->hash;
- }
- unlock(&palloc.hashlock);
- putimage(p->image);
- p->image = 0;
- p->daddr = 0;
- }
- void
- cachepage(Page *p, Image *i)
- {
- Page **l;
- /* If this ever happens it should be fixed by calling
- * uncachepage instead of panic. I think there is a race
- * with pio in which this can happen. Calling uncachepage is
- * correct - I just wanted to see if we got here.
- */
- if(p->image)
- panic("cachepage");
- incref(i);
- lock(&palloc.hashlock);
- p->image = i;
- l = &pghash(p->daddr);
- p->hash = *l;
- *l = p;
- unlock(&palloc.hashlock);
- }
- void
- cachedel(Image *i, ulong daddr)
- {
- Page *f, **l;
- lock(&palloc.hashlock);
- l = &pghash(daddr);
- for(f = *l; f; f = f->hash) {
- if(f->image == i && f->daddr == daddr) {
- lock(f);
- if(f->image == i && f->daddr == daddr){
- *l = f->hash;
- putimage(f->image);
- f->image = 0;
- f->daddr = 0;
- }
- unlock(f);
- break;
- }
- l = &f->hash;
- }
- unlock(&palloc.hashlock);
- }
- Page *
- lookpage(Image *i, ulong daddr)
- {
- Page *f;
- lock(&palloc.hashlock);
- for(f = pghash(daddr); f; f = f->hash) {
- if(f->image == i && f->daddr == daddr) {
- unlock(&palloc.hashlock);
- lock(&palloc);
- lock(f);
- if(f->image != i || f->daddr != daddr) {
- unlock(f);
- unlock(&palloc);
- return 0;
- }
- if(++f->ref == 1)
- pageunchain(f);
- unlock(&palloc);
- unlock(f);
- return f;
- }
- }
- unlock(&palloc.hashlock);
- return 0;
- }
- Pte*
- ptecpy(Pte *old)
- {
- Pte *new;
- Page **src, **dst;
- new = ptealloc();
- dst = &new->pages[old->first-old->pages];
- new->first = dst;
- for(src = old->first; src <= old->last; src++, dst++)
- if(*src) {
- if(onswap(*src))
- dupswap(*src);
- else {
- lock(*src);
- (*src)->ref++;
- unlock(*src);
- }
- new->last = dst;
- *dst = *src;
- }
- return new;
- }
- Pte*
- ptealloc(void)
- {
- Pte *new;
- new = smalloc(sizeof(Pte));
- new->first = &new->pages[PTEPERTAB];
- new->last = new->pages;
- return new;
- }
- void
- freepte(Segment *s, Pte *p)
- {
- int ref;
- void (*fn)(Page*);
- Page *pt, **pg, **ptop;
- switch(s->type&SG_TYPE) {
- case SG_PHYSICAL:
- fn = s->pseg->pgfree;
- ptop = &p->pages[PTEPERTAB];
- if(fn) {
- for(pg = p->pages; pg < ptop; pg++) {
- if(*pg == 0)
- continue;
- (*fn)(*pg);
- *pg = 0;
- }
- break;
- }
- for(pg = p->pages; pg < ptop; pg++) {
- pt = *pg;
- if(pt == 0)
- continue;
- lock(pt);
- ref = --pt->ref;
- unlock(pt);
- if(ref == 0)
- free(pt);
- }
- break;
- default:
- for(pg = p->first; pg <= p->last; pg++)
- if(*pg) {
- putpage(*pg);
- *pg = 0;
- }
- }
- free(p);
- }
- ulong
- pagenumber(Page *p)
- {
- return p-palloc.pages;
- }
- void
- checkpagerefs(void)
- {
- int s;
- ulong i, np, nwrong;
- ulong *ref;
-
- np = palloc.user;
- ref = malloc(np*sizeof ref[0]);
- if(ref == nil){
- print("checkpagerefs: out of memory\n");
- return;
- }
-
- /*
- * This may not be exact if there are other processes
- * holding refs to pages on their stacks. The hope is
- * that if you run it on a quiescent system it will still
- * be useful.
- */
- s = splhi();
- lock(&palloc);
- countpagerefs(ref, 0);
- portcountpagerefs(ref, 0);
- nwrong = 0;
- for(i=0; i<np; i++){
- if(palloc.pages[i].ref != ref[i]){
- iprint("page %#.8lux ref %d actual %lud\n",
- palloc.pages[i].pa, palloc.pages[i].ref, ref[i]);
- ref[i] = 1;
- nwrong++;
- }else
- ref[i] = 0;
- }
- countpagerefs(ref, 1);
- portcountpagerefs(ref, 1);
- iprint("%lud mistakes found\n", nwrong);
- unlock(&palloc);
- splx(s);
- }
- void
- portcountpagerefs(ulong *ref, int print)
- {
- ulong i, j, k, ns, n;
- Page **pg, *entry;
- Proc *p;
- Pte *pte;
- Segment *s;
- /*
- * Pages in segments. s->mark avoids double-counting.
- */
- n = 0;
- ns = 0;
- for(i=0; i<conf.nproc; i++){
- p = proctab(i);
- for(j=0; j<NSEG; j++){
- s = p->seg[j];
- if(s)
- s->mark = 0;
- }
- }
- for(i=0; i<conf.nproc; i++){
- p = proctab(i);
- for(j=0; j<NSEG; j++){
- s = p->seg[j];
- if(s == nil || s->mark++)
- continue;
- ns++;
- for(k=0; k<s->mapsize; k++){
- pte = s->map[k];
- if(pte == nil)
- continue;
- for(pg = pte->first; pg <= pte->last; pg++){
- entry = *pg;
- if(pagedout(entry))
- continue;
- if(print){
- if(ref[pagenumber(entry)])
- iprint("page %#.8lux in segment %#p\n", entry->pa, s);
- continue;
- }
- if(ref[pagenumber(entry)]++ == 0)
- n++;
- }
- }
- }
- }
- if(!print){
- iprint("%lud pages in %lud segments\n", n, ns);
- for(i=0; i<conf.nproc; i++){
- p = proctab(i);
- for(j=0; j<NSEG; j++){
- s = p->seg[j];
- if(s == nil)
- continue;
- if(s->ref != s->mark){
- iprint("segment %#.8lux (used by proc %lud pid %lud) has bad ref count %lud actual %lud\n",
- s, i, p->pid, s->ref, s->mark);
- }
- }
- }
- }
- }
|