123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780 |
- /*
- * This file is part of the UCB release of Plan 9. It is subject to the license
- * terms in the LICENSE file found in the top-level directory of this
- * distribution and at http://akaros.cs.berkeley.edu/files/Plan9License. No
- * part of the UCB release of Plan 9, including this file, may be copied,
- * modified, propagated, or distributed except according to the terms contained
- * in the LICENSE file.
- */
- #include "u.h"
- #include "../port/lib.h"
- #include "mem.h"
- #include "dat.h"
- #include "fns.h"
- #include "../port/error.h"
- #include <tos.h>
- #include "ureg.h"
- #include "../port/pmc.h"
- #include "io.h"
- #include "amd64.h"
- // counters. Set by assembly code.
- // interrupt enter and exit, systecm call enter and exit.
- unsigned long ire, irx, sce, scx;
- // Did we start doing an exit for the interrupts?
- // ir exit entry :-)
- unsigned long irxe;
- extern int notify(Ureg*);
- static void debugbpt(Ureg*, void*);
- static void faultamd64(Ureg*, void*);
- static void doublefault(Ureg*, void*);
- static void unexpected(Ureg*, void*);
- static void expected(Ureg*, void*);
- static void dumpstackwithureg(Ureg*);
- static Lock vctllock;
- static Vctl *vctl[256];
- typedef struct Intrtime Intrtime;
- struct Intrtime {
- uint64_t count;
- uint64_t cycles;
- };
- static Intrtime intrtimes[256];
- void*
- intrenable(int irq, void (*f)(Ureg*, void*), void* a, int tbdf, char *name)
- {
- int vno;
- Vctl *v;
- extern int ioapicintrenable(Vctl*);
- if(f == nil){
- print("intrenable: nil handler for %d, tbdf %#ux for %s\n",
- irq, tbdf, name);
- return nil;
- }
- v = malloc(sizeof(Vctl));
- v->isintr = 1;
- v->Vkey.irq = irq;
- v->Vkey.tbdf = tbdf;
- v->f = f;
- v->a = a;
- strncpy(v->name, name, KNAMELEN-1);
- v->name[KNAMELEN-1] = 0;
- ilock(&vctllock);
- vno = ioapicintrenable(v);
- if(vno == -1){
- iunlock(&vctllock);
- print("intrenable: couldn't enable irq %d, tbdf %#ux for %s\n",
- irq, tbdf, v->name);
- free(v);
- return nil;
- }
- if(vctl[vno]){
- if(vctl[v->vno]->isr != v->isr || vctl[v->vno]->eoi != v->eoi)
- panic("intrenable: handler: %s %s %#p %#p %#p %#p",
- vctl[v->vno]->name, v->name,
- vctl[v->vno]->isr, v->isr, vctl[v->vno]->eoi, v->eoi);
- }
- v->vno = vno;
- v->next = vctl[vno];
- vctl[vno] = v;
- iunlock(&vctllock);
- if(v->mask)
- v->mask(&v->Vkey, 0);
- /*
- * Return the assigned vector so intrdisable can find
- * the handler; the IRQ is useless in the wonderful world
- * of the IOAPIC.
- */
- return v;
- }
- int
- intrdisable(void* vector)
- {
- Vctl *v, *x, **ll;
- extern int ioapicintrdisable(int);
- ilock(&vctllock);
- v = vector;
- if(v == nil || vctl[v->vno] != v)
- panic("intrdisable: v %#p", v);
- for(ll = vctl+v->vno; x = *ll; ll = &x->next)
- if(v == x)
- break;
- if(x != v)
- panic("intrdisable: v %#p", v);
- if(v->mask)
- v->mask(&v->Vkey, 1);
- v->f(nil, v->a);
- *ll = v->next;
- ioapicintrdisable(v->vno);
- iunlock(&vctllock);
- free(v);
- return 0;
- }
- static int32_t
- irqallocread(Chan* c, void *vbuf, int32_t n, int64_t offset)
- {
- char *buf, *p, str[2*(11+1)+2*(20+1)+(KNAMELEN+1)+(8+1)+1];
- int m, vno;
- int32_t oldn;
- Intrtime *t;
- Vctl *v;
- if(n < 0 || offset < 0)
- error(Ebadarg);
- oldn = n;
- buf = vbuf;
- for(vno=0; vno<nelem(vctl); vno++){
- for(v=vctl[vno]; v; v=v->next){
- t = intrtimes + vno;
- m = snprint(str, sizeof str, "%11d %11d %20llud %20llud %-*.*s %.*s\n",
- vno, v->Vkey.irq, t->count, t->cycles, 8, 8, v->type, KNAMELEN, v->name);
- if(m <= offset) /* if do not want this, skip entry */
- offset -= m;
- else{
- /* skip offset bytes */
- m -= offset;
- p = str+offset;
- offset = 0;
- /* write at most max(n,m) bytes */
- if(m > n)
- m = n;
- memmove(buf, p, m);
- n -= m;
- buf += m;
- if(n == 0)
- return oldn;
- }
- }
- }
- return oldn - n;
- }
- void
- trapenable(int vno, void (*f)(Ureg*, void*), void* a, char *name)
- {
- Vctl *v;
- if(vno < 0 || vno >= 256)
- panic("trapenable: vno %d\n", vno);
- v = malloc(sizeof(Vctl));
- v->type = "trap";
- v->Vkey.tbdf = BUSUNKNOWN;
- v->f = f;
- v->a = a;
- strncpy(v->name, name, KNAMELEN);
- v->name[KNAMELEN-1] = 0;
- ilock(&vctllock);
- v->next = vctl[vno];
- vctl[vno] = v;
- iunlock(&vctllock);
- }
- static void
- nmienable(void)
- {
- int x;
- /*
- * Hack: should be locked with NVRAM access.
- */
- outb(0x70, 0x80); /* NMI latch clear */
- outb(0x70, 0);
- x = inb(0x61) & 0x07; /* Enable NMI */
- outb(0x61, 0x08|x);
- outb(0x61, x);
- }
- void
- trapinit(void)
- {
- /*
- * Need to set BPT interrupt gate - here or in vsvminit?
- */
- /*
- * Special traps.
- * Syscall() is called directly without going through trap().
- */
- trapenable(VectorBPT, debugbpt, 0, "#BP");
- trapenable(VectorPF, faultamd64, 0, "#PF");
- trapenable(Vector2F, doublefault, 0, "#DF");
- intrenable(IdtIPI, expected, 0, BUSUNKNOWN, "#IPI");
- trapenable(Vector15, unexpected, 0, "#15");
- nmienable();
- addarchfile("irqalloc", 0444, irqallocread, nil);
- }
- static char* excname[32] = {
- "#DE", /* Divide-by-Zero Error */
- "#DB", /* Debug */
- "#NMI", /* Non-Maskable-Interrupt */
- "#BP", /* Breakpoint */
- "#OF", /* Overflow */
- "#BR", /* Bound-Range */
- "#UD", /* Invalid-Opcode */
- "#NM", /* Device-Not-Available */
- "#DF", /* Double-Fault */
- "#9 (reserved)",
- "#TS", /* Invalid-TSS */
- "#NP", /* Segment-Not-Present */
- "#SS", /* Stack */
- "#GP", /* General-Protection */
- "#PF", /* Page-Fault */
- "#15 (reserved)",
- "#MF", /* x87 FPE-Pending */
- "#AC", /* Alignment-Check */
- "#MC", /* Machine-Check */
- "#XF", /* SIMD Floating-Point */
- "#20 (reserved)",
- "#21 (reserved)",
- "#22 (reserved)",
- "#23 (reserved)",
- "#24 (reserved)",
- "#25 (reserved)",
- "#26 (reserved)",
- "#27 (reserved)",
- "#28 (reserved)",
- "#29 (reserved)",
- "#30 (reserved)",
- "#31 (reserved)",
- };
- /*
- * keep interrupt service times and counts
- */
- void
- intrtime(int vno)
- {
- Proc *up = externup();
- uint32_t diff, x; /* should be uint64_t */
- x = perfticks();
- diff = x - machp()->perf.intrts;
- machp()->perf.intrts = x;
- machp()->perf.inintr += diff;
- if(up == nil && machp()->perf.inidle > diff)
- machp()->perf.inidle -= diff;
- intrtimes[vno].cycles += diff;
- intrtimes[vno].count++;
- }
- static void
- pmcnop(Mach *m)
- {
- }
- void (*_pmcupdate)(Mach *m) = pmcnop;
- /* go to user space */
- void
- kexit(Ureg* u)
- {
- Proc *up = externup();
- uint64_t t;
- Tos *tos;
- Mach *mp;
- /*
- * precise time accounting, kernel exit
- * initialized in exec, sysproc.c
- */
- tos = (Tos*)(USTKTOP-sizeof(Tos));
- cycles(&t);
- tos->kcycles += t - up->kentry;
- tos->pcycles = up->pcycles;
- tos->pid = up->pid;
- if (up->ac != nil)
- mp = up->ac;
- else
- mp = machp();
- tos->core = mp->machno;
- tos->nixtype = mp->NIX.nixtype;
- //_pmcupdate(m);
- /*
- * The process may change its core.
- * Be sure it has the right cyclefreq.
- */
- tos->cyclefreq = mp->cyclefreq;
- /* thread local storage */
- wrmsr(FSbase, up->tls);
- }
- void
- kstackok(void)
- {
- Proc *up = externup();
- if(up == nil){
- uintptr_t *stk = (uintptr_t*)machp()->stack;
- if(*stk != STACKGUARD)
- panic("trap: mach %d machstk went through bottom %p\n", machp()->machno, machp()->stack);
- } else {
- uintptr_t *stk = (uintptr_t*)up->kstack;
- if(*stk != STACKGUARD)
- panic("trap: proc %d kstack went through bottom %p\n", up->pid, up->kstack);
- }
- }
- void
- _trap(Ureg *ureg)
- {
- /*
- * If it's a real trap in this core, then we want to
- * use the hardware cr2 register.
- * We cannot do this in trap() because application cores
- * would update m->cr2 with their cr2 values upon page faults,
- * and then call trap().
- * If we do this in trap(), we would overwrite that with our own cr2.
- */
- if(ureg->type == VectorPF)
- machp()->MMU.cr2 = cr2get();
- trap(ureg);
- }
- /*
- * All traps come here. It is slower to have all traps call trap()
- * rather than directly vectoring the handler. However, this avoids a
- * lot of code duplication and possible bugs. The only exception is
- * VectorSYSCALL.
- * Trap is called with interrupts disabled via interrupt-gates.
- */
- void
- trap(Ureg* ureg)
- {
- int clockintr, vno, user;
- // cache the previous vno to see what might be causing
- // trouble
- static int lastvno;
- vno = ureg->type;
- uint64_t gsbase = rdmsr(GSbase);
- //if (sce > scx) iprint("====================");
- if (vno == 8) {
- iprint("Lstar is %p\n", (void *)rdmsr(Lstar));
- iprint("GSbase is %p\n", (void *)gsbase);
- iprint("ire %d irx %d sce %d scx %d lastvno %d\n",
- ire, irx, sce, scx, lastvno);
- iprint("irxe %d \n",
- irxe);
- die("8");
- }
- lastvno = vno;
- if (gsbase < 1ULL<<63)
- die("bogus gsbase");
- Proc *up = externup();
- char buf[ERRMAX];
- Vctl *ctl, *v;
- if (0 && machp() && up && up->pid == 6) {
- //iprint("type %x\n", ureg->type);
- if (ureg->type != 0x49)
- die("6\n");
- }
- machp()->perf.intrts = perfticks();
- user = userureg(ureg);
- if(user && (machp()->NIX.nixtype == NIXTC)){
- up->dbgreg = ureg;
- cycles(&up->kentry);
- }
- clockintr = 0;
- //_pmcupdate(machp());
- if(ctl = vctl[vno]){
- if(ctl->isintr){
- machp()->intr++;
- if(vno >= VectorPIC && vno != VectorSYSCALL)
- machp()->lastintr = ctl->Vkey.irq;
- }else
- if(up)
- up->nqtrap++;
- if(ctl->isr){
- ctl->isr(vno);
- if(islo())print("trap %d: isr %p enabled interrupts\n", vno, ctl->isr);
- }
- for(v = ctl; v != nil; v = v->next){
- if(v->f){
- v->f(ureg, v->a);
- if(islo())print("trap %d: ctlf %p enabled interrupts\n", vno, v->f);
- }
- }
- if(ctl->eoi){
- ctl->eoi(vno);
- if(islo())print("trap %d: eoi %p enabled interrupts\n", vno, ctl->eoi);
- }
- intrtime(vno);
- if(ctl->isintr){
- if(ctl->Vkey.irq == IrqCLOCK || ctl->Vkey.irq == IrqTIMER)
- clockintr = 1;
- if (ctl->Vkey.irq == IrqTIMER)
- oprof_alarm_handler(ureg);
- if(up && !clockintr)
- preempted();
- }
- }
- else if(vno < nelem(excname) && user){
- spllo();
- snprint(buf, sizeof buf, "sys: trap: %s", excname[vno]);
- postnote(up, 1, buf, NDebug);
- }
- else if(vno >= VectorPIC && vno != VectorSYSCALL){
- /*
- * An unknown interrupt.
- * Check for a default IRQ7. This can happen when
- * the IRQ input goes away before the acknowledge.
- * In this case, a 'default IRQ7' is generated, but
- * the corresponding bit in the ISR isn't set.
- * In fact, just ignore all such interrupts.
- */
- /* clear the interrupt */
- i8259isr(vno);
- iprint("cpu%d: spurious interrupt %d, last %d\n",
- machp()->machno, vno, machp()->lastintr);
- intrtime(vno);
- if(user)
- kexit(ureg);
- return;
- }
- else{
- if(vno == VectorNMI){
- nmienable();
- if(machp()->machno != 0){
- iprint("cpu%d: PC %#llux\n",
- machp()->machno, ureg->ip);
- for(;;);
- }
- }
- dumpregs(ureg);
- if(!user){
- ureg->sp = PTR2UINT(&ureg->sp);
- dumpstackwithureg(ureg);
- }
- if(vno < nelem(excname))
- panic("%s", excname[vno]);
- panic("unknown trap/intr: %d\n", vno);
- }
- splhi();
- /* delaysched set because we held a lock or because our quantum ended */
- if(up && up->delaysched && clockintr){
- if(0)
- if(user && up->ac == nil && up->nqtrap == 0 && up->nqsyscall == 0){
- if(!waserror()){
- up->ac = getac(up, -1);
- poperror();
- runacore();
- return;
- }
- }
- sched();
- splhi();
- }
- if(user){
- if(up && up->procctl || up->nnote)
- notify(ureg);
- kexit(ureg);
- }
- }
- /*
- * Dump general registers.
- */
- void
- dumpgpr(Ureg* ureg)
- {
- Proc *up = externup();
- if(up != nil)
- print("cpu%d: registers for %s %d\n",
- machp()->machno, up->text, up->pid);
- else
- print("cpu%d: registers for kernel\n", machp()->machno);
- print("ax\t%#16.16llux\n", ureg->ax);
- print("bx\t%#16.16llux\n", ureg->bx);
- print("cx\t%#16.16llux\n", ureg->cx);
- print("dx\t%#16.16llux\n", ureg->dx);
- print("di\t%#16.16llux\n", ureg->di);
- print("si\t%#16.16llux\n", ureg->si);
- print("bp\t%#16.16llux\n", ureg->bp);
- print("r8\t%#16.16llux\n", ureg->r8);
- print("r9\t%#16.16llux\n", ureg->r9);
- print("r10\t%#16.16llux\n", ureg->r10);
- print("r11\t%#16.16llux\n", ureg->r11);
- print("r12\t%#16.16llux\n", ureg->r12);
- print("r13\t%#16.16llux\n", ureg->r13);
- print("r14\t%#16.16llux\n", ureg->r14);
- print("r15\t%#16.16llux\n", ureg->r15);
- print("type\t%#llux\n", ureg->type);
- print("error\t%#llux\n", ureg->error);
- print("pc\t%#llux\n", ureg->ip);
- print("cs\t%#llux\n", ureg->cs);
- print("flags\t%#llux\n", ureg->flags);
- print("sp\t%#llux\n", ureg->sp);
- print("ss\t%#llux\n", ureg->ss);
- print("type\t%#llux\n", ureg->type);
- print("FS\t%#llux\n", rdmsr(FSbase));
- print("GS\t%#llux\n", rdmsr(GSbase));
- print("m\t%#16.16p\nup\t%#16.16p\n", machp(), up);
- }
- void
- dumpregs(Ureg* ureg)
- {
- die("dumpregs");
- dumpgpr(ureg);
- /*
- * Processor control registers.
- * If machine check exception, time stamp counter, page size extensions
- * or enhanced virtual 8086 mode extensions are supported, there is a
- * CR4. If there is a CR4 and machine check extensions, read the machine
- * check address and machine check type registers if RDMSR supported.
- */
- print("cr0\t%#16.16llux\n", cr0get());
- print("cr2\t%#16.16llux\n", machp()->MMU.cr2);
- print("cr3\t%#16.16llux\n", cr3get());
- die("dumpregs");
- // archdumpregs();
- }
- /*
- * Fill in enough of Ureg to get a stack trace, and call a function.
- * Used by debugging interface rdb.
- */
- void
- callwithureg(void (*fn)(Ureg*))
- {
- Ureg ureg;
- ureg.ip = getcallerpc(&fn);
- ureg.sp = PTR2UINT(&fn);
- fn(&ureg);
- }
- static void
- dumpstackwithureg(Ureg* ureg)
- {
- Proc *up = externup();
- uintptr_t l, v, i, estack;
- // extern char etext;
- int x;
- if (0) { //if((s = getconf("*nodumpstack")) != nil && atoi(s) != 0){
- iprint("dumpstack disabled\n");
- return;
- }
- iprint("dumpstack\n");
- x = 0;
- x += iprint("ktrace 9%s %#p %#p\n", strrchr(conffile, '/')+1, ureg->ip, ureg->sp);
- i = 0;
- if(up != nil
- // && (uintptr)&l >= (uintptr)up->kstack
- && (uintptr_t)&l <= (uintptr_t)up->kstack+KSTACK)
- estack = (uintptr_t)up->kstack+KSTACK;
- else if((uintptr_t)&l >= machp()->stack && (uintptr_t)&l <= machp()->stack+MACHSTKSZ)
- estack = machp()->stack+MACHSTKSZ;
- else{
- if(up != nil)
- iprint("&up->kstack %#p &l %#p\n", up->kstack, &l);
- else
- iprint("&m %#p &l %#p\n", machp(), &l);
- return;
- }
- x += iprint("estackx %#p\n", estack);
- for(l = (uintptr_t)&l; l < estack; l += sizeof(uintptr_t)){
- v = *(uintptr_t*)l;
- if((KTZERO < v && v < (uintptr_t)&etext)
- || ((uintptr_t)&l < v && v < estack) || estack-l < 256){
- x += iprint("%#16.16p=%#16.16p ", l, v);
- i++;
- }
- if(i == 2){
- i = 0;
- x += iprint("\n");
- }
- }
- if(i)
- iprint("\n");
- }
- void
- dumpstack(void)
- {
- callwithureg(dumpstackwithureg);
- }
- static void
- debugbpt(Ureg* ureg, void* v)
- {
- Proc *up = externup();
- char buf[ERRMAX];
- if(up == 0)
- panic("kernel bpt");
- /* restore pc to instruction that caused the trap */
- ureg->ip--;
- sprint(buf, "sys: breakpoint");
- postnote(up, 1, buf, NDebug);
- }
- static void
- doublefault(Ureg* ureg, void* v)
- {
- iprint("cr2 %p\n", (void *)cr2get());
- panic("double fault");
- }
- static void
- unexpected(Ureg* ureg, void* v)
- {
- iprint("unexpected trap %llu; ignoring\n", ureg->type);
- }
- static void
- expected(Ureg* ureg, void* v)
- {
- }
- static void
- faultamd64(Ureg* ureg, void* v)
- {
- Proc *up = externup();
- uint64_t addr;
- int ftype, user, insyscall;
- char buf[ERRMAX];
- addr = machp()->MMU.cr2;
- user = userureg(ureg);
- if(!user && mmukmapsync(addr))
- return;
- /*
- * There must be a user context.
- * If not, the usual problem is causing a fault during
- * initialisation before the system is fully up.
- */
- if(up == nil){
- panic("fault with up == nil; pc %#llux addr %#llux\n",
- ureg->ip, addr);
- }
- ftype = (ureg->error&2) ? FT_WRITE : (ureg->error&16) ? FT_EXEC : FT_READ;
- /*
- if (read) hi("read fault\n"); else hi("write fault\n");
- hi("addr "); put64(addr); hi("\n");
- */
- insyscall = up->insyscall;
- up->insyscall = 1;
- if (0)hi("call fault\n");
- if(fault(addr, ureg->ip, ftype) < 0){
- iprint("could not %s fault %p\n", faulttypes[ftype], addr);
- /*
- * It is possible to get here with !user if, for example,
- * a process was in a system call accessing a shared
- * segment but was preempted by another process which shrunk
- * or deallocated the shared segment; when the original
- * process resumes it may fault while in kernel mode.
- * No need to panic this case, post a note to the process
- * and unwind the error stack. There must be an error stack
- * (up->nerrlab != 0) if this is a system call, if not then
- * the game's a bogey.
- */
- if(!user && (!insyscall || up->nerrlab == 0))
- panic("fault: %#llux\n", addr);
- sprint(buf, "sys: trap: fault %s addr=%#llux",
- faulttypes[ftype], addr);
- postnote(up, 1, buf, NDebug);
- if(insyscall)
- error(buf);
- }
- up->insyscall = insyscall;
- }
- /*
- * return the userpc the last exception happened at
- */
- uintptr_t
- userpc(Ureg* ureg)
- {
- Proc *up = externup();
- if(ureg == nil)
- ureg = up->dbgreg;
- return ureg->ip;
- }
- /* This routine must save the values of registers the user is not permitted
- * to write from devproc and then restore the saved values before returning.
- * TODO: fix this because the segment registers are wrong for 64-bit mode.
- */
- void
- setregisters(Ureg* ureg, char* pureg, char* uva, int n)
- {
- uint64_t cs, flags, ss;
- ss = ureg->ss;
- flags = ureg->flags;
- cs = ureg->cs;
- memmove(pureg, uva, n);
- ureg->cs = cs;
- ureg->flags = (ureg->flags & 0x00ff) | (flags & 0xff00);
- ureg->ss = ss;
- }
- /* Give enough context in the ureg to produce a kernel stack for
- * a sleeping process
- */
- void
- setkernur(Ureg* ureg, Proc* p)
- {
- ureg->ip = p->sched.pc;
- ureg->sp = p->sched.sp+BY2SE;
- }
- uintptr_t
- dbgpc(Proc *p)
- {
- Ureg *ureg;
- ureg = p->dbgreg;
- if(ureg == 0)
- return 0;
- return ureg->ip;
- }
|