mmu.c 24 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063
  1. /*
  2. * Memory mappings. Life was easier when 2G of memory was enough.
  3. *
  4. * The kernel memory starts at KZERO, with the text loaded at KZERO+1M
  5. * (9load sits under 1M during the load). The memory from KZERO to the
  6. * top of memory is mapped 1-1 with physical memory, starting at physical
  7. * address 0. All kernel memory and data structures (i.e., the entries stored
  8. * into conf.mem) must sit in this physical range: if KZERO is at 0xF0000000,
  9. * then the kernel can only have 256MB of memory for itself.
  10. *
  11. * The 256M below KZERO comprises three parts. The lowest 4M is the
  12. * virtual page table, a virtual address representation of the current
  13. * page table tree. The second 4M is used for temporary per-process
  14. * mappings managed by kmap and kunmap. The remaining 248M is used
  15. * for global (shared by all procs and all processors) device memory
  16. * mappings and managed by vmap and vunmap. The total amount (256M)
  17. * could probably be reduced somewhat if desired. The largest device
  18. * mapping is that of the video card, and even though modern video cards
  19. * have embarrassing amounts of memory, the video drivers only use one
  20. * frame buffer worth (at most 16M). Each is described in more detail below.
  21. *
  22. * The VPT is a 4M frame constructed by inserting the pdb into itself.
  23. * This short-circuits one level of the page tables, with the result that
  24. * the contents of second-level page tables can be accessed at VPT.
  25. * We use the VPT to edit the page tables (see mmu) after inserting them
  26. * into the page directory. It is a convenient mechanism for mapping what
  27. * might be otherwise-inaccessible pages. The idea was borrowed from
  28. * the Exokernel.
  29. *
  30. * The VPT doesn't solve all our problems, because we still need to
  31. * prepare page directories before we can install them. For that, we
  32. * use tmpmap/tmpunmap, which map a single page at TMPADDR.
  33. */
  34. #include "u.h"
  35. #include "../port/lib.h"
  36. #include "mem.h"
  37. #include "dat.h"
  38. #include "fns.h"
  39. #include "io.h"
  40. /*
  41. * Simple segment descriptors with no translation.
  42. */
  43. #define DATASEGM(p) { 0xFFFF, SEGG|SEGB|(0xF<<16)|SEGP|SEGPL(p)|SEGDATA|SEGW }
  44. #define EXECSEGM(p) { 0xFFFF, SEGG|SEGD|(0xF<<16)|SEGP|SEGPL(p)|SEGEXEC|SEGR }
  45. #define EXEC16SEGM(p) { 0xFFFF, SEGG|(0xF<<16)|SEGP|SEGPL(p)|SEGEXEC|SEGR }
  46. #define TSSSEGM(b,p) { ((b)<<16)|sizeof(Tss),\
  47. ((b)&0xFF000000)|(((b)>>16)&0xFF)|SEGTSS|SEGPL(p)|SEGP }
  48. Segdesc gdt[NGDT] =
  49. {
  50. [NULLSEG] { 0, 0}, /* null descriptor */
  51. [KDSEG] DATASEGM(0), /* kernel data/stack */
  52. [KESEG] EXECSEGM(0), /* kernel code */
  53. [UDSEG] DATASEGM(3), /* user data/stack */
  54. [UESEG] EXECSEGM(3), /* user code */
  55. [TSSSEG] TSSSEGM(0,0), /* tss segment */
  56. [KESEG16] EXEC16SEGM(0), /* kernel code 16-bit */
  57. };
  58. static int didmmuinit;
  59. static void taskswitch(ulong, ulong);
  60. static void memglobal(void);
  61. #define vpt ((ulong*)VPT)
  62. #define VPTX(va) (((ulong)(va))>>12)
  63. #define vpd (vpt+VPTX(VPT))
  64. void
  65. mmuinit0(void)
  66. {
  67. memmove(m->gdt, gdt, sizeof gdt);
  68. }
  69. void
  70. mmuinit(void)
  71. {
  72. ulong x, *p;
  73. ushort ptr[3];
  74. didmmuinit = 1;
  75. if(0) print("vpt=%#.8ux vpd=%#p kmap=%#.8ux\n",
  76. VPT, vpd, KMAP);
  77. memglobal();
  78. m->pdb[PDX(VPT)] = PADDR(m->pdb)|PTEWRITE|PTEVALID;
  79. m->tss = malloc(sizeof(Tss));
  80. memset(m->tss, 0, sizeof(Tss));
  81. m->tss->iomap = 0xDFFF<<16;
  82. /*
  83. * We used to keep the GDT in the Mach structure, but it
  84. * turns out that that slows down access to the rest of the
  85. * page. Since the Mach structure is accessed quite often,
  86. * it pays off anywhere from a factor of 1.25 to 2 on real
  87. * hardware to separate them (the AMDs are more sensitive
  88. * than Intels in this regard). Under VMware it pays off
  89. * a factor of about 10 to 100.
  90. */
  91. memmove(m->gdt, gdt, sizeof gdt);
  92. x = (ulong)m->tss;
  93. m->gdt[TSSSEG].d0 = (x<<16)|sizeof(Tss);
  94. m->gdt[TSSSEG].d1 = (x&0xFF000000)|((x>>16)&0xFF)|SEGTSS|SEGPL(0)|SEGP;
  95. ptr[0] = sizeof(gdt)-1;
  96. x = (ulong)m->gdt;
  97. ptr[1] = x & 0xFFFF;
  98. ptr[2] = (x>>16) & 0xFFFF;
  99. lgdt(ptr);
  100. ptr[0] = sizeof(Segdesc)*256-1;
  101. x = IDTADDR;
  102. ptr[1] = x & 0xFFFF;
  103. ptr[2] = (x>>16) & 0xFFFF;
  104. lidt(ptr);
  105. /* make kernel text unwritable */
  106. for(x = KTZERO; x < (ulong)etext; x += BY2PG){
  107. p = mmuwalk(m->pdb, x, 2, 0);
  108. if(p == nil)
  109. panic("mmuinit");
  110. *p &= ~PTEWRITE;
  111. }
  112. taskswitch(PADDR(m->pdb), (ulong)m + BY2PG);
  113. ltr(TSSSEL);
  114. }
  115. /*
  116. * On processors that support it, we set the PTEGLOBAL bit in
  117. * page table and page directory entries that map kernel memory.
  118. * Doing this tells the processor not to bother flushing them
  119. * from the TLB when doing the TLB flush associated with a
  120. * context switch (write to CR3). Since kernel memory mappings
  121. * are never removed, this is safe. (If we ever remove kernel memory
  122. * mappings, we can do a full flush by turning off the PGE bit in CR4,
  123. * writing to CR3, and then turning the PGE bit back on.)
  124. *
  125. * See also mmukmap below.
  126. *
  127. * Processor support for the PTEGLOBAL bit is enabled in devarch.c.
  128. */
  129. static void
  130. memglobal(void)
  131. {
  132. int i, j;
  133. ulong *pde, *pte;
  134. /* only need to do this once, on bootstrap processor */
  135. if(m->machno != 0)
  136. return;
  137. if(!m->havepge)
  138. return;
  139. pde = m->pdb;
  140. for(i=PDX(KZERO); i<1024; i++){
  141. if(pde[i] & PTEVALID){
  142. pde[i] |= PTEGLOBAL;
  143. if(!(pde[i] & PTESIZE)){
  144. pte = KADDR(pde[i]&~(BY2PG-1));
  145. for(j=0; j<1024; j++)
  146. if(pte[j] & PTEVALID)
  147. pte[j] |= PTEGLOBAL;
  148. }
  149. }
  150. }
  151. }
  152. /*
  153. * Flush all the user-space and device-mapping mmu info
  154. * for this process, because something has been deleted.
  155. * It will be paged back in on demand.
  156. */
  157. void
  158. flushmmu(void)
  159. {
  160. int s;
  161. s = splhi();
  162. up->newtlb = 1;
  163. mmuswitch(up);
  164. splx(s);
  165. }
  166. /*
  167. * Flush a single page mapping from the tlb.
  168. */
  169. void
  170. flushpg(ulong va)
  171. {
  172. if(X86FAMILY(m->cpuidax) >= 4)
  173. invlpg(va);
  174. else
  175. putcr3(getcr3());
  176. }
  177. /*
  178. * Allocate a new page for a page directory.
  179. * We keep a small cache of pre-initialized
  180. * page directories in each mach.
  181. */
  182. static Page*
  183. mmupdballoc(void)
  184. {
  185. int s;
  186. Page *page;
  187. ulong *pdb;
  188. s = splhi();
  189. m->pdballoc++;
  190. if(m->pdbpool == 0){
  191. spllo();
  192. page = newpage(0, 0, 0);
  193. page->va = (ulong)vpd;
  194. splhi();
  195. pdb = tmpmap(page);
  196. memmove(pdb, m->pdb, BY2PG);
  197. pdb[PDX(VPT)] = page->pa|PTEWRITE|PTEVALID; /* set up VPT */
  198. tmpunmap(pdb);
  199. }else{
  200. page = m->pdbpool;
  201. m->pdbpool = page->next;
  202. m->pdbcnt--;
  203. }
  204. splx(s);
  205. return page;
  206. }
  207. static void
  208. mmupdbfree(Proc *proc, Page *p)
  209. {
  210. if(islo())
  211. panic("mmupdbfree: islo");
  212. m->pdbfree++;
  213. if(m->pdbcnt >= 10){
  214. p->next = proc->mmufree;
  215. proc->mmufree = p;
  216. }else{
  217. p->next = m->pdbpool;
  218. m->pdbpool = p;
  219. m->pdbcnt++;
  220. }
  221. }
  222. /*
  223. * A user-space memory segment has been deleted, or the
  224. * process is exiting. Clear all the pde entries for user-space
  225. * memory mappings and device mappings. Any entries that
  226. * are needed will be paged back in as necessary.
  227. */
  228. static void
  229. mmuptefree(Proc* proc)
  230. {
  231. int s;
  232. ulong *pdb;
  233. Page **last, *page;
  234. if(proc->mmupdb == nil || proc->mmuused == nil)
  235. return;
  236. s = splhi();
  237. pdb = tmpmap(proc->mmupdb);
  238. last = &proc->mmuused;
  239. for(page = *last; page; page = page->next){
  240. pdb[page->daddr] = 0;
  241. last = &page->next;
  242. }
  243. tmpunmap(pdb);
  244. splx(s);
  245. *last = proc->mmufree;
  246. proc->mmufree = proc->mmuused;
  247. proc->mmuused = 0;
  248. }
  249. static void
  250. taskswitch(ulong pdb, ulong stack)
  251. {
  252. Tss *tss;
  253. tss = m->tss;
  254. tss->ss0 = KDSEL;
  255. tss->esp0 = stack;
  256. tss->ss1 = KDSEL;
  257. tss->esp1 = stack;
  258. tss->ss2 = KDSEL;
  259. tss->esp2 = stack;
  260. putcr3(pdb);
  261. }
  262. void
  263. mmuswitch(Proc* proc)
  264. {
  265. ulong *pdb;
  266. if(proc->newtlb){
  267. mmuptefree(proc);
  268. proc->newtlb = 0;
  269. }
  270. if(proc->mmupdb){
  271. pdb = tmpmap(proc->mmupdb);
  272. pdb[PDX(MACHADDR)] = m->pdb[PDX(MACHADDR)];
  273. tmpunmap(pdb);
  274. taskswitch(proc->mmupdb->pa, (ulong)(proc->kstack+KSTACK));
  275. }else
  276. taskswitch(PADDR(m->pdb), (ulong)(proc->kstack+KSTACK));
  277. }
  278. /*
  279. * Release any pages allocated for a page directory base or page-tables
  280. * for this process:
  281. * switch to the prototype pdb for this processor (m->pdb);
  282. * call mmuptefree() to place all pages used for page-tables (proc->mmuused)
  283. * onto the process' free list (proc->mmufree). This has the side-effect of
  284. * cleaning any user entries in the pdb (proc->mmupdb);
  285. * if there's a pdb put it in the cache of pre-initialised pdb's
  286. * for this processor (m->pdbpool) or on the process' free list;
  287. * finally, place any pages freed back into the free pool (palloc).
  288. * This routine is only called from schedinit() with palloc locked.
  289. */
  290. void
  291. mmurelease(Proc* proc)
  292. {
  293. Page *page, *next;
  294. ulong *pdb;
  295. if(islo())
  296. panic("mmurelease: islo");
  297. taskswitch(PADDR(m->pdb), (ulong)m + BY2PG);
  298. if(proc->kmaptable){
  299. if(proc->mmupdb == nil)
  300. panic("mmurelease: no mmupdb");
  301. if(--proc->kmaptable->ref)
  302. panic("mmurelease: kmap ref %d\n", proc->kmaptable->ref);
  303. if(proc->nkmap)
  304. panic("mmurelease: nkmap %d\n", proc->nkmap);
  305. /*
  306. * remove kmaptable from pdb before putting pdb up for reuse.
  307. */
  308. pdb = tmpmap(proc->mmupdb);
  309. if(PPN(pdb[PDX(KMAP)]) != proc->kmaptable->pa)
  310. panic("mmurelease: bad kmap pde %#.8lux kmap %#.8lux",
  311. pdb[PDX(KMAP)], proc->kmaptable->pa);
  312. pdb[PDX(KMAP)] = 0;
  313. tmpunmap(pdb);
  314. /*
  315. * move kmaptable to free list.
  316. */
  317. pagechainhead(proc->kmaptable);
  318. proc->kmaptable = 0;
  319. }
  320. if(proc->mmupdb){
  321. mmuptefree(proc);
  322. mmupdbfree(proc, proc->mmupdb);
  323. proc->mmupdb = 0;
  324. }
  325. for(page = proc->mmufree; page; page = next){
  326. next = page->next;
  327. if(--page->ref)
  328. panic("mmurelease: page->ref %d\n", page->ref);
  329. pagechainhead(page);
  330. }
  331. if(proc->mmufree && palloc.r.p)
  332. wakeup(&palloc.r);
  333. proc->mmufree = 0;
  334. }
  335. /*
  336. * Allocate and install pdb for the current process.
  337. */
  338. static void
  339. upallocpdb(void)
  340. {
  341. int s;
  342. ulong *pdb;
  343. Page *page;
  344. if(up->mmupdb != nil)
  345. return;
  346. page = mmupdballoc();
  347. s = splhi();
  348. if(up->mmupdb != nil){
  349. /*
  350. * Perhaps we got an interrupt while
  351. * mmupdballoc was sleeping and that
  352. * interrupt allocated an mmupdb?
  353. * Seems unlikely.
  354. */
  355. mmupdbfree(up, page);
  356. splx(s);
  357. return;
  358. }
  359. pdb = tmpmap(page);
  360. pdb[PDX(MACHADDR)] = m->pdb[PDX(MACHADDR)];
  361. tmpunmap(pdb);
  362. up->mmupdb = page;
  363. putcr3(up->mmupdb->pa);
  364. splx(s);
  365. }
  366. /*
  367. * Update the mmu in response to a user fault. pa may have PTEWRITE set.
  368. */
  369. void
  370. putmmu(ulong va, ulong pa, Page*)
  371. {
  372. int old, s;
  373. Page *page;
  374. if(up->mmupdb == nil)
  375. upallocpdb();
  376. /*
  377. * We should be able to get through this with interrupts
  378. * turned on (if we get interrupted we'll just pick up
  379. * where we left off) but we get many faults accessing
  380. * vpt[] near the end of this function, and they always happen
  381. * after the process has been switched out and then
  382. * switched back, usually many times in a row (perhaps
  383. * it cannot switch back successfully for some reason).
  384. *
  385. * In any event, I'm tired of searching for this bug.
  386. * Turn off interrupts during putmmu even though
  387. * we shouldn't need to. - rsc
  388. */
  389. s = splhi();
  390. if(!(vpd[PDX(va)]&PTEVALID)){
  391. if(up->mmufree == 0){
  392. spllo();
  393. page = newpage(0, 0, 0);
  394. splhi();
  395. }
  396. else{
  397. page = up->mmufree;
  398. up->mmufree = page->next;
  399. }
  400. vpd[PDX(va)] = PPN(page->pa)|PTEUSER|PTEWRITE|PTEVALID;
  401. /* page is now mapped into the VPT - clear it */
  402. memset((void*)(VPT+PDX(va)*BY2PG), 0, BY2PG);
  403. page->daddr = PDX(va);
  404. page->next = up->mmuused;
  405. up->mmuused = page;
  406. }
  407. old = vpt[VPTX(va)];
  408. vpt[VPTX(va)] = pa|PTEUSER|PTEVALID;
  409. if(old&PTEVALID)
  410. flushpg(va);
  411. if(getcr3() != up->mmupdb->pa)
  412. print("bad cr3 %#.8lux %#.8lux\n", getcr3(), up->mmupdb->pa);
  413. splx(s);
  414. }
  415. /*
  416. * Double-check the user MMU.
  417. * Error checking only.
  418. */
  419. void
  420. checkmmu(ulong va, ulong pa)
  421. {
  422. if(up->mmupdb == 0)
  423. return;
  424. if(!(vpd[PDX(va)]&PTEVALID) || !(vpt[VPTX(va)]&PTEVALID))
  425. return;
  426. if(PPN(vpt[VPTX(va)]) != pa)
  427. print("%ld %s: va=%#08lux pa=%#08lux pte=%#08lux\n",
  428. up->pid, up->text,
  429. va, pa, vpt[VPTX(va)]);
  430. }
  431. /*
  432. * Walk the page-table pointed to by pdb and return a pointer
  433. * to the entry for virtual address va at the requested level.
  434. * If the entry is invalid and create isn't requested then bail
  435. * out early. Otherwise, for the 2nd level walk, allocate a new
  436. * page-table page and register it in the 1st level. This is used
  437. * only to edit kernel mappings, which use pages from kernel memory,
  438. * so it's okay to use KADDR to look at the tables.
  439. */
  440. ulong*
  441. mmuwalk(ulong* pdb, ulong va, int level, int create)
  442. {
  443. ulong *table;
  444. void *map;
  445. table = &pdb[PDX(va)];
  446. if(!(*table & PTEVALID) && create == 0)
  447. return 0;
  448. switch(level){
  449. default:
  450. return 0;
  451. case 1:
  452. return table;
  453. case 2:
  454. if(*table & PTESIZE)
  455. panic("mmuwalk2: va %luX entry %luX\n", va, *table);
  456. if(!(*table & PTEVALID)){
  457. /*
  458. * Have to call low-level allocator from
  459. * memory.c if we haven't set up the xalloc
  460. * tables yet.
  461. */
  462. if(didmmuinit)
  463. map = xspanalloc(BY2PG, BY2PG, 0);
  464. else
  465. map = rampage();
  466. if(map == nil)
  467. panic("mmuwalk xspanalloc failed");
  468. *table = PADDR(map)|PTEWRITE|PTEVALID;
  469. }
  470. table = KADDR(PPN(*table));
  471. return &table[PTX(va)];
  472. }
  473. }
  474. /*
  475. * Device mappings are shared by all procs and processors and
  476. * live in the virtual range VMAP to VMAP+VMAPSIZE. The master
  477. * copy of the mappings is stored in mach0->pdb, and they are
  478. * paged in from there as necessary by vmapsync during faults.
  479. */
  480. static Lock vmaplock;
  481. static int findhole(ulong *a, int n, int count);
  482. static ulong vmapalloc(ulong size);
  483. static void pdbunmap(ulong*, ulong, int);
  484. /*
  485. * Add a device mapping to the vmap range.
  486. */
  487. void*
  488. vmap(ulong pa, int size)
  489. {
  490. int osize;
  491. ulong o, va;
  492. /*
  493. * might be asking for less than a page.
  494. */
  495. osize = size;
  496. o = pa & (BY2PG-1);
  497. pa -= o;
  498. size += o;
  499. size = ROUND(size, BY2PG);
  500. if(pa == 0){
  501. print("vmap pa=0 pc=%#p\n", getcallerpc(&pa));
  502. return nil;
  503. }
  504. ilock(&vmaplock);
  505. if((va = vmapalloc(size)) == 0
  506. || pdbmap(MACHP(0)->pdb, pa|PTEUNCACHED|PTEWRITE, va, size) < 0){
  507. iunlock(&vmaplock);
  508. return 0;
  509. }
  510. iunlock(&vmaplock);
  511. /* avoid trap on local processor
  512. for(i=0; i<size; i+=4*MB)
  513. vmapsync(va+i);
  514. */
  515. USED(osize);
  516. // print(" vmap %#.8lux %d => %#.8lux\n", pa+o, osize, va+o);
  517. return (void*)(va + o);
  518. }
  519. static int
  520. findhole(ulong *a, int n, int count)
  521. {
  522. int have, i;
  523. have = 0;
  524. for(i=0; i<n; i++){
  525. if(a[i] == 0)
  526. have++;
  527. else
  528. have = 0;
  529. if(have >= count)
  530. return i+1 - have;
  531. }
  532. return -1;
  533. }
  534. /*
  535. * Look for free space in the vmap.
  536. */
  537. static ulong
  538. vmapalloc(ulong size)
  539. {
  540. int i, n, o;
  541. ulong *vpdb;
  542. int vpdbsize;
  543. vpdb = &MACHP(0)->pdb[PDX(VMAP)];
  544. vpdbsize = VMAPSIZE/(4*MB);
  545. if(size >= 4*MB){
  546. n = (size+4*MB-1) / (4*MB);
  547. if((o = findhole(vpdb, vpdbsize, n)) != -1)
  548. return VMAP + o*4*MB;
  549. return 0;
  550. }
  551. n = (size+BY2PG-1) / BY2PG;
  552. for(i=0; i<vpdbsize; i++)
  553. if((vpdb[i]&PTEVALID) && !(vpdb[i]&PTESIZE))
  554. if((o = findhole(KADDR(PPN(vpdb[i])), WD2PG, n)) != -1)
  555. return VMAP + i*4*MB + o*BY2PG;
  556. if((o = findhole(vpdb, vpdbsize, 1)) != -1)
  557. return VMAP + o*4*MB;
  558. /*
  559. * could span page directory entries, but not worth the trouble.
  560. * not going to be very much contention.
  561. */
  562. return 0;
  563. }
  564. /*
  565. * Remove a device mapping from the vmap range.
  566. * Since pdbunmap does not remove page tables, just entries,
  567. * the call need not be interlocked with vmap.
  568. */
  569. void
  570. vunmap(void *v, int size)
  571. {
  572. int i;
  573. ulong va, o;
  574. Mach *nm;
  575. Proc *p;
  576. /*
  577. * might not be aligned
  578. */
  579. va = (ulong)v;
  580. o = va&(BY2PG-1);
  581. va -= o;
  582. size += o;
  583. size = ROUND(size, BY2PG);
  584. if(size < 0 || va < VMAP || va+size > VMAP+VMAPSIZE)
  585. panic("vunmap va=%#.8lux size=%#x pc=%#.8lux\n",
  586. va, size, getcallerpc(&va));
  587. pdbunmap(MACHP(0)->pdb, va, size);
  588. /*
  589. * Flush mapping from all the tlbs and copied pdbs.
  590. * This can be (and is) slow, since it is called only rarely.
  591. * It is possible for vunmap to be called with up == nil,
  592. * e.g. from the reset/init driver routines during system
  593. * boot. In that case it suffices to flush the MACH(0) TLB
  594. * and return.
  595. */
  596. if(!active.thunderbirdsarego){
  597. putcr3(PADDR(MACHP(0)->pdb));
  598. return;
  599. }
  600. for(i=0; i<conf.nproc; i++){
  601. p = proctab(i);
  602. if(p->state == Dead)
  603. continue;
  604. if(p != up)
  605. p->newtlb = 1;
  606. }
  607. for(i=0; i<conf.nmach; i++){
  608. nm = MACHP(i);
  609. if(nm != m)
  610. nm->flushmmu = 1;
  611. }
  612. flushmmu();
  613. for(i=0; i<conf.nmach; i++){
  614. nm = MACHP(i);
  615. if(nm != m)
  616. while((active.machs&(1<<nm->machno)) && nm->flushmmu)
  617. ;
  618. }
  619. }
  620. /*
  621. * Add kernel mappings for pa -> va for a section of size bytes.
  622. */
  623. int
  624. pdbmap(ulong *pdb, ulong pa, ulong va, int size)
  625. {
  626. int pse;
  627. ulong pgsz, *pte, *table;
  628. ulong flag, off;
  629. flag = pa&0xFFF;
  630. pa &= ~0xFFF;
  631. if((MACHP(0)->cpuiddx & 0x08) && (getcr4() & 0x10))
  632. pse = 1;
  633. else
  634. pse = 0;
  635. for(off=0; off<size; off+=pgsz){
  636. table = &pdb[PDX(va+off)];
  637. if((*table&PTEVALID) && (*table&PTESIZE))
  638. panic("vmap: va=%#.8lux pa=%#.8lux pde=%#.8lux",
  639. va+off, pa+off, *table);
  640. /*
  641. * Check if it can be mapped using a 4MB page:
  642. * va, pa aligned and size >= 4MB and processor can do it.
  643. */
  644. if(pse && (pa+off)%(4*MB) == 0 && (va+off)%(4*MB) == 0 && (size-off) >= 4*MB){
  645. *table = (pa+off)|flag|PTESIZE|PTEVALID;
  646. pgsz = 4*MB;
  647. }else{
  648. pte = mmuwalk(pdb, va+off, 2, 1);
  649. if(*pte&PTEVALID)
  650. panic("vmap: va=%#.8lux pa=%#.8lux pte=%#.8lux",
  651. va+off, pa+off, *pte);
  652. *pte = (pa+off)|flag|PTEVALID;
  653. pgsz = BY2PG;
  654. }
  655. }
  656. return 0;
  657. }
  658. /*
  659. * Remove mappings. Must already exist, for sanity.
  660. * Only used for kernel mappings, so okay to use KADDR.
  661. */
  662. static void
  663. pdbunmap(ulong *pdb, ulong va, int size)
  664. {
  665. ulong vae;
  666. ulong *table;
  667. vae = va+size;
  668. while(va < vae){
  669. table = &pdb[PDX(va)];
  670. if(!(*table & PTEVALID)){
  671. panic("vunmap: not mapped");
  672. /*
  673. va = (va+4*MB-1) & ~(4*MB-1);
  674. continue;
  675. */
  676. }
  677. if(*table & PTESIZE){
  678. *table = 0;
  679. va = (va+4*MB-1) & ~(4*MB-1);
  680. continue;
  681. }
  682. table = KADDR(PPN(*table));
  683. if(!(table[PTX(va)] & PTEVALID))
  684. panic("vunmap: not mapped");
  685. table[PTX(va)] = 0;
  686. va += BY2PG;
  687. }
  688. }
  689. /*
  690. * Handle a fault by bringing vmap up to date.
  691. * Only copy pdb entries and they never go away,
  692. * so no locking needed.
  693. */
  694. int
  695. vmapsync(ulong va)
  696. {
  697. ulong entry, *table;
  698. if(va < VMAP || va >= VMAP+VMAPSIZE)
  699. return 0;
  700. entry = MACHP(0)->pdb[PDX(va)];
  701. if(!(entry&PTEVALID))
  702. return 0;
  703. if(!(entry&PTESIZE)){
  704. /* make sure entry will help the fault */
  705. table = KADDR(PPN(entry));
  706. if(!(table[PTX(va)]&PTEVALID))
  707. return 0;
  708. }
  709. vpd[PDX(va)] = entry;
  710. /*
  711. * TLB doesn't cache negative results, so no flush needed.
  712. */
  713. return 1;
  714. }
  715. /*
  716. * KMap is used to map individual pages into virtual memory.
  717. * It is rare to have more than a few KMaps at a time (in the
  718. * absence of interrupts, only two at a time are ever used,
  719. * but interrupts can stack). The mappings are local to a process,
  720. * so we can use the same range of virtual address space for
  721. * all processes without any coordination.
  722. */
  723. #define kpt (vpt+VPTX(KMAP))
  724. #define NKPT (KMAPSIZE/BY2PG)
  725. KMap*
  726. kmap(Page *page)
  727. {
  728. int i, o, s;
  729. if(up == nil)
  730. panic("kmap: up=0 pc=%#.8lux", getcallerpc(&page));
  731. if(up->mmupdb == nil)
  732. upallocpdb();
  733. if(up->nkmap < 0)
  734. panic("kmap %lud %s: nkmap=%d", up->pid, up->text, up->nkmap);
  735. /*
  736. * Splhi shouldn't be necessary here, but paranoia reigns.
  737. * See comment in putmmu above.
  738. */
  739. s = splhi();
  740. up->nkmap++;
  741. if(!(vpd[PDX(KMAP)]&PTEVALID)){
  742. /* allocate page directory */
  743. if(KMAPSIZE > BY2XPG)
  744. panic("bad kmapsize");
  745. if(up->kmaptable != nil)
  746. panic("kmaptable");
  747. spllo();
  748. up->kmaptable = newpage(0, 0, 0);
  749. splhi();
  750. vpd[PDX(KMAP)] = up->kmaptable->pa|PTEWRITE|PTEVALID;
  751. flushpg((ulong)kpt);
  752. memset(kpt, 0, BY2PG);
  753. kpt[0] = page->pa|PTEWRITE|PTEVALID;
  754. up->lastkmap = 0;
  755. splx(s);
  756. return (KMap*)KMAP;
  757. }
  758. if(up->kmaptable == nil)
  759. panic("no kmaptable");
  760. o = up->lastkmap+1;
  761. for(i=0; i<NKPT; i++){
  762. if(kpt[(i+o)%NKPT] == 0){
  763. o = (i+o)%NKPT;
  764. kpt[o] = page->pa|PTEWRITE|PTEVALID;
  765. up->lastkmap = o;
  766. splx(s);
  767. return (KMap*)(KMAP+o*BY2PG);
  768. }
  769. }
  770. panic("out of kmap");
  771. return nil;
  772. }
  773. void
  774. kunmap(KMap *k)
  775. {
  776. ulong va;
  777. va = (ulong)k;
  778. if(up->mmupdb == nil || !(vpd[PDX(KMAP)]&PTEVALID))
  779. panic("kunmap: no kmaps");
  780. if(va < KMAP || va >= KMAP+KMAPSIZE)
  781. panic("kunmap: bad address %#.8lux pc=%#p", va, getcallerpc(&k));
  782. if(!(vpt[VPTX(va)]&PTEVALID))
  783. panic("kunmap: not mapped %#.8lux pc=%#p", va, getcallerpc(&k));
  784. up->nkmap--;
  785. if(up->nkmap < 0)
  786. panic("kunmap %lud %s: nkmap=%d", up->pid, up->text, up->nkmap);
  787. vpt[VPTX(va)] = 0;
  788. flushpg(va);
  789. }
  790. /*
  791. * Temporary one-page mapping used to edit page directories.
  792. *
  793. * The fasttmp #define controls whether the code optimizes
  794. * the case where the page is already mapped in the physical
  795. * memory window.
  796. */
  797. #define fasttmp 1
  798. void*
  799. tmpmap(Page *p)
  800. {
  801. ulong i;
  802. ulong *entry;
  803. if(islo())
  804. panic("tmpaddr: islo");
  805. if(fasttmp && p->pa < -KZERO)
  806. return KADDR(p->pa);
  807. /*
  808. * PDX(TMPADDR) == PDX(MACHADDR), so this
  809. * entry is private to the processor and shared
  810. * between up->mmupdb (if any) and m->pdb.
  811. */
  812. entry = &vpt[VPTX(TMPADDR)];
  813. if(!(*entry&PTEVALID)){
  814. for(i=KZERO; i<=CPU0MACH; i+=BY2PG)
  815. print("%#p: *%#p=%#p (vpt=%#p index=%#p)\n", i, &vpt[VPTX(i)], vpt[VPTX(i)], vpt, VPTX(i));
  816. panic("tmpmap: no entry");
  817. }
  818. if(PPN(*entry) != PPN(TMPADDR-KZERO))
  819. panic("tmpmap: already mapped entry=%#.8lux", *entry);
  820. *entry = p->pa|PTEWRITE|PTEVALID;
  821. flushpg(TMPADDR);
  822. return (void*)TMPADDR;
  823. }
  824. void
  825. tmpunmap(void *v)
  826. {
  827. ulong *entry;
  828. if(islo())
  829. panic("tmpaddr: islo");
  830. if(fasttmp && (ulong)v >= KZERO && v != (void*)TMPADDR)
  831. return;
  832. if(v != (void*)TMPADDR)
  833. panic("tmpunmap: bad address");
  834. entry = &vpt[VPTX(TMPADDR)];
  835. if(!(*entry&PTEVALID) || PPN(*entry) == PPN(PADDR(TMPADDR)))
  836. panic("tmpmap: not mapped entry=%#.8lux", *entry);
  837. *entry = PPN(TMPADDR-KZERO)|PTEWRITE|PTEVALID;
  838. flushpg(TMPADDR);
  839. }
  840. /*
  841. * These could go back to being macros once the kernel is debugged,
  842. * but the extra checking is nice to have.
  843. */
  844. void*
  845. kaddr(ulong pa)
  846. {
  847. if(pa > (ulong)-KZERO)
  848. panic("kaddr: pa=%#.8lux", pa);
  849. return (void*)(pa+KZERO);
  850. }
  851. ulong
  852. paddr(void *v)
  853. {
  854. ulong va;
  855. va = (ulong)v;
  856. if(va < KZERO)
  857. panic("paddr: va=%#.8lux pc=%#p", va, getcallerpc(&v));
  858. return va-KZERO;
  859. }
  860. /*
  861. * More debugging.
  862. */
  863. void
  864. countpagerefs(ulong *ref, int print)
  865. {
  866. int i, n;
  867. Mach *mm;
  868. Page *pg;
  869. Proc *p;
  870. n = 0;
  871. for(i=0; i<conf.nproc; i++){
  872. p = proctab(i);
  873. if(p->mmupdb){
  874. if(print){
  875. if(ref[pagenumber(p->mmupdb)])
  876. iprint("page %#.8lux is proc %d (pid %lud) pdb\n",
  877. p->mmupdb->pa, i, p->pid);
  878. continue;
  879. }
  880. if(ref[pagenumber(p->mmupdb)]++ == 0)
  881. n++;
  882. else
  883. iprint("page %#.8lux is proc %d (pid %lud) pdb but has other refs!\n",
  884. p->mmupdb->pa, i, p->pid);
  885. }
  886. if(p->kmaptable){
  887. if(print){
  888. if(ref[pagenumber(p->kmaptable)])
  889. iprint("page %#.8lux is proc %d (pid %lud) kmaptable\n",
  890. p->kmaptable->pa, i, p->pid);
  891. continue;
  892. }
  893. if(ref[pagenumber(p->kmaptable)]++ == 0)
  894. n++;
  895. else
  896. iprint("page %#.8lux is proc %d (pid %lud) kmaptable but has other refs!\n",
  897. p->kmaptable->pa, i, p->pid);
  898. }
  899. for(pg=p->mmuused; pg; pg=pg->next){
  900. if(print){
  901. if(ref[pagenumber(pg)])
  902. iprint("page %#.8lux is on proc %d (pid %lud) mmuused\n",
  903. pg->pa, i, p->pid);
  904. continue;
  905. }
  906. if(ref[pagenumber(pg)]++ == 0)
  907. n++;
  908. else
  909. iprint("page %#.8lux is on proc %d (pid %lud) mmuused but has other refs!\n",
  910. pg->pa, i, p->pid);
  911. }
  912. for(pg=p->mmufree; pg; pg=pg->next){
  913. if(print){
  914. if(ref[pagenumber(pg)])
  915. iprint("page %#.8lux is on proc %d (pid %lud) mmufree\n",
  916. pg->pa, i, p->pid);
  917. continue;
  918. }
  919. if(ref[pagenumber(pg)]++ == 0)
  920. n++;
  921. else
  922. iprint("page %#.8lux is on proc %d (pid %lud) mmufree but has other refs!\n",
  923. pg->pa, i, p->pid);
  924. }
  925. }
  926. if(!print)
  927. iprint("%d pages in proc mmu\n", n);
  928. n = 0;
  929. for(i=0; i<conf.nmach; i++){
  930. mm = MACHP(i);
  931. for(pg=mm->pdbpool; pg; pg=pg->next){
  932. if(print){
  933. if(ref[pagenumber(pg)])
  934. iprint("page %#.8lux is in cpu%d pdbpool\n",
  935. pg->pa, i);
  936. continue;
  937. }
  938. if(ref[pagenumber(pg)]++ == 0)
  939. n++;
  940. else
  941. iprint("page %#.8lux is in cpu%d pdbpool but has other refs!\n",
  942. pg->pa, i);
  943. }
  944. }
  945. if(!print){
  946. iprint("%d pages in mach pdbpools\n", n);
  947. for(i=0; i<conf.nmach; i++)
  948. iprint("cpu%d: %d pdballoc, %d pdbfree\n",
  949. i, MACHP(i)->pdballoc, MACHP(i)->pdbfree);
  950. }
  951. }
  952. void
  953. checkfault(ulong, ulong)
  954. {
  955. }
  956. /*
  957. * Return the number of bytes that can be accessed via KADDR(pa).
  958. * If pa is not a valid argument to KADDR, return 0.
  959. */
  960. ulong
  961. cankaddr(ulong pa)
  962. {
  963. if(pa >= -KZERO)
  964. return 0;
  965. return -KZERO - pa;
  966. }