123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456 |
- /*
- * PL310 level 2 cache (non-architectural bag on the side)
- *
- * guaranteed to work incorrectly with default settings; must set Sharovr.
- *
- * clean & invalidate (wbinv) is buggy, so we work around erratum 588369
- * by disabling write-back and cache line-fill before, and restoring after.
- */
- #include "u.h"
- #include "../port/lib.h"
- #include "mem.h"
- #include "dat.h"
- #include "fns.h"
- #include "io.h"
- #include "../port/error.h"
- #include "arm.h"
- #define NWAYS(l2p) ((l2p)->auxctl & Assoc16way? 16: 8)
- #define L2P ((L2pl310 *)soc.l2cache)
- enum {
- L2size = 1024 * 1024, /* according to the tegra 2 manual */
- Wayszgran = 16 * KiB, /* granularity of way sizes */
- };
- typedef struct L2pl310 L2pl310;
- typedef struct Pl310op Pl310op;
- struct Pl310op {
- ulong pa;
- ulong _pad;
- ulong indexway;
- ulong way;
- };
- struct L2pl310 {
- ulong id;
- ulong type;
- uchar _pad0[0x100 - 0x8];
- ulong ctl;
- ulong auxctl;
- uchar _pad1[0x730 - 0x108]; /* boring regs */
- ulong sync;
- uchar _pad2[0x740 - 0x734];
- ulong r3p0sync; /* workaround for r3p0 bug */
- uchar _pad3[0x770 - 0x744];
- Pl310op inv; /* inv.indexway doesn't exist */
- uchar _pad4[0x7b0 - 0x780];
- Pl310op clean;
- uchar _pad5[0x7f0 - 0x7c0];
- Pl310op cleaninv;
- uchar _pad6[0xc00 - 0x7d0];
- ulong filtstart;
- ulong filtend;
- uchar _pad6[0xf40 - 0xc08];
- ulong debug;
- /* ... */
- };
- enum {
- /* ctl bits */
- L2enable = 1,
- /* auxctl bits */
- Ipref = 1<<29, /* prefetch enables */
- Dpref = 1<<28,
- Mbo = 1<<25,
- Sharovr = 1<<22, /* shared attribute override (i.e., work right!) */
- Parity = 1<<21,
- Waycfgshift= 17,
- Waycfgmask = (1<<3) - 1,
- Assoc16way = 1<<16,
- /*
- * optim'n to 0 cache lines; must be enabled in a9(?!).
- * set CpAClwr0line on all cpus 1st.
- */
- Fullline0= 1<<0,
- /* debug bits */
- Wt = 1<<1, /* write-through, not write-back */
- Nolinefill= 1<<0,
- Basecfg = Wt | Nolinefill,
- };
- static Lock l2lock;
- static int disallowed; /* by user: *l2off= in plan9.ini */
- static int l2ison;
- static int bg_op_running;
- static ulong waysmask;
- static Cacheimpl l2cacheimpl;
- static void
- awaitbgop(void)
- {
- while (bg_op_running)
- ;
- }
- static void
- getlock(void)
- {
- awaitbgop(); /* wait at normal PL first */
- ilock(&l2lock);
- awaitbgop(); /* wait under lock */
- }
- static void
- l2pl310sync(void)
- {
- L2P->sync = 0;
- coherence();
- }
- /* call this first to set sets/ways configuration */
- void
- l2pl310init(void)
- {
- int waysz, nways;
- ulong new;
- L2pl310 *l2p = L2P;
- static int configed;
- if (getconf("*l2off") != nil) {
- // iprint("l2 cache (pl310) disabled\n");
- disallowed = 1;
- return;
- }
- if (l2ison || configed)
- return;
- l2cache = &l2cacheimpl;
- cachedwb();
- /*
- * default config is:
- * l2: ext unified, 8 ways 512 sets 32 bytes/line => 128KB
- * but the tegra 2 manual says there's 1MB available.
- * ways or way-size may be fixed by hardware; the only way to tell
- * is to try to change the setting and read it back.
- */
- l2pl310sync();
- l2cache->inv();
- /* figure out number of ways */
- l2pl310sync();
- nways = NWAYS(l2p);
- if (!(l2p->auxctl & Assoc16way)) {
- l2p->auxctl |= Assoc16way;
- coherence();
- l2pl310sync();
- nways = NWAYS(l2p);
- // iprint("\nl2: was set for 8 ways, asked for 16, got %d\n", nways);
- }
- waysmask = MASK(nways);
- /* figure out way size (and thus number of sets) */
- waysz = L2size / nways;
- new = l2p->auxctl & ~(Waycfgmask << Waycfgshift) |
- (log2(waysz / Wayszgran) + 1) << Waycfgshift;
- l2p->auxctl = new;
- coherence();
- l2pl310sync();
- l2cache->inv();
- // iprint("\nl2: configed %d ways, %d sets (way size %d)\n", nways,
- // waysz / CACHELINESZ, waysz);
- if (l2p->auxctl != new)
- iprint("l2 config %#8.8lux didn't stick; is now %#8.8lux\n",
- new, l2p->auxctl);
- configed++;
- }
- void
- l2pl310info(Memcache *cp)
- {
- int pow2;
- ulong waysz;
- L2pl310 *l2p = L2P;
- memset(cp, 0, sizeof *cp);
- if (!l2ison)
- return;
- l2pl310init();
- assert((l2p->id >> 24) == 'A');
- cp->level = 2;
- cp->type = Unified;
- cp->external = Extcache;
- cp->setsways = Cara | Cawa | Cawt | Cawb;
- cp->l1ip = 3<<14; /* PIPT */
- cp->setsh = cp->waysh = 0; /* bag on the side */
- cp->linelen = CACHELINESZ;
- cp->log2linelen = log2(CACHELINESZ);
- cp->nways = NWAYS(l2p);
- pow2 = ((l2p->auxctl >> Waycfgshift) & Waycfgmask) - 1;
- if (pow2 < 0)
- pow2 = 0;
- waysz = (1 << pow2) * Wayszgran;
- cp->nsets = waysz / CACHELINESZ;
- }
- void
- l2pl310on(void)
- {
- ulong ctl;
- L2pl310 *l2p = L2P;
- if (getconf("*l2off") != nil) {
- // iprint("l2 cache (pl310) disabled\n");
- disallowed = 1;
- return;
- }
- if (l2ison)
- return;
- l2pl310init();
- l2cache->inv();
- /*
- * drain l1. can't turn it off (which would make locks not work)
- * because doing so makes references below to the l2 registers wedge
- * the system.
- */
- cacheuwbinv();
- cacheiinv();
- /*
- * this is only called once, on cpu0 at startup,
- * so we don't need locks here.
- * must do all configuration before enabling l2 cache.
- */
- l2p->filtend = 0;
- coherence();
- l2p->filtstart = 0; /* no enable bit */
- l2p->debug = 0; /* write-back, line fills allowed */
- coherence();
- ctl = l2p->auxctl;
- /* don't change number of sets & ways, but reset all else. */
- ctl &= Waycfgmask << Waycfgshift | Assoc16way;
- ctl |= Sharovr; /* actually work correctly for a change */
- ctl |= Mbo | Ipref | Dpref | Parity | Fullline0;
- l2p->auxctl = ctl;
- coherence();
- l2p->ctl |= L2enable;
- coherence();
- l2ison = 1;
- // iprint("l2 cache (pl310) now on\n");
- }
- void
- l2pl310off(void)
- {
- if (!l2ison)
- return;
- l2cache->wbinv();
- getlock();
- L2P->ctl &= ~L2enable;
- coherence();
- l2ison = 0;
- iunlock(&l2lock);
- }
- static void
- applyrange(ulong *reg, void *ava, int len)
- {
- uintptr va, endva;
- if (disallowed || !l2ison)
- return;
- if (len < 0)
- panic("l2cache*se called with negative length");
- endva = (uintptr)ava + len;
- for (va = (uintptr)ava & ~(CACHELINESZ-1); va < endva;
- va += CACHELINESZ)
- *reg = PADDR(va);
- l2pl310sync();
- }
- void
- l2pl310invse(void *va, int bytes)
- {
- uintptr start, end;
- L2pl310 *l2p = L2P;
- /*
- * if start & end addresses are not on cache-line boundaries,
- * flush first & last cachelines before invalidating.
- */
- start = (uintptr)va;
- end = start + bytes;
- getlock();
- if (start % CACHELINESZ != 0) {
- // iprint("l2pl310invse: unaligned start %#p from %#p\n", start,
- // getcallerpc(&va));
- applyrange(&l2p->clean.pa, va, 1);
- }
- if (end % CACHELINESZ != 0) {
- // iprint("l2pl310invse: unaligned end %#p from %#p\n", end,
- // getcallerpc(&va));
- applyrange(&l2p->clean.pa, (char *)va + bytes, 1);
- }
- applyrange(&l2p->inv.pa, va, bytes);
- iunlock(&l2lock);
- }
- void
- l2pl310wbse(void *va, int bytes)
- {
- getlock();
- applyrange(&L2P->clean.pa, va, bytes);
- iunlock(&l2lock);
- }
- /*
- * assume that ldrex/strex (thus locks) won't work when Wt in is effect,
- * so don't manipulate locks between setting and clearing Wt.
- */
- void
- l2pl310wbinvse(void *va, int bytes)
- {
- int odb;
- L2pl310 *l2p = L2P;
- if (!l2ison)
- return;
- getlock();
- applyrange(&l2p->clean.pa, va, bytes); /* paranoia */
- odb = l2p->debug;
- l2p->debug |= Wt | Nolinefill; /* erratum workaround */
- coherence();
- applyrange(&l2p->cleaninv.pa, va, bytes);
- l2p->debug = odb;
- iunlock(&l2lock);
- }
- /*
- * we want to wait for completion at normal PL.
- * if waiting is interrupted, interrupt code that calls
- * these ops could deadlock on a uniprocessor, so we only
- * give up l2lock before waiting on multiprocessors.
- * in this port, only cpu 0 gets interrupts other than local timer ones.
- */
- void
- l2pl310inv(void)
- {
- L2pl310 *l2p = L2P;
- if (disallowed)
- return;
- getlock();
- bg_op_running = 1;
- l2p->inv.way = waysmask;
- coherence();
- if (conf.nmach > 1)
- iunlock(&l2lock);
- while (l2p->inv.way & waysmask)
- ;
- if (conf.nmach > 1)
- ilock(&l2lock);
- l2pl310sync();
- bg_op_running = 0;
- iunlock(&l2lock);
- }
- /*
- * maximum time seen is 2542µs, typical is 625µs.
- */
- void
- l2pl310wb(void)
- {
- L2pl310 *l2p = L2P;
- if (disallowed || !l2ison)
- return;
- getlock();
- bg_op_running = 1;
- l2p->clean.way = waysmask;
- coherence();
- if (conf.nmach > 1)
- iunlock(&l2lock);
- while (l2p->clean.way & waysmask)
- ;
- if (conf.nmach > 1)
- ilock(&l2lock);
- l2pl310sync();
- bg_op_running = 0;
- iunlock(&l2lock);
- }
- void
- l2pl310wbinv(void)
- {
- int odb;
- L2pl310 *l2p = L2P;
- if (disallowed || !l2ison)
- return;
- l2pl310wb(); /* paranoia */
- getlock();
- bg_op_running = 1;
- odb = l2p->debug;
- l2p->debug |= Wt | Nolinefill; /* erratum workaround */
- coherence();
- l2p->cleaninv.way = waysmask;
- coherence();
- if (conf.nmach > 1)
- iunlock(&l2lock);
- while (l2p->cleaninv.way & waysmask)
- ;
- if (conf.nmach > 1)
- ilock(&l2lock);
- l2pl310sync();
- l2p->debug = odb;
- bg_op_running = 0;
- iunlock(&l2lock);
- }
- static Cacheimpl l2cacheimpl = {
- .info = l2pl310info,
- .on = l2pl310on,
- .off = l2pl310off,
- .inv = l2pl310inv,
- .wb = l2pl310wb,
- .wbinv = l2pl310wbinv,
- .invse = l2pl310invse,
- .wbse = l2pl310wbse,
- .wbinvse= l2pl310wbinvse,
- };
|