12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664 |
- /*
- * This file is part of the UCB release of Plan 9. It is subject to the license
- * terms in the LICENSE file found in the top-level directory of this
- * distribution and at http://akaros.cs.berkeley.edu/files/Plan9License. No
- * part of the UCB release of Plan 9, including this file, may be copied,
- * modified, propagated, or distributed except according to the terms contained
- * in the LICENSE file.
- */
- /*
- * myricom 10 Gb ethernet driver
- * © 2007 erik quanstrom, coraid
- *
- * the card is big endian.
- * we use u64int rather than uintptr to hold addresses so that
- * we don't get "warning: stupid shift" on 32-bit architectures.
- */
- #include "u.h"
- #include "../port/lib.h"
- #include "mem.h"
- #include "dat.h"
- #include "fns.h"
- #include "../port/error.h"
- #include "../port/netif.h"
- #include "etherif.h"
- #include "io.h"
- #ifndef KiB
- #define KiB 1024u /* Kibi 0x0000000000000400 */
- #define MiB 1048576u /* Mebi 0x0000000000100000 */
- #endif /* KiB */
- #define dprint(...) if(debug) print(__VA_ARGS__)
- #define pcicapdbg(...)
- #define malign(n) mallocalign((n), 4*KiB, 0, 0)
- #include "etherm10g2k.i"
- #include "etherm10g4k.i"
- static int debug = 0;
- static char Etimeout[] = "timeout";
- enum {
- Epromsz = 256,
- Maxslots= 1024,
- Align = 4096,
- Maxmtu = 9000,
- Noconf = 0xffffffff,
- Fwoffset= 1*MiB,
- Cmdoff = 0xf80000, /* command port offset */
- Fwsubmt = 0xfc0000, /* firmware submission command port offset */
- Rdmaoff = 0xfc01c0, /* rdma command port offset */
- };
- enum {
- CZero,
- Creset,
- Cversion,
- CSintrqdma, /* issue these before Cetherup */
- CSbigsz, /* in bytes bigsize = 2^n */
- CSsmallsz,
- CGsendoff,
- CGsmallrxoff,
- CGbigrxoff,
- CGirqackoff,
- CGirqdeassoff,
- CGsendrgsz,
- CGrxrgsz,
- CSintrqsz, /* 2^n */
- Cetherup, /* above parameters + mtu/mac addr must be set first. */
- Cetherdn,
- CSmtu, /* below may be issued live */
- CGcoaloff, /* in µs */
- CSstatsrate, /* in µs */
- CSstatsdma,
- Cpromisc,
- Cnopromisc,
- CSmac,
- Cenablefc,
- Cdisablefc,
- Cdmatest, /* address in d[0-1], d[2]=length */
- Cenableallmc,
- Cdisableallmc,
- CSjoinmc,
- CSleavemc,
- Cleaveallmc,
- CSstatsdma2, /* adds (unused) multicast stats */
- };
- typedef union {
- uint i[2];
- uint8_t c[8];
- } Cmd;
- typedef uint32_t Slot;
- typedef struct {
- uint16_t cksum;
- uint16_t len;
- } Slotparts;
- enum {
- SFsmall = 1,
- SFfirst = 2,
- SFalign = 4,
- SFnotso = 16,
- };
- typedef struct {
- uint32_t high;
- uint32_t low;
- uint16_t hdroff;
- uint16_t len;
- uint8_t pad;
- uint8_t nrdma;
- uint8_t chkoff;
- uint8_t flags;
- } Send;
- typedef struct {
- QLock QLock;
- Send *lanai; /* tx ring (cksum+len in lanai memory) */
- Send *host; /* tx ring (data in our memory) */
- Block **bring;
- // uchar *wcfifo; /* what the heck is a w/c fifo? */
- int size; /* of buffers in the z8's memory */
- uint32_t segsz;
- uint n; /* rxslots */
- uint m; /* mask; rxslots must be a power of two */
- uint i; /* number of segments (not frames) queued */
- uint cnt; /* number of segments sent by the card */
- uint32_t npkt;
- int64_t nbytes;
- } Tx;
- typedef struct {
- Lock Lock;
- Block *head;
- uint size; /* buffer size of each block */
- uint n; /* n free buffers */
- uint cnt;
- } Bpool;
- static Bpool smpool = { .size = 128, };
- static Bpool bgpool = { .size = Maxmtu, };
- typedef struct {
- Bpool *pool; /* free buffers */
- uint32_t *lanai; /* rx ring; we have no permanent host shadow */
- Block **host; /* called "info" in myricom driver */
- // uchar *wcfifo; /* cmd submission fifo */
- uint m;
- uint n; /* rxslots */
- uint i;
- uint cnt; /* number of buffers allocated (lifetime) */
- uint allocfail;
- } Rx;
- /* dma mapped. unix network byte order. */
- typedef struct {
- uint8_t txcnt[4];
- uint8_t linkstat[4];
- uint8_t dlink[4];
- uint8_t derror[4];
- uint8_t drunt[4];
- uint8_t doverrun[4];
- uint8_t dnosm[4];
- uint8_t dnobg[4];
- uint8_t nrdma[4];
- uint8_t txstopped;
- uint8_t down;
- uint8_t updated;
- uint8_t valid;
- } Stats;
- enum {
- Detached,
- Attached,
- Runed,
- };
- typedef struct {
- Slot *entry;
- uint64_t busaddr;
- uint m;
- uint n;
- uint i;
- } Done;
- typedef struct Ctlr Ctlr;
- typedef struct Ctlr {
- QLock QLock;
- int state;
- int kprocs;
- uint64_t port;
- Pcidev* pcidev;
- Ctlr* next;
- int active;
- int id; /* do we need this? */
- unsigned char ra[Eaddrlen];
- int ramsz;
- unsigned char *ram;
- uint32_t *irqack;
- uint32_t *irqdeass;
- uint32_t *coal;
- char eprom[Epromsz];
- uint32_t serial; /* unit serial number */
- QLock cmdl;
- Cmd *cmd; /* address of command return */
- uint64_t cprt; /* bus address of command */
- uint64_t boot; /* boot address */
- Done done;
- Tx tx;
- Rx sm;
- Rx bg;
- Stats *stats;
- uint64_t statsprt;
- Rendez rxrendez;
- Rendez txrendez;
- int msi;
- uint32_t linkstat;
- uint32_t nrdma;
- } Ctlr;
- static Ctlr *ctlrs;
- /*
- enum {
- PciCapPMG = 0x01, / * power management * /
- PciCapAGP = 0x02,
- PciCapVPD = 0x03, / * vital product data * /
- PciCapSID = 0x04, / * slot id * /
- PciCapMSI = 0x05,
- PciCapCHS = 0x06, / * compact pci hot swap * /
- PciCapPCIX = 0x07,
- PciCapHTC = 0x08, / * hypertransport irq conf * /
- PciCapVND = 0x09, / * vendor specific information * /
- PciCapHSW = 0x0C, / * hot swap * /
- PciCapPCIe = 0x10,
- PciCapMSIX = 0x11,
- };
- */
- enum {
- PcieAERC = 1,
- PcieVC,
- PcieSNC,
- PciePBC,
- };
- enum {
- AercCCR = 0x18, /* control register */
- };
- enum {
- PcieCTL = 8,
- PcieLCR = 12,
- PcieMRD = 0x7000, /* maximum read size */
- };
- /*
- static int
- pcicap(Pcidev *p, int cap)
- {
- int i, c, off;
- pcicapdbg("pcicap: %x:%d\n", p->vid, p->did);
- off = 0x34; / * 0x14 for cardbus * /
- for(i = 48; i--; ){
- pcicapdbg("\t" "loop %x\n", off);
- off = pcicfgr8(p, off);
- pcicapdbg("\t" "pcicfgr8 %x\n", off);
- if(off < 0x40)
- break;
- off &= ~3;
- c = pcicfgr8(p, off);
- pcicapdbg("\t" "pcicfgr8 %x\n", c);
- if(c == 0xff)
- break;
- if(c == cap)
- return off;
- off++;
- }
- return 0;
- }
- */
- /*
- * this function doesn't work because pcicgr32 doesn't have access
- * to the pcie extended configuration space.
- */
- static int
- pciecap(Pcidev *p, int cap)
- {
- uint off, i;
- off = 0x100;
- while(((i = pcicfgr32(p, off))&0xffff) != cap){
- off = i >> 20;
- print("pciecap offset = %u\n", off);
- if(off < 0x100 || off >= 4*KiB - 1)
- return 0;
- }
- print("pciecap found = %u\n", off);
- return off;
- }
- static int
- setpcie(Pcidev *p)
- {
- int off;
- /* set 4k writes */
- off = pcicap(p, PciCapPCIe);
- if(off < 64)
- return -1;
- off += PcieCTL;
- pcicfgw16(p, off, (pcicfgr16(p, off) & ~PcieMRD) | 5<<12);
- return 0;
- }
- static int
- whichfw(Pcidev *p)
- {
- char *s;
- int i, off, lanes, ecrc;
- uint32_t cap;
- /* check the number of configured lanes. */
- off = pcicap(p, PciCapPCIe);
- if(off < 64)
- return -1;
- off += PcieLCR;
- cap = pcicfgr16(p, off);
- lanes = (cap>>4) & 0x3f;
- /* check AERC register. we need it on. */
- off = pciecap(p, PcieAERC);
- print("%d offset\n", off);
- cap = 0;
- if(off != 0){
- off += AercCCR;
- cap = pcicfgr32(p, off);
- print("%u cap\n", cap);
- }
- ecrc = (cap>>4) & 0xf;
- /* if we don't like the aerc, kick it here. */
- print("m10g %d lanes; ecrc=%d; ", lanes, ecrc);
- if(0) { //s = getconf("myriforce")){
- i = atoi(s);
- if(i != 4*KiB || i != 2*KiB)
- i = 2*KiB;
- print("fw=%d [forced]\n", i);
- return i;
- }
- if(lanes <= 4){
- print("fw = 4096 [lanes]\n");
- return 4*KiB;
- }
- if(ecrc & 10){
- print("fw = 4096 [ecrc set]\n");
- return 4*KiB;
- }
- print("fw = 4096 [default]\n");
- return 4*KiB;
- }
- static int
- parseeprom(Ctlr *c)
- {
- int i, j, k, l, bits;
- char *s;
- dprint("m10g eprom:\n");
- s = c->eprom;
- bits = 3;
- for(i = 0; s[i] && i < Epromsz; i++){
- l = strlen(s+i);
- dprint("\t%s\n", s+i);
- if(strncmp(s+i, "MAC=", 4) == 0 && l == 4+12+5){
- bits ^= 1;
- j = i + 4;
- for(k = 0; k < 6; k++)
- c->ra[k] = strtoul(s+j+3*k, 0, 16);
- }else if(strncmp(s+i, "SN=", 3) == 0){
- bits ^= 2;
- c->serial = atoi(s+i+3);
- }
- i += l;
- }
- if(bits)
- return -1;
- return 0;
- }
- static uint16_t
- pbit16(uint16_t i)
- {
- uint16_t j;
- uint8_t *p;
- p = (uint8_t*)&j;
- p[1] = i;
- p[0] = i>>8;
- return j;
- }
- static uint16_t
- gbit16(uint8_t i[2])
- {
- uint16_t j;
- j = i[1];
- j |= i[0]<<8;
- return j;
- }
- static uint32_t
- pbit32(uint32_t i)
- {
- uint32_t j;
- uint8_t *p;
- p = (uint8_t*)&j;
- p[3] = i;
- p[2] = i>>8;
- p[1] = i>>16;
- p[0] = i>>24;
- return j;
- }
- static uint32_t
- gbit32(uint8_t i[4])
- {
- uint32_t j;
- j = i[3];
- j |= i[2]<<8;
- j |= i[1]<<16;
- j |= i[0]<<24;
- return j;
- }
- static void
- prepcmd(uint *cmd, int i)
- {
- while(i-- > 0)
- cmd[i] = pbit32(cmd[i]);
- }
- /*
- * the command looks like this (int 32bit integers)
- * cmd type
- * addr (low)
- * addr (high)
- * pad (used for dma testing)
- * response (high)
- * response (low)
- * 40 byte = 5 int pad.
- */
- uint32_t
- cmd(Ctlr *c, int type, uint64_t data)
- {
- Proc *up = externup();
- uint32_t buf[16], i;
- Cmd *cmd;
- qlock(&c->cmdl);
- cmd = c->cmd;
- cmd->i[1] = Noconf;
- memset(buf, 0, sizeof buf);
- buf[0] = type;
- buf[1] = data;
- buf[2] = data >> 32;
- buf[4] = c->cprt >> 32;
- buf[5] = c->cprt;
- prepcmd(buf, 6);
- coherence();
- memmove(c->ram + Cmdoff, buf, sizeof buf);
- if(waserror())
- nexterror();
- for(i = 0; i < 15; i++){
- if(cmd->i[1] != Noconf){
- poperror();
- i = gbit32(cmd->c);
- qunlock(&c->cmdl);
- if(cmd->i[1] != 0)
- dprint("[%x]", i);
- return i;
- }
- tsleep(&up->sleep, return0, 0, 1);
- }
- qunlock(&c->cmdl);
- iprint("m10g: cmd timeout [%x %x] cmd=%d\n",
- cmd->i[0], cmd->i[1], type);
- error(Etimeout);
- return ~0; /* silence! */
- }
- uint32_t
- maccmd(Ctlr *c, int type, uint8_t *mac)
- {
- Proc *up = externup();
- uint32_t buf[16], i;
- Cmd *cmd;
- qlock(&c->cmdl);
- cmd = c->cmd;
- cmd->i[1] = Noconf;
- memset(buf, 0, sizeof buf);
- buf[0] = type;
- buf[1] = mac[0]<<24 | mac[1]<<16 | mac[2]<<8 | mac[3];
- buf[2] = mac[4]<< 8 | mac[5];
- buf[4] = c->cprt >> 32;
- buf[5] = c->cprt;
- prepcmd(buf, 6);
- coherence();
- memmove(c->ram + Cmdoff, buf, sizeof buf);
- if(waserror())
- nexterror();
- for(i = 0; i < 15; i++){
- if(cmd->i[1] != Noconf){
- poperror();
- i = gbit32(cmd->c);
- qunlock(&c->cmdl);
- if(cmd->i[1] != 0)
- dprint("[%x]", i);
- return i;
- }
- tsleep(&up->sleep, return0, 0, 1);
- }
- qunlock(&c->cmdl);
- iprint("m10g: maccmd timeout [%x %x] cmd=%d\n",
- cmd->i[0], cmd->i[1], type);
- error(Etimeout);
- return ~0; /* silence! */
- }
- /* remove this garbage after testing */
- enum {
- DMAread = 0x10000,
- DMAwrite= 0x1,
- };
- uint32_t
- dmatestcmd(Ctlr *c, int type, uint64_t addr, int len)
- {
- Proc *up = externup();
- uint32_t buf[16], i;
- memset(buf, 0, sizeof buf);
- memset(c->cmd, Noconf, sizeof *c->cmd);
- buf[0] = Cdmatest;
- buf[1] = addr;
- buf[2] = addr >> 32;
- buf[3] = len * type;
- buf[4] = c->cprt >> 32;
- buf[5] = c->cprt;
- prepcmd(buf, 6);
- coherence();
- memmove(c->ram + Cmdoff, buf, sizeof buf);
- if(waserror())
- nexterror();
- for(i = 0; i < 15; i++){
- if(c->cmd->i[1] != Noconf){
- i = gbit32(c->cmd->c);
- if(i == 0)
- error(Eio);
- poperror();
- return i;
- }
- tsleep(&up->sleep, return0, 0, 5);
- }
- error(Etimeout);
- return ~0; /* silence! */
- }
- uint32_t
- rdmacmd(Ctlr *c, int on)
- {
- Proc *up = externup();
- uint32_t buf[16], i;
- memset(buf, 0, sizeof buf);
- c->cmd->i[0] = 0;
- coherence();
- buf[0] = c->cprt >> 32;
- buf[1] = c->cprt;
- buf[2] = Noconf;
- buf[3] = c->cprt >> 32;
- buf[4] = c->cprt;
- buf[5] = on;
- prepcmd(buf, 6);
- memmove(c->ram + Rdmaoff, buf, sizeof buf);
- if(waserror())
- nexterror();
- for(i = 0; i < 20; i++){
- if(c->cmd->i[0] == Noconf){
- poperror();
- return gbit32(c->cmd->c);
- }
- tsleep(&up->sleep, return0, 0, 1);
- }
- error(Etimeout);
- iprint("m10g: rdmacmd timeout\n");
- return ~0; /* silence! */
- }
- static int
- loadfw(Ctlr *c, int *align)
- {
- uint *f, *s, sz;
- int i;
- if((*align = whichfw(c->pcidev)) == 4*KiB){
- f = (uint32_t*)fw4k;
- sz = sizeof fw4k;
- }else{
- f = (uint32_t*)fw2k;
- sz = sizeof fw2k;
- }
- s = (uint32_t*)(c->ram + Fwoffset);
- for(i = 0; i < sz / 4; i++)
- s[i] = f[i];
- return sz & ~3;
- }
- static int
- bootfw(Ctlr *c)
- {
- int i, sz, align;
- uint buf[16];
- Cmd* cmd;
- if((sz = loadfw(c, &align)) == 0)
- return 0;
- dprint("bootfw %d bytes ... ", sz);
- cmd = c->cmd;
- memset(buf, 0, sizeof buf);
- c->cmd->i[0] = 0;
- coherence();
- buf[0] = c->cprt >> 32; /* upper dma target address */
- buf[1] = c->cprt; /* lower */
- buf[2] = Noconf; /* writeback */
- buf[3] = Fwoffset + 8,
- buf[4] = sz - 8;
- buf[5] = 8;
- buf[6] = 0;
- prepcmd(buf, 7);
- coherence();
- memmove(c->ram + Fwsubmt, buf, sizeof buf);
- for(i = 0; i < 20; i++){
- if(cmd->i[0] == Noconf)
- break;
- delay(1);
- }
- dprint("[%x %x]", gbit32(cmd->c), gbit32(cmd->c+4));
- if(i == 20){
- print("m10g: cannot load fw\n");
- return -1;
- }
- dprint("\n");
- c->tx.segsz = align;
- return 0;
- }
- #if 0
- static int
- kickthebaby(Pcidev *p, Ctlr *c)
- {
- /* don't kick the baby! */
- uint32_t code;
- pcicfgw8(p, 0x10 + c->boot, 0x3);
- pcicfgw32(p, 0x18 + c->boot, 0xfffffff0);
- code = pcicfgr32(p, 0x14 + c->boot);
- dprint("reboot status = %x\n", code);
- if(code != 0xfffffff0)
- return -1;
- return 0;
- }
- #endif
- typedef struct {
- uint8_t len[4];
- uint8_t type[4];
- char version[128];
- uint8_t globals[4];
- uint8_t ramsz[4];
- uint8_t specs[4];
- uint8_t specssz[4];
- } Fwhdr;
- enum {
- Tmx = 0x4d582020,
- Tpcie = 0x70636965,
- Teth = 0x45544820,
- Tmcp0 = 0x4d435030,
- };
- static char *
- fwtype(uint32_t type)
- {
- switch(type){
- case Tmx:
- return "mx";
- case Tpcie:
- return "PCIe";
- case Teth:
- return "eth";
- case Tmcp0:
- return "mcp0";
- }
- return "*GOK*";
- }
- static int
- chkfw(Ctlr *c)
- {
- uintptr_t off;
- Fwhdr *h;
- uint32_t type;
- off = gbit32(c->ram+0x3c);
- dprint("firmware %llx\n", (uint64_t)off);
- if((off&3) || off + sizeof *h > c->ramsz){
- print("!m10g: bad firmware %llx\n", (uint64_t)off);
- return -1;
- }
- h = (Fwhdr*)(c->ram + off);
- type = gbit32(h->type);
- dprint("\t" "type %s\n", fwtype(type));
- dprint("\t" "vers %s\n", h->version);
- dprint("\t" "ramsz %x\n", gbit32(h->ramsz));
- if(type != Teth){
- print("!m10g: bad card type %s\n", fwtype(type));
- return -1;
- }
- return bootfw(c) || rdmacmd(c, 0);
- }
- static int
- reset(Ether *e, Ctlr *c)
- {
- Proc *up = externup();
- uint32_t i, sz;
- if(waserror()){
- print("m10g: reset error\n");
- nexterror();
- return -1;
- }
- chkfw(c);
- cmd(c, Creset, 0);
- cmd(c, CSintrqsz, c->done.n * sizeof *c->done.entry);
- cmd(c, CSintrqdma, c->done.busaddr);
- c->irqack = (uint32_t*)(c->ram + cmd(c, CGirqackoff, 0));
- /* required only if we're not doing msi? */
- c->irqdeass = (uint32_t*)(c->ram + cmd(c, CGirqdeassoff, 0));
- /* this is the driver default, why fiddle with this? */
- c->coal = (uint32_t*)(c->ram + cmd(c, CGcoaloff, 0));
- *c->coal = pbit32(25);
- dprint("dma stats:\n");
- rdmacmd(c, 1);
- sz = c->tx.segsz;
- i = dmatestcmd(c, DMAread, c->done.busaddr, sz);
- print("\t" "read: %u MB/s\n", ((i>>16)*sz*2)/(i&0xffff));
- i = dmatestcmd(c, DMAwrite, c->done.busaddr, sz);
- print("\t" "write: %u MB/s\n", ((i>>16)*sz*2)/(i&0xffff));
- i = dmatestcmd(c, DMAwrite|DMAread, c->done.busaddr, sz);
- print("\t" "r/w: %u MB/s\n", ((i>>16)*sz*2*2)/(i&0xffff));
- memset(c->done.entry, 0, c->done.n * sizeof *c->done.entry);
- maccmd(c, CSmac, c->ra);
- // cmd(c, Cnopromisc, 0);
- cmd(c, Cenablefc, 0);
- e->Netif.maxmtu = Maxmtu;
- cmd(c, CSmtu, e->Netif.maxmtu);
- dprint("CSmtu %d...\n", e->Netif.maxmtu);
- poperror();
- return 0;
- }
- static void
- ctlrfree(Ctlr *c)
- {
- /* free up all the Block*s, too */
- free(c->tx.host);
- free(c->sm.host);
- free(c->bg.host);
- free(c->cmd);
- free(c->done.entry);
- free(c->stats);
- free(c);
- }
- static int
- setmem(Pcidev *p, Ctlr *c)
- {
- uint32_t i;
- uint64_t raddr;
- Done *d;
- void *mem;
- c->tx.segsz = 2048;
- c->ramsz = 2*MiB - (2*48*KiB + 32*KiB) - 0x100;
- if(c->ramsz > p->mem[0].size)
- return -1;
- raddr = p->mem[0].bar & ~0x0F;
- mem = vmap(raddr, p->mem[0].size);
- if(mem == nil){
- print("m10g: can't map %8.8lx\n", p->mem[0].bar);
- return -1;
- }
- dprint("%llx <- vmap(mem[0].size = %x)\n", raddr, p->mem[0].size);
- c->port = raddr;
- c->ram = mem;
- c->cmd = malign(sizeof *c->cmd);
- c->cprt = PCIWADDR(c->cmd);
- d = &c->done;
- d->n = Maxslots;
- d->m = d->n - 1;
- i = d->n * sizeof *d->entry;
- d->entry = malign(i);
- memset(d->entry, 0, i);
- d->busaddr = PCIWADDR(d->entry);
- c->stats = malign(sizeof *c->stats);
- memset(c->stats, 0, sizeof *c->stats);
- c->statsprt = PCIWADDR(c->stats);
- memmove(c->eprom, c->ram + c->ramsz - Epromsz, Epromsz-2);
- return setpcie(p) || parseeprom(c);
- }
- static Rx*
- whichrx(Ctlr *c, int sz)
- {
- if(sz <= smpool.size)
- return &c->sm;
- return &c->bg;
- }
- static Block*
- balloc(Rx* rx)
- {
- Block *b;
- ilock(&rx->pool->Lock);
- if((b = rx->pool->head) != nil){
- rx->pool->head = b->next;
- b->next = nil;
- rx->pool->n--;
- }
- iunlock(&rx->pool->Lock);
- return b;
- }
- static void
- smbfree(Block *b)
- {
- Bpool *p;
- b->rp = b->wp = (uint8_t*)ROUNDUP((uintptr_t)b->base, 4*KiB);
- b->flag &= ~(Bpktck|Btcpck|Budpck|Bipck);
- p = &smpool;
- ilock(&p->Lock);
- b->next = p->head;
- p->head = b;
- p->n++;
- p->cnt++;
- iunlock(&p->Lock);
- }
- static void
- bgbfree(Block *b)
- {
- Bpool *p;
- b->rp = b->wp = (uint8_t*)ROUNDUP((uintptr_t)b->base, 4*KiB);
- b->flag &= ~(Bpktck|Btcpck|Budpck|Bipck);
- p = &bgpool;
- ilock(&p->Lock);
- b->next = p->head;
- p->head = b;
- p->n++;
- p->cnt++;
- iunlock(&p->Lock);
- }
- static void
- replenish(Rx *rx)
- {
- uint32_t buf[16], i, idx, e;
- Bpool *p;
- Block *b;
- p = rx->pool;
- if(p->n < 8)
- return;
- memset(buf, 0, sizeof buf);
- e = (rx->i - rx->cnt) & ~7;
- e += rx->n;
- while(p->n >= 8 && e){
- idx = rx->cnt & rx->m;
- for(i = 0; i < 8; i++){
- b = balloc(rx);
- buf[i*2] = pbit32((uint64_t)PCIWADDR(b->wp) >> 32);
- buf[i*2+1] = pbit32(PCIWADDR(b->wp));
- rx->host[idx+i] = b;
- assert(b);
- }
- memmove(rx->lanai + 2*idx, buf, sizeof buf);
- coherence();
- rx->cnt += 8;
- e -= 8;
- }
- if(e && p->n > 7+1)
- print("should panic? pool->n = %d\n", p->n);
- }
- /*
- * future:
- * if (c->mtrr >= 0) {
- * c->tx.wcfifo = c->ram+0x200000;
- * c->sm.wcfifo = c->ram+0x300000;
- * c->bg.wcfifo = c->ram+0x340000;
- * }
- */
- static int
- nextpow(int j)
- {
- int i;
- for(i = 0; j > (1 << i); i++)
- ;
- return 1 << i;
- }
- static void*
- emalign(int sz)
- {
- void *v;
- v = malign(sz);
- if(v == nil)
- error(Enomem);
- memset(v, 0, sz);
- return v;
- }
- static void
- open0(Ether *e, Ctlr *c)
- {
- Block *b;
- int i, sz, entries;
- entries = cmd(c, CGsendrgsz, 0) / sizeof *c->tx.lanai;
- c->tx.lanai = (Send*)(c->ram + cmd(c, CGsendoff, 0));
- c->tx.host = emalign(entries * sizeof *c->tx.host);
- c->tx.bring = emalign(entries * sizeof *c->tx.bring);
- c->tx.n = entries;
- c->tx.m = entries-1;
- entries = cmd(c, CGrxrgsz, 0)/8;
- c->sm.pool = &smpool;
- cmd(c, CSsmallsz, c->sm.pool->size);
- c->sm.lanai = (uint32_t*)(c->ram + cmd(c, CGsmallrxoff, 0));
- c->sm.n = entries;
- c->sm.m = entries-1;
- c->sm.host = emalign(entries * sizeof *c->sm.host);
- c->bg.pool = &bgpool;
- c->bg.pool->size = nextpow(2 + e->Netif.maxmtu); /* 2-byte alignment pad */
- cmd(c, CSbigsz, c->bg.pool->size);
- c->bg.lanai = (uint32_t*)(c->ram + cmd(c, CGbigrxoff, 0));
- c->bg.n = entries;
- c->bg.m = entries-1;
- c->bg.host = emalign(entries * sizeof *c->bg.host);
- sz = c->sm.pool->size + 4*KiB;
- for(i = 0; i < c->sm.n; i++){
- if((b = allocb(sz)) == 0)
- break;
- b->free = smbfree;
- freeb(b);
- }
- sz = c->bg.pool->size + 4*KiB;
- for(i = 0; i < c->bg.n; i++){
- if((b = allocb(sz)) == 0)
- break;
- b->free = bgbfree;
- freeb(b);
- }
- cmd(c, CSstatsdma, c->statsprt);
- c->linkstat = ~0;
- c->nrdma = 15;
- cmd(c, Cetherup, 0);
- }
- static Block*
- nextblock(Ctlr *c)
- {
- uint i;
- uint16_t l, k;
- Block *b;
- Done *d;
- Rx *rx;
- Slot *s;
- Slotparts *sp;
- d = &c->done;
- s = d->entry;
- i = d->i & d->m;
- sp = (Slotparts *)(s + i);
- l = sp->len;
- if(l == 0)
- return 0;
- k = sp->cksum;
- s[i] = 0;
- d->i++;
- l = gbit16((uint8_t*)&l);
- //dprint("nextb: i=%d l=%d\n", d->i, l);
- rx = whichrx(c, l);
- if(rx->i >= rx->cnt){
- iprint("m10g: overrun\n");
- return 0;
- }
- i = rx->i & rx->m;
- b = rx->host[i];
- rx->host[i] = 0;
- if(b == 0){
- iprint("m10g: error rx to no block. memory is hosed.\n");
- return 0;
- }
- rx->i++;
- b->flag |= Bipck|Btcpck|Budpck;
- b->checksum = k;
- b->rp += 2;
- b->wp += 2+l;
- b->lim = b->wp; /* lie like a dog. */
- return b;
- }
- static int
- rxcansleep(void *v)
- {
- Ctlr *c;
- Slot *s;
- Slotparts *sp;
- Done *d;
- c = v;
- d = &c->done;
- s = c->done.entry;
- sp = (Slotparts *)(s + (d->i & d->m));
- if(sp->len != 0)
- return -1;
- c->irqack[0] = pbit32(3);
- return 0;
- }
- static void
- m10rx(void *v)
- {
- Ether *e;
- Ctlr *c;
- Block *b;
- e = v;
- c = e->ctlr;
- for(;;){
- replenish(&c->sm);
- replenish(&c->bg);
- sleep(&c->rxrendez, rxcansleep, c);
- while(b = nextblock(c))
- etheriq(e, b, 1);
- }
- }
- static void
- txcleanup(Tx *tx, uint32_t n)
- {
- Block *b;
- uint j, l, m;
- if(tx->npkt == n)
- return;
- l = 0;
- m = tx->m;
- /*
- * if tx->cnt == tx->i, yet tx->npkt == n-1, we just
- * caught ourselves and myricom card updating.
- */
- for(;; tx->cnt++){
- j = tx->cnt & tx->m;
- if(b = tx->bring[j]){
- tx->bring[j] = 0;
- tx->nbytes += BLEN(b);
- freeb(b);
- if(++tx->npkt == n)
- return;
- }
- if(tx->cnt == tx->i)
- return;
- if(l++ == m){
- iprint("tx ovrun: %u %lu\n", n, tx->npkt);
- return;
- }
- }
- }
- static int
- txcansleep(void *v)
- {
- Ctlr *c;
- c = v;
- if(c->tx.cnt != c->tx.i && c->tx.npkt != gbit32(c->stats->txcnt))
- return -1;
- return 0;
- }
- static void
- txproc(void *v)
- {
- Ether *e;
- Ctlr *c;
- Tx *tx;
- e = v;
- c = e->ctlr;
- tx = &c->tx;
- for(;;){
- sleep(&c->txrendez, txcansleep, c);
- txcleanup(tx, gbit32(c->stats->txcnt));
- }
- }
- static void
- submittx(Tx *tx, int n)
- {
- Send *l, *h;
- int i0, i, m;
- m = tx->m;
- i0 = tx->i & m;
- l = tx->lanai;
- h = tx->host;
- for(i = n-1; i >= 0; i--)
- memmove(l+(i + i0 & m), h+(i + i0 & m), sizeof *h);
- tx->i += n;
- // coherence();
- }
- static int
- nsegments(Block *b, int segsz)
- {
- uintptr_t bus, end, slen, len;
- int i;
- bus = PCIWADDR(b->rp);
- i = 0;
- for(len = BLEN(b); len; len -= slen){
- end = bus + segsz & ~(segsz-1);
- slen = end - bus;
- if(slen > len)
- slen = len;
- bus += slen;
- i++;
- }
- return i;
- }
- static void
- m10gtransmit(Ether *e)
- {
- uint16_t slen;
- uint32_t i, cnt, rdma, nseg, count, end, bus, len, segsz;
- uint8_t flags;
- Block *b;
- Ctlr *c;
- Send *s, *s0, *s0m8;
- Tx *tx;
- c = e->ctlr;
- tx = &c->tx;
- segsz = tx->segsz;
- qlock(&tx->QLock);
- count = 0;
- s = tx->host + (tx->i & tx->m);
- cnt = tx->cnt;
- s0 = tx->host + (cnt & tx->m);
- s0m8 = tx->host + ((cnt - 8) & tx->m);
- i = tx->i;
- for(; s >= s0 || s < s0m8; i += nseg){
- if((b = qget(e->oq)) == nil)
- break;
- flags = SFfirst|SFnotso;
- if((len = BLEN(b)) < 1520)
- flags |= SFsmall;
- rdma = nseg = nsegments(b, segsz);
- bus = PCIWADDR(b->rp);
- for(; len; len -= slen){
- end = bus + segsz & ~(segsz-1);
- slen = end - bus;
- if(slen > len)
- slen = len;
- s->low = pbit32(bus);
- s->len = pbit16(slen);
- s->nrdma = rdma;
- s->flags = flags;
- bus += slen;
- if(++s == tx->host + tx->n)
- s = tx->host;
- count++;
- flags &= ~SFfirst;
- rdma = 1;
- }
- tx->bring[i + nseg - 1 & tx->m] = b;
- if(1 || count > 0){
- submittx(tx, count);
- count = 0;
- cnt = tx->cnt;
- s0 = tx->host + (cnt & tx->m);
- s0m8 = tx->host + ((cnt - 8) & tx->m);
- }
- }
- qunlock(&tx->QLock);
- }
- static void
- checkstats(Ether *e, Ctlr *c, Stats *s)
- {
- uint32_t i;
- if(s->updated == 0)
- return;
- i = gbit32(s->linkstat);
- if(c->linkstat != i){
- e->Netif.link = i;
- if(c->linkstat = i)
- dprint("m10g: link up\n");
- else
- dprint("m10g: link down\n");
- }
- i = gbit32(s->nrdma);
- if(i != c->nrdma){
- dprint("m10g: rdma timeout %d\n", i);
- c->nrdma = i;
- }
- }
- static void
- waitintx(Ctlr *c)
- {
- int i;
- for(i = 0; i < 1024*1024; i++){
- if(c->stats->valid == 0)
- break;
- coherence();
- }
- }
- static void
- m10ginterrupt(Ureg *ureg, void *v)
- {
- Ether *e;
- Ctlr *c;
- e = v;
- c = e->ctlr;
- if(c->state != Runed || c->stats->valid == 0) /* not ready for us? */
- return;
- if(c->stats->valid & 1)
- wakeup(&c->rxrendez);
- if(gbit32(c->stats->txcnt) != c->tx.npkt)
- wakeup(&c->txrendez);
- if(c->msi == 0)
- *c->irqdeass = 0;
- else
- c->stats->valid = 0;
- waitintx(c);
- checkstats(e, c, c->stats);
- c->irqack[1] = pbit32(3);
- }
- static void
- m10gattach(Ether *e)
- {
- Proc *up = externup();
- Ctlr *c;
- char name[12];
- dprint("m10gattach\n");
- /* the original code use the anon struct to lock; qlock came before
- * the c = c->ctrl deref. I think what we're doing here is really equivalent,
- * but I wanted you to know.
- */
- c = e->ctlr;
- qlock(&c->QLock);
- if(c->state != Detached){
- qunlock(&c->QLock);
- return;
- }
- if(waserror()){
- c->state = Detached;
- qunlock(&c->QLock);
- nexterror();
- }
- reset(e, c);
- c->state = Attached;
- open0(e, c);
- if(c->kprocs == 0){
- c->kprocs++;
- snprint(name, sizeof name, "#l%drxproc", e->ctlrno);
- kproc(name, m10rx, e);
- snprint(name, sizeof name, "#l%dtxproc", e->ctlrno);
- kproc(name, txproc, e);
- }
- c->state = Runed;
- qunlock(&c->QLock);
- poperror();
- }
- static int
- m10gdetach(Ctlr *c)
- {
- dprint("m10gdetach\n");
- // reset(e->ctlr);
- vunmap(c->ram, c->pcidev->mem[0].size);
- ctlrfree(c);
- return -1;
- }
- static int
- lstcount(Block *b)
- {
- int i;
- i = 0;
- for(; b; b = b->next)
- i++;
- return i;
- }
- static int32_t
- m10gifstat(Ether *e, void *v, int32_t n, uint32_t off)
- {
- int l, lim;
- char *p;
- Ctlr *c;
- Stats s;
- c = e->ctlr;
- lim = 2*READSTR-1;
- p = malloc(lim+1);
- l = 0;
- /* no point in locking this because this is done via dma. */
- memmove(&s, c->stats, sizeof s);
- // l +=
- snprint(p+l, lim,
- "txcnt = %u\n" "linkstat = %u\n" "dlink = %u\n"
- "derror = %u\n" "drunt = %u\n" "doverrun = %u\n"
- "dnosm = %u\n" "dnobg = %u\n" "nrdma = %u\n"
- "txstopped = %u\n" "down = %u\n" "updated = %u\n"
- "valid = %u\n\n"
- "tx pkt = %lu\n" "tx bytes = %lld\n"
- "tx cnt = %u\n" "tx n = %u\n" "tx i = %u\n"
- "sm cnt = %u\n" "sm i = %u\n" "sm n = %u\n"
- "sm lst = %u\n"
- "bg cnt = %u\n" "bg i = %u\n" "bg n = %u\n"
- "bg lst = %u\n"
- "segsz = %u\n" "coal = %d\n",
- gbit32(s.txcnt), gbit32(s.linkstat), gbit32(s.dlink),
- gbit32(s.derror), gbit32(s.drunt), gbit32(s.doverrun),
- gbit32(s.dnosm), gbit32(s.dnobg), gbit32(s.nrdma),
- s.txstopped, s.down, s.updated, s.valid,
- c->tx.npkt, c->tx.nbytes,
- c->tx.cnt, c->tx.n, c->tx.i,
- c->sm.cnt, c->sm.i, c->sm.pool->n, lstcount(c->sm.pool->head),
- c->bg.cnt, c->bg.i, c->bg.pool->n, lstcount(c->bg.pool->head),
- c->tx.segsz, gbit32((uint8_t*)c->coal));
- n = readstr(off, v, n, p);
- free(p);
- return n;
- }
- //static void
- //summary(Ether *e)
- //{
- // char *buf;
- // int n, i, j;
- //
- // if(e == 0)
- // return;
- // buf = malloc(n=250);
- // if(buf == 0)
- // return;
- //
- // snprint(buf, n, "oq\n");
- // qsummary(e->oq, buf+3, n-3-1);
- // iprint("%s", buf);
- //
- // if(e->f) for(i = 0; e->f[i]; i++){
- // j = snprint(buf, n, "f%d %d\n", i, e->f[i]->type);
- // qsummary(e->f[i]->in, buf+j, n-j-1);
- // print("%s", buf);
- // }
- //
- // free(buf);
- //}
- static void
- rxring(Ctlr *c)
- {
- Done *d;
- Slot *s;
- Slotparts *sp;
- int i;
- d = &c->done;
- s = d->entry;
- for(i = 0; i < d->n; i++) {
- sp = (Slotparts *)(s + i);
- if(sp->len)
- iprint("s[%d] = %d\n", i, sp->len);
- }
- }
- enum {
- CMdebug,
- CMcoal,
- CMwakeup,
- CMtxwakeup,
- CMqsummary,
- CMrxring,
- };
- static Cmdtab ctab[] = {
- CMdebug, "debug", 2,
- CMcoal, "coal", 2,
- CMwakeup, "wakeup", 1,
- CMtxwakeup, "txwakeup", 1,
- // CMqsummary, "q", 1,
- CMrxring, "rxring", 1,
- };
- static int32_t
- m10gctl(Ether *e, void *v, int32_t n)
- {
- Proc *up = externup();
- int i;
- Cmdbuf *c;
- Cmdtab *t;
- dprint("m10gctl\n");
- if(e->ctlr == nil)
- error(Enonexist);
- c = parsecmd(v, n);
- if(waserror()){
- free(c);
- nexterror();
- }
- t = lookupcmd(c, ctab, nelem(ctab));
- switch(t->index){
- case CMdebug:
- debug = (strcmp(c->f[1], "on") == 0);
- break;
- case CMcoal:
- i = atoi(c->f[1]);
- if(i < 0 || i > 1000)
- error(Ebadarg);
- *((Ctlr*)e->ctlr)->coal = pbit32(i);
- break;
- case CMwakeup:
- wakeup(&((Ctlr*)e->ctlr)->rxrendez); /* you're kidding, right? */
- break;
- case CMtxwakeup:
- wakeup(&((Ctlr*)e->ctlr)->txrendez); /* you're kidding, right? */
- break;
- // case CMqsummary:
- // summary(e);
- // break;
- case CMrxring:
- rxring(e->ctlr);
- break;
- default:
- error(Ebadarg);
- }
- free(c);
- poperror();
- return n;
- }
- static void
- m10gshutdown(Ether *e)
- {
- dprint("m10gshutdown\n");
- m10gdetach(e->ctlr);
- }
- static void
- m10gpromiscuous(void *v, int on)
- {
- Ether *e;
- int i;
- dprint("m10gpromiscuous\n");
- e = v;
- if(on)
- i = Cpromisc;
- else
- i = Cnopromisc;
- cmd(e->ctlr, i, 0);
- }
- static int mcctab[] = { CSleavemc, CSjoinmc };
- static char *mcntab[] = { "leave", "join" };
- static void
- m10gmulticast(void *v, uint8_t *ea, int on)
- {
- Ether *e;
- int i;
- dprint("m10gmulticast\n");
- e = v;
- if((i = maccmd(e->ctlr, mcctab[on], ea)) != 0)
- print("m10g: can't %s %E: %d\n", mcntab[on], ea, i);
- }
- static void
- m10gpci(void)
- {
- Pcidev *p;
- Ctlr *t, *c;
- t = 0;
- for(p = 0; p = pcimatch(p, 0x14c1, 0x0008); ){
- c = malloc(sizeof *c);
- if(c == nil)
- continue;
- memset(c, 0, sizeof *c);
- c->pcidev = p;
- c->id = p->did<<16 | p->vid;
- c->boot = pcicap(p, PciCapVND);
- // kickthebaby(p, c);
- pcisetbme(p);
- if(setmem(p, c) == -1){
- print("m10g failed\n");
- free(c);
- /* cleanup */
- continue;
- }
- if(t)
- t->next = c;
- else
- ctlrs = c;
- t = c;
- }
- }
- static int
- m10gpnp(Ether *e)
- {
- Ctlr *c;
- if(ctlrs == nil)
- m10gpci();
- for(c = ctlrs; c != nil; c = c->next)
- if(c->active)
- continue;
- else if(e->ISAConf.port == 0 || e->ISAConf.port == c->port)
- break;
- if(c == nil)
- return -1;
- c->active = 1;
- e->ctlr = c;
- e->ISAConf.port = c->port;
- e->ISAConf.irq = c->pcidev->intl;
- e->tbdf = c->pcidev->tbdf;
- e->Netif.mbps = 10000;
- memmove(e->ea, c->ra, Eaddrlen);
- e->attach = m10gattach;
- e->detach = m10gshutdown;
- e->transmit = m10gtransmit;
- e->interrupt = m10ginterrupt;
- e->ifstat = m10gifstat;
- e->ctl = m10gctl;
- // e->power = m10gpower;
- e->shutdown = m10gshutdown;
- e->Netif.arg = e;
- e->Netif.promiscuous = m10gpromiscuous;
- e->Netif.multicast = m10gmulticast;
- return 0;
- }
- void
- etherm10glink(void)
- {
- addethercard("m10g", m10gpnp);
- }
|