123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730 |
- #include "stdinc.h"
- #include "dat.h"
- #include "fns.h"
- /*
- * disk structure conversion routines
- */
- #define U8GET(p) ((p)[0])
- #define U16GET(p) (((p)[0]<<8)|(p)[1])
- #define U32GET(p) ((u32int)(((p)[0]<<24)|((p)[1]<<16)|((p)[2]<<8)|(p)[3]))
- #define U64GET(p) (((u64int)U32GET(p)<<32)|(u64int)U32GET((p)+4))
- #define U8PUT(p,v) (p)[0]=(v)&0xFF
- #define U16PUT(p,v) (p)[0]=((v)>>8)&0xFF;(p)[1]=(v)&0xFF
- #define U32PUT(p,v) (p)[0]=((v)>>24)&0xFF;(p)[1]=((v)>>16)&0xFF;(p)[2]=((v)>>8)&0xFF;(p)[3]=(v)&0xFF
- #define U64PUT(p,v,t32) t32=(v)>>32;U32PUT(p,t32);t32=(v);U32PUT((p)+4,t32)
- int debugarena = -1; /* hack to improve error reporting */
- static struct {
- u32int m;
- char *s;
- } magics[] = {
- ArenaPartMagic, "ArenaPartMagic",
- ArenaHeadMagic, "ArenaHeadMagic",
- ArenaMagic, "ArenaMagic",
- ISectMagic, "ISectMagic",
- BloomMagic, "BloomMagic",
- };
- static char*
- fmtmagic(char *s, u32int m)
- {
- int i;
- for(i=0; i<nelem(magics); i++)
- if(magics[i].m == m)
- return magics[i].s;
- sprint(s, "%#08ux", m);
- return s;
- }
- u32int
- unpackmagic(u8int *buf)
- {
- return U32GET(buf);
- }
- void
- packmagic(u32int magic, u8int *buf)
- {
- U32PUT(buf, magic);
- }
- int
- unpackarenapart(ArenaPart *ap, u8int *buf)
- {
- u8int *p;
- u32int m;
- char fbuf[20];
- p = buf;
- m = U32GET(p);
- if(m != ArenaPartMagic){
- seterr(ECorrupt, "arena set has wrong magic number: %s expected ArenaPartMagic (%#lux)", fmtmagic(fbuf, m), ArenaPartMagic);
- return -1;
- }
- p += U32Size;
- ap->version = U32GET(p);
- p += U32Size;
- ap->blocksize = U32GET(p);
- p += U32Size;
- ap->arenabase = U32GET(p);
- p += U32Size;
- if(buf + ArenaPartSize != p)
- sysfatal("unpackarenapart unpacked wrong amount");
- return 0;
- }
- int
- packarenapart(ArenaPart *ap, u8int *buf)
- {
- u8int *p;
- p = buf;
- U32PUT(p, ArenaPartMagic);
- p += U32Size;
- U32PUT(p, ap->version);
- p += U32Size;
- U32PUT(p, ap->blocksize);
- p += U32Size;
- U32PUT(p, ap->arenabase);
- p += U32Size;
- if(buf + ArenaPartSize != p)
- sysfatal("packarenapart packed wrong amount");
- return 0;
- }
- int
- unpackarena(Arena *arena, u8int *buf)
- {
- int sz;
- u8int *p;
- u32int m;
- char fbuf[20];
- p = buf;
- m = U32GET(p);
- if(m != ArenaMagic){
- seterr(ECorrupt, "arena %d has wrong magic number: %s "
- "expected ArenaMagic (%#lux)", debugarena,
- fmtmagic(fbuf, m), ArenaMagic);
- return -1;
- }
- p += U32Size;
- arena->version = U32GET(p);
- p += U32Size;
- namecp(arena->name, (char*)p);
- p += ANameSize;
- arena->diskstats.clumps = U32GET(p);
- p += U32Size;
- arena->diskstats.cclumps = U32GET(p);
- p += U32Size;
- arena->ctime = U32GET(p);
- p += U32Size;
- arena->wtime = U32GET(p);
- p += U32Size;
- if(arena->version == ArenaVersion5){
- arena->clumpmagic = U32GET(p);
- p += U32Size;
- }
- arena->diskstats.used = U64GET(p);
- p += U64Size;
- arena->diskstats.uncsize = U64GET(p);
- p += U64Size;
- arena->diskstats.sealed = U8GET(p);
- p += U8Size;
- switch(arena->version){
- case ArenaVersion4:
- sz = ArenaSize4;
- arena->clumpmagic = _ClumpMagic;
- break;
- case ArenaVersion5:
- sz = ArenaSize5;
- break;
- default:
- seterr(ECorrupt, "arena has bad version number %d", arena->version);
- return -1;
- }
- /*
- * Additional fields for the memstats version of the stats.
- * Diskstats reflects what is committed to the index.
- * Memstats reflects what is in the arena. Originally intended
- * this to be a version 5 extension, but might as well use for
- * all the existing version 4 arenas too.
- *
- * To maintain backwards compatibility with existing venti
- * installations using the older format, we define that if
- * memstats == diskstats, then the extension fields are not
- * included (see packarena below). That is, only partially
- * indexed arenas have these fields. Fully indexed arenas
- * (in particular, sealed arenas) do not.
- */
- if(U8GET(p) == 1){
- sz += ArenaSize5a-ArenaSize5;
- p += U8Size;
- arena->memstats.clumps = U32GET(p);
- p += U32Size;
- arena->memstats.cclumps = U32GET(p);
- p += U32Size;
- arena->memstats.used = U64GET(p);
- p += U64Size;
- arena->memstats.uncsize = U64GET(p);
- p += U64Size;
- arena->memstats.sealed = U8GET(p);
- p += U8Size;
-
- /*
- * 2008/4/2
- * Packarena (below) used to have a bug in which it would
- * not zero out any existing extension fields when writing
- * the arena metadata. This would manifest itself as arenas
- * with arena->diskstats.sealed == 1 but arena->memstats.sealed == 0
- * after a server restart. Because arena->memstats.sealed wouldn't
- * be set, the server might try to fit another block into the arena
- * (and succeed), violating the append-only structure of the log
- * and invalidating any already-computed seal on the arena.
- *
- * It might end up that other fields in arena->memstats end up
- * behind arena->diskstats too, but that would be considerably
- * more rare, and the bug is fixed now. The case we need to
- * handle is just the sealed mismatch.
- *
- * If we encounter such a bogus arena, fix the sealed field.
- */
- if(arena->diskstats.sealed)
- arena->memstats.sealed = 1;
- }else
- arena->memstats = arena->diskstats;
- if(buf + sz != p)
- sysfatal("unpackarena unpacked wrong amount");
- return 0;
- }
- int
- packarena(Arena *arena, u8int *buf)
- {
- return _packarena(arena, buf, 0);
- }
- int
- _packarena(Arena *arena, u8int *buf, int forceext)
- {
- int sz;
- u8int *p;
- u32int t32;
- switch(arena->version){
- case ArenaVersion4:
- sz = ArenaSize4;
- if(arena->clumpmagic != _ClumpMagic)
- fprint(2, "warning: writing old arena tail loses clump magic 0x%lux != 0x%lux\n",
- (ulong)arena->clumpmagic, (ulong)_ClumpMagic);
- break;
- case ArenaVersion5:
- sz = ArenaSize5;
- break;
- default:
- sysfatal("packarena unknown version %d", arena->version);
- return -1;
- }
- p = buf;
- U32PUT(p, ArenaMagic);
- p += U32Size;
- U32PUT(p, arena->version);
- p += U32Size;
- namecp((char*)p, arena->name);
- p += ANameSize;
- U32PUT(p, arena->diskstats.clumps);
- p += U32Size;
- U32PUT(p, arena->diskstats.cclumps);
- p += U32Size;
- U32PUT(p, arena->ctime);
- p += U32Size;
- U32PUT(p, arena->wtime);
- p += U32Size;
- if(arena->version == ArenaVersion5){
- U32PUT(p, arena->clumpmagic);
- p += U32Size;
- }
- U64PUT(p, arena->diskstats.used, t32);
- p += U64Size;
- U64PUT(p, arena->diskstats.uncsize, t32);
- p += U64Size;
- U8PUT(p, arena->diskstats.sealed);
- p += U8Size;
-
- /*
- * Extension fields; see above.
- */
- if(forceext
- || arena->memstats.clumps != arena->diskstats.clumps
- || arena->memstats.cclumps != arena->diskstats.cclumps
- || arena->memstats.used != arena->diskstats.used
- || arena->memstats.uncsize != arena->diskstats.uncsize
- || arena->memstats.sealed != arena->diskstats.sealed){
- sz += ArenaSize5a - ArenaSize5;
- U8PUT(p, 1);
- p += U8Size;
- U32PUT(p, arena->memstats.clumps);
- p += U32Size;
- U32PUT(p, arena->memstats.cclumps);
- p += U32Size;
- U64PUT(p, arena->memstats.used, t32);
- p += U64Size;
- U64PUT(p, arena->memstats.uncsize, t32);
- p += U64Size;
- U8PUT(p, arena->memstats.sealed);
- p += U8Size;
- }else{
- /* Clear any extension fields already on disk. */
- memset(p, 0, ArenaSize5a - ArenaSize5);
- p += ArenaSize5a - ArenaSize5;
- sz += ArenaSize5a - ArenaSize5;
- }
- if(buf + sz != p)
- sysfatal("packarena packed wrong amount");
- return 0;
- }
- int
- unpackarenahead(ArenaHead *head, u8int *buf)
- {
- u8int *p;
- u32int m;
- int sz;
- char fbuf[20];
- p = buf;
- m = U32GET(p);
- if(m != ArenaHeadMagic){
- seterr(ECorrupt, "arena %d head has wrong magic number: %s "
- "expected ArenaHeadMagic (%#lux)", debugarena,
- fmtmagic(fbuf, m), ArenaHeadMagic);
- return -1;
- }
- p += U32Size;
- head->version = U32GET(p);
- p += U32Size;
- namecp(head->name, (char*)p);
- p += ANameSize;
- head->blocksize = U32GET(p);
- p += U32Size;
- head->size = U64GET(p);
- p += U64Size;
- if(head->version == ArenaVersion5){
- head->clumpmagic = U32GET(p);
- p += U32Size;
- }
- switch(head->version){
- case ArenaVersion4:
- sz = ArenaHeadSize4;
- head->clumpmagic = _ClumpMagic;
- break;
- case ArenaVersion5:
- sz = ArenaHeadSize5;
- break;
- default:
- seterr(ECorrupt, "arena head has unexpected version %d", head->version);
- return -1;
- }
- if(buf + sz != p)
- sysfatal("unpackarenahead unpacked wrong amount");
- return 0;
- }
- int
- packarenahead(ArenaHead *head, u8int *buf)
- {
- u8int *p;
- int sz;
- u32int t32;
- switch(head->version){
- case ArenaVersion4:
- sz = ArenaHeadSize4;
- if(head->clumpmagic != _ClumpMagic)
- fprint(2, "warning: writing old arena header loses clump magic 0x%lux != 0x%lux\n",
- (ulong)head->clumpmagic, (ulong)_ClumpMagic);
- break;
- case ArenaVersion5:
- sz = ArenaHeadSize5;
- break;
- default:
- sysfatal("packarenahead unknown version %d", head->version);
- return -1;
- }
- p = buf;
- U32PUT(p, ArenaHeadMagic);
- p += U32Size;
- U32PUT(p, head->version);
- p += U32Size;
- namecp((char*)p, head->name);
- p += ANameSize;
- U32PUT(p, head->blocksize);
- p += U32Size;
- U64PUT(p, head->size, t32);
- p += U64Size;
- if(head->version == ArenaVersion5){
- U32PUT(p, head->clumpmagic);
- p += U32Size;
- }
- if(buf + sz != p)
- sysfatal("packarenahead packed wrong amount");
- return 0;
- }
- static int
- checkclump(Clump *w)
- {
- if(w->encoding == ClumpENone){
- if(w->info.size != w->info.uncsize){
- seterr(ECorrupt, "uncompressed wad size mismatch");
- return -1;
- }
- }else if(w->encoding == ClumpECompress){
- if(w->info.size >= w->info.uncsize){
- seterr(ECorrupt, "compressed lump has inconsistent block sizes %d %d", w->info.size, w->info.uncsize);
- return -1;
- }
- }else{
- seterr(ECorrupt, "clump has illegal encoding");
- return -1;
- }
- return 0;
- }
- int
- unpackclump(Clump *c, u8int *buf, u32int cmagic)
- {
- u8int *p;
- u32int magic;
- p = buf;
- magic = U32GET(p);
- if(magic != cmagic){
- seterr(ECorrupt, "clump has bad magic number=%#8.8ux != %#8.8ux", magic, cmagic);
- return -1;
- }
- p += U32Size;
- c->info.type = vtfromdisktype(U8GET(p));
- p += U8Size;
- c->info.size = U16GET(p);
- p += U16Size;
- c->info.uncsize = U16GET(p);
- p += U16Size;
- scorecp(c->info.score, p);
- p += VtScoreSize;
- c->encoding = U8GET(p);
- p += U8Size;
- c->creator = U32GET(p);
- p += U32Size;
- c->time = U32GET(p);
- p += U32Size;
- if(buf + ClumpSize != p)
- sysfatal("unpackclump unpacked wrong amount");
- return checkclump(c);
- }
- int
- packclump(Clump *c, u8int *buf, u32int magic)
- {
- u8int *p;
- p = buf;
- U32PUT(p, magic);
- p += U32Size;
- U8PUT(p, vttodisktype(c->info.type));
- p += U8Size;
- U16PUT(p, c->info.size);
- p += U16Size;
- U16PUT(p, c->info.uncsize);
- p += U16Size;
- scorecp(p, c->info.score);
- p += VtScoreSize;
- U8PUT(p, c->encoding);
- p += U8Size;
- U32PUT(p, c->creator);
- p += U32Size;
- U32PUT(p, c->time);
- p += U32Size;
- if(buf + ClumpSize != p)
- sysfatal("packclump packed wrong amount");
- return checkclump(c);
- }
- void
- unpackclumpinfo(ClumpInfo *ci, u8int *buf)
- {
- u8int *p;
- p = buf;
- ci->type = vtfromdisktype(U8GET(p));
- p += U8Size;
- ci->size = U16GET(p);
- p += U16Size;
- ci->uncsize = U16GET(p);
- p += U16Size;
- scorecp(ci->score, p);
- p += VtScoreSize;
- if(buf + ClumpInfoSize != p)
- sysfatal("unpackclumpinfo unpacked wrong amount");
- }
- void
- packclumpinfo(ClumpInfo *ci, u8int *buf)
- {
- u8int *p;
- p = buf;
- U8PUT(p, vttodisktype(ci->type));
- p += U8Size;
- U16PUT(p, ci->size);
- p += U16Size;
- U16PUT(p, ci->uncsize);
- p += U16Size;
- scorecp(p, ci->score);
- p += VtScoreSize;
- if(buf + ClumpInfoSize != p)
- sysfatal("packclumpinfo packed wrong amount");
- }
- int
- unpackisect(ISect *is, u8int *buf)
- {
- u8int *p;
- u32int m;
- char fbuf[20];
- p = buf;
- m = U32GET(p);
- if(m != ISectMagic){
- seterr(ECorrupt, "index section has wrong magic number: %s expected ISectMagic (%#lux)",
- fmtmagic(fbuf, m), ISectMagic);
- return -1;
- }
- p += U32Size;
- is->version = U32GET(p);
- p += U32Size;
- namecp(is->name, (char*)p);
- p += ANameSize;
- namecp(is->index, (char*)p);
- p += ANameSize;
- is->blocksize = U32GET(p);
- p += U32Size;
- is->blockbase = U32GET(p);
- p += U32Size;
- is->blocks = U32GET(p);
- p += U32Size;
- is->start = U32GET(p);
- p += U32Size;
- is->stop = U32GET(p);
- p += U32Size;
- if(buf + ISectSize1 != p)
- sysfatal("unpackisect unpacked wrong amount");
- is->bucketmagic = 0;
- if(is->version == ISectVersion2){
- is->bucketmagic = U32GET(p);
- p += U32Size;
- if(buf + ISectSize2 != p)
- sysfatal("unpackisect unpacked wrong amount");
- }
- return 0;
- }
- int
- packisect(ISect *is, u8int *buf)
- {
- u8int *p;
- p = buf;
- U32PUT(p, ISectMagic);
- p += U32Size;
- U32PUT(p, is->version);
- p += U32Size;
- namecp((char*)p, is->name);
- p += ANameSize;
- namecp((char*)p, is->index);
- p += ANameSize;
- U32PUT(p, is->blocksize);
- p += U32Size;
- U32PUT(p, is->blockbase);
- p += U32Size;
- U32PUT(p, is->blocks);
- p += U32Size;
- U32PUT(p, is->start);
- p += U32Size;
- U32PUT(p, is->stop);
- p += U32Size;
- if(buf + ISectSize1 != p)
- sysfatal("packisect packed wrong amount");
- if(is->version == ISectVersion2){
- U32PUT(p, is->bucketmagic);
- p += U32Size;
- if(buf + ISectSize2 != p)
- sysfatal("packisect packed wrong amount");
- }
- return 0;
- }
- void
- unpackientry(IEntry *ie, u8int *buf)
- {
- u8int *p;
- p = buf;
- scorecp(ie->score, p);
- p += VtScoreSize;
- /* ie->wtime = U32GET(p); */
- p += U32Size;
- /* ie->train = U16GET(p); */
- p += U16Size;
- if(p - buf != IEntryAddrOff)
- sysfatal("unpackentry bad IEntryAddrOff amount");
- ie->ia.addr = U64GET(p);
- if(ie->ia.addr>>56) print("%.8H => %llux\n", p, ie->ia.addr);
- p += U64Size;
- ie->ia.size = U16GET(p);
- p += U16Size;
- if(p - buf != IEntryTypeOff)
- sysfatal("unpackientry bad IEntryTypeOff amount");
- ie->ia.type = vtfromdisktype(U8GET(p));
- p += U8Size;
- ie->ia.blocks = U8GET(p);
- p += U8Size;
- if(p - buf != IEntrySize)
- sysfatal("unpackientry unpacked wrong amount");
- }
- void
- packientry(IEntry *ie, u8int *buf)
- {
- u32int t32;
- u8int *p;
- p = buf;
- scorecp(p, ie->score);
- p += VtScoreSize;
- U32PUT(p, 0); /* wtime */
- p += U32Size;
- U16PUT(p, 0); /* train */
- p += U16Size;
- U64PUT(p, ie->ia.addr, t32);
- p += U64Size;
- U16PUT(p, ie->ia.size);
- p += U16Size;
- U8PUT(p, vttodisktype(ie->ia.type));
- p += U8Size;
- U8PUT(p, ie->ia.blocks);
- p += U8Size;
- if(p - buf != IEntrySize)
- sysfatal("packientry packed wrong amount");
- }
- void
- unpackibucket(IBucket *b, u8int *buf, u32int magic)
- {
- b->n = U16GET(buf);
- b->data = buf + IBucketSize;
- if(magic && magic != U32GET(buf+U16Size))
- b->n = 0;
- }
- void
- packibucket(IBucket *b, u8int *buf, u32int magic)
- {
- U16PUT(buf, b->n);
- U32PUT(buf+U16Size, magic);
- }
- void
- packbloomhead(Bloom *b, u8int *buf)
- {
- u8int *p;
- p = buf;
- U32PUT(p, BloomMagic);
- U32PUT(p+4, BloomVersion);
- U32PUT(p+8, b->nhash);
- U32PUT(p+12, b->size);
- }
- int
- unpackbloomhead(Bloom *b, u8int *buf)
- {
- u8int *p;
- u32int m;
- char fbuf[20];
- p = buf;
- m = U32GET(p);
- if(m != BloomMagic){
- seterr(ECorrupt, "bloom filter has wrong magic number: %s expected BloomMagic (%#lux)", fmtmagic(fbuf, m), (ulong)BloomMagic);
- return -1;
- }
- p += U32Size;
-
- m = U32GET(p);
- if(m != BloomVersion){
- seterr(ECorrupt, "bloom filter has wrong version %ud expected %ud", (uint)m, (uint)BloomVersion);
- return -1;
- }
- p += U32Size;
- b->nhash = U32GET(p);
- p += U32Size;
- b->size = U32GET(p);
- p += U32Size;
- if(b->size < BloomHeadSize || b->size > MaxBloomSize || (b->size&(b->size-1))){
- seterr(ECorrupt, "bloom filter has invalid size %#lux", b->size);
- return -1;
- }
- if(buf + BloomHeadSize != p)
- sysfatal("unpackarena unpacked wrong amount");
- return 0;
- }
|