harvey/sys/src/cmd/vac/vac.c

#include "stdinc.h"
#include "vac.h"
#include "dat.h"
#include "fns.h"

// TODO: qids

void
usage(void)
{
	fprint(2, "vac [-imqsv] [-a archive.vac] [-b bsize] [-d old.vac] [-f new.vac] [-e exclude]... [-h host] file...\n");
	threadexitsall("usage");
}

enum
{
	BlockSize = 8*1024,
};

struct
{
	int nfile;
	int ndir;
	vlong data;
	vlong skipdata;
	int skipfiles;
} stats;

int qdiff;
int merge;
int verbose;
char *host;
VtConn *z;
VacFs *fs;
char *archivefile;
char *vacfile;

int vacmerge(VacFile*, char*);
void vac(VacFile*, VacFile*, char*, Dir*);
void vacstdin(VacFile*, char*);
VacFile *recentarchive(VacFs*, char*);

static u64int unittoull(char*);
static void warn(char *fmt, ...);
static void removevacfile(void);

void
threadmain(int argc, char **argv)
{
	int i, j, fd, n, printstats;
	Dir *d;
	char *s;
	uvlong u;
	VacFile *f, *fdiff;
	VacFs *fsdiff;
	int blocksize;
	int outfd;
	char *stdinname;
	char *diffvac;
	uvlong qid;


	fmtinstall('F', vtfcallfmt);
	fmtinstall('H', encodefmt);
	fmtinstall('V', vtscorefmt);

	blocksize = BlockSize;
	stdinname = nil;
	printstats = 0;
	fsdiff = nil;
	diffvac = nil;

	ARGBEGIN{
	case 'V':
		chattyventi++;
		break;
	case 'a':
		archivefile = EARGF(usage());
		break;
	case 'b':
		u = unittoull(EARGF(usage()));
		if(u < 512)
			u = 512;
		if(u > VtMaxLumpSize)
			u = VtMaxLumpSize;
		blocksize = u;
		break;
	case 'd':
		diffvac = EARGF(usage());
		break;
	case 'e':
		excludepattern(EARGF(usage()));
		break;
	case 'f':
		vacfile = EARGF(usage());
		break;
	case 'h':
		host = EARGF(usage());
		break;
	case 'i':
		stdinname = EARGF(usage());
		break;
	case 'm':
		merge++;
		break;
	case 'q':
		qdiff++;
		break;
	case 's':
		printstats++;
		break;
	case 'v':
		verbose++;
		break;
	case 'x':
		loadexcludefile(EARGF(usage()));
		break;
	default:
		usage();
	}ARGEND
	
	if(argc == 0 && !stdinname)
		usage();
	
	if(archivefile && (vacfile || diffvac)){
		fprint(2, "cannot use -a with -f, -d\n");
		usage();
	}

	z = vtdial(host);
	if(z == nil)
		sysfatal("could not connect to server: %r");
	if(vtconnect(z) < 0)
		sysfatal("vtconnect: %r");

	// Setup:
	//	fs is the output vac file system
	//	f is directory in output vac to write new files
	//	fdiff is corresponding directory in existing vac
	if(archivefile){
		VacFile *fp;
		char yyyy[5];
		char mmdd[10];
		char oldpath[40];
		Tm tm;

		fdiff = nil;
		if((outfd = open(archivefile, ORDWR)) < 0){
			if(access(archivefile, 0) >= 0)
				sysfatal("open %s: %r", archivefile);
			if((outfd = create(archivefile, OWRITE, 0666)) < 0)
				sysfatal("create %s: %r", archivefile);
			atexit(removevacfile);	// because it is new
			if((fs = vacfscreate(z, blocksize, 512)) == nil)
				sysfatal("vacfscreate: %r");
		}else{
			if((fs = vacfsopen(z, archivefile, VtORDWR, 512)) == nil)
				sysfatal("vacfsopen %s: %r", archivefile);
			if((fdiff = recentarchive(fs, oldpath)) != nil){
				if(verbose)
					fprint(2, "diff %s\n", oldpath);
			}else
				if(verbose)
					fprint(2, "no recent archive to diff against\n");
		}

		// Create yyyy/mmdd.
		tm = *localtime(time(0));
		snprint(yyyy, sizeof yyyy, "%04d", tm.year+1900);
		fp = vacfsgetroot(fs);
		if((f = vacfilewalk(fp, yyyy)) == nil
		&& (f = vacfilecreate(fp, yyyy, ModeDir|0555)) == nil)
			sysfatal("vacfscreate %s: %r", yyyy);
		vacfiledecref(fp);
		fp = f;

		snprint(mmdd, sizeof mmdd, "%02d%02d", tm.mon+1, tm.mday);
		n = 0;
		while((f = vacfilewalk(fp, mmdd)) != nil){
			vacfiledecref(f);
			n++;
			snprint(mmdd+4, sizeof mmdd-4, ".%d", n);
		}
		f = vacfilecreate(fp, mmdd, ModeDir|0555);
		if(f == nil)
			sysfatal("vacfscreate %s/%s: %r", yyyy, mmdd);
		vacfiledecref(fp);

		if(verbose)
			fprint(2, "archive %s/%s\n", yyyy, mmdd);
	}else{
		if(vacfile == nil)
			outfd = 1;
		else if((outfd = create(vacfile, OWRITE, 0666)) < 0)
			sysfatal("create %s: %r", vacfile);
		atexit(removevacfile);
		if((fs = vacfscreate(z, blocksize, 512)) == nil)
			sysfatal("vacfscreate: %r");
		f = vacfsgetroot(fs);

		fdiff = nil;
		if(diffvac){
			if((fsdiff = vacfsopen(z, diffvac, VtOREAD, 128)) == nil)
				warn("vacfsopen %s: %r", diffvac);
			else
				fdiff = vacfsgetroot(fsdiff);
		}
	}

	if(stdinname)
		vacstdin(f, stdinname);
	for(i=0; i<argc; i++){
		// We can't use / and . and .. and ../.. as valid archive
		// names, so expand to the list of files in the directory.
		if(argv[i][0] == 0){
			warn("empty string given as command-line argument");
			continue;
		}
		cleanname(argv[i]);
		if(strcmp(argv[i], "/") == 0
		|| strcmp(argv[i], ".") == 0
		|| strcmp(argv[i], "..") == 0
		|| (strlen(argv[i]) > 3 && strcmp(argv[i]+strlen(argv[i])-3, "/..") == 0)){
			if((fd = open(argv[i], OREAD)) < 0){
				warn("open %s: %r", argv[i]);
				continue;
			}
			while((n = dirread(fd, &d)) > 0){
				for(j=0; j<n; j++){
					s = vtmalloc(strlen(argv[i])+1+strlen(d[j].name)+1);
					strcpy(s, argv[i]);
					strcat(s, "/");
					strcat(s, d[j].name);
					cleanname(s);
					vac(f, fdiff, s, &d[j]);
				}
				free(d);
			}
			close(fd);
			continue;
		}
		if((d = dirstat(argv[i])) == nil){
			warn("stat %s: %r", argv[i]);
			continue;
		}
		vac(f, fdiff, argv[i], d);
		free(d);
	}
	if(fdiff)
		vacfiledecref(fdiff);
	
	/*
	 * Record the maximum qid so that vacs can be merged
	 * without introducing overlapping qids.  Older versions
	 * of vac arranged that the root would have the largest
	 * qid in the file system, but we can't do that anymore
	 * (the root gets created first!).
	 */
	if(_vacfsnextqid(fs, &qid) >= 0)
		vacfilesetqidspace(f, 0, qid);
	vacfiledecref(f);

	/*
	 * Copy fsdiff's root block score into fs's slot for that,
	 * so that vacfssync will copy it into root.prev for us.
	 * Just nice documentation, no effect.
	 */
	if(fsdiff)
		memmove(fs->score, fsdiff->score, VtScoreSize);
	if(vacfssync(fs) < 0)
		fprint(2, "vacfssync: %r\n");

	fprint(outfd, "vac:%V\n", fs->score);
	atexitdont(removevacfile);
	vacfsclose(fs);
	vthangup(z);

	if(printstats){
		fprint(2,
			"%d files, %d files skipped, %d directories\n"
			"%lld data bytes written, %lld data bytes skipped\n",
			stats.nfile, stats.skipfiles, stats.ndir, stats.data, stats.skipdata);
		dup(2, 1);
		packetstats();
	}
	threadexitsall(0);
}

VacFile*
recentarchive(VacFs *fs, char *path)
{
	VacFile *fp, *f;
	VacDirEnum *de;
	VacDir vd;
	char buf[10];
	int year, mmdd, nn, n, n1;
	char *p;
	
	fp = vacfsgetroot(fs);
	de = vdeopen(fp);
	year = 0;
	if(de){
		for(; vderead(de, &vd) > 0; vdcleanup(&vd)){
			if(strlen(vd.elem) != 4)
				continue;
			if((n = strtol(vd.elem, &p, 10)) < 1900 || *p != 0)
				continue;
			if(year < n)
				year = n;
		}
	}
	vdeclose(de);
	if(year == 0){
		vacfiledecref(fp);
		return nil;
	}
	snprint(buf, sizeof buf, "%04d", year);
	if((f = vacfilewalk(fp, buf)) == nil){
		fprint(2, "warning: dirread %s but cannot walk", buf);
		vacfiledecref(fp);
		return nil;
	}
	fp = f;
	
	de = vdeopen(fp);
	mmdd = 0;
	nn = 0;
	if(de){
		for(; vderead(de, &vd) > 0; vdcleanup(&vd)){
			if(strlen(vd.elem) < 4)
				continue;
			if((n = strtol(vd.elem, &p, 10)) < 100 || n > 1231 || p != vd.elem+4)
				continue;
			if(*p == '.'){
				if(p[1] == '0' || (n1 = strtol(p+1, &p, 10)) == 0 || *p != 0)
					continue;
			}else{
				if(*p != 0)
					continue;
				n1 = 0;
			}
			if(n < mmdd || (n == mmdd && n1 < nn))
				continue;
			mmdd = n;
			nn = n1;
		}
	}
	vdeclose(de);
	if(mmdd == 0){
		vacfiledecref(fp);
		return nil;
	}
	if(nn == 0)
		snprint(buf, sizeof buf, "%04d", mmdd);
	else
		snprint(buf, sizeof buf, "%04d.%d", mmdd, nn);
	if((f = vacfilewalk(fp, buf)) == nil){
		fprint(2, "warning: dirread %s but cannot walk", buf);
		vacfiledecref(fp);
		return nil;
	}
	vacfiledecref(fp);

	sprint(path, "%04d/%s", year, buf);
	return f;
}

static void
removevacfile(void)
{
	if(vacfile)
		remove(vacfile);
}

void
plan9tovacdir(VacDir *vd, Dir *dir)
{
	memset(vd, 0, sizeof *vd);

	vd->elem = dir->name;
	vd->uid = dir->uid;
	vd->gid = dir->gid;
	vd->mid = dir->muid;
	if(vd->mid == nil)
		vd->mid = "";
	vd->mtime = dir->mtime;
	vd->mcount = 0;
	vd->ctime = dir->mtime;		/* ctime: not available on plan 9 */
	vd->atime = dir->atime;
	vd->size = dir->length;

	vd->mode = dir->mode & 0777;
	if(dir->mode & DMDIR)
		vd->mode |= ModeDir;
	if(dir->mode & DMAPPEND)
		vd->mode |= ModeAppend;
	if(dir->mode & DMEXCL)
		vd->mode |= ModeExclusive;

	vd->plan9 = 1;
	vd->p9path = dir->qid.path;
	vd->p9version = dir->qid.vers;
}


/*
 * Archive the file named name, which has stat info d,
 * into the vac directory fp (p = parent).  
 *
 * If we're doing a vac -d against another archive, the
 * equivalent directory to fp in that archive is diffp.
 */
void
vac(VacFile *fp, VacFile *diffp, char *name, Dir *d)
{
	char *elem, *s;
	static char buf[65536];
	int fd, i, n, bsize;
	vlong off;
	Dir *dk;	// kids
	VacDir vd, vddiff;
	VacFile *f, *fdiff;
	VtEntry e;

	if(!includefile(name)){
		warn("excluding %s%s", name, (d->mode&DMDIR) ? "/" : "");
		return;
	}

	if(d->mode&DMDIR)
		stats.ndir++;
	else
		stats.nfile++;

	if(merge && vacmerge(fp, name) >= 0)
		return;
	
	if(verbose)
		fprint(2, "%s%s\n", name, (d->mode&DMDIR) ? "/" : "");

	if((fd = open(name, OREAD)) < 0){
		warn("open %s: %r", name);
		return;
	}

	elem = strrchr(name, '/');
	if(elem)
		elem++;
	else
		elem = name;

	plan9tovacdir(&vd, d);
	if((f = vacfilecreate(fp, elem, vd.mode)) == nil){
		warn("vacfilecreate %s: %r", name);
		return;
	}
	if(diffp)
		fdiff = vacfilewalk(diffp, elem);
	else
		fdiff = nil;

	if(vacfilesetdir(f, &vd) < 0)
		warn("vacfilesetdir %s: %r", name);
	
	if(d->mode&DMDIR){
		while((n = dirread(fd, &dk)) > 0){
			for(i=0; i<n; i++){
				s = vtmalloc(strlen(name)+1+strlen(dk[i].name)+1);
				strcpy(s, name);
				strcat(s, "/");
				strcat(s, dk[i].name);
				vac(f, fdiff, s, &dk[i]);
				free(s);
			}
			free(dk);
		}
	}else{
		off = 0;
		bsize = fs->bsize;
		if(fdiff){
			/*
			 * Copy fdiff's contents into f by moving the score.
			 * We'll diff and update below.
			 */
			if(vacfilegetentries(fdiff, &e, nil) >= 0)
			if(vacfilesetentries(f, &e, nil) >= 0){
				bsize = e.dsize;
			
				/*
				 * Or if -q is set, and the metadata looks the same,
				 * don't even bother reading the file.
				 */
				if(qdiff && vacfilegetdir(fdiff, &vddiff) >= 0){
					if(vddiff.mtime == vd.mtime)
					if(vddiff.size == vd.size)
					if(!vddiff.plan9 || (/* vddiff.p9path == vd.p9path && */ vddiff.p9version == vd.p9version)){
						stats.skipfiles++;
						stats.nfile--;
						vdcleanup(&vddiff);
						goto Out;
					}
					
					/*
					 * Skip over presumably-unchanged prefix
					 * of an append-only file.
					 */
					if(vd.mode&ModeAppend)
					if(vddiff.size < vd.size)
					if(vddiff.plan9 && vd.plan9)
					if(vddiff.p9path == vd.p9path){
						off = vd.size/bsize*bsize;
						if(seek(fd, off, 0) >= 0)
							stats.skipdata += off;
						else{
							seek(fd, 0, 0);	// paranoia
							off = 0;
						}
					}

					vdcleanup(&vddiff);
					// XXX different verbose chatty prints for kaminsky?
				}
			}
		}
		if(qdiff && verbose)
			fprint(2, "+%s\n", name);
		while((n = readn(fd, buf, bsize)) > 0){
			if(fdiff && sha1matches(f, off/bsize, (uchar*)buf, n)){
				off += n;
				stats.skipdata += n;
				continue;
			}
			if(vacfilewrite(f, buf, n, off) < 0){
				warn("venti write %s: %r", name);
				goto Out;
			}
			stats.data += n;
			off += n;
		}
		/*
		 * Since we started with fdiff's contents,
		 * set the size in case fdiff was bigger.
		 */
		if(fdiff && vacfilesetsize(f, off) < 0)
			warn("vtfilesetsize %s: %r", name);
	}

Out:
	vacfileflush(f, 1);
	vacfiledecref(f);
	if(fdiff)
		vacfiledecref(fdiff);
	close(fd);
}

void
vacstdin(VacFile *fp, char *name)
{
	vlong off;
	VacFile *f;
	static char buf[8192];
	int n;

	if((f = vacfilecreate(fp, name, 0666)) == nil){
		warn("vacfilecreate %s: %r", name);
		return;
	}
	
	off = 0;
	while((n = read(0, buf, sizeof buf)) > 0){
		if(vacfilewrite(f, buf, n, off) < 0){
			warn("venti write %s: %r", name);
			vacfiledecref(f);
			return;
		}
		off += n;
	}
	vacfileflush(f, 1);
	vacfiledecref(f);
}

/*
 * fp is the directory we're writing.
 * mp is the directory whose contents we're merging in.
 * d is the directory entry of the file from mp that we want to add to fp.
 * vacfile is the name of the .vac file, for error messages.
 * offset is the qid that qid==0 in mp should correspond to.
 * max is the maximum qid we expect to see (not really needed).
 */
int
vacmergefile(VacFile *fp, VacFile *mp, VacDir *d, char *vacfile,
	vlong offset, vlong max)
{
	VtEntry ed, em;
	VacFile *mf;
	VacFile *f;
	
	mf = vacfilewalk(mp, d->elem);
	if(mf == nil){
		warn("could not walk %s in %s", d->elem, vacfile);
		return -1;
	}
	if(vacfilegetentries(mf, &ed, &em) < 0){
		warn("could not get entries for %s in %s", d->elem, vacfile);
		vacfiledecref(mf);
		return -1;
	}
	
	if((f = vacfilecreate(fp, d->elem, d->mode)) == nil){
		warn("vacfilecreate %s: %r", d->elem);
		vacfiledecref(mf);
		return -1;
	}
	if(d->qidspace){
		d->qidoffset += offset;
		d->qidmax += offset;
	}else{
		d->qidspace = 1;
		d->qidoffset = offset;
		d->qidmax = max;
	}
	if(vacfilesetdir(f, d) < 0
	|| vacfilesetentries(f, &ed, &em) < 0
	|| vacfilesetqidspace(f, d->qidoffset, d->qidmax) < 0){
		warn("vacmergefile %s: %r", d->elem);
		vacfiledecref(mf);
		vacfiledecref(f);
		return -1;
	}
	
	vacfiledecref(mf);
	vacfiledecref(f);
	return 0;
}

int
vacmerge(VacFile *fp, char *name)
{
	VacFs *mfs;
	VacDir vd;
	VacDirEnum *de;
	VacFile *mp;
	uvlong maxqid, offset;

	if(strlen(name) < 4 || strcmp(name+strlen(name)-4, ".vac") != 0)
		return -1;
	if((mfs = vacfsopen(z, name, VtOREAD, 100)) == nil)
		return -1;
	if(verbose)
		fprint(2, "merging %s\n", name);

	mp = vacfsgetroot(mfs);
	de = vdeopen(mp);
	if(de){
		offset = 0;
		if(vacfsgetmaxqid(mfs, &maxqid) >= 0){
			_vacfsnextqid(fs, &offset);
			vacfsjumpqid(fs, maxqid+1);
		}
		while(vderead(de, &vd) > 0){
			if(vd.qid > maxqid){
				warn("vacmerge %s: maxqid=%lld but %s has %lld",
					name, maxqid, vd.elem, vd.qid);
				vacfsjumpqid(fs, vd.qid - maxqid);
				maxqid = vd.qid;
			}
			vacmergefile(fp, mp, &vd, name,
				offset, maxqid);
			vdcleanup(&vd);
		}
		vdeclose(de);
	}
	vacfiledecref(mp);
	vacfsclose(mfs);
	return 0;
}

#define TWID64	((u64int)~(u64int)0)

static u64int
unittoull(char *s)
{
	char *es;
	u64int n;

	if(s == nil)
		return TWID64;
	n = strtoul(s, &es, 0);
	if(*es == 'k' || *es == 'K'){
		n *= 1024;
		es++;
	}else if(*es == 'm' || *es == 'M'){
		n *= 1024*1024;
		es++;
	}else if(*es == 'g' || *es == 'G'){
		n *= 1024*1024*1024;
		es++;
	}
	if(*es != '\0')
		return TWID64;
	return n;
}

static void
warn(char *fmt, ...)
{
	va_list arg;

	va_start(arg, fmt);
	fprint(2, "vac: ");
	vfprint(2, fmt, arg);
	fprint(2, "\n");
	va_end(arg);
}