Plan 9 from Bell Labs 2007-10-02

adm/timezone/NZ

@@ -10,14 +10,14 @@ NZS 43200 NZD 46800
  907466400  921981600  938916000  953431200  970365600  984880800
 1002420000 1016330400 1033869600 1047780000 1065319200 1079834400
 1096768800 1111284000 1128218400 1142733600 1159668000 1174183200
-1191722400 1205632800 1223172000 1237082400 1254621600 1269136800
-1286071200 1300586400 1317520800 1332036000 1349575200 1363485600
-1381024800 1394935200 1412474400 1426384800 1443924000 1458439200
-1475373600 1489888800 1506823200 1521338400 1538877600 1552788000
-1570327200 1584237600 1601776800 1616292000 1633226400 1647741600
-1664676000 1679191200 1696125600 1710640800 1728180000 1742090400
-1759629600 1773540000 1791079200 1805594400 1822528800 1837044000
-1853978400 1868493600 1886032800 1899943200 1917482400 1931392800
-1948932000 1963447200 1980381600 1994896800 2011831200 2026346400
-2043280800 2057796000 2075335200 2089245600 2106784800 2120695200
-2138234400
+1191117600 1207447200 1222567200 1238896800 1254016800 1270346400
+1285466400 1301796000 1316916000 1333245600 1348970400 1365300000
+1380420000 1396749600 1411869600 1428199200 1443319200 1459648800
+1474768800 1491098400 1506823200 1522548000 1538272800 1554602400
+1569722400 1586052000 1601172000 1617501600 1632621600 1648951200
+1664071200 1680400800 1696125600 1712455200 1727575200 1743904800
+1759024800 1775354400 1790474400 1806804000 1821924000 1838253600
+1853978400 1869703200 1885428000 1901757600 1916877600 1933207200
+1948327200 1964656800 1979776800 1996106400 2011226400 2027556000
+2043280800 2059005600 2074730400 2091060000 2106180000 2122509600
+2137629600

dist/replica/_plan9.db

@@ -293,14 +293,14 @@
 386/bin/ip/gping - 775 sys sys 1179372093 182147
 386/bin/ip/hogports - 775 sys sys 1148500655 42914
 386/bin/ip/httpd - 20000000775 sys sys 1068385801 0
-386/bin/ip/httpd/httpd - 775 sys sys 1190863308 295398
-386/bin/ip/httpd/imagemap - 775 sys sys 1190863311 116312
-386/bin/ip/httpd/man2html - 775 sys sys 1190863314 124866
-386/bin/ip/httpd/netlib_find - 775 sys sys 1190863316 117160
-386/bin/ip/httpd/netlib_history - 775 sys sys 1190863320 115952
-386/bin/ip/httpd/save - 775 sys sys 1190863324 132756
-386/bin/ip/httpd/webls - 775 sys sys 1190863326 132821
-386/bin/ip/httpd/wikipost - 775 sys sys 1190863330 114191
+386/bin/ip/httpd/httpd - 775 sys sys 1191273645 295390
+386/bin/ip/httpd/imagemap - 775 sys sys 1191273648 116304
+386/bin/ip/httpd/man2html - 775 sys sys 1191273650 124858
+386/bin/ip/httpd/netlib_find - 775 sys sys 1191273653 117152
+386/bin/ip/httpd/netlib_history - 775 sys sys 1191273655 115944
+386/bin/ip/httpd/save - 775 sys sys 1191273658 132748
+386/bin/ip/httpd/webls - 775 sys sys 1191273660 132813
+386/bin/ip/httpd/wikipost - 775 sys sys 1191273662 114183
 386/bin/ip/httpfile - 775 sys sys 1188447244 284004
 386/bin/ip/imap4d - 775 sys sys 1188447250 236867
 386/bin/ip/ipconfig - 775 sys sys 1186112228 159700
@@ -361,7 +361,7 @@
 386/bin/oventi - 20000000775 sys sys 1189138522 0
 386/bin/oventi/copy - 775 sys sys 1189196486 107409
 386/bin/p - 775 sys sys 1168402343 65220
-386/bin/page - 775 sys sys 1190716644 236230
+386/bin/page - 775 sys sys 1191208765 236255
 386/bin/paqfs - 775 sys sys 1188447270 109707
 386/bin/passwd - 775 sys sys 1168402344 84333
 386/bin/pbd - 775 sys sys 1085077087 4661
@@ -490,27 +490,28 @@
 386/bin/vac - 775 sys sys 1189146367 170972
 386/bin/vacfs - 775 sys sys 1189146374 175780
 386/bin/venti - 20000000775 sys sys 947360466 0
-386/bin/venti/buildindex - 775 sys sys 1190716651 256549
-386/bin/venti/checkarenas - 775 sys sys 1190716660 259018
-386/bin/venti/checkindex - 775 sys sys 1190716666 254387
-386/bin/venti/clumpstats - 775 sys sys 1190716673 243326
+386/bin/venti/buildindex - 775 sys sys 1191273668 259251
+386/bin/venti/checkarenas - 775 sys sys 1191273673 261130
+386/bin/venti/checkindex - 775 sys sys 1191273677 256560
+386/bin/venti/clumpstats - 775 sys sys 1191273682 245496
 386/bin/venti/conf - 775 sys sys 1189362423 1505
 386/bin/venti/copy - 775 sys sys 1189363200 107409
-386/bin/venti/findscore - 775 sys sys 1190716681 245347
-386/bin/venti/fixarenas - 775 sys sys 1190716688 213085
-386/bin/venti/fmtarenas - 775 sys sys 1190716697 245595
-386/bin/venti/fmtbloom - 775 sys sys 1190716703 168825
-386/bin/venti/fmtindex - 775 sys sys 1190716709 243593
-386/bin/venti/fmtisect - 775 sys sys 1190716717 243945
-386/bin/venti/mirrorarenas - 775 sys sys 1190716727 252839
-386/bin/venti/printarena - 775 sys sys 1190716735 255634
-386/bin/venti/rdarena - 775 sys sys 1190716741 245021
+386/bin/venti/findscore - 775 sys sys 1191273686 247532
+386/bin/venti/fixarenas - 775 sys sys 1191273690 212717
+386/bin/venti/fmtarenas - 775 sys sys 1191273694 247765
+386/bin/venti/fmtbloom - 775 sys sys 1191273697 168531
+386/bin/venti/fmtindex - 775 sys sys 1191273702 245763
+386/bin/venti/fmtisect - 775 sys sys 1191273706 246115
+386/bin/venti/mirrorarenas - 775 sys sys 1191273710 255223
+386/bin/venti/printarena - 775 sys sys 1191273715 257804
+386/bin/venti/rdarena - 775 sys sys 1191273719 247191
 386/bin/venti/read - 775 sys sys 1190716746 154800
+386/bin/venti/ro - 775 sys sys 1191273722 161694
 386/bin/venti/sync - 775 sys sys 1190716753 153672
-386/bin/venti/syncindex - 775 sys sys 1190716764 274328
-386/bin/venti/venti - 775 sys sys 1190863339 459617
-386/bin/venti/verifyarena - 775 sys sys 1189363307 154989
-386/bin/venti/wrarena - 775 sys sys 1190716786 282035
+386/bin/venti/syncindex - 775 sys sys 1191273727 267127
+386/bin/venti/venti - 775 sys sys 1191273735 459037
+386/bin/venti/verifyarena - 775 sys sys 1191273738 166940
+386/bin/venti/wrarena - 775 sys sys 1191273743 284205
 386/bin/venti/write - 775 sys sys 1190716792 155215
 386/bin/vncs - 775 sys sys 1188447411 471527
 386/bin/vncv - 775 sys sys 1188447423 518952
@@ -566,7 +567,7 @@
 386/lib/libframe.a - 664 sys sys 1184529910 66398
 386/lib/libgeometry.a - 664 sys sys 1168402366 50470
 386/lib/libhtml.a - 664 sys sys 1186370975 229202
-386/lib/libhttpd.a - 664 sys sys 1190863344 100872
+386/lib/libhttpd.a - 664 sys sys 1191273745 100822
 386/lib/libip.a - 664 sys sys 1187061208 34710
 386/lib/libl.a - 664 sys sys 1168402367 5372
 386/lib/libmach.a - 664 sys sys 1188530244 807480
@@ -787,7 +788,7 @@ adm/timezone/Libya - 664 adm adm 1020313577 1251
 adm/timezone/Mexico_BajaNorte - 664 adm adm 1020313577 1144
 adm/timezone/Mexico_BajaSur - 664 adm adm 1020313577 22
 adm/timezone/Mexico_General - 664 adm adm 1020313577 22
-adm/timezone/NZ - 664 adm adm 1020313577 1417
+adm/timezone/NZ - 664 adm adm 1191217396 1417
 adm/timezone/NZ_CHAT - 664 adm adm 1020313577 20
 adm/timezone/Navajo - 664 adm adm 1020369288 1518
 adm/timezone/PRC - 664 adm adm 1020369288 1516
@@ -5538,7 +5539,7 @@ rc/bin/nroff - 775 sys sys 945617208 27
 rc/bin/patch - 20000000775 sys sys 1155363651 0
 rc/bin/patch/applied - 775 sys sys 1115941614 111
 rc/bin/patch/apply - 775 sys sys 1109511606 1285
-rc/bin/patch/create - 775 sys sys 1141738939 1509
+rc/bin/patch/create - 775 sys sys 1191217103 1516
 rc/bin/patch/diff - 775 sys sys 1109511606 474
 rc/bin/patch/list - 775 sys sys 1115941576 749
 rc/bin/patch/move - 775 sys sys 1112166652 643
@@ -8204,7 +8205,7 @@ sys/src/9/port/devcons.c - 664 sys sys 1176658321 22943
 sys/src/9/port/devdraw.c - 664 sys sys 1147023550 44447
 sys/src/9/port/devdup.c - 664 sys sys 1014931172 2332
 sys/src/9/port/devenv.c - 664 sys sys 1169498893 7015
-sys/src/9/port/devfs.c - 664 sys sys 1191008235 14697
+sys/src/9/port/devfs.c - 664 sys sys 1191279047 14902
 sys/src/9/port/devkbmap.c - 664 sys sys 1130763846 3064
 sys/src/9/port/devkprof.c - 664 sys sys 1014931173 3111
 sys/src/9/port/devloopback.c - 664 sys sys 1138458368 14579
@@ -10102,7 +10103,7 @@ sys/src/cmd/cwfs/lrand.c - 664 sys sys 1171160167 1070
 sys/src/cmd/cwfs/main.c - 664 sys sys 1189033288 10146
 sys/src/cmd/cwfs/malloc.c - 664 sys sys 1174281557 2360
 sys/src/cmd/cwfs/mkfile - 664 sys sys 1174941889 201
-sys/src/cmd/cwfs/mworm.c - 664 sys sys 1174370308 4311
+sys/src/cmd/cwfs/mworm.c - 664 sys sys 1191264337 4542
 sys/src/cmd/cwfs/net.c - 664 sys sys 1174521421 9939
 sys/src/cmd/cwfs/pc.c - 664 sys sys 1174976414 1718
 sys/src/cmd/cwfs/portdat.h - 664 sys sys 1174716952 15241
@@ -14308,7 +14309,7 @@ sys/src/cmd/vc/v.out.h - 664 sys sys 1089299165 2630
 sys/src/cmd/venti - 20000000775 sys sys 1068500904 0
 sys/src/cmd/venti/copy.c - 664 sys sys 1189017556 5275
 sys/src/cmd/venti/devnull.c - 664 sys sys 1177189434 1225
-sys/src/cmd/venti/mkfile - 664 sys sys 1189017596 358
+sys/src/cmd/venti/mkfile - 664 sys sys 1191258765 439
 sys/src/cmd/venti/mkroot.c - 664 sys sys 1177189435 1178
 sys/src/cmd/venti/randtest.c - 664 sys sys 1177189435 5656
 sys/src/cmd/venti/read.c - 664 sys sys 1177189435 1289
@@ -14316,21 +14317,21 @@ sys/src/cmd/venti/readlist.c - 664 sys sys 1177189435 1934
 sys/src/cmd/venti/ro.c - 664 sys sys 1177189435 1886
 sys/src/cmd/venti/root.c - 664 sys sys 1177189435 1329
 sys/src/cmd/venti/srv - 20000000775 sys sys 1189020012 0
-sys/src/cmd/venti/srv/arena.c - 664 sys sys 1178160303 16402
+sys/src/cmd/venti/srv/arena.c - 664 sys sys 1191257299 20137
 sys/src/cmd/venti/srv/arenas.c - 664 sys sys 1178160303 8039
 sys/src/cmd/venti/srv/bloom.c - 664 sys sys 1178160303 4529
 sys/src/cmd/venti/srv/buildbuck.c - 664 sys sys 1177189435 2837
-sys/src/cmd/venti/srv/buildindex.c - 664 sys sys 1182131900 21117
-sys/src/cmd/venti/srv/checkarenas.c - 664 sys sys 1177189435 2313
+sys/src/cmd/venti/srv/buildindex.c - 664 sys sys 1191257306 21645
+sys/src/cmd/venti/srv/checkarenas.c - 664 sys sys 1191257304 2310
 sys/src/cmd/venti/srv/checkindex.c - 664 sys sys 1179342633 5978
-sys/src/cmd/venti/srv/clump.c - 664 sys sys 1177189435 5329
+sys/src/cmd/venti/srv/clump.c - 664 sys sys 1191257297 5316
 sys/src/cmd/venti/srv/clumpstats.c - 664 sys sys 1142736351 2234
 sys/src/cmd/venti/srv/cmparenas.c - 664 sys sys 1177189435 7215
 sys/src/cmd/venti/srv/conf.rc - 775 sys sys 1189304120 1416
-sys/src/cmd/venti/srv/config.c - 664 sys sys 1178160303 5566
-sys/src/cmd/venti/srv/conv.c - 664 sys sys 1178160303 14052
-sys/src/cmd/venti/srv/dat.h - 664 sys sys 1178160303 19018
-sys/src/cmd/venti/srv/dcache.c - 664 sys sys 1190092637 18695
+sys/src/cmd/venti/srv/config.c - 664 sys sys 1191257289 5659
+sys/src/cmd/venti/srv/conv.c - 664 sys sys 1191257291 14072
+sys/src/cmd/venti/srv/dat.h - 664 sys sys 1191257301 19110
+sys/src/cmd/venti/srv/dcache.c - 664 sys sys 1191257300 15583
 sys/src/cmd/venti/srv/disksched.c - 664 sys sys 1142736352 2125
 sys/src/cmd/venti/srv/dump.c - 664 sys sys 1142736352 1642
 sys/src/cmd/venti/srv/findscore.c - 664 sys sys 1179863768 2195
@@ -14339,20 +14340,20 @@ sys/src/cmd/venti/srv/fmtarenas.c - 664 sys sys 1177189436 2702
 sys/src/cmd/venti/srv/fmtbloom.c - 664 sys sys 1142736352 2293
 sys/src/cmd/venti/srv/fmtindex.c - 664 sys sys 1178160304 2599
 sys/src/cmd/venti/srv/fmtisect.c - 664 sys sys 1177189436 1454
-sys/src/cmd/venti/srv/fns.h - 664 sys sys 1178160304 9179
+sys/src/cmd/venti/srv/fns.h - 664 sys sys 1191257308 9208
 sys/src/cmd/venti/srv/graph.c - 664 sys sys 1177189436 4190
-sys/src/cmd/venti/srv/hdisk.c - 664 sys sys 1177189437 16510
-sys/src/cmd/venti/srv/httpd.c - 664 sys sys 1177189437 22784
-sys/src/cmd/venti/srv/icache.c - 664 sys sys 1186114652 8778
-sys/src/cmd/venti/srv/icachewrite.c - 664 sys sys 1190076776 7787
+sys/src/cmd/venti/srv/hdisk.c - 664 sys sys 1191257294 16482
+sys/src/cmd/venti/srv/httpd.c - 664 sys sys 1191257304 22869
+sys/src/cmd/venti/srv/icache.c - 664 sys sys 1191257303 10676
+sys/src/cmd/venti/srv/icachewrite.c - 664 sys sys 1191257310 7528
 sys/src/cmd/venti/srv/ifile.c - 664 sys sys 1189307036 2592
-sys/src/cmd/venti/srv/index.c - 664 sys sys 1178160304 17239
-sys/src/cmd/venti/srv/lump.c - 664 sys sys 1177189437 5799
+sys/src/cmd/venti/srv/index.c - 664 sys sys 1191257302 17933
+sys/src/cmd/venti/srv/lump.c - 664 sys sys 1191257295 4947
 sys/src/cmd/venti/srv/lumpcache.c - 664 sys sys 1177189437 8811
 sys/src/cmd/venti/srv/lumpqueue.c - 664 sys sys 1142736354 2722
-sys/src/cmd/venti/srv/mirrorarenas.c - 664 sys sys 1178160304 10544
+sys/src/cmd/venti/srv/mirrorarenas.c - 664 sys sys 1191257291 10802
 sys/src/cmd/venti/srv/mkfile - 664 sys sys 1189304068 1243
-sys/src/cmd/venti/srv/part.c - 664 sys sys 1180244694 5662
+sys/src/cmd/venti/srv/part.c - 664 sys sys 1191217317 4500
 sys/src/cmd/venti/srv/png.c - 664 sys sys 1142736354 3729
 sys/src/cmd/venti/srv/printarena.c - 664 sys sys 1177189438 2673
 sys/src/cmd/venti/srv/printarenapart.c - 664 sys sys 1178160304 3544
@@ -14365,17 +14366,17 @@ sys/src/cmd/venti/srv/reseal.c - 664 sys sys 1177189438 6682
 sys/src/cmd/venti/srv/round.c - 664 sys sys 1142736355 1577
 sys/src/cmd/venti/srv/score.c - 664 sys sys 1178160305 740
 sys/src/cmd/venti/srv/sortientry.c - 664 sys sys 1177189439 8325
-sys/src/cmd/venti/srv/stats.c - 664 sys sys 1177189439 3912
+sys/src/cmd/venti/srv/stats.c - 664 sys sys 1191257294 4054
 sys/src/cmd/venti/srv/stdinc.h - 664 sys sys 1177189439 169
-sys/src/cmd/venti/srv/syncarena.c - 664 sys sys 1177249927 4834
-sys/src/cmd/venti/srv/syncindex.c - 664 sys sys 1177249927 1622
-sys/src/cmd/venti/srv/syncindex0.c - 664 sys sys 1177249927 4447
+sys/src/cmd/venti/srv/syncarena.c - 664 sys sys 1191257305 4816
+sys/src/cmd/venti/srv/syncindex.c - 664 sys sys 1191257309 1277
+sys/src/cmd/venti/srv/syncindex0.c - 664 sys sys 1191257309 1951
 sys/src/cmd/venti/srv/trace.c - 664 sys sys 1142736356 709
 sys/src/cmd/venti/srv/unittoull.c - 664 sys sys 1142736356 471
 sys/src/cmd/venti/srv/unwhack.c - 664 sys sys 1142736356 3191
 sys/src/cmd/venti/srv/utils.c - 664 sys sys 1178160305 3755
-sys/src/cmd/venti/srv/venti.c - 664 sys sys 1188624404 5882
-sys/src/cmd/venti/srv/verifyarena.c - 664 sys sys 1178160305 5666
+sys/src/cmd/venti/srv/venti.c - 664 sys sys 1191257307 5584
+sys/src/cmd/venti/srv/verifyarena.c - 664 sys sys 1191257292 5776
 sys/src/cmd/venti/srv/whack.c - 664 sys sys 1142736357 6419
 sys/src/cmd/venti/srv/whack.h - 664 sys sys 1142736357 966
 sys/src/cmd/venti/srv/wrarena.c - 664 sys sys 1183678902 4647
@@ -15228,7 +15229,7 @@ sys/src/libhttpd/httpunesc.c - 664 sys sys 1014930784 888
 sys/src/libhttpd/lower.c - 664 sys sys 1014930784 226
 sys/src/libhttpd/mkfile - 664 sys sys 1035389778 421
 sys/src/libhttpd/okheaders.c - 664 sys sys 1014930784 433
-sys/src/libhttpd/parse.c - 664 sys sys 1190835951 19361
+sys/src/libhttpd/parse.c - 664 sys sys 1191217118 19346
 sys/src/libhttpd/parsereq.c - 664 sys sys 1177187800 5112
 sys/src/libhttpd/query.c - 664 sys sys 1014930785 712
 sys/src/libhttpd/redirected.c - 664 sys sys 1014930785 1555
@@ -15732,4 +15733,4 @@ usr/glenda/lib/profile - 664 glenda glenda 1105128663 890
 usr/glenda/readme.acme - 664 glenda glenda 1019860628 4753
 usr/glenda/readme.rio - 664 glenda glenda 1019860628 6370
 usr/glenda/tmp - 20000000775 glenda glenda 1018802620 0
-386/bin/page - 775 sys sys 1191208765 236255
+386/bin/cwfs - 775 sys sys 1191295136 365724

dist/replica/plan9.db

@@ -191,7 +191,7 @@
 386/bin/cpp - 775 sys sys 1188446882 149057
 386/bin/cpu - 775 sys sys 1188446885 139216
 386/bin/crop - 775 sys sys 1168402298 116374
-386/bin/cwfs - 775 sys sys 1190401259 365500
+386/bin/cwfs - 775 sys sys 1191295136 365724
 386/bin/date - 775 sys sys 1178568265 41845
 386/bin/db - 775 sys sys 1188499570 349188
 386/bin/dc - 775 sys sys 1168402299 99260
@@ -293,14 +293,14 @@
 386/bin/ip/gping - 775 sys sys 1179372093 182147
 386/bin/ip/hogports - 775 sys sys 1148500655 42914
 386/bin/ip/httpd - 20000000775 sys sys 1068385801 0
-386/bin/ip/httpd/httpd - 775 sys sys 1190863308 295398
-386/bin/ip/httpd/imagemap - 775 sys sys 1190863311 116312
-386/bin/ip/httpd/man2html - 775 sys sys 1190863314 124866
-386/bin/ip/httpd/netlib_find - 775 sys sys 1190863316 117160
-386/bin/ip/httpd/netlib_history - 775 sys sys 1190863320 115952
-386/bin/ip/httpd/save - 775 sys sys 1190863324 132756
-386/bin/ip/httpd/webls - 775 sys sys 1190863326 132821
-386/bin/ip/httpd/wikipost - 775 sys sys 1190863330 114191
+386/bin/ip/httpd/httpd - 775 sys sys 1191273645 295390
+386/bin/ip/httpd/imagemap - 775 sys sys 1191273648 116304
+386/bin/ip/httpd/man2html - 775 sys sys 1191273650 124858
+386/bin/ip/httpd/netlib_find - 775 sys sys 1191273653 117152
+386/bin/ip/httpd/netlib_history - 775 sys sys 1191273655 115944
+386/bin/ip/httpd/save - 775 sys sys 1191273658 132748
+386/bin/ip/httpd/webls - 775 sys sys 1191273660 132813
+386/bin/ip/httpd/wikipost - 775 sys sys 1191273662 114183
 386/bin/ip/httpfile - 775 sys sys 1188447244 284004
 386/bin/ip/imap4d - 775 sys sys 1188447250 236867
 386/bin/ip/ipconfig - 775 sys sys 1186112228 159700
@@ -490,27 +490,28 @@
 386/bin/vac - 775 sys sys 1189146367 170972
 386/bin/vacfs - 775 sys sys 1189146374 175780
 386/bin/venti - 20000000775 sys sys 947360466 0
-386/bin/venti/buildindex - 775 sys sys 1190716651 256549
-386/bin/venti/checkarenas - 775 sys sys 1190716660 259018
-386/bin/venti/checkindex - 775 sys sys 1190716666 254387
-386/bin/venti/clumpstats - 775 sys sys 1190716673 243326
+386/bin/venti/buildindex - 775 sys sys 1191273668 259251
+386/bin/venti/checkarenas - 775 sys sys 1191273673 261130
+386/bin/venti/checkindex - 775 sys sys 1191273677 256560
+386/bin/venti/clumpstats - 775 sys sys 1191273682 245496
 386/bin/venti/conf - 775 sys sys 1189362423 1505
 386/bin/venti/copy - 775 sys sys 1189363200 107409
-386/bin/venti/findscore - 775 sys sys 1190716681 245347
-386/bin/venti/fixarenas - 775 sys sys 1190716688 213085
-386/bin/venti/fmtarenas - 775 sys sys 1190716697 245595
-386/bin/venti/fmtbloom - 775 sys sys 1190716703 168825
-386/bin/venti/fmtindex - 775 sys sys 1190716709 243593
-386/bin/venti/fmtisect - 775 sys sys 1190716717 243945
-386/bin/venti/mirrorarenas - 775 sys sys 1190716727 252839
-386/bin/venti/printarena - 775 sys sys 1190716735 255634
-386/bin/venti/rdarena - 775 sys sys 1190716741 245021
+386/bin/venti/findscore - 775 sys sys 1191273686 247532
+386/bin/venti/fixarenas - 775 sys sys 1191273690 212717
+386/bin/venti/fmtarenas - 775 sys sys 1191273694 247765
+386/bin/venti/fmtbloom - 775 sys sys 1191273697 168531
+386/bin/venti/fmtindex - 775 sys sys 1191273702 245763
+386/bin/venti/fmtisect - 775 sys sys 1191273706 246115
+386/bin/venti/mirrorarenas - 775 sys sys 1191273710 255223
+386/bin/venti/printarena - 775 sys sys 1191273715 257804
+386/bin/venti/rdarena - 775 sys sys 1191273719 247191
 386/bin/venti/read - 775 sys sys 1190716746 154800
+386/bin/venti/ro - 775 sys sys 1191273722 161694
 386/bin/venti/sync - 775 sys sys 1190716753 153672
-386/bin/venti/syncindex - 775 sys sys 1190716764 274328
-386/bin/venti/venti - 775 sys sys 1190863339 459617
-386/bin/venti/verifyarena - 775 sys sys 1189363307 154989
-386/bin/venti/wrarena - 775 sys sys 1190716786 282035
+386/bin/venti/syncindex - 775 sys sys 1191273727 267127
+386/bin/venti/venti - 775 sys sys 1191273735 459037
+386/bin/venti/verifyarena - 775 sys sys 1191273738 166940
+386/bin/venti/wrarena - 775 sys sys 1191273743 284205
 386/bin/venti/write - 775 sys sys 1190716792 155215
 386/bin/vncs - 775 sys sys 1188447411 471527
 386/bin/vncv - 775 sys sys 1188447423 518952
@@ -566,7 +567,7 @@
 386/lib/libframe.a - 664 sys sys 1184529910 66398
 386/lib/libgeometry.a - 664 sys sys 1168402366 50470
 386/lib/libhtml.a - 664 sys sys 1186370975 229202
-386/lib/libhttpd.a - 664 sys sys 1190863344 100872
+386/lib/libhttpd.a - 664 sys sys 1191273745 100822
 386/lib/libip.a - 664 sys sys 1187061208 34710
 386/lib/libl.a - 664 sys sys 1168402367 5372
 386/lib/libmach.a - 664 sys sys 1188530244 807480
@@ -787,7 +788,7 @@ adm/timezone/Libya - 664 adm adm 1020313577 1251
 adm/timezone/Mexico_BajaNorte - 664 adm adm 1020313577 1144
 adm/timezone/Mexico_BajaSur - 664 adm adm 1020313577 22
 adm/timezone/Mexico_General - 664 adm adm 1020313577 22
-adm/timezone/NZ - 664 adm adm 1020313577 1417
+adm/timezone/NZ - 664 adm adm 1191217396 1417
 adm/timezone/NZ_CHAT - 664 adm adm 1020313577 20
 adm/timezone/Navajo - 664 adm adm 1020369288 1518
 adm/timezone/PRC - 664 adm adm 1020369288 1516
@@ -5538,7 +5539,7 @@ rc/bin/nroff - 775 sys sys 945617208 27
 rc/bin/patch - 20000000775 sys sys 1155363651 0
 rc/bin/patch/applied - 775 sys sys 1115941614 111
 rc/bin/patch/apply - 775 sys sys 1109511606 1285
-rc/bin/patch/create - 775 sys sys 1141738939 1509
+rc/bin/patch/create - 775 sys sys 1191217103 1516
 rc/bin/patch/diff - 775 sys sys 1109511606 474
 rc/bin/patch/list - 775 sys sys 1115941576 749
 rc/bin/patch/move - 775 sys sys 1112166652 643
@@ -8204,7 +8205,7 @@ sys/src/9/port/devcons.c - 664 sys sys 1176658321 22943
 sys/src/9/port/devdraw.c - 664 sys sys 1147023550 44447
 sys/src/9/port/devdup.c - 664 sys sys 1014931172 2332
 sys/src/9/port/devenv.c - 664 sys sys 1169498893 7015
-sys/src/9/port/devfs.c - 664 sys sys 1191008235 14697
+sys/src/9/port/devfs.c - 664 sys sys 1191279047 14902
 sys/src/9/port/devkbmap.c - 664 sys sys 1130763846 3064
 sys/src/9/port/devkprof.c - 664 sys sys 1014931173 3111
 sys/src/9/port/devloopback.c - 664 sys sys 1138458368 14579
@@ -10102,7 +10103,7 @@ sys/src/cmd/cwfs/lrand.c - 664 sys sys 1171160167 1070
 sys/src/cmd/cwfs/main.c - 664 sys sys 1189033288 10146
 sys/src/cmd/cwfs/malloc.c - 664 sys sys 1174281557 2360
 sys/src/cmd/cwfs/mkfile - 664 sys sys 1174941889 201
-sys/src/cmd/cwfs/mworm.c - 664 sys sys 1174370308 4311
+sys/src/cmd/cwfs/mworm.c - 664 sys sys 1191264337 4542
 sys/src/cmd/cwfs/net.c - 664 sys sys 1174521421 9939
 sys/src/cmd/cwfs/pc.c - 664 sys sys 1174976414 1718
 sys/src/cmd/cwfs/portdat.h - 664 sys sys 1174716952 15241
@@ -14308,7 +14309,7 @@ sys/src/cmd/vc/v.out.h - 664 sys sys 1089299165 2630
 sys/src/cmd/venti - 20000000775 sys sys 1068500904 0
 sys/src/cmd/venti/copy.c - 664 sys sys 1189017556 5275
 sys/src/cmd/venti/devnull.c - 664 sys sys 1177189434 1225
-sys/src/cmd/venti/mkfile - 664 sys sys 1189017596 358
+sys/src/cmd/venti/mkfile - 664 sys sys 1191258765 439
 sys/src/cmd/venti/mkroot.c - 664 sys sys 1177189435 1178
 sys/src/cmd/venti/randtest.c - 664 sys sys 1177189435 5656
 sys/src/cmd/venti/read.c - 664 sys sys 1177189435 1289
@@ -14316,21 +14317,21 @@ sys/src/cmd/venti/readlist.c - 664 sys sys 1177189435 1934
 sys/src/cmd/venti/ro.c - 664 sys sys 1177189435 1886
 sys/src/cmd/venti/root.c - 664 sys sys 1177189435 1329
 sys/src/cmd/venti/srv - 20000000775 sys sys 1189020012 0
-sys/src/cmd/venti/srv/arena.c - 664 sys sys 1178160303 16402
+sys/src/cmd/venti/srv/arena.c - 664 sys sys 1191257299 20137
 sys/src/cmd/venti/srv/arenas.c - 664 sys sys 1178160303 8039
 sys/src/cmd/venti/srv/bloom.c - 664 sys sys 1178160303 4529
 sys/src/cmd/venti/srv/buildbuck.c - 664 sys sys 1177189435 2837
-sys/src/cmd/venti/srv/buildindex.c - 664 sys sys 1182131900 21117
-sys/src/cmd/venti/srv/checkarenas.c - 664 sys sys 1177189435 2313
+sys/src/cmd/venti/srv/buildindex.c - 664 sys sys 1191257306 21645
+sys/src/cmd/venti/srv/checkarenas.c - 664 sys sys 1191257304 2310
 sys/src/cmd/venti/srv/checkindex.c - 664 sys sys 1179342633 5978
-sys/src/cmd/venti/srv/clump.c - 664 sys sys 1177189435 5329
+sys/src/cmd/venti/srv/clump.c - 664 sys sys 1191257297 5316
 sys/src/cmd/venti/srv/clumpstats.c - 664 sys sys 1142736351 2234
 sys/src/cmd/venti/srv/cmparenas.c - 664 sys sys 1177189435 7215
 sys/src/cmd/venti/srv/conf.rc - 775 sys sys 1189304120 1416
-sys/src/cmd/venti/srv/config.c - 664 sys sys 1178160303 5566
-sys/src/cmd/venti/srv/conv.c - 664 sys sys 1178160303 14052
-sys/src/cmd/venti/srv/dat.h - 664 sys sys 1178160303 19018
-sys/src/cmd/venti/srv/dcache.c - 664 sys sys 1190092637 18695
+sys/src/cmd/venti/srv/config.c - 664 sys sys 1191257289 5659
+sys/src/cmd/venti/srv/conv.c - 664 sys sys 1191257291 14072
+sys/src/cmd/venti/srv/dat.h - 664 sys sys 1191257301 19110
+sys/src/cmd/venti/srv/dcache.c - 664 sys sys 1191257300 15583
 sys/src/cmd/venti/srv/disksched.c - 664 sys sys 1142736352 2125
 sys/src/cmd/venti/srv/dump.c - 664 sys sys 1142736352 1642
 sys/src/cmd/venti/srv/findscore.c - 664 sys sys 1179863768 2195
@@ -14339,20 +14340,20 @@ sys/src/cmd/venti/srv/fmtarenas.c - 664 sys sys 1177189436 2702
 sys/src/cmd/venti/srv/fmtbloom.c - 664 sys sys 1142736352 2293
 sys/src/cmd/venti/srv/fmtindex.c - 664 sys sys 1178160304 2599
 sys/src/cmd/venti/srv/fmtisect.c - 664 sys sys 1177189436 1454
-sys/src/cmd/venti/srv/fns.h - 664 sys sys 1178160304 9179
+sys/src/cmd/venti/srv/fns.h - 664 sys sys 1191257308 9208
 sys/src/cmd/venti/srv/graph.c - 664 sys sys 1177189436 4190
-sys/src/cmd/venti/srv/hdisk.c - 664 sys sys 1177189437 16510
-sys/src/cmd/venti/srv/httpd.c - 664 sys sys 1177189437 22784
-sys/src/cmd/venti/srv/icache.c - 664 sys sys 1186114652 8778
-sys/src/cmd/venti/srv/icachewrite.c - 664 sys sys 1190076776 7787
+sys/src/cmd/venti/srv/hdisk.c - 664 sys sys 1191257294 16482
+sys/src/cmd/venti/srv/httpd.c - 664 sys sys 1191257304 22869
+sys/src/cmd/venti/srv/icache.c - 664 sys sys 1191257303 10676
+sys/src/cmd/venti/srv/icachewrite.c - 664 sys sys 1191257310 7528
 sys/src/cmd/venti/srv/ifile.c - 664 sys sys 1189307036 2592
-sys/src/cmd/venti/srv/index.c - 664 sys sys 1178160304 17239
-sys/src/cmd/venti/srv/lump.c - 664 sys sys 1177189437 5799
+sys/src/cmd/venti/srv/index.c - 664 sys sys 1191257302 17933
+sys/src/cmd/venti/srv/lump.c - 664 sys sys 1191257295 4947
 sys/src/cmd/venti/srv/lumpcache.c - 664 sys sys 1177189437 8811
 sys/src/cmd/venti/srv/lumpqueue.c - 664 sys sys 1142736354 2722
-sys/src/cmd/venti/srv/mirrorarenas.c - 664 sys sys 1178160304 10544
+sys/src/cmd/venti/srv/mirrorarenas.c - 664 sys sys 1191257291 10802
 sys/src/cmd/venti/srv/mkfile - 664 sys sys 1189304068 1243
-sys/src/cmd/venti/srv/part.c - 664 sys sys 1180244694 5662
+sys/src/cmd/venti/srv/part.c - 664 sys sys 1191217317 4500
 sys/src/cmd/venti/srv/png.c - 664 sys sys 1142736354 3729
 sys/src/cmd/venti/srv/printarena.c - 664 sys sys 1177189438 2673
 sys/src/cmd/venti/srv/printarenapart.c - 664 sys sys 1178160304 3544
@@ -14365,17 +14366,17 @@ sys/src/cmd/venti/srv/reseal.c - 664 sys sys 1177189438 6682
 sys/src/cmd/venti/srv/round.c - 664 sys sys 1142736355 1577
 sys/src/cmd/venti/srv/score.c - 664 sys sys 1178160305 740
 sys/src/cmd/venti/srv/sortientry.c - 664 sys sys 1177189439 8325
-sys/src/cmd/venti/srv/stats.c - 664 sys sys 1177189439 3912
+sys/src/cmd/venti/srv/stats.c - 664 sys sys 1191257294 4054
 sys/src/cmd/venti/srv/stdinc.h - 664 sys sys 1177189439 169
-sys/src/cmd/venti/srv/syncarena.c - 664 sys sys 1177249927 4834
-sys/src/cmd/venti/srv/syncindex.c - 664 sys sys 1177249927 1622
-sys/src/cmd/venti/srv/syncindex0.c - 664 sys sys 1177249927 4447
+sys/src/cmd/venti/srv/syncarena.c - 664 sys sys 1191257305 4816
+sys/src/cmd/venti/srv/syncindex.c - 664 sys sys 1191257309 1277
+sys/src/cmd/venti/srv/syncindex0.c - 664 sys sys 1191257309 1951
 sys/src/cmd/venti/srv/trace.c - 664 sys sys 1142736356 709
 sys/src/cmd/venti/srv/unittoull.c - 664 sys sys 1142736356 471
 sys/src/cmd/venti/srv/unwhack.c - 664 sys sys 1142736356 3191
 sys/src/cmd/venti/srv/utils.c - 664 sys sys 1178160305 3755
-sys/src/cmd/venti/srv/venti.c - 664 sys sys 1188624404 5882
-sys/src/cmd/venti/srv/verifyarena.c - 664 sys sys 1178160305 5666
+sys/src/cmd/venti/srv/venti.c - 664 sys sys 1191257307 5584
+sys/src/cmd/venti/srv/verifyarena.c - 664 sys sys 1191257292 5776
 sys/src/cmd/venti/srv/whack.c - 664 sys sys 1142736357 6419
 sys/src/cmd/venti/srv/whack.h - 664 sys sys 1142736357 966
 sys/src/cmd/venti/srv/wrarena.c - 664 sys sys 1183678902 4647
@@ -15228,7 +15229,7 @@ sys/src/libhttpd/httpunesc.c - 664 sys sys 1014930784 888
 sys/src/libhttpd/lower.c - 664 sys sys 1014930784 226
 sys/src/libhttpd/mkfile - 664 sys sys 1035389778 421
 sys/src/libhttpd/okheaders.c - 664 sys sys 1014930784 433
-sys/src/libhttpd/parse.c - 664 sys sys 1190835951 19361
+sys/src/libhttpd/parse.c - 664 sys sys 1191217118 19346
 sys/src/libhttpd/parsereq.c - 664 sys sys 1177187800 5112
 sys/src/libhttpd/query.c - 664 sys sys 1014930785 712
 sys/src/libhttpd/redirected.c - 664 sys sys 1014930785 1555

dist/replica/plan9.log

@@ -52921,3 +52921,60 @@
 1191178804 0 c sys/src/cmd/page/cache.c - 664 sys sys 1191178341 3160
 1191178804 1 c sys/src/cmd/page/view.c - 664 sys sys 1190093181 22159
 1191209404 0 c 386/bin/page - 775 sys sys 1191208765 236255
+1191218407 0 c adm/timezone/NZ - 664 adm adm 1191217396 1417
+1191218407 1 c rc/bin/patch/create - 775 sys sys 1191217103 1516
+1191218407 2 c sys/src/cmd/venti/srv/part.c - 664 sys sys 1191217317 4500
+1191218407 3 c sys/src/libhttpd/parse.c - 664 sys sys 1191217118 19346
+1191258004 0 c sys/src/cmd/venti/srv/arena.c - 664 sys sys 1191257299 20137
+1191258004 1 c sys/src/cmd/venti/srv/buildindex.c - 664 sys sys 1191257306 21645
+1191258004 2 c sys/src/cmd/venti/srv/checkarenas.c - 664 sys sys 1191257304 2310
+1191258004 3 c sys/src/cmd/venti/srv/clump.c - 664 sys sys 1191257297 5316
+1191258004 4 c sys/src/cmd/venti/srv/config.c - 664 sys sys 1191257289 5659
+1191258004 5 c sys/src/cmd/venti/srv/conv.c - 664 sys sys 1191257291 14072
+1191258004 6 c sys/src/cmd/venti/srv/dat.h - 664 sys sys 1191257301 19110
+1191258004 7 c sys/src/cmd/venti/srv/dcache.c - 664 sys sys 1191257300 15583
+1191258004 8 c sys/src/cmd/venti/srv/fns.h - 664 sys sys 1191257308 9208
+1191258004 9 c sys/src/cmd/venti/srv/hdisk.c - 664 sys sys 1191257294 16482
+1191258004 10 c sys/src/cmd/venti/srv/httpd.c - 664 sys sys 1191257304 22869
+1191258004 11 c sys/src/cmd/venti/srv/icache.c - 664 sys sys 1191257303 10676
+1191258004 12 c sys/src/cmd/venti/srv/icachewrite.c - 664 sys sys 1191257310 7528
+1191258004 13 c sys/src/cmd/venti/srv/index.c - 664 sys sys 1191257302 17933
+1191258004 14 c sys/src/cmd/venti/srv/lump.c - 664 sys sys 1191257295 4947
+1191258004 15 c sys/src/cmd/venti/srv/mirrorarenas.c - 664 sys sys 1191257291 10802
+1191258004 16 c sys/src/cmd/venti/srv/stats.c - 664 sys sys 1191257294 4054
+1191258004 17 c sys/src/cmd/venti/srv/syncarena.c - 664 sys sys 1191257305 4816
+1191258004 18 c sys/src/cmd/venti/srv/syncindex.c - 664 sys sys 1191257309 1277
+1191258004 19 c sys/src/cmd/venti/srv/syncindex0.c - 664 sys sys 1191257309 1951
+1191258004 20 c sys/src/cmd/venti/srv/venti.c - 664 sys sys 1191257307 5584
+1191258004 21 c sys/src/cmd/venti/srv/verifyarena.c - 664 sys sys 1191257292 5776
+1191259804 0 c sys/src/cmd/venti/mkfile - 664 sys sys 1191258765 439
+1191265205 0 c sys/src/cmd/cwfs/mworm.c - 664 sys sys 1191264337 4542
+1191274203 0 c 386/bin/venti/buildindex - 775 sys sys 1191273668 259251
+1191274203 1 c 386/bin/venti/checkarenas - 775 sys sys 1191273673 261130
+1191274203 2 c 386/bin/venti/checkindex - 775 sys sys 1191273677 256560
+1191274203 3 c 386/bin/venti/clumpstats - 775 sys sys 1191273682 245496
+1191274203 4 c 386/bin/venti/findscore - 775 sys sys 1191273686 247532
+1191274203 5 c 386/bin/venti/fixarenas - 775 sys sys 1191273690 212717
+1191274203 6 c 386/bin/venti/fmtarenas - 775 sys sys 1191273694 247765
+1191274203 7 c 386/bin/venti/fmtbloom - 775 sys sys 1191273697 168531
+1191274203 8 c 386/bin/venti/fmtindex - 775 sys sys 1191273702 245763
+1191274203 9 c 386/bin/venti/fmtisect - 775 sys sys 1191273706 246115
+1191274203 10 c 386/bin/venti/mirrorarenas - 775 sys sys 1191273710 255223
+1191274203 11 c 386/bin/venti/printarena - 775 sys sys 1191273715 257804
+1191274203 12 c 386/bin/venti/rdarena - 775 sys sys 1191273719 247191
+1191274203 13 a 386/bin/venti/ro - 775 sys sys 1191273722 161694
+1191274203 14 c 386/bin/venti/syncindex - 775 sys sys 1191273727 267127
+1191274203 15 c 386/bin/venti/venti - 775 sys sys 1191273735 459037
+1191274203 16 c 386/bin/venti/verifyarena - 775 sys sys 1191273738 166940
+1191274203 17 c 386/bin/venti/wrarena - 775 sys sys 1191273743 284205
+1191274203 18 c 386/bin/ip/httpd/httpd - 775 sys sys 1191273645 295390
+1191274203 19 c 386/bin/ip/httpd/imagemap - 775 sys sys 1191273648 116304
+1191274203 20 c 386/bin/ip/httpd/man2html - 775 sys sys 1191273650 124858
+1191274203 21 c 386/bin/ip/httpd/netlib_find - 775 sys sys 1191273653 117152
+1191274203 22 c 386/bin/ip/httpd/netlib_history - 775 sys sys 1191273655 115944
+1191274203 23 c 386/bin/ip/httpd/save - 775 sys sys 1191273658 132748
+1191274203 24 c 386/bin/ip/httpd/webls - 775 sys sys 1191273660 132813
+1191274203 25 c 386/bin/ip/httpd/wikipost - 775 sys sys 1191273662 114183
+1191274203 26 c 386/lib/libhttpd.a - 664 sys sys 1191273745 100822
+1191279604 0 c sys/src/9/port/devfs.c - 664 sys sys 1191279047 14902
+1191295804 0 c 386/bin/cwfs - 775 sys sys 1191295136 365724

rc/bin/patch/create

@@ -64,9 +64,8 @@ for(i in $*){
 	if not
 		echo warning: new file $i not on sources >[1=2]
 	echo $i $uniq >>$d/files
-	xchmod ug+rw $d/*
-	xchmod a+r $d/*
 }
+@{cd $d && xchmod ug+rw * && xchmod a+r *}
 
 if(~ `{cat /proc/$pid/fd | awk 'NR==2{print $NF}'} */dev/cons && test -w /dev/consctl){
 	>/dev/consctl {

sys/src/9/port/devfs.c

@@ -18,7 +18,7 @@ enum {
 	Fmirror,		/* mirror of others */
 	Fcat,			/* catenation of others */
 	Finter,			/* interleaving of others */
-	Fpart,			/* part of others */
+	Fpart,			/* part of other */
 	Fclear,			/* start over */
 
 	Blksize	= 8*1024,	/* for Finter only */
@@ -81,7 +81,7 @@ static Cmdtab configs[] = {
 	Fcat,	"cat",		0,
 	Finter,	"inter",	0,
 	Fpart,	"part",		5,
-	Fclear,	"clear",	1,	
+	Fclear,	"clear",	1,
 };
 
 static Fsdev*
@@ -117,42 +117,55 @@ setdsize(Fsdev* mp)
 {
 	int	i;
 	long	l;
+	vlong	inlen;
 	uchar	buf[128];	/* old DIRLEN plus a little should be plenty */
 	Dir	d;
 	Inner	*in;
 
 	if (mp->type != Fpart){
-		mp->start= 0;
-		mp->size = 0;
+		mp->start = 0;
+		mp->size  = 0;
 	}
 	for (i = 0; i < mp->ndevs; i++){
 		in = &mp->inner[i];
 		l = devtab[in->idev->type]->stat(in->idev, buf, sizeof buf);
 		convM2D(buf, l, &d, nil);
-		in->isize = d.length;
+		inlen = d.length;
+		in->isize = inlen;
 		switch(mp->type){
+		case Finter:
+			/* truncate to multiple of Blksize */
+			inlen &= ~(Blksize-1);
+			in->isize = inlen;
+			/* fall through */
 		case Fmirror:
-			if (mp->size == 0 || mp->size > d.length)
-				mp->size = d.length;
+			/* use size of smallest inner device */
+			if (mp->size == 0 || mp->size > inlen)
+				mp->size = inlen;
 			break;
 		case Fcat:
-			mp->size += d.length;
-			break;
-		case Finter:
-			/* truncate to multiple of Blksize */
-			d.length &= ~(Blksize-1);
-			in->isize = d.length;
-			mp->size += d.length;
+			mp->size += inlen;
 			break;
 		case Fpart:
 			/* should raise errors here? */
-			if (mp->start > d.length)
-				mp->start = d.length;
-			if (d.length < mp->start + mp->size)
-				mp->size = d.length - mp->start;
+			if (mp->start > inlen) {
+				print("#k/%s: partition start truncated from "
+					"%lld to %lld bytes\n", mp->name,
+					mp->start, inlen);
+				mp->start = inlen;	/* empty partition */
+			}
+			/* truncate partition to keep it within inner device */
+			if (inlen < mp->start + mp->size) {
+				print("#k/%s: partition truncated from "
+					"%lld to %lld bytes\n", mp->name,
+					mp->size, inlen - mp->start);
+				mp->size = inlen - mp->start;
+			}
 			break;
 		}
 	}
+	if(mp->type == Finter)
+		mp->size *= mp->ndevs;
 }
 
 static void
@@ -177,7 +190,7 @@ mpshut(Fsdev *mp)
 
 /*
  * process a single line of configuration,
- * often of the form "name idev0 idev1".
+ * often of the form "cmd newname idev0 idev1".
  */
 static void
 mconfig(char* a, long n)
@@ -271,17 +284,19 @@ mconfig(char* a, long n)
 		mp->size = size;
 	}
 	kstrdup(&mp->name, cb->f[0]);
+	if (waserror()){
+		mpshut(mp);
+		nexterror();
+	}
 	for (i = 1; i < cb->nf; i++){
 		inprv = &mp->inner[i-1];
 		kstrdup(&inprv->iname, cb->f[i]);
 		inprv->idev = namec(inprv->iname, Aopen, ORDWR, 0);
-		if (inprv->idev == nil) {
-			free(mp->name);
-			mp->name = nil;		/* free mp */
+		if (inprv->idev == nil)
 			error(Egreg);
-		}
 		mp->ndevs++;
 	}
+	poperror();
 	setdsize(mp);
 
 	poperror();
@@ -467,31 +482,27 @@ io(Fsdev *mp, Inner *in, int isread, void *a, long l, vlong off)
 			(up && up->errstr? up->errstr: ""));
 		nexterror();
 	}
-	if (isread) {
+	if (isread)
 		wl = devtab[mc->type]->read(mc, a, l, off);
-		if (wl != l)
-			error("#k: short read");
-	} else {
+	else
 		wl = devtab[mc->type]->write(mc, a, l, off);
-		if (wl != l)
-			error("#k: write error");
-	}
 	poperror();
 	return wl;
 }
 
+/* NB: a transfer could span multiple inner devices */
 static long
 catio(Fsdev *mp, int isread, void *a, long n, vlong off)
 {
 	int	i;
-	long	l, wl, res;
+	long	l, res;
 	Inner	*in;
 
 	// print("catio %d %p %ld %lld\n", isread, a, n, off);
 	res = n;
-	for (i = 0; n >= 0 && i < mp->ndevs ; i++){
+	for (i = 0; n > 0 && i < mp->ndevs; i++){
 		in = &mp->inner[i];
-		if (off > in->isize){
+		if (off >= in->isize){
 			off -= in->isize;
 			continue;		/* not there yet */
 		}
@@ -501,8 +512,8 @@ catio(Fsdev *mp, int isread, void *a, long n, vlong off)
 			l = n;
 		// print("\tdev %d %p %ld %lld\n", i, a, l, off);
 
-		wl = io(mp, in, isread, a, l, off);
-		assert(wl == l);
+		if (io(mp, in, isread, a, l, off) != l)
+			error(Eio);
 
 		a = (char*)a + l;
 		off = 0;
@@ -518,7 +529,6 @@ interio(Fsdev *mp, int isread, void *a, long n, vlong off)
 	int	i;
 	long	boff, res, l, wl, wsz;
 	vlong	woff, blk, mblk;
-	Inner	*in;
 
 	blk  = off / Blksize;
 	boff = off % Blksize;
@@ -533,16 +543,15 @@ interio(Fsdev *mp, int isread, void *a, long n, vlong off)
 		else
 			l = n;
 
-		in = &mp->inner[i];
-		wl = io(mp, in, isread, a, l, woff);
-		if (wl != l || l == 0)
+		wl = io(mp, &mp->inner[i], isread, a, l, woff);
+		if (wl != l)
 			error(Eio);
 
-		a = (char*)a + l;
-		n -= l;
 		blk++;
 		boff = 0;
 		wsz = Blksize;
+		a = (char*)a + l;
+		n -= l;
 	}
 	return res;
 }
@@ -553,7 +562,6 @@ mread(Chan *c, void *a, long n, vlong off)
 	int	i, retry;
 	long	l, res;
 	Fsdev	*mp;
-	Inner	*in;
 
 	if (c->qid.type & QTDIR)
 		return devdirread(c, a, n, 0, 0, mgen);
@@ -583,9 +591,7 @@ mread(Chan *c, void *a, long n, vlong off)
 		res = interio(mp, Isread, a, n, off);
 		break;
 	case Fpart:
-		in = &mp->inner[0];
-		res = io(mp, in, Isread, a, n, mp->start + off);
-		assert(res == n);
+		res = io(mp, &mp->inner[0], Isread, a, n, mp->start + off);
 		break;
 	case Fmirror:
 		retry = 0;
@@ -603,8 +609,7 @@ mread(Chan *c, void *a, long n, vlong off)
 			for (i = 0; i < mp->ndevs; i++){
 				if (waserror())
 					continue;
-				in = &mp->inner[i];
-				l = io(mp, in, Isread, a, n, off);
+				l = io(mp, &mp->inner[i], Isread, a, n, off);
 				poperror();
 				if (l >= 0){
 					res = l;
@@ -612,7 +617,7 @@ mread(Chan *c, void *a, long n, vlong off)
 				}
 			}
 		} while (i == mp->ndevs && ++retry <= Maxretries);
-		if (i == mp->ndevs) {
+		if (retry > Maxretries) {
 			/* no mirror had a good copy of the block */
 			print("#k/%s: byte %,lld count %ld: CAN'T READ "
 				"from mirror: %s\n", mp->name, off, n,
@@ -633,7 +638,6 @@ mwrite(Chan *c, void *a, long n, vlong off)
 	int	i, allbad, anybad, retry;
 	long	l, res;
 	Fsdev	*mp;
-	Inner	*in;
 
 	if (c->qid.type & QTDIR)
 		error(Eperm);
@@ -658,10 +662,9 @@ mwrite(Chan *c, void *a, long n, vlong off)
 		res = interio(mp, Iswrite, a, n, off);
 		break;
 	case Fpart:
-		in = &mp->inner[0];
-		res = io(mp, in, Iswrite, a, n, mp->start + off);
-		if (res > n)
-			res = n;
+		res = io(mp, &mp->inner[0], Iswrite, a, n, mp->start + off);
+		if (res != n)
+			error(Eio);
 		break;
 	case Fmirror:
 		retry = 0;
@@ -683,12 +686,9 @@ mwrite(Chan *c, void *a, long n, vlong off)
 					anybad = 1;
 					continue;
 				}
-				in = &mp->inner[i];
-				l = io(mp, in, Iswrite, a, n, off);
+				l = io(mp, &mp->inner[i], Iswrite, a, n, off);
 				poperror();
-				if (res > l)
-					res = l;	/* shortest OK write */
-				if (l == n)
+				if (l == n) 
 					allbad = 0;	/* wrote a good copy */
 				else
 					anybad = 1;

sys/src/cmd/cwfs/mworm.c

@@ -57,7 +57,7 @@ mcatread(Device *d, Off b, void *c)
 			return devread(x, b-l, c);
 		l += m;
 	}
-	print("mcatread %Z block %lld beyond end %lld\n",
+	print("mcatread past end: %Z block %lld, %lld beyond end\n",
 		d, (Wideoff)b, (Wideoff)l);
 	return 1;
 }
@@ -79,7 +79,7 @@ mcatwrite(Device *d, Off b, void *c)
 			return devwrite(x, b-l, c);
 		l += m;
 	}
-	print("mcatwrite %Z block %lld beyond end %lld\n",
+	print("mcatwrite past end: %Z block %lld, %lld beyond end\n",
 		d, (Wideoff)b, (Wideoff)l);
 	return 1;
 }
@@ -176,7 +176,8 @@ partread(Device *d, Off b, void *c)
 		size = l*100;
 	if(b < size)
 		return devread(d->part.d, base+b, c);
-	print("partread %lld %lld\n", (Wideoff)b, (Wideoff)size);
+	print("partread past end: %Z blk %lld size %lld\n",
+		d, (Wideoff)b, (Wideoff)size);
 	return 1;
 }
 
@@ -192,7 +193,8 @@ partwrite(Device *d, Off b, void *c)
 		size = l*100;
 	if(b < size)
 		return devwrite(d->part.d, base+b, c);
-	print("partwrite %lld %lld\n", (Wideoff)b, (Wideoff)size);
+	print("partwrite past end: %Z blk %lld size %lld\n",
+		d, (Wideoff)b, (Wideoff)size);
 	return 1;
 }
 
@@ -236,14 +238,19 @@ mirrread(Device *d, Off b, void *c)
 {
 	Device *x;
 
+	if (d->cat.first == nil) {
+		print("mirrread: empty mirror %Z\n", d);
+		return 1;
+	}
 	for(x=d->cat.first; x; x=x->link) {
 		if(x->size == 0)
 			x->size = devsize(x);
 		if (devread(x, b, c) == 0)	/* okay? */
 			return 0;
 	}
-	// DANGER WILL ROBINSON - all copies of this block were bad
-	print("mirrread %Z error at block %lld\n", d, (Wideoff)b);
+	// DANGER WILL ROBINSON
+	print("mirrread: all mirrors of %Z block %lld are bad\n",
+		d, (Wideoff)b);
 	return 1;
 }
 
@@ -261,7 +268,7 @@ ewrite(Device *x, Off b, void *c)
 	if(x->size == 0)
 		x->size = devsize(x);
 	if (devwrite(x, b, c) != 0) {
-		print("mirrwrite %Z error at block %lld\n", x, (Wideoff)b);
+		print("mirrwrite: error at %Z block %lld\n", x, (Wideoff)b);
 		return 1;
 	}
 	return 0;
@@ -281,5 +288,9 @@ wrmirrs1st(Device *x, Off b, void *c)	// write any mirrors of x, then x
 int
 mirrwrite(Device *d, Off b, void *c)
 {
+	if (d->cat.first == nil) {
+		print("mirrwrite: empty mirror %Z\n", d);
+		return 1;
+	}
 	return wrmirrs1st(d->cat.first, b, c);
 }

sys/src/cmd/venti/mkfile

@@ -2,6 +2,7 @@
 
 TARG=\
 	read\
+	ro\
 	sync\
 	write\
 
@@ -18,6 +19,8 @@ extra:V: $O.devnull $O.mkroot $O.randtest $O.readlist $O.ro $O.root
 all:V:		srv.all.dir
 install:V:	srv.install.dir
 installall:V:	srv.installall.dir
+safeinstall:V:	srv.safeinstall.dir
+safeinstallall:V: srv.safeinstallall.dir
 clean:V:	srv.clean.dir
 nuke:V:		srv.nuke.dir
 

sys/src/cmd/venti/srv/arena.c

@@ -16,6 +16,7 @@ static int	loadarena(Arena *arena);
 static CIBlock	*getcib(Arena *arena, int clump, int writing, CIBlock *rock);
 static void	putcib(Arena *arena, CIBlock *cib);
 static void	sumproc(void *);
+static void loadcig(Arena *arena);
 
 static QLock	sumlock;
 static Rendez	sumwait;
@@ -65,7 +66,7 @@ initarena(Part *part, u64int base, u64int size, u32int blocksize)
 	}
 
 	if(arena->diskstats.sealed && scorecmp(zeroscore, arena->score)==0)
-		backsumarena(arena);
+		sealarena(arena);
 
 	return arena;
 }
@@ -137,14 +138,23 @@ readclumpinfos(Arena *arena, int clump, ClumpInfo *cis, int n)
 	CIBlock *cib, r;
 	int i;
 
-	for(i = 0; i < n; i++){
+	/*
+	 * because the clump blocks are laid out
+	 * in reverse order at the end of the arena,
+	 * it can be a few percent faster to read
+	 * the clumps backwards, which reads the
+	 * disk blocks forwards.
+	 */
+	for(i = n-1; i >= 0; i--){
 		cib = getcib(arena, clump + i, 0, &r);
-		if(cib == nil)
-			break;
+		if(cib == nil){
+			n = i;
+			continue;
+		}
 		unpackclumpinfo(&cis[i], &cib->data->data[cib->offset]);
 		putcib(arena, cib);
 	}
-	return i;
+	return n;
 }
 
 /*
@@ -283,13 +293,12 @@ ZZZ question: should this distinguish between an arena
 filling up and real errors writing the clump?
  */
 u64int
-writeaclump(Arena *arena, Clump *c, u8int *clbuf, u64int start, u64int *pa)
+writeaclump(Arena *arena, Clump *c, u8int *clbuf)
 {
 	DBlock *b;
 	u64int a, aa;
 	u32int clump, n, nn, m, off, blocksize;
 	int ok;
-	AState as;
 
 	n = c->info.size + ClumpSize + U32Size;
 	qlock(&arena->lock);
@@ -299,10 +308,6 @@ writeaclump(Arena *arena, Clump *c, u8int *clbuf, u64int start, u64int *pa)
 		if(!arena->memstats.sealed){
 			logerr(EOk, "seal memstats %s", arena->name);
 			arena->memstats.sealed = 1;
-			as.arena = arena;
-			as.aa = start+aa;
-			as.stats = arena->memstats;
-			setdcachestate(&as);
 		}
 		qunlock(&arena->lock);
 		return TWID64;
@@ -349,7 +354,28 @@ writeaclump(Arena *arena, Clump *c, u8int *clbuf, u64int start, u64int *pa)
 	if(c->info.size < c->info.uncsize)
 		arena->memstats.cclumps++;
 
-	clump = arena->memstats.clumps++;
+	clump = arena->memstats.clumps;
+	if(clump % ArenaCIGSize == 0){
+		if(arena->cig == nil){
+			loadcig(arena);
+			if(arena->cig == nil)
+				goto NoCIG;
+		}
+		/* add aa as start of next cig */
+		if(clump/ArenaCIGSize != arena->ncig){
+			fprint(2, "bad arena cig computation %s: writing clump %d but %d cigs\n",
+				arena->name, clump, arena->ncig);
+			arena->ncig = -1;
+			vtfree(arena->cig);
+			arena->cig = nil;
+			goto NoCIG;
+		}
+		arena->cig = vtrealloc(arena->cig, (arena->ncig+1)*sizeof arena->cig[0]);
+		arena->cig[arena->ncig++].offset = aa;
+	}
+NoCIG:
+	arena->memstats.clumps++;
+
 	if(arena->memstats.clumps == 0)
 		sysfatal("clumps wrapped");
 	arena->wtime = now();
@@ -359,14 +385,6 @@ writeaclump(Arena *arena, Clump *c, u8int *clbuf, u64int start, u64int *pa)
 	writeclumpinfo(arena, clump, &c->info);
 	wbarena(arena);
 
-	/* set up for call to setdcachestate */
-	as.arena = arena;
-	as.aa = start+arena->memstats.used;
-	as.stats = arena->memstats;
-
-	/* update this before calling setdcachestate so it cannot be behind dcache.diskstate */
-	*pa = start+aa;
-	setdcachestate(&as);
 	qunlock(&arena->lock);
 
 	return aa;
@@ -415,6 +433,7 @@ setatailstate(AState *as)
 	/*
 	 * Look up as->arena to find index.
 	 */
+	needmainindex();	/* OS X linker */
 	ix = mainindex;
 	for(i=0; i<ix->narenas; i++)
 		if(ix->arenas[i] == as->arena)
@@ -515,6 +534,7 @@ sumarena(Arena *arena)
 	/*
 	 * read & sum all blocks except the last one
 	 */
+	flushdcache();
 	memset(&s, 0, sizeof s);
 	b = alloczblock(bs, 0, arena->part->blocksize);
 	e = arena->base + arena->size;
@@ -550,24 +570,19 @@ ReadErr:
 	sha1(b->data, bs-VtScoreSize, nil, &s);
 	sha1(zeroscore, VtScoreSize, nil, &s);
 	sha1(nil, 0, score, &s);
-
+	
 	/*
 	 * check for no checksum or the same
-	 *
-	 * the writepart is okay because we flushed the dcache in sealarena
 	 */
-	if(scorecmp(score, &b->data[bs - VtScoreSize]) != 0){
-		if(scorecmp(zeroscore, &b->data[bs - VtScoreSize]) != 0)
-			logerr(EOk, "overwriting mismatched checksums for arena=%s, found=%V calculated=%V",
-				arena->name, &b->data[bs - VtScoreSize], score);
-		scorecp(&b->data[bs - VtScoreSize], score);
-		if(writepart(arena->part, e, b->data, bs) < 0)
-			logerr(EOk, "sumarena can't write sum for %s: %r", arena->name);
-	}
+	if(scorecmp(score, &b->data[bs - VtScoreSize]) != 0
+	&& scorecmp(zeroscore, &b->data[bs - VtScoreSize]) != 0)
+		logerr(EOk, "overwriting mismatched checksums for arena=%s, found=%V calculated=%V",
+			arena->name, &b->data[bs - VtScoreSize], score);
 	freezblock(b);
 
 	qlock(&arena->lock);
 	scorecp(arena->score, score);
+	wbarena(arena);
 	qunlock(&arena->lock);
 }
 
@@ -586,6 +601,7 @@ wbarena(Arena *arena)
 	}
 	dirtydblock(b, DirtyArenaTrailer);
 	bad = okarena(arena)<0 || packarena(arena, b->data)<0;
+	scorecp(b->data + arena->blocksize - VtScoreSize, arena->score);
 	putdblock(b);
 	if(bad)
 		return -1;
@@ -754,3 +770,157 @@ putcib(Arena *arena, CIBlock *cib)
 	putdblock(cib->data);
 	cib->data = nil;
 }
+
+
+/*
+ * For index entry readahead purposes, the arenas are 
+ * broken into smaller subpieces, called clump info groups
+ * or cigs.  Each cig has ArenaCIGSize clumps (ArenaCIGSize
+ * is chosen to make the index entries take up about half
+ * a megabyte).  The index entries do not contain enough
+ * information to determine what the clump index is for
+ * a given address in an arena.  That info is needed both for
+ * figuring out which clump group an address belongs to 
+ * and for prefetching a clump group's index entries from
+ * the arena table of contents.  The first time clump groups
+ * are accessed, we scan the entire arena table of contents
+ * (which might be 10s of megabytes), recording the data 
+ * offset of each clump group.
+ */
+
+/* 
+ * load clump info group information by scanning entire toc.
+ */
+static void
+loadcig(Arena *arena)
+{
+	u32int i, j, ncig, nci;
+	ArenaCIG *cig;
+	ClumpInfo *ci;
+	u64int offset;
+	int ms;
+
+	if(arena->cig || arena->ncig < 0)
+		return;
+
+//	fprint(2, "loadcig %s\n", arena->name);
+	
+	ncig = (arena->memstats.clumps+ArenaCIGSize-1) / ArenaCIGSize;
+	if(ncig == 0){
+		arena->cig = vtmalloc(1);
+		arena->ncig = 0;
+		return;
+	}
+
+	ms = msec();
+	cig = vtmalloc(ncig*sizeof cig[0]);
+	ci = vtmalloc(ArenaCIGSize*sizeof ci[0]);
+	offset = 0;
+	for(i=0; i<ncig; i++){
+		nci = readclumpinfos(arena, i*ArenaCIGSize, ci, ArenaCIGSize);
+		cig[i].offset = offset;
+		for(j=0; j<nci; j++)
+			offset += ClumpSize + ci[j].size;
+		if(nci < ArenaCIGSize){
+			if(i != ncig-1){
+				vtfree(ci);
+				vtfree(cig);
+				arena->ncig = -1;
+				fprint(2, "loadcig %s: got %ud cigs, expected %ud\n", arena->name, i+1, ncig);
+				goto out;
+			}
+		}
+	}
+	vtfree(ci);
+	
+	arena->ncig = ncig;
+	arena->cig = cig;
+
+out:
+	ms = msec() - ms;
+	addstat2(StatCigLoad, 1, StatCigLoadTime, ms);
+}
+
+/*
+ * convert arena address into arena group + data boundaries.
+ */
+int
+arenatog(Arena *arena, u64int addr, u64int *gstart, u64int *glimit, int *g)
+{
+	int r, l, m;
+
+	qlock(&arena->lock);
+	if(arena->cig == nil)
+		loadcig(arena);
+	if(arena->cig == nil || arena->ncig == 0){
+		qunlock(&arena->lock);
+		return -1;
+	}
+
+	l = 1;
+	r = arena->ncig - 1;
+	while(l <= r){
+		m = (r + l) / 2;
+		if(arena->cig[m].offset <= addr)
+			l = m + 1;
+		else
+			r = m - 1;
+	}
+	l--;
+
+	*g = l;
+	*gstart = arena->cig[l].offset;
+	if(l+1 < arena->ncig)
+		*glimit = arena->cig[l+1].offset;
+	else
+		*glimit = arena->memstats.used;
+	qunlock(&arena->lock);
+	return 0;
+}
+
+/*
+ * load the clump info for group g into the index entries.
+ */
+int
+asumload(Arena *arena, int g, IEntry *entries, int nentries)
+{
+	int i, base, limit;
+	u64int addr;
+	ClumpInfo ci;
+	IEntry *ie;
+
+	if(nentries < ArenaCIGSize){
+		fprint(2, "asking for too few entries\n");
+		return -1;
+	}
+	
+	qlock(&arena->lock);
+	if(arena->cig == nil)
+		loadcig(arena);
+	if(arena->cig == nil || arena->ncig == 0 || g >= arena->ncig){
+		qunlock(&arena->lock);
+		return -1;
+	}
+	
+	addr = 0;
+	base = g*ArenaCIGSize;
+	limit = base + ArenaCIGSize;
+	if(base > arena->memstats.clumps)
+		base = arena->memstats.clumps;
+	ie = entries;
+	for(i=base; i<limit; i++){
+		if(readclumpinfo(arena, i, &ci) < 0)
+			break;
+		if(ci.type != VtCorruptType){
+			scorecp(ie->score, ci.score);
+			ie->ia.type = ci.type;
+			ie->ia.size = ci.uncsize;
+			ie->ia.blocks = (ci.size + ClumpSize + (1<<ABlockLog) - 1) >> ABlockLog;
+			ie->ia.addr = addr;
+			ie++;
+		}
+		addr += ClumpSize + ci.size;
+	}
+	qunlock(&arena->lock);
+	return ie - entries;
+}

sys/src/cmd/venti/srv/buildindex.c

@@ -36,18 +36,19 @@ static void	arenapartproc(void*);
 void
 usage(void)
 {
-	fprint(2, "usage: buildindex [-bd] [-i isect]... [-M imem] venti.conf\n");
+	fprint(2, "usage: buildindex [-b] [-i isect]... [-M imem] venti.conf\n");
 	threadexitsall("usage");
 }
 
 void
 threadmain(int argc, char *argv[])
 {
-	int fd, i, napart;
+	int fd, i, napart, nfinish, maxdisks;
 	u32int bcmem, imem;
 	Config conf;
 	Part *p;
 	
+	maxdisks = 100000;
 	ventifmtinstall();
 	imem = 256*1024*1024;
 	ARGBEGIN{
@@ -64,6 +65,9 @@ threadmain(int argc, char *argv[])
 	case 'M':
 		imem = unittoull(EARGF(usage()));
 		break;
+	case 'm':	/* temporary - might go away */
+		maxdisks = atoi(EARGF(usage()));
+		break;
 	default:
 		usage();
 		break;
@@ -132,17 +136,21 @@ threadmain(int argc, char *argv[])
 	/* start arena procs */
 	p = nil;
 	napart = 0;
+	nfinish = 0;
 	arenadonechan = chancreate(sizeof(void*), 0);
 	for(i=0; i<ix->narenas; i++){
 		if(ix->arenas[i]->part != p){
 			p = ix->arenas[i]->part;
 			vtproc(arenapartproc, p);
-			napart++;
+			if(++napart >= maxdisks){
+				recvp(arenadonechan);
+				nfinish++;
+			}
 		}
 	}
 
 	/* wait for arena procs to finish */
-	for(i=0; i<napart; i++)
+	for(nfinish=0; nfinish<napart; nfinish++)
 		recvp(arenadonechan);
 
 	/* tell index procs to finish */
@@ -222,22 +230,28 @@ arenapartproc(void *v)
 		if(a->memstats.clumps)
 			fprint(2, "%T arena %s: %d entries\n", 
 				a->name, a->memstats.clumps);
-		addr = ix->amap[i].start;
-		for(clump=0; clump<a->memstats.clumps; clump+=n){
+		/*
+		 * Running the loop backwards accesses the 
+		 * clump info blocks forwards, since they are
+		 * stored in reverse order at the end of the arena.
+		 * This speeds things slightly.
+		 */
+		addr = ix->amap[i].start + a->memstats.used;
+		for(clump=a->memstats.clumps; clump > 0; clump-=n){
 			n = ClumpChunks;
-			if(n > a->memstats.clumps - clump)
-				n = a->memstats.clumps - clump;
-			if(readclumpinfos(a, clump, cis, n) != n){
+			if(n > clump)
+				n = clump;
+			if(readclumpinfos(a, clump-n, cis, n) != n){
 				fprint(2, "%T arena %s: directory read: %r\n", a->name);
 				errors = 1;
 				break;
 			}
-			for(j=0; j<n; j++){
+			for(j=n-1; j>=0; j--){
 				ci = &cis[j];
 				ie.ia.type = ci->type;
 				ie.ia.size = ci->uncsize;
+				addr -= ci->size + ClumpSize;
 				ie.ia.addr = addr;
-				addr += ci->size + ClumpSize;
 				ie.ia.blocks = (ci->size + ClumpSize + (1<<ABlockLog)-1) >> ABlockLog;
 				scorecp(ie.score, ci->score);
 				if(ci->type == VtCorruptType)
@@ -253,6 +267,8 @@ arenapartproc(void *v)
 				}
 			}
 		}
+		if(addr != ix->amap[i].start)
+			fprint(2, "%T arena %s: clump miscalculation %lld != %lld\n", a->name, addr, ix->amap[i].start);
 	}
 	add(&arenaentries, tot);
 	add(&skipentries, nskip);

sys/src/cmd/venti/srv/checkarenas.c

@@ -24,7 +24,7 @@ checkarena(Arena *arena, int scan, int fix)
 
 	err = 0;
 	for(;;){
-		e = syncarena(arena, 0, 1000, 0, fix);
+		e = syncarena(arena, 1000, 0, fix);
 		err |= e;
 		if(!(e & SyncHeader))
 			break;

sys/src/cmd/venti/srv/clump.c

@@ -62,19 +62,17 @@ storeclump(Index *ix, ZBlock *zb, u8int *sc, int type, u32int creator, IAddr *ia
 	memset(cb->data+ClumpSize+dsize, 0, 4);
 	cl.info.size = dsize;
 
-	ia->addr = 0;
-	ia->type = type;
-	ia->size = size;
-	ia->blocks = (dsize + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog;
-
-	a = writeiclump(ix, &cl, cb->data, &ia->addr);
-
+	a = writeiclump(ix, &cl, cb->data);
 	trace(TraceLump, "storeclump exit %lld", a);
-
 	freezblock(cb);
 	if(a == TWID64)
 		return -1;
 
+	ia->addr = a;
+	ia->type = type;
+	ia->size = size;
+	ia->blocks = (dsize + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog;
+
 /*
 	qlock(&stats.lock);
 	stats.clumpwrites++;

sys/src/cmd/venti/srv/config.c

@@ -245,3 +245,9 @@ configbloom(char *file)
 	return b;
 }
 
+/* for OS X linker, which only resolves functions, not data */
+void
+needmainindex(void)
+{
+}
+

sys/src/cmd/venti/srv/conv.c

@@ -581,9 +581,9 @@ unpackientry(IEntry *ie, u8int *buf)
 
 	scorecp(ie->score, p);
 	p += VtScoreSize;
-	ie->wtime = U32GET(p);
+	/* ie->wtime = U32GET(p); */
 	p += U32Size;
-	ie->train = U16GET(p);
+	/* ie->train = U16GET(p); */
 	p += U16Size;
 	if(p - buf != IEntryAddrOff)
 		sysfatal("unpackentry bad IEntryAddrOff amount");
@@ -613,9 +613,9 @@ packientry(IEntry *ie, u8int *buf)
 
 	scorecp(p, ie->score);
 	p += VtScoreSize;
-	U32PUT(p, ie->wtime);
+	U32PUT(p, 0); /* wtime */
 	p += U32Size;
-	U16PUT(p, ie->train);
+	U16PUT(p, 0); /* train */
 	p += U16Size;
 	U64PUT(p, ie->ia.addr, t32);
 	p += U64Size;

sys/src/cmd/venti/srv/dat.h

@@ -3,6 +3,7 @@ typedef struct AMap		AMap;
 typedef struct AMapN		AMapN;
 typedef struct Arena		Arena;
 typedef struct AState	AState;
+typedef struct ArenaCIG	ArenaCIG;
 typedef struct ArenaHead	ArenaHead;
 typedef struct ArenaPart	ArenaPart;
 typedef struct ArenaTail	ArenaTail;
@@ -14,7 +15,6 @@ typedef struct Graph Graph;
 typedef struct IAddr		IAddr;
 typedef struct IBucket		IBucket;
 typedef struct IEStream		IEStream;
-#pragma incomplete IEStream
 typedef struct IEntry		IEntry;
 typedef struct IFile		IFile;
 typedef struct ISect		ISect;
@@ -29,8 +29,10 @@ typedef struct ZBlock		ZBlock;
 typedef struct Round	Round;
 typedef struct Bloom	Bloom;
 
-#define TWID32	((u32int)~(u32int)0)
-#define TWID64	((u64int)~(u64int)0)
+#pragma incomplete IEStream
+
+#define	TWID32	((u32int)~(u32int)0)
+#define	TWID64	((u64int)~(u64int)0)
 #define	TWID8	((u8int)~(u8int)0)
 
 enum
@@ -45,7 +47,6 @@ enum
 	IndexBase		= 1024*1024,	/* initial address to use in an index */
 	MaxIo			= 64*1024,	/* max size of a single read or write operation */
 	ICacheBits		= 16,		/* default bits for indexing icache */
-	ICacheDepth		= 4,		/* default depth of an icache hash chain */
 	MaxAMap			= 2*1024,	/* max. allowed arenas in an address mapping; must be < 32*1024 */
 
 	/*
@@ -148,6 +149,8 @@ enum
 	DirtyArenaCib,
 	DirtyArenaTrailer,
 	DirtyMax,
+	
+	ArenaCIGSize = 10*1024,	// about 0.5 MB worth of IEntry.
 
 	VentiZZZZZZZZ
 };
@@ -372,6 +375,14 @@ struct Arena
 	u32int		ctime;			/* first time a block was written */
 	u32int		wtime;			/* last time a block was written */
 	u32int		clumpmagic;
+	
+	ArenaCIG	*cig;
+	int	ncig;
+};
+
+struct ArenaCIG
+{
+	u64int	offset;  // from arena base
 };
 
 /*
@@ -455,6 +466,8 @@ struct Index
 	AMap		*smap;			/* mapping of buckets to index sections */
 	int		narenas;
 	AMap		*amap;			/* mapping from index addesses to arenas */
+	
+	QLock	writing;
 };
 
 /*
@@ -506,14 +519,20 @@ struct IAddr
  */
 struct IEntry
 {
-	u8int		score[VtScoreSize];
-	IEntry		*next;			/* next in hash chain */
-	IEntry		*nextdirty; 		/* next in dirty chain */
-	u32int		wtime;			/* last write time */
-	u16int		train;			/* relative train containing the most recent ref; 0 if no ref, 1 if in same car */
-	u8int		rac;			/* read ahead count */
-	u8int		dirty;		/* is dirty */
-	IAddr		ia;
+	/* on disk data - 32 bytes*/
+	u8int	score[VtScoreSize];
+	IAddr	ia;
+	
+	IEntry	*nexthash;
+	IEntry	*nextdirty;
+	IEntry	*next;
+	IEntry	*prev;
+	u8int	state;
+};
+enum {
+	IEClean = 0,
+	IEDirty = 1,
+	IESummary = 2,
 };
 
 /*
@@ -608,6 +627,9 @@ enum
 	StatIcacheFlush,
 	StatIcacheStall,
 	StatIcacheReadTime,
+	StatIcacheLookup,
+	StatScacheHit,
+	StatScachePrefetch,
 
 	StatBloomHit,
 	StatBloomMiss,
@@ -629,6 +651,9 @@ enum
 
 	StatSumRead,
 	StatSumReadBytes,
+	
+	StatCigLoad,
+	StatCigLoadTime,
 
 	NStat
 };
@@ -729,3 +754,4 @@ extern	ulong	stattime;
 #pragma varargck type "V" uchar*
 #define ODIRECT 0
 #endif
+

sys/src/cmd/venti/srv/dcache.c

@@ -55,15 +55,6 @@ struct DCache
 	u8int		*mem;			/* memory for all block descriptors */
 	int		ndirty;			/* number of dirty blocks */
 	int		maxdirty;		/* max. number of dirty blocks */
-	Channel	*ra;
-	u8int		*rabuf;
-	u32int		ramax;
-	u32int		rasize;
-	u64int		raaddr;
-	Part		*rapart;
-
-	AState	diskstate;
-	AState	state;
 };
 
 typedef struct Ra Ra;
@@ -82,7 +73,6 @@ static void	delheap(DBlock *db);
 static void	fixheap(int i, DBlock *b);
 static void	flushproc(void*);
 static void	writeproc(void*);
-static void raproc(void*);
 
 void
 initdcache(u32int mem)
@@ -109,7 +99,6 @@ initdcache(u32int mem)
 	dcache.blocks = MKNZ(DBlock, nblocks);
 	dcache.write = MKNZ(DBlock*, nblocks);
 	dcache.mem = MKNZ(u8int, (nblocks+1+128) * blocksize);
-	dcache.ra = chancreate(sizeof(Ra), 0);
 
 	last = nil;
 	p = (u8int*)(((ulong)dcache.mem+blocksize-1)&~(ulong)(blocksize-1));
@@ -121,10 +110,6 @@ initdcache(u32int mem)
 		b->next = last;
 		last = b;
 	}
-	dcache.rabuf = &p[i*blocksize];
-	dcache.ramax = 128*blocksize;
-	dcache.raaddr = 0;
-	dcache.rapart = nil;
 
 	dcache.free = last;
 	dcache.nheap = 0;
@@ -133,136 +118,6 @@ initdcache(u32int mem)
 
 	vtproc(flushproc, nil);
 	vtproc(delaykickroundproc, &dcache.round);
-	vtproc(raproc, nil);
-}
-
-void
-setdcachestate(AState *a)
-{
-	trace(TraceBlock, "setdcachestate %s 0x%llux clumps %d", a->arena ? a->arena->name : nil, a->aa, a->stats.clumps);
-	qlock(&dcache.lock);
-	dcache.state = *a;
-	qunlock(&dcache.lock);
-}
-
-AState
-diskstate(void)
-{
-	AState a;
-
-	qlock(&dcache.lock);
-	a = dcache.diskstate;
-	qunlock(&dcache.lock);
-	return a;
-}
-
-static void
-raproc(void *v)
-{
-	Ra ra;
-	DBlock *b;
-
-	USED(v);
-	while(recv(dcache.ra, &ra) == 1){
-		if(ra.part->size <= ra.addr)
-			continue;
-		b = _getdblock(ra.part, ra.addr, OREAD, 2);
-		putdblock(b);
-	}
-}
-
-/*
- * We do readahead a whole arena at a time now,
- * so dreadahead is a no-op.  The original implementation
- * is in unused_dreadahead below.
- */
-void
-dreadahead(Part *part, u64int addr, int miss)
-{
-	USED(part);
-	USED(addr);
-	USED(miss);
-}
-
-void
-unused_dreadahead(Part *part, u64int addr, int miss)
-{
-	Ra ra;
-	static struct {
-		Part *part;
-		u64int addr;
-	} lastmiss;
-	static struct {
-		Part *part;
-		u64int addr;
-		int dir;
-	} lastra;
-
-	if(miss){
-		if(lastmiss.part==part && lastmiss.addr==addr-dcache.size){
-		XRa:
-			lastra.part = part;
-			lastra.dir = addr-lastmiss.addr;
-			lastra.addr = addr+lastra.dir;
-			ra.part = part;
-			ra.addr = lastra.addr;
-			nbsend(dcache.ra, &ra);
-		}else if(lastmiss.part==part && lastmiss.addr==addr+dcache.size){
-			addr -= dcache.size;
-			goto XRa;
-		}
-	}else{
-		if(lastra.part==part && lastra.addr==addr){
-			lastra.addr += lastra.dir;
-			ra.part = part;
-			ra.addr = lastra.addr;
-			nbsend(dcache.ra, &ra);
-		}
-	}
-
-	if(miss){
-		lastmiss.part = part;
-		lastmiss.addr = addr;
-	}
-}
-
-int
-rareadpart(Part *part, u64int addr, u8int *buf, uint n, int load)
-{
-	uint nn;
-	static RWLock ralock;
-
-	rlock(&ralock);
-	if(dcache.rapart==part && dcache.raaddr <= addr && addr+n <= dcache.raaddr+dcache.rasize){
-		memmove(buf, dcache.rabuf+(addr-dcache.raaddr), n);
-		runlock(&ralock);
-		return 0;
-	}
-	if(load != 2 || addr >= part->size){	/* addr >= part->size: let readpart do the error */	
-		runlock(&ralock);
-		diskaccess(0);
-		return readpart(part, addr, buf, n);
-	}
-
-	runlock(&ralock);
-	wlock(&ralock);
-fprint(2, "raread %s %llx\n", part->name, addr);
-	nn = dcache.ramax;
-	if(addr+nn > part->size)
-		nn = part->size - addr;
-	diskaccess(0);
-	if(readpart(part, addr, dcache.rabuf, nn) < 0){
-		wunlock(&ralock);
-		return -1;
-	}
-	memmove(buf, dcache.rabuf, n);	
-	dcache.rapart = part;
-	dcache.rasize = nn;
-	dcache.raaddr = addr;
-	wunlock(&ralock);
-
-	addstat(StatApartReadBytes, nn-n);
-	return 0;
 }
 
 static u32int
@@ -313,16 +168,8 @@ _getdblock(Part *part, u64int addr, int mode, int load)
 again:
 	for(b = dcache.heads[h]; b != nil; b = b->next){
 		if(b->part == part && b->addr == addr){
-			/*
-			qlock(&stats.lock);
-			stats.pchit++;
-			qunlock(&stats.lock);
-			*/
-			if(load){
+			if(load)
 				addstat(StatDcacheHit, 1);
-				if(load != 2 && mode != OWRITE)
-					dreadahead(part, b->addr, 0);
-			}
 			goto found;
 		}
 	}
@@ -367,8 +214,6 @@ ZZZ this is not reasonable
 	b->addr = addr;
 	b->part = part;
 	b->size = 0;
-	if(load != 2 && mode != OWRITE)
-		dreadahead(part, b->addr, 1);
 
 found:
 	b->ref++;
@@ -405,7 +250,8 @@ found:
 				memset(&b->data[b->size], 0, size - b->size);
 			else{
 				trace(TraceBlock, "getdblock readpart %s 0x%llux", part->name, addr);
-				if(rareadpart(part, addr + b->size, &b->data[b->size], size - b->size, load) < 0){
+				diskaccess(0);
+				if(readpart(part, addr + b->size, &b->data[b->size], size - b->size) < 0){
 					b->mode = ORDWR;	/* so putdblock wunlocks */
 					putdblock(b);
 					return nil;
@@ -768,7 +614,6 @@ flushproc(void *v)
 	int i, j, n;
 	ulong t0;
 	DBlock *b, **write;
-	AState as;
 
 	USED(v);
 	threadsetname("flushproc");
@@ -779,10 +624,6 @@ flushproc(void *v)
 		t0 = nsec()/1000;
 		trace(TraceProc, "build t=%lud", (ulong)(nsec()/1000)-t0);
 
-		qlock(&dcache.lock);
-		as = dcache.state;
-		qunlock(&dcache.lock);
-
 		write = dcache.write;
 		n = 0;
 		for(i=0; i<dcache.nblocks; i++){
@@ -819,7 +660,6 @@ flushproc(void *v)
 		 */
 		trace(TraceProc, "undirty.%d t=%lud", j, (ulong)(nsec()/1000)-t0);
 		qlock(&dcache.lock);
-		dcache.diskstate = as;
 		for(i=0; i<n; i++){
 			b = write[i];
 			--dcache.ndirty;

sys/src/cmd/venti/srv/fns.h

@@ -6,8 +6,11 @@ void		addstat(int, int);
 void		addstat2(int, int, int, int);
 ZBlock		*alloczblock(u32int size, int zeroed, uint alignment);
 Arena		*amapitoa(Index *index, u64int a, u64int *aa);
+Arena		*amapitoag(Index *index, u64int a, u64int *gstart, u64int *glimit, int *g);
 u64int		arenadirsize(Arena *arena, u32int clumps);
+int		arenatog(Arena *arena, u64int aa, u64int *gstart, u64int *glimit, int *g);
 void		arenaupdate(Arena *arena, u32int size, u8int *score);
+int		asumload(Arena *arena, int g, IEntry *entries, int maxentries);
 void		backsumarena(Arena *arena);
 void	binstats(long (*fn)(Stats *s0, Stats *s1, void*), void *arg, long t0, long t1, Statbin *bin, int nbin);
 int		bloominit(Bloom*, vlong, uchar*);
@@ -26,7 +29,6 @@ void		delaykickroundproc(void*);
 void		dirtydblock(DBlock*, int);
 void		diskaccess(int);
 void		disksched(void);
-AState	diskstate(void);
 void		*emalloc(ulong);
 void		emptydcache(void);
 void		emptyicache(void);
@@ -64,6 +66,8 @@ int		iaddrcmp(IAddr *ia1, IAddr *ia2);
 IEntry*	icachedirty(u32int, u32int, u64int);
 ulong	icachedirtyfrac(void);
 void		icacheclean(IEntry*);
+int		icachelookup(u8int *score, int type, IAddr *ia);
+AState	icachestate(void);
 int		ientrycmp(const void *vie1, const void *vie2);
 char		*ifileline(IFile *f);
 int		ifilename(IFile *f, char *dst);
@@ -76,7 +80,7 @@ ArenaPart	*initarenapart(Part *part);
 int		initarenasum(void);
 void		initbloomfilter(Index*);
 void		initdcache(u32int mem);
-void		initicache(int bits, int depth);
+void		initicache(u32int mem);
 void		initicachewrite(void);
 IEStream	*initiestream(Part *part, u64int off, u64int clumps, u32int size);
 ISect		*initisect(Part *part);
@@ -87,7 +91,7 @@ Part*		initpart(char *name, int mode);
 void		initround(Round*, char*, int);
 int		initventi(char *config, Config *conf);
 void		insertlump(Lump *lump, Packet *p);
-int		insertscore(u8int *score, IAddr *ia, int write);
+int		insertscore(u8int *score, IAddr *ia, int state, AState *as);
 void		kickdcache(void);
 void		kickicache(void);
 void		kickround(Round*, int wait);
@@ -97,14 +101,14 @@ DBlock	*loadibucket(Index *index, u8int *score, ISect **is, u32int *buck, IBucke
 int		loadientry(Index *index, u8int *score, int type, IEntry *ie);
 void		logerr(int severity, char *fmt, ...);
 Lump		*lookuplump(u8int *score, int type);
-int		_lookupscore(u8int *score, int type, IAddr *ia, int *rac);
-int		lookupscore(u8int *score, int type, IAddr *ia, int *rac);
+int		lookupscore(u8int *score, int type, IAddr *ia);
 int		maparenas(AMap *am, Arena **arenas, int n, char *what);
 void		markbloomfilter(Bloom*, u8int*);
 uint		msec(void);
 int		namecmp(char *s, char *t);
 void		namecp(char *dst, char *src);
 int		nameok(char *name);
+void		needmainindex(void);
 void		needzeroscore(void);
 Arena		*newarena(Part *part, u32int, char *name, u64int base, u64int size, u32int blocksize);
 ArenaPart	*newarenapart(Part *part, u32int blocksize, u32int tabsize);
@@ -152,7 +156,6 @@ int		runconfig(char *config, Config*);
 int		scorecmp(u8int *, u8int *);
 void		scoremem(u8int *score, u8int *buf, int size);
 void		setatailstate(AState*);
-void		setdcachestate(AState*);
 void		seterr(int severity, char *fmt, ...);
 void		setstat(int, long);
 void		settrace(char *type);
@@ -166,9 +169,8 @@ int		strscore(char *s, u8int *score);
 int		stru32int(char *s, u32int *r);
 int		stru64int(char *s, u64int *r);
 void		sumarena(Arena *arena);
-int		syncarena(Arena *arena, u64int start, u32int n, int zok, int fix);
-int		syncarenaindex(Index *ix, Arena *arena, u32int clump, u64int a, int fix, int *pflush, int check);
-int		syncindex(Index *ix, int fix, int mustflushicache, int check);
+int		syncarena(Arena *arena, u32int n, int zok, int fix);
+int		syncindex(Index *ix);
 void		trace(char *type, char*, ...);
 void		traceinit(void);
 int		u64log2(u64int v);
@@ -197,12 +199,12 @@ void		wbbloomhead(Bloom*);
 int		wbisect(ISect *is);
 int		wbindex(Index *ix);
 int		whackblock(u8int *dst, u8int *src, int ssize);
-u64int		writeaclump(Arena *a, Clump *c, u8int *clbuf, u64int, u64int*);
+u64int		writeaclump(Arena *a, Clump *c, u8int *clbuf);
 u32int		writearena(Arena *arena, u64int aa, u8int *clbuf, u32int n);
 int		writebloom(Bloom*);
 int		writeclumpinfo(Arena *arean, int clump, ClumpInfo *ci);
 int		writepng(Hio*, Memimage*);
-u64int		writeiclump(Index *ix, Clump *c, u8int *clbuf, u64int*);
+u64int		writeiclump(Index *ix, Clump *c, u8int *clbuf);
 int		writelump(Packet *p, u8int *score, int type, u32int creator, uint ms);
 int		writepart(Part *part, u64int addr, u8int *buf, u32int n);
 int		writeqlump(Lump *u, Packet *p, int creator, uint ms);

sys/src/cmd/venti/srv/hdisk.c

@@ -547,7 +547,7 @@ debugread(HConnect *c, u8int *score)
 	Lump *u;
 	IAddr ia;
 	IEntry ie;
-	int i, rac;
+	int i;
 	Arena *arena;
 	u64int aa;
 	ZBlock *zb;
@@ -561,7 +561,7 @@ debugread(HConnect *c, u8int *score)
 	}
 	
 	hprint(&c->hout, "<h2>index search %V</h2><pre>\n", score);
-	if(_lookupscore(score, -1, &ia, nil) < 0)
+	if(icachelookup(score, -1, &ia) < 0)
 		hprint(&c->hout, "  icache: not found\n");
 	else
 		hprint(&c->hout, "  icache: addr=%#llx size=%d type=%d blocks=%d\n",
@@ -585,12 +585,12 @@ debugread(HConnect *c, u8int *score)
 			hprint(&c->hout, " -cache");
 		putlump(u);
 		
-		if(lookupscore(score, type, &ia, &rac) < 0){
+		if(lookupscore(score, type, &ia) < 0){
 			hprint(&c->hout, " -lookup\n");
 			continue;
 		}
-		hprint(&c->hout, "\n  lookupscore: addr=%#llx size=%d blocks=%d rac=%d\n",
-			ia.addr, ia.size, ia.blocks, rac);
+		hprint(&c->hout, "\n  lookupscore: addr=%#llx size=%d blocks=%d\n",
+			ia.addr, ia.size, ia.blocks);
 		
 		arena = amapitoa(mainindex, ia.addr, &aa);
 		if(arena == nil){

sys/src/cmd/venti/srv/httpd.c

@@ -565,11 +565,11 @@ darena(Hio *hout, Arena *arena)
 	if(scorecmp(zeroscore, arena->score) != 0)
 		hprint(hout, "\tscore=%V\n", arena->score);
 
-	hprint(hout, "\tmem: clumps=%d compressed clumps=%d data=%,lld compressed data=%,lld storage=%,lld\n",
+	hprint(hout, "\twritten: clumps=%d compressed clumps=%d data=%,lld compressed data=%,lld storage=%,lld\n",
 		arena->memstats.clumps, arena->memstats.cclumps, arena->memstats.uncsize,
 		arena->memstats.used - arena->memstats.clumps * ClumpSize,
 		arena->memstats.used + arena->memstats.clumps * ClumpInfoSize);
-	hprint(hout, "\tdisk: clumps=%d compressed clumps=%d data=%,lld compressed data=%,lld storage=%,lld\n",
+	hprint(hout, "\tindexed: clumps=%d compressed clumps=%d data=%,lld compressed data=%,lld storage=%,lld\n",
 		arena->diskstats.clumps, arena->diskstats.cclumps, arena->diskstats.uncsize,
 		arena->diskstats.used - arena->diskstats.clumps * ClumpSize,
 		arena->diskstats.used + arena->diskstats.clumps * ClumpInfoSize);
@@ -895,7 +895,7 @@ static char* graphname[] =
 
 	"icachehit",
 	"icachemiss",
-	"icachelookup",
+	"icacheread",
 	"icachewrite",
 	"icachefill",
 	"icacheprefetch",
@@ -904,6 +904,9 @@ static char* graphname[] =
 	"icacheflush",
 	"icachestall",
 	"icachelookuptime",
+	"icachelookup",
+	"scachehit",
+	"scacheprefetch",
 
 	"bloomhit",
 	"bloommiss",
@@ -925,6 +928,9 @@ static char* graphname[] =
 
 	"sumread",
 	"sumreadbyte",
+	
+	"cigload",
+	"cigloadtime",
 };
 
 static int

sys/src/cmd/venti/srv/icache.c

@@ -2,236 +2,429 @@
 #include "dat.h"
 #include "fns.h"
 
+int icacheprefetch = 1;
+
 typedef struct ICache ICache;
+typedef struct IHash IHash;
+typedef struct ISum ISum;
+
 struct ICache
 {
-	QLock	lock;			/* locks hash table & all associated data */
+	QLock	lock;
 	Rendez	full;
-	IEntry	**heads;		/* heads of all the hash chains */
-	int	bits;			/* bits to use for indexing heads */
-	u32int	size;			/* number of heads; == 1 << bits, should be < entries */
-	IEntry	*base;			/* all allocated hash table entries */
-	IEntry	*free;
-	u32int	entries;		/* elements in base */
-	IEntry	*dirty;		/* chain of dirty elements */
-	u32int	ndirty;
+	IHash	*hash;
+	IEntry	*entries;
+	int		nentries;
+	IEntry	free;
+	IEntry	clean;
+	IEntry	dirty;
 	u32int	maxdirty;
-	u32int	unused;			/* index of first unused element in base */
-	u32int	stolen;			/* last head from which an element was stolen */
+	u32int	ndirty;
+	AState	as;
 
-	Arena	*last[4];
-	Arena	*lastload;
-	int		nlast;
+	ISum	**sum;
+	int		nsum;
+	IHash	*shash;
+	IEntry	*sentries;
+	int		nsentries;
 };
 
-int icacheprefetch = 0;		/* interferes with playing music via vacfs */
-
 static ICache icache;
 
-static IEntry	*icachealloc(IAddr *ia, u8int *score);
-
 /*
- * bits is the number of bits in the icache hash table
- * depth is the average depth
- * memory usage is about (1<<bits) * depth * sizeof(IEntry) + (1<<bits) * sizeof(IEntry*)
+ * Hash table of IEntries
  */
-void
-initicache(int bits, int depth)
+
+struct IHash
 {
-	icache.bits = bits;
-	icache.size = 1 << bits;
-	icache.entries = depth * icache.size;
-	icache.maxdirty = icache.entries/2;
-	icache.base = MKNZ(IEntry, icache.entries);
-	icache.heads = MKNZ(IEntry*, icache.size);
-	icache.full.l = &icache.lock;
-	setstat(StatIcacheSize, icache.entries);
-}
+	int bits;
+	u32int size;
+	IEntry **table;
+};
 
-ulong
-icachedirtyfrac(void)
+static IHash*
+mkihash(int size1)
 {
-	return (vlong)icache.ndirty*IcacheFrac / icache.entries;
+	u32int size;
+	int bits;
+	IHash *ih;
+	
+	bits = 0;
+	size = 1;
+	while(size < size1){
+		bits++;
+		size <<= 1;
+	}
+	
+	ih = vtmallocz(sizeof(IHash)+size*sizeof(ih->table[0]));
+	ih->table = (IEntry**)(ih+1);
+	ih->bits = bits;
+	ih->size = size;
+	return ih;
 }
 
-u32int
-hashbits(u8int *sc, int bits)
+static IEntry*
+ihashlookup(IHash *ih, u8int score[VtScoreSize], int type)
 {
-	u32int v;
-
-	v = (sc[0] << 24) | (sc[1] << 16) | (sc[2] << 8) | sc[3];
-	if(bits < 32)
-		 v >>= (32 - bits);
-	return v;
+	u32int h;
+	IEntry *ie;
+	
+	h = hashbits(score, ih->bits);
+	for(ie=ih->table[h]; ie; ie=ie->nexthash)
+		if((type == -1 || type == ie->ia.type) && scorecmp(score, ie->score) == 0)
+			return ie;
+	return nil;
 }
 
 static void
-loadarenaclumps(Arena *arena, u64int aa)
+ihashdelete(IHash *ih, IEntry *ie, char *what)
 {
-	ulong i;
-	ClumpInfo ci;
-	IAddr ia;
-
-	for(i=0; i<arena->memstats.clumps; i++){
-		if(readclumpinfo(arena, i, &ci) < 0)
-			break;
-		ia.type = ci.type;
-		ia.size = ci.uncsize;
-		ia.blocks = (ci.size + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog;
-		ia.addr = aa;
-		aa += ClumpSize + ci.size;
-		if(ia.type != VtCorruptType)
-			insertscore(ci.score, &ia, 0);
-	}
+	u32int h;
+	IEntry **l;
+	
+	h = hashbits(ie->score, ih->bits);
+	for(l=&ih->table[h]; *l; l=&(*l)->nexthash)
+		if(*l == ie){
+			*l = ie->nexthash;
+			return;
+		}
+	fprint(2, "warning: %s %V not found in ihashdelete\n", what, ie->score);
 }
 
-int
-_lookupscore(u8int *score, int type, IAddr *ia, int *rac)
+static void
+ihashinsert(IHash *ih, IEntry *ie)
 {
 	u32int h;
-	IEntry *ie, *last;
-
-	qlock(&icache.lock);
-	h = hashbits(score, icache.bits);
-	last = nil;
-	for(ie = icache.heads[h]; ie != nil; ie = ie->next){
-		if((ie->ia.type == type || type == -1) && scorecmp(ie->score, score)==0){
-			if(last != nil)
-				last->next = ie->next;
-			else
-				icache.heads[h] = ie->next;
-			addstat(StatIcacheHit, 1);
-			if(rac)
-				ie->rac = 1;
-			trace(TraceLump, "lookupscore incache");
-			ie->next = icache.heads[h];
-			icache.heads[h] = ie;
-
-			*ia = ie->ia;
-			if(rac)
-				*rac = ie->rac;
-			qunlock(&icache.lock);
-			return 0;
-		}
-		last = ie;
-	}
-	addstat(StatIcacheMiss, 1);
-	qunlock(&icache.lock);
-	return -1;
+	
+	h = hashbits(ie->score, ih->bits);
+	ie->nexthash = ih->table[h];
+	ih->table[h] = ie;
 }
 
 
 /*
-ZZZ need to think about evicting the correct IEntry,
-and writing back the wtime.
- * look up data score in the index cache
- * if this fails, pull it in from the disk index table, if it exists.
- *
- * must be called with the lump for this score locked
+ * IEntry lists.
  */
-int
-lookupscore(u8int *score, int type, IAddr *ia, int *rac)
+
+static IEntry*
+popout(IEntry *ie)
 {
-	IEntry d, *ie;
-	u32int h;
-	u64int aa;
-	Arena *load;
-	int i, ret;
-	uint ms;
+	if(ie->prev == nil && ie->next == nil)
+		return ie;
+	ie->prev->next = ie->next;
+	ie->next->prev = ie->prev;
+	ie->next = nil;
+	ie->prev = nil;
+	return ie;
+}
 
-	aa = 0;
-	ms = msec();
-	
-	trace(TraceLump, "lookupscore %V.%d", score, type);
+static IEntry*
+poplast(IEntry *list)
+{
+	if(list->prev == list)
+		return nil;
+	return popout(list->prev);
+}
 
-	ret = 0;
-	if(_lookupscore(score, type, ia, rac) < 0){
-		if(loadientry(mainindex, score, type, &d) < 0){
-			ret = -1;
-			goto out;
-		}
+static IEntry*
+pushfirst(IEntry *list, IEntry *ie)
+{
+	popout(ie);
+	ie->prev = list;
+	ie->next = list->next;
+	ie->prev->next = ie;
+	ie->next->prev = ie;
+	return ie;
+}
 
-		/* failed in cache but found on disk - fill cache. */
-		trace(TraceLump, "lookupscore loaded");
-		addstat(StatIcacheFill, 1);
+/*
+ * Arena summary cache.
+ */
+struct ISum
+{
+	QLock	lock;
+	IEntry	*entries;
+	int	nentries;
+	int	loaded;
+	u64int addr;
+	u64int limit;
+	Arena *arena;
+	int g;
+};
 
-		/*
-		 * no one else can load an entry for this score,
-		 * since we have this score's lump's lock.
-		 */
-		qlock(&icache.lock);
-	
-		/*
-		 * If we notice that all the hits are coming from one arena,
-		 * load the table of contents for that arena into the cache.
-		 */
-		load = nil;
-		h = hashbits(score, icache.bits);
-		ie = icachealloc(&d.ia, score);
-		if(icacheprefetch){
-			icache.last[icache.nlast++%nelem(icache.last)] = amapitoa(mainindex, ie->ia.addr, &aa);
-			aa = ie->ia.addr - aa;	/* compute base addr of arena */
-			for(i=0; i<nelem(icache.last); i++)
-				if(icache.last[i] != icache.last[0])
-					break;
-			if(i==nelem(icache.last) && icache.lastload != icache.last[0]){
-				load = icache.last[0];
-				icache.lastload = load;
+static ISum*
+scachelookup(u64int addr)
+{
+	int i;
+	ISum *s;
+
+	for(i=0; i<icache.nsum; i++){
+		s = icache.sum[i];
+		if(s->addr <= addr && addr < s->limit){
+			if(i > 0){
+				memmove(icache.sum+1, icache.sum, i*sizeof icache.sum[0]);
+				icache.sum[0] = s;
 			}
+			return s;
 		}
+	}
+	return nil;
+}
+
+static void
+sumclear(ISum *s)
+{
+	int i;
+
+	for(i=0; i<s->nentries; i++)
+		ihashdelete(icache.shash, &s->entries[i], "scache");
+	s->nentries = 0;
+	s->loaded = 0;
+	s->addr = 0;
+	s->limit = 0;
+	s->arena = nil;
+	s->g = 0;
+}
+
+static ISum*
+scacheevict(void)
+{
+	ISum *s;
+	int i;
 	
-		ie->next = icache.heads[h];
-		icache.heads[h] = ie;
-	
-		*ia = ie->ia;
-		*rac = ie->rac;
-	
-		qunlock(&icache.lock);
-		if(load){
-			trace(TraceProc, "preload 0x%llux", aa);
-			loadarenaclumps(load, aa);
+	for(i=icache.nsum-1; i>=0; i--){
+		s = icache.sum[i];
+		if(canqlock(&s->lock)){
+			if(i > 0){
+				memmove(icache.sum+1, icache.sum, i*sizeof icache.sum[0]);
+				icache.sum[0] = s;
+			}
+			sumclear(s);
+			return s;
 		}
 	}
+	return nil;
+}
 
-out:
-	ms = msec() - ms;
-	addstat2(StatIcacheRead, 1, StatIcacheReadTime, ms);
+static void
+scachehit(u64int addr)
+{
+	scachelookup(addr);	/* for move-to-front */
+}
 
-	return ret;
+static void
+scachesetup(ISum *s, u64int addr)
+{
+	u64int addr0, limit;
+	int g;
+
+	s->arena = amapitoag(mainindex, addr, &addr0, &limit, &g);
+	s->addr = addr0;
+	s->limit = limit;
+	s->g = g;
+}
+
+static void
+scacheload(ISum *s)
+{
+	int i, n;
+
+	s->loaded = 1;
+	n = asumload(s->arena, s->g, s->entries, ArenaCIGSize);
+	/*
+	 * n can be less then ArenaCIGSize, either if the clump group
+	 * is the last in the arena and is only partially filled, or if there
+	 * are corrupt clumps in the group -- those are not returned.
+	 */
+	for(i=0; i<n; i++){
+		s->entries[i].ia.addr += s->addr;
+		ihashinsert(icache.shash, &s->entries[i]);
+	}
+//fprint(2, "%T scacheload %s %d - %d entries\n", s->arena->name, s->g, n);
+	addstat(StatScachePrefetch, n);
+	s->nentries = n;
+}
+
+static ISum*
+scachemiss(u64int addr)
+{
+	ISum *s;
+
+	s = scachelookup(addr);
+	if(s == nil){
+		/* first time: make an entry in the cache but don't populate it yet */
+		s = scacheevict();
+		if(s == nil)
+			return nil;
+		scachesetup(s, addr);
+		qunlock(&s->lock);
+		return nil;
+	}
+
+	/* second time: load from disk */
+	qlock(&s->lock);
+	if(s->loaded || !icacheprefetch){
+		qunlock(&s->lock);
+		return nil;
+	}
+	
+	return s;	/* locked */
 }
 
 /*
- * insert a new element in the hash table.
+ * Index cache.
  */
-int
-insertscore(u8int *score, IAddr *ia, int write)
+
+void
+initicache(u32int mem0)
 {
-	IEntry *ie, se;
-	u32int h;
+	u32int mem;
+	int i, entries, scache;
+	
+	icache.full.l = &icache.lock;
 
-	trace(TraceLump, "insertscore enter");
-	if(write)
-		addstat(StatIcacheWrite, 1);
-	else
-		addstat(StatIcachePrefetch, 1);
+	mem = mem0;
+	entries = mem / (sizeof(IEntry)+sizeof(IEntry*));
+	scache = (entries/8) / ArenaCIGSize;
+	entries -= entries/8;
+	if(scache < 4)
+		scache = 4;
+	if(scache > 16)
+		scache = 16;
+	if(entries < 1000)
+		entries = 1000;
+fprint(2, "icache %,d bytes = %,d entries; %d scache\n", mem0, entries, scache);
+
+	icache.clean.prev = icache.clean.next = &icache.clean;
+	icache.dirty.prev = icache.dirty.next = &icache.dirty;
+	icache.free.prev = icache.free.next = &icache.free;
+	
+	icache.hash = mkihash(entries);
+	icache.nentries = entries;
+	setstat(StatIcacheSize, entries);
+	icache.entries = vtmallocz(entries*sizeof icache.entries[0]);
+	icache.maxdirty = entries / 2;
+	for(i=0; i<entries; i++)
+		pushfirst(&icache.free, &icache.entries[i]);
+
+	icache.nsum = scache;
+	icache.sum = vtmallocz(scache*sizeof icache.sum[0]);
+	icache.sum[0] = vtmallocz(scache*sizeof icache.sum[0][0]);
+	icache.nsentries = scache * ArenaCIGSize;
+	icache.sentries = vtmallocz(scache*ArenaCIGSize*sizeof icache.sentries[0]);
+	icache.shash = mkihash(scache*ArenaCIGSize);
+	for(i=0; i<scache; i++){
+		icache.sum[i] = icache.sum[0] + i;
+		icache.sum[i]->entries = icache.sentries + i*ArenaCIGSize;
+	}
+}
 
-	qlock(&icache.lock);
-	h = hashbits(score, icache.bits);
 
-	ie = icachealloc(ia, score);
-	if(write){
+static IEntry*
+evictlru(void)
+{
+	IEntry *ie;
+	
+	ie = poplast(&icache.clean);
+	if(ie == nil)
+		return nil;
+	ihashdelete(icache.hash, ie, "evictlru");
+	return ie;
+}
+
+static void
+icacheinsert(u8int score[VtScoreSize], IAddr *ia, int state)
+{
+	IEntry *ie;
+
+	if((ie = poplast(&icache.free)) == nil && (ie = evictlru()) == nil){
+		addstat(StatIcacheStall, 1);
+		while((ie = poplast(&icache.free)) == nil && (ie = evictlru()) == nil){
+			// Could safely return here if state == IEClean.
+			// But if state == IEDirty, have to wait to make
+			// sure we don't lose an index write.  
+			// Let's wait all the time.
+			flushdcache();
+			kickicache();
+			rsleep(&icache.full);
+		}
+		addstat(StatIcacheStall, -1);
+	}
+
+	memmove(ie->score, score, VtScoreSize);
+	ie->state = state;
+	ie->ia = *ia;
+	if(state == IEClean){
+		addstat(StatIcachePrefetch, 1);
+		pushfirst(&icache.clean, ie);
+	}else{
+		addstat(StatIcacheWrite, 1);
+		assert(state == IEDirty);
 		icache.ndirty++;
 		setstat(StatIcacheDirty, icache.ndirty);
 		delaykickicache();
-		ie->dirty = 1;
+		pushfirst(&icache.dirty, ie);
+	}
+	ihashinsert(icache.hash, ie);
+}
+
+int
+icachelookup(u8int score[VtScoreSize], int type, IAddr *ia)
+{
+	IEntry *ie;
+
+	qlock(&icache.lock);
+	addstat(StatIcacheLookup, 1);
+	if((ie = ihashlookup(icache.hash, score, type)) != nil){
+		*ia = ie->ia;
+		if(ie->state == IEClean)
+			pushfirst(&icache.clean, ie);
+		addstat(StatIcacheHit, 1);
+		qunlock(&icache.lock);
+		return 0;
 	}
-	ie->next = icache.heads[h];
-	icache.heads[h] = ie;
 
-	se = *ie;
+	if((ie = ihashlookup(icache.shash, score, type)) != nil){
+		*ia = ie->ia;
+		icacheinsert(score, &ie->ia, IEClean);
+		scachehit(ie->ia.addr);
+		addstat(StatScacheHit, 1);
+		qunlock(&icache.lock);
+		return 0;
+	}
+	addstat(StatIcacheMiss, 1);
 	qunlock(&icache.lock);
 
-	if(write && icache.ndirty >= icache.maxdirty)
+	return -1;
+}
+
+int
+insertscore(u8int score[VtScoreSize], IAddr *ia, int state, AState *as)
+{
+	ISum *toload;
+
+	qlock(&icache.lock);
+	icacheinsert(score, ia, state);
+	if(state == IEClean)
+		toload = scachemiss(ia->addr);
+	else{
+		assert(state == IEDirty);
+		toload = nil;
+		if(as == nil)
+			fprint(2, "%T insertscore IEDirty without as; called from %lux\n", getcallerpc(&score));
+		else{
+			if(icache.as.aa > as->aa)
+				fprint(2, "%T insertscore: aa moving backward: %#llux -> %#llux\n", icache.as.aa, as->aa);
+			icache.as = *as;
+		}
+	}
+	qunlock(&icache.lock);
+	if(toload){
+		scacheload(toload);
+		qunlock(&toload->lock);
+	}
+	
+	if(icache.ndirty >= icache.maxdirty)
 		kickicache();
 
 	/*
@@ -240,125 +433,81 @@ insertscore(u8int *score, IAddr *ia, int write)
 	 * the lump, meaning any searches for this block
 	 * will hit in the lump cache until after we return.
 	 */
-	markbloomfilter(mainindex->bloom, score);
+	if(state == IEDirty)
+		markbloomfilter(mainindex->bloom, score);
 
 	return 0;
 }
 
-/*
- * allocate a index cache entry which hasn't been used in a while.
- * must be called with icache.lock locked
- * if the score is already in the table, update the entry.
- */
-static IEntry *
-icachealloc(IAddr *ia, u8int *score)
+static int
+lookupscore_untimed(u8int score[VtScoreSize], int type, IAddr *ia)
 {
-	int i;
-	IEntry *ie, *last, *clean, *lastclean;
-	u32int h;
+	IEntry d;
 
-	h = hashbits(score, icache.bits);
-	last = nil;
-	for(ie = icache.heads[h]; ie != nil; ie = ie->next){
-		if(ie->ia.type == ia->type && scorecmp(ie->score, score)==0){
-			if(last != nil)
-				last->next = ie->next;
-			else
-				icache.heads[h] = ie->next;
-			trace(TraceLump, "icachealloc hit");
-			ie->rac = 1;
-			return ie;
-		}
-		last = ie;
-	}
+	if(icachelookup(score, type, ia) >= 0)
+		return 0;
 
-	h = icache.unused;
-	if(h < icache.entries){
-		ie = &icache.base[h++];
-		icache.unused = h;
-		trace(TraceLump, "icachealloc unused");
-		goto Found;
-	}
+	addstat(StatIcacheFill, 1);
+	if(loadientry(mainindex, score, type, &d) < 0)
+		return -1;
 	
-	if((ie = icache.free) != nil){
-		icache.free = ie->next;
-		goto Found;
-	}
+	insertscore(score, &d.ia, IEClean, nil);
+	*ia = d.ia;
+	return 0;
+}
 
-	h = icache.stolen;
-	for(i=0;; i++){
-		h++;
-		if(h >= icache.size)
-			h = 0;
-		if(i == icache.size){
-			trace(TraceLump, "icachealloc sleep");
-			addstat(StatIcacheStall, 1);
-			while(icache.ndirty == icache.entries){
-				/*
-				 * This is a bit suspect.  Kickicache will wake up the
-				 * icachewritecoord, but if all the index entries are for
-				 * unflushed disk blocks, icachewritecoord won't be
-				 * able to do much.  It always rewakes everyone when
-				 * it thinks it is done, though, so at least we'll go around
-				 * the while loop again.  Also, if icachewritecoord sees
-				 * that the disk state hasn't change at all since the last
-				 * time around, it kicks the disk.  This needs to be
-				 * rethought, but it shouldn't deadlock anymore.
-				 */
-				kickicache();
-				rsleep(&icache.full);
-			}
-			addstat(StatIcacheStall, -1);
-			i = 0;
-		}
-		lastclean = nil;
-		clean = nil;
-		last = nil;
-		for(ie=icache.heads[h]; ie; last=ie, ie=ie->next){
-			if(!ie->dirty){
-				clean = ie;
-				lastclean = last;
-			}
-		}
-		if(clean){
-			if(lastclean)
-				lastclean->next = clean->next;
-			else
-				icache.heads[h] = clean->next;
-			clean->next = nil;
-			icache.stolen = h;
-			ie = clean;
-			trace(TraceLump, "icachealloc steal");
-			goto Found;
-		}
-	}
+int
+lookupscore(u8int score[VtScoreSize], int type, IAddr *ia)
+{
+	int ms, ret;
+	
+	ms = msec();
+	ret = lookupscore_untimed(score, type, ia);
+	ms = msec() - ms;
+	addstat2(StatIcacheRead, 1, StatIcacheReadTime, ms);
+	return ret;
+}
+	
+u32int
+hashbits(u8int *sc, int bits)
+{
+	u32int v;
 
-Found:
-	ie->ia = *ia;
-	scorecp(ie->score, score);
-	ie->rac = 0;	
-	return ie;
+	v = (sc[0] << 24) | (sc[1] << 16) | (sc[2] << 8) | sc[3];
+	if(bits < 32)
+		 v >>= (32 - bits);
+	return v;
 }
 
+ulong
+icachedirtyfrac(void)
+{
+	return (vlong)icache.ndirty*IcacheFrac / icache.nentries;
+}
+
+/*
+ * Return a singly-linked list of dirty index entries.
+ * with 32-bit hash numbers between lo and hi
+ * and address < limit.
+ */
 IEntry*
 icachedirty(u32int lo, u32int hi, u64int limit)
 {
-	int i;
 	u32int h;
 	IEntry *ie, *dirty;
 
 	dirty = nil;
 	trace(TraceProc, "icachedirty enter");
 	qlock(&icache.lock);
-	for(i=0; i<icache.size; i++)
-	for(ie = icache.heads[i]; ie; ie=ie->next)
-		if(ie->dirty && ie->ia.addr != 0 && ie->ia.addr < limit){
+	for(ie = icache.dirty.next; ie != &icache.dirty; ie=ie->next){
+		if(ie->state == IEDirty && ie->ia.addr < limit){
 			h = hashbits(ie->score, 32);
 			if(lo <= h && h <= hi){
 				ie->nextdirty = dirty;
 				dirty = ie;
 			}
 		}
+	}
 	qunlock(&icache.lock);
 	trace(TraceProc, "icachedirty exit");
 	if(dirty == nil)
@@ -366,36 +515,59 @@ icachedirty(u32int lo, u32int hi, u64int limit)
 	return dirty;
 }
 
+AState
+icachestate(void)
+{
+	AState as;
+
+	qlock(&icache.lock);
+	as = icache.as;
+	qunlock(&icache.lock);
+	return as;
+}
+
+/*
+ * The singly-linked non-circular list of index entries ie
+ * has been written to disk.  Move them to the clean list.
+ */
 void
 icacheclean(IEntry *ie)
 {
-	trace(TraceProc, "icachedirty enter");
+	IEntry *next;
+	
+	trace(TraceProc, "icacheclean enter");
 	qlock(&icache.lock);
-	for(; ie; ie=ie->nextdirty){
+	for(; ie; ie=next){
+		assert(ie->state == IEDirty);
+		next = ie->nextdirty;
+		ie->nextdirty = nil;
+		popout(ie); /* from icache.dirty */
 		icache.ndirty--;
-		ie->dirty = 0;
+		ie->state = IEClean;
+		pushfirst(&icache.clean, ie);
 	}
 	setstat(StatIcacheDirty, icache.ndirty);
 	rwakeupall(&icache.full);
 	qunlock(&icache.lock);
-	trace(TraceProc, "icachedirty exit");
+	trace(TraceProc, "icacheclean exit");
 }
 
 void
 emptyicache(void)
 {
 	int i;
-	IEntry *ie, **lie;
+	IEntry *ie;
+	ISum *s;
 	
 	qlock(&icache.lock);
-	for(i=0; i<icache.size; i++)
-	for(lie=&icache.heads[i]; (ie=*lie); ){
-		if(ie->dirty == 0){
-			*lie = ie->next;
-			ie->next = icache.free;
-			icache.free = ie;
-		}else
-			lie = &ie->next;
+	while((ie = evictlru()) != nil)
+		pushfirst(&icache.free, ie);
+	for(i=0; i<icache.nsum; i++){
+		s = icache.sum[i];
+		qlock(&s->lock);
+		sumclear(s);
+		qunlock(&s->lock);
 	}
 	qunlock(&icache.lock);
 }
+

sys/src/cmd/venti/srv/icachewrite.c

@@ -12,7 +12,7 @@ static void icachewritecoord(void*);
 static IEntry *iesort(IEntry*);
 
 int icachesleeptime = 1000;	/* milliseconds */
-int minicachesleeptime = 50;
+int minicachesleeptime = 0;
 
 enum
 {
@@ -85,7 +85,7 @@ nextchunk(Index *ix, ISect *is, IEntry **pie, u64int *paddr, uint *pnbuf)
 static int
 icachewritesect(Index *ix, ISect *is, u8int *buf)
 {
-	int err, h, bsize, t;
+	int err, i, werr, h, bsize, t;
 	u32int lo, hi;
 	u64int addr, naddr;
 	uint nbuf, off;
@@ -115,7 +115,8 @@ icachewritesect(Index *ix, ISect *is, u8int *buf)
 		}
 		if(t < minicachesleeptime)
 			t = minicachesleeptime;
-		sleep(t);
+		if(t > 0)
+			sleep(t);
 		trace(TraceProc, "icachewritesect nextchunk");
 		chunk = nextchunk(ix, is, &iedirty, &addr, &nbuf);
 
@@ -169,33 +170,29 @@ skipit:
 					break;
 			}
 			packibucket(&ib, buf+off, is->bucketmagic);
-			/*
-			 * XXX This is not quite right - it's good that we 
-			 * update the cached block (if any) here, but
-			 * since the block doesn't get written until writepart
-			 * below, we also need to make sure that the cache 
-			 * doesn't load the stale block before we write it to
-			 * disk below.  We could lock the disk cache during
-			 * the writepart, but that's pretty annoying.
-			 * Another possibility would be never to cache
-			 * index partition blocks.  The hit rate on those is
-			 * miniscule anyway.
-			 */
-			if((b = _getdblock(is->part, naddr, ORDWR, 0)) != nil){
-				memmove(b->data, buf+off, bsize);
-				putdblock(b);
-			}
 		}
 
 		diskaccess(1);
 
 		trace(TraceProc, "icachewritesect writepart", addr, nbuf);
-		if(writepart(is->part, addr, buf, nbuf) < 0 ||
-		    flushpart(is->part) < 0){
+		werr = 0;
+		if(writepart(is->part, addr, buf, nbuf) < 0 || flushpart(is->part) < 0)
+			werr = -1;
+
+		for(i=0; i<nbuf; i+=bsize){
+			if((b = _getdblock(is->part, addr+i, ORDWR, 0)) != nil){
+				memmove(b->data, buf+i, bsize);
+				putdblock(b);
+			}
+		}
+
+		if(werr < 0){
 			fprint(2, "%s: part %s addr 0x%llux: icachewritesect "
 				"writepart: %r\n", argv0, is->part->name, addr);
+			err = -1;
 			continue;
 		}
+		
 		addstat(StatIsectWriteBytes, nbuf);
 		addstat(StatIsectWrite, 1);
 		icacheclean(chunk);
@@ -245,18 +242,20 @@ icachewritecoord(void *v)
 	threadsetname("icachewritecoord");
 
 	ix = mainindex;
-	iwrite.as = diskstate();
+	iwrite.as = icachestate();
 
 	for(;;){
 		trace(TraceProc, "icachewritecoord sleep");
 		waitforkick(&iwrite.round);
 		trace(TraceWork, "start");
-		as = diskstate();
+		as = icachestate();
 		if(as.arena==iwrite.as.arena && as.aa==iwrite.as.aa){
 			/* will not be able to do anything more than last flush - kick disk */
+			fprint(2, "icache: nothing to do - kick dcache\n");
 			trace(TraceProc, "icachewritecoord kick dcache");
 			kickdcache();
 			trace(TraceProc, "icachewritecoord kicked dcache");
+			goto SkipWork;	/* won't do anything; don't bother rewriting bloom filter */
 		}
 		iwrite.as = as;
 
@@ -274,9 +273,11 @@ icachewritecoord(void *v)
 				err |= recvul(ix->bloom->writedonechan);
 
 			trace(TraceProc, "icachewritecoord donewrite err=%d", err);
-			if(err == 0)
+			if(err == 0){
 				setatailstate(&iwrite.as);
+			}
 		}
+	SkipWork:
 		icacheclean(nil);	/* wake up anyone waiting */
 		trace(TraceWork, "finish");
 		addstat(StatIcacheFlush, 1);

sys/src/cmd/venti/srv/index.c

@@ -541,20 +541,33 @@ ZZZ question: should this distinguish between an arena
 filling up and real errors writing the clump?
  */
 u64int
-writeiclump(Index *ix, Clump *c, u8int *clbuf, u64int *pa)
+writeiclump(Index *ix, Clump *c, u8int *clbuf)
 {
 	u64int a;
 	int i;
+	IAddr ia;
+	AState as;
 
 	trace(TraceLump, "writeiclump enter");
+	qlock(&ix->writing);
 	for(i = ix->mapalloc; i < ix->narenas; i++){
-		a = writeaclump(ix->arenas[i], c, clbuf, ix->amap[i].start, pa);
+		a = writeaclump(ix->arenas[i], c, clbuf);
 		if(a != TWID64){
-			ix->mapalloc = i;	/* assuming write is atomic, race is okay */
+			ix->mapalloc = i;
+			ia.addr = ix->amap[i].start + a;
+			ia.type = c->info.type;
+			ia.size = c->info.uncsize;
+			ia.blocks = (c->info.size + ClumpSize + (1<<ABlockLog) - 1) >> ABlockLog;
+			as.arena = ix->arenas[i];
+			as.aa = ia.addr;
+			as.stats = as.arena->memstats;
+			insertscore(c->info.score, &ia, IEDirty, &as);
+			qunlock(&ix->writing);
 			trace(TraceLump, "writeiclump exit");
-			return a;
+			return ia.addr;
 		}
 	}
+	qunlock(&ix->writing);
 
 	seterr(EAdmin, "no space left in arenas");
 	trace(TraceLump, "writeiclump failed");
@@ -596,6 +609,25 @@ print("want arena %d for %llux\n", l, a);
 	return ix->arenas[l];
 }
 
+/*
+ * convert an arena index to the bounds of the containing arena group.
+ */
+Arena*
+amapitoag(Index *ix, u64int a, u64int *gstart, u64int *glimit, int *g)
+{
+	u64int aa;
+	Arena *arena;
+	
+	arena = amapitoa(ix, a, &aa);
+	if(arena == nil)
+		return nil;
+	if(arenatog(arena, aa, gstart, glimit, g) < 0)
+		return nil;
+	*gstart += a - aa;
+	*glimit += a - aa;
+	return arena;
+}
+
 int
 iaddrcmp(IAddr *ia1, IAddr *ia2)
 {

sys/src/cmd/venti/srv/lump.c

@@ -7,7 +7,7 @@ int			queuewrites = 0;
 int			writestodevnull = 0;
 int			verifywrites = 0;
 
-static Packet		*readilump(Lump *u, IAddr *ia, u8int *score, int rac);
+static Packet		*readilump(Lump *u, IAddr *ia, u8int *score);
 
 /*
  * Some of this logic is duplicated in hdisk.c
@@ -19,7 +19,6 @@ readlump(u8int *score, int type, u32int size, int *cached)
 	Packet *p;
 	IAddr ia;
 	u32int n;
-	int rac;
 
 	trace(TraceLump, "readlump enter");
 /*
@@ -49,7 +48,7 @@ readlump(u8int *score, int type, u32int size, int *cached)
 	if(cached)
 		*cached = 0;
 
-	if(lookupscore(score, type, &ia, &rac) < 0){
+	if(lookupscore(score, type, &ia) < 0){
 		/* ZZZ place to check for someone trying to guess scores */
 		seterr(EOk, "no block with score %V/%d exists", score, type);
 
@@ -64,7 +63,7 @@ readlump(u8int *score, int type, u32int size, int *cached)
 	}
 
 	trace(TraceLump, "readlump readilump");
-	p = readilump(u, &ia, score, rac);
+	p = readilump(u, &ia, score);
 	putlump(u);
 
 	trace(TraceLump, "readlump exit");
@@ -134,9 +133,8 @@ writeqlump(Lump *u, Packet *p, int creator, uint ms)
 	Packet *old;
 	IAddr ia;
 	int ok;
-	int rac;
 
-	if(lookupscore(u->score, u->type, &ia, &rac) == 0){
+	if(lookupscore(u->score, u->type, &ia) == 0){
 		if(verifywrites == 0){
 			/* assume the data is here! */
 			packetfree(p);
@@ -149,7 +147,7 @@ writeqlump(Lump *u, Packet *p, int creator, uint ms)
 		 * if the read fails,
 		 * assume it was corrupted data and store the block again
 		 */
-		old = readilump(u, &ia, u->score, rac);
+		old = readilump(u, &ia, u->score);
 		if(old != nil){
 			ok = 0;
 			if(packetcmp(p, old) != 0){
@@ -175,8 +173,6 @@ writeqlump(Lump *u, Packet *p, int creator, uint ms)
 	flat = packet2zblock(p, packetsize(p));
 	ok = storeclump(mainindex, flat, u->score, u->type, creator, &ia);
 	freezblock(flat);
-	if(ok == 0)
-		ok = insertscore(u->score, &ia, 1);
 	if(ok == 0)
 		insertlump(u, p);
 	else
@@ -193,39 +189,14 @@ writeqlump(Lump *u, Packet *p, int creator, uint ms)
 	return ok;
 }
 
-static void
-lreadahead(u64int a, Arena *arena, u64int aa, int n)
-{	
-	u8int buf[ClumpSize];
-	Clump cl;
-	IAddr ia;
-
-	while(n > 0) {
-		if (aa >= arena->memstats.used)
-			break;
-		if(readarena(arena, aa, buf, ClumpSize) < ClumpSize)
-			break;
-		if(unpackclump(&cl, buf, arena->clumpmagic) < 0)
-			break;
-		ia.addr = a;
-		ia.type = cl.info.type;
-		ia.size = cl.info.uncsize;
-		ia.blocks = (cl.info.size + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog;
-		insertscore(cl.info.score, &ia, 0);
-		a += ClumpSize + cl.info.size;
-		aa += ClumpSize + cl.info.size;
-		n--;
-	}
-}
-
 static Packet*
-readilump(Lump *u, IAddr *ia, u8int *score, int rac)
+readilump(Lump *u, IAddr *ia, u8int *score)
 {
 	Arena *arena;
 	ZBlock *zb;
 	Packet *p, *pp;
 	Clump cl;
-	u64int a, aa;
+	u64int aa;
 	u8int sc[VtScoreSize];
 
 	trace(TraceLump, "readilump enter");
@@ -258,13 +229,6 @@ readilump(Lump *u, IAddr *ia, u8int *score, int rac)
 		return nil;
 	}
 
-	if(rac == 0) {
-		trace(TraceLump, "readilump readahead");
-		a = ia->addr + ClumpSize + cl.info.size;
-		aa += ClumpSize + cl.info.size;
-		lreadahead(a, arena, aa, 20);
-	}
-
 	trace(TraceLump, "readilump success");
 	p = zblock2packet(zb, cl.info.uncsize);
 	freezblock(zb);

sys/src/cmd/venti/srv/mirrorarenas.c

@@ -22,13 +22,14 @@ Part *src;
 Part *dst;
 int force;
 int verbose;
+int dosha1 = 1;
 char *status;
 uvlong astart, aend;
 
 void
 usage(void)
 {
-	fprint(2, "usage: mirrorarenas [-v] src dst [ranges]\n");
+	fprint(2, "usage: mirrorarenas [-sv] src dst [ranges]\n");
 	threadexitsall("usage");
 }
 
@@ -92,6 +93,7 @@ ewritepart(Part *p, u64int offset, u8int *buf, u32int count)
  * src with writing dst during copy.  This is an easy factor of two
  * (almost) in performance.
  */
+static Write wsync;
 static void
 writeproc(void *v)
 {
@@ -99,7 +101,7 @@ writeproc(void *v)
 	
 	USED(v);
 	while((w = recvp(writechan)) != nil){
-		if(w->n == 0)
+		if(w == &wsync)
 			continue;
 		if(ewritepart(dst, w->o, w->p, w->n) < 0)
 			w->error = 1;
@@ -146,11 +148,7 @@ copy(uvlong start, uvlong end, char *what, DigestState *ds)
 	/*
 	 * wait for queued write to finish
 	 */
-	w[i].p = nil;
-	w[i].o = 0;
-	w[i].n = 0;
-	w[i].error = 0;
-	sendp(writechan, &w[i]);
+	sendp(writechan, &wsync);
 	i = 1-i;
 	if(w[i].error)
 		return -1;
@@ -240,7 +238,7 @@ void
 mirror(Arena *sa, Arena *da)
 {
 	vlong v, si, di, end;
-	int clumpmax, blocksize;
+	int clumpmax, blocksize, sealed;
 	static uchar buf[MaxIoSize];
 	ArenaHead h;
 	DigestState xds, *ds;
@@ -305,7 +303,8 @@ mirror(Arena *sa, Arena *da)
 
 	shaoff = 0;
 	ds = nil;
-	if(sa->diskstats.sealed && scorecmp(sa->score, zeroscore) != 0){
+	sealed = sa->diskstats.sealed && scorecmp(sa->score, zeroscore) != 0;
+	if(sealed && dosha1){
 		/* start sha1 state with header */
 		memset(&xds, 0, sizeof xds);
 		ds = &xds;
@@ -362,7 +361,7 @@ mirror(Arena *sa, Arena *da)
 	if(ewritepart(dst, end, buf, blocksize) < 0)
 		return;
 
-	if(ds){
+	if(sealed){
 		/*
 		 * ... but on the final pass, copy the encoding
 		 * of the tail information from the source
@@ -375,20 +374,27 @@ mirror(Arena *sa, Arena *da)
 		if(asha1(dst, shaoff, end, ds) < 0
 		|| copy(end, end+blocksize-VtScoreSize, "tail", ds) < 0)
 			return;
-		memset(buf, 0, VtScoreSize);
-		sha1(buf, VtScoreSize, da->score, ds);
-		if(scorecmp(sa->score, da->score) == 0){
+		if(dosha1){
+			memset(buf, 0, VtScoreSize);
+			sha1(buf, VtScoreSize, da->score, ds);
+			if(scorecmp(sa->score, da->score) == 0){
+				if(verbose)
+					chat("%T %s: %V sealed mirrored\n", sa->name, sa->score);
+				if(ewritepart(dst, end+blocksize-VtScoreSize, da->score, VtScoreSize) < 0)
+					return;
+			}else{
+				chat("%T %s: sealing dst: score mismatch: %V vs %V\n", sa->name, sa->score, da->score);
+				memset(&xds, 0, sizeof xds);
+				asha1(dst, base-blocksize, end+blocksize-VtScoreSize, &xds);
+				sha1(buf, VtScoreSize, 0, &xds);
+				chat("%T   reseal: %V\n", da->score);
+				status = "errors";
+			}
+		}else{
 			if(verbose)
-				chat("%T %s: %V sealed mirrored\n", sa->name, sa->score);
-			if(ewritepart(dst, end+blocksize-VtScoreSize, da->score, VtScoreSize) < 0)
+				chat("%T %s: %V mirrored\n", sa->name, sa->score);
+			if(ewritepart(dst, end+blocksize-VtScoreSize, sa->score, VtScoreSize) < 0)
 				return;
-		}else{
-			chat("%T %s: sealing dst: score mismatch: %V vs %V\n", sa->name, sa->score, da->score);
-			memset(&xds, 0, sizeof xds);
-			asha1(dst, base-blocksize, end+blocksize-VtScoreSize, &xds);
-			sha1(buf, VtScoreSize, 0, &xds);
-			chat("%T   reseal: %V\n", da->score);
-			status = "errors";
 		}
 	}else{
 		chat("%T %s: %,lld used mirrored\n",
@@ -462,6 +468,9 @@ threadmain(int argc, char **argv)
 	case 'v':
 		verbose++;
 		break;
+	case 's':
+		dosha1 = 0;
+		break;
 	default:
 		usage();
 	}ARGEND

sys/src/cmd/venti/srv/part.c

@@ -168,85 +168,47 @@ enum {
 	Maxxfer = 64*1024,	/* for NCR SCSI controllers; was 128K */
 };
 
-/*
- * Read/write some amount of data between a block device or file and a memory buffer.
- *
- * Most Unix systems require that when accessing a block device directly,
- * the buffer, offset, and count are all multiples of the device block size,
- * making this a lot more complicated than it otherwise would be.
- * 
- * Most of our callers will make things easy on us, but for some callers it's best
- * if we just do the work here, with only one place to get it right (hopefully).
- * 
- * If everything is aligned properly, prwb will try to do big transfers in the main 
- * body of the loop: up to MaxIo bytes at a time.  If everything isn't aligned properly,
- * we work one block at a time.
- */
-#undef min
-#define min(a, b) ((a) < (b) ? (a) : (b))
-static int
-prwb(char *name, int fd, int isread, u64int offset, void *vbuf, u32int count, u32int blocksize)
-{
-	char *op;
-	u8int *buf, *freetmp, *dst;
-	u32int icount, opsize;
-	int r;
-
-	USED(blocksize);
-	icount = count;
-	buf = vbuf;
-	op = isread ? "read" : "write";
-	dst = buf;
-	freetmp = nil;
-	while(count > 0){
-		opsize = min(count, Maxxfer /* blocksize */);
-		if(isread)
-			r = pread(fd, dst, opsize, offset);
-		else
-			r = pwrite(fd, dst, opsize, offset);
-		if(r <= 0)
-			goto Error;
-		offset += r;
-		count -= r;
-		dst += r;
-		if(r != opsize)
-			goto Error;
-	}
-	return icount;
-
-Error:
-	seterr(EAdmin, "%s %s offset 0x%llux count %ud buf %p returned %d: %r",
-		op, name, offset, opsize, dst, r);
-	if(freetmp)
-		free(freetmp);
-	return -1;
-}
+static int reopen(Part*);
 
 int
-rwpart(Part *part, int isread, u64int offset, u8int *buf, u32int count)
+rwpart(Part *part, int isread, u64int offset0, u8int *buf0, u32int count0)
 {
+	u32int count, opsize;
 	int n;
-	u32int blocksize;
+	u8int *buf;
+	u64int offset;
 
 	trace(TraceDisk, "%s %s %ud at 0x%llx", 
-		isread ? "read" : "write", part->name, count, offset);
-	if(offset >= part->size || offset+count > part->size){
+		isread ? "read" : "write", part->name, count0, offset0);
+	if(offset0 >= part->size || offset0+count0 > part->size){
 		seterr(EStrange, "out of bounds %s offset 0x%llux count %ud to partition %s size 0x%llux",
-			isread ? "read" : "write", offset, count, part->name,
+			isread ? "read" : "write", offset0, count0, part->name,
 			part->size);
 		return -1;
 	}
 
-	blocksize = part->fsblocksize;
-	if(blocksize == 0)
-		blocksize = part->blocksize;
-	if(blocksize == 0)
-		blocksize = 4096;
-
-	n = prwb(part->filename, part->fd, isread, part->offset+offset,
-		buf, count, blocksize);
+	buf = buf0;
+	count = count0;
+	offset = offset0;
+	while(count > 0){
+		opsize = count;
+		if(opsize > Maxxfer)
+			opsize = Maxxfer;
+		if(isread)
+			n = pread(part->fd, buf, opsize, offset);
+		else
+			n = pwrite(part->fd, buf, opsize, offset);
+		if(n <= 0){
+			seterr(EAdmin, "%s %s offset 0x%llux count %ud buf %p returned %d: %r",
+				isread ? "read" : "write", part->filename, offset, opsize, buf, n);
+			return -1;
+		}
+		offset += n;
+		count -= n;
+		buf += n;
+	}
 
-	return n;
+	return count0;
 }
 
 int
@@ -285,4 +247,3 @@ readfile(char *name)
 	freepart(p);
 	return b;
 }
-

sys/src/cmd/venti/srv/stats.c

@@ -60,6 +60,9 @@ Statdesc statdesc[NStat] =
 	{ "index cache flushes", },
 	{ "index cache stalls", },
 	{ "index cache read time", },
+	{ "index cache lookups" },
+	{ "index cache summary hits" },
+	{ "index cache summary prefetches" },
 
 	{ "bloom filter hits", },
 	{ "bloom filter misses", },
@@ -81,6 +84,9 @@ Statdesc statdesc[NStat] =
 
 	{ "sum reads", },
 	{ "sum read bytes", },
+	
+	{ "cig loads" },
+	{ "cig load time" },
 };
 
 QLock statslock;

sys/src/cmd/venti/srv/syncarena.c

@@ -25,7 +25,7 @@ clumpinfocmp(ClumpInfo *c, ClumpInfo *d)
  * returns 0 if ok, flags if error occurred
  */
 int
-syncarena(Arena *arena, u64int start, u32int n, int zok, int fix)
+syncarena(Arena *arena, u32int n, int zok, int fix)
 {
 	ZBlock *lump;
 	Clump cl;
@@ -53,7 +53,7 @@ syncarena(Arena *arena, u64int start, u32int n, int zok, int fix)
 			fprint(2, "%s: illegal clump magic number=%#8.8ux at clump=%d\n", arena->name, magic, clump);
 			/* err |= SyncDataErr; */
 			if(fix && writeclumpmagic(arena, aa, ClumpFreeMagic) < 0){
-				fprint(2, "can't write corrected clump free magic: %r");
+				fprint(2, "%s: can't write corrected clump free magic: %r", arena->name);
 				err |= SyncFixErr;
 			}
 			break;
@@ -136,9 +136,8 @@ syncarena(Arena *arena, u64int start, u32int n, int zok, int fix)
 	|| cclumps != arena->memstats.cclumps
 	|| uncsize != arena->memstats.uncsize){
 		err |= SyncHeader;
-		fprint(2, "arena %s: start=%lld fix=%d flush=%d %lld->%lld %ud->%ud %ud->%ud %lld->%lld\n",
+		fprint(2, "arena %s: fix=%d flush=%d %lld->%lld %ud->%ud %ud->%ud %lld->%lld\n",
 			arena->name,
-			start,
 			fix,
 			flush,
 			used, arena->memstats.used,

sys/src/cmd/venti/srv/syncindex.c

@@ -6,7 +6,7 @@ static	int	verbose;
 void
 usage(void)
 {
-	fprint(2, "usage: syncindex [-fv] [-B blockcachesize] config\n");
+	fprint(2, "usage: syncindex [-v] [-B blockcachesize] config\n");
 	threadexitsall("usage");
 }
 
@@ -16,9 +16,7 @@ void
 threadmain(int argc, char *argv[])
 {
 	u32int bcmem, icmem;
-	int fix;
 
-	fix = 0;
 	bcmem = 0;
 	icmem = 0;
 	ARGBEGIN{
@@ -28,9 +26,6 @@ threadmain(int argc, char *argv[])
 	case 'I':
 		icmem = unittoull(EARGF(usage()));
 		break;
-	case 'f':
-		fix++;
-		break;
 	case 'v':
 		verbose++;
 		break;
@@ -39,9 +34,6 @@ threadmain(int argc, char *argv[])
 		break;
 	}ARGEND
 
-	if(!fix)
-		readonly = 1;
-
 	if(argc != 1)
 		usage();
 
@@ -56,21 +48,17 @@ threadmain(int argc, char *argv[])
 	if(0) fprint(2, "initialize %d bytes of disk block cache\n", bcmem);
 	initdcache(bcmem);
 	initlumpcache(1*1024*1024, 1024/8);
-	icmem = u64log2(icmem / (sizeof(IEntry)+sizeof(IEntry*)) / ICacheDepth);
-	if(icmem < 4)
-		icmem = 4;
-	if(1) fprint(2, "initialize %d bytes of index cache for %d index entries\n",
-		(sizeof(IEntry)+sizeof(IEntry*)) * (1 << icmem) * ICacheDepth,
-		(1 << icmem) * ICacheDepth);
-	initicache(icmem, ICacheDepth);
+	initicache(icmem);
 	initicachewrite();
 	if(mainindex->bloom)
 		startbloomproc(mainindex->bloom);
 
 	if(verbose)
 		printindex(2, mainindex);
-	if(syncindex(mainindex, fix, 1, 0) < 0)
+	if(syncindex(mainindex) < 0)
 		sysfatal("failed to sync index=%s: %r\n", mainindex->name);
+	flushicache();
+	flushdcache();
 
 	threadexitsall(0);
 }

sys/src/cmd/venti/srv/syncindex0.c

@@ -2,184 +2,92 @@
 #include "dat.h"
 #include "fns.h"
 
-enum
+static int
+syncarenaindex(Arena *arena, u64int a0)
 {
-	ClumpChunks	= 32*1024
-};
-
-static int missing, wrong;
-
-/*
- * shell sort is plenty good enough
- * because we're going to do a bunch of disk i/o's
- */
-static void
-sortclumpinfo(ClumpInfo *ci, int *s, int n)
-{
-	int i, j, m, t;
-
-	for(m = (n + 3) / 5; m > 0; m = (m + 1) / 3){
-		for(i = n - m; i-- > 0;){
-			for(j = i + m; j < n; j += m){
-				if(memcmp(ci[s[j - m]].score, ci[s[j]].score, VtScoreSize) <= 0)
-					break;
-				t = s[j];
-				s[j] = s[j - m];
-				s[j - m] = t;
-			}
-		}
-	}
-}
-
-int
-syncarenaindex(Index *ix, Arena *arena, u32int clump, u64int a, int fix, int *pflush, int check)
-{
-	Packet *pack;
-	IEntry ie;
+	int ok;
+	u32int clump;
+	u64int a;
+	ClumpInfo ci;
 	IAddr ia;
-	ClumpInfo *ci, *cis;
-	u64int *addrs;
-	int i, n, ok, *s, flush;
-
-	trace(TraceProc, "syncarenaindex enter");
+	AState as;
+	
+	if(arena->diskstats.clumps == arena->memstats.clumps)
+		return 0;
+	
+	memset(&as, 0, sizeof as);
+	as.arena = arena;
+	as.stats = arena->diskstats;
 
-	flush = 0;
-	cis = MKN(ClumpInfo, ClumpChunks);
-	addrs = MKN(u64int, ClumpChunks);
-	s = MKN(int, ClumpChunks);
 	ok = 0;
-	for(; clump < arena->memstats.clumps; clump += n){
-		n = ClumpChunks;
-		if(n > arena->memstats.clumps - clump)
-			n = arena->memstats.clumps - clump;
-		n = readclumpinfos(arena, clump, cis, n);
-		if(n <= 0){
-			fprint(2, "arena directory read failed\n");
+	a = a0 + arena->diskstats.used;
+	for(clump=arena->diskstats.clumps; clump < arena->memstats.clumps; clump++){
+		if(readclumpinfo(arena, clump, &ci) < 0){
+			fprint(2, "%s: clump %d: cannot read clumpinfo\n",
+				arena->name, clump);
 			ok = -1;
 			break;
 		}
 
-		for(i = 0; i < n; i++){
-			addrs[i] = a;
-			a += cis[i].size + ClumpSize;
-			s[i] = i;
-		}
-
-		sortclumpinfo(cis, s, n);
+		ia.type = ci.type;
+		ia.size = ci.uncsize;
+		ia.addr = a;
+		ia.blocks = (ClumpSize + ci.size + (1 << ABlockLog) - 1) >> ABlockLog;
+		a += ClumpSize + ci.size;
 
-		for(i = 0; i < n; i++){
-			ci = &cis[s[i]];
-			ia.type = ci->type;
-			ia.size = ci->uncsize;
-			ia.addr = addrs[s[i]];
-			ia.blocks = (ci->size + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog;
-
-			if(!check)
-				goto Add;
-			if(loadientry(ix, ci->score, ci->type, &ie) < 0){
-				trace(TraceProc, "syncarenaindex missing block %V.%d", ci->score, ci->type);
-				missing++;
-			if(0)	fprint(2, "missing block type=%d score=%V\n", ci->type, ci->score);
-			}else if(iaddrcmp(&ia, &ie.ia) != 0){
-				trace(TraceProc, "syncarenaindex mismatched entry");
-				fprint(2, "\nmismatched index entry and clump at %d\n", clump + i);
-				fprint(2, "\tclump: type=%d size=%d blocks=%d addr=%lld\n", ia.type, ia.size, ia.blocks, ia.addr);
-				fprint(2, "\tindex: type=%d size=%d block=%d addr=%lld\n", ie.ia.type, ie.ia.size, ie.ia.blocks, ie.ia.addr);
-				pack = readlump(ie.score, ie.ia.type, ie.ia.size, nil);
-				packetfree(pack);
-				if(pack != nil){
-					fprint(2, "duplicated lump\n");
-					continue;
-				}
-				wrong++;
-			}else
-				continue;
-		Add:
-			if(!fix){
-				ok = -1;
-				continue;
-			}
-			flush = 1;
-			trace(TraceProc, "syncarenaindex insert %V", ci->score);
-			insertscore(ci->score, &ia, 1);
-		}
-
-		if(0 && clump / 1000 != (clump + n) / 1000)
-			fprint(2, ".");
-	}
-	free(cis);
-	free(addrs);
-	free(s);
-	if(flush){
-		flushdcache();
-		*pflush = 1;
+		as.stats.used += ClumpSize + ci.size;
+		as.stats.uncsize += ia.size;
+		as.stats.clumps++;
+		if(ci.uncsize > ci.size)
+			as.stats.cclumps++;
+		as.aa = a;
+		insertscore(ci.score, &ia, IEDirty, &as);
 	}
+	flushdcache();
 	return ok;
 }
 
 int
-syncindex(Index *ix, int fix, int mustflush, int check)
+syncindex(Index *ix)
 {
 	Arena *arena;
-	AState as;
-	u64int a;
-	int i, e, e1, ok, ok1, flush;
+	int i, e, e1, ok;
 
 	ok = 0;
-	flush = 0;
 	for(i = 0; i < ix->narenas; i++){
 		trace(TraceProc, "syncindex start %d", i);
 		arena = ix->arenas[i];
-		/*
-		 * Syncarena will scan through the arena looking for blocks
-		 * that have been forgotten.  It will update arena->memstats.used,
-		 * so save the currenct copy as the place to start the 
-		 * syncarenaindex scan.
-		 */
-		a = arena->memstats.used;
-		e = syncarena(arena, ix->amap[i].start, TWID32, fix, fix);
+		e = syncarena(arena, TWID32, 1, 1);
 		e1 = e;
-		if(fix)
-			e1 &= ~(SyncHeader|SyncCIZero|SyncCIErr);
-		if(e1 == SyncHeader)
+		e1 &= ~(SyncHeader|SyncCIZero|SyncCIErr);
+		if(e & SyncHeader)
 			fprint(2, "arena %s: header is out-of-date\n", arena->name);
-		if(e1)
+		if(e1){
+			fprint(2, "arena %s: %x\n", arena->name, e1);
 			ok = -1;
-		else{
-			/*
-			 * use diskstats not memstats here, because diskstats
-			 * is what has been indexed; memstats is what has 
-			 * made it to disk (confusing names).
-			 */
-			ok1 = syncarenaindex(ix, arena,
-					arena->diskstats.clumps,
-					ix->amap[i].start + arena->diskstats.used,
-					fix, &flush, check);
-			if(ok1 < 0)
-				fprint(2, "syncarenaindex: %r\n");
-			if(fix && ok1==0 && (e & SyncHeader) && wbarena(arena) < 0)
-				fprint(2, "arena=%s header write failed: %r\n", arena->name);
-			ok |= ok1;
+			continue;
+		}
+		flushdcache();
+		
+		if(arena->memstats.clumps == arena->diskstats.clumps)
+			continue;
+		
+		fprint(2, "%T %s: indexing %d clumps...\n",
+			arena->name,
+			arena->memstats.clumps - arena->diskstats.clumps);
 
-			as.arena = arena;
-			as.aa = ix->amap[i].start + arena->memstats.used;
-			as.stats = arena->memstats;
-			setdcachestate(&as);
+		if(syncarenaindex(arena, ix->amap[i].start) < 0){
+			fprint(2, "arena %s: syncarenaindex: %r\n", arena->name);
+			ok = -1;
+			continue;
+		}
+		if(wbarena(arena) < 0){
+			fprint(2, "arena %s: wbarena: %r\n", arena->name);
+			ok = -1;
+			continue;
 		}
-	}
-	if(missing || wrong)
-		fprint(2, "syncindex: %d missing entries, %d wrong entries (flush=%d)\n", missing, wrong, flush);
-	if(fix && wbindex(ix) < 0){
-		fprint(2, "can't write back index header for %s: %r\n", ix->name);
-		return -1;
-	}
-	if(fix && flush){
 		flushdcache();
-		if(mustflush){
-			flushicache();
-			flushdcache();
-		}else
-			kickicache();
+		delaykickicache();
 	}
 	return ok;
 }

sys/src/cmd/venti/srv/venti.c

@@ -106,9 +106,6 @@ threadmain(int argc, char *argv[])
 	if(configfile == nil)
 		configfile = "venti.conf";
 
-	if(initarenasum() < 0)
-		fprint(2, "warning: can't initialize arena summing process: %r");
-
 	fprint(2, "conf...");
 	if(initventi(configfile, &config) < 0)
 		sysfatal("can't init server: %r");
@@ -146,13 +143,7 @@ threadmain(int argc, char *argv[])
 		mem, mem / (8 * 1024));
 	initlumpcache(mem, mem / (8 * 1024));
 
-	icmem = u64log2(icmem / (sizeof(IEntry)+sizeof(IEntry*)) / ICacheDepth);
-	if(icmem < 4)
-		icmem = 4;
-	if(0) fprint(2, "initialize %d bytes of index cache for %d index entries\n",
-		(sizeof(IEntry)+sizeof(IEntry*)) * (1 << icmem) * ICacheDepth,
-		(1 << icmem) * ICacheDepth);
-	initicache(icmem, ICacheDepth);
+	initicache(icmem);
 	initicachewrite();
 
 	/*
@@ -170,7 +161,7 @@ threadmain(int argc, char *argv[])
 		startbloomproc(mainindex->bloom);
 
 	fprint(2, "sync...");
-	if(!readonly && syncindex(mainindex, 1, 0, 0) < 0)
+	if(!readonly && syncindex(mainindex) < 0)
 		sysfatal("can't sync server: %r");
 
 	if(!readonly && queuewrites){
@@ -182,6 +173,9 @@ threadmain(int argc, char *argv[])
 		}
 	}
 
+	if(initarenasum() < 0)
+		fprint(2, "warning: can't initialize arena summing process: %r");
+
 	fprint(2, "announce %s...", vaddr);
 	ventisrv = vtlisten(vaddr);
 	if(ventisrv == nil)

sys/src/cmd/venti/srv/verifyarena.c

@@ -7,6 +7,7 @@ static int	fd;
 static uchar	*data;
 static int	blocksize;
 static int	sleepms;
+static vlong offset0;
 
 void
 usage(void)
@@ -22,7 +23,7 @@ preadblock(uchar *buf, int n, vlong off)
 
 	for(nr = 0; nr < n; nr += m){
 		m = n - nr;
-		m = pread(fd, &buf[nr], m, off+nr);
+		m = pread(fd, &buf[nr], m, offset0+off+nr);
 		if(m <= 0){
 			if(m == 0)
 				werrstr("early eof");
@@ -175,7 +176,8 @@ threadmain(int argc, char *argv[])
 	char *p, *q, *table, *f[10], line[256];
 	vlong start, stop;
 	ArenaPart ap;
-	
+	Part *part;
+
 	needzeroscore();
 	ventifmtinstall();
 	blocksize = MaxIoSize;
@@ -201,8 +203,10 @@ threadmain(int argc, char *argv[])
 		threadexitsall(nil);
 	}
 	
-	if((fd = open(argv[0], OREAD)) < 0)
-		sysfatal("open %s: %r", argv[0]);
+	if((part = initpart(argv[0], OREAD)) == nil)
+		sysfatal("open partition %s: %r", argv[0]);
+	fd = part->fd;
+	offset0 = part->offset;
 
 	if(preadblock(data, 8192, PartBlank) < 0)
 		sysfatal("read arena part header: %r");
@@ -249,7 +253,7 @@ threadmain(int argc, char *argv[])
 				fprint(2, "%T %s: bad start,stop %lld,%lld\n", f[0], stop, start);
 				continue;
 			}
-			if(seek(fd, start, 0) < 0)
+			if(seek(fd, offset0+start, 0) < 0)
 				fprint(2, "%T %s: seek to start: %r\n", f[0]);
 			verifyarena(f[0], stop - start);
 		}

sys/src/libhttpd/parse.c

@@ -110,7 +110,7 @@ static	int	lexbase64(Hlex*);
 static	ulong	digtoul(char *s, char **e);
 
 /*
- * flush an clean up junk from a request
+ * flush and clean up junk from a request
  */
 void
 hreqcleanup(HConnect *c)
@@ -753,7 +753,7 @@ lexbase64(Hlex *h)
 	lex1(h, 1);
 
 	while((c = getc(h)) >= 0){
-		if(!(isascii(c) && isalnum(c) || c == '+' || c == '/')){
+		if(!isalnum(c) && c != '+' && c != '/'){
 			ungetc(h);
 			break;
 		}