harvey/sys/src/cmd/dict/comfix.awk

# when raw index has a lot of entries like
# 1578324	problematico, a, ci, che
# apply this algorithm:
#  treat things after comma as suffixes
#  for each suffix:
#      if single letter, replace last letter
#      else search backwards for beginning of suffix
#      and if it leads to an old suffix of approximately
#      the same length, put replace that suffix
# This will still leave some commas to fix by hand
# Usage: awk -F'	' -f comfix.awk rawindex > newrawindex

NF == 2	{
		i = index($2, ",")
		if(i == 0 || length($2) == 0)
			print $0
		else {
			n = split($2, a, /,[ ]*/)
			w = a[1]
			printf "%s\t%s\n", $1, w
			for(i = 2; i <= n; i++) {
				suf = a[i]
				m = matchsuflen(w, suf)
				if(m) {
					nw = substr(w, 1, length(w)-m) suf
					printf "%s\t%s\n", $1, nw
				} else
					printf "%s\t%s\n", $1, w ", " suf
			}
		}
	}
NF != 2 {
	print $0
	}

function matchsuflen(w, suf,		wlen,suflen,c,pat,k,d)
{
	wlen = length(w)
	suflen = length(suf)
	if(suflen == 1)
		return 1
	else {
		c = substr(suf, 1, 1)
		for (k = 1; k <= wlen ; k++)
			if(substr(w, wlen-k+1, 1) == c)
				break
		if(k > wlen)
			return 0
		d = k-suflen
		if(d < 0)
			d = -d
		if(d > 3)
			return 0
		return k
	}
}