deblob-main 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311
  1. #! /bin/sh
  2. # Copyright (C) 2008-2016 Alexandre Oliva <lxoliva@fsfla.org>
  3. # This program is part of GNU Linux-libre, a GNU project that
  4. # publishes scripts to clean up Linux so as to make it suitable for
  5. # use in the GNU Project and in Free System Distributions.
  6. # This program is free software; you can redistribute it and/or modify
  7. # it under the terms of the GNU General Public License as published by
  8. # the Free Software Foundation; either version 2 of the License, or
  9. # (at your option) any later version.
  10. # This program is distributed in the hope that it will be useful, but
  11. # WITHOUT ANY WARRANTY; without even the implied warranty of
  12. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  13. # General Public License for more details.
  14. # You should have received a copy of the GNU General Public License
  15. # along with this program; if not, write to the Free Software
  16. # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
  17. # USA
  18. # deblob-main - prepare a GNU Linux-libre tarball out of a non-libre
  19. # Linux tarball. It expects the Linux release (mver, say 3.0) as the
  20. # first argument, the gnu sub-release (extra) as the second optional
  21. # argument, and the patch release (sver, say .13) as an optional third
  22. # argument. mver and sver are pasted together to form kver.
  23. # linux-$kver.tar.bz2 and deblob-$mver must exist in the current
  24. # directory, and the line that sets kver and extra in deblob-$mver
  25. # must match mver and extra.
  26. # The resulting tarball is put in linux-libre-$kver-gnu$extra.tar.bz2.
  27. # An uncompressed xdelta that produces linux-libre-$kver-gnu$extra.tar
  28. # out of linux-$kver.tar is put in linux-libre-$kver-gnu$extra.xdelta.
  29. # This xdelta can be distributed to enable third parties to easily
  30. # reconstruct the binary tarball starting out of sources downloaded
  31. # from kernel.org, but without distributing non-Free Software
  32. # yourself, because xdelta (unlike patches) is not reversible: the
  33. # removed bits are not present in it at all.
  34. # xdelta version 3 uses different command line syntax, and it switched
  35. # to the more standardized but less efficient vcdiff file format.
  36. # This script will also produce a vcdiff file if xdelta3 is present,
  37. # and it expects the xdelta program to use the version 1 syntax.
  38. # To enable you to check the differences between the tarballs, a patch
  39. # file is generated in linux-libre-$kver-gnu$extra.patch. This patch
  40. # file contains the non-Free blobs, even though in reversed form, so
  41. # its distribution is discouraged.
  42. # The tar files and binary deltas are finally compressed with bzip2,
  43. # and optionally with lzip and xz too, if the compressors are
  44. # available.
  45. # At the end, the script attempts to generate a digital signature for
  46. # the newly-created tarball. This is the last thing the script does,
  47. # so interrupting it at that point to skip the signing won't fail to
  48. # do anything else.
  49. # It is safe to interrupt the script at any other point. When it gets
  50. # a ^C (other than during signing), it starts cleaning up all of its
  51. # temporary and output files. If you insist, it may leave junk
  52. # behind, and then it will refuse to run again before you clean it up
  53. # by hand. It takes extra care to avoid overwriting useful files.
  54. # If deblob-$mver finds any unexpected situation, it will error out,
  55. # and then deblob-main will quit. Pass --force to deblob-main, before
  56. # any other argument, for deblob-main to ignore any such situations.
  57. case $1 in
  58. --force) force=--force; shift;;
  59. *) force=;;
  60. esac
  61. # We don't want e.g. diff output translations to affect us.
  62. LC_ALL=C; export LC_ALL
  63. LANGUAGE=C; export LANGUAGE
  64. mver=$1 extra=$2 sver=$3
  65. kver=$mver$sver gnu=gnu$extra
  66. deblob= dir=`echo "$0" | sed 's,[^/]*$,,;s,^$,.,;s,/*$,,'`
  67. if test -f linux-$kver.tar; then
  68. zext=tar zcmd=
  69. elif test -f linux-$kver.tar.bz2; then
  70. zext=tar.bz2 zcmd=bunzip2
  71. elif test -f linux-$kver.tar.xz; then
  72. zext=tar.xz zcmd=unxz
  73. elif test -f linux-$kver.tar.lz; then
  74. zext=tar.lz zcmd="lzip -d"
  75. elif test -f linux-$kver.tar.gz; then
  76. zext=tar.gz zcmd=gunzip
  77. elif test -f linux-$kver.tgz; then
  78. zext=tgz zcmd=gunzip
  79. else
  80. echo linux-$kver.tar not found, tried .bz2, .xz, .lz, .gz and .tgz too >&2
  81. exit 1
  82. fi
  83. if test -f deblob-$mver; then
  84. deblob=deblob-$mver
  85. elif test -f deblob; then
  86. deblob=deblob
  87. elif test -f $dir/deblob-$mver; then
  88. cp $dir/deblob-$mver deblob
  89. deblob=deblob
  90. else
  91. echo deblob does not exist >&2
  92. exit 1
  93. fi
  94. x1="kver=$mver extra=$extra"
  95. x2=`grep "^kver=[^ ]* extra=" $deblob`
  96. if test "$x1" = "$x2"; then
  97. :
  98. else
  99. echo deblob script does not match command-line arguments >&2
  100. echo expected: $x1 >&2
  101. echo found : $x2 >&2
  102. exit 1
  103. fi
  104. cleanup=
  105. for f in \
  106. linux-libre-$kver-$gnu.tar.bz2 \
  107. linux-libre-$kver-$gnu.tar.bz2.asc \
  108. linux-libre-$kver-$gnu.tar.bz2.sign \
  109. linux-libre-$kver-$gnu.tar.xz \
  110. linux-libre-$kver-$gnu.tar.xz.asc \
  111. linux-libre-$kver-$gnu.tar.xz.sign \
  112. linux-libre-$kver-$gnu.tar.lz \
  113. linux-libre-$kver-$gnu.tar.lz.asc \
  114. linux-libre-$kver-$gnu.tar.lz.sign \
  115. linux-libre-$kver-$gnu.tar \
  116. linux-libre-$kver-$gnu.tar.asc \
  117. linux-libre-$kver-$gnu.tar.sign \
  118. linux-libre-$kver-$gnu.patch \
  119. linux-libre-$kver-$gnu.log \
  120. linux-libre-$kver-$gnu.vcdiff \
  121. linux-libre-$kver-$gnu.vcdiff.bz2 \
  122. linux-libre-$kver-$gnu.vcdiff.bz2.asc \
  123. linux-libre-$kver-$gnu.vcdiff.bz2.sign \
  124. linux-libre-$kver-$gnu.vcdiff.xz \
  125. linux-libre-$kver-$gnu.vcdiff.xz.asc \
  126. linux-libre-$kver-$gnu.vcdiff.xz.sign \
  127. linux-libre-$kver-$gnu.vcdiff.lz \
  128. linux-libre-$kver-$gnu.vcdiff.lz.asc \
  129. linux-libre-$kver-$gnu.vcdiff.lz.sign \
  130. linux-libre-$kver-$gnu.xdelta \
  131. linux-libre-$kver-$gnu.xdelta.bz2 \
  132. linux-libre-$kver-$gnu.xdelta.bz2.asc \
  133. linux-libre-$kver-$gnu.xdelta.bz2.sign \
  134. linux-libre-$kver-$gnu.xdelta.xz \
  135. linux-libre-$kver-$gnu.xdelta.xz.asc \
  136. linux-libre-$kver-$gnu.xdelta.xz.sign \
  137. linux-libre-$kver-$gnu.xdelta.lz \
  138. linux-libre-$kver-$gnu.xdelta.lz.asc \
  139. linux-libre-$kver-$gnu.xdelta.lz.sign \
  140. ; do
  141. if test -f $f; then
  142. echo $f already exists >&2
  143. exit 1
  144. fi
  145. cleanup="$cleanup $f"
  146. done
  147. for d in \
  148. linux-$kver \
  149. linux-libre-$kver-$gnu \
  150. orig-linux-$kver \
  151. ; do
  152. if test -d $d; then
  153. echo $d already exists >&2
  154. exit 1
  155. fi
  156. cleanup="$cleanup $d"
  157. done
  158. if test -f $dir/deblob-$kver; then
  159. if cmp $dir/deblob-$kver $deblob; then
  160. :
  161. else
  162. echo $dir/deblob-$kver and $deblob are different >&2
  163. exit 1
  164. fi
  165. fi
  166. if test ! -f deblob-check; then
  167. if test -f $dir/deblob-check; then
  168. cp $dir/deblob-check deblob-check
  169. fi
  170. else
  171. if test -f $dir/deblob-check; then
  172. if cmp $dir/deblob-check deblob-check; then
  173. :
  174. else
  175. echo $dir/deblob-check and deblob-check are different >&2
  176. exit 1
  177. fi
  178. fi
  179. fi
  180. trap 'status=$?; echo cleaning up...; rm -rf $cleanup; (exit $status); exit' 0 1 2 15
  181. set -e
  182. if test -n "$zcmd"; then
  183. echo Uncompressing linux-$kver.$zext into linux-$kver.tar
  184. rm -rf linux-$kver.tar
  185. cleanup="$cleanup linux-$kver.tar"
  186. $zcmd < linux-$kver.$zext > linux-$kver.tar
  187. fi
  188. echo Extracting linux-$kver.tar into linux-$kver
  189. rm -rf linux-$kver
  190. tar -xf linux-$kver.tar
  191. rm -rf linux-libre-$kver-$gnu linux-libre-$kver-$gnu.tar
  192. echo Copying linux-$kver to linux-libre-$kver-$gnu
  193. cp linux-$kver.tar linux-libre-$kver-$gnu.tar
  194. cp -lR linux-$kver/. linux-libre-$kver-$gnu
  195. rm -f linux-libre-$kver-$gnu.log linux-libre-$kver-$gnu.log.tmp
  196. echo Deblobbing within linux-libre-$kver-$gnu, saving output to linux-libre-$kver-$gnu.log
  197. # We can't just pipe deblob into tee, for then we fail to detect
  198. # error conditions. Use file renaming to tell whether we succeeded.
  199. if (cd linux-libre-$kver-$gnu && /bin/sh ../$deblob $force) 2>&1; then
  200. mv linux-libre-$kver-$gnu.log.tmp linux-libre-$kver-$gnu.log
  201. fi | tee linux-libre-$kver-$gnu.log.tmp
  202. if test ! -f linux-libre-$kver-$gnu.log; then
  203. mv linux-libre-$kver-$gnu.log.tmp linux-libre-$kver-$gnu.log
  204. echo $deblob failed, aborting >&2
  205. exit 1
  206. fi
  207. rm -f linux-libre-$kver-$gnu.patch
  208. # Do not copy these scripts for now, deblob-check regards itself as a blob.
  209. # cp -p $0 $deblob deblob-check linux-libre-$kver-$gnu
  210. echo Generating linux-libre-$kver-$gnu.patch
  211. diff -druN linux-$kver linux-libre-$kver-$gnu > linux-libre-$kver-$gnu.patch || :
  212. echo Removing removed or modified files from linux-libre-$kver-$gnu.tar
  213. diff -rq linux-$kver linux-libre-$kver-$gnu |
  214. sed -n "
  215. s,^Only in \\(linux-$kver\\(/.*\\)\\?\\): \\(.*\\),\1/\3,p;
  216. s,^Files \\(linux-$kver\\)/\\(.*\\) and linux-libre-$kver-$gnu/\\2 differ,\\1/\\2,p;
  217. " |
  218. xargs tar --delete -f linux-libre-$kver-$gnu.tar
  219. echo Adding modified or added files to linux-libre-$kver-$gnu.tar
  220. rm -rf orig-linux-$kver
  221. mv linux-$kver orig-linux-$kver
  222. mv linux-libre-$kver-$gnu linux-$kver
  223. diff -rq orig-linux-$kver linux-$kver |
  224. sed -n "
  225. s,^Files orig-\\(linux-$kver/.*\\) and \\1 differ,\\1,p;
  226. s,^Only in \\(linux-$kver\\(/.*\\)\\?\\): \\(.*\\),\\1/\\3,p;
  227. " |
  228. xargs tar --append -f linux-libre-$kver-$gnu.tar
  229. echo Wiping out extracted trees
  230. rm -rf linux-$kver orig-linux-$kver
  231. echo Creating vcdiff between linux-$kver.tar and linux-libre-$kver-$gnu.tar
  232. xdelta3 -e -9 -S djw -s linux-$kver.tar linux-libre-$kver-$gnu.tar linux-libre-$kver-$gnu.vcdiff || : # don't fail if xdelta3 is not present
  233. echo Creating xdelta between linux-$kver.tar and linux-libre-$kver-$gnu.tar
  234. xdelta delta -0 linux-$kver.tar linux-libre-$kver-$gnu.tar linux-libre-$kver-$gnu.xdelta || : # xdelta returns nonzero on success
  235. cleanup="linux-libre-$kver-$gnu.tar linux-libre-$kver-$gnu.xdelta"
  236. echo Compressing binary deltas and linux-libre-$kver-$gnu.tar
  237. rm -f linux-$kver.tar
  238. if test -f linux-libre-$kver-$gnu.xdelta; then
  239. bzip2 -k9 linux-libre-$kver-$gnu.xdelta
  240. xz -k9 linux-libre-$kver-$gnu.xdelta || :
  241. lzip -k9s64MiB linux-libre-$kver-$gnu.xdelta || :
  242. fi
  243. bzip2 -k9 linux-libre-$kver-$gnu.tar
  244. xz -k9 linux-libre-$kver-$gnu.tar || :
  245. lzip -k9s64MiB linux-libre-$kver-$gnu.tar || :
  246. echo Done except for signing, feel free to interrupt
  247. for f in \
  248. linux-libre-$kver-$gnu.tar \
  249. linux-libre-$kver-$gnu.tar.bz2 \
  250. linux-libre-$kver-$gnu.tar.xz \
  251. linux-libre-$kver-$gnu.tar.lz \
  252. linux-libre-$kver-$gnu.vcdiff \
  253. linux-libre-$kver-$gnu.xdelta \
  254. linux-libre-$kver-$gnu.xdelta.bz2 \
  255. linux-libre-$kver-$gnu.xdelta.xz \
  256. linux-libre-$kver-$gnu.xdelta.lz \
  257. ; do
  258. if test -f $f; then
  259. gpg -a --detach-sign $f
  260. mv $f.asc $f.sign
  261. fi
  262. done
  263. rm -f $cleanup
  264. cleanup=
  265. trap 'status=$?; (exit $status); exit' 0 1 2 15
  266. echo All set, please review linux-libre-$kver-$gnu.patch
  267. exit 0