build.go 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535
  1. // Build builds code as directed by json files.
  2. // We slurp in the JSON, and recursively process includes.
  3. // At the end, we issue a single cc command for all the files.
  4. // Compilers are fast.
  5. //
  6. // ENVIRONMENT
  7. //
  8. // Needed: HARVEY, ARCH
  9. //
  10. // HARVEY should point to a harvey root.
  11. // A best-effort to autodetect the harvey root is made if not explicitly set.
  12. //
  13. // Optional: CC, AR, LD, RANLIB, STRIP, SH, TOOLPREFIX
  14. //
  15. // These all control how the needed tools are found.
  16. //
  17. package main
  18. import (
  19. "encoding/json"
  20. "flag"
  21. "fmt"
  22. "io/ioutil"
  23. "log"
  24. "os"
  25. "os/exec"
  26. "path"
  27. "path/filepath"
  28. "regexp"
  29. "strings"
  30. )
  31. type kernconfig struct {
  32. Code []string
  33. Dev []string
  34. Ip []string
  35. Link []string
  36. Sd []string
  37. Uart []string
  38. VGA []string
  39. }
  40. type kernel struct {
  41. Systab string
  42. Config kernconfig
  43. Ramfiles map[string]string
  44. }
  45. type build struct {
  46. // jsons is unexported so can not be set in a .json file
  47. jsons map[string]bool
  48. path string
  49. name string
  50. // Projects name a whole subproject which is built independently of
  51. // this one. We'll need to be able to use environment variables at some point.
  52. Projects []string
  53. Pre []string
  54. Post []string
  55. Cflags []string
  56. Oflags []string
  57. Include []string
  58. SourceFiles []string
  59. ObjectFiles []string
  60. Libs []string
  61. Env []string
  62. ToolOpts map[string][]string
  63. // cmd's
  64. SourceFilesCmd []string
  65. // Targets.
  66. Program string
  67. Library string
  68. Install string // where to place the resulting binary/lib
  69. Kernel *kernel
  70. }
  71. type buildfile map[string]build
  72. // UnmarshalJSON works like the stdlib unmarshal would, except it adjusts all
  73. // paths.
  74. func (bf *buildfile) UnmarshalJSON(s []byte) error {
  75. r := make(map[string]build)
  76. if err := json.Unmarshal(s, &r); err != nil {
  77. return err
  78. }
  79. for k, b := range r {
  80. // we're getting a copy of the struct, remember.
  81. b.jsons = make(map[string]bool)
  82. b.Projects = adjust(b.Projects)
  83. b.Libs = adjust(b.Libs)
  84. b.Cflags = adjust(b.Cflags)
  85. b.ObjectFiles = adjust(b.ObjectFiles)
  86. b.Include = adjust(b.Include)
  87. b.Install = fromRoot(b.Install)
  88. for i, e := range b.Env {
  89. b.Env[i] = os.ExpandEnv(e)
  90. }
  91. r[k] = b
  92. }
  93. *bf = r
  94. return nil
  95. }
  96. var (
  97. cwd string
  98. harvey string
  99. regexpAll = []*regexp.Regexp{regexp.MustCompile(".")}
  100. // findTools looks at all env vars and absolutizes these paths
  101. // also respects TOOLPREFIX
  102. tools = map[string]string{
  103. "cc": "gcc",
  104. "ar": "ar",
  105. "ld": "ld",
  106. "ranlib": "ranlib",
  107. "strip": "strip",
  108. "sh": "sh",
  109. }
  110. arch = map[string]bool{
  111. "amd64": true,
  112. "riscv": true,
  113. }
  114. debugPrint = flag.Bool("debug", false, "enable debug prints")
  115. shellhack = flag.Bool("shellhack", false, "spawn every command in a shell (forced on if LD_PRELOAD is set)")
  116. )
  117. func debug(fmt string, s ...interface{}) {
  118. if *debugPrint {
  119. log.Printf(fmt, s...)
  120. }
  121. }
  122. // fail with message, if err is not nil
  123. func failOn(err error) {
  124. if err != nil {
  125. log.Fatalf("%v", err)
  126. }
  127. }
  128. func adjust(s []string) []string {
  129. for i, v := range s {
  130. s[i] = fromRoot(v)
  131. }
  132. return s
  133. }
  134. // return the given absolute path as an absolute path rooted at the harvey tree.
  135. func fromRoot(p string) string {
  136. p = os.ExpandEnv(p)
  137. if path.IsAbs(p) {
  138. return path.Join(harvey, p)
  139. }
  140. return p
  141. }
  142. // Sh sends cmd to a shell. It's needed to enable $LD_PRELOAD tricks,
  143. // see https://github.com/Harvey-OS/harvey/issues/8#issuecomment-131235178
  144. func sh(cmd *exec.Cmd) {
  145. shell := exec.Command(tools["sh"])
  146. shell.Env = cmd.Env
  147. if cmd.Args[0] == tools["sh"] && cmd.Args[1] == "-c" {
  148. cmd.Args = cmd.Args[2:]
  149. }
  150. commandString := strings.Join(cmd.Args, " ")
  151. if shStdin, e := shell.StdinPipe(); e == nil {
  152. go func() {
  153. defer shStdin.Close()
  154. io.WriteString(shStdin, commandString)
  155. }()
  156. } else {
  157. log.Fatalf("cannot pipe [%v] to %s: %v", commandString, tools["sh"], e)
  158. }
  159. shell.Stderr = os.Stderr
  160. shell.Stdout = os.Stdout
  161. debug("%q | sh\n", commandString)
  162. failOn(shell.Run())
  163. }
  164. func process(f, which string, b *build) {
  165. r := regexp.MustCompile(which)
  166. if b.jsons[f] {
  167. return
  168. }
  169. b.jsons[f] = true
  170. log.Printf("Including %v", f)
  171. d, err := ioutil.ReadFile(f)
  172. failOn(err)
  173. var builds buildfile
  174. failOn(json.Unmarshal(d, &builds))
  175. for n, build := range builds {
  176. log.Printf("Merging %v", n)
  177. for t,v := range build.ToolOpts {
  178. b.ToolOpts[t] = v
  179. }
  180. b.SourceFiles = append(b.SourceFiles, build.SourceFiles...)
  181. b.Cflags = append(b.Cflags, build.Cflags...)
  182. b.Oflags = append(b.Oflags, build.Oflags...)
  183. b.Pre = append(b.Pre, build.Pre...)
  184. b.Post = append(b.Post, build.Post...)
  185. b.Libs = append(b.Libs, build.Libs...)
  186. b.Projects = append(b.Projects, build.Projects...)
  187. b.Env = append(b.Env, build.Env...)
  188. b.SourceFilesCmd = append(b.SourceFilesCmd, build.SourceFilesCmd...)
  189. b.Program += build.Program
  190. b.Library += build.Library
  191. if build.Install != "" {
  192. if b.Install != "" {
  193. log.Fatalf("In file %s (target %s) included by %s (target %s): redefined Install.", f, n, build.path, build.name)
  194. }
  195. b.Install = build.Install
  196. }
  197. b.ObjectFiles = append(b.ObjectFiles, build.ObjectFiles...)
  198. // For each source file, assume we create an object file with the last char replaced
  199. // with 'o'. We can get smarter later.
  200. for _, v := range build.SourceFiles {
  201. f := path.Base(v)
  202. o := f[:len(f)-1] + "o"
  203. b.ObjectFiles = append(b.ObjectFiles, o)
  204. }
  205. for _, v := range build.Include {
  206. if !path.IsAbs(v) {
  207. wd := path.Dir(f)
  208. v = path.Join(wd, v)
  209. }
  210. include(v, b)
  211. }
  212. }
  213. }
  214. func appendIfMissing(s []string, v string) []string {
  215. for _, a := range s {
  216. if a == v {
  217. return s
  218. }
  219. }
  220. return append(s, v)
  221. }
  222. func process(f string, r []*regexp.Regexp) []build {
  223. log.Printf("Processing %v", f)
  224. var builds buildfile
  225. var results []build
  226. d, err := ioutil.ReadFile(f)
  227. failOn(err)
  228. failOn(json.Unmarshal(d, &builds))
  229. for n, build := range builds {
  230. build.name = n
  231. build.jsons = make(map[string]bool)
  232. build.ToolOpts = make(map[string][]string)
  233. skip := true
  234. for _, re := range r {
  235. if re.MatchString(build.name) {
  236. skip = false
  237. break
  238. }
  239. }
  240. if skip {
  241. continue
  242. }
  243. log.Printf("Run %v", build.name)
  244. build.jsons[f] = true
  245. build.path = path.Dir(f)
  246. // For each source file, assume we create an object file with the last char replaced
  247. // with 'o'. We can get smarter later.
  248. for _, v := range build.SourceFiles {
  249. f := path.Base(v)
  250. o := f[:len(f)-1] + "o"
  251. build.ObjectFiles = appendIfMissing(build.ObjectFiles, o)
  252. }
  253. for _, v := range build.Include {
  254. include(v, &build)
  255. }
  256. results = append(results, build)
  257. }
  258. return results
  259. }
  260. func buildkernel(b *build) {
  261. if b.Kernel == nil {
  262. return
  263. }
  264. codebuf := confcode(b.path, b.Kernel)
  265. failOn(ioutil.WriteFile(b.name+".c", codebuf, 0666))
  266. }
  267. func compile(b *build) {
  268. log.Printf("Building %s\n", b.name)
  269. // N.B. Plan 9 has a very well defined include structure, just three things:
  270. // /amd64/include, /sys/include, .
  271. args := []string{
  272. "-std=c11", "-c",
  273. "-I", fromRoot("/$ARCH/include"),
  274. "-I", fromRoot("/sys/include"),
  275. "-I", ".",
  276. }
  277. if toolOpts, ok := b.ToolOpts[tools["cc"]]; ok {
  278. args = append(args, toolOpts...)
  279. }
  280. args = append(args, b.Cflags...)
  281. if len(b.SourceFilesCmd) > 0 {
  282. for _, i := range b.SourceFilesCmd {
  283. cmd := exec.Command(tools["cc"], append(args, i)...)
  284. run(b, *shellhack, cmd)
  285. }
  286. return
  287. }
  288. args = append(args, b.SourceFiles...)
  289. cmd := exec.Command(tools["cc"], args...)
  290. run(b, *shellhack, cmd)
  291. }
  292. func link(b *build) {
  293. log.Printf("Linking %s\n", b.name)
  294. if len(b.SourceFilesCmd) > 0 {
  295. for _, n := range b.SourceFilesCmd {
  296. // Split off the last element of the file
  297. var ext = filepath.Ext(n)
  298. if len(ext) == 0 {
  299. log.Fatalf("refusing to overwrite extension-less source file %v", n)
  300. continue
  301. }
  302. n = n[:len(n)-len(ext)]
  303. f := path.Base(n)
  304. o := f[:len(f)] + ".o"
  305. args := []string{"-o", n, o}
  306. args = append(args, b.Oflags...)
  307. if toolOpts, ok := b.ToolOpts[tools["ld"]]; ok {
  308. args = append(args, toolOpts...)
  309. }
  310. args = append(args, "-L", fromRoot("/$ARCH/lib"))
  311. args = append(args, b.Libs...)
  312. run(b, *shellhack, exec.Command(tools["ld"], args...))
  313. }
  314. return
  315. }
  316. args := []string{"-o", b.Program}
  317. if toolOpts, ok := b.ToolOpts[tools["ld"]]; ok {
  318. args = append(args, toolOpts...)
  319. }
  320. args = append(args, b.ObjectFiles...)
  321. args = append(args, b.Oflags...)
  322. args = append(args, "-L", fromRoot("/$ARCH/lib"))
  323. args = append(args, b.Libs...)
  324. run(b, *shellhack, exec.Command(tools["ld"], args...))
  325. }
  326. func install(b *build) {
  327. if b.Install == "" {
  328. return
  329. }
  330. log.Printf("Installing %s\n", b.name)
  331. failOn(os.MkdirAll(b.Install, 0755))
  332. switch {
  333. case len(b.SourceFilesCmd) > 0:
  334. for _, n := range b.SourceFilesCmd {
  335. ext := filepath.Ext(n)
  336. exe := n[:len(n)-len(ext)]
  337. move(exe, b.Install)
  338. }
  339. case len(b.Program) > 0:
  340. move(b.Program, b.Install)
  341. case len(b.Library) > 0:
  342. libpath := path.Join(b.Install, b.Library)
  343. args := append([]string{"-rs", libpath}, b.ObjectFiles...)
  344. run(b, *shellhack, exec.Command(tools["ar"], args...))
  345. run(b, *shellhack, exec.Command(tools["ranlib"], libpath))
  346. }
  347. }
  348. func move(from, to string) {
  349. final := path.Join(to, from)
  350. log.Printf("move %s %s\n", from, final)
  351. _ = os.Remove(final)
  352. failOn(os.Link(from, final))
  353. failOn(os.Remove(from))
  354. }
  355. func run(b *build, pipe bool, cmd *exec.Cmd) {
  356. if b != nil {
  357. cmd.Env = append(os.Environ(), b.Env...)
  358. }
  359. cmd.Stdout = os.Stdout
  360. cmd.Stderr = os.Stderr
  361. if pipe {
  362. // Sh sends cmd to a shell. It's needed to enable $LD_PRELOAD tricks, see https://github.com/Harvey-OS/harvey/issues/8#issuecomment-131235178
  363. shell := exec.Command(tools["sh"])
  364. shell.Env = cmd.Env
  365. shell.Stderr = os.Stderr
  366. shell.Stdout = os.Stdout
  367. commandString := strings.Join(cmd.Args, " ")
  368. shStdin, err := shell.StdinPipe()
  369. if err != nil {
  370. log.Fatalf("cannot pipe [%v] to %s: %v", commandString, tools["sh"], err)
  371. }
  372. go func() {
  373. defer shStdin.Close()
  374. io.WriteString(shStdin, commandString)
  375. }()
  376. log.Printf("%q | sh\n", commandString)
  377. failOn(shell.Run())
  378. return
  379. }
  380. log.Println(strings.Join(cmd.Args, " "))
  381. failOn(cmd.Run())
  382. }
  383. func projects(b *build, r []*regexp.Regexp) {
  384. for _, v := range b.Projects {
  385. log.Printf("Doing %s\n", strings.TrimSuffix(v, ".json"))
  386. project(v, r)
  387. }
  388. }
  389. // assumes we are in the wd of the project.
  390. func project(bf string, which []*regexp.Regexp) {
  391. cwd, err := os.Getwd()
  392. failOn(err)
  393. debug("Start new project cwd is %v", cwd)
  394. defer os.Chdir(cwd)
  395. dir := path.Dir(bf)
  396. root := path.Base(bf)
  397. debug("CD to %v and build using %v", dir, root)
  398. failOn(os.Chdir(dir))
  399. builds := process(root, which)
  400. debug("Processing %v: %d target", root, len(builds))
  401. for _, b := range builds {
  402. debug("Processing %v: %v", b.name, b)
  403. projects(&b, regexpAll)
  404. for _, c := range b.Pre {
  405. // this is a hack: we just pass the command through as an exec.Cmd
  406. run(&b, true, exec.Command(c))
  407. }
  408. buildkernel(&b)
  409. if len(b.SourceFiles) > 0 || len(b.SourceFilesCmd) > 0 {
  410. compile(&b)
  411. }
  412. if b.Program != "" || len(b.SourceFilesCmd) > 0 {
  413. link(&b)
  414. }
  415. install(&b)
  416. for _, c := range b.Post {
  417. run(&b, true, exec.Command(c))
  418. }
  419. }
  420. install(b)
  421. run(b, b.Post)
  422. }
  423. func main() {
  424. // A small amount of setup is done in the paths*.go files. They are
  425. // OS-specific path setup/manipulation. "harvey" is set there and $PATH is
  426. // adjusted.
  427. var err error
  428. findTools(os.Getenv("TOOLPREFIX"))
  429. flag.Parse()
  430. cwd, err = os.Getwd()
  431. failOn(err)
  432. a := os.Getenv("ARCH")
  433. if a == "" || !arch[a] {
  434. s := []string{}
  435. for i := range arch {
  436. s = append(s, i)
  437. }
  438. log.Fatalf("You need to set the ARCH environment variable from: %v", s)
  439. }
  440. // ensure this is exported, in case we used a default value
  441. os.Setenv("HARVEY", harvey)
  442. if os.Getenv("LD_PRELOAD") != "" {
  443. log.Println("Using shellhack")
  444. *shellhack = true
  445. }
  446. // If no args, assume 'build.json'
  447. // If 1 arg, that's a dir or file name.
  448. // if two args, that's a dir and a regular expression.
  449. f := "build.json"
  450. if len(flag.Args()) > 0 {
  451. f = flag.Arg(0)
  452. }
  453. bf, err := findBuildfile(f)
  454. failOn(err)
  455. re := []*regexp.Regexp{regexp.MustCompile(".")}
  456. if len(flag.Args()) > 1 {
  457. re = re[:0]
  458. for _, r := range flag.Args()[1:] {
  459. rx, err := regexp.Compile(r)
  460. failOn(err)
  461. re = append(re, rx)
  462. }
  463. }
  464. project(bf, re)
  465. }
  466. func findTools(toolprefix string) {
  467. var err error
  468. for k, v := range tools {
  469. if x := os.Getenv(strings.ToUpper(k)); x != "" {
  470. v = x
  471. }
  472. v, err = exec.LookPath(toolprefix + v)
  473. failOn(err)
  474. tools[k] = v
  475. }
  476. }
  477. // disambiguate the buildfile argument
  478. func findBuildfile(f string) (string, error) {
  479. try := []string{
  480. f,
  481. path.Join(f, "build.json"),
  482. fromRoot(path.Join("/sys/src", f+".json")),
  483. fromRoot(path.Join("/sys/src", f, "build.json")),
  484. }
  485. for _, p := range try {
  486. if fi, err := os.Stat(p); err == nil && !fi.IsDir() {
  487. return p, nil
  488. }
  489. }
  490. return "", fmt.Errorf("unable to find buildfile (tried %s)", strings.Join(try, ", "))
  491. }