2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 import sys, os, getopt, glob, copy, os.path, re
6 sys.path.append('/usr/share/oe')
10 import itertools, optparse
12 parsespin = itertools.cycle( r'|/-\\' )
15 __build_cache_fail = []
21 __world_target = Set()
22 __ignored_dependencies = Set()
23 __depcmds = { "clean": None,
28 oefile_config_priorities = []
32 def handle_options( args ):
33 parser = optparse.OptionParser( version = "OpenEmbedded Build Infrastructure Core version %s, %%prog version %s" % ( oe.__version__, __version__ ),
34 usage = """%prog [options] [package ...]
36 Builds specified packages, expecting that the .oe files
37 it has to work from are in OEFILES
38 Default packages are all packages in OEFILES.
39 Default OEFILES are the .oe files in the current directory.""" )
41 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
42 action = "store_false", dest = "abort", default = True )
44 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
45 action = "store_true", dest = "force", default = False )
48 parser.add_option( "-c", "--cmd", help = "specify command to pass to oebuild. Valid commands are "
49 "'fetch' (fetch all sources), "
50 "'unpack' (unpack the sources), "
51 "'patch' (apply the patches), "
52 "'configure' (configure the source tree), "
53 "'compile' (compile the source tree), "
54 "'stage' (install libraries and headers needed for subsequent packages), "
55 "'install' (install libraries and executables), and"
56 "'package' (package files into the selected package format)",
57 action = "store", dest = "cmd", default = "build" )
59 parser.add_option( "-r", "--read", help = "read the specified file before oe.conf",
60 action = "append", dest = "file", default = [] )
62 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
63 action = "store_true", dest = "verbose", default = False )
65 parser.add_option( "-n", "--dry-run", help = "don't call oebuild, just go through the motions",
66 action = "store_true", dest = "dry_run", default = False )
68 parser.add_option( "-p", "--parse-only", help = "quit after parsing the OE files (developers only)",
69 action = "store_true", dest = "parse_only", default = False )
71 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
72 action = "store_true", dest = "disable_psyco", default = False )
74 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
75 action = "store_true", dest = "show_versions", default = False )
77 options, args = parser.parse_args( args )
78 return options, args[1:]
80 def try_build(fn, virtual):
81 if fn in __building_list:
82 oe.error("%s depends on itself (eventually)" % fn)
83 oe.error("upwards chain is: %s" % (" -> ".join(__build_path)))
86 __building_list.append(fn)
88 the_data = make.pkgdata[fn]
89 item = oe.data.getVar('PN', the_data, 1)
90 pathstr = "%s (%s)" % (item, virtual)
91 __build_path.append(pathstr)
93 depends_list = (oe.data.getVar('DEPENDS', the_data, 1) or "").split()
94 if make.options.verbose:
95 oe.note("current path: %s" % (" -> ".join(__build_path)))
96 oe.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
102 oldcmd = make.options.cmd
103 make.options.cmd = __depcmd
105 for d in depends_list:
106 if d in __ignored_dependencies:
110 if buildPackage(d) == 0:
111 oe.error("dependency %s (for %s) not satisfied" % (d,item))
113 if make.options.abort:
117 make.options.cmd = oldcmd
123 oe.event.fire(oe.event.PkgStarted(item, make.pkgdata[fn]))
125 __stats["attempt"] += 1
126 if not make.options.dry_run:
127 oe.build.exec_task('do_%s' % make.options.cmd, make.pkgdata[fn])
128 oe.event.fire(oe.event.PkgSucceeded(item, make.pkgdata[fn]))
129 __build_cache.append(fn)
131 except oe.build.FuncFailed:
133 oe.error("task stack execution failed")
134 oe.event.fire(oe.event.PkgFailed(item, make.pkgdata[fn]))
135 __build_cache_fail.append(fn)
137 except oe.build.EventException:
139 (type, value, traceback) = sys.exc_info()
141 oe.error("%s event exception, aborting" % oe.event.getName(e))
142 oe.event.fire(oe.event.PkgFailed(item, make.pkgdata[fn]))
143 __build_cache_fail.append(fn)
146 __building_list.remove(fn)
147 __build_path.remove(pathstr)
151 preferred_versions = {}
154 for p in make.pkgdata.keys():
155 pn = oe.data.getVar('PN', make.pkgdata[p], 1)
156 if not pkg_pn.has_key(pn):
161 for pn in pkg_pn.keys():
165 priority = oefile_priority[f]
166 if not priorities.has_key(priority):
167 priorities[priority] = []
168 priorities[priority].append(f)
169 p_list = priorities.keys()
170 p_list.sort(lambda a, b: a - b)
173 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
175 # If there is a PREFERRED_VERSION, find the highest-priority oefile providing that
176 # version. If not, find the latest version provided by an oefile in the
177 # highest-priority set.
178 for pn in pkg_pn.keys():
179 preferred_file = None
181 preferred_v = oe.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
184 m = re.match('(.*)_(.*)', preferred_v)
186 preferred_v = m.group(1)
187 preferred_r = m.group(2)
189 for file_set in pkg_pn[pn]:
191 the_data = make.pkgdata[f]
192 pv = oe.data.getVar('PV', the_data, 1)
193 pr = oe.data.getVar('PR', the_data, 1)
194 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
196 preferred_ver = (pv, pr)
201 pv_str = '%s-%s' % (preferred_v, preferred_r)
204 if preferred_file is None:
205 oe.note("preferred version %s of %s not available" % (pv_str, pn))
207 oe.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
209 # get highest priority file set
210 files = pkg_pn[pn][0]
215 the_data = make.pkgdata[f]
216 pv = oe.data.getVar('PV', the_data, 1)
217 pr = oe.data.getVar('PR', the_data, 1)
218 dp = int(oe.data.getVar('DEFAULT_PREFERENCE', the_data, 1) or "0")
220 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
224 if preferred_file is None:
225 preferred_file = latest_f
226 preferred_ver = latest
228 preferred_versions[pn] = (preferred_ver, preferred_file)
229 latest_versions[pn] = (latest, latest_f)
231 pkg_list = pkg_pn.keys()
235 pref = preferred_versions[p]
236 latest = latest_versions[p]
239 prefstr = pref[0][0] + "-" + pref[0][1]
243 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
246 def buildPackage(item):
249 discriminated = False
251 if not providers.has_key(item):
252 oe.error("Nothing provides %s" % item)
255 all_p = providers[item]
258 if p in __build_cache:
259 oe.debug(1, "already built %s in this run\n" % p)
263 preferred_versions = {}
265 # Collate providers by PN
268 the_data = make.pkgdata[p]
269 pn = oe.data.getVar('PN', the_data, 1)
270 if not pkg_pn.has_key(pn):
274 oe.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
277 for pn in pkg_pn.keys():
281 priority = oefile_priority[f]
282 if not priorities.has_key(priority):
283 priorities[priority] = []
284 priorities[priority].append(f)
285 p_list = priorities.keys()
286 p_list.sort(lambda a, b: a - b)
289 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
291 # If there is a PREFERRED_VERSION, find the highest-priority oefile providing that
292 # version. If not, find the latest version provided by an oefile in the
293 # highest-priority set.
294 for pn in pkg_pn.keys():
295 preferred_file = None
297 preferred_v = oe.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
300 m = re.match('(.*)_(.*)', preferred_v)
302 preferred_v = m.group(1)
303 preferred_r = m.group(2)
305 for file_set in pkg_pn[pn]:
307 the_data = make.pkgdata[f]
308 pv = oe.data.getVar('PV', the_data, 1)
309 pr = oe.data.getVar('PR', the_data, 1)
310 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
312 preferred_ver = (pv, pr)
317 pv_str = '%s-%s' % (preferred_v, preferred_r)
320 if preferred_file is None:
321 oe.note("preferred version %s of %s not available" % (pv_str, pn))
323 oe.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
325 if preferred_file is None:
326 # get highest priority file set
327 files = pkg_pn[pn][0]
332 the_data = make.pkgdata[f]
333 pv = oe.data.getVar('PV', the_data, 1)
334 pr = oe.data.getVar('PR', the_data, 1)
335 dp = int(oe.data.getVar('DEFAULT_PREFERENCE', the_data, 1) or "0")
337 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
341 preferred_file = latest_f
342 preferred_ver = latest
344 oe.debug(1, "selecting %s as latest version of provider %s" % (preferred_file, pn))
346 preferred_versions[pn] = (preferred_ver, preferred_file)
347 eligible.append(preferred_file)
350 if p in __build_cache_fail:
351 oe.debug(1, "rejecting already-failed %s" % p)
354 if len(eligible) == 0:
355 oe.error("no eligible providers for %s" % item)
358 # look to see if one of them is already staged, or marked as preferred.
359 # if so, bump it to the head of the queue
361 the_data = make.pkgdata[p]
362 pn = oe.data.getVar('PN', the_data, 1)
363 pv = oe.data.getVar('PV', the_data, 1)
364 pr = oe.data.getVar('PR', the_data, 1)
365 tmpdir = oe.data.getVar('TMPDIR', the_data, 1)
366 stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr)
367 if os.path.exists(stamp):
368 (newvers, fn) = preferred_versions[pn]
369 if not fn in eligible:
370 # package was made ineligible by already-failed check
372 oldver = "%s-%s" % (pv, pr)
373 newver = '-'.join(newvers)
374 if (newver != oldver):
375 extra_chat = "; upgrading from %s to %s" % (oldver, newver)
378 if make.options.verbose:
379 oe.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat))
381 eligible = [fn] + eligible
385 prefervar = oe.data.getVar('PREFERRED_PROVIDER_%s' % item, make.cfg, 1)
387 __preferred[item] = prefervar
389 if __preferred.has_key(item):
391 the_data = make.pkgdata[p]
392 pn = oe.data.getVar('PN', the_data, 1)
393 if __preferred[item] == pn:
394 if make.options.verbose:
395 oe.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
397 eligible = [p] + eligible
401 if len(eligible) > 1 and discriminated == False:
404 providers_list.append(oe.data.getVar('PN', make.pkgdata[fn], 1))
405 oe.note("multiple providers are available (%s);" % ", ".join(providers_list))
406 oe.note("consider defining PREFERRED_PROVIDER_%s" % item)
408 # run through the list until we find one that we can build
410 oe.debug(2, "selecting %s to satisfy %s" % (fn, item))
411 if try_build(fn, item):
414 oe.note("no buildable providers for %s" % item)
417 def build_depgraph():
422 if oedebug or progress.p == p: return
424 if os.isatty(sys.stdout.fileno()):
425 sys.stdout.write("\rNOTE: Building provider hash: [%s%s] (%02d%%)" % ( "#" * (p/5), " " * ( 20 - p/5 ), p ) )
429 sys.stdout.write("NOTE: Building provider hash, please wait...\n")
431 sys.stdout.write("done.\n")
434 def calc_oefile_priority(filename):
435 for (regex, pri) in oefile_config_priorities:
436 if regex.match(filename):
440 # Handle PREFERRED_PROVIDERS
441 for p in (oe.data.getVar('PREFERRED_PROVIDERS', make.cfg, 1) or "").split():
442 (providee, provider) = p.split(':')
443 if __preferred.has_key(providee) and __preferred[providee] != provider:
444 oe.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, __preferred[providee]))
445 __preferred[providee] = provider
447 # Calculate priorities for each file
448 for p in make.pkgdata.keys():
449 oefile_priority[p] = calc_oefile_priority(p)
451 n = len(make.pkgdata.keys())
456 oe.debug(1, "OEMAKE building providers hashes")
458 # Build forward and reverse provider hashes
459 # Forward: virtual -> [filenames]
460 # Reverse: PN -> [virtuals]
461 for f in make.pkgdata.keys():
464 pn = oe.data.getVar('PN', d, 1)
465 provides = Set([pn] + (oe.data.getVar("PROVIDES", d, 1) or "").split())
467 if not pn_provides.has_key(pn):
468 pn_provides[pn] = Set()
469 pn_provides[pn] |= provides
471 for provide in provides:
472 if not providers.has_key(provide):
473 providers[provide] = []
474 providers[provide].append(f)
476 deps = (oe.data.getVar("DEPENDS", d, 1) or "").split()
487 sys.stdout.write("\n")
489 # Build package list for "oemake world"
490 oe.debug(1, "OEMAKE collating packages for \"world\"")
491 for f in make.pkgdata.keys():
493 if oe.data.getVar('BROKEN', d, 1) or oe.data.getVar('EXCLUDE_FROM_WORLD', d, 1):
494 oe.debug(2, "OEMAKE skipping %s due to BROKEN/EXCLUDE_FROM_WORLD" % f)
497 pn = oe.data.getVar('PN', d, 1)
498 for p in pn_provides[pn]:
499 if p.startswith('virtual/'):
500 oe.debug(2, "OEMAKE skipping %s due to %s provider starting with virtual/" % (f, p))
503 for pf in providers[p]:
504 if oe.data.getVar('PN', make.pkgdata[pf], 1) != pn:
505 oe.debug(2, "OEMAKE skipping %s due to both us and %s providing %s" % (f, pf, p))
509 __world_target.add(pn)
511 def myProgressCallback( x, y, f ):
514 if os.isatty(sys.stdout.fileno()):
515 sys.stdout.write("\rNOTE: Parsing .oe files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
519 sys.stdout.write("Parsing .oe files, please wait...")
522 sys.stdout.write("done.")
530 if __name__ == "__main__":
532 if "OEDEBUG" in os.environ:
533 oedebug = int(os.environ["OEDEBUG"])
535 make.options, args = handle_options( sys.argv )
537 if not make.options.cmd:
538 make.options.cmd = "build"
540 if make.options.cmd in __depcmds:
541 __depcmd=__depcmds[make.options.cmd]
543 __depcmd=make.options.cmd
546 make.cfg = oe.data.init()
549 for f in make.options.file:
551 make.cfg = oe.parse.handle(f, make.cfg)
553 oe.fatal("Unable to open %s" % f)
556 make.cfg = oe.parse.handle("conf/oe.conf", make.cfg)
558 oe.fatal("Unable to open oe.conf")
560 if not oe.data.getVar("BUILDNAME", make.cfg):
561 oe.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), make.cfg)
563 buildname = oe.data.getVar("BUILDNAME", make.cfg)
565 ignore = oe.data.getVar("ASSUME_PROVIDED", make.cfg, 1) or ""
566 __ignored_dependencies = ignore.split()
568 collections = oe.data.getVar("OEFILE_COLLECTIONS", make.cfg, 1)
570 collection_list = collections.split()
571 for c in collection_list:
572 regex = oe.data.getVar("OEFILE_PATTERN_%s" % c, make.cfg, 1)
574 oe.error("OEFILE_PATTERN_%s not defined" % c)
576 priority = oe.data.getVar("OEFILE_PRIORITY_%s" % c, make.cfg, 1)
578 oe.error("OEFILE_PRIORITY_%s not defined" % c)
581 cre = re.compile(regex)
583 oe.error("OEFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
587 oefile_config_priorities.append((cre, pri))
589 oe.error("invalid value for OEFILE_PRIORITY_%s: \"%s\"" % (c, priority))
593 if not pkgs_to_build:
595 pkgs_to_build.extend(args)
596 if not pkgs_to_build:
597 oepkgs = oe.data.getVar('OEPKGS', make.cfg, 1)
599 pkgs_to_build = oepkgs.split()
600 if not pkgs_to_build and not make.options.show_versions:
601 print "Nothing to build. Use 'oemake world' to build everything."
604 __stats["attempt"] = 0
605 __stats["success"] = 0
609 # Import Psyco if available and not disabled
610 if not make.options.disable_psyco:
615 oe.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
617 psyco.bind( make.collect_oefiles )
619 oe.note("You have disabled Psyco. This decreases performance.")
622 oe.debug(1, "OEMAKE collecting .oe files")
623 make.collect_oefiles( myProgressCallback )
624 oe.debug(1, "OEMAKE parsing complete")
627 if make.options.parse_only:
628 print "Requested parsing .oe files only. Exiting."
633 if make.options.show_versions:
637 if 'world' in pkgs_to_build:
638 pkgs_to_build.remove('world')
639 for t in __world_target:
640 pkgs_to_build.append(t)
642 oe.event.fire(oe.event.BuildStarted(buildname, pkgs_to_build, make.cfg))
644 for k in pkgs_to_build:
647 if buildPackage(k) == 0:
650 except oe.build.EventException:
651 oe.error("Build of " + k + " failed")
655 if make.options.abort:
658 oe.event.fire(oe.event.BuildCompleted(buildname, pkgs_to_build, make.cfg))
660 print "Build statistics:"
661 print " Attempted builds: %d" % __stats["attempt"]
662 if __stats["fail"] != 0:
663 print " Failed builds: %d" % __stats["fail"]
664 if __stats["deps"] != 0:
665 print " Dependencies not satisfied: %d" % __stats["deps"]
666 if __stats["fail"] != 0 or __stats["deps"] != 0:
670 except KeyboardInterrupt:
671 print "\nNOTE: KeyboardInterrupt - Build not completed."