2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright (C) 2003, 2004 Chris Larson
6 # Copyright (C) 2003, 2004 Phil Blundell
8 # This program is free software; you can redistribute it and/or modify it under
9 # the terms of the GNU General Public License as published by the Free Software
10 # Foundation; either version 2 of the License, or (at your option) any later
13 # This program is distributed in the hope that it will be useful, but WITHOUT
14 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
15 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License along with
18 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
19 # Place, Suite 330, Boston, MA 02111-1307 USA.
21 import sys, os, getopt, glob, copy, os.path, re
22 sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
26 import itertools, optparse
28 parsespin = itertools.cycle( r'|/-\\' )
31 __build_cache_fail = []
37 __world_target = Set()
38 __ignored_dependencies = Set()
39 __depcmds = { "clean": None,
44 bbfile_config_priorities = []
48 def handle_options( args ):
49 parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
50 usage = """%prog [options] [package ...]
52 Executes the specified task (default is 'build') for a given set of BitBake files.
53 It expects that BBFILES is defined, which is a space seperated list of files to
54 be executed. BBFILES does support wildcards.
55 Default packages to be executed are all packages in BBFILES.
56 Default BBFILES are the .bb files in the current directory.""" )
58 parser.add_option( "-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES.",
59 action = "store", dest = "buildfile", default = None )
61 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
62 action = "store_false", dest = "abort", default = True )
64 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
65 action = "store_true", dest = "force", default = False )
68 parser.add_option( "-c", "--cmd", help = "Specify task to execute",
69 action = "store", dest = "cmd", default = "build" )
71 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
72 action = "append", dest = "file", default = [] )
74 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
75 action = "store_true", dest = "verbose", default = False )
77 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
78 action = "store_true", dest = "dry_run", default = False )
80 parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
81 action = "store_true", dest = "parse_only", default = False )
83 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
84 action = "store_true", dest = "disable_psyco", default = False )
86 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
87 action = "store_true", dest = "show_versions", default = False )
89 options, args = parser.parse_args( args )
90 return options, args[1:]
92 def try_build(fn, virtual):
93 if fn in __building_list:
94 bb.error("%s depends on itself (eventually)" % fn)
95 bb.error("upwards chain is: %s" % (" -> ".join(__build_path)))
98 __building_list.append(fn)
100 the_data = make.pkgdata[fn]
101 item = bb.data.getVar('PN', the_data, 1)
102 pathstr = "%s (%s)" % (item, virtual)
103 __build_path.append(pathstr)
105 depends_list = (bb.data.getVar('DEPENDS', the_data, 1) or "").split()
106 if make.options.verbose:
107 bb.note("current path: %s" % (" -> ".join(__build_path)))
108 bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
114 oldcmd = make.options.cmd
115 make.options.cmd = __depcmd
117 for d in depends_list:
118 if d in __ignored_dependencies:
122 if buildPackage(d) == 0:
123 bb.error("dependency %s (for %s) not satisfied" % (d,item))
125 if make.options.abort:
129 make.options.cmd = oldcmd
135 bb.event.fire(bb.event.PkgStarted(item, make.pkgdata[fn]))
137 __stats["attempt"] += 1
138 if not make.options.dry_run:
139 bb.build.exec_task('do_%s' % make.options.cmd, make.pkgdata[fn])
140 bb.event.fire(bb.event.PkgSucceeded(item, make.pkgdata[fn]))
141 __build_cache.append(fn)
143 except bb.build.FuncFailed:
145 bb.error("task stack execution failed")
146 bb.event.fire(bb.event.PkgFailed(item, make.pkgdata[fn]))
147 __build_cache_fail.append(fn)
149 except bb.build.EventException:
151 (type, value, traceback) = sys.exc_info()
153 bb.error("%s event exception, aborting" % bb.event.getName(e))
154 bb.event.fire(bb.event.PkgFailed(item, make.pkgdata[fn]))
155 __build_cache_fail.append(fn)
158 __building_list.remove(fn)
159 __build_path.remove(pathstr)
163 preferred_versions = {}
166 for p in make.pkgdata.keys():
167 pn = bb.data.getVar('PN', make.pkgdata[p], 1)
168 if not pkg_pn.has_key(pn):
173 for pn in pkg_pn.keys():
177 priority = bbfile_priority[f]
178 if not priorities.has_key(priority):
179 priorities[priority] = []
180 priorities[priority].append(f)
181 p_list = priorities.keys()
182 p_list.sort(lambda a, b: a - b)
185 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
187 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
188 # version. If not, find the latest version provided by an bbfile in the
189 # highest-priority set.
190 for pn in pkg_pn.keys():
191 preferred_file = None
193 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
196 m = re.match('(.*)_(.*)', preferred_v)
198 preferred_v = m.group(1)
199 preferred_r = m.group(2)
201 for file_set in pkg_pn[pn]:
203 the_data = make.pkgdata[f]
204 pv = bb.data.getVar('PV', the_data, 1)
205 pr = bb.data.getVar('PR', the_data, 1)
206 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
208 preferred_ver = (pv, pr)
213 pv_str = '%s-%s' % (preferred_v, preferred_r)
216 if preferred_file is None:
217 bb.note("preferred version %s of %s not available" % (pv_str, pn))
219 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
221 # get highest priority file set
222 files = pkg_pn[pn][0]
227 the_data = make.pkgdata[f]
228 pv = bb.data.getVar('PV', the_data, 1)
229 pr = bb.data.getVar('PR', the_data, 1)
230 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', the_data, 1) or "0")
232 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
236 if preferred_file is None:
237 preferred_file = latest_f
238 preferred_ver = latest
240 preferred_versions[pn] = (preferred_ver, preferred_file)
241 latest_versions[pn] = (latest, latest_f)
243 pkg_list = pkg_pn.keys()
247 pref = preferred_versions[p]
248 latest = latest_versions[p]
251 prefstr = pref[0][0] + "-" + pref[0][1]
255 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
258 def buildPackage(item):
261 discriminated = False
263 if not providers.has_key(item):
264 bb.error("Nothing provides %s" % item)
267 all_p = providers[item]
270 if p in __build_cache:
271 bb.debug(1, "already built %s in this run\n" % p)
275 preferred_versions = {}
277 # Collate providers by PN
280 the_data = make.pkgdata[p]
281 pn = bb.data.getVar('PN', the_data, 1)
282 if not pkg_pn.has_key(pn):
286 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
289 for pn in pkg_pn.keys():
293 priority = bbfile_priority[f]
294 if not priorities.has_key(priority):
295 priorities[priority] = []
296 priorities[priority].append(f)
297 p_list = priorities.keys()
298 p_list.sort(lambda a, b: a - b)
301 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
303 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
304 # version. If not, find the latest version provided by an bbfile in the
305 # highest-priority set.
306 for pn in pkg_pn.keys():
307 preferred_file = None
309 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
312 m = re.match('(.*)_(.*)', preferred_v)
314 preferred_v = m.group(1)
315 preferred_r = m.group(2)
317 for file_set in pkg_pn[pn]:
319 the_data = make.pkgdata[f]
320 pv = bb.data.getVar('PV', the_data, 1)
321 pr = bb.data.getVar('PR', the_data, 1)
322 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
324 preferred_ver = (pv, pr)
329 pv_str = '%s-%s' % (preferred_v, preferred_r)
332 if preferred_file is None:
333 bb.note("preferred version %s of %s not available" % (pv_str, pn))
335 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
337 if preferred_file is None:
338 # get highest priority file set
339 files = pkg_pn[pn][0]
344 the_data = make.pkgdata[f]
345 pv = bb.data.getVar('PV', the_data, 1)
346 pr = bb.data.getVar('PR', the_data, 1)
347 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', the_data, 1) or "0")
349 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
353 preferred_file = latest_f
354 preferred_ver = latest
356 bb.debug(1, "selecting %s as latest version of provider %s" % (preferred_file, pn))
358 preferred_versions[pn] = (preferred_ver, preferred_file)
359 eligible.append(preferred_file)
362 if p in __build_cache_fail:
363 bb.debug(1, "rejecting already-failed %s" % p)
366 if len(eligible) == 0:
367 bb.error("no eligible providers for %s" % item)
370 # look to see if one of them is already staged, or marked as preferred.
371 # if so, bump it to the head of the queue
373 the_data = make.pkgdata[p]
374 pn = bb.data.getVar('PN', the_data, 1)
375 pv = bb.data.getVar('PV', the_data, 1)
376 pr = bb.data.getVar('PR', the_data, 1)
377 tmpdir = bb.data.getVar('TMPDIR', the_data, 1)
378 stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr)
379 if os.path.exists(stamp):
380 (newvers, fn) = preferred_versions[pn]
381 if not fn in eligible:
382 # package was made ineligible by already-failed check
384 oldver = "%s-%s" % (pv, pr)
385 newver = '-'.join(newvers)
386 if (newver != oldver):
387 extra_chat = "; upgrading from %s to %s" % (oldver, newver)
390 if make.options.verbose:
391 bb.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat))
393 eligible = [fn] + eligible
397 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, make.cfg, 1)
399 __preferred[item] = prefervar
401 if __preferred.has_key(item):
403 the_data = make.pkgdata[p]
404 pn = bb.data.getVar('PN', the_data, 1)
405 if __preferred[item] == pn:
406 if make.options.verbose:
407 bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
409 eligible = [p] + eligible
413 if len(eligible) > 1 and discriminated == False:
416 providers_list.append(bb.data.getVar('PN', make.pkgdata[fn], 1))
417 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
418 bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
420 # run through the list until we find one that we can build
422 bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
423 if try_build(fn, item):
426 bb.note("no buildable providers for %s" % item)
429 def build_depgraph():
434 if bbdebug or progress.p == p: return
436 if os.isatty(sys.stdout.fileno()):
437 sys.stdout.write("\rNOTE: Building provider hash: [%s%s] (%02d%%)" % ( "#" * (p/5), " " * ( 20 - p/5 ), p ) )
441 sys.stdout.write("NOTE: Building provider hash, please wait...\n")
443 sys.stdout.write("done.\n")
446 def calc_bbfile_priority(filename):
447 for (regex, pri) in bbfile_config_priorities:
448 if regex.match(filename):
452 # Handle PREFERRED_PROVIDERS
453 for p in (bb.data.getVar('PREFERRED_PROVIDERS', make.cfg, 1) or "").split():
454 (providee, provider) = p.split(':')
455 if __preferred.has_key(providee) and __preferred[providee] != provider:
456 bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, __preferred[providee]))
457 __preferred[providee] = provider
459 # Calculate priorities for each file
460 for p in make.pkgdata.keys():
461 bbfile_priority[p] = calc_bbfile_priority(p)
463 n = len(make.pkgdata.keys())
468 bb.debug(1, "BBMAKE building providers hashes")
470 # Build forward and reverse provider hashes
471 # Forward: virtual -> [filenames]
472 # Reverse: PN -> [virtuals]
473 for f in make.pkgdata.keys():
476 pn = bb.data.getVar('PN', d, 1)
477 provides = Set([pn] + (bb.data.getVar("PROVIDES", d, 1) or "").split())
479 if not pn_provides.has_key(pn):
480 pn_provides[pn] = Set()
481 pn_provides[pn] |= provides
483 for provide in provides:
484 if not providers.has_key(provide):
485 providers[provide] = []
486 providers[provide].append(f)
488 deps = (bb.data.getVar("DEPENDS", d, 1) or "").split()
499 sys.stdout.write("\n")
501 # Build package list for "bbmake world"
502 bb.debug(1, "BBMAKE collating packages for \"world\"")
503 for f in make.pkgdata.keys():
505 if bb.data.getVar('BROKEN', d, 1) or bb.data.getVar('EXCLUDE_FROM_WORLD', d, 1):
506 bb.debug(2, "BBMAKE skipping %s due to BROKEN/EXCLUDE_FROM_WORLD" % f)
509 pn = bb.data.getVar('PN', d, 1)
510 for p in pn_provides[pn]:
511 if p.startswith('virtual/'):
512 bb.debug(2, "BBMAKE skipping %s due to %s provider starting with virtual/" % (f, p))
515 for pf in providers[p]:
516 if bb.data.getVar('PN', make.pkgdata[pf], 1) != pn:
517 bb.debug(2, "BBMAKE skipping %s due to both us and %s providing %s" % (f, pf, p))
521 __world_target.add(pn)
523 def myProgressCallback( x, y, f ):
526 if os.isatty(sys.stdout.fileno()):
527 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
531 sys.stdout.write("Parsing .bb files, please wait...")
534 sys.stdout.write("done.")
537 def executeOneBB( fn ):
539 d = bb.parse.handle(fn, make.cfg)
541 bb.fatal("Unable to open %s" % fn)
543 name = bb.data.getVar('PN', d, 1)
544 bb.event.fire(bb.event.PkgStarted(name, d))
546 __stats["attempt"] += 1
547 if not make.options.dry_run:
548 bb.build.exec_task('do_%s' % make.options.cmd, d)
549 bb.event.fire(bb.event.PkgSucceeded(name, d))
550 __build_cache.append(fn)
551 except bb.build.FuncFailed:
553 bb.error("task stack execution failed")
554 bb.event.fire(bb.event.PkgFailed(name, d))
555 __build_cache_fail.append(fn)
556 except bb.build.EventException:
558 (type, value, traceback) = sys.exc_info()
560 bb.error("%s event exception, aborting" % bb.event.getName(e))
561 bb.event.fire(bb.event.PkgFailed(name, d))
562 __build_cache_fail.append(fn)
568 __stats["attempt"] = 0
569 __stats["success"] = 0
574 print "Build statistics:"
575 print " Attempted builds: %d" % __stats["attempt"]
576 if __stats["fail"] != 0:
577 print " Failed builds: %d" % __stats["fail"]
578 if __stats["deps"] != 0:
579 print " Dependencies not satisfied: %d" % __stats["deps"]
580 if __stats["fail"] != 0 or __stats["deps"] != 0:
584 if __name__ == "__main__":
586 if "BBDEBUG" in os.environ:
587 bbdebug = int(os.environ["BBDEBUG"])
589 make.options, args = handle_options( sys.argv )
591 if not make.options.cmd:
592 make.options.cmd = "build"
594 if make.options.cmd in __depcmds:
595 __depcmd=__depcmds[make.options.cmd]
597 __depcmd=make.options.cmd
600 make.cfg = bb.data.init()
603 for f in make.options.file:
605 make.cfg = bb.parse.handle(f, make.cfg)
607 bb.fatal("Unable to open %s" % f)
610 make.cfg = bb.parse.handle(os.path.join('conf', 'bitbake.conf'), make.cfg)
612 bb.fatal("Unable to open %s" % os.path.join('conf', 'bitbake.conf'))
614 if not bb.data.getVar("BUILDNAME", make.cfg):
615 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), make.cfg)
617 buildname = bb.data.getVar("BUILDNAME", make.cfg)
619 bf = make.options.buildfile
624 ignore = bb.data.getVar("ASSUME_PROVIDED", make.cfg, 1) or ""
625 __ignored_dependencies = ignore.split()
627 collections = bb.data.getVar("BBFILE_COLLECTIONS", make.cfg, 1)
629 collection_list = collections.split()
630 for c in collection_list:
631 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, make.cfg, 1)
633 bb.error("BBFILE_PATTERN_%s not defined" % c)
635 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, make.cfg, 1)
637 bb.error("BBFILE_PRIORITY_%s not defined" % c)
640 cre = re.compile(regex)
642 bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
646 bbfile_config_priorities.append((cre, pri))
648 bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
652 if not pkgs_to_build:
654 pkgs_to_build.extend(args)
655 if not pkgs_to_build:
656 bbpkgs = bb.data.getVar('BBPKGS', make.cfg, 1)
658 pkgs_to_build = bbpkgs.split()
659 if not pkgs_to_build and not make.options.show_versions:
660 print "Nothing to build. Use 'bbmake world' to build everything."
664 # Import Psyco if available and not disabled
665 if not make.options.disable_psyco:
670 bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
672 psyco.bind( make.collect_bbfiles )
674 bb.note("You have disabled Psyco. This decreases performance.")
677 bb.debug(1, "BBMAKE collecting .bb files")
678 make.collect_bbfiles( myProgressCallback )
679 bb.debug(1, "BBMAKE parsing complete")
682 if make.options.parse_only:
683 print "Requested parsing .bb files only. Exiting."
688 if make.options.show_versions:
692 if 'world' in pkgs_to_build:
693 pkgs_to_build.remove('world')
694 for t in __world_target:
695 pkgs_to_build.append(t)
697 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, make.cfg))
699 for k in pkgs_to_build:
702 if buildPackage(k) == 0:
705 except bb.build.EventException:
706 bb.error("Build of " + k + " failed")
710 if make.options.abort:
713 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, make.cfg))
717 except KeyboardInterrupt:
718 print "\nNOTE: KeyboardInterrupt - Build not completed."