2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright (C) 2003, 2004 Chris Larson
6 # Copyright (C) 2003, 2004 Phil Blundell
8 # This program is free software; you can redistribute it and/or modify it under
9 # the terms of the GNU General Public License as published by the Free Software
10 # Foundation; either version 2 of the License, or (at your option) any later
13 # This program is distributed in the hope that it will be useful, but WITHOUT
14 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
15 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License along with
18 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
19 # Place, Suite 330, Boston, MA 02111-1307 USA.
21 import sys, os, getopt, glob, copy, os.path, re
22 sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
26 import itertools, optparse
28 parsespin = itertools.cycle( r'|/-\\' )
31 __build_cache_fail = []
37 __world_target = Set()
38 __ignored_dependencies = Set()
42 bbfile_config_priorities = []
46 def handle_options( args ):
47 parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
48 usage = """%prog [options] [package ...]
50 Executes the specified task (default is 'build') for a given set of BitBake files.
51 It expects that BBFILES is defined, which is a space seperated list of files to
52 be executed. BBFILES does support wildcards.
53 Default BBFILES are the .bb files in the current directory.""" )
55 parser.add_option( "-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES.",
56 action = "store", dest = "buildfile", default = None )
58 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
59 action = "store_false", dest = "abort", default = True )
61 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
62 action = "store_true", dest = "force", default = False )
65 parser.add_option( "-c", "--cmd", help = "Specify task to execute",
66 action = "store", dest = "cmd", default = "build" )
68 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
69 action = "append", dest = "file", default = [] )
71 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
72 action = "store_true", dest = "verbose", default = False )
73 parser.add_option( "-D", "--debug", help = "Increase the debug level",
74 action = "count", dest="debug", default = 0)
76 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
77 action = "store_true", dest = "dry_run", default = False )
79 parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
80 action = "store_true", dest = "parse_only", default = False )
82 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
83 action = "store_true", dest = "disable_psyco", default = False )
85 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
86 action = "store_true", dest = "show_versions", default = False )
88 options, args = parser.parse_args( args )
89 return options, args[1:]
91 def try_build(fn, virtual):
92 if fn in __building_list:
93 bb.error("%s depends on itself (eventually)" % fn)
94 bb.error("upwards chain is: %s" % (" -> ".join(__build_path)))
97 __building_list.append(fn)
99 the_data = make.pkgdata[fn]
100 item = bb.data.getVar('PN', the_data, 1)
101 pathstr = "%s (%s)" % (item, virtual)
102 __build_path.append(pathstr)
104 depends_list = (bb.data.getVar('DEPENDS', the_data, 1) or "").split()
105 if make.options.verbose:
106 bb.note("current path: %s" % (" -> ".join(__build_path)))
107 bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
112 depcmd = make.options.cmd
113 bbdepcmd = bb.data.getVarFlag('do_%s' % make.options.cmd, 'bbdepcmd', make.pkgdata[fn])
114 if bbdepcmd is not None:
121 oldcmd = make.options.cmd
122 make.options.cmd = depcmd
124 for d in depends_list:
125 if d in __ignored_dependencies:
129 if buildPackage(d) == 0:
130 bb.error("dependency %s (for %s) not satisfied" % (d,item))
132 if make.options.abort:
136 make.options.cmd = oldcmd
142 bb.event.fire(bb.event.PkgStarted(item, make.pkgdata[fn]))
144 __stats["attempt"] += 1
145 if not make.options.dry_run:
146 bb.build.exec_task('do_%s' % make.options.cmd, make.pkgdata[fn])
147 bb.event.fire(bb.event.PkgSucceeded(item, make.pkgdata[fn]))
148 __build_cache.append(fn)
150 except bb.build.FuncFailed:
152 bb.error("task stack execution failed")
153 bb.event.fire(bb.event.PkgFailed(item, make.pkgdata[fn]))
154 __build_cache_fail.append(fn)
156 except bb.build.EventException:
158 (type, value, traceback) = sys.exc_info()
160 bb.error("%s event exception, aborting" % bb.event.getName(e))
161 bb.event.fire(bb.event.PkgFailed(item, make.pkgdata[fn]))
162 __build_cache_fail.append(fn)
165 __building_list.remove(fn)
166 __build_path.remove(pathstr)
170 preferred_versions = {}
173 for p in make.pkgdata.keys():
174 pn = bb.data.getVar('PN', make.pkgdata[p], 1)
175 if not pkg_pn.has_key(pn):
180 for pn in pkg_pn.keys():
184 priority = bbfile_priority[f]
185 if not priorities.has_key(priority):
186 priorities[priority] = []
187 priorities[priority].append(f)
188 p_list = priorities.keys()
189 p_list.sort(lambda a, b: a - b)
192 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
194 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
195 # version. If not, find the latest version provided by an bbfile in the
196 # highest-priority set.
197 for pn in pkg_pn.keys():
198 preferred_file = None
200 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
203 m = re.match('(.*)_(.*)', preferred_v)
205 preferred_v = m.group(1)
206 preferred_r = m.group(2)
208 for file_set in pkg_pn[pn]:
210 the_data = make.pkgdata[f]
211 pv = bb.data.getVar('PV', the_data, 1)
212 pr = bb.data.getVar('PR', the_data, 1)
213 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
215 preferred_ver = (pv, pr)
220 pv_str = '%s-%s' % (preferred_v, preferred_r)
223 if preferred_file is None:
224 bb.note("preferred version %s of %s not available" % (pv_str, pn))
226 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
228 # get highest priority file set
229 files = pkg_pn[pn][0]
234 the_data = make.pkgdata[f]
235 pv = bb.data.getVar('PV', the_data, 1)
236 pr = bb.data.getVar('PR', the_data, 1)
237 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', the_data, 1) or "0")
239 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
243 if preferred_file is None:
244 preferred_file = latest_f
245 preferred_ver = latest
247 preferred_versions[pn] = (preferred_ver, preferred_file)
248 latest_versions[pn] = (latest, latest_f)
250 pkg_list = pkg_pn.keys()
254 pref = preferred_versions[p]
255 latest = latest_versions[p]
258 prefstr = pref[0][0] + "-" + pref[0][1]
262 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
265 def buildPackage(item):
268 discriminated = False
270 if not providers.has_key(item):
271 bb.error("Nothing provides %s" % item)
274 all_p = providers[item]
277 if p in __build_cache:
278 bb.debug(1, "already built %s in this run\n" % p)
282 preferred_versions = {}
284 # Collate providers by PN
287 the_data = make.pkgdata[p]
288 pn = bb.data.getVar('PN', the_data, 1)
289 if not pkg_pn.has_key(pn):
293 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
296 for pn in pkg_pn.keys():
300 priority = bbfile_priority[f]
301 if not priorities.has_key(priority):
302 priorities[priority] = []
303 priorities[priority].append(f)
304 p_list = priorities.keys()
305 p_list.sort(lambda a, b: a - b)
308 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
310 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
311 # version. If not, find the latest version provided by an bbfile in the
312 # highest-priority set.
313 for pn in pkg_pn.keys():
314 preferred_file = None
316 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
319 m = re.match('(.*)_(.*)', preferred_v)
321 preferred_v = m.group(1)
322 preferred_r = m.group(2)
324 for file_set in pkg_pn[pn]:
326 the_data = make.pkgdata[f]
327 pv = bb.data.getVar('PV', the_data, 1)
328 pr = bb.data.getVar('PR', the_data, 1)
329 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
331 preferred_ver = (pv, pr)
336 pv_str = '%s-%s' % (preferred_v, preferred_r)
339 if preferred_file is None:
340 bb.note("preferred version %s of %s not available" % (pv_str, pn))
342 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
344 if preferred_file is None:
345 # get highest priority file set
346 files = pkg_pn[pn][0]
351 the_data = make.pkgdata[f]
352 pv = bb.data.getVar('PV', the_data, 1)
353 pr = bb.data.getVar('PR', the_data, 1)
354 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', the_data, 1) or "0")
356 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
360 preferred_file = latest_f
361 preferred_ver = latest
363 bb.debug(1, "selecting %s as latest version of provider %s" % (preferred_file, pn))
365 preferred_versions[pn] = (preferred_ver, preferred_file)
366 eligible.append(preferred_file)
369 if p in __build_cache_fail:
370 bb.debug(1, "rejecting already-failed %s" % p)
373 if len(eligible) == 0:
374 bb.error("no eligible providers for %s" % item)
377 # look to see if one of them is already staged, or marked as preferred.
378 # if so, bump it to the head of the queue
380 the_data = make.pkgdata[p]
381 pn = bb.data.getVar('PN', the_data, 1)
382 pv = bb.data.getVar('PV', the_data, 1)
383 pr = bb.data.getVar('PR', the_data, 1)
384 tmpdir = bb.data.getVar('TMPDIR', the_data, 1)
385 stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr)
386 if os.path.exists(stamp):
387 (newvers, fn) = preferred_versions[pn]
388 if not fn in eligible:
389 # package was made ineligible by already-failed check
391 oldver = "%s-%s" % (pv, pr)
392 newver = '-'.join(newvers)
393 if (newver != oldver):
394 extra_chat = "; upgrading from %s to %s" % (oldver, newver)
397 if make.options.verbose:
398 bb.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat))
400 eligible = [fn] + eligible
404 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, make.cfg, 1)
406 __preferred[item] = prefervar
408 if __preferred.has_key(item):
410 the_data = make.pkgdata[p]
411 pn = bb.data.getVar('PN', the_data, 1)
412 if __preferred[item] == pn:
413 if make.options.verbose:
414 bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
416 eligible = [p] + eligible
420 if len(eligible) > 1 and discriminated == False:
423 providers_list.append(bb.data.getVar('PN', make.pkgdata[fn], 1))
424 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
425 bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
427 # run through the list until we find one that we can build
429 bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
430 if try_build(fn, item):
433 bb.note("no buildable providers for %s" % item)
436 def build_depgraph():
441 if bbdebug or progress.p == p: return
443 if os.isatty(sys.stdout.fileno()):
444 sys.stdout.write("\rNOTE: Building provider hash: [%s%s] (%02d%%)" % ( "#" * (p/5), " " * ( 20 - p/5 ), p ) )
448 sys.stdout.write("NOTE: Building provider hash, please wait...\n")
450 sys.stdout.write("done.\n")
453 def calc_bbfile_priority(filename):
454 for (regex, pri) in bbfile_config_priorities:
455 if regex.match(filename):
459 # Handle PREFERRED_PROVIDERS
460 for p in (bb.data.getVar('PREFERRED_PROVIDERS', make.cfg, 1) or "").split():
461 (providee, provider) = p.split(':')
462 if __preferred.has_key(providee) and __preferred[providee] != provider:
463 bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, __preferred[providee]))
464 __preferred[providee] = provider
466 # Calculate priorities for each file
467 for p in make.pkgdata.keys():
468 bbfile_priority[p] = calc_bbfile_priority(p)
470 n = len(make.pkgdata.keys())
475 bb.debug(1, "BBMAKE building providers hashes")
477 # Build forward and reverse provider hashes
478 # Forward: virtual -> [filenames]
479 # Reverse: PN -> [virtuals]
480 for f in make.pkgdata.keys():
483 pn = bb.data.getVar('PN', d, 1)
484 provides = Set([pn] + (bb.data.getVar("PROVIDES", d, 1) or "").split())
486 if not pn_provides.has_key(pn):
487 pn_provides[pn] = Set()
488 pn_provides[pn] |= provides
490 for provide in provides:
491 if not providers.has_key(provide):
492 providers[provide] = []
493 providers[provide].append(f)
495 deps = (bb.data.getVar("DEPENDS", d, 1) or "").split()
506 sys.stdout.write("\n")
508 # Build package list for "bitbake world"
509 bb.debug(1, "BBMAKE collating packages for \"world\"")
510 for f in make.pkgdata.keys():
512 if bb.data.getVar('BROKEN', d, 1) or bb.data.getVar('EXCLUDE_FROM_WORLD', d, 1):
513 bb.debug(2, "BBMAKE skipping %s due to BROKEN/EXCLUDE_FROM_WORLD" % f)
516 pn = bb.data.getVar('PN', d, 1)
517 for p in pn_provides[pn]:
518 if p.startswith('virtual/'):
519 bb.debug(2, "BBMAKE skipping %s due to %s provider starting with virtual/" % (f, p))
522 for pf in providers[p]:
523 if bb.data.getVar('PN', make.pkgdata[pf], 1) != pn:
524 bb.debug(2, "BBMAKE skipping %s due to both us and %s providing %s" % (f, pf, p))
528 __world_target.add(pn)
530 def myProgressCallback( x, y, f ):
533 if os.isatty(sys.stdout.fileno()):
534 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
538 sys.stdout.write("Parsing .bb files, please wait...")
541 sys.stdout.write("done.")
544 def executeOneBB( fn ):
546 d = bb.parse.handle(fn, make.cfg)
548 bb.fatal("Unable to open %s" % fn)
550 name = bb.data.getVar('PN', d, 1)
551 bb.event.fire(bb.event.PkgStarted(name, d))
553 __stats["attempt"] += 1
554 if not make.options.dry_run:
555 bb.build.exec_task('do_%s' % make.options.cmd, d)
556 bb.event.fire(bb.event.PkgSucceeded(name, d))
557 __build_cache.append(fn)
558 except bb.build.FuncFailed:
560 bb.error("task stack execution failed")
561 bb.event.fire(bb.event.PkgFailed(name, d))
562 __build_cache_fail.append(fn)
563 except bb.build.EventException:
565 (type, value, traceback) = sys.exc_info()
567 bb.error("%s event exception, aborting" % bb.event.getName(e))
568 bb.event.fire(bb.event.PkgFailed(name, d))
569 __build_cache_fail.append(fn)
575 __stats["attempt"] = 0
576 __stats["success"] = 0
581 print "Build statistics:"
582 print " Attempted builds: %d" % __stats["attempt"]
583 if __stats["fail"] != 0:
584 print " Failed builds: %d" % __stats["fail"]
585 if __stats["deps"] != 0:
586 print " Dependencies not satisfied: %d" % __stats["deps"]
587 if __stats["fail"] != 0 or __stats["deps"] != 0:
591 if __name__ == "__main__":
593 make.options, args = handle_options( sys.argv )
595 if not make.options.cmd:
596 make.options.cmd = "build"
598 if make.options.debug:
599 bb.debug_level = make.options.debug
602 make.cfg = bb.data.init()
605 for f in make.options.file:
607 make.cfg = bb.parse.handle(f, make.cfg)
609 bb.fatal("Unable to open %s" % f)
612 make.cfg = bb.parse.handle(os.path.join('conf', 'bitbake.conf'), make.cfg)
614 bb.fatal("Unable to open %s" % os.path.join('conf', 'bitbake.conf'))
616 if not bb.data.getVar("BUILDNAME", make.cfg):
617 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), make.cfg)
619 buildname = bb.data.getVar("BUILDNAME", make.cfg)
621 bf = make.options.buildfile
623 executeOneBB( os.path.abspath(bf) )
626 ignore = bb.data.getVar("ASSUME_PROVIDED", make.cfg, 1) or ""
627 __ignored_dependencies = ignore.split()
629 collections = bb.data.getVar("BBFILE_COLLECTIONS", make.cfg, 1)
631 collection_list = collections.split()
632 for c in collection_list:
633 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, make.cfg, 1)
635 bb.error("BBFILE_PATTERN_%s not defined" % c)
637 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, make.cfg, 1)
639 bb.error("BBFILE_PRIORITY_%s not defined" % c)
642 cre = re.compile(regex)
644 bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
648 bbfile_config_priorities.append((cre, pri))
650 bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
654 if not pkgs_to_build:
656 pkgs_to_build.extend(args)
657 if not pkgs_to_build:
658 bbpkgs = bb.data.getVar('BBPKGS', make.cfg, 1)
660 pkgs_to_build = bbpkgs.split()
661 if not pkgs_to_build and not make.options.show_versions:
662 print "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help'"
663 print "for usage information."
667 # Import Psyco if available and not disabled
668 if not make.options.disable_psyco:
673 bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
675 psyco.bind( make.collect_bbfiles )
677 bb.note("You have disabled Psyco. This decreases performance.")
680 bb.debug(1, "BBMAKE collecting .bb files")
681 make.collect_bbfiles( myProgressCallback )
682 bb.debug(1, "BBMAKE parsing complete")
685 if make.options.parse_only:
686 print "Requested parsing .bb files only. Exiting."
691 if make.options.show_versions:
695 if 'world' in pkgs_to_build:
696 pkgs_to_build.remove('world')
697 for t in __world_target:
698 pkgs_to_build.append(t)
700 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, make.cfg))
702 for k in pkgs_to_build:
705 if buildPackage(k) == 0:
708 except bb.build.EventException:
709 bb.error("Build of " + k + " failed")
713 if make.options.abort:
716 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, make.cfg))
720 except KeyboardInterrupt:
721 print "\nNOTE: KeyboardInterrupt - Build not completed."