2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright (C) 2003, 2004 Chris Larson
6 # Copyright (C) 2003, 2004 Phil Blundell
8 # This program is free software; you can redistribute it and/or modify it under
9 # the terms of the GNU General Public License as published by the Free Software
10 # Foundation; either version 2 of the License, or (at your option) any later
13 # This program is distributed in the hope that it will be useful, but WITHOUT
14 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
15 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License along with
18 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
19 # Place, Suite 330, Boston, MA 02111-1307 USA.
21 import sys, os, getopt, glob, copy, os.path, re
22 sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
26 import itertools, optparse
28 parsespin = itertools.cycle( r'|/-\\' )
31 __build_cache_fail = []
37 __world_target = Set()
38 __ignored_dependencies = Set()
39 __depcmds = { "clean": None,
44 bbfile_config_priorities = []
48 def handle_options( args ):
49 parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
50 usage = """%prog [options] [package ...]
52 Executes the specified task (default is 'build') for a given set of BitBake files.
53 It expects that BBFILES is defined, which is a space seperated list of files to
54 be executed. BBFILES does support wildcards.
55 Default BBFILES are the .bb files in the current directory.""" )
57 parser.add_option( "-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES.",
58 action = "store", dest = "buildfile", default = None )
60 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
61 action = "store_false", dest = "abort", default = True )
63 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
64 action = "store_true", dest = "force", default = False )
67 parser.add_option( "-c", "--cmd", help = "Specify task to execute",
68 action = "store", dest = "cmd", default = "build" )
70 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
71 action = "append", dest = "file", default = [] )
73 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
74 action = "store_true", dest = "verbose", default = False )
76 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
77 action = "store_true", dest = "dry_run", default = False )
79 parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
80 action = "store_true", dest = "parse_only", default = False )
82 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
83 action = "store_true", dest = "disable_psyco", default = False )
85 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
86 action = "store_true", dest = "show_versions", default = False )
88 options, args = parser.parse_args( args )
89 return options, args[1:]
91 def try_build(fn, virtual):
92 if fn in __building_list:
93 bb.error("%s depends on itself (eventually)" % fn)
94 bb.error("upwards chain is: %s" % (" -> ".join(__build_path)))
97 __building_list.append(fn)
99 the_data = make.pkgdata[fn]
100 item = bb.data.getVar('PN', the_data, 1)
101 pathstr = "%s (%s)" % (item, virtual)
102 __build_path.append(pathstr)
104 depends_list = (bb.data.getVar('DEPENDS', the_data, 1) or "").split()
105 if make.options.verbose:
106 bb.note("current path: %s" % (" -> ".join(__build_path)))
107 bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
113 oldcmd = make.options.cmd
114 make.options.cmd = __depcmd
116 for d in depends_list:
117 if d in __ignored_dependencies:
121 if buildPackage(d) == 0:
122 bb.error("dependency %s (for %s) not satisfied" % (d,item))
124 if make.options.abort:
128 make.options.cmd = oldcmd
134 bb.event.fire(bb.event.PkgStarted(item, make.pkgdata[fn]))
136 __stats["attempt"] += 1
137 if not make.options.dry_run:
138 bb.build.exec_task('do_%s' % make.options.cmd, make.pkgdata[fn])
139 bb.event.fire(bb.event.PkgSucceeded(item, make.pkgdata[fn]))
140 __build_cache.append(fn)
142 except bb.build.FuncFailed:
144 bb.error("task stack execution failed")
145 bb.event.fire(bb.event.PkgFailed(item, make.pkgdata[fn]))
146 __build_cache_fail.append(fn)
148 except bb.build.EventException:
150 (type, value, traceback) = sys.exc_info()
152 bb.error("%s event exception, aborting" % bb.event.getName(e))
153 bb.event.fire(bb.event.PkgFailed(item, make.pkgdata[fn]))
154 __build_cache_fail.append(fn)
157 __building_list.remove(fn)
158 __build_path.remove(pathstr)
162 preferred_versions = {}
165 for p in make.pkgdata.keys():
166 pn = bb.data.getVar('PN', make.pkgdata[p], 1)
167 if not pkg_pn.has_key(pn):
172 for pn in pkg_pn.keys():
176 priority = bbfile_priority[f]
177 if not priorities.has_key(priority):
178 priorities[priority] = []
179 priorities[priority].append(f)
180 p_list = priorities.keys()
181 p_list.sort(lambda a, b: a - b)
184 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
186 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
187 # version. If not, find the latest version provided by an bbfile in the
188 # highest-priority set.
189 for pn in pkg_pn.keys():
190 preferred_file = None
192 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
195 m = re.match('(.*)_(.*)', preferred_v)
197 preferred_v = m.group(1)
198 preferred_r = m.group(2)
200 for file_set in pkg_pn[pn]:
202 the_data = make.pkgdata[f]
203 pv = bb.data.getVar('PV', the_data, 1)
204 pr = bb.data.getVar('PR', the_data, 1)
205 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
207 preferred_ver = (pv, pr)
212 pv_str = '%s-%s' % (preferred_v, preferred_r)
215 if preferred_file is None:
216 bb.note("preferred version %s of %s not available" % (pv_str, pn))
218 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
220 # get highest priority file set
221 files = pkg_pn[pn][0]
226 the_data = make.pkgdata[f]
227 pv = bb.data.getVar('PV', the_data, 1)
228 pr = bb.data.getVar('PR', the_data, 1)
229 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', the_data, 1) or "0")
231 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
235 if preferred_file is None:
236 preferred_file = latest_f
237 preferred_ver = latest
239 preferred_versions[pn] = (preferred_ver, preferred_file)
240 latest_versions[pn] = (latest, latest_f)
242 pkg_list = pkg_pn.keys()
246 pref = preferred_versions[p]
247 latest = latest_versions[p]
250 prefstr = pref[0][0] + "-" + pref[0][1]
254 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
257 def buildPackage(item):
260 discriminated = False
262 if not providers.has_key(item):
263 bb.error("Nothing provides %s" % item)
266 all_p = providers[item]
269 if p in __build_cache:
270 bb.debug(1, "already built %s in this run\n" % p)
274 preferred_versions = {}
276 # Collate providers by PN
279 the_data = make.pkgdata[p]
280 pn = bb.data.getVar('PN', the_data, 1)
281 if not pkg_pn.has_key(pn):
285 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
288 for pn in pkg_pn.keys():
292 priority = bbfile_priority[f]
293 if not priorities.has_key(priority):
294 priorities[priority] = []
295 priorities[priority].append(f)
296 p_list = priorities.keys()
297 p_list.sort(lambda a, b: a - b)
300 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
302 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
303 # version. If not, find the latest version provided by an bbfile in the
304 # highest-priority set.
305 for pn in pkg_pn.keys():
306 preferred_file = None
308 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
311 m = re.match('(.*)_(.*)', preferred_v)
313 preferred_v = m.group(1)
314 preferred_r = m.group(2)
316 for file_set in pkg_pn[pn]:
318 the_data = make.pkgdata[f]
319 pv = bb.data.getVar('PV', the_data, 1)
320 pr = bb.data.getVar('PR', the_data, 1)
321 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
323 preferred_ver = (pv, pr)
328 pv_str = '%s-%s' % (preferred_v, preferred_r)
331 if preferred_file is None:
332 bb.note("preferred version %s of %s not available" % (pv_str, pn))
334 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
336 if preferred_file is None:
337 # get highest priority file set
338 files = pkg_pn[pn][0]
343 the_data = make.pkgdata[f]
344 pv = bb.data.getVar('PV', the_data, 1)
345 pr = bb.data.getVar('PR', the_data, 1)
346 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', the_data, 1) or "0")
348 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
352 preferred_file = latest_f
353 preferred_ver = latest
355 bb.debug(1, "selecting %s as latest version of provider %s" % (preferred_file, pn))
357 preferred_versions[pn] = (preferred_ver, preferred_file)
358 eligible.append(preferred_file)
361 if p in __build_cache_fail:
362 bb.debug(1, "rejecting already-failed %s" % p)
365 if len(eligible) == 0:
366 bb.error("no eligible providers for %s" % item)
369 # look to see if one of them is already staged, or marked as preferred.
370 # if so, bump it to the head of the queue
372 the_data = make.pkgdata[p]
373 pn = bb.data.getVar('PN', the_data, 1)
374 pv = bb.data.getVar('PV', the_data, 1)
375 pr = bb.data.getVar('PR', the_data, 1)
376 tmpdir = bb.data.getVar('TMPDIR', the_data, 1)
377 stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr)
378 if os.path.exists(stamp):
379 (newvers, fn) = preferred_versions[pn]
380 if not fn in eligible:
381 # package was made ineligible by already-failed check
383 oldver = "%s-%s" % (pv, pr)
384 newver = '-'.join(newvers)
385 if (newver != oldver):
386 extra_chat = "; upgrading from %s to %s" % (oldver, newver)
389 if make.options.verbose:
390 bb.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat))
392 eligible = [fn] + eligible
396 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, make.cfg, 1)
398 __preferred[item] = prefervar
400 if __preferred.has_key(item):
402 the_data = make.pkgdata[p]
403 pn = bb.data.getVar('PN', the_data, 1)
404 if __preferred[item] == pn:
405 if make.options.verbose:
406 bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
408 eligible = [p] + eligible
412 if len(eligible) > 1 and discriminated == False:
415 providers_list.append(bb.data.getVar('PN', make.pkgdata[fn], 1))
416 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
417 bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
419 # run through the list until we find one that we can build
421 bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
422 if try_build(fn, item):
425 bb.note("no buildable providers for %s" % item)
428 def build_depgraph():
433 if bbdebug or progress.p == p: return
435 if os.isatty(sys.stdout.fileno()):
436 sys.stdout.write("\rNOTE: Building provider hash: [%s%s] (%02d%%)" % ( "#" * (p/5), " " * ( 20 - p/5 ), p ) )
440 sys.stdout.write("NOTE: Building provider hash, please wait...\n")
442 sys.stdout.write("done.\n")
445 def calc_bbfile_priority(filename):
446 for (regex, pri) in bbfile_config_priorities:
447 if regex.match(filename):
451 # Handle PREFERRED_PROVIDERS
452 for p in (bb.data.getVar('PREFERRED_PROVIDERS', make.cfg, 1) or "").split():
453 (providee, provider) = p.split(':')
454 if __preferred.has_key(providee) and __preferred[providee] != provider:
455 bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, __preferred[providee]))
456 __preferred[providee] = provider
458 # Calculate priorities for each file
459 for p in make.pkgdata.keys():
460 bbfile_priority[p] = calc_bbfile_priority(p)
462 n = len(make.pkgdata.keys())
467 bb.debug(1, "BBMAKE building providers hashes")
469 # Build forward and reverse provider hashes
470 # Forward: virtual -> [filenames]
471 # Reverse: PN -> [virtuals]
472 for f in make.pkgdata.keys():
475 pn = bb.data.getVar('PN', d, 1)
476 provides = Set([pn] + (bb.data.getVar("PROVIDES", d, 1) or "").split())
478 if not pn_provides.has_key(pn):
479 pn_provides[pn] = Set()
480 pn_provides[pn] |= provides
482 for provide in provides:
483 if not providers.has_key(provide):
484 providers[provide] = []
485 providers[provide].append(f)
487 deps = (bb.data.getVar("DEPENDS", d, 1) or "").split()
498 sys.stdout.write("\n")
500 # Build package list for "bitbake world"
501 bb.debug(1, "BBMAKE collating packages for \"world\"")
502 for f in make.pkgdata.keys():
504 if bb.data.getVar('BROKEN', d, 1) or bb.data.getVar('EXCLUDE_FROM_WORLD', d, 1):
505 bb.debug(2, "BBMAKE skipping %s due to BROKEN/EXCLUDE_FROM_WORLD" % f)
508 pn = bb.data.getVar('PN', d, 1)
509 for p in pn_provides[pn]:
510 if p.startswith('virtual/'):
511 bb.debug(2, "BBMAKE skipping %s due to %s provider starting with virtual/" % (f, p))
514 for pf in providers[p]:
515 if bb.data.getVar('PN', make.pkgdata[pf], 1) != pn:
516 bb.debug(2, "BBMAKE skipping %s due to both us and %s providing %s" % (f, pf, p))
520 __world_target.add(pn)
522 def myProgressCallback( x, y, f ):
525 if os.isatty(sys.stdout.fileno()):
526 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
530 sys.stdout.write("Parsing .bb files, please wait...")
533 sys.stdout.write("done.")
536 def executeOneBB( fn ):
538 d = bb.parse.handle(fn, make.cfg)
540 bb.fatal("Unable to open %s" % fn)
542 name = bb.data.getVar('PN', d, 1)
543 bb.event.fire(bb.event.PkgStarted(name, d))
545 __stats["attempt"] += 1
546 if not make.options.dry_run:
547 bb.build.exec_task('do_%s' % make.options.cmd, d)
548 bb.event.fire(bb.event.PkgSucceeded(name, d))
549 __build_cache.append(fn)
550 except bb.build.FuncFailed:
552 bb.error("task stack execution failed")
553 bb.event.fire(bb.event.PkgFailed(name, d))
554 __build_cache_fail.append(fn)
555 except bb.build.EventException:
557 (type, value, traceback) = sys.exc_info()
559 bb.error("%s event exception, aborting" % bb.event.getName(e))
560 bb.event.fire(bb.event.PkgFailed(name, d))
561 __build_cache_fail.append(fn)
567 __stats["attempt"] = 0
568 __stats["success"] = 0
573 print "Build statistics:"
574 print " Attempted builds: %d" % __stats["attempt"]
575 if __stats["fail"] != 0:
576 print " Failed builds: %d" % __stats["fail"]
577 if __stats["deps"] != 0:
578 print " Dependencies not satisfied: %d" % __stats["deps"]
579 if __stats["fail"] != 0 or __stats["deps"] != 0:
583 if __name__ == "__main__":
585 if "BBDEBUG" in os.environ:
586 bbdebug = int(os.environ["BBDEBUG"])
588 make.options, args = handle_options( sys.argv )
590 if not make.options.cmd:
591 make.options.cmd = "build"
593 if make.options.cmd in __depcmds:
594 __depcmd=__depcmds[make.options.cmd]
596 __depcmd=make.options.cmd
599 make.cfg = bb.data.init()
602 for f in make.options.file:
604 make.cfg = bb.parse.handle(f, make.cfg)
606 bb.fatal("Unable to open %s" % f)
609 make.cfg = bb.parse.handle(os.path.join('conf', 'bitbake.conf'), make.cfg)
611 bb.fatal("Unable to open %s" % os.path.join('conf', 'bitbake.conf'))
613 if not bb.data.getVar("BUILDNAME", make.cfg):
614 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), make.cfg)
616 buildname = bb.data.getVar("BUILDNAME", make.cfg)
618 bf = make.options.buildfile
623 ignore = bb.data.getVar("ASSUME_PROVIDED", make.cfg, 1) or ""
624 __ignored_dependencies = ignore.split()
626 collections = bb.data.getVar("BBFILE_COLLECTIONS", make.cfg, 1)
628 collection_list = collections.split()
629 for c in collection_list:
630 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, make.cfg, 1)
632 bb.error("BBFILE_PATTERN_%s not defined" % c)
634 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, make.cfg, 1)
636 bb.error("BBFILE_PRIORITY_%s not defined" % c)
639 cre = re.compile(regex)
641 bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
645 bbfile_config_priorities.append((cre, pri))
647 bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
651 if not pkgs_to_build:
653 pkgs_to_build.extend(args)
654 if not pkgs_to_build:
655 bbpkgs = bb.data.getVar('BBPKGS', make.cfg, 1)
657 pkgs_to_build = bbpkgs.split()
658 if not pkgs_to_build and not make.options.show_versions:
659 print "Nothing to build. Use 'bitbake world' to build everything."
663 # Import Psyco if available and not disabled
664 if not make.options.disable_psyco:
669 bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
671 psyco.bind( make.collect_bbfiles )
673 bb.note("You have disabled Psyco. This decreases performance.")
676 bb.debug(1, "BBMAKE collecting .bb files")
677 make.collect_bbfiles( myProgressCallback )
678 bb.debug(1, "BBMAKE parsing complete")
681 if make.options.parse_only:
682 print "Requested parsing .bb files only. Exiting."
687 if make.options.show_versions:
691 if 'world' in pkgs_to_build:
692 pkgs_to_build.remove('world')
693 for t in __world_target:
694 pkgs_to_build.append(t)
696 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, make.cfg))
698 for k in pkgs_to_build:
701 if buildPackage(k) == 0:
704 except bb.build.EventException:
705 bb.error("Build of " + k + " failed")
709 if make.options.abort:
712 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, make.cfg))
716 except KeyboardInterrupt:
717 print "\nNOTE: KeyboardInterrupt - Build not completed."