2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright (C) 2003, 2004 Chris Larson
6 # Copyright (C) 2003, 2004 Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005 Holger Hans Peter Freyther
9 # Copyright (C) 2005 ROAD GmbH
11 # This program is free software; you can redistribute it and/or modify it under
12 # the terms of the GNU General Public License as published by the Free Software
13 # Foundation; either version 2 of the License, or (at your option) any later
16 # This program is distributed in the hope that it will be useful, but WITHOUT
17 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
20 # You should have received a copy of the GNU General Public License along with
21 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
22 # Place, Suite 330, Boston, MA 02111-1307 USA.
24 import sys, os, getopt, glob, copy, os.path, re
25 sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
29 import itertools, optparse
31 parsespin = itertools.cycle( r'|/-\\' )
36 #============================================================================#
38 #============================================================================#
39 class BBParsingStatus:
41 The initial idea for this status class is to use the data when it is
42 already loaded instead of loading it from various place over and over
47 self.cache_dirty = False
49 self.bbfile_priority = {}
50 self.bbfile_config_priorities = []
51 self.ignored_depedencies = None
52 self.possible_world = []
53 self.world_target = Set()
59 self.all_depends = Set()
61 def handle_bb_data(self, file_name, bb_data, cached):
63 We will fill the dictionaries with the stuff we
64 need for building the tree more fast
70 self.cache_dirty = True
72 pn = bb.data.getVar('PN', bb_data, True)
73 pv = bb.data.getVar('PV', bb_data, True)
74 pr = bb.data.getVar('PR', bb_data, True)
75 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', bb_data, True) or "0")
76 provides = Set([pn] + (bb.data.getVar("PROVIDES", bb_data, 1) or "").split())
77 depends = (bb.data.getVar("DEPENDS", bb_data, True) or "").split()
80 # build PackageName to FileName lookup table
81 if not self.pkg_pn.has_key(pn):
83 self.pkg_pn[pn].append(file_name)
85 # build FileName to PackageName lookup table
86 self.pkg_fn[file_name] = pn
87 self.pkg_pvpr[file_name] = (pv,pr)
88 self.pkg_dp[file_name] = dp
90 # Build forward and reverse provider hashes
91 # Forward: virtual -> [filenames]
92 # Reverse: PN -> [virtuals]
93 if not self.pn_provides.has_key(pn):
94 self.pn_provides[pn] = Set()
95 self.pn_provides[pn] |= provides
97 for provide in provides:
98 if not self.providers.has_key(provide):
99 self.providers[provide] = []
100 self.providers[provide].append(file_name)
103 self.all_depends.add(dep)
105 # Collect files we may need for possible world-dep
107 if not bb.data.getVar('BROKEN', bb_data, True) and not bb.data.getVar('EXCLUDE_FROM_WORLD', bb_data, True):
108 self.possible_world.append(file_name)
111 #============================================================================#
113 #============================================================================#
116 Manage build statistics for one run
125 print "Build statistics:"
126 print " Attempted builds: %d" % self.attempt
128 print " Failed builds: %d" % self.fail
130 print " Dependencies not satisfied: %d" % self.deps
131 if self.fail or self.deps: return 1
135 #============================================================================#
137 #============================================================================#
140 Manages one bitbake build run
142 def __init__( self ):
143 self.build_cache_fail = []
144 self.build_cache = []
145 self.building_list = []
147 self.consider_msgs_cache = []
149 self.stats = BBStatistics()
152 def tryBuild( self, fn, virtual ):
153 if fn in self.building_list:
154 bb.error("%s depends on itself (eventually)" % fn)
155 bb.error("upwards chain is: %s" % (" -> ".join(self.build_path)))
158 the_data = make.pkgdata[fn]
159 item = self.status.pkg_fn[fn]
161 self.building_list.append(fn)
163 pathstr = "%s (%s)" % (item, virtual)
164 self.build_path.append(pathstr)
166 depends_list = (bb.data.getVar('DEPENDS', the_data, 1) or "").split()
167 if make.options.verbose:
168 bb.note("current path: %s" % (" -> ".join(self.build_path)))
169 bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
174 depcmd = make.options.cmd
175 bbdepcmd = bb.data.getVarFlag('do_%s' % make.options.cmd, 'bbdepcmd', the_data)
176 if bbdepcmd is not None:
183 oldcmd = make.options.cmd
184 make.options.cmd = depcmd
186 for d in depends_list:
187 if d in self.status.ignored_dependencies:
191 if self.buildPackage(d) == 0:
192 bb.error("dependency %s (for %s) not satisfied" % (d,item))
194 if make.options.abort:
198 make.options.cmd = oldcmd
204 if bb.build.stamp_is_current('do_%s' % make.options.cmd, the_data):
205 self.build_cache.append(fn)
208 bb.event.fire(bb.event.PkgStarted(item, the_data))
210 self.stats.attempt += 1
211 if not make.options.dry_run:
212 bb.build.exec_task('do_%s' % make.options.cmd, the_data)
213 bb.event.fire(bb.event.PkgSucceeded(item, the_data))
214 self.build_cache.append(fn)
216 except bb.build.FuncFailed:
218 bb.error("task stack execution failed")
219 bb.event.fire(bb.event.PkgFailed(item, the_data))
220 self.build_cache_fail.append(fn)
222 except bb.build.EventException:
224 (type, value, traceback) = sys.exc_info()
226 bb.error("%s event exception, aborting" % bb.event.getName(e))
227 bb.event.fire(bb.event.PkgFailed(item, the_data))
228 self.build_cache_fail.append(fn)
231 self.building_list.remove(fn)
232 self.build_path.remove(pathstr)
234 def showVersions( self ):
235 pkg_pn = self.status.pkg_pn
236 preferred_versions = {}
240 for pn in pkg_pn.keys():
244 priority = self.status.bbfile_priority[f]
245 if not priorities.has_key(priority):
246 priorities[priority] = []
247 priorities[priority].append(f)
248 p_list = priorities.keys()
249 p_list.sort(lambda a, b: a - b)
252 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
254 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
255 # version. If not, find the latest version provided by an bbfile in the
256 # highest-priority set.
257 for pn in pkg_pn.keys():
258 preferred_file = None
260 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
263 m = re.match('(.*)_(.*)', preferred_v)
265 preferred_v = m.group(1)
266 preferred_r = m.group(2)
268 for file_set in pkg_pn[pn]:
270 pv,pr = self.status.pkg_pvpr[f]
271 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
273 preferred_ver = (pv, pr)
278 pv_str = '%s-%s' % (preferred_v, preferred_r)
281 if preferred_file is None:
282 bb.note("preferred version %s of %s not available" % (pv_str, pn))
284 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
286 # get highest priority file set
287 files = pkg_pn[pn][0]
292 pv,pr = self.status.pkg_pvpr[f]
293 dp = self.status.pkg_dp[f]
295 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
299 if preferred_file is None:
300 preferred_file = latest_f
301 preferred_ver = latest
303 preferred_versions[pn] = (preferred_ver, preferred_file)
304 latest_versions[pn] = (latest, latest_f)
306 pkg_list = pkg_pn.keys()
310 pref = preferred_versions[p]
311 latest = latest_versions[p]
314 prefstr = pref[0][0] + "-" + pref[0][1]
318 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
321 def buildPackage( self, item ):
324 discriminated = False
326 if not self.status.providers.has_key(item):
327 bb.error("Nothing provides %s" % item)
330 all_p = self.status.providers[item]
333 if p in self.build_cache:
334 bb.debug(1, "already built %s in this run\n" % p)
338 preferred_versions = {}
340 # Collate providers by PN
343 pn = self.status.pkg_fn[p]
344 if not pkg_pn.has_key(pn):
348 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
351 for pn in pkg_pn.keys():
355 priority = self.status.bbfile_priority[f]
356 if not priorities.has_key(priority):
357 priorities[priority] = []
358 priorities[priority].append(f)
359 p_list = priorities.keys()
360 p_list.sort(lambda a, b: a - b)
363 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
365 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
366 # version. If not, find the latest version provided by an bbfile in the
367 # highest-priority set.
368 for pn in pkg_pn.keys():
369 preferred_file = None
371 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
374 m = re.match('(.*)_(.*)', preferred_v)
376 preferred_v = m.group(1)
377 preferred_r = m.group(2)
379 for file_set in pkg_pn[pn]:
381 pv,pr = self.status.pkg_pvpr[f]
382 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
384 preferred_ver = (pv, pr)
389 pv_str = '%s-%s' % (preferred_v, preferred_r)
392 if preferred_file is None:
393 bb.note("preferred version %s of %s not available" % (pv_str, pn))
395 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
397 if preferred_file is None:
398 # get highest priority file set
399 files = pkg_pn[pn][0]
404 pv,pr = self.status.pkg_pvpr[f]
405 dp = self.status.pkg_dp[f]
407 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
411 preferred_file = latest_f
412 preferred_ver = latest
414 bb.debug(1, "selecting %s as latest version of provider %s" % (preferred_file, pn))
416 preferred_versions[pn] = (preferred_ver, preferred_file)
417 eligible.append(preferred_file)
420 if p in self.build_cache_fail:
421 bb.debug(1, "rejecting already-failed %s" % p)
424 if len(eligible) == 0:
425 bb.error("no eligible providers for %s" % item)
428 # look to see if one of them is already staged, or marked as preferred.
429 # if so, bump it to the head of the queue
431 the_data = make.pkgdata[p]
432 pn = bb.data.getVar('PN', the_data, 1)
433 pv = bb.data.getVar('PV', the_data, 1)
434 pr = bb.data.getVar('PR', the_data, 1)
435 tmpdir = bb.data.getVar('TMPDIR', the_data, 1)
436 stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr)
437 if os.path.exists(stamp):
438 (newvers, fn) = preferred_versions[pn]
439 if not fn in eligible:
440 # package was made ineligible by already-failed check
442 oldver = "%s-%s" % (pv, pr)
443 newver = '-'.join(newvers)
444 if (newver != oldver):
445 extra_chat = "; upgrading from %s to %s" % (oldver, newver)
448 if make.options.verbose:
449 bb.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat))
451 eligible = [fn] + eligible
455 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, make.cfg, 1)
457 self.preferred[item] = prefervar
459 if self.preferred.has_key(item):
461 pn = self.status.pkg_fn[p]
462 if self.preferred[item] == pn:
463 if make.options.verbose:
464 bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
466 eligible = [p] + eligible
470 if len(eligible) > 1 and discriminated == False:
471 if item not in self.consider_msgs_cache:
474 providers_list.append(self.status.pkg_fn[fn])
475 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
476 bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
477 self.consider_msgs_cache.append(item)
480 # run through the list until we find one that we can build
482 bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
483 if self.tryBuild(fn, item):
486 bb.note("no buildable providers for %s" % item)
489 def buildDepgraph( self ):
490 all_depends = self.status.all_depends
491 pn_provides = self.status.pn_provides
493 def calc_bbfile_priority(filename):
494 for (regex, pri) in self.status.bbfile_config_priorities:
495 if regex.match(filename):
499 # Handle PREFERRED_PROVIDERS
500 for p in (bb.data.getVar('PREFERRED_PROVIDERS', make.cfg, 1) or "").split():
501 (providee, provider) = p.split(':')
502 if self.preferred.has_key(providee) and self.preferred[providee] != provider:
503 bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.preferred[providee]))
504 self.preferred[providee] = provider
506 # Calculate priorities for each file
507 for p in make.pkgdata.keys():
508 self.status.bbfile_priority[p] = calc_bbfile_priority(p)
510 # Build package list for "bitbake world"
511 bb.debug(1, "collating packages for \"world\"")
512 for f in self.status.possible_world:
514 pn = self.status.pkg_fn[f]
516 for p in pn_provides[pn]:
517 if p.startswith('virtual/'):
518 bb.debug(2, "skipping %s due to %s provider starting with virtual/" % (f, p))
521 for pf in self.status.providers[p]:
522 if self.status.pkg_fn[pf] != pn:
523 bb.debug(2, "skipping %s due to both us and %s providing %s" % (f, pf, p))
527 self.status.world_target.add(pn)
529 # drop reference count now
530 self.status.possible_world = None
531 self.status.all_depends = None
533 def myProgressCallback( self, x, y, f, file_data, from_cache ):
534 # feed the status with new input
535 self.status.handle_bb_data(f, file_data, from_cache)
539 if os.isatty(sys.stdout.fileno()):
540 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
544 sys.stdout.write("Parsing .bb files, please wait...")
547 sys.stdout.write("done.")
550 def executeOneBB( self, fn ):
552 d = bb.parse.handle(fn, make.cfg)
554 bb.fatal("Unable to open %s" % fn)
556 if make.options.parse_only:
557 print "Requested parsing .bb files only. Exiting."
560 name = bb.data.getVar('PN', d, 1)
561 bb.event.fire(bb.event.PkgStarted(name, d))
563 self.stats.attempt += 1
564 if make.options.force:
565 bb.data.setVarFlag('do_%s' % make.options.cmd, 'force', 1, d)
566 if not make.options.dry_run:
567 bb.build.exec_task('do_%s' % make.options.cmd, d)
568 bb.event.fire(bb.event.PkgSucceeded(name, d))
569 self.build_cache.append(fn)
570 except bb.build.FuncFailed:
572 bb.error("task stack execution failed")
573 bb.event.fire(bb.event.PkgFailed(name, d))
574 self.build_cache_fail.append(fn)
575 except bb.build.EventException:
577 (type, value, traceback) = sys.exc_info()
579 bb.error("%s event exception, aborting" % bb.event.getName(e))
580 bb.event.fire(bb.event.PkgFailed(name, d))
581 self.build_cache_fail.append(fn)
583 def interactiveMode( self ):
587 print "Sorry, the shell is not available yet."
592 def cook( self, args ):
593 if not make.options.cmd:
594 make.options.cmd = "build"
596 if make.options.debug:
597 bb.debug_level = make.options.debug
599 make.cfg = bb.data.init()
601 for f in make.options.file:
603 make.cfg = bb.parse.handle(f, make.cfg)
605 bb.fatal("Unable to open %s" % f)
608 make.cfg = bb.parse.handle(os.path.join('conf', 'bitbake.conf'), make.cfg)
610 bb.fatal("Unable to open %s" % os.path.join('conf', 'bitbake.conf'))
612 if not bb.data.getVar("BUILDNAME", make.cfg):
613 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), make.cfg)
615 buildname = bb.data.getVar("BUILDNAME", make.cfg)
617 if make.options.interactive:
618 self.interactiveMode()
620 bf = make.options.buildfile
622 self.executeOneBB( os.path.abspath(bf) )
623 sys.exit( self.stats.show() )
625 # initialise the parsing status now we know we will need deps
626 self.status = BBParsingStatus()
628 ignore = bb.data.getVar("ASSUME_PROVIDED", make.cfg, 1) or ""
629 self.status.ignored_dependencies = Set( ignore.split() )
631 collections = bb.data.getVar("BBFILE_COLLECTIONS", make.cfg, 1)
633 collection_list = collections.split()
634 for c in collection_list:
635 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, make.cfg, 1)
637 bb.error("BBFILE_PATTERN_%s not defined" % c)
639 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, make.cfg, 1)
641 bb.error("BBFILE_PRIORITY_%s not defined" % c)
644 cre = re.compile(regex)
646 bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
650 self.status.bbfile_config_priorities.append((cre, pri))
652 bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
656 if not pkgs_to_build:
658 pkgs_to_build.extend(args)
659 if not pkgs_to_build:
660 bbpkgs = bb.data.getVar('BBPKGS', make.cfg, 1)
662 pkgs_to_build = bbpkgs.split()
663 if not pkgs_to_build and not make.options.show_versions:
664 print "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help'"
665 print "for usage information."
669 # Import Psyco if available and not disabled
670 if not make.options.disable_psyco:
675 bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
677 psyco.bind( make.collect_bbfiles )
679 bb.note("You have disabled Psyco. This decreases performance.")
682 bb.debug(1, "collecting .bb files")
683 make.collect_bbfiles( self.myProgressCallback )
684 bb.debug(1, "parsing complete")
687 if make.options.parse_only:
688 print "Requested parsing .bb files only. Exiting."
693 if make.options.show_versions:
696 if 'world' in pkgs_to_build:
697 pkgs_to_build.remove('world')
698 for t in self.status.world_target:
699 pkgs_to_build.append(t)
701 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, make.cfg))
703 for k in pkgs_to_build:
706 if self.buildPackage(k) == 0:
709 except bb.build.EventException:
710 bb.error("Build of " + k + " failed")
714 if make.options.abort:
717 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, make.cfg))
719 sys.exit( self.stats.show() )
721 except KeyboardInterrupt:
722 print "\nNOTE: KeyboardInterrupt - Build not completed."
725 #============================================================================#
727 #============================================================================#
729 if __name__ == "__main__":
731 parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
732 usage = """%prog [options] [package ...]
734 Executes the specified task (default is 'build') for a given set of BitBake files.
735 It expects that BBFILES is defined, which is a space seperated list of files to
736 be executed. BBFILES does support wildcards.
737 Default BBFILES are the .bb files in the current directory.""" )
739 parser.add_option( "-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES.",
740 action = "store", dest = "buildfile", default = None )
742 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
743 action = "store_false", dest = "abort", default = True )
745 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
746 action = "store_true", dest = "force", default = False )
748 parser.add_option( "-i", "--interactive", help = "drop into the interactive mode.",
749 action = "store_true", dest = "interactive", default = False )
751 parser.add_option( "-c", "--cmd", help = "Specify task to execute",
752 action = "store", dest = "cmd", default = "build" )
754 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
755 action = "append", dest = "file", default = [] )
757 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
758 action = "store_true", dest = "verbose", default = False )
760 parser.add_option( "-D", "--debug", help = "Increase the debug level",
761 action = "count", dest="debug", default = 0)
763 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
764 action = "store_true", dest = "dry_run", default = False )
766 parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
767 action = "store_true", dest = "parse_only", default = False )
769 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
770 action = "store_true", dest = "disable_psyco", default = False )
772 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
773 action = "store_true", dest = "show_versions", default = False )
775 options, args = parser.parse_args( sys.argv )
777 make.options = options
779 cooker.cook( args[1:] )