2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright (C) 2003, 2004 Chris Larson
6 # Copyright (C) 2003, 2004 Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005 Holger Hans Peter Freyther
9 # Copyright (C) 2005 ROAD GmbH
11 # This program is free software; you can redistribute it and/or modify it under
12 # the terms of the GNU General Public License as published by the Free Software
13 # Foundation; either version 2 of the License, or (at your option) any later
16 # This program is distributed in the hope that it will be useful, but WITHOUT
17 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
20 # You should have received a copy of the GNU General Public License along with
21 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
22 # Place, Suite 330, Boston, MA 02111-1307 USA.
24 import sys, os, getopt, glob, copy, os.path, re
25 sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
29 import itertools, optparse
31 parsespin = itertools.cycle( r'|/-\\' )
36 #============================================================================#
38 #============================================================================#
39 class BBParsingStatus:
41 The initial idea for this status class is to use the data when it is
42 already loaded instead of loading it from various place over and over
47 self.cache_dirty = False
49 self.bbfile_priority = {}
50 self.bbfile_config_priorities = []
51 self.ignored_depedencies = None
52 self.possible_world = []
53 self.world_target = Set()
59 self.all_depends = Set()
61 def handle_bb_data(self, file_name, bb_data, cached):
63 We will fill the dictionaries with the stuff we
64 need for building the tree more fast
70 self.cache_dirty = True
72 pn = bb.data.getVar('PN', bb_data, True)
73 pv = bb.data.getVar('PV', bb_data, True)
74 pr = bb.data.getVar('PR', bb_data, True)
75 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', bb_data, True) or "0")
76 provides = Set([pn] + (bb.data.getVar("PROVIDES", bb_data, 1) or "").split())
77 depends = (bb.data.getVar("DEPENDS", bb_data, True) or "").split()
80 # build PackageName to FileName lookup table
81 if pn not in self.pkg_pn:
83 self.pkg_pn[pn].append(file_name)
85 # build FileName to PackageName lookup table
86 self.pkg_fn[file_name] = pn
87 self.pkg_pvpr[file_name] = (pv,pr)
88 self.pkg_dp[file_name] = dp
90 # Build forward and reverse provider hashes
91 # Forward: virtual -> [filenames]
92 # Reverse: PN -> [virtuals]
93 if pn not in self.pn_provides:
94 self.pn_provides[pn] = Set()
95 self.pn_provides[pn] |= provides
97 for provide in provides:
98 if provide not in self.providers:
99 self.providers[provide] = []
100 self.providers[provide].append(file_name)
103 self.all_depends.add(dep)
105 # Collect files we may need for possible world-dep
107 if not bb.data.getVar('BROKEN', bb_data, True) and not bb.data.getVar('EXCLUDE_FROM_WORLD', bb_data, True):
108 self.possible_world.append(file_name)
111 #============================================================================#
113 #============================================================================#
116 Manage build statistics for one run
125 print "Build statistics:"
126 print " Attempted builds: %d" % self.attempt
128 print " Failed builds: %d" % self.fail
130 print " Dependencies not satisfied: %d" % self.deps
131 if self.fail or self.deps: return 1
135 #============================================================================#
137 #============================================================================#
140 Manages one bitbake build run
143 ParsingStatus = BBParsingStatus # make it visible from the shell
144 Statistics = BBStatistics # make it visible from the shell
146 def __init__( self ):
147 self.build_cache_fail = []
148 self.build_cache = []
149 self.building_list = []
151 self.consider_msgs_cache = []
153 self.stats = BBStatistics()
156 def tryBuildPackage( self, fn, item, the_data ):
157 """Build one package"""
158 bb.event.fire(bb.event.PkgStarted(item, the_data))
160 self.stats.attempt += 1
161 if make.options.force:
162 bb.data.setVarFlag('do_%s' % make.options.cmd, 'force', 1, the_data)
163 if not make.options.dry_run:
164 bb.build.exec_task('do_%s' % make.options.cmd, the_data)
165 bb.event.fire(bb.event.PkgSucceeded(item, the_data))
166 self.build_cache.append(fn)
168 except bb.build.FuncFailed:
170 bb.error("task stack execution failed")
171 bb.event.fire(bb.event.PkgFailed(item, the_data))
172 self.build_cache_fail.append(fn)
174 except bb.build.EventException:
176 (type, value, traceback) = sys.exc_info()
178 bb.error("%s event exception, aborting" % bb.event.getName(e))
179 bb.event.fire(bb.event.PkgFailed(item, the_data))
180 self.build_cache_fail.append(fn)
183 def tryBuild( self, fn, virtual ):
184 """Build a provider and its dependencies"""
185 if fn in self.building_list:
186 bb.error("%s depends on itself (eventually)" % fn)
187 bb.error("upwards chain is: %s" % (" -> ".join(self.build_path)))
190 the_data = make.pkgdata[fn]
191 item = self.status.pkg_fn[fn]
193 self.building_list.append(fn)
195 pathstr = "%s (%s)" % (item, virtual)
196 self.build_path.append(pathstr)
198 depends_list = (bb.data.getVar('DEPENDS', the_data, 1) or "").split()
199 if make.options.verbose:
200 bb.note("current path: %s" % (" -> ".join(self.build_path)))
201 bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
206 depcmd = make.options.cmd
207 bbdepcmd = bb.data.getVarFlag('do_%s' % make.options.cmd, 'bbdepcmd', the_data)
208 if bbdepcmd is not None:
215 oldcmd = make.options.cmd
216 make.options.cmd = depcmd
218 for dependency in depends_list:
219 if dependency in self.status.ignored_dependencies:
223 if self.buildProvider( dependency ) == 0:
224 bb.error("dependency %s (for %s) not satisfied" % (dependency,item))
226 if make.options.abort:
230 make.options.cmd = oldcmd
236 if bb.build.stamp_is_current('do_%s' % make.options.cmd, the_data):
237 self.build_cache.append(fn)
240 return self.tryBuildPackage( fn, item, the_data )
243 self.building_list.remove(fn)
244 self.build_path.remove(pathstr)
246 def findBestProvider( self, pn ):
248 If there is a PREFERRED_VERSION, find the highest-priority bbfile
249 providing that version. If not, find the latest version provided by
250 an bbfile in the highest-priority set.
252 pkg_pn = self.status.pkg_pn
257 priority = self.status.bbfile_priority[f]
258 if priority not in priorities:
259 priorities[priority] = []
260 priorities[priority].append(f)
261 p_list = priorities.keys()
262 p_list.sort(lambda a, b: a - b)
265 tmp_pn = [priorities[p]] + tmp_pn
268 preferred_file = None
270 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
272 m = re.match('(.*)_(.*)', preferred_v)
274 preferred_v = m.group(1)
275 preferred_r = m.group(2)
279 for file_set in pkg_pn[pn]:
281 pv,pr = self.status.pkg_pvpr[f]
282 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
284 preferred_ver = (pv, pr)
289 pv_str = '%s-%s' % (preferred_v, preferred_r)
292 if preferred_file is None:
293 bb.note("preferred version %s of %s not available" % (pv_str, pn))
295 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
297 # get highest priority file set
298 files = pkg_pn[pn][0]
302 for file_name in files:
303 pv,pr = self.status.pkg_pvpr[file_name]
304 dp = self.status.pkg_dp[file_name]
306 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
310 if preferred_file is None:
311 preferred_file = latest_f
312 preferred_ver = latest
314 return (latest,latest_f,preferred_ver, preferred_file)
316 def showVersions( self ):
317 pkg_pn = self.status.pkg_pn
318 preferred_versions = {}
322 for pn in pkg_pn.keys():
323 (last_ver,last_file,pref_ver,pref_file) = self.findBestProvider(pn)
324 preferred_versions[pn] = (pref_ver, pref_file)
325 latest_versions[pn] = (last_ver, last_file)
327 pkg_list = pkg_pn.keys()
331 pref = preferred_versions[p]
332 latest = latest_versions[p]
335 prefstr = pref[0][0] + "-" + pref[0][1]
339 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
342 def buildProvider( self, item ):
345 discriminated = False
347 if item not in self.status.providers:
348 bb.error("Nothing provides %s" % item)
351 all_p = self.status.providers[item]
354 if p in self.build_cache:
355 bb.debug(1, "already built %s in this run\n" % p)
359 preferred_versions = {}
361 # Collate providers by PN
364 pn = self.status.pkg_fn[p]
369 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
371 for pn in pkg_pn.keys():
372 preferred_versions[pn] = self.findBestProvider(pn)[2:4]
373 eligible.append(preferred_versions[pn][1])
376 if p in self.build_cache_fail:
377 bb.debug(1, "rejecting already-failed %s" % p)
380 if len(eligible) == 0:
381 bb.error("no eligible providers for %s" % item)
384 # look to see if one of them is already staged, or marked as preferred.
385 # if so, bump it to the head of the queue
387 the_data = make.pkgdata[p]
388 pn = bb.data.getVar('PN', the_data, 1)
389 pv = bb.data.getVar('PV', the_data, 1)
390 pr = bb.data.getVar('PR', the_data, 1)
391 tmpdir = bb.data.getVar('TMPDIR', the_data, 1)
392 stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr)
393 if os.path.exists(stamp):
394 (newvers, fn) = preferred_versions[pn]
395 if not fn in eligible:
396 # package was made ineligible by already-failed check
398 oldver = "%s-%s" % (pv, pr)
399 newver = '-'.join(newvers)
400 if (newver != oldver):
401 extra_chat = "; upgrading from %s to %s" % (oldver, newver)
404 if make.options.verbose:
405 bb.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat))
407 eligible = [fn] + eligible
411 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, make.cfg, 1)
413 self.preferred[item] = prefervar
415 if item in self.preferred:
417 pn = self.status.pkg_fn[p]
418 if self.preferred[item] == pn:
419 if make.options.verbose:
420 bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
422 eligible = [p] + eligible
426 if len(eligible) > 1 and discriminated == False:
427 if item not in self.consider_msgs_cache:
430 providers_list.append(self.status.pkg_fn[fn])
431 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
432 bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
433 self.consider_msgs_cache.append(item)
436 # run through the list until we find one that we can build
438 bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
439 if self.tryBuild(fn, item):
442 bb.note("no buildable providers for %s" % item)
445 def buildDepgraph( self ):
446 all_depends = self.status.all_depends
447 pn_provides = self.status.pn_provides
449 def calc_bbfile_priority(filename):
450 for (regex, pri) in self.status.bbfile_config_priorities:
451 if regex.match(filename):
455 # Handle PREFERRED_PROVIDERS
456 for p in (bb.data.getVar('PREFERRED_PROVIDERS', make.cfg, 1) or "").split():
457 (providee, provider) = p.split(':')
458 if providee in self.preferred and self.preferred[providee] != provider:
459 bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.preferred[providee]))
460 self.preferred[providee] = provider
462 # Calculate priorities for each file
463 for p in make.pkgdata.keys():
464 self.status.bbfile_priority[p] = calc_bbfile_priority(p)
466 # Build package list for "bitbake world"
467 bb.debug(1, "collating packages for \"world\"")
468 for f in self.status.possible_world:
470 pn = self.status.pkg_fn[f]
472 for p in pn_provides[pn]:
473 if p.startswith('virtual/'):
474 bb.debug(2, "skipping %s due to %s provider starting with virtual/" % (f, p))
477 for pf in self.status.providers[p]:
478 if self.status.pkg_fn[pf] != pn:
479 bb.debug(2, "skipping %s due to both us and %s providing %s" % (f, pf, p))
483 self.status.world_target.add(pn)
485 # drop reference count now
486 self.status.possible_world = None
487 self.status.all_depends = None
489 def myProgressCallback( self, x, y, f, file_data, from_cache ):
490 # feed the status with new input
491 self.status.handle_bb_data(f, file_data, from_cache)
495 if os.isatty(sys.stdout.fileno()):
496 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
500 sys.stdout.write("Parsing .bb files, please wait...")
503 sys.stdout.write("done.")
506 def interactiveMode( self ):
507 """Drop off into a shell"""
510 except ImportError, details:
511 bb.fatal("Sorry, shell not available (%s)" % details )
516 def parseConfigurationFile( self, afile ):
518 make.cfg = bb.parse.handle( afile, make.cfg )
520 bb.fatal( "Unable to open %s" % afile )
521 except bb.parse.ParseError:
522 bb.fatal( "Unable to parse %s" % afile )
524 def handleCollections( self, collections ):
525 """Handle collections"""
527 collection_list = collections.split()
528 for c in collection_list:
529 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, make.cfg, 1)
531 bb.error("BBFILE_PATTERN_%s not defined" % c)
533 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, make.cfg, 1)
535 bb.error("BBFILE_PRIORITY_%s not defined" % c)
538 cre = re.compile(regex)
540 bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
544 self.status.bbfile_config_priorities.append((cre, pri))
546 bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
549 def cook( self, args ):
550 if not make.options.cmd:
551 make.options.cmd = "build"
553 if make.options.debug:
554 bb.debug_level = make.options.debug
556 make.cfg = bb.data.init()
558 for f in make.options.file:
559 self.parseConfigurationFile( f )
561 self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
563 if not bb.data.getVar("BUILDNAME", make.cfg):
564 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), make.cfg)
566 buildname = bb.data.getVar("BUILDNAME", make.cfg)
568 if make.options.interactive:
569 self.interactiveMode()
571 if make.options.buildfile is not None:
572 bf = os.path.abspath( make.options.buildfile )
574 bbfile_data = bb.parse.handle(bf, make.cfg)
576 bb.fatal("Unable to open %s" % bf)
578 item = bb.data.getVar('PN', bbfile_data, 1)
579 self.tryBuildPackage( bf, item, bbfile_data )
580 sys.exit( self.stats.show() )
582 # initialise the parsing status now we know we will need deps
583 self.status = BBParsingStatus()
585 ignore = bb.data.getVar("ASSUME_PROVIDED", make.cfg, 1) or ""
586 self.status.ignored_dependencies = Set( ignore.split() )
588 self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", make.cfg, 1) )
592 if not pkgs_to_build:
594 pkgs_to_build.extend(args)
595 if not pkgs_to_build:
596 bbpkgs = bb.data.getVar('BBPKGS', make.cfg, 1)
598 pkgs_to_build = bbpkgs.split()
599 if not pkgs_to_build and not make.options.show_versions and not make.options.interactive:
600 print "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help'"
601 print "for usage information."
604 # Import Psyco if available and not disabled
605 if not make.options.disable_psyco:
610 bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
612 psyco.bind( make.collect_bbfiles )
614 bb.note("You have disabled Psyco. This decreases performance.")
617 bb.debug(1, "collecting .bb files")
618 make.collect_bbfiles( self.myProgressCallback )
619 bb.debug(1, "parsing complete")
622 if make.options.parse_only:
623 print "Requested parsing .bb files only. Exiting."
626 bb.data.update_data( make.cfg )
629 if make.options.show_versions:
632 if 'world' in pkgs_to_build:
633 pkgs_to_build.remove('world')
634 for t in self.status.world_target:
635 pkgs_to_build.append(t)
637 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, make.cfg))
639 for k in pkgs_to_build:
642 if self.buildProvider( k ) == 0:
645 except bb.build.EventException:
646 bb.error("Build of " + k + " failed")
650 if make.options.abort:
653 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, make.cfg))
655 sys.exit( self.stats.show() )
657 except KeyboardInterrupt:
658 print "\nNOTE: KeyboardInterrupt - Build not completed."
661 #============================================================================#
663 #============================================================================#
665 if __name__ == "__main__":
667 parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
668 usage = """%prog [options] [package ...]
670 Executes the specified task (default is 'build') for a given set of BitBake files.
671 It expects that BBFILES is defined, which is a space seperated list of files to
672 be executed. BBFILES does support wildcards.
673 Default BBFILES are the .bb files in the current directory.""" )
675 parser.add_option( "-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES.",
676 action = "store", dest = "buildfile", default = None )
678 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
679 action = "store_false", dest = "abort", default = True )
681 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
682 action = "store_true", dest = "force", default = False )
684 parser.add_option( "-i", "--interactive", help = "drop into the interactive mode.",
685 action = "store_true", dest = "interactive", default = False )
687 parser.add_option( "-c", "--cmd", help = "Specify task to execute",
688 action = "store", dest = "cmd", default = "build" )
690 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
691 action = "append", dest = "file", default = [] )
693 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
694 action = "store_true", dest = "verbose", default = False )
696 parser.add_option( "-D", "--debug", help = "Increase the debug level",
697 action = "count", dest="debug", default = 0)
699 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
700 action = "store_true", dest = "dry_run", default = False )
702 parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
703 action = "store_true", dest = "parse_only", default = False )
705 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
706 action = "store_true", dest = "disable_psyco", default = False )
708 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
709 action = "store_true", dest = "show_versions", default = False )
711 options, args = parser.parse_args( sys.argv )
713 make.options = options
715 cooker.cook( args[1:] )