runqueue.py: Change failed tasks handling so all failed tasks are reported, not just...
[vuplus_bitbake] / bin / bitbake
index 8ec8d64..253ee09 100755 (executable)
@@ -31,7 +31,7 @@ import itertools, optparse
 
 parsespin = itertools.cycle( r'|/-\\' )
 
-__version__ = "1.7.0"
+__version__ = "1.7.4"
 
 #============================================================================#
 # BBStatistics
@@ -124,11 +124,11 @@ class BBCooker:
         If build_depends is empty, we're dealing with a runtime depends
         """
 
-        the_data = self.bb_cache.loadDataFull(fn, self)
+        the_data = self.bb_cache.loadDataFull(fn, self.configuration.data)
 
         item = self.status.pkg_fn[fn]
 
-        if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
+        if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data) and not self.configuration.force:
             self.build_cache.append(fn)
             return True
 
@@ -167,7 +167,7 @@ class BBCooker:
             self.cb = None
             self.bb_cache = bb.cache.init(self)
             try:
-                self.configuration.data = self.bb_cache.loadDataFull(self.configuration.buildfile, self)
+                self.configuration.data = self.bb_cache.loadDataFull(self.configuration.buildfile, self.configuration.data)
             except IOError, e:
                 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to read %s: %s" % ( self.configuration.buildfile, e ))
             except Exception, e:
@@ -179,6 +179,7 @@ class BBCooker:
         except Exception, e:
             bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
         # emit the metadata which isnt valid shell
+        data.expandKeys( self.configuration.data )     
         for e in self.configuration.data.keys():
             if data.getVarFlag( e, 'python', self.configuration.data ):
                 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
@@ -357,6 +358,7 @@ class BBCooker:
 
         localdata = data.createCopy(self.configuration.data)
         bb.data.update_data(localdata)
+        bb.data.expandKeys(localdata)
 
         def calc_bbfile_priority(filename):
             for (regex, pri) in self.status.bbfile_config_priorities:
@@ -403,11 +405,8 @@ class BBCooker:
             self.status.possible_world = None
             self.status.all_depends    = None
 
-    def myProgressCallback( self, x, y, f, bb_cache, from_cache ):
-        # feed the status with new input
-
-        bb_cache.handle_data(f, self.status)
-
+    def myProgressCallback( self, x, y, f, from_cache ):
+        """Update any tty with the progress change"""
         if os.isatty(sys.stdout.fileno()):
             sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
             sys.stdout.flush()
@@ -427,6 +426,7 @@ class BBCooker:
             bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
         else:
             bb.data.update_data( self.configuration.data )
+            bb.data.expandKeys(localdata)
             shell.start( self )
             sys.exit( 0 )
 
@@ -535,9 +535,23 @@ class BBCooker:
         if self.configuration.buildfile is not None:
             bf = os.path.abspath( self.configuration.buildfile )
             try:
-                bbfile_data = bb.parse.handle(bf, self.configuration.data)
-            except IOError:
-                bb.msg.fatal(bb.msg.domain.Parsing, "Unable to open %s" % bf)
+                os.stat(bf)
+            except OSError:
+                (filelist, masked) = self.collect_bbfiles()
+                regexp = re.compile(self.configuration.buildfile)
+                matches = []
+                for f in filelist:
+                    if regexp.search(f) and os.path.isfile(f):
+                        bf = f
+                        matches.append(f)
+                if len(matches) != 1:
+                    bb.msg.error(bb.msg.domain.Parsing, "Unable to match %s (%s matches found):" % (self.configuration.buildfile, len(matches)))
+                    for f in matches:
+                        bb.msg.error(bb.msg.domain.Parsing, "    %s" % f)
+                    sys.exit(1)
+                bf = matches[0]                    
+
+            bbfile_data = bb.parse.handle(bf, self.configuration.data)
 
             item = bb.data.getVar('PN', bbfile_data, 1)
             try:
@@ -578,13 +592,14 @@ class BBCooker:
             except ImportError:
                 bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
             else:
-                psyco.bind( self.collect_bbfiles )
+                psyco.bind( self.parse_bbfiles )
         else:
             bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
 
         try:
             bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
-            self.collect_bbfiles( self.myProgressCallback )
+            (filelist, masked) = self.collect_bbfiles()
+            self.parse_bbfiles(filelist, masked, self.myProgressCallback)
             bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
             print
             if self.configuration.parse_only:
@@ -609,14 +624,18 @@ class BBCooker:
 
             bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, self.configuration.event_data))
 
-            taskdata = bb.taskdata.TaskData()
+            localdata = data.createCopy(self.configuration.data)
+            bb.data.update_data(localdata)
+            bb.data.expandKeys(localdata)
+
+            taskdata = bb.taskdata.TaskData(self.configuration.abort)
 
             runlist = []
             try:
                 for k in pkgs_to_build:
-                    taskdata.add_provider(self.configuration.data, self.status, k)
+                    taskdata.add_provider(localdata, self.status, k)
                     runlist.append([k, "do_%s" % self.configuration.cmd])
-                taskdata.add_unresolved(self.configuration.data, self.status)
+                taskdata.add_unresolved(localdata, self.status)
             except bb.providers.NoProvider:
                 sys.exit(1)
 
@@ -624,10 +643,10 @@ class BBCooker:
             rq.prepare_runqueue(self.configuration.data, self.status, taskdata, runlist)
             try:
                 failures = rq.execute_runqueue(self, self.configuration.data, self.status, taskdata, runlist)
-            except runqueue.TaskFailure, (fnid, fn, taskname):
-                bb.msg.error(bb.msg.domain.Build, "'%s, %s' failed" % (fn, taskname))
+            except runqueue.TaskFailure, fnids:
+                for fnid in fnids:
+                    bb.msg.error(bb.msg.domain.Build, "'%s' failed" % taskdata.fn_index[fnid])
                 sys.exit(1)
-
             bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, self.configuration.event_data, failures))
 
             sys.exit( self.stats.show() )
@@ -655,9 +674,8 @@ class BBCooker:
             return []
         return finddata.readlines()
 
-    def collect_bbfiles( self, progressCallback ):
+    def collect_bbfiles( self ):
         """Collect all available .bb build files"""
-        self.cb = progressCallback
         parsed, cached, skipped, masked = 0, 0, 0, 0
         self.bb_cache = bb.cache.init(self)
 
@@ -679,61 +697,78 @@ class BBCooker:
                     continue
             newfiles += glob.glob(f) or [ f ]
 
-        bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1) or ""
+        bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
+
+        if not bbmask:
+            return (newfiles, 0)
+
         try:
             bbmask_compiled = re.compile(bbmask)
         except sre_constants.error:
             bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
 
+        finalfiles = []
         for i in xrange( len( newfiles ) ):
             f = newfiles[i]
             if bbmask and bbmask_compiled.search(f):
-                bb.msg.debug(1, bb.msg.domain.Collection, "bbmake: skipping %s" % f, f)
+                bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
                 masked += 1
                 continue
-            bb.msg.debug(1, bb.msg.domain.Collection, "bbmake: parsing %s" % f, f)
+            finalfiles.append(f)
+
+        return (finalfiles, masked)
+
+    def parse_bbfiles(self, filelist, masked, progressCallback = None):
+        parsed, cached, skipped = 0, 0, 0
+        for i in xrange( len( filelist ) ):
+            f = filelist[i]
+
+            bb.msg.debug(1, bb.msg.domain.Collection, "parsing %s" % f)
 
             # read a file's metadata
             try:
-                fromCache, skip = self.bb_cache.loadData(f, self)
+                fromCache, skip = self.bb_cache.loadData(f, self.configuration.data)
                 if skip:
                     skipped += 1
-                    bb.msg.debug(2, bb.msg.domain.Collection, "Skipping %s" % f, f)
+                    bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
                     self.bb_cache.skip(f)
                     continue
                 elif fromCache: cached += 1
                 else: parsed += 1
                 deps = None
 
+                # Disabled by RP as was no longer functional
                 # allow metadata files to add items to BBFILES
                 #data.update_data(self.pkgdata[f])
-                addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
-                if addbbfiles:
-                    for aof in addbbfiles.split():
-                        if not files.count(aof):
-                            if not os.path.isabs(aof):
-                                aof = os.path.join(os.path.dirname(f),aof)
-                            files.append(aof)
+                #addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
+                #if addbbfiles:
+                #    for aof in addbbfiles.split():
+                #        if not files.count(aof):
+                #            if not os.path.isabs(aof):
+                #                aof = os.path.join(os.path.dirname(f),aof)
+                #            files.append(aof)
+
+                self.bb_cache.handle_data(f, self.status)
 
                 # now inform the caller
-                if self.cb is not None:
-                    self.cb( i + 1, len( newfiles ), f, self.bb_cache, fromCache )
+                if progressCallback is not None:
+                    progressCallback( i + 1, len( filelist ), f, fromCache )
 
             except IOError, e:
                 self.bb_cache.remove(f)
-                bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e), f)
+                bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
                 pass
             except KeyboardInterrupt:
                 self.bb_cache.sync()
                 raise
             except Exception, e:
                 self.bb_cache.remove(f)
-                bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f), f)
+                bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
             except:
                 self.bb_cache.remove(f)
                 raise
 
-        if self.cb is not None:
+        if progressCallback is not None:
             print "\r" # need newline after Handling Bitbake files message
             bb.msg.note(1, bb.msg.domain.Collection, "Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ))