lib/bb/fetch: Add ability to fetchers to check URL validity without downloading

diff --git a/ChangeLog b/ChangeLog
index 27de599..871f260 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -29,6 +29,7 @@ Changes in BitBake 1.8.x:
        - Work around refs/HEAD issues with git over http (#3410)
        - Add proxy support to the CVS fetcher (from Cyril Chemparathy)
        - Improve runfetchcmd so errors are seen and various GIT variables are exported
+       - Add ability to fetchers to check URL validity without downloading
 
 Changes in BitBake 1.8.10:
        - Psyco is available only for x86 - do not use it on other architectures.

diff --git a/lib/bb/fetch/__init__.py b/lib/bb/fetch/__init__.py
index 41eebb2..c697f47 100644 (file)
--- a/lib/bb/fetch/__init__.py
+++ b/lib/bb/fetch/__init__.py
@@ -162,6 +162,22 @@ def go(d):
                 Fetch.write_md5sum(u, ud, d)
             bb.utils.unlockfile(lf)
 
+
+def checkstatus(d):
+    """
+    Check all urls exist upstream
+    init must have previously been called
+    """
+    urldata = init([], d, True)
+
+    for u in urldata:
+        ud = urldata[u]
+        m = ud.method
+        bb.msg.note(1, bb.msg.domain.Fetcher, "Testing URL %s" % u)
+        ret = m.checkstatus(u, ud, d)
+        if not ret:
+            bb.msg.fatal(bb.msg.domain.Fetcher, "URL %s doesn't work" % u)
+
 def localpaths(d):
     """
     Return a list of the local filenames, assuming successful fetch
@@ -364,6 +380,14 @@ class Fetch(object):
         """
         raise NoMethodError("Missing implementation for url")
 
+    def checkstatus(self, url, urldata, d):
+        """
+        Check the status of a URL
+        Assumes localpath was called first
+        """
+        bb.msg.note(1, bb.msg.domain.Fetcher, "URL %s could not be checked for status since no method exists." % url)
+        return True
+
     def getSRCDate(urldata, d):
         """
         Return the SRC Date for the component

diff --git a/lib/bb/fetch/local.py b/lib/bb/fetch/local.py
index 5e480a2..a39cdce 100644 (file)
--- a/lib/bb/fetch/local.py
+++ b/lib/bb/fetch/local.py
@@ -59,3 +59,11 @@ class Local(Fetch):
         """Fetch urls (no-op for Local method)"""
         # no need to fetch local files, we'll deal with them in place.
         return 1
+
+    def checkstatus(self, url, urldata, d):
+        """
+        Check the status of the url
+        """
+        if os.path.exists(urldata.localpath):
+           return True
+        return False

diff --git a/lib/bb/fetch/wget.py b/lib/bb/fetch/wget.py
index f8ade45..a5979de 100644 (file)
--- a/lib/bb/fetch/wget.py
+++ b/lib/bb/fetch/wget.py
@@ -48,11 +48,13 @@ class Wget(Fetch):
 
         return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
 
-    def go(self, uri, ud, d):
+    def go(self, uri, ud, d, checkonly = False):
         """Fetch urls"""
 
         def fetch_uri(uri, ud, d):
-            if os.path.exists(ud.localpath):
+            if checkonly:
+                fetchcmd = data.getVar("FETCHCOMMAND", d, 1) + " " + data.getVar("FETCHOPTION_checkonly", d, 1)
+            elif os.path.exists(ud.localpath):
                 # file exists, but we didnt complete it.. trying again..
                 fetchcmd = data.getVar("RESUMECOMMAND", d, 1)
             else:
@@ -83,10 +85,10 @@ class Wget(Fetch):
             newuri = uri_replace(uri, find, replace, d)
             if newuri != uri:
                 if fetch_uri(newuri, ud, localdata):
-                    return
+                    return True
 
         if fetch_uri(uri, ud, localdata):
-            return
+            return True
 
         # try mirrors
         mirrors = [ i.split() for i in (data.getVar('MIRRORS', localdata, 1) or "").split('\n') if i ]
@@ -94,6 +96,10 @@ class Wget(Fetch):
             newuri = uri_replace(uri, find, replace, d)
             if newuri != uri:
                 if fetch_uri(newuri, ud, localdata):
-                    return
+                    return True
 
         raise FetchError(uri)
+
+
+    def checkstatus(self, uri, ud, d):
+        return self.go(uri, ud, d, True)