svn fetcher: Add SRCREV support

diff --git a/ChangeLog b/ChangeLog
index 77011af..2acac58 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -4,7 +4,7 @@ Changes in Bitbake 1.8.x:
          (requires new FETCHCMD_svn definition in bitbake.conf)
        - Change SVNDIR layout to be more unique (fixes #2644 and #2624)
        - Import persistent data store from trunk
-       - Sync fetcher code with that in trunk
+       - Sync fetcher code with that in trunk, adding SRCREV support for svn
        - Add ConfigParsed Event after configuration parsing is complete
 
 Changes in Bitbake 1.8.6:

diff --git a/lib/bb/fetch/__init__.py b/lib/bb/fetch/__init__.py
index 9d1ecfe..cc46b81 100644 (file)
--- a/lib/bb/fetch/__init__.py
+++ b/lib/bb/fetch/__init__.py
@@ -87,10 +87,14 @@ def fetcher_init(d):
     Calls before this must not hit the cache.
     """
     pd = persist_data.PersistData(d)
-    # Clear any cached data
+    # Clear any cached url data
     pd.delDomain("BB_URLDATA")
-    # Make sure our domain exists
+    # When to drop SCM head revisions should be controled by user policy
+    pd.delDomain("BB_URI_HEADREVS")
+    # Make sure our domains exist
     pd.addDomain("BB_URLDATA")
+    pd.addDomain("BB_URI_HEADREVS")
+    pd.addDomain("BB_URI_LOCALCOUNT")
 
 # Function call order is usually:
 #   1. init
@@ -164,6 +168,35 @@ def localpaths(d, urldata = None):
 
     return local
 
+def get_srcrev(d):
+    """
+    Return the version string for the current package
+    (usually to be used as PV)
+    Most packages usually only have one SCM so we just pass on the call.
+    In the multi SCM case, we build a value based on SRCREV_FORMAT which must 
+    have been set.
+    """
+    urldata, pd, fn = getdata(d)
+    if len(urldata) == 0:
+        src_uri = bb.data.getVar('SRC_URI', d, 1).split(" ")
+        urldata = init(src_uri, d, True)
+
+    scms = []
+    for u in urldata:
+        ud = urldata[u]
+        if ud.method.suppports_srcrev():
+            scms.append(u)
+
+    if len(scms) == 0:
+        bb.msg.error(bb.msg.domain.Fetcher, "SRCREV was used yet no valid SCM was found in SRC_URI")
+        raise ParameterError
+
+    if len(scms) == 1:
+        return ud.method.sortable_revision(scms[0], urldata[scms[0]], d)
+
+    bb.msg.error(bb.msg.domain.Fetcher, "Sorry, support for SRCREV_FORMAT still needs to be written")
+    raise ParameterError
+
 def localpath(url, d, cache = True):
     """
     Called from the parser with cache=False since the cache isn't ready 
@@ -259,6 +292,12 @@ class Fetch(object):
         """
         return False
 
+    def suppports_srcrev(self):
+        """
+        The fetcher supports auto source revisions (SRCREV)
+        """
+        return False
+
     def go(self, url, urldata, d):
         """
         Fetch urls
@@ -346,6 +385,32 @@ class Fetch(object):
         md5out.close()
     write_md5sum = staticmethod(write_md5sum)
 
+    def latest_revision(self, url, ud, d):
+        """
+        Look in the cache for the latest revision, if not present ask the SCM.
+        """
+        if not self._latest_revision:
+            raise ParameterError
+
+        pd = persist_data.PersistData(d)
+        key = self._revision_key(url, ud, d)
+        rev = pd.getValue("BB_URI_HEADREVS", key)
+        if rev != None:
+            return str(rev)
+
+        rev = self._latest_revision(url, ud, d)
+        pd.setValue("BB_URI_HEADREVS", key, rev)
+        return rev
+
+    def sortable_revision(self, url, ud, d):
+        """
+        
+        """
+        if not self._sortable_revision:
+            raise ParameterError
+
+        return self._sortable_revision(url, ud, d)
+
 import cvs
 import git
 import local

diff --git a/lib/bb/fetch/svn.py b/lib/bb/fetch/svn.py
index 5229838..fdba725 100644 (file)
--- a/lib/bb/fetch/svn.py
+++ b/lib/bb/fetch/svn.py
@@ -45,24 +45,38 @@ class Svn(Fetch):
             raise MissingParameterError("svn method needs a 'module' parameter")
 
         ud.module = ud.parm["module"]
-        ud.moddir = ud.module.replace('/', '.')
 
-        ud.revision = ""
+        # Create paths to svn checkouts
+        relpath = ud.path
+        if relpath.startswith('/'):
+            # Remove leading slash as os.path.join can't cope
+            relpath = relpath[1:]
+        ud.pkgdir = os.path.join(data.expand('${SVNDIR}', d), ud.host, relpath)
+        ud.moddir = os.path.join(ud.pkgdir, ud.module)
+
         if 'rev' in ud.parm:
             ud.date = ""
             ud.revision = ud.parm['rev']
-        elif ud.date == "now":
-            ud.date = ""
-            # FIXME caching
+        elif 'date' in ud.date:
+            ud.date = ud.parm['date']
+            ud.revision = ""
+        else:
             ud.revision = self.latest_revision(url, ud, d)
-
-        ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.moddir, ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d)
+            #
+            # ***Nasty hack***
+            # If DATE in unexpanded PV, use ud.date (which is set from SRCDATE)
+            # Will warn people to switch to SRCREV here
+            #
+            pv = data.getVar("PV", d, 0)
+            if "DATE" in pv:
+                ud.revision = ""
+            else:
+                ud.date = ""
+
+        ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d)
 
         return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
 
-    def forcefetch(self, url, ud, d):
-        return ud.force
-
     def _buildsvncommand(self, ud, d, command):
         """
         Build up an svn commandline based on ud
@@ -83,10 +97,6 @@ class Svn(Fetch):
 
         # either use the revision, or SRCDATE in braces,
         options = []
-        if ud.revision:
-            options.append("-r %s" % ud.revision)
-        elif ud.date:
-            options.append("-r {%s}" % ud.date)
 
         if ud.user:
             options.append("--username %s" % ud.user)
@@ -94,14 +104,20 @@ class Svn(Fetch):
         if ud.pswd:
             options.append("--password %s" % ud.pswd)
 
-        if command is "fetch":
-            svncmd = "%s co %s %s://%s/%s %s" % (basecmd, " ".join(options), proto, svnroot, ud.module, ud.module)
-        elif command is "update":
-            svncmd = "%s update %s" % (basecmd, " ".join(options))
-        elif command is "info":
+        if command is "info":
             svncmd = "%s info %s %s://%s/%s" % (basecmd, " ".join(options), proto, svnroot, ud.module)
         else:
-            raise FetchError("Invalid svn command %s" % command)
+            if ud.revision:
+                options.append("-r %s" % ud.revision)
+            elif ud.date:
+                options.append("-r {%s}" % ud.date)
+
+            if command is "fetch":
+                svncmd = "%s co %s %s://%s/%s %s" % (basecmd, " ".join(options), proto, svnroot, ud.module, ud.module)
+            elif command is "update":
+                svncmd = "%s update %s" % (basecmd, " ".join(options))
+            else:
+                raise FetchError("Invalid svn command %s" % command)
 
         if svn_rsh:
             svncmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svncmd)
@@ -116,32 +132,25 @@ class Svn(Fetch):
             bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping svn checkout." % ud.localpath)
             return
 
-        pkg = data.expand('${PN}', d)
-        relpath = ud.path
-        if relpath.startswith('/'):
-            # Remove leading slash as os.path.join can't cope
-            relpath = relpath[1:]
-        pkgdir = os.path.join(data.expand('${SVNDIR}', d), ud.host, relpath)
-        moddir = os.path.join(pkgdir, ud.module)
-        bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + moddir + "'")
+        bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'")
 
-        if os.access(os.path.join(moddir, '.svn'), os.R_OK):
+        if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):
             svnupdatecmd = self._buildsvncommand(ud, d, "update")
             bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc)
             # update sources there
-            os.chdir(moddir)
+            os.chdir(ud.moddir)
             bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svnupdatecmd)
             runfetchcmd(svnupdatecmd, d)
         else:
             svnfetchcmd = self._buildsvncommand(ud, d, "fetch")
             bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
             # check out sources there
-            bb.mkdirhier(pkgdir)
-            os.chdir(pkgdir)
+            bb.mkdirhier(ud.pkgdir)
+            os.chdir(ud.pkgdir)
             bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svnfetchcmd)
             runfetchcmd(svnfetchcmd, d)
 
-        os.chdir(pkgdir)
+        os.chdir(ud.pkgdir)
         # tar them up to a defined filename
         try:
             runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.module)), d)
@@ -153,7 +162,21 @@ class Svn(Fetch):
                 pass
             raise t, v, tb
 
-    def latest_revision(self, url, ud, d):
+    def suppports_srcrev(self):
+        return True
+
+    def _revision_key(self, url, ud, d):
+        """
+        Return a unique key for the url
+        """
+        return "svn:" + ud.moddir
+
+    def _latest_revision(self, url, ud, d):
+        """
+        Return the latest upstream revision number
+        """
+        bb.msg.debug(2, bb.msg.domain.Fetcher, "SVN fetcher hitting network for %s" % url)
+
         output = runfetchcmd("LANG= LC_ALL= " + self._buildsvncommand(ud, d, "info"), d, True)
 
         revision = None
@@ -163,7 +186,11 @@ class Svn(Fetch):
 
         return revision
 
-    def sortable_revision(self, url, ud, d):
+    def _sortable_revision(self, url, ud, d):
+        """
+        Return a sortable revision number which in our case is the revision number
+        (use the cached version to avoid network access)
+        """
 
         return self.latest_revision(url, ud, d)