code.vuplus.com Git - vuplus_bitbake/blob - bin/oe/fetch.py

   1 #!/usr/bin/python
   2 """
   3 OpenEmbedded 'Fetch' implementations
   4
   5 Classes for obtaining upstream sources for the
   6 OpenEmbedded (http://openembedded.org) build infrastructure.
   7
   8 NOTE that it requires Python 2.x due to its use of static methods.
   9
  10 Copyright: (c) 2003 Chris Larson
  11
  12 Based on functions from the base oe module, Copyright 2003 Holger Schurig
  13 """
  14
  15 import os, re, string
  16 import oe
  17 import oe.data
  18
  19 class FetchError(Exception):
  20         """Exception raised when a download fails"""
  21
  22 class NoMethodError(Exception):
  23         """Exception raised when there is no method to obtain a supplied url or set of urls"""
  24
  25 class MissingParameterError(Exception):
  26         """Exception raised when a fetch method is missing a critical parameter in the url"""
  27
  28 #decodeurl("cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;module=familiar/dist/ipkg;tag=V0-99-81")
  29 #('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', {'tag': 'V0-99-81', 'module': 'familiar/dist/ipkg'})
  30
  31 def uri_replace(uri, uri_find, uri_replace, d = oe.data.init()):
  32 #       oe.note("uri_replace: operating on %s" % uri)
  33         if not uri or not uri_find or not uri_replace:
  34                 oe.debug(1, "uri_replace: passed an undefined value, not replacing")
  35         uri_decoded = list(oe.decodeurl(uri))
  36         uri_find_decoded = list(oe.decodeurl(uri_find))
  37         uri_replace_decoded = list(oe.decodeurl(uri_replace))
  38         result_decoded = ['','','','','',{}]
  39         for i in uri_find_decoded:
  40                 loc = uri_find_decoded.index(i)
  41                 result_decoded[loc] = uri_decoded[loc]
  42                 import types
  43                 if type(i) == types.StringType:
  44                         import re
  45                         if (re.match(i, uri_decoded[loc])):
  46                                 result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc])
  47                                 if uri_find_decoded.index(i) == 2:
  48                                         if d:
  49                                                 localfn = oe.fetch.localpath(uri, d)
  50                                                 if localfn:
  51                                                         result_decoded[loc] = os.path.dirname(result_decoded[loc]) + "/" + os.path.basename(oe.fetch.localpath(uri, d))
  52 #                               oe.note("uri_replace: matching %s against %s and replacing with %s" % (i, uri_decoded[loc], uri_replace_decoded[loc]))
  53                         else:
  54 #                               oe.note("uri_replace: no match")
  55                                 return uri
  56 #               else:
  57 #                       for j in i.keys():
  58 #                               FIXME: apply replacements against options
  59         return oe.encodeurl(result_decoded)
  60
  61 methods = []
  62
  63 def init(urls = [], d = oe.data.init()):
  64         for m in methods:
  65                 m.urls = []
  66
  67         for u in urls:
  68                 for m in methods:
  69                         m.data = d
  70                         if m.supports(u, d):
  71                                 m.urls.append(u)
  72
  73 def go(d = oe.data.init()):
  74         """Fetch all urls"""
  75         for m in methods:
  76                 if m.urls:
  77                         m.go(d)
  78
  79 def localpaths(d):
  80         """Return a list of the local filenames, assuming successful fetch"""
  81         local = []
  82         for m in methods:
  83                 for u in m.urls:
  84                         local.append(m.localpath(u, d))
  85         return local
  86
  87 def localpath(url, d = oe.data.init()):
  88         for m in methods:
  89                 if m.supports(url, d):
  90                         return m.localpath(url, d)
  91         return url
  92
  93 class Fetch(object):
  94         """Base class for 'fetch'ing data"""
  95
  96         def __init__(self, urls = []):
  97                 self.urls = []
  98                 for url in urls:
  99                         if self.supports(oe.decodeurl(url), d) is 1:
 100                                 self.urls.append(url)
 101
 102         def supports(url, d):
 103                 """Check to see if this fetch class supports a given url.
 104                    Expects supplied url in list form, as outputted by oe.decodeurl().
 105                 """
 106                 return 0
 107         supports = staticmethod(supports)
 108
 109         def localpath(url, d = oe.data.init()):
 110                 """Return the local filename of a given url assuming a successful fetch.
 111                 """
 112                 return url
 113         localpath = staticmethod(localpath)
 114
 115         def setUrls(self, urls):
 116                 self.__urls = urls
 117
 118         def getUrls(self):
 119                 return self.__urls
 120
 121         urls = property(getUrls, setUrls, None, "Urls property")
 122
 123         def setData(self, data):
 124                 self.__data = data
 125
 126         def getData(self):
 127                 return self.__data
 128
 129         data = property(getData, setData, None, "Data property")
 130
 131         def go(self, urls = []):
 132                 """Fetch urls"""
 133                 raise NoMethodError("Missing implementation for url")
 134
 135 class Wget(Fetch):
 136         """Class to fetch urls via 'wget'"""
 137         def supports(url, d):
 138                 """Check to see if a given url can be fetched using wget.
 139                    Expects supplied url in list form, as outputted by oe.decodeurl().
 140                 """
 141                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(url, d))
 142                 return type in ['http','https','ftp']
 143         supports = staticmethod(supports)
 144
 145         def localpath(url, d):
 146                 # strip off parameters
 147                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(url, d))
 148                 if "localpath" in parm:
 149                         # if user overrides local path, use it.
 150                         return parm["localpath"]
 151                 url = oe.encodeurl([type, host, path, user, pswd, {}])
 152                 return os.path.join(oe.data.getVar("DL_DIR", d), os.path.basename(url))
 153         localpath = staticmethod(localpath)
 154
 155         def go(self, d = oe.data.init(), urls = []):
 156                 """Fetch urls"""
 157                 def fetch_uri(uri, basename, dl, md5, d):
 158                         if os.path.exists(dl):
 159                                 # file exists, but we didnt complete it.. trying again..
 160                                 fetchcmd = oe.data.getVar("RESUMECOMMAND", d, 1)
 161                         else:
 162                                 fetchcmd = oe.data.getVar("FETCHCOMMAND", d, 1)
 163
 164                         oe.note("fetch " + uri)
 165                         fetchcmd = fetchcmd.replace("${URI}", uri)
 166                         fetchcmd = fetchcmd.replace("${FILE}", basename)
 167                         oe.debug(2, "executing " + fetchcmd)
 168                         ret = os.system(fetchcmd)
 169                         if ret != 0:
 170                                 return False
 171
 172                         # supposedly complete.. write out md5sum
 173                         if oe.which(oe.data.getVar('PATH', d, 1), 'md5sum'):
 174                                 md5pipe = os.popen('md5sum ' + dl)
 175                                 md5data = md5pipe.readline().split()[0]
 176                                 md5pipe.close()
 177                                 md5out = file(md5, 'w')
 178                                 md5out.write(md5data)
 179                                 md5out.close()
 180                         else:
 181                                 md5out = file(md5, 'w')
 182                                 md5out.write("")
 183                                 md5out.close()
 184                         return True
 185
 186                 if not urls:
 187                         urls = self.urls
 188
 189                 from copy import deepcopy
 190                 localdata = deepcopy(d)
 191                 oe.data.setVar('OVERRIDES', "wget:" + oe.data.getVar('OVERRIDES', localdata), localdata)
 192                 oe.data.update_data(localdata)
 193
 194                 for uri in urls:
 195                         (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(uri, localdata))
 196                         basename = os.path.basename(path)
 197                         dl = self.localpath(uri, d)
 198                         dl = oe.data.expand(dl, localdata)
 199                         md5 = dl + '.md5'
 200
 201                         if os.path.exists(md5):
 202                                 # complete, nothing to see here..
 203                                 continue
 204
 205                         if fetch_uri(uri, basename, dl, md5, localdata):
 206                                 continue
 207
 208                         # try mirrors
 209                         mirror = 0
 210                         mirrors = [string.split(i) for i in string.split(oe.data.getVar('MIRRORS', localdata, 1), '\n') if i]
 211                         completed = 0
 212                         for (find, replace) in mirrors:
 213                                 newuri = uri_replace(uri, find, replace)
 214                                 if newuri != uri:
 215                                         if fetch_uri(newuri, basename, dl, md5, localdata):
 216                                                 completed = 1
 217                                                 break
 218
 219                         if not completed:
 220                                 raise FetchError(uri)
 221
 222                 del localdata
 223
 224
 225 methods.append(Wget())
 226
 227 class Cvs(Fetch):
 228         """Class to fetch a module or modules from cvs repositories"""
 229         def supports(url, d):
 230                 """Check to see if a given url can be fetched with cvs.
 231                    Expects supplied url in list form, as outputted by oe.decodeurl().
 232                 """
 233                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(url, d))
 234                 return type in ['cvs', 'pserver']
 235         supports = staticmethod(supports)
 236
 237         def localpath(url, d):
 238                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(url, d))
 239                 if "localpath" in parm:
 240                         # if user overrides local path, use it.
 241                         return parm["localpath"]
 242
 243                 if not "module" in parm:
 244                         raise MissingParameterError("cvs method needs a 'module' parameter")
 245                 else:
 246                         module = parm["module"]
 247                 if 'tag' in parm:
 248                         tag = parm['tag']
 249                 else:
 250                         tag = ""
 251                 if 'date' in parm:
 252                         date = parm['date']
 253                 else:
 254                         date = oe.data.getVar("CVSDATE", d, 1) or oe.data.getVar("DATE", d, 1)
 255
 256                 return os.path.join(oe.data.getVar("DL_DIR", d, 1),oe.data.expand('%s_%s_%s_%s.tar.gz' % (string.replace(module, '/', '.'), host, tag, date), d))
 257         localpath = staticmethod(localpath)
 258
 259         def go(self, d = oe.data.init(), urls = []):
 260                 """Fetch urls"""
 261                 if not urls:
 262                         urls = self.urls
 263
 264                 from copy import deepcopy
 265                 localdata = deepcopy(d)
 266                 oe.data.setVar('OVERRIDES', "cvs:%s" % oe.data.getVar('OVERRIDES', localdata), localdata)
 267                 oe.data.update_data(localdata)
 268
 269                 for loc in urls:
 270                         (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(loc, localdata))
 271                         if not "module" in parm:
 272                                 raise MissingParameterError("cvs method needs a 'module' parameter")
 273                         else:
 274                                 module = parm["module"]
 275
 276                         dlfile = self.localpath(loc, localdata)
 277                         dldir = oe.data.getVar('DL_DIR', localdata, 1)
 278                         # if local path contains the cvs
 279                         # module, consider the dir above it to be the
 280                         # download directory
 281 #                       pos = dlfile.find(module)
 282 #                       if pos:
 283 #                               dldir = dlfile[:pos]
 284 #                       else:
 285 #                               dldir = os.path.dirname(dlfile)
 286
 287                         # setup cvs options
 288                         options = []
 289                         if 'tag' in parm:
 290                                 tag = parm['tag']
 291                         else:
 292                                 tag = ""
 293
 294                         if 'date' in parm:
 295                                 date = parm['date']
 296                         else:
 297                                 date = oe.data.getVar("CVSDATE", d, 1) or oe.data.getVar("DATE", d, 1)
 298
 299                         if "method" in parm:
 300                                 method = parm["method"]
 301                         else:
 302                                 method = "pserver"
 303
 304                         tarfn = oe.data.expand('%s_%s_%s_%s.tar.gz' % (string.replace(module, '/', '.'), host, tag, date), localdata)
 305                         oe.data.setVar('TARFILES', dlfile, localdata)
 306                         oe.data.setVar('TARFN', tarfn, localdata)
 307
 308                         if os.access(os.path.join(dldir, tarfn), os.R_OK):
 309                                 oe.debug(1, "%s already exists, skipping cvs checkout." % tarfn)
 310                                 return
 311
 312                         if date:
 313                                 options.append("-D %s" % date)
 314                         if tag:
 315                                 options.append("-r %s" % tag)
 316
 317                         olddir = os.path.abspath(os.getcwd())
 318                         os.chdir(oe.data.expand(dldir, localdata))
 319
 320                         # setup cvsroot
 321                         cvsroot = ":" + method + ":" + user
 322                         if pswd:
 323                                 cvsroot += ":" + pswd
 324                         cvsroot += "@" + host + ":" + path
 325
 326                         oe.data.setVar('CVSROOT', cvsroot, localdata)
 327                         oe.data.setVar('CVSCOOPTS', string.join(options), localdata)
 328                         oe.data.setVar('CVSMODULE', module, localdata)
 329                         cvscmd = oe.data.getVar('FETCHCOMMAND', localdata, 1)
 330
 331                         # create temp directory
 332                         oe.debug(2, "Fetch: creating temporary directory")
 333                         oe.mkdirhier(oe.data.expand('${WORKDIR}', localdata))
 334                         oe.data.setVar('TMPBASE', oe.data.expand('${WORKDIR}/oecvs.XXXXXX', localdata), localdata)
 335                         tmppipe = os.popen(oe.data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
 336                         tmpfile = tmppipe.readline().strip()
 337                         if not tmpfile:
 338                                 oe.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.")
 339                                 raise FetchError(module)
 340
 341                         # check out sources there
 342                         os.chdir(tmpfile)
 343                         oe.note("Fetch " + loc)
 344                         oe.debug(1, "Running %s" % cvscmd)
 345                         myret = os.system(cvscmd)
 346                         if myret != 0:
 347                                 try:
 348                                         os.rmdir(tmpfile)
 349                                 except OSError:
 350                                         pass
 351                                 raise FetchError(module)
 352
 353                         os.chdir(os.path.join(tmpfile, os.path.dirname(module)))
 354                         # tar them up to a defined filename
 355                         myret = os.system("tar -czvf %s %s" % (os.path.join(dldir,tarfn), os.path.basename(module)))
 356                         if myret != 0:
 357                                 try:
 358                                         os.unlink(tarfn)
 359                                 except OSError:
 360                                         pass
 361                         # cleanup
 362                         os.system('rm -rf %s' % tmpfile)
 363                         os.chdir(olddir)
 364                 del localdata
 365
 366 methods.append(Cvs())
 367
 368 class Bk(Fetch):
 369         def supports(url, d):
 370                 """Check to see if a given url can be fetched via bitkeeper.
 371                    Expects supplied url in list form, as outputted by oe.decodeurl().
 372                 """
 373                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(url, d))
 374                 return type in ['bk']
 375         supports = staticmethod(supports)
 376
 377 methods.append(Bk())
 378
 379 class Local(Fetch):
 380         def supports(url, d):
 381                 """Check to see if a given url can be fetched in the local filesystem.
 382                    Expects supplied url in list form, as outputted by oe.decodeurl().
 383                 """
 384                 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(url, d))
 385                 return type in ['file','patch']
 386         supports = staticmethod(supports)
 387
 388         def localpath(url, d):
 389                 """Return the local filename of a given url assuming a successful fetch.
 390                 """
 391                 return url.split("://")[1]
 392         localpath = staticmethod(localpath)
 393
 394         def go(self, urls = []):
 395                 """Fetch urls (no-op for Local method)"""
 396                 # no need to fetch local files, we'll deal with them in place.
 397                 return 1
 398
 399 methods.append(Local())