code.vuplus.com Git - vuplus_openembedded/blob - classes/packaged-staging.bbclass

   1 #
   2 # Populate builds using prebuilt packages where possible to speed up builds
   3 # and allow staging to be reconstructed.
   4 #
   5 # To use it add that line to conf/local.conf:
   6 #
   7 # INHERIT += "packaged-staging"
   8 #
   9
  10
  11 #
  12 # bitbake.conf set PSTAGING_ACTIVE = "0", this class sets to "1" if we're active
  13 #
  14 PSTAGE_PKGVERSION = "${PV}-${PR}"
  15 PSTAGE_PKGARCH    = "${BUILD_SYS}"
  16 PSTAGE_EXTRAPATH  ?= ""
  17 PSTAGE_PKGPATH    = "${DISTRO}${PSTAGE_EXTRAPATH}"
  18 PSTAGE_PKGPN      = "${@bb.data.expand('staging-${PN}-${MULTIMACH_ARCH}${TARGET_VENDOR}-${TARGET_OS}', d).replace('_', '-')}"
  19 PSTAGE_PKGNAME    = "${PSTAGE_PKGPN}_${PSTAGE_PKGVERSION}_${PSTAGE_PKGARCH}.ipk"
  20 PSTAGE_PKG        = "${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}/${PSTAGE_PKGNAME}"
  21
  22 PSTAGE_NATIVEDEPENDS = "\
  23     shasum-native \
  24     stagemanager-native \
  25     "
  26
  27 BB_STAMP_WHITELIST = "${PSTAGE_NATIVEDEPENDS}"
  28
  29 python () {
  30     import bb
  31     pstage_allowed = True
  32
  33     # These classes encode staging paths into the binary data so can only be
  34     # reused if the path doesn't change/
  35     if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('sdk', d):
  36         path = bb.data.getVar('PSTAGE_PKGPATH', d, 1)
  37         path = path + bb.data.getVar('TMPDIR', d, 1).replace('/', '-')
  38         bb.data.setVar('PSTAGE_PKGPATH', path, d)
  39
  40     # PSTAGE_NATIVEDEPENDS lists the packages we need before we can use packaged
  41     # staging. There will always be some packages we depend on.
  42     if bb.data.inherits_class('native', d):
  43         pn = bb.data.getVar('PN', d, True)
  44         nativedeps = bb.data.getVar('PSTAGE_NATIVEDEPENDS', d, True).split()
  45         if pn in nativedeps:
  46             pstage_allowed = False
  47
  48     # Images aren't of interest to us
  49     if bb.data.inherits_class('image', d):
  50         pstage_allowed = False
  51
  52     # Add task dependencies if we're active, otherwise mark packaged staging
  53     # as inactive.
  54     if pstage_allowed:
  55         deps = bb.data.getVarFlag('do_setscene', 'depends', d) or ""
  56         deps += " stagemanager-native:do_populate_staging"
  57         bb.data.setVarFlag('do_setscene', 'depends', deps, d)
  58
  59         policy = bb.data.getVar("BB_STAMP_POLICY", d, True)
  60         if policy == "whitelist" or policy == "full":
  61            deps = bb.data.getVarFlag('do_setscene', 'recrdeptask', d) or ""
  62            deps += " do_setscene"
  63            bb.data.setVarFlag('do_setscene', 'recrdeptask', deps, d)
  64
  65         bb.data.setVar("PSTAGING_ACTIVE", "1", d)
  66     else:
  67         bb.data.setVar("PSTAGING_ACTIVE", "0", d)
  68 }
  69
  70 DEPLOY_DIR_PSTAGE   ?= "${DEPLOY_DIR}/pstage"
  71 PSTAGE_MACHCONFIG   = "${DEPLOY_DIR_PSTAGE}/opkg.conf"
  72
  73 PSTAGE_PKGMANAGER = "stage-manager-ipkg"
  74
  75 PSTAGE_BUILD_CMD        = "stage-manager-ipkg-build -o 0 -g 0"
  76 PSTAGE_INSTALL_CMD      = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -force-depends -o ${TMPDIR} install"
  77 PSTAGE_UPDATE_CMD       = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR} update"
  78 PSTAGE_REMOVE_CMD       = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -force-depends -o ${TMPDIR} remove"
  79 PSTAGE_LIST_CMD         = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR} list_installed"
  80
  81 PSTAGE_TMPDIR_STAGE     = "${WORKDIR}/staging-pkg"
  82
  83 def pstage_manualclean(srcname, destvarname, d):
  84         import os, bb
  85
  86         src = os.path.join(bb.data.getVar('PSTAGE_TMPDIR_STAGE', d, True), srcname)
  87         dest = bb.data.getVar(destvarname, d, True)
  88
  89         for walkroot, dirs, files in os.walk(src):
  90                 for file in files:
  91                         filepath = os.path.join(walkroot, file).replace(src, dest)
  92                         bb.note("rm %s" % filepath)
  93                         os.system("rm %s" % filepath)
  94
  95 def pstage_set_pkgmanager(d):
  96     import bb
  97     path = bb.data.getVar("PATH", d, 1)
  98     pkgmanager = bb.which(path, 'opkg-cl')
  99     if pkgmanager == "":
 100         pkgmanager = bb.which(path, 'ipkg-cl')
 101     if pkgmanager != "":
 102         bb.data.setVar("PSTAGE_PKGMANAGER", pkgmanager, d)
 103
 104
 105 def pstage_cleanpackage(pkgname, d):
 106         import os, bb
 107
 108         path = bb.data.getVar("PATH", d, 1)
 109         pstage_set_pkgmanager(d)
 110         list_cmd = bb.data.getVar("PSTAGE_LIST_CMD", d, True)
 111
 112         bb.note("Checking if staging package installed")
 113         lf = bb.utils.lockfile(bb.data.expand("${STAGING_DIR}/staging.lock", d))
 114         ret = os.system("PATH=\"%s\" %s | grep %s" % (path, list_cmd, pkgname))
 115         if ret == 0:
 116                 bb.note("Yes. Uninstalling package from staging...")
 117                 removecmd = bb.data.getVar("PSTAGE_REMOVE_CMD", d, 1)
 118                 ret = os.system("PATH=\"%s\" %s %s" % (path, removecmd, pkgname))
 119                 if ret != 0:
 120                         bb.note("Failure removing staging package")
 121         else:
 122                 bb.note("No. Manually removing any installed files")
 123                 pstage_manualclean("staging", "STAGING_DIR", d)
 124                 pstage_manualclean("cross", "CROSS_DIR", d)
 125                 pstage_manualclean("deploy", "DEPLOY_DIR", d)
 126
 127         bb.utils.unlockfile(lf)
 128
 129 do_clean_prepend() {
 130         """
 131         Clear the build and temp directories
 132         """
 133
 134         removepkg = bb.data.expand("${PSTAGE_PKGPN}", d)
 135         pstage_cleanpackage(removepkg, d)
 136
 137         stagepkg = bb.data.expand("${PSTAGE_PKG}", d)
 138         bb.note("Removing staging package %s" % stagepkg)
 139         os.system('rm -rf ' + stagepkg)
 140 }
 141
 142 staging_helper () {
 143         # Assemble appropriate opkg.conf
 144         conffile=${PSTAGE_MACHCONFIG}
 145         mkdir -p ${DEPLOY_DIR_PSTAGE}/pstaging_lists
 146         if [ ! -e $conffile ]; then
 147                 ipkgarchs="${BUILD_SYS}"
 148                 priority=1
 149                 for arch in $ipkgarchs; do
 150                         echo "arch $arch $priority" >> $conffile
 151                         priority=$(expr $priority + 5)
 152                 done
 153                 echo "dest root /" >> $conffile
 154         fi
 155         if [ ! -e ${TMPDIR}${layout_libdir}/opkg/info/ ]; then
 156                 mkdir -p ${TMPDIR}${layout_libdir}/opkg/info/
 157         fi
 158         if [ ! -e ${TMPDIR}${layout_libdir}/ipkg/ ]; then
 159                 cd ${TMPDIR}${layout_libdir}/
 160                 ln -s opkg/ ipkg
 161         fi
 162 }
 163
 164 PSTAGE_TASKS_COVERED = "fetch unpack munge patch configure qa_configure rig_locales compile sizecheck install deploy package populate_staging package_write_deb package_write_ipk package_write package_stage qa_staging"
 165
 166 SCENEFUNCS += "packagestage_scenefunc"
 167
 168 python packagestage_scenefunc () {
 169     import os
 170
 171     if bb.data.getVar("PSTAGING_ACTIVE", d, 1) == "0":
 172         return
 173
 174     bb.build.exec_func("staging_helper", d)
 175
 176     removepkg = bb.data.expand("${PSTAGE_PKGPN}", d)
 177     pstage_cleanpackage(removepkg, d)
 178
 179     stagepkg = bb.data.expand("${PSTAGE_PKG}", d)
 180
 181     if os.path.exists(stagepkg):
 182         path = bb.data.getVar("PATH", d, 1)
 183         pstage_set_pkgmanager(d)
 184         file = bb.data.getVar("FILE", d, True)
 185         bb.debug(2, "Packaged staging active for %s\n" % file)
 186
 187         #
 188         # Install the staging package somewhere temporarily so we can extract the stamp files
 189         #
 190         bb.mkdirhier(bb.data.expand("${WORKDIR}/tstage/${layout_libdir}/opkg/info/ ", d))
 191         cmd = bb.data.expand("${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -force-depends -o ${WORKDIR}/tstage install", d)
 192         ret = os.system("PATH=\"%s\" %s %s" % (path, cmd, stagepkg))
 193         if ret != 0:
 194             bb.fatal("Couldn't install the staging package to a temp directory")
 195
 196         #
 197         # Copy the stamp files into the main stamps directoy
 198         #
 199         cmd = bb.data.expand("cp -dpR ${WORKDIR}/tstage/stamps/* ${TMPDIR}/stamps/", d)
 200         ret = os.system(cmd)
 201         if ret != 0:
 202             bb.fatal("Couldn't copy the staging package stamp files")
 203
 204         #
 205         # Iterate over the stamps seeing if they're valid. If we find any that
 206         # are invalid or the task wasn't in the taskgraph, assume caution and
 207         # do a rebuild.
 208         #
 209         # FIXME - some tasks are safe to ignore in the task graph. e.g. package_write_*
 210         stageok = True
 211         taskscovered = bb.data.getVar("PSTAGE_TASKS_COVERED", d, True).split()
 212         stamp = bb.data.getVar("STAMP", d, True)
 213         for task in taskscovered:
 214             task = 'do_' + task
 215             stampfn = "%s.%s" % (stamp, task)
 216             bb.debug(1, "Checking %s" % (stampfn))
 217             if os.path.exists(stampfn):
 218                 stageok = bb.runqueue.check_stamp_fn(file, task, d)
 219                 bb.debug(1, "Result %s" % (stageok))
 220                 if not stageok:
 221                     break
 222
 223         # Remove the stamps and files we added above
 224         # FIXME - we should really only remove the stamps we added
 225         os.system('rm -f ' + stamp + '.*')
 226         os.system(bb.data.expand("rm -rf ${WORKDIR}/tstage", d))
 227
 228         if stageok:
 229             bb.note("Staging package found, using it for %s." % file)
 230             installcmd = bb.data.getVar("PSTAGE_INSTALL_CMD", d, 1)
 231             lf = bb.utils.lockfile(bb.data.expand("${STAGING_DIR}/staging.lock", d))
 232             ret = os.system("PATH=\"%s\" %s %s" % (path, installcmd, stagepkg))
 233             bb.utils.unlockfile(lf)
 234             if ret != 0:
 235                 bb.note("Failure installing prestage package")
 236
 237             bb.build.make_stamp("do_stage_package_populated", d)
 238         else:
 239             bb.note("Staging package found but invalid for %s" % file)
 240
 241 }
 242 packagestage_scenefunc[cleandirs] = "${PSTAGE_TMPDIR_STAGE}"
 243 packagestage_scenefunc[dirs] = "${STAGING_DIR}"
 244
 245 addhandler packagedstage_stampfixing_eventhandler
 246 python packagedstage_stampfixing_eventhandler() {
 247     from bb.event import getName
 248     import os
 249
 250     if getName(e) == "StampUpdate":
 251         taskscovered = bb.data.getVar("PSTAGE_TASKS_COVERED", e.data, 1).split()
 252         for (fn, task) in e.targets:
 253             # strip off 'do_'
 254             task = task[3:]
 255             if task in taskscovered:
 256                 stamp = "%s.do_stage_package_populated" % e.stampPrefix[fn]
 257                 if os.path.exists(stamp):
 258                     # We're targetting a task which was skipped with packaged staging
 259                     # so we need to remove the autogenerated stamps.
 260                     for task in taskscovered:
 261                         dir = "%s.do_%s" % (e.stampPrefix[fn], task)
 262                         os.system('rm -f ' + dir)
 263                     os.system('rm -f ' + stamp)
 264
 265     return NotHandled
 266 }
 267
 268 populate_staging_preamble () {
 269         if [ "$PSTAGING_ACTIVE" = "1" ]; then
 270                 stage-manager -p ${STAGING_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-staging -u || true
 271                 stage-manager -p ${CROSS_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-cross -u || true
 272         fi
 273 }
 274
 275 populate_staging_postamble () {
 276         if [ "$PSTAGING_ACTIVE" = "1" ]; then
 277                 # list the packages currently installed in staging
 278                 # ${PSTAGE_LIST_CMD} | awk '{print $1}' > ${DEPLOY_DIR_PSTAGE}/installed-list
 279
 280                 # exitcode == 5 is ok, it means the files change
 281                 set +e
 282                 stage-manager -p ${STAGING_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-staging -u -d ${PSTAGE_TMPDIR_STAGE}/staging
 283                 exitcode=$?
 284                 if [ "$exitcode" != "5" -a "$exitcode" != "0" ]; then
 285                         exit $exitcode
 286                 fi
 287                 stage-manager -p ${CROSS_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-cross -u -d ${PSTAGE_TMPDIR_STAGE}/cross
 288                 if [ "$exitcode" != "5" -a "$exitcode" != "0" ]; then
 289                         exit $exitcode
 290                 fi
 291                 set -e
 292         fi
 293 }
 294
 295 do_populate_staging[lockfiles] = "${STAGING_DIR}/staging.lock"
 296 do_populate_staging[dirs] =+ "${DEPLOY_DIR_PSTAGE}"
 297 python do_populate_staging_prepend() {
 298     bb.build.exec_func("populate_staging_preamble", d)
 299 }
 300
 301 python do_populate_staging_append() {
 302     bb.build.exec_func("populate_staging_postamble", d)
 303 }
 304
 305
 306 staging_packager () {
 307
 308         mkdir -p ${PSTAGE_TMPDIR_STAGE}/CONTROL
 309         mkdir -p ${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}
 310
 311         echo "Package: ${PSTAGE_PKGPN}"         >  ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 312         echo "Version: ${PSTAGE_PKGVERSION}"    >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 313         echo "Description: ${DESCRIPTION}"      >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 314         echo "Section: ${SECTION}"              >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 315         echo "Priority: Optional"               >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 316         echo "Maintainer: ${MAINTAINER}"        >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 317         echo "Architecture: ${PSTAGE_PKGARCH}"  >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 318
 319         # Protect against empty SRC_URI
 320         if [ "${SRC_URI}" != "" ] ; then
 321                 echo "Source: ${SRC_URI}"               >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 322         else
 323                 echo "Source: OpenEmbedded"               >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 324         fi
 325
 326         ${PSTAGE_BUILD_CMD} ${PSTAGE_TMPDIR_STAGE} ${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}
 327 }
 328
 329 staging_package_installer () {
 330         #${PSTAGE_INSTALL_CMD} ${PSTAGE_PKG}
 331
 332         STATUSFILE=${TMPDIR}${layout_libdir}/opkg/status
 333         echo "Package: ${PSTAGE_PKGPN}"        >> $STATUSFILE
 334         echo "Version: ${PSTAGE_PKGVERSION}"   >> $STATUSFILE
 335         echo "Status: install user installed"  >> $STATUSFILE
 336         echo "Architecture: ${PSTAGE_PKGARCH}" >> $STATUSFILE
 337         echo "" >> $STATUSFILE
 338
 339         CTRLFILE=${TMPDIR}${layout_libdir}/opkg/info/${PSTAGE_PKGPN}.control
 340         echo "Package: ${PSTAGE_PKGPN}"        > $CTRLFILE
 341         echo "Version: ${PSTAGE_PKGVERSION}"   >> $CTRLFILE
 342         echo "Architecture: ${PSTAGE_PKGARCH}" >> $CTRLFILE
 343
 344         cd ${PSTAGE_TMPDIR_STAGE}
 345         find -type f | grep -v ./CONTROL | sed -e 's/^\.//' > ${TMPDIR}${layout_libdir}/opkg/info/${PSTAGE_PKGPN}.list
 346 }
 347
 348 python do_package_stage () {
 349     if bb.data.getVar("PSTAGING_ACTIVE", d, 1) != "1":
 350         return
 351
 352     #
 353     # Handle deploy/ packages
 354     #
 355     bb.build.exec_func("read_subpackage_metadata", d)
 356     stagepath = bb.data.getVar("PSTAGE_TMPDIR_STAGE", d, 1)
 357     tmpdir = bb.data.getVar("TMPDIR", d, True)
 358     packages = (bb.data.getVar('PACKAGES', d, 1) or "").split()
 359     if len(packages) > 0:
 360         if bb.data.inherits_class('package_ipk', d):
 361             ipkpath = bb.data.getVar('DEPLOY_DIR_IPK', d, True).replace(tmpdir, stagepath)
 362         if bb.data.inherits_class('package_deb', d):
 363             debpath = bb.data.getVar('DEPLOY_DIR_DEB', d, True).replace(tmpdir, stagepath)
 364
 365         for pkg in packages:
 366             pkgname = bb.data.getVar('PKG_%s' % pkg, d, 1)
 367             if not pkgname:
 368                 pkgname = pkg
 369             arch = bb.data.getVar('PACKAGE_ARCH_%s' % pkg, d, 1)
 370             if not arch:
 371                 arch = bb.data.getVar('PACKAGE_ARCH', d, 1)
 372             pr = bb.data.getVar('PR_%s' % pkg, d, 1)
 373             if not pr:
 374                 pr = bb.data.getVar('PR', d, 1)
 375             if not packaged(pkg, d):
 376                 continue
 377             if bb.data.inherits_class('package_ipk', d):
 378                 srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "_" + arch + ".ipk", d)
 379                 srcfile = bb.data.expand("${DEPLOY_DIR_IPK}/" + arch + "/" + srcname, d)
 380                 if os.path.exists(srcfile):
 381                     destpath = ipkpath + "/" + arch + "/"
 382                     bb.mkdirhier(destpath)
 383                     bb.copyfile(srcfile, destpath + srcname)
 384
 385             if bb.data.inherits_class('package_deb', d):
 386                 if arch == 'all':
 387                     srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "_all.deb", d)
 388                 else:
 389                     srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "_${DPKG_ARCH}.deb", d)
 390                 srcfile = bb.data.expand("${DEPLOY_DIR_DEB}/" + arch + "/" + srcname, d)
 391                 if os.path.exists(srcfile):
 392                     destpath = debpath + "/" + arch + "/"
 393                     bb.mkdirhier(destpath)
 394                     bb.copyfile(srcfile, destpath + srcname)
 395
 396     #
 397     # Handle stamps/ files
 398     #
 399     stampfn = bb.data.getVar("STAMP", d, True)
 400     destdir = os.path.dirname(stampfn.replace(tmpdir, stagepath))
 401     bb.mkdirhier(destdir)
 402     # We need to include the package_stage stamp in the staging package so create one
 403     bb.build.make_stamp("do_package_stage", d)
 404     os.system("cp -dpR %s.do_* %s/" % (stampfn, destdir))
 405
 406     pstage_set_pkgmanager(d)
 407     bb.build.exec_func("staging_helper", d)
 408     bb.build.exec_func("staging_packager", d)
 409     lf = bb.utils.lockfile(bb.data.expand("${STAGING_DIR}/staging.lock", d))
 410     bb.build.exec_func("staging_package_installer", d)
 411     bb.utils.unlockfile(lf)
 412 }
 413
 414 #
 415 # Note an assumption here is that do_deploy runs before do_package_write/do_populate_staging
 416 #
 417 addtask package_stage after do_package_write do_populate_staging before do_build
 418
 419 do_package_stage_all () {
 420         :
 421 }
 422 do_package_stage_all[recrdeptask] = "do_package_stage"
 423 addtask package_stage_all after do_package_stage before do_build