code.vuplus.com Git - vuplus_openembedded/blob - classes/packaged-staging.bbclass

   1 #
   2 # Populate builds using prebuilt packages where possible to speed up builds
   3 # and allow staging to be reconstructed.
   4 #
   5 # To use it add that line to conf/local.conf:
   6 #
   7 # INHERIT += "packaged-staging"
   8 #
   9
  10
  11 #
  12 # bitbake.conf set PSTAGING_ACTIVE = "0", this class sets to "1" if we're active
  13 #
  14 PSTAGE_PKGVERSION = "${PV}-${PR}"
  15 PSTAGE_PKGARCH    = "${BUILD_SYS}"
  16 PSTAGE_EXTRAPATH  ?= ""
  17 PSTAGE_PKGPATH    = "${DISTRO}${PSTAGE_EXTRAPATH}"
  18 PSTAGE_PKGPN      = "${@bb.data.expand('staging-${PN}-${MULTIMACH_ARCH}${TARGET_VENDOR}-${TARGET_OS}', d).replace('_', '-')}"
  19 PSTAGE_PKGNAME    = "${PSTAGE_PKGPN}_${PSTAGE_PKGVERSION}_${PSTAGE_PKGARCH}.ipk"
  20 PSTAGE_PKG        = "${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}/${PSTAGE_PKGNAME}"
  21
  22 # multimachine.bbclass will override this but add a default in case we're not using it
  23 MULTIMACH_ARCH ?= "${PACKAGE_ARCH}"
  24
  25 PSTAGE_NATIVEDEPENDS = "\
  26     shasum-native \
  27     stagemanager-native \
  28     "
  29
  30 BB_STAMP_WHITELIST = "${PSTAGE_NATIVEDEPENDS}"
  31
  32 python () {
  33     import bb
  34     pstage_allowed = True
  35
  36     # These classes encode staging paths into the binary data so can only be
  37     # reused if the path doesn't change/
  38     if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('sdk', d):
  39         path = bb.data.getVar('PSTAGE_PKGPATH', d, 1)
  40         path = path + bb.data.getVar('TMPDIR', d, 1).replace('/', '-')
  41         bb.data.setVar('PSTAGE_PKGPATH', path, d)
  42
  43     # PSTAGE_NATIVEDEPENDS lists the packages we need before we can use packaged
  44     # staging. There will always be some packages we depend on.
  45     if bb.data.inherits_class('native', d):
  46         pn = bb.data.getVar('PN', d, True)
  47         nativedeps = bb.data.getVar('PSTAGE_NATIVEDEPENDS', d, True).split()
  48         if pn in nativedeps:
  49             pstage_allowed = False
  50
  51     # Images aren't of interest to us
  52     if bb.data.inherits_class('image', d):
  53         pstage_allowed = False
  54
  55     # Add task dependencies if we're active, otherwise mark packaged staging
  56     # as inactive.
  57     if pstage_allowed:
  58         deps = bb.data.getVarFlag('do_setscene', 'depends', d) or ""
  59         deps += " stagemanager-native:do_populate_staging"
  60         bb.data.setVarFlag('do_setscene', 'depends', deps, d)
  61
  62         policy = bb.data.getVar("BB_STAMP_POLICY", d, True)
  63         if policy == "whitelist" or policy == "full":
  64            deps = bb.data.getVarFlag('do_setscene', 'recrdeptask', d) or ""
  65            deps += " do_setscene"
  66            bb.data.setVarFlag('do_setscene', 'recrdeptask', deps, d)
  67
  68         bb.data.setVar("PSTAGING_ACTIVE", "1", d)
  69     else:
  70         bb.data.setVar("PSTAGING_ACTIVE", "0", d)
  71 }
  72
  73 DEPLOY_DIR_PSTAGE   = "${DEPLOY_DIR}/pstage"
  74 PSTAGE_MACHCONFIG   = "${DEPLOY_DIR_PSTAGE}/opkg.conf"
  75
  76 PSTAGE_PKGMANAGER = "stage-manager-ipkg"
  77
  78 PSTAGE_BUILD_CMD        = "stage-manager-ipkg-build -o 0 -g 0"
  79 PSTAGE_INSTALL_CMD      = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -force-depends -o ${TMPDIR} install"
  80 PSTAGE_UPDATE_CMD       = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR} update"
  81 PSTAGE_REMOVE_CMD       = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -force-depends -o ${TMPDIR} remove"
  82 PSTAGE_LIST_CMD         = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR} list_installed"
  83
  84 PSTAGE_TMPDIR_STAGE     = "${WORKDIR}/staging-pkg"
  85
  86 def pstage_manualclean(srcname, destvarname, d):
  87         import os, bb
  88
  89         src = os.path.join(bb.data.getVar('PSTAGE_TMPDIR_STAGE', d, True), srcname)
  90         dest = bb.data.getVar(destvarname, d, True)
  91
  92         for walkroot, dirs, files in os.walk(src):
  93                 for file in files:
  94                         filepath = os.path.join(walkroot, file).replace(src, dest)
  95                         bb.note("rm %s" % filepath)
  96                         os.system("rm %s" % filepath)
  97
  98 def pstage_set_pkgmanager(d):
  99     import bb
 100     path = bb.data.getVar("PATH", d, 1)
 101     pkgmanager = bb.which(path, 'opkg-cl')
 102     if pkgmanager == "":
 103         pkgmanager = bb.which(path, 'ipkg-cl')
 104     if pkgmanager != "":
 105         bb.data.setVar("PSTAGE_PKGMANAGER", pkgmanager, d)
 106
 107
 108 def pstage_cleanpackage(pkgname, d):
 109         import os, bb
 110
 111         path = bb.data.getVar("PATH", d, 1)
 112         pstage_set_pkgmanager(d)
 113         list_cmd = bb.data.getVar("PSTAGE_LIST_CMD", d, True)
 114
 115         bb.note("Checking if staging package installed")
 116         lf = bb.utils.lockfile(bb.data.expand("${STAGING_DIR}/staging.lock", d))
 117         ret = os.system("PATH=\"%s\" %s | grep %s" % (path, list_cmd, pkgname))
 118         if ret == 0:
 119                 bb.note("Yes. Uninstalling package from staging...")
 120                 removecmd = bb.data.getVar("PSTAGE_REMOVE_CMD", d, 1)
 121                 ret = os.system("PATH=\"%s\" %s %s" % (path, removecmd, pkgname))
 122                 if ret != 0:
 123                         bb.note("Failure removing staging package")
 124         else:
 125                 bb.note("No. Manually removing any installed files")
 126                 pstage_manualclean("staging", "STAGING_DIR", d)
 127                 pstage_manualclean("cross", "CROSS_DIR", d)
 128                 pstage_manualclean("deploy", "DEPLOY_DIR", d)
 129
 130         bb.utils.unlockfile(lf)
 131
 132 do_clean_prepend() {
 133         """
 134         Clear the build and temp directories
 135         """
 136
 137         removepkg = bb.data.expand("${PSTAGE_PKGPN}", d)
 138         pstage_cleanpackage(removepkg, d)
 139
 140         stagepkg = bb.data.expand("${PSTAGE_PKG}", d)
 141         bb.note("Removing staging package %s" % stagepkg)
 142         os.system('rm -rf ' + stagepkg)
 143 }
 144
 145 staging_helper () {
 146         # Assemble appropriate opkg.conf
 147         conffile=${PSTAGE_MACHCONFIG}
 148         mkdir -p ${DEPLOY_DIR_PSTAGE}/pstaging_lists
 149         if [ ! -e $conffile ]; then
 150                 ipkgarchs="${BUILD_SYS}"
 151                 priority=1
 152                 for arch in $ipkgarchs; do
 153                         echo "arch $arch $priority" >> $conffile
 154                         priority=$(expr $priority + 5)
 155                 done
 156                 echo "dest root /" >> $conffile
 157         fi
 158         if [ ! -e ${TMPDIR}${layout_libdir}/opkg/info/ ]; then
 159                 mkdir -p ${TMPDIR}${layout_libdir}/opkg/info/
 160         fi
 161         if [ ! -e ${TMPDIR}${layout_libdir}/ipkg/ ]; then
 162                 cd ${TMPDIR}${layout_libdir}/
 163                 ln -s opkg/ ipkg
 164         fi
 165 }
 166
 167 PSTAGE_TASKS_COVERED = "fetch unpack munge patch configure qa_configure rig_locales compile sizecheck install deploy package populate_staging package_write_deb package_write_ipk package_write package_stage qa_staging"
 168
 169 SCENEFUNCS += "packagestage_scenefunc"
 170
 171 python packagestage_scenefunc () {
 172     import os
 173
 174     if bb.data.getVar("PSTAGING_ACTIVE", d, 1) == "0":
 175         return
 176
 177     bb.build.exec_func("staging_helper", d)
 178
 179     removepkg = bb.data.expand("${PSTAGE_PKGPN}", d)
 180     pstage_cleanpackage(removepkg, d)
 181
 182     stagepkg = bb.data.expand("${PSTAGE_PKG}", d)
 183
 184     if os.path.exists(stagepkg):
 185         path = bb.data.getVar("PATH", d, 1)
 186         pstage_set_pkgmanager(d)
 187         file = bb.data.getVar("FILE", d, True)
 188         bb.debug(2, "Packaged staging active for %s\n" % file)
 189
 190         #
 191         # Install the staging package somewhere temporarily so we can extract the stamp files
 192         #
 193         bb.mkdirhier(bb.data.expand("${WORKDIR}/tstage/${layout_libdir}/opkg/info/ ", d))
 194         cmd = bb.data.expand("${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -force-depends -o ${WORKDIR}/tstage install", d)
 195         ret = os.system("PATH=\"%s\" %s %s" % (path, cmd, stagepkg))
 196         if ret != 0:
 197             bb.fatal("Couldn't install the staging package to a temp directory")
 198
 199         #
 200         # Copy the stamp files into the main stamps directoy
 201         #
 202         cmd = bb.data.expand("cp -dpR ${WORKDIR}/tstage/stamps/* ${TMPDIR}/stamps/", d)
 203         ret = os.system(cmd)
 204         if ret != 0:
 205             bb.fatal("Couldn't copy the staging package stamp files")
 206
 207         #
 208         # Iterate over the stamps seeing if they're valid. If we find any that
 209         # are invalid or the task wasn't in the taskgraph, assume caution and
 210         # do a rebuild.
 211         #
 212         # FIXME - some tasks are safe to ignore in the task graph. e.g. package_write_*
 213         stageok = True
 214         taskscovered = bb.data.getVar("PSTAGE_TASKS_COVERED", d, True).split()
 215         stamp = bb.data.getVar("STAMP", d, True)
 216         for task in taskscovered:
 217             task = 'do_' + task
 218             stampfn = "%s.%s" % (stamp, task)
 219             bb.debug(1, "Checking %s" % (stampfn))
 220             if os.path.exists(stampfn):
 221                 stageok = bb.runqueue.check_stamp_fn(file, task, d)
 222                 bb.debug(1, "Result %s" % (stageok))
 223                 if not stageok:
 224                     break
 225
 226         # Remove the stamps and files we added above
 227         # FIXME - we should really only remove the stamps we added
 228         os.system('rm -f ' + stamp + '.*')
 229         os.system(bb.data.expand("rm -rf ${WORKDIR}/tstage", d))
 230
 231         if stageok:
 232             bb.note("Staging package found, using it for %s." % file)
 233             installcmd = bb.data.getVar("PSTAGE_INSTALL_CMD", d, 1)
 234             lf = bb.utils.lockfile(bb.data.expand("${STAGING_DIR}/staging.lock", d))
 235             ret = os.system("PATH=\"%s\" %s %s" % (path, installcmd, stagepkg))
 236             bb.utils.unlockfile(lf)
 237             if ret != 0:
 238                 bb.note("Failure installing prestage package")
 239
 240             bb.build.make_stamp("do_stage_package_populated", d)
 241         else:
 242             bb.note("Staging package found but invalid for %s" % file)
 243
 244 }
 245 packagestage_scenefunc[cleandirs] = "${PSTAGE_TMPDIR_STAGE}"
 246 packagestage_scenefunc[dirs] = "${STAGING_DIR}"
 247
 248 addhandler packagedstage_stampfixing_eventhandler
 249 python packagedstage_stampfixing_eventhandler() {
 250     from bb.event import getName
 251     import os
 252
 253     if getName(e) == "StampUpdate":
 254         taskscovered = bb.data.getVar("PSTAGE_TASKS_COVERED", e.data, 1).split()
 255         for (fn, task) in e.targets:
 256             # strip off 'do_'
 257             task = task[3:]
 258             if task in taskscovered:
 259                 stamp = "%s.do_stage_package_populated" % e.stampPrefix[fn]
 260                 if os.path.exists(stamp):
 261                     # We're targetting a task which was skipped with packaged staging
 262                     # so we need to remove the autogenerated stamps.
 263                     for task in taskscovered:
 264                         dir = "%s.do_%s" % (e.stampPrefix[fn], task)
 265                         os.system('rm -f ' + dir)
 266                     os.system('rm -f ' + stamp)
 267
 268     return NotHandled
 269 }
 270
 271 populate_staging_preamble () {
 272         if [ "$PSTAGING_ACTIVE" = "1" ]; then
 273                 stage-manager -p ${STAGING_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-staging -u || true
 274                 stage-manager -p ${CROSS_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-cross -u || true
 275         fi
 276 }
 277
 278 populate_staging_postamble () {
 279         if [ "$PSTAGING_ACTIVE" = "1" ]; then
 280                 # list the packages currently installed in staging
 281                 # ${PSTAGE_LIST_CMD} | awk '{print $1}' > ${DEPLOY_DIR_PSTAGE}/installed-list
 282
 283                 # exitcode == 5 is ok, it means the files change
 284                 set +e
 285                 stage-manager -p ${STAGING_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-staging -u -d ${PSTAGE_TMPDIR_STAGE}/staging
 286                 exitcode=$?
 287                 if [ "$exitcode" != "5" -a "$exitcode" != "0" ]; then
 288                         exit $exitcode
 289                 fi
 290                 stage-manager -p ${CROSS_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-cross -u -d ${PSTAGE_TMPDIR_STAGE}/cross
 291                 if [ "$exitcode" != "5" -a "$exitcode" != "0" ]; then
 292                         exit $exitcode
 293                 fi
 294                 set -e
 295         fi
 296 }
 297
 298 do_populate_staging[lockfiles] = "${STAGING_DIR}/staging.lock"
 299 do_populate_staging[dirs] =+ "${DEPLOY_DIR_PSTAGE}"
 300 python do_populate_staging_prepend() {
 301     bb.build.exec_func("populate_staging_preamble", d)
 302 }
 303
 304 python do_populate_staging_append() {
 305     bb.build.exec_func("populate_staging_postamble", d)
 306 }
 307
 308
 309 staging_packager () {
 310
 311         mkdir -p ${PSTAGE_TMPDIR_STAGE}/CONTROL
 312         mkdir -p ${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}
 313
 314         echo "Package: ${PSTAGE_PKGPN}"         >  ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 315         echo "Version: ${PSTAGE_PKGVERSION}"    >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 316         echo "Description: ${DESCRIPTION}"      >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 317         echo "Section: ${SECTION}"              >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 318         echo "Priority: Optional"               >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 319         echo "Maintainer: ${MAINTAINER}"        >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 320         echo "Architecture: ${PSTAGE_PKGARCH}"  >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 321
 322         # Protect against empty SRC_URI
 323         if [ "${SRC_URI}" != "" ] ; then
 324                 echo "Source: ${SRC_URI}"               >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 325         else
 326                 echo "Source: OpenEmbedded"               >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 327         fi
 328
 329         ${PSTAGE_BUILD_CMD} ${PSTAGE_TMPDIR_STAGE} ${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}
 330 }
 331
 332 staging_package_installer () {
 333         #${PSTAGE_INSTALL_CMD} ${PSTAGE_PKG}
 334
 335         STATUSFILE=${TMPDIR}${layout_libdir}/opkg/status
 336         echo "Package: ${PSTAGE_PKGPN}"        >> $STATUSFILE
 337         echo "Version: ${PSTAGE_PKGVERSION}"   >> $STATUSFILE
 338         echo "Status: install user installed"  >> $STATUSFILE
 339         echo "Architecture: ${PSTAGE_PKGARCH}" >> $STATUSFILE
 340         echo "" >> $STATUSFILE
 341
 342         CTRLFILE=${TMPDIR}${layout_libdir}/opkg/info/${PSTAGE_PKGPN}.control
 343         echo "Package: ${PSTAGE_PKGPN}"        > $CTRLFILE
 344         echo "Version: ${PSTAGE_PKGVERSION}"   >> $CTRLFILE
 345         echo "Architecture: ${PSTAGE_PKGARCH}" >> $CTRLFILE
 346
 347         cd ${PSTAGE_TMPDIR_STAGE}
 348         find -type f | grep -v ./CONTROL | sed -e 's/^\.//' > ${TMPDIR}${layout_libdir}/opkg/info/${PSTAGE_PKGPN}.list
 349 }
 350
 351 python do_package_stage () {
 352     if bb.data.getVar("PSTAGING_ACTIVE", d, 1) != "1":
 353         return
 354
 355     #
 356     # Handle deploy/ packages
 357     #
 358     bb.build.exec_func("read_subpackage_metadata", d)
 359     stagepath = bb.data.getVar("PSTAGE_TMPDIR_STAGE", d, 1)
 360     tmpdir = bb.data.getVar("TMPDIR", d, True)
 361     packages = (bb.data.getVar('PACKAGES', d, 1) or "").split()
 362     if len(packages) > 0:
 363         if bb.data.inherits_class('package_ipk', d):
 364             ipkpath = bb.data.getVar('DEPLOY_DIR_IPK', d, True).replace(tmpdir, stagepath)
 365         if bb.data.inherits_class('package_deb', d):
 366             debpath = bb.data.getVar('DEPLOY_DIR_DEB', d, True).replace(tmpdir, stagepath)
 367
 368         for pkg in packages:
 369             pkgname = bb.data.getVar('PKG_%s' % pkg, d, 1)
 370             if not pkgname:
 371                 pkgname = pkg
 372             arch = bb.data.getVar('PACKAGE_ARCH_%s' % pkg, d, 1)
 373             if not arch:
 374                 arch = bb.data.getVar('PACKAGE_ARCH', d, 1)
 375             pr = bb.data.getVar('PR_%s' % pkg, d, 1)
 376             if not pr:
 377                 pr = bb.data.getVar('PR', d, 1)
 378             if not packaged(pkg, d):
 379                 continue
 380             if bb.data.inherits_class('package_ipk', d):
 381                 srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "_" + arch + ".ipk", d)
 382                 srcfile = bb.data.expand("${DEPLOY_DIR_IPK}/" + arch + "/" + srcname, d)
 383                 if os.path.exists(srcfile):
 384                     destpath = ipkpath + "/" + arch + "/"
 385                     bb.mkdirhier(destpath)
 386                     bb.copyfile(srcfile, destpath + srcname)
 387
 388             if bb.data.inherits_class('package_deb', d):
 389                 if arch == 'all':
 390                     srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "_all.deb", d)
 391                 else:
 392                     srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "_${DPKG_ARCH}.deb", d)
 393                 srcfile = bb.data.expand("${DEPLOY_DIR_DEB}/" + arch + "/" + srcname, d)
 394                 if os.path.exists(srcfile):
 395                     destpath = debpath + "/" + arch + "/"
 396                     bb.mkdirhier(destpath)
 397                     bb.copyfile(srcfile, destpath + srcname)
 398
 399     #
 400     # Handle stamps/ files
 401     #
 402     stampfn = bb.data.getVar("STAMP", d, True)
 403     destdir = os.path.dirname(stampfn.replace(tmpdir, stagepath))
 404     bb.mkdirhier(destdir)
 405     # We need to include the package_stage stamp in the staging package so create one
 406     bb.build.make_stamp("do_package_stage", d)
 407     os.system("cp -dpR %s.do_* %s/" % (stampfn, destdir))
 408
 409     pstage_set_pkgmanager(d)
 410     bb.build.exec_func("staging_helper", d)
 411     bb.build.exec_func("staging_packager", d)
 412     lf = bb.utils.lockfile(bb.data.expand("${STAGING_DIR}/staging.lock", d))
 413     bb.build.exec_func("staging_package_installer", d)
 414     bb.utils.unlockfile(lf)
 415 }
 416
 417 #
 418 # Note an assumption here is that do_deploy runs before do_package_write/do_populate_staging
 419 #
 420 addtask package_stage after do_package_write do_populate_staging before do_build
 421
 422 do_package_stage_all () {
 423         :
 424 }
 425 do_package_stage_all[recrdeptask] = "do_package_stage"
 426 addtask package_stage_all after do_package_stage before do_build