code.vuplus.com Git - vuplus_openembedded/blob - classes/packaged-staging.bbclass

   1 #
   2 # Populate builds using prebuilt packages where possible to speed up builds
   3 # and allow staging to be reconstructed.
   4 #
   5 # To use it add that line to conf/local.conf:
   6 #
   7 # INHERIT += "packaged-staging"
   8 #
   9
  10
  11 #
  12 # bitbake.conf set PSTAGING_ACTIVE = "0", this class sets to "1" if we're active
  13 #
  14 PSTAGE_PKGVERSION = "${PV}-${PR}"
  15 PSTAGE_PKGARCH    = "${BUILD_SYS}"
  16 PSTAGE_EXTRAPATH  ?= "/${OELAYOUT_ABI}/${DISTRO_PR}/"
  17 PSTAGE_PKGPATH    = "${DISTRO}${PSTAGE_EXTRAPATH}"
  18 PSTAGE_PKGPN      = "${@bb.data.expand('staging-${PN}-${MULTIMACH_ARCH}${TARGET_VENDOR}-${TARGET_OS}', d).replace('_', '-')}"
  19 PSTAGE_PKGNAME    = "${PSTAGE_PKGPN}_${PSTAGE_PKGVERSION}_${PSTAGE_PKGARCH}.ipk"
  20 PSTAGE_PKG        ?= "${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}/${PSTAGE_PKGNAME}"
  21
  22 PSTAGE_NATIVEDEPENDS = "\
  23     shasum-native \
  24     stagemanager-native \
  25     "
  26
  27 BB_STAMP_WHITELIST = "${PSTAGE_NATIVEDEPENDS}"
  28
  29 python () {
  30     pstage_allowed = True
  31
  32     # These classes encode staging paths into the binary data so can only be
  33     # reused if the path doesn't change/
  34     if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('sdk', d):
  35         path = bb.data.getVar('PSTAGE_PKGPATH', d, 1)
  36         path = path + bb.data.getVar('TMPDIR', d, 1).replace('/', '-')
  37         bb.data.setVar('PSTAGE_PKGPATH', path, d)
  38
  39     # PSTAGE_NATIVEDEPENDS lists the packages we need before we can use packaged
  40     # staging. There will always be some packages we depend on.
  41     if bb.data.inherits_class('native', d):
  42         pn = bb.data.getVar('PN', d, True)
  43         nativedeps = bb.data.getVar('PSTAGE_NATIVEDEPENDS', d, True).split()
  44         if pn in nativedeps:
  45             pstage_allowed = False
  46
  47     # Images aren't of interest to us
  48     if bb.data.inherits_class('image', d):
  49         pstage_allowed = False
  50
  51     if bb.data.getVar('PSTAGING_DISABLED', d, True) == "1":
  52         pstage_allowed = False
  53
  54     # Add task dependencies if we're active, otherwise mark packaged staging
  55     # as inactive.
  56     if pstage_allowed:
  57         deps = bb.data.getVarFlag('do_setscene', 'depends', d) or ""
  58         deps += " stagemanager-native:do_populate_staging"
  59         bb.data.setVarFlag('do_setscene', 'depends', deps, d)
  60
  61         policy = bb.data.getVar("BB_STAMP_POLICY", d, True)
  62         if policy == "whitelist" or policy == "full":
  63            deps = bb.data.getVarFlag('do_setscene', 'recrdeptask', d) or ""
  64            deps += " do_setscene"
  65            bb.data.setVarFlag('do_setscene', 'recrdeptask', deps, d)
  66
  67         bb.data.setVar("PSTAGING_ACTIVE", "1", d)
  68     else:
  69         bb.data.setVar("PSTAGING_ACTIVE", "0", d)
  70 }
  71
  72 DEPLOY_DIR_PSTAGE   ?= "${DEPLOY_DIR}/pstage"
  73 PSTAGE_MACHCONFIG   = "${DEPLOY_DIR_PSTAGE}/opkg.conf"
  74
  75 PSTAGE_PKGMANAGER = "stage-manager-ipkg"
  76
  77 PSTAGE_BUILD_CMD        = "stage-manager-ipkg-build -o 0 -g 0"
  78 PSTAGE_INSTALL_CMD      = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -force-depends -o ${TMPDIR} install"
  79 PSTAGE_UPDATE_CMD       = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR} update"
  80 PSTAGE_REMOVE_CMD       = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -force-depends -o ${TMPDIR} remove"
  81 PSTAGE_LIST_CMD         = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR} list_installed"
  82
  83 PSTAGE_TMPDIR_STAGE     = "${WORKDIR}/staging-pkg"
  84
  85 def pstage_manualclean(srcname, destvarname, d):
  86         src = os.path.join(bb.data.getVar('PSTAGE_TMPDIR_STAGE', d, True), srcname)
  87         dest = bb.data.getVar(destvarname, d, True)
  88
  89         for walkroot, dirs, files in os.walk(src):
  90                 for file in files:
  91                         filepath = os.path.join(walkroot, file).replace(src, dest)
  92                         bb.note("rm %s" % filepath)
  93                         os.system("rm %s" % filepath)
  94
  95 def pstage_set_pkgmanager(d):
  96     path = bb.data.getVar("PATH", d, 1)
  97     pkgmanager = bb.which(path, 'opkg-cl')
  98     if pkgmanager == "":
  99         pkgmanager = bb.which(path, 'ipkg-cl')
 100     if pkgmanager != "":
 101         bb.data.setVar("PSTAGE_PKGMANAGER", pkgmanager, d)
 102
 103
 104 def pstage_cleanpackage(pkgname, d):
 105         path = bb.data.getVar("PATH", d, 1)
 106         pstage_set_pkgmanager(d)
 107         list_cmd = bb.data.getVar("PSTAGE_LIST_CMD", d, True)
 108
 109         bb.debug(2, "Checking if staging package installed")
 110         lf = bb.utils.lockfile(bb.data.expand("${SYSROOT_LOCK}", d))
 111         ret = os.system("PATH=\"%s\" %s | grep %s" % (path, list_cmd, pkgname))
 112         if ret == 0:
 113                 bb.debug(1, "Uninstalling package from staging...")
 114                 removecmd = bb.data.getVar("PSTAGE_REMOVE_CMD", d, 1)
 115                 ret = os.system("PATH=\"%s\" %s %s" % (path, removecmd, pkgname))
 116                 if ret != 0:
 117                         bb.note("Failure removing staging package")
 118         else:
 119                 bb.debug(1, "Manually removing any installed files from staging...")
 120                 pstage_manualclean("staging", "STAGING_DIR", d)
 121                 pstage_manualclean("cross", "CROSS_DIR", d)
 122                 pstage_manualclean("deploy", "DEPLOY_DIR", d)
 123
 124         bb.utils.unlockfile(lf)
 125
 126 do_clean_prepend() {
 127         """
 128         Clear the build and temp directories
 129         """
 130
 131         removepkg = bb.data.expand("${PSTAGE_PKGPN}", d)
 132         pstage_cleanpackage(removepkg, d)
 133
 134         stagepkg = bb.data.expand("${PSTAGE_PKG}", d)
 135         bb.note("Removing staging package %s" % base_path_out(stagepkg, d))
 136         os.system('rm -rf ' + stagepkg)
 137 }
 138
 139 staging_helper () {
 140         # Assemble appropriate opkg.conf
 141         conffile=${PSTAGE_MACHCONFIG}
 142         mkdir -p ${DEPLOY_DIR_PSTAGE}/pstaging_lists
 143         if [ ! -e $conffile ]; then
 144                 ipkgarchs="${BUILD_SYS}"
 145                 priority=1
 146                 for arch in $ipkgarchs; do
 147                         echo "arch $arch $priority" >> $conffile
 148                         priority=$(expr $priority + 5)
 149                 done
 150                 echo "dest root /" >> $conffile
 151         fi
 152         if [ ! -e ${TMPDIR}${libdir_native}/opkg/info/ ]; then
 153                 mkdir -p ${TMPDIR}${libdir_native}/opkg/info/
 154         fi
 155         if [ ! -e ${TMPDIR}${libdir_native}/ipkg/ ]; then
 156                 ln -sf opkg/ ${TMPDIR}${libdir_native}/ipkg || true
 157         fi
 158 }
 159
 160 PSTAGE_TASKS_COVERED = "fetch unpack munge patch configure qa_configure rig_locales compile sizecheck install deploy package populate_staging package_write_deb package_write_ipk package_write package_stage qa_staging"
 161
 162 SCENEFUNCS += "packagestage_scenefunc"
 163
 164 python packagestage_scenefunc () {
 165     if bb.data.getVar("PSTAGING_ACTIVE", d, 1) == "0":
 166         return
 167
 168     bb.build.exec_func("staging_helper", d)
 169
 170     removepkg = bb.data.expand("${PSTAGE_PKGPN}", d)
 171
 172     pstage_cleanpackage(removepkg, d)
 173
 174     stagepkg = bb.data.expand("${PSTAGE_PKG}", d)
 175
 176     if os.path.exists(stagepkg):
 177         path = bb.data.getVar("PATH", d, 1)
 178         pstage_set_pkgmanager(d)
 179         file = bb.data.getVar("FILE", d, True)
 180         bb.debug(2, "Packaged staging active for %s\n" % file)
 181
 182         #
 183         # Install the staging package somewhere temporarily so we can extract the stamp files
 184         #
 185         bb.mkdirhier(bb.data.expand("${WORKDIR}/tstage/${libdir_native}/opkg/info/ ", d))
 186         cmd = bb.data.expand("${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -force-depends -o ${WORKDIR}/tstage install", d)
 187         ret = os.system("PATH=\"%s\" %s %s" % (path, cmd, stagepkg))
 188         if ret != 0:
 189             bb.fatal("Couldn't install the staging package to a temp directory")
 190
 191         #
 192         # Copy the stamp files into the main stamps directoy
 193         #
 194         cmd = bb.data.expand("cp -dpR ${WORKDIR}/tstage/stamps/* ${TMPDIR}/stamps/", d)
 195         ret = os.system(cmd)
 196         if ret != 0:
 197             bb.fatal("Couldn't copy the staging package stamp files")
 198
 199         #
 200         # Iterate over the stamps seeing if they're valid. If we find any that
 201         # are invalid or the task wasn't in the taskgraph, assume caution and
 202         # do a rebuild.
 203         #
 204         # FIXME - some tasks are safe to ignore in the task graph. e.g. package_write_*
 205         stageok = True
 206         taskscovered = bb.data.getVar("PSTAGE_TASKS_COVERED", d, True).split()
 207         stamp = bb.data.getVar("STAMP", d, True)
 208         for task in taskscovered:
 209             task = 'do_' + task
 210             stampfn = "%s.%s" % (stamp, task)
 211             bb.debug(1, "Checking %s" % (stampfn))
 212             if os.path.exists(stampfn):
 213                 stageok = bb.runqueue.check_stamp_fn(file, task, d)
 214                 bb.debug(1, "Result %s" % (stageok))
 215                 if not stageok:
 216                     break
 217
 218         # Remove the stamps and files we added above
 219         # FIXME - we should really only remove the stamps we added
 220         os.system('rm -f ' + stamp + '.*')
 221         os.system(bb.data.expand("rm -rf ${WORKDIR}/tstage", d))
 222
 223         if stageok:
 224             bb.note("Staging package found, using it for %s." % file)
 225             installcmd = bb.data.getVar("PSTAGE_INSTALL_CMD", d, 1)
 226             lf = bb.utils.lockfile(bb.data.expand("${SYSROOT_LOCK}", d))
 227             ret = os.system("PATH=\"%s\" %s %s" % (path, installcmd, stagepkg))
 228             bb.utils.unlockfile(lf)
 229             if ret != 0:
 230                 bb.note("Failure installing prestage package")
 231
 232             bb.build.make_stamp("do_stage_package_populated", d)
 233         else:
 234             bb.note("Staging package found but invalid for %s" % file)
 235
 236 }
 237 packagestage_scenefunc[cleandirs] = "${PSTAGE_TMPDIR_STAGE}"
 238 packagestage_scenefunc[dirs] = "${STAGING_DIR}"
 239
 240 addhandler packagedstage_stampfixing_eventhandler
 241 python packagedstage_stampfixing_eventhandler() {
 242     if bb.event.getName(e) == "StampUpdate":
 243         taskscovered = bb.data.getVar("PSTAGE_TASKS_COVERED", e.data, 1).split()
 244         for (fn, task) in e.targets:
 245             # strip off 'do_'
 246             task = task[3:]
 247             if task in taskscovered:
 248                 stamp = "%s.do_stage_package_populated" % e.stampPrefix[fn]
 249                 if os.path.exists(stamp):
 250                     # We're targetting a task which was skipped with packaged staging
 251                     # so we need to remove the autogenerated stamps.
 252                     for task in taskscovered:
 253                         dir = "%s.do_%s" % (e.stampPrefix[fn], task)
 254                         os.system('rm -f ' + dir)
 255                     os.system('rm -f ' + stamp)
 256
 257     return NotHandled
 258 }
 259
 260 populate_staging_preamble () {
 261         if [ "$PSTAGING_ACTIVE" = "1" ]; then
 262                 stage-manager -p ${STAGING_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-staging -u || true
 263                 stage-manager -p ${CROSS_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-cross -u || true
 264         fi
 265 }
 266
 267 populate_staging_postamble () {
 268         if [ "$PSTAGING_ACTIVE" = "1" ]; then
 269                 # list the packages currently installed in staging
 270                 # ${PSTAGE_LIST_CMD} | awk '{print $1}' > ${DEPLOY_DIR_PSTAGE}/installed-list
 271
 272                 # exitcode == 5 is ok, it means the files change
 273                 set +e
 274                 stage-manager -p ${STAGING_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-staging -u -d ${PSTAGE_TMPDIR_STAGE}/staging
 275                 exitcode=$?
 276                 if [ "$exitcode" != "5" -a "$exitcode" != "0" ]; then
 277                         exit $exitcode
 278                 fi
 279                 stage-manager -p ${CROSS_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-cross -u -d ${PSTAGE_TMPDIR_STAGE}/cross/${BASE_PACKAGE_ARCH}
 280                 if [ "$exitcode" != "5" -a "$exitcode" != "0" ]; then
 281                         exit $exitcode
 282                 fi
 283                 set -e
 284         fi
 285 }
 286
 287 packagedstaging_fastpath () {
 288         if [ "$PSTAGING_ACTIVE" = "1" ]; then
 289                 mkdir -p ${PSTAGE_TMPDIR_STAGE}/staging/
 290                 mkdir -p ${PSTAGE_TMPDIR_STAGE}/cross/
 291                 cp -fpPR ${SYSROOT_DESTDIR}${STAGING_DIR}/* ${PSTAGE_TMPDIR_STAGE}/staging/ || /bin/true
 292                 cp -fpPR ${SYSROOT_DESTDIR}${CROSS_DIR}/* ${PSTAGE_TMPDIR_STAGE}/cross/ || /bin/true
 293         fi
 294 }
 295
 296 do_populate_staging[dirs] =+ "${DEPLOY_DIR_PSTAGE}"
 297 python populate_staging_prehook() {
 298     bb.build.exec_func("populate_staging_preamble", d)
 299 }
 300
 301 python populate_staging_posthook() {
 302     bb.build.exec_func("populate_staging_postamble", d)
 303 }
 304
 305
 306 staging_packager () {
 307
 308         mkdir -p ${PSTAGE_TMPDIR_STAGE}/CONTROL
 309         mkdir -p ${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}
 310
 311         echo "Package: ${PSTAGE_PKGPN}"         >  ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 312         echo "Version: ${PSTAGE_PKGVERSION}"    >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 313         echo "Description: ${DESCRIPTION}"      >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 314         echo "Section: ${SECTION}"              >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 315         echo "Priority: Optional"               >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 316         echo "Maintainer: ${MAINTAINER}"        >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 317         echo "Architecture: ${PSTAGE_PKGARCH}"  >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 318
 319         # Protect against empty SRC_URI
 320         if [ "${SRC_URI}" != "" ] ; then
 321                 echo "Source: ${SRC_URI}"               >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 322         else
 323                 echo "Source: OpenEmbedded"               >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 324         fi
 325
 326         ${PSTAGE_BUILD_CMD} ${PSTAGE_TMPDIR_STAGE} ${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}
 327 }
 328
 329 staging_package_installer () {
 330         #${PSTAGE_INSTALL_CMD} ${PSTAGE_PKG}
 331
 332         STATUSFILE=${TMPDIR}${libdir_native}/opkg/status
 333         echo "Package: ${PSTAGE_PKGPN}"        >> $STATUSFILE
 334         echo "Version: ${PSTAGE_PKGVERSION}"   >> $STATUSFILE
 335         echo "Status: install user installed"  >> $STATUSFILE
 336         echo "Architecture: ${PSTAGE_PKGARCH}" >> $STATUSFILE
 337         echo "" >> $STATUSFILE
 338
 339         CTRLFILE=${TMPDIR}${libdir_native}/opkg/info/${PSTAGE_PKGPN}.control
 340         echo "Package: ${PSTAGE_PKGPN}"        > $CTRLFILE
 341         echo "Version: ${PSTAGE_PKGVERSION}"   >> $CTRLFILE
 342         echo "Architecture: ${PSTAGE_PKGARCH}" >> $CTRLFILE
 343
 344         cd ${PSTAGE_TMPDIR_STAGE}
 345         find -type f | grep -v ./CONTROL | sed -e 's/^\.//' > ${TMPDIR}${libdir_native}/opkg/info/${PSTAGE_PKGPN}.list
 346 }
 347
 348 python do_package_stage () {
 349     if bb.data.getVar("PSTAGING_ACTIVE", d, 1) != "1":
 350         return
 351
 352     #
 353     # Handle deploy/ packages
 354     #
 355     bb.build.exec_func("read_subpackage_metadata", d)
 356     stagepath = bb.data.getVar("PSTAGE_TMPDIR_STAGE", d, 1)
 357     tmpdir = bb.data.getVar("TMPDIR", d, True)
 358     packages = (bb.data.getVar('PACKAGES', d, 1) or "").split()
 359     if len(packages) > 0:
 360         if bb.data.inherits_class('package_ipk', d):
 361             ipkpath = bb.data.getVar('DEPLOY_DIR_IPK', d, True).replace(tmpdir, stagepath)
 362         if bb.data.inherits_class('package_deb', d):
 363             debpath = bb.data.getVar('DEPLOY_DIR_DEB', d, True).replace(tmpdir, stagepath)
 364         if bb.data.inherits_class('package_rpm', d):
 365             rpmpath = bb.data.getVar('DEPLOY_DIR_RPM', d, True).replace(tmpdir, stagepath)
 366
 367         for pkg in packages:
 368             pkgname = bb.data.getVar('PKG_%s' % pkg, d, 1)
 369             if not pkgname:
 370                 pkgname = pkg
 371             arch = bb.data.getVar('PACKAGE_ARCH_%s' % pkg, d, 1)
 372             if not arch:
 373                 arch = bb.data.getVar('PACKAGE_ARCH', d, 1)
 374             pr = bb.data.getVar('PR_%s' % pkg, d, 1)
 375             if not pr:
 376                 pr = bb.data.getVar('PR', d, 1)
 377             if not packaged(pkg, d):
 378                 continue
 379             if bb.data.inherits_class('package_ipk', d):
 380                 srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "${DISTRO_PR}" + "_" + arch + ".ipk", d)
 381                 srcfile = bb.data.expand("${DEPLOY_DIR_IPK}/" + arch + "/" + srcname, d)
 382                 if os.path.exists(srcfile):
 383                     destpath = ipkpath + "/" + arch + "/"
 384                     bb.mkdirhier(destpath)
 385                     bb.copyfile(srcfile, destpath + srcname)
 386
 387             if bb.data.inherits_class('package_deb', d):
 388                 if arch == 'all':
 389                     srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "${DISTRO_PR}" + "_all.deb", d)
 390                 else:
 391                     srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "${DISTRO_PR}" + "_${DPKG_ARCH}.deb", d)
 392                 srcfile = bb.data.expand("${DEPLOY_DIR_DEB}/" + arch + "/" + srcname, d)
 393                 if os.path.exists(srcfile):
 394                     destpath = debpath + "/" + arch + "/"
 395                     bb.mkdirhier(destpath)
 396                     bb.copyfile(srcfile, destpath + srcname)
 397
 398             if bb.data.inherits_class('package_rpm', d):
 399                 version = bb.data.getVar('PV', d, 1)
 400                 version = version.replace('-', '+')
 401                 bb.data.setVar('RPMPV', version, d)
 402                 srcname = bb.data.expand(pkgname + "-${RPMPV}-" + pr + "${DISTRO_PR}" + ".${TARGET_ARCH}.rpm", d)
 403                 srcfile = bb.data.expand("${DEPLOY_DIR_RPM}/" + arch + "/" + srcname, d)
 404                 if os.path.exists(srcfile):
 405                     destpath = rpmpath + "/" + arch + "/"
 406                     bb.mkdirhier(destpath)
 407                     bb.copyfile(srcfile, destpath + srcname)
 408
 409
 410     #
 411     # Handle stamps/ files
 412     #
 413     stampfn = bb.data.getVar("STAMP", d, True)
 414     destdir = os.path.dirname(stampfn.replace(tmpdir, stagepath))
 415     bb.mkdirhier(destdir)
 416     # We need to include the package_stage stamp in the staging package so create one
 417     bb.build.make_stamp("do_package_stage", d)
 418     os.system("cp -dpR %s.do_* %s/" % (stampfn, destdir))
 419
 420     pstage_set_pkgmanager(d)
 421     bb.build.exec_func("staging_helper", d)
 422     bb.build.exec_func("staging_packager", d)
 423     lf = bb.utils.lockfile(bb.data.expand("${SYSROOT_LOCK}", d))
 424     bb.build.exec_func("staging_package_installer", d)
 425     bb.utils.unlockfile(lf)
 426 }
 427
 428 #
 429 # Note an assumption here is that do_deploy runs before do_package_write/do_populate_staging
 430 #
 431 addtask package_stage after do_package_write do_populate_staging before do_build
 432
 433 do_package_stage_all () {
 434         :
 435 }
 436 do_package_stage_all[recrdeptask] = "do_package_stage"
 437 addtask package_stage_all after do_package_stage before do_build