From cbd57c100522ec764817d312d66de60af372c7cd Mon Sep 17 00:00:00 2001 From: "Carl D. Roth" Date: Fri, 4 Nov 2016 12:35:50 -0700 Subject: [PATCH 01/12] Refactor boot loader config upgrade code --- .../src/python/onl/install/BaseInstall.py | 77 +++++++++++++++---- 1 file changed, 62 insertions(+), 15 deletions(-) diff --git a/packages/base/all/vendor-config-onl/src/python/onl/install/BaseInstall.py b/packages/base/all/vendor-config-onl/src/python/onl/install/BaseInstall.py index bb60b0b7..5c7c25f7 100644 --- a/packages/base/all/vendor-config-onl/src/python/onl/install/BaseInstall.py +++ b/packages/base/all/vendor-config-onl/src/python/onl/install/BaseInstall.py @@ -84,6 +84,10 @@ class Base: self.log.error("not implemented") return 1 + def upgradeBootLoader(self): + self.log.error("not implemented") + return 1 + def shutdown(self): zf, self.zf = self.zf, None if zf: zf.close() @@ -511,19 +515,7 @@ class GrubInstaller(SubprocessMixin, Base): def installLoader(self): - ctx = {} - - kernel = self.im.platformConf['grub']['kernel'] - ctx['kernel'] = kernel['='] if type(kernel) == dict else kernel - ctx['args'] = self.im.platformConf['grub']['args'] - ctx['platform'] = self.im.installerConf.installer_platform - ctx['serial'] = self.im.platformConf['grub']['serial'] - - ctx['boot_menu_entry'] = sysconfig.installer.menu_name - ctx['boot_loading_name'] = sysconfig.installer.os_name - kernels = [] - for f in set(os.listdir(self.im.installerConf.installer_dir) + self.zf.namelist()): if 'kernel' in f: kernels.append(f) @@ -535,11 +527,10 @@ class GrubInstaller(SubprocessMixin, Base): initrd = i break - cf = GRUB_TPL % ctx - - self.log.info("Installing kernel") dev = self.blkidParts['ONL-BOOT'] + self.log.info("Installing kernel to %s", dev.device) + with MountContext(dev.device, log=self.log) as ctx: def _cp(b, dstname=None): if dstname is None: @@ -548,6 +539,29 @@ class GrubInstaller(SubprocessMixin, Base): self.installerCopy(b, dst, optional=True) [_cp(e) for e in kernels] _cp(initrd, "%s.cpio.gz" % self.im.installerConf.installer_platform) + + return 0 + + def installGrubCfg(self): + + dev = self.blkidParts['ONL-BOOT'] + + self.log.info("Installing grub.cfg to %s", dev.device) + + ctx = {} + + kernel = self.im.platformConf['grub']['kernel'] + ctx['kernel'] = kernel['='] if type(kernel) == dict else kernel + ctx['args'] = self.im.platformConf['grub']['args'] + ctx['platform'] = self.im.installerConf.installer_platform + ctx['serial'] = self.im.platformConf['grub']['serial'] + + ctx['boot_menu_entry'] = sysconfig.installer.menu_name + ctx['boot_loading_name'] = sysconfig.installer.os_name + + cf = GRUB_TPL % ctx + + with MountContext(dev.device, log=self.log) as ctx: d = os.path.join(ctx.dir, "grub") self.makedirs(d) dst = os.path.join(ctx.dir, 'grub/grub.cfg') @@ -611,6 +625,9 @@ class GrubInstaller(SubprocessMixin, Base): code = self.installLoader() if code: return code + code = self.installGrubCfg() + if code: return code + code = self.installBootConfig() if code: return code @@ -635,6 +652,17 @@ class GrubInstaller(SubprocessMixin, Base): return 1 return self.installGpt() + def upgradeBootLoader(self): + """Upgrade the boot loader settings.""" + + code = self.findGpt() + if code: return code + + code = self.installGrubCfg() + if code: return code + + return 0 + def shutdown(self): Base.shutdown(self) @@ -864,5 +892,24 @@ class UbootInstaller(SubprocessMixin, Base): return self.installUboot() + def upgradeBootLoader(self): + """Upgrade the boot loader settings as part of a loader upgrade.""" + + self.partedDevice = parted.getDevice(self.device) + self.partedDisk = parted.newDisk(self.partedDevice) + if self.partedDisk.type != 'msdos': + self.log.error("disk %s has wrong label %s", + self.device, self.partedDisk.type) + return 1 + + self.blkidParts = BlkidParser(log=self.log.getChild("blkid")) + + # XXX boot-config (and saved boot-config) should be unchanged during loader upgrade + + code = self.installUbootEnv() + if code: return code + + return 0 + def shutdown(self): Base.shutdown(self) From 3f0299027961a4d0d48c1eadb54f00acf4045ffe Mon Sep 17 00:00:00 2001 From: "Carl D. Roth" Date: Fri, 4 Nov 2016 12:36:18 -0700 Subject: [PATCH 02/12] Added fit extraction context object, add attach/detach support --- .../src/python/onl/install/InstallUtils.py | 84 +++++++++++++++++-- 1 file changed, 75 insertions(+), 9 deletions(-) diff --git a/packages/base/all/vendor-config-onl/src/python/onl/install/InstallUtils.py b/packages/base/all/vendor-config-onl/src/python/onl/install/InstallUtils.py index 830e042e..c30e307a 100644 --- a/packages/base/all/vendor-config-onl/src/python/onl/install/InstallUtils.py +++ b/packages/base/all/vendor-config-onl/src/python/onl/install/InstallUtils.py @@ -10,6 +10,8 @@ import tempfile import string import shutil +import Fit + class SubprocessMixin: V1 = "V1" @@ -210,8 +212,8 @@ class MountContext(SubprocessMixin): self.label = label self.fsType = fsType self.dir = None - self.hostDir = None - self.mounted = False + self.hostDir = self.__hostDir = None + self.mounted = self.__mounted = False self.log = log or logging.getLogger("mount") if self.device and self.label: @@ -245,7 +247,7 @@ class MountContext(SubprocessMixin): self.mounted = True return self - def __exit__(self, type, value, tb): + def shutdown(self): mounted = False if self.mounted: @@ -263,8 +265,18 @@ class MountContext(SubprocessMixin): if self.hostDir is not None: self.rmdir(self.hostDir) + def __exit__(self, type, value, tb): + self.shutdown() return False + def detach(self): + self.__mounted, self.mounted = self.mounted, False + self.__hostDir, self.hostDir = self.hostDir, None + + def attach(self): + self.mounted = self.__mounted + self.hostDir = self.__hostDir + class BlkidEntry: def __init__(self, device, **kwargs): @@ -665,6 +677,9 @@ class InitrdContext(SubprocessMixin): self.ilog.setLevel(logging.INFO) self.log = self.hlog + self.__initrd = None + self.__dir = None + def _unpack(self): self.dir = self.mkdtemp(prefix="chroot-", suffix=".d") @@ -783,7 +798,7 @@ class InitrdContext(SubprocessMixin): return self - def __exit__(self, type, value, tb): + def shutdown(self): p = ProcMountsParser() dirs = [e.dir for e in p.mounts if e.dir.startswith(self.dir)] @@ -797,23 +812,74 @@ class InitrdContext(SubprocessMixin): cmd = ('umount', p,) self.check_call(cmd, vmode=self.V1) - if self.initrd is not None: + if self.initrd and self.dir: self.log.debug("cleaning up chroot in %s", self.dir) self.rmtree(self.dir) - else: + elif self.dir: self.log.debug("saving chroot in %s", self.dir) + def __exit__(self, type, value, tb): + self.shutdown() return False + def detach(self): + self.__initrd, self.initrd = self.initrd, None + self.__dir, self.dir = self.dir, None + @classmethod - def mkChroot(self, initrd, log=None): - with InitrdContext(initrd=initrd, log=log) as ctx: + def mkChroot(cls, initrd, log=None): + with cls(initrd=initrd, log=log) as ctx: initrdDir = ctx.dir - ctx.initrd = None + ctx.detach() # save the unpacked directory, do not clean it up # (it's inside this chroot anyway) return initrdDir +class FitInitrdContext(SubprocessMixin): + + def __init__(self, path, log=None): + self.fitPath = path + self.log = log or logging.getLogger(self.__class__.__name__) + self.initrd = self.__initrd = None + + def __enter__(self): + self.log.debug("parsing FIT image in %s", self.fitPath) + p = Fit.Parser(path=self.fitPath, log=self.log) + node = p.getInitrdNode() + if node is None: + raise ValueError("cannot find initrd node in FDT") + prop = node.properties.get('data', None) + if prop is None: + raise ValueError("cannot find initrd data property in FDT") + + with open(device) as fd: + self.log.debug("reading initrd at [%x:%x]", + prop.offset, prop.offset+prop.sz) + fd.seek(prop.offset, 0) + buf = fd.read(prop.sz) + + fno, self.initrd = tempfile.mkstemp(prefix="initrd-", + suffix=".img") + self.log.debug("+ cat > %s", self.initrd) + with os.fdopen(fno, "w") as fd: + fd.write(buf) + return self + + def shutdown(self): + initrd, self.initrd = self.initrd, None + if initrd and os.path.exists(initrd): + self.unlink(initrd) + + def __exit__(self, eType, eValue, eTrace): + self.shutdown() + return False + + def detach(self): + self.__initrd, self.initrd = self.initrd, None + + def attach(self): + self.initrd = self.__initrd + class ChrootSubprocessMixin: chrootDir = None From cf1bfca4c05204d246eb192099638cbe090aed64 Mon Sep 17 00:00:00 2001 From: "Carl D. Roth" Date: Fri, 4 Nov 2016 12:36:55 -0700 Subject: [PATCH 03/12] Refactor to use fit extraction context object --- .../src/python/onl/install/ShellApp.py | 113 +++++++++++------- 1 file changed, 67 insertions(+), 46 deletions(-) diff --git a/packages/base/all/vendor-config-onl/src/python/onl/install/ShellApp.py b/packages/base/all/vendor-config-onl/src/python/onl/install/ShellApp.py index c0ff2fa2..0d85d7f3 100644 --- a/packages/base/all/vendor-config-onl/src/python/onl/install/ShellApp.py +++ b/packages/base/all/vendor-config-onl/src/python/onl/install/ShellApp.py @@ -13,7 +13,7 @@ from InstallUtils import InitrdContext, MountContext from InstallUtils import SubprocessMixin from InstallUtils import ProcMountsParser, ProcMtdParser from InstallUtils import BlkidParser -import Fit +from InstallUtils import FitInitrdContext import onl.platform.current @@ -46,30 +46,8 @@ class AppBase(SubprocessMixin): return 0 def _runFitShell(self, device): - self.log.debug("parsing FIT image in %s", device) - p = Fit.Parser(path=device, log=self.log) - node = p.getInitrdNode() - if node is None: - self.log.error("cannot find initrd node in FDT") - return 1 - prop = node.properties.get('data', None) - if prop is None: - self.log.error("cannot find initrd data property in FDT") - return 1 - with open(device) as fd: - self.log.debug("reading initrd at [%x:%x]", - prop.offset, prop.offset+prop.sz) - fd.seek(prop.offset, 0) - buf = fd.read(prop.sz) - try: - fno, initrd = tempfile.mkstemp(prefix="initrd-", - suffix=".img") - self.log.debug("+ cat > %s", initrd) - with os.fdopen(fno, "w") as fd: - fd.write(buf) - return self._runInitrdShell(initrd) - finally: - self.unlink(initrd) + with FitInitrdContext(path=device, log=self.log) as ctx: + return self._runInitrdShell(ctx.initrd) def shutdown(self): pass @@ -108,11 +86,22 @@ class AppBase(SubprocessMixin): app.shutdown() sys.exit(code) -class Onie(AppBase): +class OnieBootContext: + """XXX roth -- overlap with onl.install.ShellApp.Onie, + also with onl.install.App.OnieHelper from system-upgrade branch... - PROG = "onie-shell" + XXX roth -- refactor all three bits of code here + """ - def run(self): + def __init__(self, log=None): + self.log = log or logging.getLogger(self.__class__.__name__) + + self.initrd = None + + self.pm = self.blkid = self.mtd = None + self.ictx = self.mctx = self.fctx = None + + def __enter__(self): self.pm = ProcMountsParser() self.blkid = BlkidParser(log=self.log.getChild("blkid")) @@ -138,21 +127,25 @@ class Onie(AppBase): self.log.debug("found ONIE boot mounted at %s", onieDir) initrd = _g(onieDir) if initrd is None: - self.log.warn("cannot find ONIE initrd on %s", onieDir) - else: - self.log.debug("found ONIE initrd at %s", initrd) - return _runInitrdShell(initrd) + raise ValueError("cannot find ONIE initrd on %s" % onieDir) + self.log.debug("found ONIE initrd at %s", initrd) + with InitrdContext(initrd=initrd, log=self.log) as self.ictx: + self.initrd = initrd + self.ictx.detach() + return self - with MountContext(dev, log=self.log) as ctx: + with MountContext(dev, log=self.log) as self.mctx: initrd = _g(ctx.dir) if initrd is None: - self.log.warn("cannot find ONIE initrd on %s", dev) - else: - self.log.debug("found ONIE initrd at %s", initrd) - return self._runInitrdShell(initrd) + raise ValueError("cannot find ONIE initrd on %s" % dev) + self.log.debug("found ONIE initrd at %s", initrd) + with InitrdContext(initrd=initrd, log=self.log) as self.ictx: + self.initrd = initrd + self.mctx.detach() + self.ictx.detach() + return self - self.log.warn("cannot find an ONIE initrd") - return 1 + raise ValueError("cannot find an ONIE initrd") # try to find onie initrd on a mounted fs (GRUB); # for ONIE images this is usually /mnt/onie-boot @@ -164,7 +157,10 @@ class Onie(AppBase): part.device, part.dir) else: self.log.debug("found ONIE initrd at %s", initrd) - return self._runInitrdShell(initrd) + with InitrdContext(initrd=initrd, log=self.log) as self.ictx: + self.initrd = initrd + self.ictx.detach() + return self # grovel through MTD devices (u-boot) parts = [p for p in self.mtd.parts if p.label == "onie"] @@ -172,13 +168,38 @@ class Onie(AppBase): part = parts[0] self.log.debug("found ONIE MTD device %s", part.charDevice or part.blockDevice) - return self._runFitShell(part.blockDevice) - elif self.mtd.mounts: - self.log.error("cannot find ONIE MTD device") - return 1 + with FitInitrdContext(part.blockDevice, log=self.log) as self.fctx: + with InitrdContext(initrd=self.fctx.initrd, log=self.log) as self.ictx: + self.initrd = self.fctx.initrd + self.fctx.detach() + self.ictx.detach() + return self - self.log.error("cannot find ONIE initrd") - return 1 + if self.mtd.mounts: + raise ValueError("cannot find ONIE MTD device") + + raise ValueError("cannot find ONIE initrd") + + def shutdown(self): + ctx, self.fctx = self.fctx, None: + if ctx is not None: ctx.shutdown() + ctx, self.ictx = self.ictx, None: + if ctx is not None: ctx.shutdown() + ctx, self.mctx = self.mctx, None: + if ctx is not None: ctx.shutdown() + + def __exit__(self, eType, eValue, eTrace): + self.shutdown() + return False + +class Onie(AppBase): + """XXX roth -- refactor in from loader.py code.""" + + PROG = "onie-shell" + + def run(self): + with OnieBootContext(log=self.log) as ctx: + return self._runInitrdShell(ctx.initrd) class Loader(AppBase): From 8979e5ebbab05f9ced5dbf98eecb06d2937bc291 Mon Sep 17 00:00:00 2001 From: "Carl D. Roth" Date: Fri, 4 Nov 2016 12:37:05 -0700 Subject: [PATCH 04/12] Added fixmes --- .../all/vendor-config-onl/src/python/onl/upgrade/loader.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/base/all/vendor-config-onl/src/python/onl/upgrade/loader.py b/packages/base/all/vendor-config-onl/src/python/onl/upgrade/loader.py index 6637c9a7..b7ac149d 100755 --- a/packages/base/all/vendor-config-onl/src/python/onl/upgrade/loader.py +++ b/packages/base/all/vendor-config-onl/src/python/onl/upgrade/loader.py @@ -75,6 +75,8 @@ class LoaderUpgrade_Fit(LoaderUpgradeBase): with OnlMountContextReadWrite("ONL-BOOT", self.logger) as d: self.copyfile(fit_image, os.path.join(d.directory, "%s.itb" % (self.platform.platform()))) + # XXX re-install the firmware environment + self.reboot() @@ -110,6 +112,8 @@ class LoaderUpgrade_x86_64(LoaderUpgradeBase): #if os.path.exists(src): # self.copyfile(src, dst) + # XXX re-install the grub config + self.reboot() From c85967e4dbf8473198e9743c997053f2a257649d Mon Sep 17 00:00:00 2001 From: "Carl D. Roth" Date: Mon, 7 Nov 2016 10:50:17 -0800 Subject: [PATCH 05/12] Handle unmount when detached --- .../src/python/onl/install/InstallUtils.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/base/all/vendor-config-onl/src/python/onl/install/InstallUtils.py b/packages/base/all/vendor-config-onl/src/python/onl/install/InstallUtils.py index c30e307a..830abf00 100644 --- a/packages/base/all/vendor-config-onl/src/python/onl/install/InstallUtils.py +++ b/packages/base/all/vendor-config-onl/src/python/onl/install/InstallUtils.py @@ -801,16 +801,20 @@ class InitrdContext(SubprocessMixin): def shutdown(self): p = ProcMountsParser() - dirs = [e.dir for e in p.mounts if e.dir.startswith(self.dir)] + if self.dir is not None: + dirs = [e.dir for e in p.mounts if e.dir.startswith(self.dir)] + else: + dirs = [] # XXX probabaly also kill files here # umount any nested mounts - self.log.debug("un-mounting mounts points in chroot %s", self.dir) - dirs.sort(reverse=True) - for p in dirs: - cmd = ('umount', p,) - self.check_call(cmd, vmode=self.V1) + if dirs: + self.log.debug("un-mounting mounts points in chroot %s", self.dir) + dirs.sort(reverse=True) + for p in dirs: + cmd = ('umount', p,) + self.check_call(cmd, vmode=self.V1) if self.initrd and self.dir: self.log.debug("cleaning up chroot in %s", self.dir) From ad913e98c5c02965d166388c5d24636b8ab6ce52 Mon Sep 17 00:00:00 2001 From: "Carl D. Roth" Date: Mon, 7 Nov 2016 17:53:49 -0800 Subject: [PATCH 06/12] Minor fixes after fallout from git pull --- .../vendor-config-onl/src/python/onl/install/InstallUtils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/base/all/vendor-config-onl/src/python/onl/install/InstallUtils.py b/packages/base/all/vendor-config-onl/src/python/onl/install/InstallUtils.py index a071d3b2..a831d20b 100644 --- a/packages/base/all/vendor-config-onl/src/python/onl/install/InstallUtils.py +++ b/packages/base/all/vendor-config-onl/src/python/onl/install/InstallUtils.py @@ -698,6 +698,7 @@ class InitrdContext(SubprocessMixin): self.__initrd = None self.__dir = None + self._hasDevTmpfs = False def _unpack(self): self.dir = self.mkdtemp(prefix="chroot-", @@ -895,7 +896,7 @@ class FitInitrdContext(SubprocessMixin): if prop is None: raise ValueError("cannot find initrd data property in FDT") - with open(device) as fd: + with open(self.fitPath) as fd: self.log.debug("reading initrd at [%x:%x]", prop.offset, prop.offset+prop.sz) fd.seek(prop.offset, 0) From c730004e2bc087175346e79913b3ba573e463dab Mon Sep 17 00:00:00 2001 From: "Carl D. Roth" Date: Mon, 7 Nov 2016 17:54:36 -0800 Subject: [PATCH 07/12] Initial working version of OnieBootContext --- .../src/python/onl/install/ShellApp.py | 39 ++++++++++++------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/packages/base/all/vendor-config-onl/src/python/onl/install/ShellApp.py b/packages/base/all/vendor-config-onl/src/python/onl/install/ShellApp.py index 711c6d78..fe534512 100644 --- a/packages/base/all/vendor-config-onl/src/python/onl/install/ShellApp.py +++ b/packages/base/all/vendor-config-onl/src/python/onl/install/ShellApp.py @@ -88,11 +88,7 @@ class AppBase(SubprocessMixin, object): sys.exit(code) class OnieBootContext: - """XXX roth -- overlap with onl.install.ShellApp.Onie, - also with onl.install.App.OnieHelper from system-upgrade branch... - - XXX roth -- refactor all three bits of code here - """ + """Find the ONIE initrd and umpack/mount it.""" def __init__(self, log=None): self.log = log or logging.getLogger(self.__class__.__name__) @@ -100,15 +96,17 @@ class OnieBootContext: self.initrd = None self.pm = self.blkid = self.mtd = None - self.ictx = self.mctx = self.fctx = None + self.ictx = self.dctx = self.fctx = None + self.onieDir = None + self.initrdDir = None + + self.__ictx = self.__dctx = self.__fctx = None def __enter__(self): self.pm = ProcMountsParser() self.blkid = BlkidParser(log=self.log.getChild("blkid")) self.mtd = ProcMtdParser(log=self.log.getChild("mtd")) - self.dctx = None - self.onieDir = None def _g(d): pat = os.path.join(d, "onie/initrd.img*") @@ -133,18 +131,21 @@ class OnieBootContext: self.log.debug("found ONIE initrd at %s", initrd) with InitrdContext(initrd=initrd, log=self.log) as self.ictx: self.initrd = initrd + self.initrdDir = self.ictx.dir self.ictx.detach() return self # else, try to mount the directory containing the initrd - with MountContext(dev, log=self.log) as self.mctx: + with MountContext(dev, log=self.log) as self.dctx: initrd = _g(self.dctx.dir) if initrd is None: raise ValueError("cannot find ONIE initrd on %s" % dev) + self.onieDir = self.dctx.dir + self.dctx.detach() self.log.debug("found ONIE initrd at %s", initrd) with InitrdContext(initrd=initrd, log=self.log) as self.ictx: self.initrd = initrd - self.mctx.detach() + self.initrdDir = self.ictx.dir self.ictx.detach() return self @@ -163,6 +164,7 @@ class OnieBootContext: self.log.debug("found ONIE initrd at %s", initrd) with InitrdContext(initrd=initrd, log=self.log) as self.ictx: self.initrd = initrd + self.initrdDir = self.ictx.dir self.ictx.detach() return self @@ -176,6 +178,7 @@ class OnieBootContext: with InitrdContext(initrd=self.fctx.initrd, log=self.log) as self.ictx: self.initrd = self.fctx.initrd self.fctx.detach() + self.initrdDir = self.ictx.dir self.ictx.detach() return self @@ -185,17 +188,27 @@ class OnieBootContext: raise ValueError("cannot find ONIE initrd") def shutdown(self): - ctx, self.fctx = self.fctx, None: + ctx, self.fctx = self.fctx, None if ctx is not None: ctx.shutdown() - ctx, self.ictx = self.ictx, None: + ctx, self.ictx = self.ictx, None if ctx is not None: ctx.shutdown() - ctx, self.mctx = self.mctx, None: + ctx, self.dctx = self.dctx, None if ctx is not None: ctx.shutdown() def __exit__(self, eType, eValue, eTrace): self.shutdown() return False + def detach(self): + self.__fctx, self.fctx = self.fctx, None + self.__ictx, self.ictx = self.ictx, None + self.__dctx, self.dctx = self.dctx, None + + def attach(self): + self.fctx = self.__fctx + self.ictx = self.__ictx + self.dctx = self.__dctx + class Onie(AppBase): """XXX roth -- refactor in from loader.py code.""" From 9e1547906006a81963a70b819a567e102db7754d Mon Sep 17 00:00:00 2001 From: "Carl D. Roth" Date: Mon, 7 Nov 2016 17:54:50 -0800 Subject: [PATCH 08/12] Refactor to use OnieBootContext --- .../src/python/onl/install/App.py | 47 +++++-------------- 1 file changed, 12 insertions(+), 35 deletions(-) diff --git a/packages/base/all/vendor-config-onl/src/python/onl/install/App.py b/packages/base/all/vendor-config-onl/src/python/onl/install/App.py index 5ab00ed7..cd18cb97 100644 --- a/packages/base/all/vendor-config-onl/src/python/onl/install/App.py +++ b/packages/base/all/vendor-config-onl/src/python/onl/install/App.py @@ -17,31 +17,9 @@ import time from InstallUtils import InitrdContext from InstallUtils import SubprocessMixin from InstallUtils import ProcMountsParser -from ShellApp import Onie +from ShellApp import OnieBootContext import ConfUtils, BaseInstall -class OnieHelper(Onie): - """Unpack the initrd, but keep it around.""" - - UMOUNT = False - # leave self.onieDir mounted - - ictx = None - - def _runInitrdShell(self, initrd): - with InitrdContext(initrd, log=self.log) as self.ictx: - self.initrdDir = self.ictx.dir - self.ictx.detach() - - def shutdown(self): - - self.ictx.attach() - self.ictx.shutdown() - - if self.dctx is not None: - self.dctx.attach() - self.dctx.shutdown() - class App(SubprocessMixin, object): def __init__(self, url=None, @@ -66,7 +44,7 @@ class App(SubprocessMixin, object): self.nextUpdate = None - self.onieHelper = None + self.octx = None def run(self): @@ -160,15 +138,13 @@ class App(SubprocessMixin, object): ##self.log.info("using native GRUB") ##self.grubEnv = ConfUtils.GrubEnv(log=self.log.getChild("grub")) - self.onieHelper = OnieHelper(log=self.log) - code = self.onieHelper.run() - if code: - self.log.warn("cannot find ONIE initrd") + with OnieBootContext(log=self.log) as self.octx: + self.octx.detach() - if self.onieHelper.onieDir is not None: - self.log.info("using native ONIE initrd+chroot GRUB (%s)", self.onieHelper.onieDir) - self.grubEnv = ConfUtils.ChrootGrubEnv(self.onieHelper.initrdDir, - bootDir=self.onieHelper.onieDir, + if self.octx.onieDir is not None: + self.log.info("using native ONIE initrd+chroot GRUB (%s)", self.octx.onieDir) + self.grubEnv = ConfUtils.ChrootGrubEnv(self.octx.initrdDir, + bootDir=self.octx.onieDir, path="/grub/grubenv", log=self.log.getChild("grub")) # direct access using ONIE initrd as a chroot @@ -340,9 +316,10 @@ class App(SubprocessMixin, object): if installer is not None: installer.shutdown() - h, self.onieHelper = self.onieHelper, None - if h is not None: - h.shutdown() + ctx, self.octx = self.octx, None + if ctx: + ctx.attach() + ctx.shutdown() def post_mortem(self): self.log.info("re-attaching to tty") From 1eaf262f71a05363c4ce5213c02340cfb8a17b94 Mon Sep 17 00:00:00 2001 From: "Carl D. Roth" Date: Mon, 7 Nov 2016 17:55:06 -0800 Subject: [PATCH 09/12] Refactor to use OnieBootContext --- .../src/python/onl/install/SystemInstall.py | 31 ++++++------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/packages/base/all/vendor-config-onl/src/python/onl/install/SystemInstall.py b/packages/base/all/vendor-config-onl/src/python/onl/install/SystemInstall.py index b3339550..59d9ad31 100644 --- a/packages/base/all/vendor-config-onl/src/python/onl/install/SystemInstall.py +++ b/packages/base/all/vendor-config-onl/src/python/onl/install/SystemInstall.py @@ -16,7 +16,7 @@ import subprocess from onl.install.InstallUtils import InitrdContext from onl.install.InstallUtils import ProcMountsParser from onl.install.ConfUtils import MachineConf, InstallerConf -from onl.install.ShellApp import Onie, Upgrader +from onl.install.ShellApp import OnieBootContext, Upgrader from onl.install.InstallUtils import SubprocessMixin import onl.install.App @@ -45,8 +45,6 @@ class App(SubprocessMixin): self.force = force - self.onieHelper = None - def _runInitrd(self, helper, path): with InitrdContext(initrd=path, log=self.log) as ctx: @@ -55,22 +53,14 @@ class App(SubprocessMixin): prefix="installer-", suffix=".d") chroot_idir = abs_idir[len(ctx.dir):] - self.onieHelper = onl.install.App.OnieHelper(log=self.log) - code = self.onieHelper.run() - if code: - self.log.error("cannot find/unpack ONIE initrd") - return code - self.log.info("onie directory is %s", self.onieHelper.onieDir) - self.log.info("initrd directory is %s", self.onieHelper.initrdDir) + with OnieBootContext(log=self.log) as octx: + self.log.info("onie directory is %s", octx.onieDir) + self.log.info("initrd directory is %s", octx.initrdDir) - src = os.path.join(self.onieHelper.initrdDir, "etc/machine.conf") - dst = os.path.join(ctx.dir, "etc/machine.conf") - self.log.debug("+ /bin/cp %s %s", src, dst) - shutil.copy2(src, dst) - - h, self.onieHelper = self.onieHelper, None - if h is not None: - h.shutdown() + src = os.path.join(octx.initrdDir, "etc/machine.conf") + dst = os.path.join(ctx.dir, "etc/machine.conf") + self.log.debug("+ /bin/cp %s %s", src, dst) + shutil.copy2(src, dst) src = "/etc/fw_env.config" if os.path.exists(src): @@ -214,10 +204,7 @@ class App(SubprocessMixin): return code def shutdown(self): - - h, self.onieHelper = self.onieHelper, None - if h is not None: - h.shutdown() + pass @classmethod def main(cls): From 4c9c1fa7451ff70e887a9627f60d939064b0d051 Mon Sep 17 00:00:00 2001 From: "Carl D. Roth" Date: Tue, 8 Nov 2016 16:21:58 -0800 Subject: [PATCH 10/12] Works for ppc, x86 probably needs a chroot --- .../src/python/onl/upgrade/loader.py | 88 ++++++++++++++++++- 1 file changed, 85 insertions(+), 3 deletions(-) diff --git a/packages/base/all/vendor-config-onl/src/python/onl/upgrade/loader.py b/packages/base/all/vendor-config-onl/src/python/onl/upgrade/loader.py index b7ac149d..101e8f39 100755 --- a/packages/base/all/vendor-config-onl/src/python/onl/upgrade/loader.py +++ b/packages/base/all/vendor-config-onl/src/python/onl/upgrade/loader.py @@ -11,6 +11,10 @@ import fnmatch from onl.upgrade import ubase from onl.sysconfig import sysconfig from onl.mounts import OnlMountManager, OnlMountContextReadOnly, OnlMountContextReadWrite +from onl.install import BaseInstall, ConfUtils +from onl.install.ShellApp import OnieBootContext +import onl.platform.current +import onl.versions class LoaderUpgradeBase(ubase.BaseUpgrade): name="loader" @@ -57,9 +61,10 @@ class LoaderUpgradeBase(ubase.BaseUpgrade): * A single reboot will be required to complete this upgrade. """ - class LoaderUpgrade_Fit(LoaderUpgradeBase): + installer_klass = BaseInstall.UbootInstaller + def do_upgrade(self, forced=False): fit_image = None @@ -75,13 +80,50 @@ class LoaderUpgrade_Fit(LoaderUpgradeBase): with OnlMountContextReadWrite("ONL-BOOT", self.logger) as d: self.copyfile(fit_image, os.path.join(d.directory, "%s.itb" % (self.platform.platform()))) - # XXX re-install the firmware environment + onlPlatform = onl.platform.current.OnlPlatform() + + with OnieBootContext(log=self.logger) as octx: + path = os.path.join(octx.initrdDir, "etc/machine.conf") + machineConf = ConfUtils.MachineConf(path=path) + + installerConf = ConfUtils.InstallerConf(path="/dev/null") + # start with an empty installerConf, fill it in piece by piece + + installerConf.installer_platform = onlPlatform.platform() + installerConf.installer_arch = machineConf.onie_arch + installerConf.installer_platform_dir = os.path.join("/lib/platform-config", + onlPlatform.platform()) + + mfPath = os.path.join(sysconfig.upgrade.loader.package.dir, "manifest.json") + mf = onl.versions.OnlVersionManifest(mfPath) + installerConf.onl_version = mf.RELEASE_ID + + grubEnv = ConfUtils.ProxyGrubEnv(installerConf, + bootDir="/mnt/onie-boot", + path="/grub/grubenv", + chroot=False, + log=self.logger.getChild("grub")) + + ubootEnv = ConfUtils.UbootEnv(log=self.logger.getChild("u-boot")) + + installer = self.installer_klass(machineConf=machineConf, + installerConf=installerConf, + platformConf=onlPlatform.platform_config, + grubEnv=grubEnv, + ubootEnv=ubootEnv, + force=True, + log=self.logger) + + installer.upgradeBootLoader() + installer.shutdown() self.reboot() class LoaderUpgrade_x86_64(LoaderUpgradeBase): + installer_klass = BaseInstall.GrubInstaller + def do_upgrade(self, forced=False): X86_64_UPGRADE_DIR=sysconfig.upgrade.loader.package.dir @@ -112,7 +154,47 @@ class LoaderUpgrade_x86_64(LoaderUpgradeBase): #if os.path.exists(src): # self.copyfile(src, dst) - # XXX re-install the grub config + # XXX re-install the firmware environment + import pdb + pdb.set_trace() + + onlPlatform = onl.platform.current.OnlPlatform() + + with OnieBootContext(log=self.logger) as octx: + path = os.path.join(octx.initrdDir, "etc/machine.conf") + machineConf = ConfUtils.MachineConf(path=path) + + # hold on to the ONIE boot context for grub access + + installerConf = ConfUtils.InstallerConf(path="/dev/null") + + # XXX fill in installerConf fields + installerConf.installer_platform = onlPlatform.platform() + installerConf.installer_arch = machineConf.onie_arch + installerConf.installer_platform_dir = os.path.join("/lib/platform-config", + onlPlatform.platform()) + + mfPath = os.path.join(sysconfig.upgrade.loader.package.dir, "manifest.json") + mf = onl.versions.OnlVersionManifest(mfPath) + installerConf.onl_version = mf.RELEASE_ID + + grubEnv = ConfUtils.ChrootGrubEnv(octx.initrdDir, + bootDir=octx.onieDir, + path="/grub/grubenv", + log=self.logger.getChild("grub")) + + ubootEnv = None + + installer = self.installer_klass(machineConf=machineConf, + installerConf=installerConf, + platformConf=onlPlatform.platform_config, + grubEnv=grubEnv, + ubootEnv=ubootEnv, + force=True, + log=self.logger) + + installer.upgradeBootLoader() + installer.shutdown() self.reboot() From 46c8c0a0be39d2b9e9a8d00e0b98de7ceddd3a70 Mon Sep 17 00:00:00 2001 From: "Carl D. Roth" Date: Wed, 9 Nov 2016 12:38:18 -0800 Subject: [PATCH 11/12] Working for x86 --- .../src/python/onl/install/BaseInstall.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/packages/base/all/vendor-config-onl/src/python/onl/install/BaseInstall.py b/packages/base/all/vendor-config-onl/src/python/onl/install/BaseInstall.py index 196d6b52..922f781c 100644 --- a/packages/base/all/vendor-config-onl/src/python/onl/install/BaseInstall.py +++ b/packages/base/all/vendor-config-onl/src/python/onl/install/BaseInstall.py @@ -571,7 +571,8 @@ class GrubInstaller(SubprocessMixin, Base): with MountContext(dev.device, log=self.log) as ctx: d = os.path.join(ctx.dir, "grub") - self.makedirs(d) + if not os.path.exists(d): + self.makedirs(d) dst = os.path.join(ctx.dir, 'grub/grub.cfg') with open(dst, "w") as fd: fd.write(cf) @@ -663,8 +664,7 @@ class GrubInstaller(SubprocessMixin, Base): def upgradeBootLoader(self): """Upgrade the boot loader settings.""" - code = self.findGpt() - if code: return code + self.blkidParts = BlkidParser(log=self.log.getChild("blkid")) code = self.installGrubCfg() if code: return code @@ -909,13 +909,6 @@ class UbootInstaller(SubprocessMixin, Base): def upgradeBootLoader(self): """Upgrade the boot loader settings as part of a loader upgrade.""" - self.partedDevice = parted.getDevice(self.device) - self.partedDisk = parted.newDisk(self.partedDevice) - if self.partedDisk.type != 'msdos': - self.log.error("disk %s has wrong label %s", - self.device, self.partedDisk.type) - return 1 - self.blkidParts = BlkidParser(log=self.log.getChild("blkid")) # XXX boot-config (and saved boot-config) should be unchanged during loader upgrade From a394f2b84c504d37698c020d1e7385c0dafdef0a Mon Sep 17 00:00:00 2001 From: "Carl D. Roth" Date: Wed, 9 Nov 2016 12:38:48 -0800 Subject: [PATCH 12/12] Unmount /mnt/onl/boot prior to invoking the installer --- .../src/python/onl/upgrade/loader.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/base/all/vendor-config-onl/src/python/onl/upgrade/loader.py b/packages/base/all/vendor-config-onl/src/python/onl/upgrade/loader.py index 101e8f39..29128db9 100755 --- a/packages/base/all/vendor-config-onl/src/python/onl/upgrade/loader.py +++ b/packages/base/all/vendor-config-onl/src/python/onl/upgrade/loader.py @@ -11,7 +11,7 @@ import fnmatch from onl.upgrade import ubase from onl.sysconfig import sysconfig from onl.mounts import OnlMountManager, OnlMountContextReadOnly, OnlMountContextReadWrite -from onl.install import BaseInstall, ConfUtils +from onl.install import BaseInstall, ConfUtils, InstallUtils from onl.install.ShellApp import OnieBootContext import onl.platform.current import onl.versions @@ -120,7 +120,7 @@ class LoaderUpgrade_Fit(LoaderUpgradeBase): self.reboot() -class LoaderUpgrade_x86_64(LoaderUpgradeBase): +class LoaderUpgrade_x86_64(LoaderUpgradeBase, InstallUtils.SubprocessMixin): installer_klass = BaseInstall.GrubInstaller @@ -154,9 +154,16 @@ class LoaderUpgrade_x86_64(LoaderUpgradeBase): #if os.path.exists(src): # self.copyfile(src, dst) - # XXX re-install the firmware environment - import pdb - pdb.set_trace() + # installer assumes that partitions are unmounted + + self.log = self.logger + # ha ha, SubprocessMixin api is different + + pm = InstallUtils.ProcMountsParser() + for m in pm.mounts: + if m.dir.startswith('/mnt/onl'): + self.logger.warn("unmounting %s (--force)", m.dir) + self.check_call(('umount', m.dir,)) onlPlatform = onl.platform.current.OnlPlatform()