diff --git a/packages/base/all/vendor-config-onl/src/python/onl/install/App.py b/packages/base/all/vendor-config-onl/src/python/onl/install/App.py index 55630010..65968f4c 100644 --- a/packages/base/all/vendor-config-onl/src/python/onl/install/App.py +++ b/packages/base/all/vendor-config-onl/src/python/onl/install/App.py @@ -8,28 +8,110 @@ import sys, os import logging import imp import glob -import distutils.sysconfig +import argparse +import shutil +import urllib +import tempfile +import time from InstallUtils import InitrdContext from InstallUtils import SubprocessMixin +from InstallUtils import ProcMountsParser import ConfUtils, BaseInstall class App(SubprocessMixin): - def __init__(self, log=None): + def __init__(self, url=None, force=False, log=None): if log is not None: self.log = log else: self.log = logging.getLogger(self.__class__.__name__) + self.url = url + self.force = force + # remote-install mode + self.installer = None self.machineConf = None self.installerConf = None self.onlPlatform = None + # local-install mode + + self.nextUpdate = None def run(self): + if self.url is not None: + return self.runUrl() + else: + return self.runLocal() + + def runUrl(self): + pm = ProcMountsParser() + for m in pm.mounts: + if m.dir.startswith('/mnt/onl'): + if not self.force: + self.log.error("directory %s is still mounted", m.dir) + return 1 + self.log.warn("unmounting %s (--force)", m.dir) + self.check_call(('umount', m.dir,)) + + def reporthook(blocks, bsz, sz): + if time.time() < self.nextUpdate: return + self.nextUpdate = time.time() + 0.25 + if sz: + pct = blocks * bsz * 100 / sz + sys.stderr.write("downloaded %d%% ...\r" % pct) + else: + icon = "|/-\\"[blocks % 4] + sys.stderr.write("downloading ... %s\r" % icon) + + p = tempfile.mktemp(prefix="installer-", + suffix=".bin") + try: + self.log.info("downloading installer from %s --> %s", + self.url, p) + self.nextUpdate = 0 + if os.isatty(sys.stdout.fileno()): + dst, headers = urllib.urlretrieve(self.url, p, reporthook) + else: + dst, headers = urllib.urlretrieve(self.url, p) + sys.stdout.write("\n") + + self.log.debug("+ chmod +x %s", p) + os.chmod(p, 0755) + + env = {} + env.update(os.environ) + + if os.path.exists("/etc/onl/platform"): + self.log.debug("enabling unzip features for ONL") + env['SFX_UNZIP'] = '1' + self.log.debug("+ export SFX_UNZIP=1") + env['SFX_LOOP'] = '1' + self.log.debug("+ export SFX_LOOP=1") + env['SFX_PIPE'] = '1' + self.log.debug("+ export SFX_PIPE=1") + + self.log.debug("enabling in-place fixups") + env['SFX_INPLACE'] = '1' + self.log.debug("+ export SFX_INPLACE=1") + + self.log.info("invoking installer...") + try: + self.check_call((p,), env=env) + except subprocess.CalledProcessError as ex: + self.log.error("installer failed") + return ex.returncode + finally: + os.unlink(p) + + self.log.info("please reboot this system now.") + return 0 + + def runLocal(self): + self.log.info("getting installer configuration") if os.path.exists(ConfUtils.MachineConf.PATH): self.machineConf = ConfUtils.MachineConf() @@ -100,6 +182,7 @@ class App(SubprocessMixin): platformConf=self.onlPlatform.platform_config, grubEnv=self.grubEnv, ubootEnv=self.ubootEnv, + force=self.force, log=self.log) try: code = self.installer.run() @@ -230,17 +313,33 @@ class App(SubprocessMixin): logger.addHandler(hnd) logger.propagate = False - debug = 'installer_debug' in os.environ - if debug: + onie_verbose = 'onie_verbose' in os.environ + installer_debug = 'installer_debug' in os.environ + + ap = argparse.ArgumentParser() + ap.add_argument('-v', '--verbose', action='store_true', + default=onie_verbose, + help="Enable verbose logging") + ap.add_argument('-D', '--debug', action='store_true', + default=installer_debug, + help="Enable python debugging") + ap.add_argument('-U', '--url', type=str, + help="Install from a remote URL") + ap.add_argument('-F', '--force', action='store_true', + help="Unmount filesystems before install") + ops = ap.parse_args() + + if ops.verbose: logger.setLevel(logging.DEBUG) - app = cls(log=logger) + app = cls(url=ops.url, force=ops.force, + log=logger) try: code = app.run() except: logger.exception("runner failed") code = 1 - if debug: + if ops.debug: app.post_mortem() app.shutdown() diff --git a/packages/base/all/vendor-config-onl/src/python/onl/install/BaseInstall.py b/packages/base/all/vendor-config-onl/src/python/onl/install/BaseInstall.py index 895a0876..da4c62ab 100644 --- a/packages/base/all/vendor-config-onl/src/python/onl/install/BaseInstall.py +++ b/packages/base/all/vendor-config-onl/src/python/onl/install/BaseInstall.py @@ -11,6 +11,8 @@ import logging import StringIO import parted import yaml +import zipfile +import shutil from InstallUtils import SubprocessMixin from InstallUtils import MountContext, BlkidParser, PartedParser @@ -44,6 +46,7 @@ class Base: def __init__(self, machineConf=None, installerConf=None, platformConf=None, grubEnv=None, ubootEnv=None, + force=False, log=None): self.im = self.installmeta(installerConf=installerConf, machineConf=machineConf, @@ -52,6 +55,9 @@ class Base: ubootEnv = ubootEnv) self.log = log or logging.getLogger(self.__class__.__name__) + self.force = False + # unmount filesystems as needed + self.device = None # target device, initialize this later @@ -69,16 +75,65 @@ class Base: self.configArchive = None # backup of ONL-CONFIG during re-partitioning + self.zf = None + # zipfile handle to installer archive + def run(self): self.log.error("not implemented") return 1 def shutdown(self): - pass + zf, self.zf = self.zf, None + if zf: zf.close() + + def installerCopy(self, basename, dst, optional=False): + """Copy the file as-is, or get it from the installer zip.""" + + src = os.path.join(self.im.installerConf.installer_dir, basename) + if os.path.exists(src): + self.copy2(src, dst) + return + + if basename in self.zf.namelist(): + self.log.debug("+ unzip -p %s %s > %s", + self.im.installerConf.installer_zip, basename, dst) + with self.zf.open(basename, "r") as rfd: + with open(dst, "wb") as wfd: + shutil.copyfileobj(rfd, wfd) + return + + if not optional: + raise ValueError("missing installer file %s" % basename) + + def installerDd(self, basename, device): + + p = os.path.join(self.im.installerConf.installer_dir, basename) + if os.path.exists(p): + cmd = ('dd', + 'if=' + basename, + 'of=' + device,) + self.check_call(cmd, vmode=self.V2) + return + + if basename in self.zf.namelist(): + self.log.debug("+ unzip -p %s %s | dd of=%s", + self.im.installerConf.installer_zip, basename, device) + with self.zf.open(basename, "r") as rfd: + with open(device, "rb+") as wfd: + shutil.copyfileobj(rfd, wfd) + return + + raise ValueError("cannot find file %s" % basename) + + def installerExists(self, basename): + if basename in os.listdir(self.im.installerConf.installer_dir): return True + if basename in self.zf.namelist(): return True + return False def installSwi(self): - swis = [x for x in os.listdir(self.im.installerConf.installer_dir) if x.endswith('.swi')] + files = os.listdir(self.im.installerConf.installer_dir) + self.zf.namelist() + swis = [x for x in files if x.endswith('.swi')] if not swis: self.log.info("No ONL Software Image available for installation.") self.log.info("Post-install ZTN installation will be required.") @@ -88,13 +143,12 @@ class Base: return base = swis[0] - src = os.path.join(self.im.installerConf.installer_dir, base) self.log.info("Installing ONL Software Image (%s)...", base) dev = self.blkidParts['ONL-IMAGES'] with MountContext(dev.device, log=self.log) as ctx: dst = os.path.join(ctx.dir, base) - self.copy2(src, dst) + self.installerCopy(base, dst) return 0 @@ -268,18 +322,18 @@ class Base: self.log.info("Installing boot-config to %s", dev.device) - src = os.path.join(self.im.installerConf.installer_dir, 'boot-config') - ##src = os.path.join(self.im.installerConf.installer_platform_dir, 'boot-config') + basename = 'boot-config' with MountContext(dev.device, log=self.log) as ctx: - dst = os.path.join(ctx.dir, 'boot-config') - self.copy2(src, dst) + dst = os.path.join(ctx.dir, basename) + self.installerCopy(basename, dst) + with open(dst) as fd: + buf = fd.read() - with open(src) as fd: - ecf = fd.read().encode('base64', 'strict').strip() - if self.im.grub and self.im.grubEnv is not None: - setattr(self.im.grubEnv, 'boot_config_default', ecf) - if self.im.uboot and self.im.ubootEnv is not None: - setattr(self.im.ubootEnv, 'boot-config-default', ecf) + ecf = buf.encode('base64', 'strict').strip() + if self.im.grub and self.im.grubEnv is not None: + setattr(self.im.grubEnv, 'boot_config_default', ecf) + if self.im.uboot and self.im.ubootEnv is not None: + setattr(self.im.ubootEnv, 'boot-config-default', ecf) return 0 @@ -288,9 +342,19 @@ class Base: pm = ProcMountsParser() for m in pm.mounts: if m.device.startswith(self.device): - self.log.error("mount %s on %s will be erased by install", - m.dir, m.device) - return 1 + if not self.force: + self.log.error("mount %s on %s will be erased by install", + m.dir, m.device) + return 1 + else: + self.log.warn("unmounting %s from %s (--force)", + m.dir, m.device) + try: + self.check_call(('umount', m.dir,)) + except subprocess.CalledProcessError: + self.log.error("cannot unmount") + return 1 + return 0 GRUB_TPL = """\ @@ -426,14 +490,15 @@ class GrubInstaller(SubprocessMixin, Base): self.log.info("Installing kernel") dev = self.blkidParts['ONL-BOOT'] + + files = set(os.listdir(self.im.installerConf.installer_dir) + self.zf.namelist()) + files = [b for b in files if b.startswith('kernel-') or b.startswith('onl-loader-initrd-')] + with MountContext(dev.device, log=self.log) as ctx: def _cp(b): - src = os.path.join(self.im.installerConf.installer_dir, b) - if not os.path.isfile(src): return - if b.startswith('kernel-') or b.startswith('onl-loader-initrd-'): - dst = os.path.join(ctx.dir, b) - self.copy2(src, dst) - [_cp(e) for e in os.listdir(self.im.installerConf.installer_dir)] + dst = os.path.join(ctx.dir, b) + self.installerCopy(b, dst, optional=True) + [_cp(e) for e in files] d = os.path.join(ctx.dir, "grub") self.makedirs(d) @@ -485,6 +550,11 @@ class GrubInstaller(SubprocessMixin, Base): self.im.grubEnv.__dict__['bootPart'] = dev.device self.im.grubEnv.__dict__['bootDir'] = None + # get a handle to the installer zip + p = os.path.join(self.im.installerConf.installer_dir, + self.im.installerConf.installer_zip) + self.zf = zipfile.ZipFile(p) + code = self.installSwi() if code: return code @@ -513,7 +583,7 @@ class GrubInstaller(SubprocessMixin, Base): return self.installGpt() def shutdown(self): - pass + Base.shutdown(self) class UbootInstaller(SubprocessMixin, Base): @@ -598,42 +668,37 @@ class UbootInstaller(SubprocessMixin, Base): def installLoader(self): - loaderSrc = None c1 = self.im.platformConf['flat_image_tree'].get('itb', None) if type(c1) == dict: c1 = c1.get('=', None) c2 = ("%s.itb" % (self.im.installerConf.installer_platform,)) c3 = "onl-loader-fit.itb" - loaderSrc = None + loaderBasename = None for c in (c1, c2, c3): if c is None: continue - p = os.path.join(self.im.installerConf.installer_dir, c) - if os.path.exists(p): - loaderSrc = p + if self.installerExists(c): + loaderBasename = c break - if not loaderSrc: + if not loaderBasename: self.log.error("The platform loader file is missing.") return 1 - self.log.info("Installing the ONL loader from %s...", loaderSrc) + self.log.info("Installing the ONL loader from %s...", loaderBasename) if self.rawLoaderDevice is not None: self.log.info("Installing ONL loader %s --> %s...", - loaderSrc, self.rawLoaderDevice) - cmd = ('dd', - 'if=' + loaderSrc, - 'of=' + self.rawLoaderDevice,) - self.check_call(cmd, vmode=self.V2) - else: - dev = self.blkidParts['ONL-BOOT'] - basename = os.path.split(loaderSrc)[1] - self.log.info("Installing ONL loader %s --> %s:%s...", - loaderSrc, dev.device, basename) - with MountContext(dev.device, log=self.log) as ctx: - dst = os.path.join(ctx.dir, basename) - self.copy2(loaderSrc, dst) + loaderBasename, self.rawLoaderDevice) + self.installerDd(loaderBasename, self.rawLoaderDevice) + return 0 + + dev = self.blkidParts['ONL-BOOT'] + self.log.info("Installing ONL loader %s --> %s:%s...", + loaderBasename, dev.device, loaderBasename) + with MountContext(dev.device, log=self.log) as ctx: + dst = os.path.join(ctx.dir, loaderBasename) + self.installerCopy(loaderBasename, dst) return 0 @@ -713,6 +778,11 @@ class UbootInstaller(SubprocessMixin, Base): self.rawLoaderDevice = self.device + str(partIdx+1) break + # get a handle to the installer zip + p = os.path.join(self.im.installerConf.installer_dir, + self.im.installerConf.installer_zip) + self.zf = zipfile.ZipFile(p) + code = self.installSwi() if code: return code @@ -744,4 +814,4 @@ class UbootInstaller(SubprocessMixin, Base): return self.installUboot() def shutdown(self): - pass + Base.shutdown(self)