Installer updates

- initial support for self-updating within ONL via a URL
- lazy unpack support

packages/base/all/vendor-config-onl/src/python/onl/install/App.py

@@ -8,28 +8,110 @@ import sys, os
 import logging
 import imp
 import glob
-import distutils.sysconfig
+import argparse
+import shutil
+import urllib
+import tempfile
+import time
 
 from InstallUtils import InitrdContext
 from InstallUtils import SubprocessMixin
+from InstallUtils import ProcMountsParser
 import ConfUtils, BaseInstall
 
 class App(SubprocessMixin):
 
-    def __init__(self, log=None):
+    def __init__(self, url=None, force=False, log=None):
 
         if log is not None:
             self.log = log
         else:
             self.log = logging.getLogger(self.__class__.__name__)
 
+        self.url = url
+        self.force = force
+        # remote-install mode
+
         self.installer = None
         self.machineConf = None
         self.installerConf = None
         self.onlPlatform = None
+        # local-install mode
+
+        self.nextUpdate = None
 
     def run(self):
 
+        if self.url is not None:
+            return self.runUrl()
+        else:
+            return self.runLocal()
+
+    def runUrl(self):
+        pm = ProcMountsParser()
+        for m in pm.mounts:
+            if m.dir.startswith('/mnt/onl'):
+                if not self.force:
+                    self.log.error("directory %s is still mounted", m.dir)
+                    return 1
+                self.log.warn("unmounting %s (--force)", m.dir)
+                self.check_call(('umount', m.dir,))
+
+        def reporthook(blocks, bsz, sz):
+            if time.time() < self.nextUpdate: return
+            self.nextUpdate = time.time() + 0.25
+            if sz:
+                pct = blocks * bsz * 100 / sz
+                sys.stderr.write("downloaded %d%% ...\r" % pct)
+            else:
+                icon = "|/-\\"[blocks % 4]
+                sys.stderr.write("downloading ... %s\r" % icon)
+
+        p = tempfile.mktemp(prefix="installer-",
+                            suffix=".bin")
+        try:
+            self.log.info("downloading installer from %s --> %s",
+                          self.url, p)
+            self.nextUpdate = 0
+            if os.isatty(sys.stdout.fileno()):
+                dst, headers = urllib.urlretrieve(self.url, p, reporthook)
+            else:
+                dst, headers = urllib.urlretrieve(self.url, p)
+            sys.stdout.write("\n")
+
+            self.log.debug("+ chmod +x %s", p)
+            os.chmod(p, 0755)
+
+            env = {}
+            env.update(os.environ)
+
+            if os.path.exists("/etc/onl/platform"):
+                self.log.debug("enabling unzip features for ONL")
+                env['SFX_UNZIP'] = '1'
+                self.log.debug("+ export SFX_UNZIP=1")
+                env['SFX_LOOP'] = '1'
+                self.log.debug("+ export SFX_LOOP=1")
+                env['SFX_PIPE'] = '1'
+                self.log.debug("+ export SFX_PIPE=1")
+
+            self.log.debug("enabling in-place fixups")
+            env['SFX_INPLACE'] = '1'
+            self.log.debug("+ export SFX_INPLACE=1")
+
+            self.log.info("invoking installer...")
+            try:
+                self.check_call((p,), env=env)
+            except subprocess.CalledProcessError as ex:
+                self.log.error("installer failed")
+                return ex.returncode
+        finally:
+            os.unlink(p)
+
+        self.log.info("please reboot this system now.")
+        return 0
+
+    def runLocal(self):
+
         self.log.info("getting installer configuration")
         if os.path.exists(ConfUtils.MachineConf.PATH):
             self.machineConf = ConfUtils.MachineConf()
@@ -100,6 +182,7 @@ class App(SubprocessMixin):
                                 platformConf=self.onlPlatform.platform_config,
                                 grubEnv=self.grubEnv,
                                 ubootEnv=self.ubootEnv,
+                                force=self.force,
                                 log=self.log)
         try:
             code = self.installer.run()
@@ -230,17 +313,33 @@ class App(SubprocessMixin):
         logger.addHandler(hnd)
         logger.propagate = False
 
-        debug = 'installer_debug' in os.environ
-        if debug:
+        onie_verbose = 'onie_verbose' in os.environ
+        installer_debug = 'installer_debug' in os.environ
+
+        ap = argparse.ArgumentParser()
+        ap.add_argument('-v', '--verbose', action='store_true',
+                        default=onie_verbose,
+                        help="Enable verbose logging")
+        ap.add_argument('-D', '--debug', action='store_true',
+                        default=installer_debug,
+                        help="Enable python debugging")
+        ap.add_argument('-U', '--url', type=str,
+                        help="Install from a remote URL")
+        ap.add_argument('-F', '--force', action='store_true',
+                        help="Unmount filesystems before install")
+        ops = ap.parse_args()
+
+        if ops.verbose:
             logger.setLevel(logging.DEBUG)
 
-        app = cls(log=logger)
+        app = cls(url=ops.url, force=ops.force,
+                  log=logger)
         try:
             code = app.run()
         except:
             logger.exception("runner failed")
             code = 1
-            if debug:
+            if ops.debug:
                 app.post_mortem()
 
         app.shutdown()

packages/base/all/vendor-config-onl/src/python/onl/install/BaseInstall.py

@@ -11,6 +11,8 @@ import logging
 import StringIO
 import parted
 import yaml
+import zipfile
+import shutil
 
 from InstallUtils import SubprocessMixin
 from InstallUtils import MountContext, BlkidParser, PartedParser
@@ -44,6 +46,7 @@ class Base:
     def __init__(self,
                  machineConf=None, installerConf=None, platformConf=None,
                  grubEnv=None, ubootEnv=None,
+                 force=False,
                  log=None):
         self.im = self.installmeta(installerConf=installerConf,
                                    machineConf=machineConf,
@@ -52,6 +55,9 @@ class Base:
                                    ubootEnv = ubootEnv)
         self.log = log or logging.getLogger(self.__class__.__name__)
 
+        self.force = False
+        # unmount filesystems as needed
+
         self.device = None
         # target device, initialize this later
 
@@ -69,16 +75,65 @@ class Base:
         self.configArchive = None
         # backup of ONL-CONFIG during re-partitioning
 
+        self.zf = None
+        # zipfile handle to installer archive
+
     def run(self):
         self.log.error("not implemented")
         return 1
 
     def shutdown(self):
-        pass
+        zf, self.zf = self.zf, None
+        if zf: zf.close()
+
+    def installerCopy(self, basename, dst, optional=False):
+        """Copy the file as-is, or get it from the installer zip."""
+
+        src = os.path.join(self.im.installerConf.installer_dir, basename)
+        if os.path.exists(src):
+            self.copy2(src, dst)
+            return
+
+        if basename in self.zf.namelist():
+            self.log.debug("+ unzip -p %s %s > %s",
+                           self.im.installerConf.installer_zip, basename, dst)
+            with self.zf.open(basename, "r") as rfd:
+                with open(dst, "wb") as wfd:
+                    shutil.copyfileobj(rfd, wfd)
+            return
+
+        if not optional:
+            raise ValueError("missing installer file %s" % basename)
+
+    def installerDd(self, basename, device):
+
+        p = os.path.join(self.im.installerConf.installer_dir, basename)
+        if os.path.exists(p):
+            cmd = ('dd',
+                   'if=' + basename,
+                   'of=' + device,)
+            self.check_call(cmd, vmode=self.V2)
+            return
+
+        if basename in self.zf.namelist():
+            self.log.debug("+ unzip -p %s %s | dd of=%s",
+                           self.im.installerConf.installer_zip, basename, device)
+            with self.zf.open(basename, "r") as rfd:
+                with open(device, "rb+") as wfd:
+                    shutil.copyfileobj(rfd, wfd)
+            return
+
+        raise ValueError("cannot find file %s" % basename)
+
+    def installerExists(self, basename):
+        if basename in os.listdir(self.im.installerConf.installer_dir): return True
+        if basename in self.zf.namelist(): return True
+        return False
 
     def installSwi(self):
 
-        swis = [x for x in os.listdir(self.im.installerConf.installer_dir) if x.endswith('.swi')]
+        files = os.listdir(self.im.installerConf.installer_dir) + self.zf.namelist()
+        swis = [x for x in files if x.endswith('.swi')]
         if not swis:
             self.log.info("No ONL Software Image available for installation.")
             self.log.info("Post-install ZTN installation will be required.")
@@ -88,13 +143,12 @@ class Base:
             return
 
         base = swis[0]
-        src = os.path.join(self.im.installerConf.installer_dir, base)
 
         self.log.info("Installing ONL Software Image (%s)...", base)
         dev = self.blkidParts['ONL-IMAGES']
         with MountContext(dev.device, log=self.log) as ctx:
             dst = os.path.join(ctx.dir, base)
-            self.copy2(src, dst)
+            self.installerCopy(base, dst)
 
         return 0
 
@@ -268,18 +322,18 @@ class Base:
 
         self.log.info("Installing boot-config to %s", dev.device)
 
-        src = os.path.join(self.im.installerConf.installer_dir, 'boot-config')
-        ##src = os.path.join(self.im.installerConf.installer_platform_dir, 'boot-config')
+        basename = 'boot-config'
         with MountContext(dev.device, log=self.log) as ctx:
-            dst = os.path.join(ctx.dir, 'boot-config')
-            self.copy2(src, dst)
+            dst = os.path.join(ctx.dir, basename)
+            self.installerCopy(basename, dst)
+            with open(dst) as fd:
+                buf = fd.read()
 
-        with open(src) as fd:
-            ecf = fd.read().encode('base64', 'strict').strip()
-            if self.im.grub and self.im.grubEnv is not None:
-                setattr(self.im.grubEnv, 'boot_config_default', ecf)
-            if self.im.uboot and self.im.ubootEnv is not None:
-                setattr(self.im.ubootEnv, 'boot-config-default', ecf)
+        ecf = buf.encode('base64', 'strict').strip()
+        if self.im.grub and self.im.grubEnv is not None:
+            setattr(self.im.grubEnv, 'boot_config_default', ecf)
+        if self.im.uboot and self.im.ubootEnv is not None:
+            setattr(self.im.ubootEnv, 'boot-config-default', ecf)
 
         return 0
 
@@ -288,9 +342,19 @@ class Base:
         pm = ProcMountsParser()
         for m in pm.mounts:
             if m.device.startswith(self.device):
-                self.log.error("mount %s on %s will be erased by install",
-                               m.dir, m.device)
-                return 1
+                if not self.force:
+                    self.log.error("mount %s on %s will be erased by install",
+                                   m.dir, m.device)
+                    return 1
+                else:
+                    self.log.warn("unmounting %s from %s (--force)",
+                                  m.dir, m.device)
+                    try:
+                        self.check_call(('umount', m.dir,))
+                    except subprocess.CalledProcessError:
+                        self.log.error("cannot unmount")
+                        return 1
+
         return 0
 
 GRUB_TPL = """\
@@ -426,14 +490,15 @@ class GrubInstaller(SubprocessMixin, Base):
 
         self.log.info("Installing kernel")
         dev = self.blkidParts['ONL-BOOT']
+
+        files = set(os.listdir(self.im.installerConf.installer_dir) + self.zf.namelist())
+        files = [b for b in files if b.startswith('kernel-') or b.startswith('onl-loader-initrd-')]
+
         with MountContext(dev.device, log=self.log) as ctx:
             def _cp(b):
-                src = os.path.join(self.im.installerConf.installer_dir, b)
-                if not os.path.isfile(src): return
-                if b.startswith('kernel-') or b.startswith('onl-loader-initrd-'):
-                    dst = os.path.join(ctx.dir, b)
-                    self.copy2(src, dst)
-            [_cp(e) for e in os.listdir(self.im.installerConf.installer_dir)]
+                dst = os.path.join(ctx.dir, b)
+                self.installerCopy(b, dst, optional=True)
+            [_cp(e) for e in files]
 
             d = os.path.join(ctx.dir, "grub")
             self.makedirs(d)
@@ -485,6 +550,11 @@ class GrubInstaller(SubprocessMixin, Base):
         self.im.grubEnv.__dict__['bootPart'] = dev.device
         self.im.grubEnv.__dict__['bootDir'] = None
 
+        # get a handle to the installer zip
+        p = os.path.join(self.im.installerConf.installer_dir,
+                         self.im.installerConf.installer_zip)
+        self.zf = zipfile.ZipFile(p)
+
         code = self.installSwi()
         if code: return code
 
@@ -513,7 +583,7 @@ class GrubInstaller(SubprocessMixin, Base):
         return self.installGpt()
 
     def shutdown(self):
-        pass
+        Base.shutdown(self)
 
 class UbootInstaller(SubprocessMixin, Base):
 
@@ -598,42 +668,37 @@ class UbootInstaller(SubprocessMixin, Base):
 
     def installLoader(self):
 
-        loaderSrc = None
         c1 = self.im.platformConf['flat_image_tree'].get('itb', None)
         if type(c1) == dict: c1 = c1.get('=', None)
         c2 = ("%s.itb"
               % (self.im.installerConf.installer_platform,))
         c3 = "onl-loader-fit.itb"
 
-        loaderSrc = None
+        loaderBasename = None
         for c in (c1, c2, c3):
             if c is None: continue
-            p = os.path.join(self.im.installerConf.installer_dir, c)
-            if os.path.exists(p):
-                loaderSrc = p
+            if self.installerExists(c):
+                loaderBasename = c
                 break
 
-        if not loaderSrc:
+        if not loaderBasename:
             self.log.error("The platform loader file is missing.")
             return 1
 
-        self.log.info("Installing the ONL loader from %s...", loaderSrc)
+        self.log.info("Installing the ONL loader from %s...", loaderBasename)
 
         if self.rawLoaderDevice is not None:
             self.log.info("Installing ONL loader %s --> %s...",
-                          loaderSrc, self.rawLoaderDevice)
-            cmd = ('dd',
-                   'if=' + loaderSrc,
-                   'of=' + self.rawLoaderDevice,)
-            self.check_call(cmd, vmode=self.V2)
-        else:
-            dev = self.blkidParts['ONL-BOOT']
-            basename = os.path.split(loaderSrc)[1]
-            self.log.info("Installing ONL loader %s --> %s:%s...",
-                          loaderSrc, dev.device, basename)
-            with MountContext(dev.device, log=self.log) as ctx:
-                dst = os.path.join(ctx.dir, basename)
-                self.copy2(loaderSrc, dst)
+                          loaderBasename, self.rawLoaderDevice)
+            self.installerDd(loaderBasename, self.rawLoaderDevice)
+            return 0
+
+        dev = self.blkidParts['ONL-BOOT']
+        self.log.info("Installing ONL loader %s --> %s:%s...",
+                      loaderBasename, dev.device, loaderBasename)
+        with MountContext(dev.device, log=self.log) as ctx:
+            dst = os.path.join(ctx.dir, loaderBasename)
+            self.installerCopy(loaderBasename, dst)
 
         return 0
 
@@ -713,6 +778,11 @@ class UbootInstaller(SubprocessMixin, Base):
                 self.rawLoaderDevice = self.device + str(partIdx+1)
                 break
 
+        # get a handle to the installer zip
+        p = os.path.join(self.im.installerConf.installer_dir,
+                         self.im.installerConf.installer_zip)
+        self.zf = zipfile.ZipFile(p)
+
         code = self.installSwi()
         if code: return code
 
@@ -744,4 +814,4 @@ class UbootInstaller(SubprocessMixin, Base):
         return self.installUboot()
 
     def shutdown(self):
-        pass
+        Base.shutdown(self)