From d7958005f3938a9d8b1742379e2405955d561272 Mon Sep 17 00:00:00 2001 From: "Carl D. Roth" Date: Mon, 28 Nov 2016 12:06:22 -0800 Subject: [PATCH 1/3] Updated plugin api - single run() method is called with a mode argument - plugin objects are persistent (re-entrant) for the lifetime of the install - installer grovels the filesystem and/or zip file for plugin files --- builds/any/installer/grub/builds/Makefile | 4 +- builds/any/installer/sample-postinstall.py | 8 ++- builds/any/installer/sample-preinstall.py | 8 ++- builds/any/installer/uboot/builds/Makefile | 4 +- .../src/python/onl/install/BaseInstall.py | 71 ++++++++++++------- .../src/python/onl/install/Plugin.py | 18 ++++- tools/mkinstaller.py | 25 ++++--- 7 files changed, 92 insertions(+), 46 deletions(-) diff --git a/builds/any/installer/grub/builds/Makefile b/builds/any/installer/grub/builds/Makefile index 933f727f..cd392e1a 100644 --- a/builds/any/installer/grub/builds/Makefile +++ b/builds/any/installer/grub/builds/Makefile @@ -18,8 +18,8 @@ MKINSTALLER_OPTS = \ --swi onl-swi:$(ARCH) \ --preinstall-script $(ONL)/builds/any/installer/sample-preinstall.sh \ --postinstall-script $(ONL)/builds/any/installer/sample-postinstall.sh \ - --preinstall-plugin $(ONL)/builds/any/installer/sample-preinstall.py \ - --postinstall-plugin $(ONL)/builds/any/installer/sample-postinstall.py \ + --plugin $(ONL)/builds/any/installer/sample-preinstall.py \ + --plugin $(ONL)/builds/any/installer/sample-postinstall.py \ # THIS LINE INTENTIONALLY LEFT BLANK __installer: diff --git a/builds/any/installer/sample-postinstall.py b/builds/any/installer/sample-postinstall.py index 1f5605d2..b7049e6b 100644 --- a/builds/any/installer/sample-postinstall.py +++ b/builds/any/installer/sample-postinstall.py @@ -54,6 +54,10 @@ import onl.install.Plugin class Plugin(onl.install.Plugin.Plugin): - def run(self): - self.log.info("hello from postinstall plugin") + def run(self, mode): + + if mode == self.PLUGIN_POSTINSTALL: + self.log.info("hello from postinstall plugin") + return 0 + return 0 diff --git a/builds/any/installer/sample-preinstall.py b/builds/any/installer/sample-preinstall.py index 09b2b524..704d9a6c 100644 --- a/builds/any/installer/sample-preinstall.py +++ b/builds/any/installer/sample-preinstall.py @@ -44,6 +44,10 @@ import onl.install.Plugin class Plugin(onl.install.Plugin.Plugin): - def run(self): - self.log.info("hello from preinstall plugin") + def run(self, mode): + + if mode == self.PLUGIN_PREINSTALL: + self.log.info("hello from preinstall plugin") + return 0 + return 0 diff --git a/builds/any/installer/uboot/builds/Makefile b/builds/any/installer/uboot/builds/Makefile index 800463e4..dc3bc4b8 100644 --- a/builds/any/installer/uboot/builds/Makefile +++ b/builds/any/installer/uboot/builds/Makefile @@ -18,8 +18,8 @@ MKINSTALLER_OPTS = \ --swi onl-swi:$(ARCH) \ --preinstall-script $(ONL)/builds/any/installer/sample-preinstall.sh \ --postinstall-script $(ONL)/builds/any/installer/sample-postinstall.sh \ - --preinstall-plugin $(ONL)/builds/any/installer/sample-preinstall.py \ - --postinstall-plugin $(ONL)/builds/any/installer/sample-postinstall.py \ + --plugin $(ONL)/builds/any/installer/sample-preinstall.py \ + --plugin $(ONL)/builds/any/installer/sample-postinstall.py \ # THIS LINE INTENTIONALLY LEFT BLANK __installer: diff --git a/packages/base/all/vendor-config-onl/src/python/onl/install/BaseInstall.py b/packages/base/all/vendor-config-onl/src/python/onl/install/BaseInstall.py index b9528b4d..77df42fa 100644 --- a/packages/base/all/vendor-config-onl/src/python/onl/install/BaseInstall.py +++ b/packages/base/all/vendor-config-onl/src/python/onl/install/BaseInstall.py @@ -14,6 +14,7 @@ import yaml import zipfile import shutil import imp +import fnmatch, glob from InstallUtils import SubprocessMixin from InstallUtils import MountContext, BlkidParser, PartedParser @@ -90,6 +91,9 @@ class Base: self.zf = None # zipfile handle to installer archive + self.plugins = [] + # dynamically-detected plugins + def run(self): self.log.error("not implemented") return 1 @@ -99,6 +103,11 @@ class Base: return 1 def shutdown(self): + + plugins, self.plugins = self.plugins, [] + for plugin in plugins: + plugin.shutdown() + zf, self.zf = self.zf, None if zf: zf.close() @@ -415,38 +424,27 @@ class Base: return 0 - def preinstall(self): - return self.runPlugin("preinstall.py") - - def postinstall(self): - return self.runPlugin("postinstall.py") - - def runPluginFile(self, pyPath): + def loadPluginsFromFile(self, pyPath): + self.log.info("loading plugins from %s", pyPath) with open(pyPath) as fd: sfx = ('.py', 'U', imp.PY_SOURCE,) mod = imp.load_module("plugin", fd, pyPath, sfx) for attr in dir(mod): klass = getattr(mod, attr) if isinstance(klass, type) and issubclass(klass, Plugin): - self.log.info("%s: running plugin %s", pyPath, attr) + self.log.info("%s: found plugin %s", pyPath, attr) plugin = klass(self) - try: - code = plugin.run() - except: - self.log.exception("plugin failed") - code = 1 - plugin.shutdown() - if code: return code + self.plugins.append(plugin) - return 0 + def loadPlugins(self): - def runPlugin(self, basename): + pat = os.path.join(self.im.installerConf.installer_dir, "plugins", "*.py") + for src in glob.glob(pat): + self.loadPluginsFromFile(src) - src = os.path.join(self.im.installerConf.installer_dir, basename) - if os.path.exists(src): - return self.runPluginFile(src) - - if basename in self.zf.namelist(): + pat = "plugins/*.py" + for basename in self.zf.namelist(): + if not fnmatch.fnmatch(basename, pat): continue try: src = None with self.zf.open(basename, "r") as rfd: @@ -454,11 +452,24 @@ class Base: suffix=".py") with os.fdopen(wfno, "w") as wfd: shutil.copyfileobj(rfd, wfd) - return self.runPluginFile(src) + self.loadPluginsFromFile(src) finally: if src and os.path.exists(src): os.unlink(src) + return 0 + + def runPlugins(self, mode): + self.log.info("running plugins: %s", mode) + for plugin in self.plugins: + try: + code = plugin.run(mode) + except: + self.log.exception("plugin failed") + code = 1 + if code: return code + return 0 + GRUB_TPL = """\ serial %(serial)s terminal_input serial @@ -654,7 +665,10 @@ class GrubInstaller(SubprocessMixin, Base): self.im.installerConf.installer_zip) self.zf = zipfile.ZipFile(p) - code = self.preinstall() + code = self.loadPlugins() + if code: return code + + code = self.runPlugins(Plugin.PLUGIN_PREINSTALL) if code: return code code = self.findGpt() @@ -712,7 +726,7 @@ class GrubInstaller(SubprocessMixin, Base): code = self.installGrub() if code: return code - code = self.postinstall() + code = self.runPlugins(Plugin.PLUGIN_POSTINSTALL) if code: return code self.log.info("ONL loader install successful.") @@ -896,7 +910,10 @@ class UbootInstaller(SubprocessMixin, Base): self.im.installerConf.installer_zip) self.zf = zipfile.ZipFile(p) - code = self.preinstall() + code = self.loadPlugins() + if code: return code + + code = self.runPlugins(Plugin.PLUGIN_PREINSTALL) if code: return code code = self.assertUnmounted() @@ -968,7 +985,7 @@ class UbootInstaller(SubprocessMixin, Base): code = self.installUbootEnv() if code: return code - code = self.postinstall() + code = self.runPlugins(Plugin.PLUGIN_POSTINSTALL) if code: return code return 0 diff --git a/packages/base/all/vendor-config-onl/src/python/onl/install/Plugin.py b/packages/base/all/vendor-config-onl/src/python/onl/install/Plugin.py index f1e97713..e854cece 100644 --- a/packages/base/all/vendor-config-onl/src/python/onl/install/Plugin.py +++ b/packages/base/all/vendor-config-onl/src/python/onl/install/Plugin.py @@ -5,13 +5,25 @@ Base class for installer plugins. class Plugin(object): + PLUGIN_PREINSTALL = "preinstall" + PLUGIN_POSTINSTALL = "postinstall" + def __init__(self, installer): self.installer = installer self.log = self.installer.log.getChild("plugin") - def run(self): - self.log.warn("not implemented") - return 0 + def run(self, mode): + + if mode == self.PLUGIN_PREINSTALL: + self.log.warn("pre-install plugin not implemented") + return 0 + + if mode == self.PLUGIN_POSTINSTALL: + self.log.warn("post-install plugin not implemented") + return 0 + + self.log.warn("invalid plugin mode %s", repr(mode)) + return 1 def shutdown(self): pass diff --git a/tools/mkinstaller.py b/tools/mkinstaller.py index 6348a252..1d1375c2 100755 --- a/tools/mkinstaller.py +++ b/tools/mkinstaller.py @@ -191,10 +191,8 @@ if __name__ == '__main__': ap.add_argument("--postinstall-script", help="Specify a preinstall script (runs after installer)") - ap.add_argument("--preinstall-plugin", - help="Specify a preinstall plugin (runs from within the installer chroot)") - ap.add_argument("--postinstall-plugin", - help="Specify a postinstall plugin (runs from within the installer chroot)") + ap.add_argument("--plugin", action='append', + help="Specify a Python plugin (runs from within the installer chroot)") ops = ap.parse_args() installer = InstallerShar(ops.arch, ops.work_dir) @@ -235,15 +233,26 @@ if __name__ == '__main__': if not os.path.exists(hookdir): os.makedirs(hookdir) + plugindir = os.path.join(installer.work_dir, "tmp/plugins") + if not os.path.exists(plugindir): + os.makedirs(plugindir) + if ops.preinstall_script: installer.add_file_as(ops.preinstall_script, "preinstall.sh") if ops.postinstall_script: installer.add_file_as(ops.postinstall_script, "postinstall.sh") - if ops.preinstall_plugin: - installer.add_file_as(ops.preinstall_plugin, "preinstall.py") - if ops.postinstall_plugin: - installer.add_file_as(ops.postinstall_plugin, "postinstall.py") + for plugin in ops.plugin: + basename = os.path.split(plugin)[1] + basename = os.path.splitext(basename)[0] + dst = tempfile.mktemp(dir=plugindir, + prefix=basename+'-', + suffix='.py') + shutil.copy(plugin, dst) + + l = os.listdir(plugindir) + if l: + installer.add_dir(plugindir) iname = os.path.abspath(ops.out) installer.build(iname) From 6448819434e2e1acf3597aeeaca5736d7fec5e34 Mon Sep 17 00:00:00 2001 From: "Carl D. Roth" Date: Mon, 28 Nov 2016 13:31:34 -0800 Subject: [PATCH 2/3] Updated plugin api docs --- builds/any/installer/sample-postinstall.py | 8 ++++++++ builds/any/installer/sample-preinstall.py | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/builds/any/installer/sample-postinstall.py b/builds/any/installer/sample-postinstall.py index b7049e6b..cf1db0b4 100644 --- a/builds/any/installer/sample-postinstall.py +++ b/builds/any/installer/sample-postinstall.py @@ -26,6 +26,8 @@ of the installer Python script) will 2. instantiate an instance of each class, with the installer object initialized as the 'installer' attribute 3. invoke the 'run' method (which must be overridden by implementors) + For a post-install plugin, the 'mode' argument is set to + PLUGIN_POSTINSTALL. 4. invoke the 'shutdown' method (by default, a no-op) The 'run' method should return zero on success. In any other case, the @@ -45,6 +47,12 @@ e.g. MountContext. The OnlMountContextReadWrite object and their siblings won't work here because the mtab.yml file is not populated within the loader environment. +A post-install plugin should execute any post-install actions when +'mode' is set to PLUGIN_POSTINSTALL. If 'mode' is set to any other +value, the plugin should ignore it and return zero. The plugin run() +method is invoked multiple times during the installer with different +values of 'mode'. The 'shutdown()' method is called only once. + When using MountContxt, the system state in the installer object can help (self.installer.blkidParts in particular). diff --git a/builds/any/installer/sample-preinstall.py b/builds/any/installer/sample-preinstall.py index 704d9a6c..cd29c2dc 100644 --- a/builds/any/installer/sample-preinstall.py +++ b/builds/any/installer/sample-preinstall.py @@ -26,6 +26,8 @@ of the installer Python script) will 2. instantiate an instance of each class, with the installer object initialized as the 'installer' attribute 3. invoke the 'run' method (which must be overridden by implementors) + For a pre-install plugin, the 'mode' argument is set to + PLUGIN_PREINSTALL. 4. invoke the 'shutdown' method (by default, a no-op) The 'run' method should return zero on success. In any other case, the @@ -38,6 +40,12 @@ prepped/initialized/scanned yet. As per the ONL installer API, the installer starts with *no* filesystems mounted, not even the ones from a prior install. +A pre-install plugin should execute any pre-install actions when +'mode' is set to PLUGIN_PREINSTALL. If 'mode' is set to any other +value, the plugin should ignore it and return zero. The plugin run() +method is invoked multiple times during the installer with different +values of 'mode'. The 'shutdown()' method is called only once. + """ import onl.install.Plugin From ee551d596835ca1439b23aaade35f2037d23109b Mon Sep 17 00:00:00 2001 From: "Carl D. Roth" Date: Mon, 28 Nov 2016 13:34:08 -0800 Subject: [PATCH 3/3] Updated plugin api docs --- builds/any/installer/sample-postinstall.py | 2 +- builds/any/installer/sample-preinstall.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/builds/any/installer/sample-postinstall.py b/builds/any/installer/sample-postinstall.py index cf1db0b4..1efd4aa3 100644 --- a/builds/any/installer/sample-postinstall.py +++ b/builds/any/installer/sample-postinstall.py @@ -5,7 +5,7 @@ Example Python script for post-install hooks. Add this as a postinstall hook to your installer via the 'mkinstaller.py' command line: -$ mkinstaller.py ... --postinstall-plugin sample-postinstall.py ... +$ mkinstaller.py ... --plugin sample-postinstall.py ... At install time, this script will diff --git a/builds/any/installer/sample-preinstall.py b/builds/any/installer/sample-preinstall.py index cd29c2dc..b0e1c114 100644 --- a/builds/any/installer/sample-preinstall.py +++ b/builds/any/installer/sample-preinstall.py @@ -5,7 +5,7 @@ Example Python script for pre-install hooks. Add this as a preinstall hook to your installer via the 'mkinstaller.py' command line: -$ mkinstaller.py ... --preinstall-plugin sample-preinstall.py ... +$ mkinstaller.py ... --plugin sample-preinstall.py ... At install time, this script will