diff --git a/builds/any/installer/installer.sh.in b/builds/any/installer/installer.sh.in index 76b33987..e3d968aa 100644 --- a/builds/any/installer/installer.sh.in +++ b/builds/any/installer/installer.sh.in @@ -528,7 +528,7 @@ installer_force_umount() { dev=$1; shift mpt=$1; shift case "$mpt" in - /mnt/*) + /mnt/*|/boot/*) installer_say "Unmounting $mpt (--force)" umount "$mpt" ;; diff --git a/builds/any/rootfs/jessie/common/amd64-base-packages.yml b/builds/any/rootfs/jessie/common/amd64-base-packages.yml index a07863f3..efaf425b 100644 --- a/builds/any/rootfs/jessie/common/amd64-base-packages.yml +++ b/builds/any/rootfs/jessie/common/amd64-base-packages.yml @@ -11,3 +11,5 @@ - hw-management - sx-kernel - onl-kernel-3.16-lts-x86-64-all-modules +- efibootmgr +- gdisk diff --git a/builds/any/rootfs/jessie/common/overlay/etc/mtab.yml b/builds/any/rootfs/jessie/common/overlay/etc/mtab.yml index 598e4c69..e04f3666 100644 --- a/builds/any/rootfs/jessie/common/overlay/etc/mtab.yml +++ b/builds/any/rootfs/jessie/common/overlay/etc/mtab.yml @@ -17,3 +17,11 @@ mounts: mount: ro dir: /mnt/onl/boot fsck: false + + # ESP (EFI system partition) + EFI-BOOT: + mount: ro + dir: /boot/efi + fsck: false + label: EFI System + optional: true diff --git a/builds/any/rootfs/stretch/common/amd64-base-packages.yml b/builds/any/rootfs/stretch/common/amd64-base-packages.yml index a07863f3..efaf425b 100644 --- a/builds/any/rootfs/stretch/common/amd64-base-packages.yml +++ b/builds/any/rootfs/stretch/common/amd64-base-packages.yml @@ -11,3 +11,5 @@ - hw-management - sx-kernel - onl-kernel-3.16-lts-x86-64-all-modules +- efibootmgr +- gdisk diff --git a/builds/any/rootfs/stretch/common/overlay/etc/mtab.yml b/builds/any/rootfs/stretch/common/overlay/etc/mtab.yml index 598e4c69..e04f3666 100644 --- a/builds/any/rootfs/stretch/common/overlay/etc/mtab.yml +++ b/builds/any/rootfs/stretch/common/overlay/etc/mtab.yml @@ -17,3 +17,11 @@ mounts: mount: ro dir: /mnt/onl/boot fsck: false + + # ESP (EFI system partition) + EFI-BOOT: + mount: ro + dir: /boot/efi + fsck: false + label: EFI System + optional: true diff --git a/packages/base/all/initrds/loader-initrd-files/src/bin/switchroot b/packages/base/all/initrds/loader-initrd-files/src/bin/switchroot index 91d08334..2eb8ead6 100644 --- a/packages/base/all/initrds/loader-initrd-files/src/bin/switchroot +++ b/packages/base/all/initrds/loader-initrd-files/src/bin/switchroot @@ -49,7 +49,13 @@ done <${mtab} rm -f ${mtab} mount --move /proc /newroot/proc +if [ -d /sys/firmware/efi/efivars ]; then + umount /sys/firmware/efi/efivars || : +fi mount --move /sys /newroot/sys +if [ -d /newroot/sys/firmware/efi/efivars ]; then + mount -t efivarfs efivarfs /newroot/sys/firmware/efi/efivars +fi mount --move /dev /newroot/dev # Switch to /newroot if possible, else re-execute /init @@ -58,3 +64,8 @@ if [ -x /newroot/sbin/init ]; then else exec /init fi + +# Local variables: +# sh-indentation: 4 +# sh-basic-offset: 4 +# End: diff --git a/packages/base/all/initrds/loader-initrd-files/src/bin/sysinit b/packages/base/all/initrds/loader-initrd-files/src/bin/sysinit index b80f85fb..910b3fcf 100755 --- a/packages/base/all/initrds/loader-initrd-files/src/bin/sysinit +++ b/packages/base/all/initrds/loader-initrd-files/src/bin/sysinit @@ -35,6 +35,9 @@ trap "restoreconsole; reboot -f" EXIT # Mount special filesystems mount -t proc proc /proc mount -t sysfs sysfs /sys +if [ -d /sys/firmware/efi/efivars ]; then + mount -t efivarfs efivarfs /sys/firmware/efi/efivars +fi mount -o remount,size=1M /dev case "$(stat -f -c "%T" /tmp)" in tmpfs|ramfs) ;; @@ -144,4 +147,5 @@ trap - EXIT # Local variables: # sh-basic-offset: 4 +# sh-indentation: 4 # End: diff --git a/packages/base/all/initrds/loader-initrd-files/src/etc/mtab.yml b/packages/base/all/initrds/loader-initrd-files/src/etc/mtab.yml index cb7635b1..051105c7 100644 --- a/packages/base/all/initrds/loader-initrd-files/src/etc/mtab.yml +++ b/packages/base/all/initrds/loader-initrd-files/src/etc/mtab.yml @@ -18,3 +18,11 @@ mounts: mount: rw dir: /mnt/onl/boot fsck: true + + # ESP (EFI system partition) + EFI-BOOT: + mount: ro + dir: /boot/efi + fsck: false + label: EFI System + optional: true diff --git a/packages/base/all/vendor-config-onl/src/lib/install/lib.sh b/packages/base/all/vendor-config-onl/src/lib/install/lib.sh index 62bfb6cf..72f6290f 100644 --- a/packages/base/all/vendor-config-onl/src/lib/install/lib.sh +++ b/packages/base/all/vendor-config-onl/src/lib/install/lib.sh @@ -113,6 +113,9 @@ installer_mkchroot() { mkdir -p ${rootdir}/dev/pts fi mount -t devpts devpts "${rootdir}/dev/pts" + if test -d "${rootdir}/sys/firmware/efi/efivars"; then + mount -t efivarfs efivarfs "${rootdir}/sys/firmware/efi/efivars" + fi if test ${TMPDIR+set}; then # make the tempdir available to the chroot diff --git a/packages/base/all/vendor-config-onl/src/python/onl/install/BaseInstall.py b/packages/base/all/vendor-config-onl/src/python/onl/install/BaseInstall.py index 3068f658..2ca43c17 100755 --- a/packages/base/all/vendor-config-onl/src/python/onl/install/BaseInstall.py +++ b/packages/base/all/vendor-config-onl/src/python/onl/install/BaseInstall.py @@ -19,8 +19,12 @@ import fnmatch, glob from InstallUtils import SubprocessMixin from InstallUtils import MountContext, BlkidParser, PartedParser from InstallUtils import ProcMountsParser +from InstallUtils import GdiskParser +from InstallUtils import OnieSubprocess from Plugin import Plugin +import onl.install.ConfUtils + import onl.YamlUtils from onl.sysconfig import sysconfig @@ -509,14 +513,33 @@ menuentry %(boot_menu_entry)s { initrd /%(platform)s.cpio.gz } +set onie_boot_label="ONIE-BOOT" +set onie_boot_uuid="%(onie_boot_uuid)s" +# filesystem UUID, *not* GPT partition GUID, *not* GPT partition unique GUID +# (tee hee, GPT GRUB cannot grok partition attributes) + +function onie_boot_uefi { + set root='(hd0,gpt1)' + search --no-floppy --fs-uuid --set=root "${onie_boot_uuid}" + echo 'Loading ONIE ...' + chainloader /EFI/onie/grubx64.efi +} + +function onie_boot_dos { + search --no-floppy --label --set=root "${onie_boot_label}" + set saved_entry="0" + save_env saved_entry + echo 'Loading ONIE ...' + chainloader +1 +} + # Menu entry to chainload ONIE menuentry ONIE { - %(set_root_para)s - search --no-floppy %(set_search_para2)s --set=root %(onie_boot)s - %(set_save_entry_para)s - %(set_save_env_para)s - echo 'Loading ONIE ...' - chainloader %(set_chainloader_para)s + if [ -n "${onie_boot_uuid}" ]; then + onie_boot_uefi + else + onie_boot_dos + fi } """ @@ -528,7 +551,14 @@ class GrubInstaller(SubprocessMixin, Base): def __init__(self, *args, **kwargs): Base.__init__(self, *args, **kwargs) - self.isUEFI = False + + self.espDevice = None + self.espFsUuid = None + # optionally fill in ESP partition information + + @property + def isUEFI(self): + return os.path.isdir('/sys/firmware/efi/efivars') def findGpt(self): self.blkidParts = BlkidParser(log=self.log.getChild("blkid")) @@ -612,6 +642,42 @@ class GrubInstaller(SubprocessMixin, Base): return 0 + def findEsp(self): + """Find the block device holding the EFI System Partition. + + XXX assume boot (ESP) partition is on the same device as GRUB + """ + + self.log.info("extracting partition UUIDs for %s", self.device) + + if isinstance(self.im.grubEnv, onl.install.ConfUtils.GrubEnv): + # direct (or chroot) access + gp = GdiskParser(self.device, + subprocessContext=self.im.grubEnv, + log=self.log) + else: + # indirect access using onie-shell + ctx = OnieSubprocess(log=self.log.getChild("onie")) + gp = GdiskParser(self.device, + subprocessContext=ctx, + log=self.log) + + espParts = [x for x in gp.parts if x.isEsp] + if not espParts: + self.log.error("cannot find ESP partition on %s", self.device) + return 1 + self.espDevice = espParts[0].device + self.log.info("found ESP partition %s", self.espDevice) + + espParts = [x for x in self.blkidParts if x.device==self.espDevice] + if not espParts: + self.log.error("cannot find blkid entry for ESP partition on %s", self.espDevice) + return 1 + self.espFsUuid = espParts[0].uuid + self.log.info("found ESP filesystem UUID %s", self.espFsUuid) + + return 0 + def installLoader(self): kernels = [] @@ -659,20 +725,9 @@ class GrubInstaller(SubprocessMixin, Base): ctx['boot_loading_name'] = sysconfig.installer.os_name if self.isUEFI: - ctx['set_root_para'] = "set root='(hd0,gpt1)'" - ctx['set_search_para2'] = "--fs-uuid" - ctx['set_save_entry_para'] = "" - ctx['set_save_env_para'] = "" - dev_UEFI = self.blkidParts['EFI System'] - ctx['onie_boot'] = dev_UEFI.uuid - ctx['set_chainloader_para'] = "/EFI/onie/grubx64.efi" + ctx['onie_boot_uuid'] = self.espFsUuid else: - ctx['set_root_para'] = "" - ctx['set_search_para2'] = "--label" - ctx['set_save_entry_para'] = "set saved_entry=\"0\"" - ctx['set_save_env_para'] = "save_env saved_entry" - ctx['onie_boot'] = "ONIE-BOOT" - ctx['set_chainloader_para'] = "+1" + ctx['onie_boot_uuid'] = "" cf = GRUB_TPL % ctx @@ -688,7 +743,7 @@ class GrubInstaller(SubprocessMixin, Base): def installGrub(self): self.log.info("Installing GRUB to %s", self.partedDevice.path) - self.im.grubEnv.install(self.partedDevice.path, self.isUEFI) + self.im.grubEnv.install(self.partedDevice.path) return 0 def installGpt(self): @@ -707,6 +762,13 @@ class GrubInstaller(SubprocessMixin, Base): code = self.findGpt() if code: return code + if self.isUEFI: + code = self.findEsp() + if code: return code + self.im.grubEnv.__dict__['espPart'] = self.espDevice + else: + self.im.grubEnv.__dict__['espPart'] = None + self.log.info("Installing to %s starting at partition %d", self.device, self.minpart) @@ -775,8 +837,6 @@ class GrubInstaller(SubprocessMixin, Base): if label != 'gpt': self.log.error("invalid GRUB label in platform config: %s", label) return 1 - if os.path.isdir('/sys/firmware/efi/efivars'): - self.isUEFI = True return self.installGpt() diff --git a/packages/base/all/vendor-config-onl/src/python/onl/install/ConfUtils.py b/packages/base/all/vendor-config-onl/src/python/onl/install/ConfUtils.py index 62e5fc66..3e63541b 100755 --- a/packages/base/all/vendor-config-onl/src/python/onl/install/ConfUtils.py +++ b/packages/base/all/vendor-config-onl/src/python/onl/install/ConfUtils.py @@ -7,7 +7,11 @@ import os import logging import subprocess from InstallUtils import SubprocessMixin, ChrootSubprocessMixin, MountContext +from InstallUtils import OnieSubprocess from cStringIO import StringIO +import re + +from onl.sysconfig import sysconfig class ConfBase: @@ -90,13 +94,16 @@ class GrubEnv(SubprocessMixin): INSTALL = "grub-install" EDITENV = "grub-editenv" + EFIBOOTMGR = "efibootmgr" # system default ENV_PATH = "/grub/grubenv" # override me + EFI_BOOT_RE = re.compile("Boot([0-9a-fA-F]*)[*] (.*)") + def __init__(self, - bootDir=None, bootPart=None, + bootDir=None, bootPart=None, espPart=None, path=None, log=None): @@ -108,13 +115,16 @@ class GrubEnv(SubprocessMixin): self.__dict__['bootPart'] = bootPart # location of GRUB boot files (mounted directory or unmounted partition) + self.__dict__['espPart'] = espPart + # location of EFI System Partition + self.__dict__['path'] = path or self.ENV_PATH # path to grubenv, relative to above self.__dict__['log'] = log or logging.getLogger("grub") - def mountCtx(self, device): - return MountContext(device, fsType='ext4', log=self.log) + def mountCtx(self, device, fsType='ext4'): + return MountContext(device, fsType=fsType, log=self.log) def asDict(self): if self.bootPart: @@ -164,36 +174,83 @@ class GrubEnv(SubprocessMixin): cmd = (self.EDITENV, p, 'unset', attr,) self.check_call(cmd) + @property + def isUEFI(self): + return os.path.isdir('/sys/firmware/efi/efivars') + def install(self, device): - if self.bootDir is not None: - self.check_call((self.INSTALL, '--boot-directory=' + self.bootDir, device,)) - elif self.bootPart is not None: - with self.mountCtx(self.bootPart) as ctx: - self.check_call((self.INSTALL, '--boot-directory=' + ctx.dir, device,)) + + uidx = None + if self.isUEFI: + buf = self.check_output((self.EFIBOOTMGR,)) + for line in buf.splitlines(False): + m = self.EFI_BOOT_RE.match(line) + if m: + if m.group(2) == sysconfig.installer.os_name: + uidx = m.group(1) + break + if uidx is not None: + self.check_output((self.EFIBOOTMGR, '-b', uidx, '-B',)) + + grubOpts = [] + if self.isUEFI: + grubOpts.append('--target=x86_64-efi') + grubOpts.append('--no-nvram') + grubOpts.append('--recheck') + + grubOpts.append('--bootloader-id=ONL') + # All ONL-derived distros should be able to use + # the same profile + + def _install(): + if self.bootDir is not None: + self.check_call([self.INSTALL, '--boot-directory=' + self.bootDir,] + + grubOpts + + [device,]) + elif self.bootPart is not None: + with self.mountCtx(self.bootPart) as ctx: + self.check_call([self.INSTALL, '--boot-directory=' + ctx.dir,] + + grubOpts + + [device,]) + else: + self.check_call([self.INSTALL,] + grubOpts + [device,]) + + if self.espPart is not None: + with self.mountCtx(self.espPart, fsType=None) as ctx: + grubOpts.append('--efi-directory=' + ctx.dir) + _install() else: - self.check_call((self.INSTALL, device,)) + _install() + + if self.isUEFI: + self.check_call((self.EFIBOOTMGR, + '--create', + '--label', sysconfig.installer.os_name, + '--disk', device, + '--part', '1', + '--loader', '/EFI/ONL/grubx64.efi',)) class ChrootGrubEnv(ChrootSubprocessMixin, GrubEnv): def __init__(self, chrootDir, mounted=False, - bootDir=None, bootPart=None, + bootDir=None, bootPart=None, espPart=None, path=None, log=None): self.__dict__['chrootDir'] = chrootDir self.__dict__['mounted'] = mounted GrubEnv.__init__(self, - bootDir=bootDir, bootPart=bootPart, + bootDir=bootDir, bootPart=bootPart, espPart=espPart, path=path, log=log) - def mountCtx(self, device): + def mountCtx(self, device, fsType='ext4'): return MountContext(device, - chroot=self.chrootDir, fsType='ext4', + chroot=self.chrootDir, fsType=fsType, log=self.log) -class ProxyGrubEnv: +class ProxyGrubEnv(SubprocessMixin): """Pretend to manipulate the GRUB environment. Instead, write a trace of shell commands to a log @@ -211,7 +268,7 @@ class ProxyGrubEnv: def __init__(self, installerConf, - bootDir=None, chroot=True, bootPart=None, + bootDir=None, chroot=True, bootPart=None, espPart=None, path=None, log=None): @@ -226,6 +283,9 @@ class ProxyGrubEnv: self.__dict__['bootPart'] = bootPart # location of GRUB boot files (mounted directory or unmounted partition) + self.__dict__['espPart'] = espPart + # location of EFI System Partition + self.__dict__['chroot'] = chroot # True of the bootDir is inside the chroot, # else bootDir is in the host's file namespace @@ -261,7 +321,8 @@ class ProxyGrubEnv: % (self.path.lstrip('/'),)) cmds.append("mpt=$(mktemp -t -d)") cmds.append("mount %s $mpt" % self.bootPart) - cmds.append(("sts=0; %s %s set %s=\"%s\" || sts=$?" + cmds.append("sts=0") + cmds.append(("%s %s set %s=\"%s\" || sts=$?" % (self.EDITENV, p, attr, val,))) cmds.append("umount $mpt") cmds.append("rmdir $mpt") @@ -291,7 +352,8 @@ class ProxyGrubEnv: % (self.path.lstrip('/'),)) cmds.append("mpt=$(mktemp -t -d)") cmds.append("mount %s $mpt" % self.bootPart) - cmds.append(("sts=0; %s %s unset %s || sts=$?" + cmds.append("sts=0") + cmds.append(("%s %s unset %s || sts=$?" % (self.EDITENV, p, attr,))) cmds.append("umount $mpt") cmds.append("rmdir $mpt") @@ -303,35 +365,83 @@ class ProxyGrubEnv: fd.write(cmd) fd.write("\n") - def install(self, device, isUEFI=False): + @property + def isUEFI(self): + return os.path.isdir('/sys/firmware/efi/efivars') + + def install(self, device): self.log.warn("deferring commands to %s...", self.installerConf.installer_postinst) + + cmds.append("os_name=\"%s\"" % sysconfig.installer.os_name) + + if self.isUEFI: + sub = OnieSubprocess(log=self.log.getChild("onie")) + cmd = (self.EFIBOOTMGR,) + buf = sub.check_output(cmd) + bidx = None + for line in buf.splitlines(False): + m = self.EFI_BOOT_RE.match(line) + if m: + if m.group(2) == sysconfig.installer.os_name: + uidx = m.group(1) + break + + if uidx is not None: + cmds.append("%s -b %s -B || sts=$?" % (self.EFIBOOTMGR, bidx,)) + + grubOpts = [] + if self.isUEFI: + grubOpts.append('--target=x86_64-efi') + grubOpts.append('--no-nvram') + grubOpts.append('--bootloader-id=ONL') + grubOpts.append('--efi-directory=/boot/efi') + grubOpts.append('--recheck') + cmds = [] + + if self.bootPart and not self.bootDir: + cmds.append("bootMpt=$(mktemp -t -d)") + cmds.append("mount %s $bootMpt" % self.bootPart) + + if self.espPart is not None: + cmds.append("espMpt=$(mktemp -t -d)") + cmds.append("mount %s $espMpt" % self.espPart) + + cmds.append("sts=0") + if self.bootDir and self.chroot: p = os.pat.join(self.installerConf.installer_chroot, self.bootDir.lstrip('/')) - cmds.append(("%s --boot-directory=\"%s\" %s" % (self.INSTALL, p, device,))) + cmd = ([self.INSTALL, '--boot-directory=' + p,] + + grubOpts + + [device,]) + cmds.append(" ".join(cmd) + " || sts=$?") elif self.bootDir: p = self.bootDir - cmds.append(("%s --boot-directory=\"%s\" %s" % (self.INSTALL, p, device,))) + cmd = ([self.INSTALL, '--boot-directory=' + p,] + + grubOpts + + [device,]) + cmds.append(" ".join(cmd) + " || sts=$?") elif self.bootPart: - cmds.append("mpt=$(mktemp -t -d)") - cmds.append("mount %s $mpt" % self.bootPart) - if isUEFI: - cmds.append("[ -n \"$(efibootmgr -v | grep 'Open Network Linux')\" ] && (efibootmgr -b $(efibootmgr | grep \"Open Network Linux\" | sed 's/^.*Boot//g'| sed 's/** Open.*$//g') -B)") - cmds.append(("sts=0; %s --target=x86_64-efi --no-nvram --bootloader-id=ONL --efi-directory=/boot/efi --boot-directory=\"$mpt\" --recheck %s || sts=$?" - % (self.INSTALL, device,))) - cmds.append("test $sts -eq 0") - cmds.append(("sts=0; %s --quiet --create --label \"Open Network Linux\" --disk %s --part 1 --loader /EFI/ONL/grubx64.efi || sts=$?" - % (self.EFIBOOTMGR , device,))) - else: - cmds.append(("sts=0; %s --boot-directory=\"$mpt\" %s || sts=$?" - % (self.INSTALL, device,))) - cmds.append("umount $mpt") - cmds.append("rmdir $mpt") - cmds.append("test $sts -eq 0") + cmd = ([self.INSTALL, '--boot-directory=\"$bootMpt\"',] + + grubOpts + + [device,]) + cmds.append(" ".join(cmd) + " || sts=$?") else: - cmds.append(("%s %s" - % (self.INSTALL, device,))) + cmd = ([self.INSTALL,] + + grubOpts + + [device,]) + cmds.append(" ".join(cmd) + " || sts=$?") + + if self.bootPart and not self.bootDir: + cmds.append("umount $bootMpt") + cmds.append("rmdir $bootMpt") + + if self.espPart is not None: + cmds.append("umount $espMpt") + cmds.append("rmdir $espMpt") + + cmds.append("test $sts -eq 0") with open(self.installerConf.installer_postinst, "a") as fd: for cmd in cmds: diff --git a/packages/base/all/vendor-config-onl/src/python/onl/install/InstallUtils.py b/packages/base/all/vendor-config-onl/src/python/onl/install/InstallUtils.py index 1cc40e4c..ccb4615e 100644 --- a/packages/base/all/vendor-config-onl/src/python/onl/install/InstallUtils.py +++ b/packages/base/all/vendor-config-onl/src/python/onl/install/InstallUtils.py @@ -9,6 +9,7 @@ import subprocess import tempfile import string import shutil +import re import Fit, Legacy @@ -620,6 +621,186 @@ class PartedParser(SubprocessMixin): def __len__(self): return len(self.parts) +class GdiskDiskEntry: + + DEVICE_RE = re.compile("Disk ([^:]*): .*") + BLOCKS_RE = re.compile("Disk [^:]*: ([0-9][0-9]*) sectors") + LBSZ_RE = re.compile("Logical sector size: ([0-9][0-9]*) bytes") + GUID_RE = re.compile("Disk identifier [(]GUID[)]: ([0-9a-fA-F-][0-9a-fA-F-]*)") + + def __init__(self, device, blocks, lbsz, guid): + self.device = device + + self.blocks = blocks + self.lbsz = lbsz + self.guid = guid + + @classmethod + def fromOutput(cls, buf): + + m = cls.BLOCKS_RE.search(buf) + if m: + blocks = int(m.group(1)) + else: + raise ValueError("cannot get block count") + + m = cls.DEVICE_RE.search(buf) + if m: + device = m.group(1) + else: + raise ValueError("cannot get block count") + + m = cls.LBSZ_RE.search(buf) + if m: + lbsz = int(m.group(1)) + else: + raise ValueError("cannot get block size") + + m = cls.GUID_RE.search(buf) + if m: + guid = m.group(1) + else: + raise ValueError("cannot get block size") + + return cls(device, blocks, lbsz, guid) + +class GdiskPartEntry: + + PGUID_RE = re.compile("Partition GUID code: ([0-9a-fA-F-][0-9a-fA-F-]*) [(]([^)]*)[)]") + PGUID2_RE = re.compile("Partition GUID code: ([0-9a-fA-F-][0-9a-fA-F-]*)") + GUID_RE = re.compile("Partition unique GUID: ([0-9a-fA-F-][0-9a-fA-F-]*)") + START_RE = re.compile("First sector: ([0-9][0-9]*)") + END_RE = re.compile("Last sector: ([0-9][0-9]*)") + SIZE_RE = re.compile("Partition size: ([0-9][0-9]*) sectors") + NAME_RE = re.compile("Partition name: [']([^']+)[']") + + ESP_PGUID = "c12a7328-f81f-11d2-ba4b-00a0c93ec93b" + GRUB_PGUID = "21686148-6449-6e6f-744e-656564454649" + ONIE_PGUID = "7412f7d5-a156-4b13-81dc-867174929325" + + def __init__(self, device, pguid, guid, start, end, sz, pguidName=None, name=None): + self.device = device + self.pguid = pguid + self.pguidName = pguidName + self.guid = guid + self.name = name + self.start = start + self.end = end + self.sz = sz + + @property + def isEsp(self): + return self.pguid == self.ESP_PGUID + + @property + def isGrub(self): + return self.pguid == self.GRUB_PGUID + + @property + def isOnie(self): + return self.pguid == self.ONIE_PGUID + + @classmethod + def fromOutput(cls, partDevice, buf): + + m = cls.PGUID_RE.search(buf) + if m: + pguid = m.group(1).lower() + pguidName = m.group(2) + else: + m = cls.PGUID2_RE.search(buf) + if m: + pguid = m.group(1).lower() + pguidName = None + else: + raise ValueError("cannot get partition GUID") + + m = cls.GUID_RE.search(buf) + if m: + guid = m.group(1).lower() + else: + raise ValueError("cannot get partition unique GUID") + + m = cls.START_RE.search(buf) + if m: + start = int(m.group(1)) + else: + raise ValueError("cannot get partition start") + + m = cls.END_RE.search(buf) + if m: + end = int(m.group(1)) + else: + raise ValueError("cannot get partition end") + + m = cls.SIZE_RE.search(buf) + if m: + sz = int(m.group(1)) + else: + raise ValueError("cannot get partition size") + + m = cls.NAME_RE.search(buf) + if m: + name = m.group(1) + else: + name = None + + return cls(partDevice, + pguid, guid, start, end, sz, + pguidName=pguidName, + name=name) + +class GdiskParser(SubprocessMixin): + + def __init__(self, device, subprocessContext=subprocess, log=None): + self.device = device + self.log = log or logging.getLogger("parted") + self.subprocessContext = subprocessContext + self.parse() + + def parse(self): + + cmd = ('sgdisk', '-p', self.device,) + buf = self.subprocessContext.check_output(cmd) + self.disk = GdiskDiskEntry.fromOutput(buf) + + parts = {} + pidx = 1 + for line in buf.splitlines(): + + line = line.strip() + if not line: continue + if not line[0] in string.digits: continue + + partno = int(line.split()[0]) + + partDevice = "%s%d" % (self.device, pidx,) + pidx += 1 + # linux partitions may be numbered differently, + # if there are holes in the GPT partition table + + cmd = ('sgdisk', '-i', str(partno), self.device,) + try: + buf = self.subprocessContext.check_output(cmd) + except subprocess.CalledProcessError as ex: + sys.stdout.write(ex.output) + self.log.warn("sgdisk failed with code %s", ex.returncode) + continue + # skip this partition, but otherwise do not give up + + ent = GdiskPartEntry.fromOutput(partDevice, buf) + parts[partno] = ent + + self.parts = [] + for partno in sorted(parts.keys()): + self.parts.append(parts[partno]) + + if self.disk is None: + raise ValueError("no partition table found") + + def __len__(self): + return len(self.parts) + class ProcMountsEntry: def __init__(self, device, dir, fsType, flags={}): @@ -821,6 +1002,11 @@ class InitrdContext(SubprocessMixin): cmd = ('mount', '-t', 'sysfs', 'sysfs', dst,) self.check_call(cmd, vmode=self.V1) + dst = os.path.join(self.dir, "sys/firmware/efi/efivars") + if os.path.exists(dst): + cmd = ('mount', '-t', 'efivarfs', 'efivarfs', dst,) + self.check_call(cmd, vmode=self.V1) + # maybe mount devtmpfs if self._hasDevTmpfs: dst = os.path.join(self.dir, "dev") @@ -1022,9 +1208,55 @@ class ChrootSubprocessMixin: cmd = ['chroot', self.chrootDir,] + list(cmd) if not self.mounted: - with InitrdContext(self.chrootDir, log=self.log) as ctx: + with InitrdContext(dir=self.chrootDir, log=self.log) as ctx: self.log.debug("+ " + " ".join(cmd)) return subprocess.check_output(cmd, *args, cwd=cwd, **kwargs) else: self.log.debug("+ " + " ".join(cmd)) return subprocess.check_output(cmd, *args, cwd=cwd, **kwargs) + +class OnieSubprocess: + """Simple subprocess mixin that defers to onie-shell.""" + + def __init__(self, log=None): + self.log = log or logging.getLogger("onie") + + def check_call(self, *args, **kwargs): + args = list(args) + kwargs = dict(kwargs) + + cwd = kwargs.pop('cwd', None) + if cwd is not None: + raise ValueError("cwd not supported") + + if args: + cmd = args.pop(0) + else: + cmd = kwargs.pop('cmd') + if isinstance(cmd, basestring): + cmd = ('onie-shell', '-c', 'IFS=;' + cmd,) + else: + cmd = ['onie-shell', '-c',] + " ".join(cmd) + + self.log.debug("+ " + " ".join(cmd)) + subprocess.check_call(cmd, *args, cwd=cwd, **kwargs) + + def check_output(self, *args, **kwargs): + args = list(args) + kwargs = dict(kwargs) + + cwd = kwargs.pop('cwd', None) + if cwd is not None: + raise ValueError("cwd not supported") + + if args: + cmd = args.pop(0) + else: + cmd = kwargs.pop('cmd') + if isinstance(cmd, basestring): + cmd = ('onie-shell', '-c', 'IFS=;' + cmd,) + else: + cmd = ['onie-shell', '-c',] + " ".join(list(cmd)) + + self.log.debug("+ " + " ".join(cmd)) + return subprocess.check_output(cmd, *args, cwd=cwd, **kwargs) diff --git a/packages/base/all/vendor-config-onl/src/python/onl/install/ShellApp.py b/packages/base/all/vendor-config-onl/src/python/onl/install/ShellApp.py index e3a5e505..7c5f2c6c 100644 --- a/packages/base/all/vendor-config-onl/src/python/onl/install/ShellApp.py +++ b/packages/base/all/vendor-config-onl/src/python/onl/install/ShellApp.py @@ -88,7 +88,7 @@ class AppBase(SubprocessMixin, object): sys.exit(code) class OnieBootContext: - """Find the ONIE initrd and umpack/mount it.""" + """Find the ONIE initrd and unpack/mount it.""" def __init__(self, log=None): self.log = log or logging.getLogger(self.__class__.__name__) @@ -128,6 +128,7 @@ class OnieBootContext: initrd = _g(parts[0].dir) if initrd is None: raise ValueError("cannot find ONIE initrd on %s" % parts[0].dir) + self.onieDir = parts[0].dir self.log.debug("found ONIE initrd at %s", initrd) with InitrdContext(initrd=initrd, log=self.log) as self.ictx: self.initrd = initrd diff --git a/packages/base/all/vendor-config-onl/src/python/onl/mounts/__init__.py b/packages/base/all/vendor-config-onl/src/python/onl/mounts/__init__.py index 3c64c0eb..bf591df6 100755 --- a/packages/base/all/vendor-config-onl/src/python/onl/mounts/__init__.py +++ b/packages/base/all/vendor-config-onl/src/python/onl/mounts/__init__.py @@ -138,35 +138,53 @@ class OnlMountManager(object): def init(self, timeout=5): - for(k, v) in self.mdata['mounts'].iteritems(): - # - # Get the partition device for the given label. - # The timeout logic is here to handle waiting for the - # block devices to arrive at boot. - # - t = timeout - while t >= 0: - try: - v['device'] = subprocess.check_output("blkid -L %s" % k, shell=True).strip() - break - except subprocess.CalledProcessError: - self.logger.debug("Block label %s does not yet exist..." % k) - time.sleep(1) - t -= 1 + now = time.time() + future = now + timeout - if 'device' not in v: - self.logger.error("Timeout waiting for block label %s after %d seconds." % (k, timeout)) - self.missing = k + md = self.mdata['mounts'] + optional = set(x for x in md if md[x].get('optional', False)) + pending = set(x for x in md if not md[x].get('optional', False)) + + def _discover(k): + v = md[k] + lbl = v.get('label', k) + + try: + v['device'] = subprocess.check_output(('blkid', '-L', lbl,)).strip() + except subprocess.CalledProcessError: return False - # - # Make the mount point for future use. - # if not os.path.isdir(v['dir']): - self.logger.debug("Make directory '%s'..." % v['dir']) + self.logger.debug("Make directory '%s'...", v['dir']) os.makedirs(v['dir']) - self.logger.debug("%s @ %s" % (k, v['device'])) + self.logger.debug("%s @ %s", k, v['dir']) + return True + + while True: + + now = time.time() + if now > future: + break + + pending_ = pending + pending = [k for k in pending_ if not _discover(k)] + optional_ = optional + optional = [k for k in optional_ if not _discover(k)] + + if not pending: break + if pending != pending_: continue + if optional != optional_: continue + + self.logger.debug("Still waiting for block devices: %s", + " ".join(pending+optional)) + time.sleep(0.25) + + if pending: + for k in pending+optional: + self.logger.error("Timeout waiting for block label %s after %d seconds.", k, timeout) + + # ignore the any optional labels that were not found def __fsck(self, label, device): self.logger.info("Running fsck on %s [ %s ]..." % (label, device)) @@ -202,20 +220,37 @@ class OnlMountManager(object): raise ValueError("invalid labels argument.") if 'all' in labels: - labels = filter(lambda l: l != 'all', labels) + self.mdata['mounts'].keys() + labels = list(labels) + labels.remove('all') + labels = labels + self.mdata['mounts'].keys() + + def _f(label): + """skip labels that do not resolve to a block device (ideally, optional ones)""" + mpt = self.mdata['mounts'][label] + dev = mpt.get('device', None) + opt = mpt.get('optional', False) + if dev: return True + if not opt: return True + return False rv = [] for l in list(set(labels)): - if self.__label_entry("ONL-%s" % l.upper(), False): - rv.append("ONL-%s" % l.upper()) - elif self.__label_entry(l.upper(), False): - rv.append(l.upper()) - elif self.__label_entry(l): - rv.append(l) - else: - pass - return rv; + lbl = "ONL-%s" % l.upper() + if self.__label_entry(lbl, False) and _f(lbl): + rv.append("ONL-%s" % l.upper()) + continue + + lbl = l.upper() + if self.__label_entry(lbl, False) and _f(lbl): + rv.append(l.upper()) + continue + + lbl = l + if self.__label_entry(lbl) and _f(lbl): + rv.append(l) + + return rv def fsck(self, labels, force=False): labels = self.validate_labels(labels)