Merge pull request #86 from carlroth/master

Support persistent, directory-based installs
This commit is contained in:
Jeffrey Townsend
2016-07-26 14:33:08 -07:00
committed by GitHub
49 changed files with 1156 additions and 207 deletions

View File

@@ -81,7 +81,7 @@ docker-debug: docker_check
versions:
$(ONL)/tools/make-versions.py --import-file=$(ONL)/tools/onlvi --class-name=OnlVersionImplementation --output-dir $(ONL)/make --force
$(ONL)/tools/make-versions.py --import-file=$(ONL)/tools/onlvi --class-name=OnlVersionImplementation --output-dir $(ONL)/make/versions --force
relclean:
@find $(ONL)/RELEASE -name "ONL-*" -delete

View File

@@ -1 +0,0 @@
!include $ONL/builds/any/installer/APKG.yml ARCH=amd64

View File

@@ -0,0 +1 @@
include $(ONL)/make/pkg.mk

View File

@@ -0,0 +1,2 @@
!include $ONL/builds/any/installer/APKG.yml ARCH=amd64 BOOTMODE=installed

View File

@@ -0,0 +1,3 @@
BOOTMODE=INSTALLED
include $(ONL)/make/config.amd64.mk
include $(ONL)/builds/any/installer/grub/builds/Makefile

View File

@@ -0,0 +1,4 @@
NETDEV=ma1
NETAUTO=dhcp
BOOTMODE=INSTALLED
SWI=images::latest

View File

@@ -0,0 +1 @@
include $(ONL)/make/pkg.mk

View File

@@ -0,0 +1 @@
!include $ONL/builds/any/installer/APKG.yml ARCH=amd64 BOOTMODE=swi

View File

@@ -1,2 +1,3 @@
BOOTMODE=SWI
include $(ONL)/make/config.amd64.mk
include $(ONL)/builds/any/installer/grub/builds/Makefile

View File

@@ -12,7 +12,7 @@ common:
maintainer: support@bigswitch.com
packages:
- name: onl-installer
- name: onl-installer-$BOOTMODE
summary: Open Network Linux $ARCH Installer
files:

View File

@@ -2,9 +2,13 @@ ifndef ARCH
$(error $$ARCH not set)
endif
ifndef BOOTMODE
$(error $$BOOTMODE not set)
endif
# Hardcoded to match ONL File naming conventions.
include $(ONL)/make/versions/version-onl.mk
INSTALLER_NAME=$(FNAME_PRODUCT_VERSION)_ONL-OS_$(FNAME_BUILD_ID)_$(UARCH)_INSTALLER
INSTALLER_NAME=$(FNAME_PRODUCT_VERSION)_ONL-OS_$(FNAME_BUILD_ID)_$(UARCH)_$(BOOTMODE)_INSTALLER
__installer:
$(ONL)/tools/mkinstaller.py --arch $(ARCH) --boot-config boot-config --initrd onl-loader-initrd:$(ARCH) onl-loader-initrd-$(ARCH).cpio.gz --swi onl-swi:$(ARCH) --out $(INSTALLER_NAME)

View File

@@ -2,9 +2,13 @@ ifndef ARCH
$(error $$ARCH not set)
endif
ifndef BOOTMODE
$(error $$BOOTMODE not set)
endif
# Hardcoded to match ONL File naming conventions.
include $(ONL)/make/versions/version-onl.mk
INSTALLER_NAME=$(FNAME_PRODUCT_VERSION)_ONL-OS_$(FNAME_BUILD_ID)_$(UARCH)_INSTALLER
INSTALLER_NAME=$(FNAME_PRODUCT_VERSION)_ONL-OS_$(FNAME_BUILD_ID)_$(UARCH)_$(BOOTMODE)_INSTALLER
__installer:
$(ONL)/tools/mkinstaller.py --arch $(ARCH) --boot-config boot-config --fit onl-loader-fit:$(ARCH) onl-loader-fit.itb --swi onl-swi:$(ARCH) --out $(INSTALLER_NAME)

View File

@@ -1 +0,0 @@
!include $ONL/builds/any/installer/APKG.yml ARCH=armel

View File

@@ -0,0 +1 @@
include $(ONL)/make/pkg.mk

View File

@@ -0,0 +1 @@
!include $ONL/builds/any/installer/APKG.yml ARCH=armel BOOTMODE=installed

View File

@@ -0,0 +1,3 @@
BOOTMODE=INSTALLED
include $(ONL)/make/config.armel.mk
include $(ONL)/builds/any/installer/uboot/builds/Makefile

View File

@@ -0,0 +1,4 @@
NETDEV=ma1
NETAUTO=dhcp
BOOTMODE=INSTALLED
SWI=images::latest

View File

@@ -0,0 +1 @@
include $(ONL)/make/pkg.mk

View File

@@ -0,0 +1 @@
!include $ONL/builds/any/installer/APKG.yml ARCH=armel BOOTMODE=swi

View File

@@ -0,0 +1 @@
*INSTALLER

View File

@@ -1,2 +1,3 @@
BOOTMODE=SWI
include $(ONL)/make/config.armel.mk
include $(ONL)/builds/any/installer/uboot/builds/Makefile

View File

@@ -1 +0,0 @@
!include $ONL/builds/any/installer/APKG.yml ARCH=powerpc

View File

@@ -0,0 +1 @@
include $(ONL)/make/pkg.mk

View File

@@ -0,0 +1 @@
!include $ONL/builds/any/installer/APKG.yml ARCH=powerpc BOOTMODE=installed

View File

@@ -0,0 +1 @@
*INSTALLER

View File

@@ -0,0 +1,3 @@
BOOTMODE=INSTALLED
include $(ONL)/make/config.powerpc.mk
include $(ONL)/builds/any/installer/uboot/builds/Makefile

View File

@@ -0,0 +1,4 @@
NETDEV=ma1
NETAUTO=dhcp
BOOTMODE=INSTALLED
SWI=images::latest

View File

@@ -0,0 +1 @@
include $(ONL)/make/pkg.mk

View File

@@ -0,0 +1,2 @@
!include $ONL/builds/any/installer/APKG.yml ARCH=powerpc BOOTMODE=swi

View File

@@ -0,0 +1 @@
*INSTALLER

View File

@@ -1,2 +1,3 @@
BOOTMODE=SWI
include $(ONL)/make/config.powerpc.mk
include $(ONL)/builds/any/installer/uboot/builds/Makefile

View File

@@ -28,11 +28,15 @@
set -e
unset testonly help cache
CR="
"
while [ "$1" ]; do
case "$1" in
-h|--help) help=1 ;;
-t|--testonly) testonly=1 ;;
--cache) shift; cache="$1" ;;
--rootfs) shift; rootfs="$1" ;;
*) break ;;
esac
shift
@@ -48,19 +52,30 @@ Usage: $0 [-h|--help] [-t|--testonly] [--rootfs ROOTFS] --cache LOCATION [SWI]
Loads and boots a software image (SWI). The load method depends on the
format of the SWI argument:
DEV:PATH
/mnt/onl/DEV/PATH
Loads a SWI file from local storage device DEV (e.g. flash).
http://[USER:PASSWORD@]SERVER[:PORT]/PATH
ftp://[USER:PASSWORD@]SERVER[:PORT]/PATH
ssh|scp://USER:PASSWORD[:PORT]@SERVER/PATH
tftp://SERVER[:PORT]/PATH
DEV:SWI-PATH
Mount /dev/DEV to find a SWI
LABEL-ISH:SWI-PATH
Mount a filesysem labeled LABEL to find a SWI
/mnt/DEV/SWI-PATH
/mnt/onl/LABEL-ISH/SWI-PATH
Loads a SWI file from local storage device (e.g. flash).
http://[USER:PASSWORD@]SERVER[:PORT]/SWI-PATH
ftp://[USER:PASSWORD@]SERVER[:PORT]/SWI-PATH
ssh|scp://USER:PASSWORD[:PORT]@SERVER/SWI-PATH
tftp://SERVER[:PORT]/SWI-PATH
Downloads a SWI file via HTTP, FTP, SSH or TFTP.
nfs://SERVER[:PORT]/PATH
If PATH is a file, mounts the parent directory via NFS and loads the
SWI file. If PATH is a directory, mounts the directory and loads
the SWI without unpacking (PATH must end with / and the directory
must contain an unpacked SWI).
nfs://SERVER[:PORT]/SWI-PATH
Mounts the parent directory via NFS and loads the SWI file.
nfs://SERVER[:PORT]/[ROOTFS-PATH/]
Mounts the parent directory via NFS and uses ROOTFS-PATH
as the unpacked SWI contents (path must end with '/')
dir:DEV[:/ROOTFS-PATH]
Mounts /dev/DEV to find a root filesystem
dir:LABEL-ISH[:/ROOTFS-PATH]
Mounts a fileystem labeled LABEL to find a root filesystem
dir:/mnt/DEV[/ROOTFS-PATH]
dir:/mnt/onl/LABEL-ISH[/ROOTFS-PATH]
Mounts a directory on a local storage device to find a root filesystem
EOF
exit 1
@@ -70,60 +85,30 @@ shift
[ ! "${testonly}" ] || set -x
unset swipath host port dir file dev user password
unset swipath host bhost port dir file dev user password scope
case "${SWI}" in
http:*|ftp:*)
echo "Downloading ${SWI}"
wget -O /tmp/swi0 "${SWI}"
mv /tmp/swi0 /tmp/swi
swipath=/tmp/swi
;;
scp:*|ssh:*)
echo "Downloading ${SWI}"
eval $(echo "${SWI}" | sed -n 's#\(scp\|ssh\)://\([^:]*\):\([^@]*\)@\([^/:]*\)\(:\([0-9]\+\)\)\?/\(.*\)#user="\2" password="\3" host="\4" port="\6" file="\7"#p')
[ "${port}" ] || port=22
DROPBEAR_PASSWORD="${password}" dbclient -y -p ${port} -l "${user}" "${host}" "cat /${file}" >/tmp/swi0
mv /tmp/swi0 /tmp/swi
swipath=/tmp/swi
;;
tftp:*)
echo "Downloading ${SWI}"
eval $(echo "${SWI}" | sed -n 's#tftp://\([^/:]*\|\[[^]/]*\]\)\(:\([0-9]\+\)\)\?/\(.*\)#host="\1" port="\3" file="\4"#p')
tftp -g -r "${file}" -l /tmp/swi0 "${host}" ${port}
mv /tmp/swi0 /tmp/swi
swipath=/tmp/swi
;;
nfs:*)
eval $(echo "${SWI}" | sed -n 's#nfs://\([^/:]*\|\[[^]/]*\]\)\(:\([0-9]\+\)\)\?\(.*\)/\(.*\)#host="\1" port="\3" dir="\4" file="\5"#p')
[ "${dir}" ] || dir=/
[ "${port}" ] || port=0
echo "Mounting nfs://${host}:${port}${dir}"
umount -l /tmp/nfs 2>/dev/null || :
mkdir -p /tmp/nfs
mount -t nfs -o "nolock,port=${port}" "${host}:${dir}" /tmp/nfs
if [ "${file}" ]; then
swipath="/tmp/nfs/${file}"
[ ! -d ${swipath} ] || { echo "${SWI} must be a SWI file (use ${SWI}/ for a SWI directory)"; exit 1; }
else
swipath=/tmp/nfs
nfs://*/|dir:*)
echo "Mounting ${SWI}"
swipath=$(swimount $SWI)
if [ "$rootfs" ]; then
[ -d "${swipath}/${rootfs}" ] || { echo "${SWI}${rootfs} must be an unpacked rootfs"; exit 1; }
mount -t nfs -o "nolock,port=${port}" "${host}:${dir}/${rootfs}" "${swipath}/${rootfs}"
fi
;;
http:*|ftp:*|scp://*|ssh://*|tftp://*|nfs://*)
echo "Downloading ${SWI}"
swipath=$(swiget $SWI)
;;
*)
# Parse dev:file or dev:/file or /mnt/onl/dev/file
parselocal='s#\(\([^:/]*\):/\?\|/mnt/onl/\([^/]*\)/\)\?\(.*\)#dev="\2\3" file="\4"#p'
eval $(echo "${SWI}" | sed -n "${parselocal}")
if [ "${dev}" ] ; then
# Wait for /mnt/dev to show up
:
else
# Assume file is relative, parse absolutified file
eval $(realpath "${file}" | sed -n "${parselocal}")
SWI="${dev}:${file}"
fi
swipath="/mnt/onl/${dev}/${file}"
[ -f "${swipath}" ] || { echo "${SWI} not found or not a file"; exit 1; }
echo "Locating ${SWI}"
swipath=$(swiget $SWI)
;;
esac
case "$SWI" in
*::latest)
swistamp=${SWI%:latest}${swipath##*/}
;;
*)
swistamp=$SWI
;;
esac
@@ -143,7 +128,38 @@ if [ -n "$cache" ]; then
python /bin/swicache.py "${swipath}" "${cache}"
fi
. /lib/boot1
if [ "$testonly" ]; then
echo "swipath=$swipath rootfs=$rootfs"
echo "Stop here"
exit 0
fi
if [ -d "${swipath}" ]; then
# rootfs is a directory
mkdir -p /newroot
umount -l /newroot 2>/dev/null || :
mount --bind "${swipath}/${rootfs}" /newroot
else
swiprep --overlay "${swipath}${rootfs}" --unmount --swiref "$swistamp" /newroot
swiprep --record "${swipath}${rootfs}" --swiref "$swistamp" /newroot
fi
#
# The file /lib/boot-custom can be provided by customized builds to
# add functionality before the root is switched.
#
if [ -f /lib/boot-custom ]; then
. /lib/boot-custom
fi
echo "Switching rootfs" # limit 16 chars since serial buffer is not flushed
kill -QUIT 1 # exec /bin/switchroot as PID 1
sleep 30
echo "Boot failed"
exit 1
# Local variables:
# mode: sh
# sh-basic-offset: 4
# End:

View File

@@ -0,0 +1,339 @@
#!/usr/bin/python
"""swiget
Retrieve/resolve a SWI file to the local filesystem path.
"""
import sys, os
import time
import urllib
import subprocess
import shutil
import logging
import tempfile
import re
import json
import zipfile
import onl.install.InstallUtils
MountContext = onl.install.InstallUtils.MountContext
BlkidParser = onl.install.InstallUtils.BlkidParser
ProcMountsParser = onl.install.InstallUtils.ProcMountsParser
import onl.mounts
OnlMountManager = onl.mounts.OnlMountManager
import onl.network
HostInfo = onl.network.HostInfo
SWI_TIMESTAMP_FMT = "%Y-%m-%d.%H:%M"
SWI_TIMESTAMP_RE = re.compile(r"""
[0-9][0-9][0-9][0-9] [-] [0-9][0-9] [-] [0-9][0-9]
[.]
[0-9][0-9] [:] [0-9][0-9]
""", re.VERBOSE)
SWI_FNAME_TIMESTAMP_FMT = "%Y-%m-%d.%H%M"
SWI_FNAME_TIMESTAMP_RE = re.compile(r"""
[0-9][0-9][0-9][0-9] [-] [0-9][0-9] [-] [0-9][0-9]
[.]
[0-9][0-9] [0-9][0-9]
""", re.VERBOSE)
def versionSortKey(vstr):
m = SWI_TIMESTAMP_RE.search(vstr)
if m is not None:
try:
return time.mktime(time.strptime(m.group(0), SWI_TIMESTAMP_FMT))
except ValueError:
pass
m = SWI_FNAME_TIMESTAMP_RE.search(vstr)
if m is not None:
try:
return time.mktime(time.strptime(m.group(0), SWI_FNAME_TIMESTAMP_FMT))
except ValueError:
pass
return None
def manifestSortKey(mdata):
mdata = mdata.get('version', {})
vstr = mdata.get('BUILD_TIMESTAMP', None)
if vstr is not None:
try:
return time.mktime(time.strptime(vstr, SWI_TIMESTAMP_FMT))
except ValueError:
pass
vstr = mdata.get('FNAME_BUILD_TIMESTAMP', None)
if vstr is not None:
try:
return time.mktime(time.strptime(vstr, SWI_FNAME_TIMESTAMP_FMT))
except ValueError:
pass
return None
def swiSortKey(swiPath):
try:
zf = zipfile.ZipFile(swiPath, "r")
except zipfile.BadZipFile:
return os.path.getmtime(swiPath)
try:
fd = zf.open("manifest.json", "r")
except KeyError:
fd = None
if fd is not None:
data = json.load(fd)
fd.close()
else:
data = {}
ts = manifestSortKey(data)
if ts is not None: return ts
try:
fd = zf.open("version", "r")
except KeyError:
fd = None
if fd is not None:
vstr = fd.read()
fd.close()
else:
vstr = ""
ts = versionSortKey(vstr)
if ts is not None: return ts
ts = versionSortKey(swiPath)
if ts is not None: return ts
# maybe the timestamp is embedded into the filename
# else, use the filesytem mtime, which is less reliable
return os.path.getmtime(swiPath)
class Runner(onl.install.InstallUtils.SubprocessMixin):
def __init__(self, log):
self.log = log
self.nextUpdate = None
def reporthook(self, blocks, bsz, sz):
if time.time() < self.nextUpdate: return
self.nextUpdate = time.time() + 0.25
if sz:
pct = blocks * bsz * 100 / sz
sys.stderr.write("downloaded %d%% ...\r" % pct)
else:
icon = "|/-\\"[blocks % 4]
sys.stderr.write("downloading ... %s\r" % icon)
def get(self, SWI):
# XXX lots of fixups required here for proper IPv6 support
if SWI.startswith('http://') or SWI.startswith('ftp://'):
self.log.info("retrieving %s via HTTP/FTP", SWI)
dst = tempfile.mktemp(prefix="swiget-", suffix=".swi")
if os.isatty(sys.stdout.fileno()):
dst, headers = urllib.urlretrieve(SWI, dst, self.reporthook)
else:
dst, headers = urllib.urlretrieve(SWI, dst)
sys.stderr.write("\n")
return dst
if SWI.startswith('scp://') or SWI.startswith('ssh://'):
buf = SWI[6:]
h, sep, r = buf.partition('/')
if not sep:
self.log.error("invalid SWI %s", SWI)
return None
hinfo = HostInfo.fromString(h)
rcmd = "cat /%s" % r
cmd = ['dbclient', '-y', hinfo.host, rcmd,]
if hinfo.port is not None:
cmd[2:2] = ['-p', str(hinfo.port),]
if hinfo.user is not None:
cmd[2:2] = ['-l', hinfo.user,]
env = {}
env.update(os.environ)
if hinfo.password is not None:
env['DROPBEAR_PASSWORD'] = hinfo.password
dst = tempfile.mktemp(prefix="swiget-", suffix=".swi")
with open(dst, "w") as fd:
self.check_call(cmd, stdout=fd, env=env)
return dst
if SWI.startswith('tftp://'):
buf = SWI[7:]
h, sep, r = buf.partition('/')
if not sep:
self.log.error("invalid SWI %s", SWI)
return None
hinfo = HostInfo.fromString(h)
port = hinfo.port or 69
dst = tempfile.mktemp(prefix="swiget-", suffix=".swi")
cmd = ('tftp', '-g', '-r', r, '-l', dst, hinfo.host, str(hinfo.port),)
self.check_call(cmd)
return dst
if SWI.startswith('nfs://'):
buf = SWI[6:]
h, sep, r = buf.partition('/')
if not sep:
self.log.error("invalid SWI %s", SWI)
return None
hinfo = HostInfo.fromString(h)
if '/' in r:
root, base = os.path.split(r)
else:
root, base = '/', r
src = "%s:/%s" % (hinfo.bhost, root,)
mpt = self.mkdtemp(prefix="swiget-nfs-", suffix=".d")
cmd = ['mount', '-t', 'nfs', src, mpt,]
if hinfo.port is not None:
cmd[3:3] = ['-o', "ro,nolock,port=%s" % hinfo.port,]
else:
cmd[3:3] = ['-o', 'ro,nolock',]
self.check_call(cmd)
dst = tempfile.mktemp(prefix="swiget-", suffix=".swi")
try:
src = "%s/%s" % (mpt, base,)
self.copy2(src, dst)
except:
if os.path.exists(dst):
self.unlink(dst)
raise
finally:
self.check_call(('umount', mpt,))
self.rmdir(mpt)
return dst
blkid = BlkidParser(log=self.log)
if ':' in SWI:
devspec, sep, r = SWI.partition(':')
p = "/dev/%s" % devspec
if os.path.exists(p):
return self.blockdevCopy(p, r)
try:
part = blkid[devspec]
except IndexError:
part = None
if part is not None:
return self.blockdevCopy(part.device, r)
mm = OnlMountManager("/etc/mtab.yml", self.log)
label = mpt = None
for k, v in mm.mdata['mounts'].items():
if v['dir'].endswith('/' + devspec):
label = k
mpt = v['dir']
break
if label is not None:
try:
part = blkid[label]
except IndexError:
part = None
if part is not None:
return self.blockdevCopy(part.device, r, dir=mpt)
self.log.error("cannot find device specifier for %s", SWI)
return None
# local file
if SWI.startswith('/') and os.path.exists(SWI):
return SWI
# possibly unmounted dirctory
if SWI.startswith('/') and not os.path.exists(SWI):
mm = OnlMountManager("/etc/mtab.yml", self.log)
label = mpt = path = None
for k, v in mm.mdata['mounts'].items():
if SWI.startswith(v['dir'] + '/'):
label = k
mpt = v['dir']
path = SWI[len(mpt)+1:]
break
if label is not None:
try:
part = blkid[label]
except IndexError:
part = None
if part is not None:
return self.blockdevCopy(part.device, path, dir=mpt)
self.log.error("invalid SWI %s", SWI)
return None
def blockdevCopy(self, dev, src, dir=None):
def latest(d):
l = [x for x in os.listdir(d) if x.endswith('.swi')]
l = [os.path.join(d, x) for x in l]
l.sort(key=swiSortKey)
return l[-1]
pm = ProcMountsParser()
parts = [x for x in pm.mounts if x.device == dev]
if parts:
if src == ':latest':
dst = latest(parts[0].dir)
self.log.info("found 'latest' swi %s", dst)
else:
dst = os.path.join(parts[0].dir, src)
if not os.path.exists(dst):
self.log.error("missing SWI: %s", dst)
return None
return dst
with MountContext(device=dev, log=self.log) as ctx:
if src == ':latest':
dst = latest(ctx.dir)
self.log.info("found 'latest' swi %s", dst)
else:
dst = os.path.join(ctx.dir, src)
if not os.path.exists(dst):
self.log.error("missing SWI: %s:%s", dev, src)
return None
# move to its proper location as per mtab
# XXX perms may not be right here
if dir is not None:
self.check_call(('mount', '--move', ctx.dir, dir,))
ctx.mounted = False
return os.path.join(dir, src)
ctx.mounted = False
ctx.hostDir = None
return dst
@classmethod
def main(cls):
SWI = sys.argv[1]
logging.basicConfig()
logger = logging.getLogger("swiget")
logger.setLevel(logging.DEBUG)
r = cls(logger)
dst = r.get(SWI)
if dst is None:
sys.exit(1)
sys.stdout.write(dst + "\n")
sys.exit(0)
main = Runner.main
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,184 @@
#!/usr/bin/python
"""swimount
Mount a directory specified in $SWI.
"""
import os, sys
import shutil
import logging
import onl.install.InstallUtils
BlkidParser = onl.install.InstallUtils.BlkidParser
import onl.mounts
MountContext = onl.install.InstallUtils.MountContext
OnlMountManager = onl.mounts.OnlMountManager
ProcMountsParser = onl.install.InstallUtils.ProcMountsParser
import onl.network
HostInfo = onl.network.HostInfo
class Runner(onl.install.InstallUtils.SubprocessMixin):
def __init__(self, log):
self.log = log
self.blkid = BlkidParser(log=self.log)
def mount(self, SWI):
if SWI.startswith('nfs://'):
buf = SWI[6:]
buf = buf.rstrip('/')
h, sep, r = buf.partition('/')
if not sep:
self.log.error("invalid SWI %s", SWI)
return None
hinfo = HostInfo.fromString(h)
if not r:
r = '/'
else:
r = '/' + r
src = "%s:%s" % (hinfo.bhost, r,)
mpt = self.mkdtemp(prefix="swimount-nfs-", suffix=".d")
cmd = ['mount', '-t', 'nfs', src, mpt,]
if hinfo.port is not None:
cmd[3:3] = ['-o', "nolock,port=%s" % hinfo.port,]
else:
cmd[3:3] = ['-o', 'nolock',]
self.check_call(cmd)
return mpt
# local path specifier
if not SWI.startswith('dir:'):
self.log.error("invalid SWI %s", SWI)
return None
SWI = SWI[4:]
if ':' in SWI:
devspec, sep, r = SWI.partition(':')
return self.devspecMount(devspec, r)
if not SWI.startswith('/'):
return self.devspecMount(SWI, '/')
if not os.path.isdir(SWI):
self.log.error("invalid SWI %s (not a directory)", SWI)
return None
# local directory, but actually mounted
if os.stat('/').st_dev != os.stat(SWI).st_dev:
return SWI
# possibly unmounted dirctory
mm = OnlMountManager("/etc/mtab.yml", self.log)
label = mpt = path = None
for k, v in mm.mdata['mounts'].items():
if SWI == v['dir']:
label = k
mpt = v['dir']
path = '/'
break
if SWI.startswith(v['dir'] + '/'):
label = k
mpt = v['dir']
path = SWI[len(mpt)+1:]
break
if label is not None:
try:
part = self.blkid[label]
except IndexError:
part = None
if part is not None:
return self.blockdevMount(part.device, path, dir=mpt)
self.log.error("invalid SWI %s", SWI)
return None
def devspecMount(self, devspec, path):
"""Find using a device specifier."""
p = "/dev/%s" % devspec
if os.path.exists(p):
return self.blockdevMount(p, r)
try:
part = self.blkid[devspec]
except IndexError:
part = None
if part is not None:
return self.blockdevMount(part.device, r)
mm = OnlMountManager("/etc/mtab.yml", self.log)
label = mpt = None
for k, v in mm.mdata['mounts'].items():
if v['dir'].endswith('/' + devspec):
label = k
mpt = v['dir']
break
if label is not None:
try:
part = self.blkid[label]
except IndexError:
part = None
if part is not None:
return self.blockdevMount(part.device, path, dir=mpt)
self.log.error("cannot find device specifier for %s", SWI)
return None
def blockdevMount(self, dev, src, dir=None):
pm = ProcMountsParser()
parts = [x for x in pm.mounts if x.device == dev]
if parts:
dst = parts[0].dir
if src and src != '/': dst = os.path.join(dst, src)
if not os.path.exists(dst):
self.log.error("missing SWI: %s", dst)
return None
self.check_call(('mount', '-o', 'rw,remount', dst,))
return dst
with MountContext(device=dev, log=self.log) as ctx:
dst = ctx.dir
if src and src != '/': dst = os.path.join(dst, src)
if not os.path.exists(dst):
self.log.error("missing SWI: %s:%s", dev, src)
return None
# move to its proper location as per mtab
# XXX perms may not be right here
if dir is not None:
self.check_call(('mount', '-o', 'rw,remount', ctx.dir,))
self.check_call(('mount', '--move', ctx.dir, dir,))
ctx.mounted = False
dst = dir
if src and src != '/': dst = os.path.join(dst, src)
return dst
ctx.mounted = False
ctx.hostDir = None
return dst
@classmethod
def main(cls):
SWI = sys.argv[1]
logging.basicConfig()
logger = logging.getLogger("swimount")
logger.setLevel(logging.DEBUG)
r = cls(logger)
dst = r.mount(SWI)
if dst is None:
sys.exit(1)
sys.stdout.write(dst + "\n")
sys.exit(0)
main = Runner.main
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,246 @@
#!/bin/sh
#
######################################################################
#
# swiprep
#
# unpack a SWI to a directory in preparation for boot
#
######################################################################
swipath=
destdir=
swiref=
mode_install=
mode_overlay=
mode_record=
flag_unmount=
while test $# -gt 0; do
case "$1" in
--install)
mode_install=1
mode_overlay=
shift
continue
;;
--overlay)
mode_install=
mode_overlay=1
shift
continue
;;
--record)
shift
mode_record=1
continue
;;
--unmount)
shift
flag_unmount=1
continue
;;
--swiref)
shift
swiref=$1
shift
continue
;;
-*)
echo "*** invalid option $1" 1>&2
exit 1
;;
esac
if test "$swipath"; then
:
else
swipath=$1
shift
continue
fi
if test "$destdir"; then
:
else
destdir=$1
shift
continue
fi
break
done
if test "$swipath"; then
:
else
echo "*** missing swipath" 1>&2
exit 1
fi
if test "$destdir"; then
:
else
echo "*** missing destdir" 1>&2
exit 1
fi
case "${mode_install}:${mode_overlay}:${mode_record}" in
::)
echo "*** missing --install or --overlay or --record" 1>&2
exit 1
;;
esac
if test $# -gt 0; then
echo "*** extra arguments" 1>&2
exit 1
fi
if test "${mode_install}${mode_overlay}"; then
mkdir -p "$destdir"
rm -fr "$destdir"/* "$destdir"/.??*
if grep -q " $destdir " /proc/mounts; then
mkdir "$destdir/lost+found"
fi
fi
if test "$flag_unmount"; then
umount -l "$destdir" 2>/dev/null || :
if test "$mode_overlay"; then
mkdir -p "${destdir}.lower" "${destdir}.upper"
umount -l "${destdir}.lower" 2>/dev/null || :
umount -l "${destdir}.upper" 2>/dev/null || :
fi
fi
if test "$mode_install"; then
workdir=$(mktemp -d "$destdir"/swiprep-XXXXXX)
else
workdir=$(mktemp -t -d swiprep-XXXXXX)
fi
if test "$mode_record"; then
echo "recording SWI $swipath --> $workdir"
else
echo "extracting SWI $swipath --> $workdir"
fi
do_cleanup() {
cd /
rm -fr $workdir
}
trap "do_cleanup" 0 1
ARCH_LIST=
case $(uname -m) in
ppc)
ARCH_LIST="ppc powerpc"
;;
x86_64)
ARCH_LIST="x86_64 amd64"
;;
armv7l)
ARCH_LIST="armel"
;;
*)
q;;
esac
if test "${mode_install}${mode_overlay}"; then
for arch in $ARCH_LIST; do
unzip -pq "$swipath" "rootfs-${arch}.sqsh" > "$workdir/rootfs.sqsh"
if test -s "$workdir/rootfs.sqsh"; then break; fi
done
if test ! -s "$workdir/rootfs.sqsh"; then
echo "*** cannot find a valid rootfs" 1>&2
exit 1
fi
fi
if test "$mode_install"; then
echo "extracting rootfs $workdir/rootfs.sqsh --> $destdir"
unsquashfs -f -d "$destdir" "$workdir/rootfs.sqsh"
if test ! -f "$destdir/lib/vendor-config/onl/install/lib.sh"; then
echo "*** invalid squashfs contents" 1>&2
exit 1
fi
fi
if test "$mode_overlay"; then
# keep the squashfs file around
mv $workdir/rootfs.sqsh /tmp/.rootfs
if grep -q overlayfs /proc/filesystems; then
mount -t squashfs -o loop /tmp/.rootfs "${destdir}.lower"
mount -t tmpfs -o size=15%,mode=0755 none "${destdir}.upper"
mount -t overlayfs -o "lowerdir=${destdir}.lower,upperdir=${destdir}.upper" none "$destdir"
elif grep -q overlay /proc/filesystems; then
mount -t squashfs -o loop /tmp/.rootfs "${destdir}.lower"
mount -t tmpfs -o size=15%,mode=0755 none "${destdir}.upper"
mkdir "${destdir}.upper/upper"
mkdir "${destdir}.upper/work"
mount -t overlay "-olowerdir=${destdir}.lower,upperdir=${destdir}.upper/upper,workdir=${destdir}.upper/work" overlay "$destdir"
else
echo "OverlayFS not found in kernel"
fi
fi
rm -f $workdir/rootfs.sqsh
if test "${mode_install}${mode_overlay}"; then
# Install any SWI data packages.
unzip -pq "$swipath" swi-data.tar.gz > "$workdir/swi-data.tar.gz"
if test -s "$workdir/swi-data.tar.gz"; then
echo "installing SWI data into /boot..."
tar -C "$destdir/boot" -xzf "$workdir/swi-data.tar.gz"
fi
mkdir -p "$destdir/etc/onl"
for thing in /etc/onl/*; do
if test $thing != "sysconfig"; then
cp -R $thing "$destdir/etc/onl/."
fi
done
if [ -f /etc/fw_env.config ]; then
cat /etc/fw_env.config > "$destdir/etc/fw_env.config"
fi
fi
if test "$mode_record"; then
vdestdir="$destdir/etc/onl/upgrade/swi"
else
vdestdir="$destdir/etc/onl/rootfs"
fi
# If there are SWI version file(s) put them in /etc/onl
if test ! -f "$vdestdir/version"; then
unzip -pq "$swipath" version > "$workdir/version"
if test -s "$workdir/version"; then
mkdir -p "$vdestdir"
cp "$workdir/version" "$vdestdir/version"
fi
fi
if test ! -f "$vdestdir/manifest.json"; then
unzip -pq "$swipath" manifest.json > "$workdir/manifest.json"
if test -s "$workdir/manifest.json"; then
mkdir -p "$vdestdir"
cp "$workdir/manifest.json" "$vdestdir/manifest.json"
fi
fi
if test -z "$swiref"; then
swiref=$swipath
fi
if test "$mode_record"; then
mkdir -p "$destdir/etc/onl/upgrade/swi"
echo "$swiref" > "$destdir/etc/onl/upgrade/swi/SWI"
else
echo "$swiref" > "$destdir/etc/onl/SWI"
fi
exit 0
# Local variables:
# mode: sh
# sh-indentation: 2
# End:

View File

@@ -0,0 +1,87 @@
############################################################
#
# Bootmode: installed
#
# Boot the installed rootfs from the INSTALLED BOOTPARAM value.
#
############################################################
. /lib/msgs
. /etc/onl/BOOTPARAMS
# make sure /mnt/onl/data exists
if [ ! -d /mnt/onl/data ]; then
msg_error "Missing /mnt/onl/data, disk boot cannot continue"
exit 200
fi
# make sure it's mounted as per mtab.yml
d1=$(stat -f -c '%d' /mnt/onl)
d2=$(stat -f -c '%d' /mnt/onl/data)
if [ "$d1" -eq "$d2" ]; then
msg_error "Unmounted /mnt/onl/data, disk boot cannot continue"
exit 200
fi
do_unpack=
swiget_workdir=
swi_workdir=
do_cleanup() {
cd /
rm -fr $swiget_workdir $swi_workdir
}
trap "do_cleanup" 0 1
case "$SWI" in
""|dir:*|nfs://*/)
msg_info "*** missing SWI file, will attempt to boot existing image"
if [ ! -s /mnt/onl/data/etc/onl/SWI ]; then
msg_error "Un-populated /mnt/onl/data, cannot continue"
exit 200
fi
;;
*)
# resolve this SWI as a file and unpack it
swiget_workdir=${TMPDIR-"/tmp"}
swiget_workdir=$(mktemp -d $swiget_workdir/bootmodes-XXXXXX)
TMPDIR=$swiget_workdir swipath=$(swiget $SWI)
# do some sort of test to see if it's populated
if [ ! -s /mnt/onl/data/etc/onl/SWI ]; then
msg_info "Un-populated /mnt/onl/data, will (re-)image"
do_unpack=1
else
msg_info "Found valid (current) image at /mnt/onl/data"
fi
;;
esac
case "$SWI" in
*::latest)
swistamp=${SWI%:latest}${swipath##*/}
;;
*)
swistamp=$SWI
;;
esac
sed -i -e '/^SWI=/d' /etc/onl/BOOTPARAMS
echo "SWI=dir:data:/" >> /etc/onl/BOOTPARAMS
if [ "$do_unpack" ]; then
swiprep --install "$swipath" --swiref "$swistamp" /mnt/onl/data
fi
swiprep --record "$swipath" --swiref "$swistamp" /mnt/onl/data
trap - 0 1
do_cleanup
. /bootmodes/swi
# Local variables:
# mode: sh
# sh-basic-offset: 4
# End:

View File

@@ -14,16 +14,6 @@ if [ ! "${SWI}" ]; then
exit 200
fi
if [ "${SWI}" = "images::latest" ]; then
# Boot the latest (by mtime) SWI in the images partition.
SWI=`ls /mnt/onl/images/*.swi -t | head -n1`
if [ -z "${SWI}" ]; then
msg_error "No SWI available in /mnt/onl/images. SWI booting cannot continue."
exit 200
fi
fi
#
# The SWI setting can be a list of URLs
#

View File

@@ -1,123 +0,0 @@
# -*- sh -*-
############################################################
# <bsn.cl fy=2013 v=onl>
#
# Copyright 2013, 2014 BigSwitch Networks, Inc.
#
# Licensed under the Eclipse Public License, Version 1.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.eclipse.org/legal/epl-v10.html
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
# either express or implied. See the License for the specific
# language governing permissions and limitations under the
# License.
#
# </bsn.cl>
############################################################
#
# boot1
#
# Copies/mounts rootfs from swi and switches root
#
######################################################################
mkdir -p /newroot
umount -l /newroot 2>/dev/null || :
if [ -d "${swipath}" ]; then
# rootfs is a directory
mount --bind "${swipath}/${rootfs}" /newroot
else
# rootfs is a squashfs
echo "Booting: ${swipath}"
mkdir -p /newroot.lower /newroot.upper
umount -l /newroot.lower 2>/dev/null || :
umount -l /newroot.upper 2>/dev/null || :
rm -f /tmp/rootfs
uarch=`uname -m`
ARCH_LIST=
case $uarch in
ppc)
ARCH_LIST="ppc powerpc"
;;
x86_64)
ARCH_LIST="x86_64 amd64"
;;
armv7l)
ARCH_LIST="armel"
;;
*)
;;
esac
for arch in $ARCH_LIST; do
unzip -pq "${swipath}" "rootfs-$arch.sqsh" >/tmp/rootfs
[ ! -s /tmp/rootfs ] || break
done
if [ ! -s /tmp/rootfs ]; then
echo "${swipath} does not contain a rootfs image for the current architecture ($uarch). Booting cannot continue."
exit 1
fi
if grep -q overlayfs /proc/filesystems; then
mount -t squashfs -o loop /tmp/rootfs /newroot.lower
mount -t tmpfs -o size=15%,mode=0755 none /newroot.upper
mount -t overlayfs -o lowerdir=/newroot.lower,upperdir=/newroot.upper none /newroot
elif grep -q overlay /proc/filesystems; then
mount -t squashfs -o loop /tmp/rootfs /newroot.lower
mount -t tmpfs -o size=15%,mode=0755 none /newroot.upper
mkdir /newroot.upper/upper
mkdir /newroot.upper/work
mount -t overlay -olowerdir=/newroot.lower,upperdir=/newroot.upper/upper,workdir=/newroot.upper/work overlay /newroot
else
echo "OverlayFS not found in kernel"
fi
fi
mkdir -p /newroot/etc/onl
for thing in /etc/onl/*; do
if [ $thing != "sysconfig" ]; then
cp -R $thing /newroot/etc/onl
fi
done
if [ -f /etc/fw_env.config ]; then
cat /etc/fw_env.config >/newroot/etc/fw_env.config
fi
unzip -oq "${swipath}" swi-data.tar.gz -d /tmp
# Install any SWI data packages.
if [ -s /tmp/swi-data.tar.gz ]; then
echo "Installing SWI data into /boot..."
tar -C /newroot/boot -xzf /tmp/swi-data.tar.gz
fi
# If there is a SWI version file put it in /etc/onl/swi_version
unzip -oq "${swipath}" version -d /tmp
if [ -f /tmp/version ]; then
cp /tmp/version /newroot/etc/onl/swi_version
fi
#
# The file /lib/boot-custom can be provided by customized builds to
# add functionality before the root is switched.
#
if [ -f /lib/boot-custom ]; then
. /lib/boot-custom
fi
echo "Switching rootfs" # limit 16 chars since serial buffer is not flushed
kill -QUIT 1 # exec /bin/switchroot as PID 1
sleep 30
# Local variables:
# sh-basic-offset: 4
# End:

View File

@@ -0,0 +1,71 @@
#!/usr/bin/python
"""63.upgrade-swi
SWI upgrade hook
This is currently a place-holder, we don't actually have a mechanism
to upgrade the SWI, only a means to detect if a SWI upgrade is indicated.
"""
import os
import re
from onl.upgrade import ubase
class Swi_Upgrade(ubase.BaseUpgrade):
name="swi"
Name="SWI"
title="SWI Upgrade Check"
atype="A SWI"
current_version_key="Current SWI Version"
next_version_key="Next SWI Version"
def init_versions(self):
"""Scrape out the SWI versions.
Use recorded version manifest, or a bare version file.
XXX probably can also scrape it from the SWI filename,
but FNAME variants are not 1:1 translatible.
"""
self.current_version = self.next_version = None
data = self.load_json("/etc/onl/rootfs/manifest.json", default={}).get('version', {})
vstr = data.get('RELEASE_ID', None)
if self.next_version is None and vstr is not None:
self.next_version = vstr
if self.next_version is None and os.path.exists("/etc/onl/rootfs/version"):
with open("/etc/onl/rootfs/version") as fd:
self.next_version = fd.read().strip()
data = self.load_json("/etc/onl/upgrade/swi/manifest.json", default={}).get('version', {})
vstr = data.get('RELEASE_ID', None)
if self.current_version is None and vstr is not None:
self.current_version = vstr
if self.current_version is None and os.path.exists("/etc/onl/upgrade/swi/version"):
with open("/etc/onl/upgrade/swi/version") as fd:
self.next_version = fd.read().strip()
def summarize(self):
self.logger.info("Current SWI Version: %s" % self.current_version)
self.logger.info(" Next SWI Version: %s" % self.next_version)
self.logger.info("")
def upgrade_notes(self):
return """
* A single reboot will be required to complete this upgrade.
"""
def do_upgrade(self, forced=False):
"""Execute the upgradee.
SWI upgrade depends on a new SWI being available (DUH).
"""
self.logger.info("THIS STEP INTENTIONALLY LEFT BLANK")
if __name__ == '__main__':
Swi_Upgrade().main()

View File

@@ -260,7 +260,9 @@ class MountContext(SubprocessMixin):
cmd = ('umount', self.hostDir,)
self.check_call(cmd, vmode=self.V1)
self.rmdir(self.hostDir)
if self.hostDir is not None:
self.rmdir(self.hostDir)
return False
class BlkidEntry:

View File

@@ -41,12 +41,19 @@ class MountManager(object):
def mount(self, device, directory, mode='r', timeout=5):
mountargs = [ str(mode) ]
current = self.is_mounted(device, directory)
if current:
currentItems = [x for x in self.mounts.iteritems() if x[1]['dev'] == device]
if currentItems:
currentDirectory, current = currentItems[0]
if current['mode'] == mode:
# Already mounted as requested.
self.logger.debug("%s already mounted @ %s with mode %s. Doing nothing." % (device, directory, mode))
return True
elif directory != currentDirectory:
# Already mounted, at a different location (e.g. '/'), but not in the requested mode.
self.logger.debug("%s mounted @ %s (%s) with mode %s. It will be remounted %s.",
device, directory, currentDirectory, current['mode'], mode)
mountargs.append('remount')
directory = currentDirectory
else:
# Already mounted, but not in the requested mode.
self.logger.debug("%s mounted @ %s with mode %s. It will be remounted %s." % (device, directory, current['mode'], mode))

View File

@@ -1,3 +1,82 @@
"""__init__.py
"""
import subprocess
class HostInfo:
def __init__(self, host,
user=None, password=None,
port=None,
scope=None):
self.user = user
self.password = password
self.port = port
if scope is None and host.startswith('fe80:'):
cmd = ('ip', '-6', '-oneline', 'route', 'show',)
for line in subprocess.check_output(cmd).splitlines(False):
words = line.split()
if words[0].startswith('fe80:'):
scope = words[2]
break
if scope is not None:
self.host = host + '%' + scope
else:
self.host = host
# try to canonicalize the scope
# add scope to host
# set bhost to bracketed host
if ':' in host:
self.bhost = '[' + self.host + ']'
else:
self.bhost = self.host
@classmethod
def fromString(cls, hinfo):
buf = hinfo
l, sep, r = buf.partition('@')
if sep:
uinfo, buf = l, r
l, sep, r = uinfo.partition(':')
if sep:
u, p = l, r
else:
u, p = uinfo, None
else:
u = p = None
if not buf.startswith('['):
s = None
l, sep, r = buf.partition(':')
if sep:
h, p = l, int(r)
else:
h, p = buf, None
else:
l, sep, r = buf.partition(']')
if not sep:
raise ValueError("invalid host specifier %s" % hinfo)
h = l[1:]
if r and r.startswith(':'):
p = int(r[1:])
elif not r:
p = None
else:
raise ValueError("invalid host specifier %s" % hinfo)
i = h.find('%25')
if i > -1:
h, s = h[:i], h[i+3:]
else:
l, sep, r = h.partition('%')
if sep:
h, s = l, r
else:
s = None
return cls(h, port=p, user=u, password=p, scope=s)

View File

@@ -7,6 +7,7 @@ import subprocess
import json
import pprint
import yaml
import time
class OnlVersionsGenerator(object):
def __init__(self, ops):
@@ -20,8 +21,12 @@ class OnlVersionsGenerator(object):
raise ValueError("The import file %s does not contain a class named %s" % (ops.import_file, ops.class_name))
self.ops = ops
self.build_sha1 = subprocess.check_output("git rev-list HEAD -1", shell=True).strip()
self.build_timestamp =subprocess.check_output("date +%Y-%m-%d.%H:%M", shell=True).strip()
cmd = ('git', 'rev-list', 'HEAD', '-1',)
self.build_sha1 = subprocess.check_output(cmd).strip()
fmt = "%Y-%m-%d.%H:%M"
self.build_timestamp = time.strftime(fmt, time.localtime())
def generate_all(self):
for product in self.implementation.PRODUCTS: