Merge pull request #1 from jnealtowns/master

Synching to upstream

.gitignore

@@ -15,6 +15,7 @@ dependmodules.x
 
 # Local module manifest.
 .manifest.mk
+RELEASE/
 
 
 

REPO/.gitignore

@@ -1 +1,6 @@
 extracts
+*.deb
+Packages
+Packages.gz
+
+

REPO/packages/.gitignore

@@ -1,3 +0,0 @@
-*.deb
-Packages
-Packages.gz

builds/.gitignore

@@ -0,0 +1,2 @@
+*.swi
+*.md5sum

builds/Makefile

@@ -0,0 +1 @@
+include $(ONL)/make/pkg.mk

builds/amd64/Makefile

@@ -0,0 +1 @@
+include $(ONL)/make/pkg.mk

builds/amd64/rootfs/PKG.yml

@@ -0,0 +1 @@
+!include $ONL/builds/any/rootfs/APKG.yml ARCH=amd64

builds/amd64/rootfs/builds/Makefile

@@ -6,7 +6,7 @@ include $(ONL)/make/config.amd64.mk
 #
 export PLATFORM_LIST=$(shell onlpm --list-platforms --arch amd64 --csv )
 
-RFS_CONFIG := $(ONL)/packages/base/any/rootfs/standard/standard.yml
+RFS_CONFIG := $(ONL)/builds/any/rootfs/standard/standard.yml
 RFS_DIR := rootfs-amd64.d
 RFS_CPIO := rootfs-amd64.cpio.gz
 RFS_SQUASH := rootfs-amd64.sqsh

builds/amd64/swi/Makefile

@@ -0,0 +1 @@
+include $(ONL)/make/pkg.mk

builds/amd64/swi/PKG.yml

@@ -0,0 +1,2 @@
+!include $ONL/builds/any/swi/APKG.yml ARCH=amd64
+

builds/any/rootfs/APKG.yml

@@ -3,13 +3,12 @@ prerequisites:
 
 common:
   arch: $ARCH
-  version: 1.0.0
+  version: $FNAME_RELEASE_ID
   copyright: Copyright 2013, 2014, 2015 Big Switch Networks
   maintainer: support@bigswitch.com
 
 packages:
   - name: onl-rootfs
-    version: 1.0.0
     summary: Open Network Linux Root Filesystem
 
     files:

builds/any/rootfs/standard/standard.yml

@@ -8,8 +8,8 @@
 #
 ############################################################
 Packages: &Packages
-  - !include $ONL/packages/base/any/rootfs/common/common-packages.yml
-  - !include $ONL/packages/base/any/rootfs/common/${ARCH}-packages.yml
+  - !include $ONL/builds/any/rootfs/common/common-packages.yml
+  - !include $ONL/builds/any/rootfs/common/${ARCH}-packages.yml
   - !script  $ONL/tools/onl-platform-pkgs.py ${PLATFORM_LIST}
 
 Multistrap:
@@ -30,16 +30,16 @@ Multistrap:
     omitdebsrc: true
 
   Local-All:
-    source: ${ONL}/REPO/packages/all
+    source: ${ONL}/REPO/${DEBIAN_SUITE}/packages/all
     omitdebsrc: true
 
   Local-Arch:
-    source: ${ONL}/REPO/packages/${ARCH}
+    source: ${ONL}/REPO/${DEBIAN_SUITE}/packages/${ARCH}
     omitdebsrc: true
 
 Configure:
   overlays:
-    - ${ONL}/packages/base/any/rootfs/common/overlay
+    - ${ONL}/builds/any/rootfs/common/overlay
 
   update-rc.d:
     - 'initdev defaults'

builds/any/swi/APKG.yml

@@ -0,0 +1,35 @@
+
+prerequisites:
+  broken: true
+  packages: [ "onl-rootfs:$ARCH" ]
+
+common:
+  arch: $ARCH
+  version: $FNAME_RELEASE_ID
+  copyright: Copyright 2013, 2014, 2015 Big Switch Networks
+  maintainer: support@bigswitch.com
+
+packages:
+  - name: onl-swi
+    summary: Open Network Linux Switch Image (All $ARCH) Platforms)
+
+    files:
+      builds/*.swi          : $$PKG_INSTALL/
+      builds/*.md5sum       : $$PKG_INSTALL/
+
+    changelog:  Change changes changes.,
+
+
+release:
+  - builds/*.swi : $ARCH/
+  - builds/*.md5sum : $ARCH/
+
+
+
+
+
+
+
+
+
+

builds/powerpc/Makefile

@@ -0,0 +1 @@
+include $(ONL)/make/pkg.mk

builds/powerpc/rootfs/PKG.yml

@@ -0,0 +1 @@
+!include $ONL/builds/any/rootfs/APKG.yml ARCH=powerpc

builds/powerpc/rootfs/builds/Makefile

@@ -6,7 +6,7 @@ include $(ONL)/make/config.powerpc.mk
 #
 export PLATFORM_LIST=$(shell onlpm --list-platforms --arch powerpc --csv )
 
-RFS_CONFIG := $(ONL)/packages/base/any/rootfs/standard/standard.yml
+RFS_CONFIG := $(ONL)/builds/any/rootfs/standard/standard.yml
 RFS_DIR := rootfs-powerpc.d
 RFS_CPIO := rootfs-powerpc.cpio.gz
 RFS_SQUASH := rootfs-powerpc.sqsh

builds/powerpc/swi/Makefile

@@ -0,0 +1 @@
+include $(ONL)/make/pkg.mk

builds/powerpc/swi/PKG.yml

@@ -0,0 +1,2 @@
+!include $ONL/builds/any/swi/APKG.yml ARCH=powerpc
+

builds/swi/.gitignore

@@ -1,2 +0,0 @@
-*.swi
-*.swi.md5sum

make/swi.mk

@@ -17,6 +17,7 @@ FILENAMER := $(ONL)/tools/filenamer.py
 endif
 
 swi: FORCE
+	$(ONL_V_at) rm -rf *.swi*
 	$(ONL_V_at) $(ONLPM) $(LINK_OPTIONS)
 	$(ONL_V_at) zip -n rootfs-$(ARCH).sqsh - rootfs-$(ARCH).sqsh manifest.json > $@
 	$(ONL_V_at) mv $@ `$(FILENAMER) --type swi --manifest manifest.json $@`

packages/base/amd64/rootfs/PKG.yml

@@ -1 +0,0 @@
-!include $ONL/packages/base/any/rootfs/APKG.yml ARCH=amd64

packages/base/powerpc/rootfs/PKG.yml

@@ -1,2 +0,0 @@
-!include $ONL/packages/base/any/rootfs/APKG.yml ARCH=powerpc
-

packages/platforms/accton/powerpc-accton-as4600-54t/platform-config/r0/info

@@ -1,18 +0,0 @@
-** Package Group: /home/jeff/ws/branches/ONL-NEXT-GEN/packages/platforms/accton/powerpc-accton-as4600-54t/platform-config/r0/PKG.yml
-packages:
-- arch: powerpc
-  changelog: Changes
-  copyright: Copyright 2013, 2014, 2015 Big Switch Networks
-  depends: onl-platform-config-accton
-  files:
-    src/lib: /lib/platform-config/powerpc-accton-as4600-54t-r0/onl
-    src/python: /usr/lib/python2.7/dist-packages/onl/platform/
-  maintainer: support@bigswitch.com
-  name: onl-platform-config-powerpc-accton-as4600-54t-r0
-  summary: ONL Platform Configuration Package for the powerpc-accton-as4600-54t-r0
-  version: 1.0.0
-prerequisites:
-  packages:
-  - onl-vendor-config-accton:all
-
-

setup.env

@@ -10,7 +10,7 @@
 export ONL=$( cd "$(dirname "${BASH_SOURCE[0]}" )" && pwd)
 
 # The ONL package dir is here:
-export ONLPM_OPTION_PACKAGEDIRS="$ONL/packages"
+export ONLPM_OPTION_PACKAGEDIRS="$ONL/packages:$ONL/builds"
 
 # The ONL repo dir is here:
 export ONLPM_OPTION_REPO="$ONL/REPO"
@@ -22,16 +22,20 @@ export PATH="$ONL/tools/scripts:$ONL/tools:$PATH"
 # Default parallel build settings
 export MAKEFLAGS="-j64"
 
-# These submodules are required for almost everything.
-$ONL/tools/submodules.py $ONL sm/infra
-$ONL/tools/submodules.py $ONL sm/bigcode
-
 # Version files
 $ONL/tools/make-versions.py --import-file=$ONL/tools/onlvi --class-name=OnlVersionImplementation --output-dir $ONL/make
 
+# Make version values available in the package environment
+export ONLPM_OPTION_INCLUDE_ENV_JSON="$ONL/make/version-onl.json"
+
 #
 # buildroot download mirror. We suggest you setup a local repository containing these contents for faster local builds.
 #
 export BUILDROOTMIRROR=${BUILDROOTMIRROR:-"http://buildroot.hw.bigswitch.com/dl"}
 
+# These submodules are required for almost everything.
+$ONL/tools/submodules.py $ONL sm/infra
+$ONL/tools/submodules.py $ONL sm/bigcode
+
+
 

tools/onlpm.py

@@ -19,6 +19,10 @@ import onlyaml
 import onlu
 from string import Template
 import re
+import json
+import lsb_release
+
+g_dist_codename = lsb_release.get_distro_information().get('CODENAME')
 
 logger = onlu.init_logging('onlpm')
 
@@ -200,58 +204,12 @@ class OnlPackage(object):
         return True
 
 
-
-    def filepath(self, filename):
-        """Return the file path for a given package file."""
-        p = None
-        if os.path.isabs(filename):
-            p = filename
-        else:
-            p = os.path.join(self.dir, filename)
-        logger.debug("package file: %s" % p)
-        return p
-
-
     def _validate_files(self):
         """Validate the existence of the required input files for the current package."""
-
-        files = []
-        if type(self.pkg['files']) is dict:
-            for (s,d) in self.pkg['files'].iteritems():
-                files.append((s,d))
-        elif type(self.pkg['files']) is list:
-            for e in self.pkg['files']:
-                if type(e) is dict:
-                    for (s,d) in e.iteritems():
-                        files.append((s,d))
-                elif type(e) in [ list, tuple ]:
-                    if len(e) != 2:
-                        raise OnlPackageError("Too many filenames: '%s'" % (e))
-                    else:
-                        files.append((e[0], e[1]))
-                else:
-                    raise OnlPackageError("File entry '%s' is invalid." % (e))
-
-        #
-        # Make sure all source files exist.
-        #
-        for f in files:
-            fp = self.filepath(f[0])
-            if not os.path.exists(fp):
-                raise OnlPackageError("Source file or directory '%s' does not exist." % fp)
-
-        #
-        # Perform destination-file substitutions.
-        #
-        flist = []
-        for f in files:
-            s = Template(f[1])
-            dst = s.substitute(dict(PKG=self.pkg['name'], PKG_INSTALL='/usr/share/onl/packages/%s/%s' % (self.pkg['arch'], self.pkg['name'])))
-            flist.append((f[0], dst))
-
-        self.pkg['files'] = flist
-
-
+        self.pkg['files'] = onlu.validate_src_dst_file_tuples(self.dir,
+                                                              self.pkg['files'],
+                                                              dict(PKG=self.pkg['name'], PKG_INSTALL='/usr/share/onl/packages/%s/%s' % (self.pkg['arch'], self.pkg['name'])),
+                                                              OnlPackageError)
     def _validate(self):
         """Validate the package contents."""
 
@@ -320,8 +278,6 @@ class OnlPackage(object):
             if dst.startswith('/'):
                 dst = dst[1:]
 
-            src = self.filepath(src)
-
             if os.path.isdir(src):
                 #
                 # Copy entire src directory to target directory
@@ -571,6 +527,20 @@ class OnlPackageGroup(object):
             for p in self.packages:
                 products.append(p.build(dir_=dir_))
 
+
+        if 'release' in self._pkgs:
+            release_list = onlu.validate_src_dst_file_tuples(self._pkgs['__directory'],
+                                                      self._pkgs['release'],
+                                                      dict(),
+                                                      OnlPackageError)
+            for f in release_list:
+                # Todo -- customize
+                dst = os.path.join(os.getenv('ONL'), 'RELEASE', g_dist_codename, f[1])
+                if not os.path.exists(dst):
+                    os.makedirs(dst)
+                logger.info("Releasing %s -> %s" % (os.path.basename(f[0]), dst))
+                shutil.copy(f[0], dst)
+
         return products
 
     def clean(self, dir_=None):
@@ -595,7 +565,7 @@ class OnlPackageRepo(object):
 
         root : The root directory that should be used for this repository."""
 
-        self.root = root
+        root = os.path.join(root, g_dist_codename)
 
         if not os.path.exists(root):
             os.makedirs(root)
@@ -1048,8 +1018,15 @@ if __name__ == '__main__':
     ap.add_argument("--skip-missing", action='store_true')
     ap.add_argument("--try-arches", nargs='+', metavar='ARCH')
     ap.add_argument("--in-repo", nargs='+', metavar='PACKAGE')
+    ap.add_argument("--include-env-json", default=os.environ.get('ONLPM_OPTION_INCLUDE_ENV_JSON', None))
     ops = ap.parse_args()
 
+    if ops.include_env_json:
+        for j in ops.include_env_json.split(':'):
+            data = json.load(open(j))
+            for (k, v) in data.iteritems():
+                os.environ[k] = v
+
     #
     # The packagedirs and repo options can be inherited from the environment
     # to make it easier to customize the settings for a given project.

tools/onlu.py

@@ -10,6 +10,8 @@ from collections import Iterable
 import sys
 import os
 import fcntl
+import glob
+from string import Template
 
 logger = None
 
@@ -156,3 +158,55 @@ class Lock(object):
     def __del__(self):
         self.handle.close()
 
+
+def filepath(absdir, relpath, eklass):
+    """Return the absolute path for the given absdir/repath file."""
+    p = None
+    if os.path.isabs(relpath):
+        p = relpath
+    else:
+        p = os.path.join(absdir, relpath)
+
+    # Globs that result in a single file are allowed:
+    g = glob.glob(p)
+    if len(g) is 0:
+        raise eklass("'%s' did not match any files." % p)
+    elif len(g) > 1:
+            raise eklass("'%s' matched too many files %s" % (p, g))
+    else:
+        p = g[0]
+
+    return p
+
+def validate_src_dst_file_tuples(absdir, data, dstsubs, eklass):
+    files = []
+    if type(data) is dict:
+        for (s,d) in data.iteritems():
+            files.append((s,d))
+    elif type(data) is list:
+        for e in data:
+            if type(e) is dict:
+                for (s,d) in e.iteritems():
+                    files.append((s,d))
+            elif type(e) in [ list, tuple ]:
+                if len(e) != 2:
+                    raise eklass("Too many filenames: '%s'" % (e))
+                else:
+                    files.append((e[0], e[1]))
+            else:
+                raise eklass("File entry '%s' is invalid." % (e))
+
+    #
+    # Validate/process source files.
+    # Process dst paths.
+    #
+    flist = []
+    for f in files:
+        src = filepath(absdir, f[0], eklass)
+        if not os.path.exists(src):
+            raise eklass("Source file or directory '%s' does not exist." % src)
+        t = Template(f[1])
+        dst = t.substitute(dstsubs)
+        flist.append((src, dst))
+
+    return flist

tools/onlvi.py

@@ -50,3 +50,5 @@ class OnlVersionImplementation(object):
     def V_RELEASE_ID(self, data):
         return "%s,%s" % (self.V_VERSION_ID(data), self.V_BUILD_ID(data))
 
+    def V_FNAME_RELEASE_ID(self, data):
+        return "%s-%s" % (self.V_VERSION_ID(data), self.V_FNAME_BUILD_ID(data))