Added JSON schema validation to run_tests.sh

* Python3 scripts are now compatible with versions <3.6
 * improved and prettified run_tests.sh

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

.travis.yml

@@ -1,7 +1,7 @@
 language: c
 before_install:
 - sudo apt-get -qq update
-- sudo apt-get install -y build-essential make binutils gcc autoconf automake libtool pkg-config git libpcap-dev libgcrypt-dev libgpg-error-dev libjson-c-dev libmaxminddb-dev netcat-openbsd
+- sudo apt-get install -y build-essential make binutils gcc autoconf automake libtool pkg-config git libpcap-dev libgcrypt-dev libgpg-error-dev libjson-c-dev libmaxminddb-dev netcat-openbsd python3 python3-jsonschema
 script:
 - git clone https://github.com/ntop/nDPI.git
 - cd nDPI &&
@@ -10,6 +10,7 @@ script:
   make install -j4 &&
   cd ..
 - make -f Makefile.old NDPI_WITH_GCRYPT=yes CUSTOM_LIBNDPI=./nDPI/_install/lib/libndpi.a ENABLE_DEBUG=yes ENABLE_SANITIZER=yes all examples
+- env python3 --version
 - test/run_tests.sh nDPI
 - make -f Makefile.old clean
 - PKG_CONFIG_PATH="$(realpath ./nDPI/_install/lib/pkgconfig)" make -f Makefile.old PKG_CONFIG_BIN=pkg-config ENABLE_DEBUG=yes ENABLE_SANITIZER=yes all examples

dependencies/nDPIsrvd.py

@@ -6,17 +6,22 @@ import base64
 import json
 import re
 import os
-import scapy.all
 import stat
 import socket
 import sys
+
 try:
     from colorama import Back, Fore, Style
     USE_COLORAMA=True
-except ModuleNotFoundError:
-    print('Python module colorama not found, using fallback.')
+except ImportError:
+    sys.stderr.write('Python module colorama not found, using fallback.\n')
     USE_COLORAMA=False
 
+try:
+    import scapy.all
+except ImportError:
+    sys.stderr.write('Python module scapy not found, PCAP generation will fail!\n')
+
 DEFAULT_HOST = '127.0.0.1'
 DEFAULT_PORT = 7000
 DEFAULT_UNIX = '/tmp/ndpid-distributor.sock'
@@ -108,6 +113,52 @@ class FlowManager:
 
         return flow
 
+class nDPIsrvdException(Exception):
+    UNSUPPORTED_ADDRESS_TYPE = 1
+    BUFFER_CAPACITY_REACHED  = 2
+    SOCKET_CONNECTION_BROKEN = 3
+    INVALID_LINE_RECEIVED    = 4
+    CALLBACK_RETURNED_FALSE  = 5
+
+    def __init__(self, etype):
+        self.etype = etype
+    def __str__(self):
+        return 'nDPIsrvdException type {}'.format(self.etype)
+
+class UnsupportedAddressType(nDPIsrvdException):
+    def __init__(self, addr):
+        super().__init__(nDPIsrvdException.UNSUPPORTED_ADDRESS_TYPE)
+        self.addr = addr
+    def __str__(self):
+        return '{}'.format(str(self.addr))
+
+class BufferCapacityReached(nDPIsrvdException):
+    def __init__(self, current_length, max_length):
+        super().__init__(nDPIsrvdException.BUFFER_CAPACITY_REACHED)
+        self.current_length = current_length
+        self.max_length = max_length
+    def __str__(self):
+        return '{} of {} bytes'.format(self.current_length, self.max_length)
+
+class SocketConnectionBroken(nDPIsrvdException):
+    def __init__(self):
+        super().__init__(nDPIsrvdException.SOCKET_CONNECTION_BROKEN)
+    def __str__(self):
+        return 'Disconnected.'
+
+class InvalidLineReceived(nDPIsrvdException):
+    def __init__(self, packet_buffer):
+        super().__init__(nDPIsrvdException.INVALID_LINE_RECEIVED)
+        self.packet_buffer = packet_buffer
+    def __str__(self):
+        return 'Received JSON line is invalid.'
+
+class CallbackReturnedFalse(nDPIsrvdException):
+    def __init__(self):
+        super().__init__(nDPIsrvdException.CALLBACK_RETURNED_FALSE)
+    def __str__(self):
+        return 'Callback returned False, abort.'
+
 class nDPIsrvdSocket:
     def __init__(self):
         self.sock_family = None
@@ -119,7 +170,7 @@ class nDPIsrvdSocket:
         elif type(addr) is str:
             self.sock_family = socket.AF_UNIX
         else:
-            raise RuntimeError('Unsupported address type:: {}'.format(str(addr)))
+            raise UnsupportedAddressType(addr)
 
         self.sock = socket.socket(self.sock_family, socket.SOCK_STREAM)
         self.sock.connect(addr)
@@ -130,12 +181,15 @@ class nDPIsrvdSocket:
 
     def receive(self):
         if len(self.buffer) == NETWORK_BUFFER_MAX_SIZE:
-            raise RuntimeError('Buffer capacity reached ({} bytes), check if it is in sync with nDPId\'s NETWORK_BUFFER_MAX_SIZE.'.format(NETWORK_BUFFER_MAX_SIZE))
+            raise BufferCapacityReached(len(self.buffer), NETWORK_BUFFER_MAX_SIZE)
 
-        recvd = self.sock.recv(NETWORK_BUFFER_MAX_SIZE - len(self.buffer))
+        try:
+            recvd = self.sock.recv(NETWORK_BUFFER_MAX_SIZE - len(self.buffer))
+        except ConnectionResetError:
+            raise SocketConnectionBroken()
 
         if len(recvd) == 0:
-            raise RuntimeError('Socket connection broken.')
+            raise SocketConnectionBroken()
         self.buffer += recvd
 
         new_data_avail = False
@@ -146,9 +200,9 @@ class nDPIsrvdSocket:
                 if starts_with_digits is None:
                     if len(self.buffer) < NETWORK_BUFFER_MIN_SIZE:
                         break
-                    raise RuntimeError('Invalid packet received: {}'.format(self.buffer))
-                self.msglen = int(starts_with_digits[1])
-                self.digitlen = len(starts_with_digits[1])
+                    raise InvalidLineReceived(self.buffer)
+                self.msglen = int(starts_with_digits.group(1))
+                self.digitlen = len(starts_with_digits.group(1))
 
             if len(self.buffer) >= self.msglen + self.digitlen:
                 recvd = self.buffer[self.digitlen:self.msglen + self.digitlen]
@@ -179,7 +233,7 @@ class nDPIsrvdSocket:
         while True:
             if self.receive() > 0:
                 if self.parse(callback, global_user_data) is False:
-                    raise RuntimeError('Callback returned False, abort.')
+                    raise CallbackReturnedFalse()
                     break;
 
 class PcapPacket:

examples/py-flow-info/flow-info.py

@@ -7,7 +7,7 @@ sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId')
 try:
     import nDPIsrvd
     from nDPIsrvd import nDPIsrvdSocket, TermColor
-except ModuleNotFoundError:
+except ImportError:
     sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
     import nDPIsrvd
     from nDPIsrvd import nDPIsrvdSocket, TermColor

examples/py-flow-undetected-to-pcap/flow-undetected-to-pcap.py

@@ -7,7 +7,7 @@ sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId')
 try:
     import nDPIsrvd
     from nDPIsrvd import nDPIsrvdSocket, TermColor
-except ModuleNotFoundError:
+except ImportError:
     sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
     import nDPIsrvd
     from nDPIsrvd import nDPIsrvdSocket, TermColor

examples/py-json-stdout/json-stdout.py

@@ -7,7 +7,7 @@ sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId')
 try:
     import nDPIsrvd
     from nDPIsrvd import nDPIsrvdSocket, TermColor
-except ModuleNotFoundError:
+except ImportError:
     sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
     import nDPIsrvd
     from nDPIsrvd import nDPIsrvdSocket, TermColor

examples/py-risky-flow-to-pcap/risky-flow-to-pcap.py

@@ -8,7 +8,7 @@ sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId')
 try:
     import nDPIsrvd
     from nDPIsrvd import nDPIsrvdSocket, TermColor
-except ModuleNotFoundError:
+except ImportError:
     sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
     import nDPIsrvd
     from nDPIsrvd import nDPIsrvdSocket, TermColor

examples/py-schema-validation/py-schema-validation.py

@@ -7,7 +7,7 @@ sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId')
 try:
     import nDPIsrvd
     from nDPIsrvd import nDPIsrvdSocket, TermColor
-except ModuleNotFoundError:
+except ImportError:
     sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
     import nDPIsrvd
     from nDPIsrvd import nDPIsrvdSocket, TermColor
@@ -43,4 +43,7 @@ if __name__ == '__main__':
 
     nsock = nDPIsrvdSocket()
     nsock.connect(address)
-    nsock.loop(onJsonLineRecvd, Stats())
+    try:
+        nsock.loop(onJsonLineRecvd, Stats())
+    except nDPIsrvd.SocketConnectionBroken as err:
+        sys.stderr.write('\n{}\n'.format(err))

schema/daemon_event_schema.json

@@ -59,6 +59,9 @@
         "tcp-max-post-end-flow-time": {
             "type": "number"
         },
+        "max-packets-per-flow-to-process": {
+            "type": "number"
+        },
         "max-packets-per-flow-to-send": {
             "type": "number"
         }

test/run_tests.sh

@@ -5,15 +5,6 @@ set -e
 LINE_SPACES=${LINE_SPACES:-48}
 MYDIR="$(realpath "$(dirname ${0})")"
 nDPId_test_EXEC="${2:-"$(realpath "${MYDIR}/../nDPId-test")"}"
-nDPI_SOURCE_ROOT="$(realpath "${1}")"
-LOCKFILE="$(realpath "${0}").lock"
-
-touch "${LOCKFILE}"
-exec 42< "${LOCKFILE}"
-flock -x -n 42 || {
-    printf '%s\n' "Could not aquire file lock for ${0}. Already running instance?";
-    exit 1;
-}
 
 if [ $# -ne 1 -a $# -ne 2 ]; then
 cat <<EOF
@@ -21,14 +12,30 @@ usage: ${0} [path-to-nDPI-source-root] [path-to-nDPId-test-exec]
 
 	path-to-nDPId-test-exec defaults to ${nDPId_test_EXEC}
 EOF
-exit 1
+exit 2
 fi
 
+nDPI_SOURCE_ROOT="$(realpath "${1}")"
+LOCKFILE="$(realpath "${0}").lock"
+
+touch "${LOCKFILE}"
+exec 42< "${LOCKFILE}"
+flock -x -n 42 || {
+    printf '%s\n' "Could not aquire file lock for ${0}. Already running instance?";
+    exit 3;
+}
+function sighandler()
+{
+    rm -f "${LOCKFILE}"
+    exit 4
+}
+trap sighandler SIGINT SIGTERM
+
 if [ ! -x "${nDPId_test_EXEC}" ]; then
 cat <<EOF
 Required nDPId-test executable does not exist; ${nDPId_test_EXEC}
 EOF
-exit 1
+exit 5
 fi
 
 nDPI_TEST_DIR="${nDPI_SOURCE_ROOT}/tests/pcap"
@@ -37,12 +44,17 @@ cat <<EOF
 nDPId-test......: ${nDPId_test_EXEC}
 nDPI source root: ${nDPI_TEST_DIR}
 
+--------------------------
+-- nDPId PCAP diff tests --
+--------------------------
+
 EOF
 
 cd "${nDPI_TEST_DIR}"
 mkdir -p /tmp/nDPId-test-stderr
 set +e
-RETVAL=0
+TESTS_FAILED=0
+
 for pcap_file in $(ls *.pcap*); do
     printf '%s\n' "${nDPId_test_EXEC} ${pcap_file}" \
         >"/tmp/nDPId-test-stderr/${pcap_file}.out"
@@ -56,7 +68,9 @@ for pcap_file in $(ls *.pcap*); do
     if [ $? -eq 0 ]; then
         if [ ! -r "${MYDIR}/results/${pcap_file}.out" ]; then
             printf '%s\n' '[NEW]'
-            RETVAL=1
+            mv -v "${MYDIR}/results/${pcap_file}.out.new" \
+                  "${MYDIR}/results/${pcap_file}.out"
+            TESTS_FAILED=$((TESTS_FAILED + 1))
         elif diff -u0 "${MYDIR}/results/${pcap_file}.out" \
                       "${MYDIR}/results/${pcap_file}.out.new" >/dev/null; then
             printf '%s\n' '[OK]'
@@ -66,26 +80,71 @@ for pcap_file in $(ls *.pcap*); do
                      "${MYDIR}/results/${pcap_file}.out.new"
             mv -v "${MYDIR}/results/${pcap_file}.out.new" \
                   "${MYDIR}/results/${pcap_file}.out"
-            RETVAL=1
+            TESTS_FAILED=$((TESTS_FAILED + 1))
         fi
     else
         printf '%s\n' '[FAIL]'
         printf '%s\n' '----------------------------------------'
         printf '%s\n' "-- STDERR of ${pcap_file}"
         cat "/tmp/nDPId-test-stderr/${pcap_file}.out"
-        RETVAL=1
+        TESTS_FAILED=$((TESTS_FAILED + 1))
     fi
 
     rm -f "${MYDIR}/results/${pcap_file}.out.new"
 done
 
+cat <<EOF
+
+----------------------------
+-- JSON schema validation --
+----------------------------
+
+EOF
+
 cd "${MYDIR}"
 for out_file in $(ls results/*.out); do
     pcap_file="${nDPI_TEST_DIR}/$(basename ${out_file%.out})"
     if [ ! -r "${pcap_file}" ]; then
         printf "%-${LINE_SPACES}s\t%s\n" "$(basename ${pcap_file})" '[MISSING]'
-        RETVAL=1
+        TESTS_FAILED=$((TESTS_FAILED + 1))
+    else
+        printf "SCHEMA %-${LINE_SPACES}s\t" "$(basename ${pcap_file})"
+        printf '%s\n' '*** JSON schema validation ***' >>"/tmp/nDPId-test-stderr/$(basename ${pcap_file}).out"
+        if [ ! -r "${out_file}" ]; then
+            printf ' %s\n' '[MISSING]'
+            TESTS_FAILED=$((TESTS_FAILED + 1))
+            continue
+        fi
+        cat "${out_file}" | nc -q 1 -l 127.0.0.1 9000 &
+        nc_pid=$!
+        ${MYDIR}/../examples/py-schema-validation/py-schema-validation.py \
+            --host 127.0.0.1 --port 9000 2>>"/tmp/nDPId-test-stderr/$(basename ${pcap_file}).out"
+        if [ $? -eq 0 ]; then
+            printf ' %s\n' '[OK]'
+        else
+            printf ' %s\n' '[FAIL]'
+            printf '%s\n' '----------------------------------------'
+            printf '%s\n' "-- STDERR of $(basename ${pcap_file})"
+            cat "/tmp/nDPId-test-stderr/$(basename ${pcap_file}).out"
+            TESTS_FAILED=$((TESTS_FAILED + 1))
+        fi
+        kill -SIGTERM ${nc_pid} 2>/dev/null
+        wait ${nc_pid} 2>/dev/null
     fi
 done
 
-exit ${RETVAL}
+if [ ${TESTS_FAILED} -eq 0 ]; then
+cat <<EOF
+
+--------------------------
+-- All tests succeeded. --
+--------------------------
+EOF
+    exit 0
+else
+cat <<EOF
+
+*** ${TESTS_FAILED} tests failed. ***
+EOF
+    exit 1
+fi