github-personal/nDPId
1
0
Fork 0
mirror of https://github.com/outbackdingo/nDPId.git synced 2026-01-28 02:19:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: github-personal:main
Branches Tags
github-personal:main github-personal:tmp github-personal:add/apple-bsd-port github-personal:add/event-io-abstraction github-personal:ndpi-example-proposal github-personal:add/tls-proxy-support github-personal:add/nDPId-UDP-endpoint
github-personal:1.6 github-personal:1.6rc4 github-personal:1.6rc3 github-personal:1.6rc2 github-personal:1.6rc1 github-personal:1.5 github-personal:1.4 github-personal:1.3 github-personal:1.2 github-personal:1.1 github-personal:1.0
..
compare: github-personal:add/nDPId-UDP-endpoint
Branches Tags
github-personal:main github-personal:tmp github-personal:add/apple-bsd-port github-personal:add/event-io-abstraction github-personal:ndpi-example-proposal github-personal:add/tls-proxy-support github-personal:add/nDPId-UDP-endpoint
github-personal:1.6 github-personal:1.6rc4 github-personal:1.6rc3 github-personal:1.6rc2 github-personal:1.6rc1 github-personal:1.5 github-personal:1.4 github-personal:1.3 github-personal:1.2 github-personal:1.1 github-personal:1.0

2 Commits
main ... add/nDPId-

Author SHA1 Message Date
lns
f0415cabfa nDPId: improved error messages if UNIX/UDP endpoint refuses connections/datagrams 
Signed-off-by: lns <matzeton@googlemail.com>
 2022-08-21 19:05:42 +02:00
Toni Uhlig
98645285c8 nDPId: support for custom UDP endpoints 
Signed-off-by: Toni Uhlig <matzeton@googlemail.com>
 2022-08-15 11:07:13 +02:00
4634 changed files with 75555 additions and 371451 deletions
Expand all files Collapse all files

19
.circleci/config.yml
View File

@@ -1,19 +0,0 @@
version: 2.1
jobs:
build:
docker:
- image: ubuntu:latest
steps:
- checkout
- run: export DEBIAN_FRONTEND=noninteractive
- run: apt-get update -qq
- run: |
env DEBIAN_FRONTEND=noninteractive \
apt-get install -y -qq \
coreutils wget git unzip make cmake binutils gcc g++ autoconf automake flex bison texinfo \
libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev zlib1g-dev
- run: |
cmake -S . -B build -DENABLE_SYSTEMD=ON -DBUILD_EXAMPLES=ON -DBUILD_NDPI=ON
- run: |
cmake --build build --verbose

38
.github/workflows/build-archlinux.yml vendored
View File

@@ -1,38 +0,0 @@
name: ArchLinux PKGBUILD
on:
push:
branches:
- main
- tmp
pull_request:
branches:
- main
types: [opened, synchronize, reopened]
release:
types: [created]
jobs:
build:
runs-on: ubuntu-latest
env:
CMAKE_C_FLAGS: -Werror
steps:
- uses: actions/checkout@v4
with:
submodules: false
fetch-depth: 1
- name: Prepare for ArchLinux packaging
run: |
sudo chmod -R 0777 .
mv -v packages/archlinux packages/ndpid-testing
- uses: 2m/arch-pkgbuild-builder@v1.16
with:
debug: true
target: 'pkgbuild'
pkgname: 'packages/ndpid-testing'
- name: Upload PKG
uses: actions/upload-artifact@v4
with:
name: nDPId-archlinux-packages
path: packages/ndpid-testing/*.pkg.tar.zst

59
.github/workflows/build-centos.yml vendored
View File

@@ -1,59 +0,0 @@
name: CentOs
on:
push:
branches:
- main
- tmp
pull_request:
branches:
- main
types: [opened, synchronize, reopened]
release:
types: [created]
jobs:
centos8:
runs-on: ubuntu-latest
container: 'centos:8'
steps:
- uses: actions/checkout@v4
with:
submodules: false
fetch-depth: 1
- name: Install CentOs Prerequisites
run: |
sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
yum -y update
yum -y install curl gpg
curl 'https://packages.ntop.org/centos/ntop.repo' > /etc/yum.repos.d/ntop.repo
curl 'https://packages.ntop.org/centos/RPM-GPG-KEY-deri' | gpg --import
yum -y install yum-utils dnf-plugins-core epel-release
dnf config-manager --set-enabled powertools
yum -y update
yum -y install rpm-build gcc gcc-c++ autoconf automake make cmake flex bison gettext pkg-config libtool ndpi-dev libpcap-devel zlib-devel python3.8 git wget unzip /usr/lib64/libasan.so.5.0.0 /usr/lib64/libubsan.so.1.0.0
repoquery -l ndpi-dev
- name: Configure nDPId
run: |
mkdir build && cd build
cmake .. -DENABLE_SYSTEMD=ON -DBUILD_EXAMPLES=ON -DENABLE_SANITIZER=ON -DNDPI_NO_PKGCONFIG=ON -DSTATIC_LIBNDPI_INSTALLDIR=/usr
- name: Build nDPId
run: |
make -C build all VERBOSE=1
- name: CPack RPM
run: |
cd ./build && cpack -G RPM && cd ..
- name: Upload RPM
uses: actions/upload-artifact@v4
with:
name: nDPId-centos-packages
path: build/*.rpm
- name: Upload on Failure
uses: actions/upload-artifact@v4
if: failure()
with:
name: autoconf-config-log
path: |
build/CMakeCache.txt
libnDPI/config.log

25
.github/workflows/build-docker.yml vendored
View File

@@ -1,25 +0,0 @@
name: Docker Build
on:
push:
branches:
- 'main'
jobs:
docker:
runs-on: ubuntu-latest
steps:
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push
uses: docker/build-push-action@v4
with:
push: true
tags: utoni/ndpid:latest

39
.github/workflows/build-freebsd.yml vendored
View File

@@ -1,39 +0,0 @@
name: FreeBSD Build
on:
schedule:
# At the end of every day
- cron: '0 0 * * *'
push:
branches:
- main
- tmp
pull_request:
branches:
- main
types: [opened, synchronize, reopened]
release:
types: [created]
jobs:
test:
runs-on: ubuntu-latest
name: Build and Test
steps:
- uses: actions/checkout@v4
- name: Test in FreeBSD
id: test
uses: vmactions/freebsd-vm@main
with:
usesh: true
prepare: |
pkg install -y bash autoconf automake cmake gmake libtool gettext pkgconf gcc \
git wget unzip flock \
json-c flex bison libpcap curl openssl dbus
run: |
echo "Working Directory: $(pwd)"
echo "User.............: $(whoami)"
echo "FreeBSD Version..: $(freebsd-version)"
# TODO: Make examples I/O event agnostic i.e. use nio
cmake -S . -B build -DBUILD_NDPI=ON -DBUILD_EXAMPLES=OFF #-DENABLE_CURL=ON -DENABLE_DBUS=ON
cmake --build build

54
.github/workflows/build-openwrt.yml vendored
View File

@@ -1,54 +0,0 @@
name: OpenWrt Build
on:
schedule:
# At the end of every day
- cron: '0 0 * * *'
push:
branches:
- main
- tmp
pull_request:
branches:
- main
types: [opened, synchronize, reopened]
release:
types: [created]
jobs:
build:
name: ${{ matrix.arch }} ${{ matrix.target }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- arch: arm_cortex-a9_vfpv3-d16
target: mvebu-cortexa9
- arch: arm_cortex-a15_neon-vfpv4
target: armvirt-32
- arch: x86_64
target: x86-64
steps:
- uses: actions/checkout@v4
with:
submodules: false
fetch-depth: 1
- name: Build
uses: openwrt/gh-action-sdk@main
env:
ARCH: ${{ matrix.arch }}-snapshot
FEED_DIR: ${{ github.workspace }}/packages/openwrt
FEEDNAME: ndpid_openwrt_packages_ci
PACKAGES: nDPId-testing
V: s
- name: Store packages
uses: actions/upload-artifact@v4
with:
name: nDPId-${{ matrix.arch}}-${{ matrix.target }}
path: bin/packages/${{ matrix.arch }}/ndpid_openwrt_packages_ci/*.ipk

45
.github/workflows/build-rpm.yml vendored
View File

@@ -1,45 +0,0 @@
name: RPM Build
on:
schedule:
# At the end of every day
- cron: '0 0 * * *'
push:
branches:
- main
- tmp
pull_request:
branches:
- main
types: [opened, synchronize, reopened]
release:
types: [created]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install Ubuntu Prerequisites
run: |
sudo apt-get update
sudo apt-get install fakeroot alien autoconf automake cmake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev zlib1g-dev libcurl4-openssl-dev libdbus-1-dev
- name: Build RPM package
run: |
cmake -S . -B build-rpm -DBUILD_EXAMPLES=ON -DBUILD_NDPI=ON -DCMAKE_BUILD_TYPE=Release
cmake --build build-rpm --parallel
cd build-rpm
cpack -G RPM
cd ..
- name: Convert/Install RPM package
run: |
fakeroot alien --scripts --to-deb --verbose ./build-rpm/nDPId-*.rpm
sudo dpkg -i ./ndpid_*.deb
- name: Upload RPM
uses: actions/upload-artifact@v4
with:
name: nDPId-rpm-packages
path: build-rpm/*.rpm

373
.github/workflows/build.yml vendored
View File

@@ -1,409 +1,88 @@
name: Build
on:
schedule:
# At the end of every day
- cron: '0 0 * * *'
push:
branches:
- main
- tmp
pull_request:
branches:
- main
types: [opened, synchronize, reopened]
release:
types: [created]
jobs:
test:
name: ${{ matrix.os }} ${{ matrix.compiler }}
name: ${{ matrix.os }} ${{ matrix.gcrypt }}
runs-on: ${{ matrix.os }}
env:
CMAKE_C_COMPILER: ${{ matrix.compiler }}
CMAKE_C_FLAGS: -Werror ${{ matrix.cflags }}
CMAKE_C_EXE_LINKER_FLAGS: ${{ matrix.ldflags }}
CMAKE_MODULE_LINKER_FLAGS: ${{ matrix.ldflags }}
DYLD_LIBRARY_PATH: /usr/local/lib
CMAKE_C_FLAGS: -Werror
strategy:
fail-fast: true
matrix:
os: ["ubuntu-latest", "ubuntu-18.04"]
ndpid_gcrypt: ["-DNDPI_WITH_GCRYPT=OFF", "-DNDPI_WITH_GCRYPT=ON"]
ndpid_zlib: ["-DENABLE_ZLIB=OFF", "-DENABLE_ZLIB=ON"]
ndpi_min_version: ["4.4"]
include:
- compiler: "gcc"
- compiler: "default-cc"
os: "ubuntu-latest"
ndpi_build: "-DBUILD_NDPI=ON"
ndpid_examples: "-DBUILD_EXAMPLES=ON"
ndpid_rust_examples: "-DBUILD_RUST_EXAMPLES=ON"
ndpid_gcrypt: "-DNDPI_WITH_GCRYPT=OFF"
ndpid_zlib: "-DENABLE_ZLIB=ON"
ndpid_extras: "-DENABLE_CRYPTO=ON"
sanitizer: "-DENABLE_SANITIZER=OFF -DENABLE_SANITIZER_THREAD=OFF"
coverage: "-DENABLE_COVERAGE=OFF"
poll: "-DFORCE_POLL=OFF"
upload: true
upload_suffix: ""
ndpi_min_version: "5.0"
- compiler: "gcc"
os: "ubuntu-latest"
ndpi_build: "-DBUILD_NDPI=ON"
ndpid_examples: "-DBUILD_EXAMPLES=ON"
ndpid_rust_examples: ""
ndpid_gcrypt: "-DNDPI_WITH_GCRYPT=ON"
ndpid_zlib: "-DENABLE_ZLIB=ON"
ndpid_extras: "-DENABLE_CRYPTO=ON -DNDPI_WITH_MAXMINDDB=ON -DNDPI_WITH_PCRE=ON -DENABLE_MEMORY_PROFILING=ON"
sanitizer: "-DENABLE_SANITIZER=OFF -DENABLE_SANITIZER_THREAD=OFF"
coverage: "-DENABLE_COVERAGE=OFF"
poll: "-DFORCE_POLL=OFF"
upload: true
upload_suffix: "-host-gcrypt"
ndpi_min_version: "5.0"
- compiler: "clang"
os: "ubuntu-latest"
ndpi_build: "-DBUILD_NDPI=ON"
ndpid_examples: "-DBUILD_EXAMPLES=ON"
ndpid_rust_examples: ""
ndpid_gcrypt: "-DNDPI_WITH_GCRYPT=OFF"
ndpid_zlib: "-DENABLE_ZLIB=OFF"
ndpid_extras: ""
sanitizer: "-DENABLE_SANITIZER=OFF -DENABLE_SANITIZER_THREAD=OFF"
coverage: "-DENABLE_COVERAGE=OFF"
poll: "-DFORCE_POLL=OFF"
upload: true
upload_suffix: "-no-zlib"
ndpi_min_version: "5.0"
- compiler: "gcc"
os: "ubuntu-latest"
ndpi_build: "-DBUILD_NDPI=ON"
ndpid_examples: "-DBUILD_EXAMPLES=ON"
ndpid_rust_examples: ""
ndpid_gcrypt: "-DNDPI_WITH_GCRYPT=OFF"
ndpid_zlib: "-DENABLE_ZLIB=ON"
ndpid_extras: ""
sanitizer: "-DENABLE_SANITIZER=ON"
coverage: "-DENABLE_COVERAGE=ON"
poll: "-DFORCE_POLL=ON"
upload: false
ndpi_min_version: "5.0"
- compiler: "clang"
os: "ubuntu-latest"
ndpi_build: "-DBUILD_NDPI=ON"
ndpid_examples: "-DBUILD_EXAMPLES=ON"
ndpid_rust_examples: ""
ndpid_gcrypt: "-DNDPI_WITH_GCRYPT=OFF"
ndpid_zlib: "-DENABLE_ZLIB=ON"
ndpid_extras: "-DENABLE_CRYPTO=ON"
sanitizer: "-DENABLE_SANITIZER=ON"
coverage: "-DENABLE_COVERAGE=OFF"
poll: "-DFORCE_POLL=OFF"
upload: false
ndpi_min_version: "5.0"
- compiler: "clang-12"
os: "ubuntu-22.04"
ndpi_build: "-DBUILD_NDPI=ON"
ndpid_examples: "-DBUILD_EXAMPLES=ON"
ndpid_rust_examples: ""
ndpid_gcrypt: "-DNDPI_WITH_GCRYPT=OFF"
ndpid_zlib: "-DENABLE_ZLIB=ON"
ndpid_extras: ""
sanitizer: "-DENABLE_SANITIZER_THREAD=ON"
coverage: "-DENABLE_COVERAGE=OFF"
poll:
upload: false
ndpi_min_version: "5.0"
os: "ubuntu-latest"
- compiler: "gcc-10"
os: "ubuntu-22.04"
ndpi_build: "-DBUILD_NDPI=ON"
ndpid_examples: "-DBUILD_EXAMPLES=ON"
ndpid_rust_examples: ""
ndpid_gcrypt: "-DNDPI_WITH_GCRYPT=OFF"
ndpid_zlib: "-DENABLE_ZLIB=OFF"
ndpid_extras: ""
sanitizer: "-DENABLE_SANITIZER=ON"
coverage: "-DENABLE_COVERAGE=OFF"
poll: "-DFORCE_POLL=ON"
upload: false
ndpi_min_version: "5.0"
- compiler: "gcc-9"
os: "ubuntu-22.04"
ndpi_build: "-DBUILD_NDPI=ON"
ndpid_examples: "-DBUILD_EXAMPLES=ON"
ndpid_rust_examples: ""
ndpid_gcrypt: "-DNDPI_WITH_GCRYPT=OFF"
ndpid_zlib: "-DENABLE_ZLIB=ON"
ndpid_extras: ""
sanitizer: "-DENABLE_SANITIZER=ON"
coverage: "-DENABLE_COVERAGE=OFF"
poll: "-DFORCE_POLL=OFF"
upload: false
ndpi_min_version: "5.0"
- compiler: "cc"
os: "macOS-13"
ndpi_build: "-DBUILD_NDPI=OFF"
ndpid_examples: "-DBUILD_EXAMPLES=OFF"
ndpid_rust_examples: ""
ndpid_gcrypt: "-DNDPI_WITH_GCRYPT=OFF"
ndpid_zlib: "-DENABLE_ZLIB=ON"
ndpid_extras: ""
examples: "-DBUILD_EXAMPLES=OFF"
sanitizer: "-DENABLE_SANITIZER=OFF"
coverage: "-DENABLE_COVERAGE=OFF"
poll:
upload: false
ndpi_min_version: "5.0"
os: "ubuntu-latest"
- compiler: "gcc-7"
os: "ubuntu-latest"
steps:
- name: Print Matrix
run: |
echo '----------------------------------------'
echo '| OS.......: ${{ matrix.os }}'
echo '| CC.......: ${{ matrix.compiler }}'
echo "| CFLAGS...: $CMAKE_C_FLAGS"
echo "| LDFLAGS..: $CMAKE_C_EXE_LINKER_FLAGS"
echo '|---------------------------------------'
echo '| nDPI min.: ${{ matrix.ndpi_min_version }}'
echo '| GCRYPT...: ${{ matrix.ndpid_gcrypt }}'
echo '| ZLIB.....: ${{ matrix.ndpid_zlib }}'
echo '| Extras...: ${{ matrix.ndpid_extras }}'
echo '| ForcePoll: ${{ matrix.poll }}'
echo '|---------------------------------------'
echo '| SANITIZER: ${{ matrix.sanitizer }}'
echo '| COVERAGE.: ${{ matrix.coverage }}'
echo '|---------------------------------------'
echo '| UPLOAD...: ${{ matrix.upload }}'
echo '----------------------------------------'
- uses: actions/checkout@v4
- uses: actions/checkout@v2
with:
submodules: false
fetch-depth: 1
- name: Install MacOS Prerequisites
if: startsWith(matrix.os, 'macOS')
run: |
brew install coreutils automake make unzip
wget 'https://www.tcpdump.org/release/libpcap-1.10.4.tar.gz'
tar -xzvf libpcap-1.10.4.tar.gz
cd libpcap-1.10.4
./configure && make install
cd ..
wget 'https://github.com/ntop/nDPI/archive/refs/heads/dev.zip' -O libndpi-dev.zip
unzip libndpi-dev.zip
cd nDPI-dev
./autogen.sh
./configure --prefix=/usr/local --with-only-libndpi && make install
- name: Fix kernel mmap rnd bits on Ubuntu
if: startsWith(matrix.os, 'ubuntu')
run: |
# Workaround for compatinility between latest kernel and sanitizer
# See https://github.com/actions/runner-images/issues/9491
sudo sysctl vm.mmap_rnd_bits=28
fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
- name: Install Ubuntu Prerequisites
if: startsWith(matrix.os, 'ubuntu')
run: |
sudo apt-get update
sudo apt-get install autoconf automake cmake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev zlib1g-dev libcurl4-openssl-dev libdbus-1-dev
sudo apt-get install ${{ matrix.compiler }} lcov iproute2
- name: Install Ubuntu Prerequisites (Rust/Cargo)
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.ndpid_rust_examples, '-DBUILD_RUST_EXAMPLES=ON')
run: |
sudo apt-get install cargo
sudo apt-get install autoconf automake cmake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev zlib1g-dev
sudo apt-get install ${{ matrix.compiler }} lcov
- name: Install Ubuntu Prerequisites (libgcrypt)
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.ndpid_gcrypt, '-DNDPI_WITH_GCRYPT=ON')
run: |
sudo apt-get install libgcrypt20-dev
- name: Install Ubuntu Prerequisites (zlib)
- name: Install Ubuntu Prerequisities (zlib)
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.ndpid_zlib, '-DENABLE_ZLIB=ON')
run: |
sudo apt-get install zlib1g-dev
- name: Install Ubuntu Prerequisites (libmaxminddb, libpcre2)
if: startsWith(matrix.ndpid_extras, '-D')
run: |
sudo apt-get install libmaxminddb-dev libpcre2-dev
- name: Install Ubuntu Prerequisites (libnl-genl-3-dev)
if: startsWith(matrix.ndpi_build, '-DBUILD_NDPI=ON') && startsWith(matrix.coverage, '-DENABLE_COVERAGE=OFF') && startsWith(matrix.sanitizer, '-DENABLE_SANITIZER=ON') && startsWith(matrix.ndpid_gcrypt, '-DNDPI_WITH_GCRYPT=OFF') && startsWith(matrix.ndpid_zlib, '-DENABLE_ZLIB=ON')
run: |
sudo apt-get install libnl-genl-3-dev
- name: Checking Network Buffer Size
run: |
C_VAL=$(cat config.h | sed -n 's/^#define\s\+NETWORK_BUFFER_MAX_SIZE\s\+\([0-9]\+\).*$/\1/gp')
PY_VAL=$(cat dependencies/nDPIsrvd.py | sed -n 's/^NETWORK_BUFFER_MAX_SIZE = \([0-9]\+\).*$/\1/gp')
test ${C_VAL} = ${PY_VAL}
- name: Configure nDPId
run: |
cmake -S . -B build -Werror=dev -Werror=deprecated -DCMAKE_C_COMPILER="$CMAKE_C_COMPILER" -DCMAKE_C_FLAGS="$CMAKE_C_FLAGS" -DCMAKE_MODULE_LINKER_FLAGS="$CMAKE_MODULE_LINKER_FLAGS" -DCMAKE_C_EXE_LINKER_FLAGS="$CMAKE_C_EXE_LINKER_FLAGS" \
-DENABLE_DBUS=ON -DENABLE_CURL=ON -DENABLE_SYSTEMD=ON \
${{ matrix.poll }} ${{ matrix.coverage }} ${{ matrix.sanitizer }} ${{ matrix.ndpi_build }} \
${{ matrix.ndpid_examples }} ${{ matrix.ndpid_rust_examples }} ${{ matrix.ndpid_zlib }} ${{ matrix.ndpid_gcrypt }} ${{ matrix.ndpid_extras }}
mkdir build && cd build
cmake .. -DENABLE_COVERAGE=ON -DBUILD_EXAMPLES=ON -DBUILD_NDPI=ON -DENABLE_SANITIZER=ON ${{ matrix.ndpid_zlib }} ${{ matrix.ndpid_gcrypt }}
- name: Build nDPId
run: |
cmake --build build --verbose
- name: Build single nDPId/nDPIsrvd executables (invoke CC directly - dynamic nDPI lib)
if: startsWith(matrix.ndpi_build, '-DBUILD_NDPI=OFF') && startsWith(matrix.coverage, '-DENABLE_COVERAGE=OFF') && startsWith(matrix.ndpid_gcrypt, '-DNDPI_WITH_GCRYPT=OFF')
run: |
pkg-config --cflags --libs libndpi
cc -Wall -Wextra -std=gnu99 \
${{ matrix.poll }} -DENABLE_MEMORY_PROFILING=1 \
nDPId.c nio.c utils.c \
$(pkg-config --cflags libndpi) -I. -I./dependencies -I./dependencies/jsmn -I./dependencies/uthash/include \
-o /tmp/a.out \
-lpcap $(pkg-config --libs libndpi) -pthread -lm
cc -Wall -Wextra -std=gnu99 \
${{ matrix.poll }} -DENABLE_MEMORY_PROFILING=1 \
nDPIsrvd.c nio.c utils.c \
-I. -I./dependencies -I./dependencies/jsmn -I./dependencies/uthash/include \
-o /tmp/a.out
- name: Build single nDPId/nDPIsrvd executables (invoke CC directly - static nDPI lib)
if: startsWith(matrix.ndpi_build, '-DBUILD_NDPI=ON') && startsWith(matrix.coverage, '-DENABLE_COVERAGE=OFF') && startsWith(matrix.sanitizer, '-DENABLE_SANITIZER=ON') && startsWith(matrix.ndpid_gcrypt, '-DNDPI_WITH_GCRYPT=OFF') && startsWith(matrix.ndpid_zlib, '-DENABLE_ZLIB=ON')
run: |
cc -Wall -Wextra -std=gnu99 ${{ matrix.poll }} -DENABLE_ZLIB=1 -DENABLE_MEMORY_PROFILING=1 \
-fsanitize=address -fsanitize=undefined -fno-sanitize=alignment -fsanitize=enum -fsanitize=leak \
nDPId.c nio.c utils.c \
-I./build/libnDPI/include/ndpi -I. -I./dependencies -I./dependencies/jsmn -I./dependencies/uthash/include \
-o /tmp/a.out \
-lpcap ./build/libnDPI/lib/libndpi.a -pthread -lm -lz
make -C build all VERBOSE=1
- name: Test EXEC
run: |
./build/nDPId-test
./build/nDPId-test || test $? -eq 1
./build/nDPId -h || test $? -eq 1
./build/nDPIsrvd -h || test $? -eq 1
- name: Test DIFF
if: startsWith(matrix.os, 'macOS') == false && startsWith(matrix.ndpid_gcrypt, '-DNDPI_WITH_GCRYPT=OFF')
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.ndpid_gcrypt, '-DNDPI_WITH_GCRYPT=OFF')
run: |
./test/run_tests.sh ./libnDPI ./build/nDPId-test
./test/run_config_tests.sh ./libnDPI ./build/nDPId-test
- name: Daemon
if: startsWith(matrix.compiler, 'gcc') || endsWith(matrix.compiler, 'clang')
run: |
make -C ./build daemon VERBOSE=1
make -C ./build daemon VERBOSE=1
- name: Coverage
if: startsWith(matrix.coverage, '-DENABLE_COVERAGE=ON')
run: |
make -C ./build coverage
- name: Dist
if: startsWith(matrix.os, 'macOS') == false && matrix.upload == false
run: |
make -C ./build dist
RAND_ID=$(( ( RANDOM ) + 1 ))
mkdir "nDPId-dist-${RAND_ID}"
cd "nDPId-dist-${RAND_ID}"
tar -xjf ../nDPId-*.tar.bz2
cd ./nDPId-*
cmake -S . -B ./build \
-DENABLE_DBUS=ON -DENABLE_CURL=ON -DENABLE_SYSTEMD=ON \
${{ matrix.poll }} ${{ matrix.coverage }} ${{ matrix.sanitizer }} ${{ matrix.ndpi_build }} \
${{ matrix.ndpid_examples }} ${{ matrix.ndpid_rust_examples }} ${{ matrix.ndpid_zlib }} ${{ matrix.ndpid_gcrypt }} ${{ matrix.ndpid_extras }}
cd ../..
rm -rf "nDPId-dist-${RAND_ID}"
- name: CPack DEB
if: startsWith(matrix.os, 'macOS') == false
run: |
cd ./build && cpack -G DEB && \
sudo dpkg -i nDPId-*.deb && \
sudo apt purge ndpid && \
sudo dpkg -i nDPId-*.deb && cd ..
- name: Upload DEB
if: startsWith(matrix.os, 'macOS') == false && matrix.upload
uses: actions/upload-artifact@v4
with:
name: nDPId-debian-packages_${{ matrix.compiler }}${{ matrix.upload_suffix }}
path: build/*.deb
- name: Test systemd
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.compiler, 'gcc')
run: |
ip -c address
sudo systemctl daemon-reload
sudo systemctl enable ndpid@lo
sudo systemctl start ndpid@lo
SYSTEMCTL_RET=3; while (( $SYSTEMCTL_RET == 3 )); do systemctl is-active ndpid@lo.service; SYSTEMCTL_RET=$?; sleep 1; done
sudo systemctl status ndpisrvd.service ndpid@lo.service
sudo systemctl show ndpisrvd.service ndpid@lo.service -p SubState,ActiveState
sudo dpkg -i ./build/nDPId-*.deb
sudo systemctl status ndpisrvd.service ndpid@lo.service
sudo systemctl show ndpisrvd.service ndpid@lo.service -p SubState,ActiveState
sudo systemctl stop ndpisrvd.service
journalctl --no-tail --no-pager -u ndpisrvd.service -u ndpid@lo.service
- name: Build PF_RING and nDPId (invoke CC directly - dynamic nDPI lib)
if: startsWith(matrix.ndpi_build, '-DBUILD_NDPI=ON') && startsWith(matrix.coverage, '-DENABLE_COVERAGE=OFF') && startsWith(matrix.sanitizer, '-DENABLE_SANITIZER=ON') && startsWith(matrix.ndpid_gcrypt, '-DNDPI_WITH_GCRYPT=OFF') && startsWith(matrix.ndpid_zlib, '-DENABLE_ZLIB=ON')
run: |
git clone --depth=1 https://github.com/ntop/PF_RING.git
cd PF_RING/userland && ./configure && make && sudo make install prefix=/usr
cd ../..
cc -Wall -Wextra -std=gnu99 ${{ matrix.poll }} -DENABLE_PFRING=1 -DENABLE_ZLIB=1 -DENABLE_MEMORY_PROFILING=1 \
-fsanitize=address -fsanitize=undefined -fno-sanitize=alignment -fsanitize=enum -fsanitize=leak \
nDPId.c npfring.c nio.c utils.c \
-I. -I./dependencies -I./dependencies/jsmn -I./dependencies/uthash/include \
-I./build/libnDPI/include/ndpi \
-I./PF_RING/userland/lib -I./PF_RING/kernel \
-o /tmp/a.out \
-ldl /usr/lib/libpfring.a -lpcap ./build/libnDPI/lib/libndpi.a -pthread -lm -lz
cd ./build && cpack -G DEB && cd ..
- name: Build against libnDPI-${{ matrix.ndpi_min_version }}
if: startsWith(matrix.os, 'ubuntu')
run: |
mkdir build-local-ndpi && cd build-local-ndpi
WGET_RET=0
wget 'https://github.com/ntop/nDPI/archive/refs/tags/${{ matrix.ndpi_min_version }}.tar.gz' || { WGET_RET=$?; true; }
echo "wget returned: ${WGET_RET}"
test $WGET_RET -ne 8 && { \
tar -xzvf ${{ matrix.ndpi_min_version }}.tar.gz; }
test $WGET_RET -ne 8 || { \
echo "::warning file=nDPId.c::New libnDPI release required to build against release tarball, falling back to dev branch."; \
wget 'http://github.com/ntop/nDPI/archive/refs/heads/dev.tar.gz'; \
WGET_RET=$?; \
tar -xzvf dev.tar.gz; \
mv -v 'nDPI-dev' 'nDPI-${{ matrix.ndpi_min_version }}'; }
test $WGET_RET -ne 0 || { cd nDPI-${{ matrix.ndpi_min_version }}; \
NDPI_CONFIGURE_ARGS=''; \
test 'x${{ matrix.ndpid_gcrypt }}' != 'x-DNDPI_WITH_GCRYPT=ON' || NDPI_CONFIGURE_ARGS="$NDPI_CONFIGURE_ARGS --with-local-libgcrypt"; \
test 'x${{ matrix.sanitizer }}' != 'x-DENABLE_SANITIZER=ON' || NDPI_CONFIGURE_ARGS="$NDPI_CONFIGURE_ARGS --with-sanitizer"; \
echo "Configure arguments: '$NDPI_CONFIGURE_ARGS'"; \
./autogen.sh; \
./configure --prefix=/usr --with-only-libndpi $NDPI_CONFIGURE_ARGS CC="${{ matrix.compiler }}" CXX=false \
CFLAGS="$CMAKE_C_FLAGS" && make && sudo make install; cd ..; }
ls -alhR /usr/include/ndpi
cd ..
test $WGET_RET -ne 0 || { echo "Running CMake.. (pkgconfig)"; \
cmake -S . -B ./build-local-pkgconfig \
-DCMAKE_C_COMPILER="$CMAKE_C_COMPILER" -DCMAKE_C_FLAGS="$CMAKE_C_FLAGS" \
-DCMAKE_C_EXE_LINKER_FLAGS="$CMAKE_C_EXE_LINKER_FLAGS" \
-DBUILD_NDPI=OFF -DBUILD_EXAMPLES=ON \
-DENABLE_DBUS=ON -DENABLE_CURL=ON -DENABLE_SYSTEMD=ON \
${{ matrix.poll }} ${{ matrix.coverage }} \
${{ matrix.sanitizer }} ${{ matrix.ndpid_examples }} ${{ matrix.ndpid_rust_examples }}; }
test $WGET_RET -ne 0 || { echo "Running Make.. (pkgconfig)"; \
cmake --build ./build-local-pkgconfig --verbose; }
test $WGET_RET -ne 0 || { echo "Testing Executable.. (pkgconfig)"; \
./build-local-pkgconfig/nDPId-test; \
./build-local-pkgconfig/nDPId -h || test $? -eq 1; \
./build-local-pkgconfig/nDPIsrvd -h || test $? -eq 1; }
test $WGET_RET -ne 0 || { echo "Running CMake.. (static)"; \
cmake -S . -B ./build-local-static \
-DCMAKE_C_COMPILER="$CMAKE_C_COMPILER" -DCMAKE_C_FLAGS="$CMAKE_C_FLAGS" \
-DCMAKE_C_EXE_LINKER_FLAGS="$CMAKE_C_EXE_LINKER_FLAGS" \
-DBUILD_NDPI=OFF -DBUILD_EXAMPLES=ON \
-DENABLE_DBUS=ON -DENABLE_CURL=ON -DENABLE_SYSTEMD=ON \
-DNDPI_NO_PKGCONFIG=ON -DSTATIC_LIBNDPI_INSTALLDIR=/usr \
${{ matrix.poll }} ${{ matrix.coverage }} ${{ matrix.ndpid_gcrypt }} \
${{ matrix.sanitizer }} ${{ matrix.ndpid_examples }} ${{ matrix.ndpid_rust_examples }}; }
test $WGET_RET -ne 0 || { echo "Running Make.. (static)"; \
cmake --build ./build-local-static --verbose; }
test $WGET_RET -ne 0 || { echo "Testing Executable.. (static)"; \
./build-local-static/nDPId-test; \
./build-local-static/nDPId -h || test $? -eq 1; \
./build-local-static/nDPIsrvd -h || test $? -eq 1; }
test $WGET_RET -ne 0 || test ! -d ./PF_RING || { echo "Running CMake.. (PF_RING)"; \
cmake -S . -B ./build-local-pfring \
-DCMAKE_C_COMPILER="$CMAKE_C_COMPILER" -DCMAKE_C_FLAGS="$CMAKE_C_FLAGS" \
-DCMAKE_C_EXE_LINKER_FLAGS="$CMAKE_C_EXE_LINKER_FLAGS" \
-DBUILD_NDPI=OFF -DBUILD_EXAMPLES=ON -DENABLE_PFRING=ON \
-DENABLE_DBUS=ON -DENABLE_CURL=ON -DENABLE_SYSTEMD=ON \
-DNDPI_NO_PKGCONFIG=ON -DSTATIC_LIBNDPI_INSTALLDIR=/usr \
-DPFRING_LINK_STATIC=OFF \
-DPFRING_INSTALLDIR=/usr -DPFRING_KERNEL_INC="$(realpath ./PF_RING/kernel)" \
${{ matrix.poll }} ${{ matrix.coverage }} ${{ matrix.ndpid_gcrypt }} \
${{ matrix.sanitizer }} ${{ matrix.ndpid_examples }} ${{ matrix.ndpid_rust_examples }}; }
test $WGET_RET -ne 0 || test ! -d ./PF_RING || { echo "Running Make.. (PF_RING)"; \
cmake --build ./build-local-pfring --verbose; }
test $WGET_RET -ne 0 || test ! -d ./PF_RING || { echo "Testing Executable.. (PF_RING)"; \
./build-local-pfring/nDPId-test; \
./build-local-pfring/nDPId -h || test $? -eq 1; \
./build-local-pfring/nDPIsrvd -h || test $? -eq 1; }
test $WGET_RET -eq 0 -o $WGET_RET -eq 8
wget 'https://github.com/ntop/nDPI/archive/refs/tags/${{ matrix.ndpi_min_version }}.tar.gz'
tar -xzvf ${{ matrix.ndpi_min_version }}.tar.gz && cd nDPI-${{ matrix.ndpi_min_version }} && ./autogen.sh --prefix=/usr --with-only-libndpi CC=${{ matrix.compiler }} CXX=false CFLAGS='-Werror' && sudo make install && cd ..
cmake .. -DENABLE_COVERAGE=ON -DBUILD_EXAMPLES=ON -DBUILD_NDPI=OFF -DENABLE_SANITIZER=ON ${{ matrix.ndpi_min_version }}
make all VERBOSE=1

66
.github/workflows/sonarcloud.yml vendored
View File

@@ -1,66 +0,0 @@
on:
push:
branches:
- main
- tmp
pull_request:
types: [opened, synchronize, reopened]
name: Sonarcloud Scan
jobs:
sonarcloud:
runs-on: ubuntu-latest
env:
BUILD_WRAPPER_OUT_DIR: build_wrapper_output_directory
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Python 3.8 for gcovr
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: install gcovr 5.0
run: |
pip install gcovr==5.0 # 5.1 is not supported
- name: Install sonar-scanner and build-wrapper
uses: SonarSource/sonarcloud-github-c-cpp@v3.2.0
- name: Install Prerequisites
run: |
sudo apt-get update
sudo apt-get install autoconf automake cmake lcov \
libtool pkg-config gettext \
libjson-c-dev flex bison \
libcurl4-openssl-dev libpcap-dev zlib1g-dev
- name: Run build-wrapper
run: |
build-wrapper-linux-x86-64 --out-dir ${{ env.BUILD_WRAPPER_OUT_DIR }} ./scripts/build-sonarcloud.sh
- name: Run tests
run: |
for file in $(ls libnDPI/tests/cfgs/*/pcap/*.pcap libnDPI/tests/cfgs/*/pcap/*.pcapng libnDPI/tests/cfgs/*/pcap/*.cap); do \
echo -n "${file} "; \
cd ./build-sonarcloud; \
./nDPId-test "../${file}" >/dev/null 2>/dev/null; \
cd ..; \
echo "[ok]"; \
done
mkdir -p gcov_report
cd gcov_report
gcov ../build-sonarcloud/CMakeFiles/nDPId-test.dir/nDPId-test.c.o
cd ..
- name: Run sonar-scanner
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
run: |
sonar-scanner \
--define sonar.projectName=nDPId \
--define sonar.projectVersion=1.7 \
--define sonar.sourceEncoding=UTF-8 \
--define sonar.branch.name=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} \
--define sonar.organization=lnslbrty \
--define sonar.projectKey=lnslbrty_nDPId \
--define sonar.python.version=3.8 \
--define sonar.cfamily.compile-commands=${{ env.BUILD_WRAPPER_OUT_DIR }}/compile_commands.json \
--define sonar.cfamily.gcov.reportsPath=gcov_report \
--define sonar.exclusions=build-sonarcloud/**,libnDPI/**,test/results/**,dependencies/jsmn/**,dependencies/uthash/**,examples/js-rt-analyzer-frontend/**,examples/js-rt-analyzer/**,examples/c-collectd/www/**,examples/py-flow-dashboard/assets/**

99
.gitlab-ci.yml
View File

@@ -3,126 +3,47 @@ image: debian:stable
stages:
- build_and_test
variables:
GIT_CLONE_PATH: '$CI_BUILDS_DIR/$CI_JOB_ID/$CI_PROJECT_NAME'
before_script:
- export DEBIAN_FRONTEND=noninteractive
- apt-get update -qq
- >
apt-get install -y -qq \
coreutils sudo \
build-essential make cmake binutils gcc clang autoconf automake \
libtool pkg-config git wget unzip \
build-essential make cmake binutils gcc autoconf automake \
libtool pkg-config git \
libpcap-dev libgpg-error-dev libjson-c-dev zlib1g-dev \
netcat-openbsd python3 python3-jsonschema tree lcov iproute2
netcat-openbsd python3 python3-jsonschema tree lcov
after_script:
- test -r /tmp/nDPIsrvd.log && cat /tmp/nDPIsrvd.log
- test -r /tmp/nDPId.log && cat /tmp/nDPId.log
- cat /tmp/nDPIsrvd.log
- cat /tmp/nDPId.log
build_and_test_static_libndpi_tsan:
build_and_test:
script:
# test for NETWORK_BUFFER_MAX_SIZE C and Python value equality
- C_VAL=$(cat config.h | sed -n 's/^#define\s\+NETWORK_BUFFER_MAX_SIZE\s\+\([0-9]\+\).*$/\1/gp')
- PY_VAL=$(cat dependencies/nDPIsrvd.py | sed -n 's/^NETWORK_BUFFER_MAX_SIZE = \([0-9]\+\).*$/\1/gp')
- test ${C_VAL} = ${PY_VAL}
# test for nDPId_PACKETS_PLEN_MAX C and Python value equality
- C_VAL=$(cat config.h | sed -n 's/^#define\s\+nDPId_PACKETS_PLEN_MAX\s\+\([0-9]\+\).*$/\1/gp')
- PY_VAL=$(cat dependencies/nDPIsrvd.py | sed -n 's/^nDPId_PACKETS_PLEN_MAX = \([0-9]\+\).*$/\1/gp')
- test ${C_VAL} = ${PY_VAL}
# static linked build
- mkdir build-clang-tsan
- cd build-clang-tsan
- env CMAKE_C_FLAGS='-Werror' CMAKE_C_COMPILER='clang' cmake .. -DBUILD_EXAMPLES=ON -DBUILD_NDPI=ON -DBUILD_NDPI_FORCE_GIT_UPDATE=ON -DENABLE_SANITIZER_THREAD=ON -DENABLE_ZLIB=ON
- make clean-libnDPI
- make libnDPI
- tree libnDPI
- make install VERBOSE=1 DESTDIR="$(realpath ../_install)"
- cd ..
- ./_install/usr/local/bin/nDPId-test
- ./test/run_tests.sh ./libnDPI ./_install/usr/local/bin/nDPId-test
artifacts:
expire_in: 1 week
paths:
- _install/
stage: build_and_test
build_and_test_static_libndpi:
script:
- mkdir build-cmake-submodule
- cd build-cmake-submodule
- env CMAKE_C_FLAGS='-Werror' cmake .. -DENABLE_SYSTEMD=ON -DBUILD_EXAMPLES=ON -DBUILD_NDPI=ON -DBUILD_NDPI_FORCE_GIT_UPDATE=ON -DENABLE_ZLIB=ON
- make clean-libnDPI
- env CMAKE_C_FLAGS='-Werror' cmake .. -DENABLE_COVERAGE=ON -DBUILD_EXAMPLES=ON -DBUILD_NDPI=ON -DENABLE_SANITIZER=ON -DENABLE_ZLIB=ON
- make libnDPI
- tree libnDPI
- make install VERBOSE=1 DESTDIR="$(realpath ../_install)"
- cpack -G DEB
- sudo dpkg -i nDPId-*.deb
- cd ..
- test -x /bin/systemctl && sudo systemctl daemon-reload
- test -x /bin/systemctl && sudo systemctl enable ndpid@lo
- test -x /bin/systemctl && sudo systemctl start ndpid@lo
- test -x /bin/systemctl && sudo systemctl status ndpisrvd.service ndpid@lo.service
- test -x /bin/systemctl && sudo systemctl stop ndpid@lo
- ./build-cmake-submodule/nDPId-test
- ./test/run_tests.sh ./libnDPI ./build-cmake-submodule/nDPId-test
- >
if ldd ./build-cmake-submodule/nDPId | grep -qoEi libndpi; then \
echo 'nDPId linked against a static libnDPI should not contain a shared linked libnDPI.' >&2; false; fi
- cc -Wall -Wextra -std=gnu99 nDPId.c nio.c utils.c -I./build-cmake-submodule/libnDPI/include/ndpi -I. -I./dependencies -I./dependencies/jsmn -I./dependencies/uthash/include -o /tmp/a.out -lpcap ./build-cmake-submodule/libnDPI/lib/libndpi.a -pthread -lm -lz
artifacts:
expire_in: 1 week
paths:
- build-cmake-submodule/*.deb
- _install/
stage: build_and_test
build_and_test_static_libndpi_coverage:
script:
- mkdir build-cmake-submodule
- cd build-cmake-submodule
- env CMAKE_C_FLAGS='-Werror' cmake .. -DENABLE_SYSTEMD=ON -DENABLE_COVERAGE=ON -DBUILD_EXAMPLES=ON -DBUILD_NDPI=ON -DBUILD_NDPI_FORCE_GIT_UPDATE=ON -DENABLE_SANITIZER=ON -DENABLE_ZLIB=ON
- make clean-libnDPI
- make libnDPI
- tree libnDPI
- make install VERBOSE=1 DESTDIR="$(realpath ../_install)"
- cd ..
- ./build-cmake-submodule/nDPId-test
- ./test/run_tests.sh ./libnDPI ./build-cmake-submodule/nDPId-test
# generate coverage report
- make -C ./build-cmake-submodule coverage || true
- make -C ./build-cmake-submodule coverage
- >
if ldd build/nDPId | grep -qoEi libndpi; then \
echo 'nDPId linked against a static libnDPI should not contain a shared linked libnDPI.' >&2; false; fi
artifacts:
expire_in: 1 week
paths:
- build-cmake-submodule/coverage_report
- _install/
stage: build_and_test
build_dynamic_libndpi:
script:
# pkg-config dynamic linked build
- git clone https://github.com/ntop/nDPI.git
- cd nDPI
- ./autogen.sh
- ./configure --prefix="$(realpath ../_install)" --enable-option-checking=fatal
- make install V=s
- cd ..
- tree ./_install
- mkdir build
- cd build
- export CMAKE_PREFIX_PATH="$(realpath ../_install)"
- export PKG_CONFIG_PATH="$(realpath ../build-cmake-submodule/libnDPI/lib/pkgconfig)"
- env CMAKE_C_FLAGS='-Werror' cmake .. -DBUILD_EXAMPLES=ON -DENABLE_SANITIZER=ON -DENABLE_MEMORY_PROFILING=ON -DENABLE_ZLIB=ON
- make all VERBOSE=1
- make install VERBOSE=1 DESTDIR="$(realpath ../_install)"
- cd ..
- tree ./_install
- ./build/nDPId-test
- ./build/nDPId-test || test $? -eq 1
- ./build/nDPId -h || test $? -eq 1
- ./build/nDPIsrvd -h || test $? -eq 1
# dameon start/stop test
- NUSER=nobody make -C ./build daemon VERBOSE=1
- NUSER=nobody make -C ./build daemon VERBOSE=1

9
.gitmodules vendored
View File

@@ -3,12 +3,3 @@
url = https://github.com/ntop/nDPI
branch = dev
update = rebase
[submodule "examples/js-rt-analyzer"]
path = examples/js-rt-analyzer
url = https://gitlab.com/verzulli/ndpid-rt-analyzer.git
[submodule "examples/js-rt-analyzer-frontend"]
path = examples/js-rt-analyzer-frontend
url = https://gitlab.com/verzulli/ndpid-rt-analyzer-frontend.git
[submodule "examples/cxx-graph"]
path = examples/cxx-graph
url = https://github.com/utoni/nDPId-Graph.git

83
CHANGELOG.md
View File

@@ -1,83 +0,0 @@
# CHANGELOG
#### nDPId 1.7 (Oct 2024)
- Read and parse configuration files for nDPId (+ libnDPI) and nDPIsrvd
- Added loading risk domains from a file (`-R`, thanks to @UnveilTech)
- Added Filebeat configuration file
- Improved hostname handling; will now always be part of `analyse`/`end`/`idle` events (if dissected)
- Improved Documentation (INSTALL / Schema)
- Added PF\_RING support
- Improved nDPIsrvd-analyse to write global stats to a CSV
- Added global (heap) memory stats for daemon status events (if enabled)
- Fixed IPv6 address/netmask retrieval on some systems
- Improved nDPIsrvd-collect; gauges and counters are now handled the right way
- Added nDPId Grafana dashboard
- Fixed `detection-update` event bug; was thrown even if nothing changed
- Fixed `not-detected` event spam if detection not completed (in some rare cases)
- Improved InfluxDB push daemon (severity parsing / gauge handling)
- Improved zLib compression
- Fixed nDPIsrvd-collectd missing escape character
#### nDPId 1.6 (Nov 2023)
- Added Event I/O abstraction layer (supporting only poll/epoll by now)
- Support for OSX and *BSD systems
- Added proper DLT_RAW dissection for IPv4 and IPv6
- Improved TCP timeout handling if FIN/RST seen which caused Midstream TCP flows when there shouldn't be any
- Fixed a crash if `nDPId -o value=''` was used
- Added OpenWrt packaging
- Added new flow event "analyse" used to give some statistical information about active flows
- Added new analyse event daemon which generates CSV files from such events
- Fixed a crash in nDPIsrvd if a collector closes a connection
- Support `nDPId` to send it's data to a UDP endpoint instead of a nDPIsrvd collector
- Added events and flow states documentation
- Added basic systemd support
- Fixed a bug in base64 encoding which could lead to invalid base64 strings
- Added some machine learning examples
- Fixed various smaller bugs
- Fixed nDPIsrvd bug which causes invalid JSON messages sent to Distributors
#### nDPId 1.5 (Apr 2022)
- Improved nDPId cross compilation
- zLib flow memory compression (Experimental!)
- Memory profiling for nDPId-test
- JSMN with parent link support for subtoken iteration
- Refactored nDPIsrvd buffer and buffer bloat handling
- Upgraded JSMN/uthash
- Improved nDPIsrvd.(h|py) debugging capability for client apps
- Advanced flow usage logging usable for memory profiling
- Support for dissection additional layer2/layer3 protocols
- Serialize more JSON information
- Add TCP/IP support for nDPIsrvd
- Improved nDPIsrvd connection lost behaviour
- Reworked Python/C distributor API
- Support read()/recv() timeouts and nonblocking I/O
#### nDPId 1.4 (Jun 2021)
- Use layer4 specific flow timeouts for nDPId
- Reworked layer4 flow length names and calculations (use only layer4 payload w/o any previous headers) for nDPId
- Build system cleanup and cosmetics
#### nDPId 1.3 (May 2021)
- Added missing datalink layer types
#### nDPId 1.2 (May 2021)
- OpenWrt compatible build system
#### nDPId 1.1 (May 2021)
- Added License information
#### nDPId 1.0 (May 2021)
- First public release

518
CMakeLists.txt
View File

@@ -1,85 +1,37 @@
cmake_minimum_required(VERSION 3.12.4)
project(nDPId C)
if(CMAKE_COMPILER_IS_GNUCXX)
execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION)
if (GCC_VERSION VERSION_GREATER 4.7 OR GCC_VERSION VERSION_EQUAL 4.7)
message(STATUS "${CMAKE_C_COMPILER} supports C11 standard.")
else ()
message(FATAL_ERROR "C Compiler with C11 standard needed. Therefore a gcc compiler with a version equal or higher than 4.7 is needed.")
endif()
endif(CMAKE_COMPILER_IS_GNUCXX)
set(CMAKE_C_STANDARD 11)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -D_DEFAULT_SOURCE=1 -D_GNU_SOURCE=1")
if("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}")
message(FATAL_ERROR "In-source builds are not allowed.\n"
"Please remove ${PROJECT_SOURCE_DIR}/CMakeCache.txt\n"
"and\n"
"${PROJECT_SOURCE_DIR}/CMakeFiles\n"
"Create a build directory somewhere and run CMake again.\n"
"Or run: 'cmake -S ${PROJECT_SOURCE_DIR} -B ./your-custom-build-dir [CMAKE-OPTIONS]'")
"Create a build directory somewhere and run CMake again.")
endif()
set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake)
find_package(PkgConfig REQUIRED)
set(CPACK_PACKAGE_CONTACT "toni@impl.cc")
set(CPACK_DEBIAN_PACKAGE_NAME "nDPId")
set(CPACK_DEBIAN_PACKAGE_SECTION "network")
set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "nDPId is a set of daemons and tools to capture, process and classify network traffic.")
set(CPACK_DEBIAN_PACKAGE_MAINTAINER "Toni Uhlig <toni@impl.cc>")
set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_SOURCE_DIR}/packages/debian/preinst;${CMAKE_SOURCE_DIR}/packages/debian/prerm;${CMAKE_SOURCE_DIR}/packages/debian/postrm")
set(CPACK_DEBIAN_PACKAGE_CONTROL_STRICT_PERMISSION TRUE)
set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
set(CPACK_DEBIAN_DEBUGINFO_PACKAGE ON)
set(CPACK_RPM_PACKAGE_LICENSE "GPL-3")
set(CPACK_RPM_PACKAGE_VENDOR "Toni Uhlig")
set(CPACK_RPM_PACKAGE_URL "https://www.github.com/utoni/nDPId.git")
set(CPACK_RPM_PACKAGE_DESCRIPTION "nDPId is a set of daemons and tools to capture, process and classify network traffic.")
set(CPACK_RPM_PRE_INSTALL_SCRIPT_FILE "${CMAKE_SOURCE_DIR}/packages/redhat/pre_install")
set(CPACK_RPM_PRE_UNINSTALL_SCRIPT_FILE "${CMAKE_SOURCE_DIR}/packages/redhat/pre_uninstall")
set(CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_SOURCE_DIR}/packages/redhat/post_uninstall")
set(CPACK_STRIP_FILES ON)
set(CPACK_PACKAGE_VERSION_MAJOR 1)
set(CPACK_PACKAGE_VERSION_MINOR 7)
set(CPACK_PACKAGE_VERSION_MINOR 5)
set(CPACK_PACKAGE_VERSION_PATCH 0)
# Note: CPACK_PACKAGING_INSTALL_PREFIX and CMAKE_INSTALL_PREFIX are *not* the same.
# It is used only to ease environment file loading via systemd.
set(CPACK_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}")
set(CMAKE_MACOSX_RPATH 1)
include(CPack)
include(CheckFunctionExists)
include(CheckLibraryExists)
include(CheckEpoll)
check_epoll(HAS_EPOLL)
if(HAS_EPOLL)
option(FORCE_POLL "Force the use of poll() instead of epoll()." OFF)
if(NOT FORCE_POLL)
set(EPOLL_DEFS "-DENABLE_EPOLL=1")
endif()
else()
if(BUILD_EXAMPLES)
message(FATAL_ERROR "Examples are using epoll event I/O. Without epoll available, you can not build/run those.")
endif()
endif()
if(NOT MATH_FUNCTION_EXISTS AND NOT NEED_LINKING_AGAINST_LIBM)
CHECK_FUNCTION_EXISTS(log2f MATH_FUNCTION_EXISTS)
if(NOT MATH_FUNCTION_EXISTS)
unset(MATH_FUNCTION_EXISTS CACHE)
list(APPEND CMAKE_REQUIRED_LIBRARIES m)
CHECK_FUNCTION_EXISTS(log2f MATH_FUNCTION_EXISTS)
if(MATH_FUNCTION_EXISTS)
set(NEED_LINKING_AGAINST_LIBM TRUE CACHE BOOL "" FORCE)
else()
check_library_exists(m sqrt "" NEED_LINKING_AGAINST_LIBM)
if(NOT NEED_LINKING_AGAINST_LIBM)
# Was not able to figure out if explicit linkage against libm is required.
# Forcing libm linkage. Good idea?
set(NEED_LINKING_AGAINST_LIBM TRUE CACHE BOOL "" FORCE)
endif()
endif()
endif()
list(APPEND CMAKE_REQUIRED_LIBRARIES m)
CHECK_FUNCTION_EXISTS(log2f MATH_FUNCTION_EXISTS)
if(MATH_FUNCTION_EXISTS)
set(NEED_LINKING_AGAINST_LIBM TRUE CACHE BOOL "" FORCE)
else()
message(FATAL_ERROR "Failed making the log2f() function available")
endif()
endif()
endif()
if(NEED_LINKING_AGAINST_LIBM)
@@ -93,85 +45,9 @@ option(ENABLE_SANITIZER "Enable ASAN/LSAN/UBSAN." OFF)
option(ENABLE_SANITIZER_THREAD "Enable TSAN (does not work together with ASAN)." OFF)
option(ENABLE_MEMORY_PROFILING "Enable dynamic memory tracking." OFF)
option(ENABLE_ZLIB "Enable zlib support for nDPId (experimental)." OFF)
option(ENABLE_SYSTEMD "Install systemd components." OFF)
option(ENABLE_CRYPTO "Enable OpenSSL cryptographic support in nDPId/nDPIsrvd." OFF)
option(BUILD_EXAMPLES "Build C examples." ON)
option(BUILD_RUST_EXAMPLES "Build Rust examples." OFF)
if(BUILD_EXAMPLES)
option(ENABLE_DBUS "Build DBus notification example." OFF)
option(ENABLE_CURL "Build influxdb data write example." OFF)
endif()
option(ENABLE_PFRING "Enable PF_RING support for nDPId (experimental)" OFF)
option(BUILD_NDPI "Clone and build nDPI from github." OFF)
if(ENABLE_PFRING)
option(PFRING_LINK_STATIC "Link against a static version of pfring." ON)
set(PFRING_KERNEL_INC "" CACHE STRING "Path to PFRING kernel module include directory.")
set(PFRING_DEFS "-DENABLE_PFRING=1")
if(PFRING_KERNEL_INC STREQUAL "")
message(FATAL_ERROR "PFRING_KERNEL_INC needs to be set to the PFRING kernel module include directory.")
endif()
if(NOT EXISTS "${PFRING_KERNEL_INC}/linux/pf_ring.h")
message(FATAL_ERROR "Expected to find <linux/pf_ring.h> below ${PFRING_KERNEL_INC}, but none found.")
endif()
set(PFRING_INSTALLDIR "/opt/PF_RING/usr" CACHE STRING "")
set(PFRING_INC "${PFRING_INSTALLDIR}/include")
if(NOT EXISTS "${PFRING_INC}")
message(FATAL_ERROR "Include directory \"${PFRING_INC}\" does not exist!")
endif()
if(PFRING_LINK_STATIC)
if(CMAKE_SIZEOF_VOID_P EQUAL 8)
if(EXISTS "${PFRING_INSTALLDIR}/lib64")
set(STATIC_PFRING_LIB "${PFRING_INSTALLDIR}/lib64/libpfring.a")
else()
set(STATIC_PFRING_LIB "${PFRING_INSTALLDIR}/lib/libpfring.a")
endif()
else()
if(EXISTS "${PFRING_INSTALLDIR}/lib32")
set(STATIC_PFRING_LIB "${PFRING_INSTALLDIR}/lib32/libpfring.a")
else()
set(STATIC_PFRING_LIB "${PFRING_INSTALLDIR}/lib/libpfring.a")
endif()
endif()
if(NOT EXISTS "${STATIC_PFRING_LIB}")
message(FATAL_ERROR "Static library \"${STATIC_PFRING_LIB}\" does not exist!")
endif()
else()
if(CMAKE_SIZEOF_VOID_P EQUAL 8)
if(EXISTS "${PFRING_INSTALLDIR}/lib64")
find_library(PF_RING_LIB pfring PATHS "${PFRING_INSTALLDIR}/lib64")
else()
find_library(PF_RING_LIB pfring PATHS "${PFRING_INSTALLDIR}/lib")
endif()
else()
if(EXISTS "${PFRING_INSTALLDIR}/lib32")
find_library(PF_RING_LIB pfring PATHS "${PFRING_INSTALLDIR}/lib32")
else()
find_library(PF_RING_LIB pfring PATHS "${PFRING_INSTALLDIR}/lib")
endif()
endif()
if(NOT PF_RING_LIB)
message(FATAL_ERROR "libpfring.so not found below ${PFRING_INSTALLDIR}/{lib,lib32,lib64}")
endif()
endif()
if(NOT EXISTS "${PFRING_INSTALLDIR}/include/pfring.h")
message(FATAL_ERROR "Expected to find <include/pfring.h> inside ${PFRING_INSTALLDIR}, but none found.")
endif()
else()
unset(PFRING_INSTALLDIR CACHE)
unset(PFRING_INC CACHE)
unset(STATIC_PFRING_LIB CACHE)
unset(PFRING_LINK_STATIC CACHE)
endif()
if(BUILD_NDPI)
option(BUILD_NDPI_FORCE_GIT_UPDATE "Forcefully instruments nDPI build script to update the git submodule." OFF)
unset(NDPI_NO_PKGCONFIG CACHE)
unset(STATIC_LIBNDPI_INSTALLDIR CACHE)
else()
@@ -196,40 +72,27 @@ else()
unset(NDPI_WITH_MAXMINDDB CACHE)
endif()
if(ENABLE_PFRING)
set(NDPID_PFRING_SRCS npfring.c)
endif()
if(ENABLE_CRYPTO)
set(CRYPTO_SRCS ncrypt.c)
endif()
add_executable(nDPId nDPId.c ${NDPID_PFRING_SRCS} ${CRYPTO_SRCS} nio.c utils.c)
add_executable(nDPIsrvd nDPIsrvd.c nio.c utils.c)
add_executable(nDPId-test nDPId-test.c ${NDPID_PFRING_SRCS} ${CRYPTO_SRCS})
set(CROSS_COMPILE_TRIPLET "" CACHE STRING "Host triplet used to enable cross compiling.")
add_custom_target(umask_check)
add_custom_command(
TARGET umask_check
PRE_BUILD
COMMAND ${CMAKE_SOURCE_DIR}/scripts/umask-check.sh
)
add_dependencies(nDPId umask_check)
add_executable(nDPId nDPId.c utils.c)
add_executable(nDPIsrvd nDPIsrvd.c utils.c)
add_executable(nDPId-test nDPId-test.c)
add_custom_target(dist)
add_custom_command(
TARGET dist
PRE_BUILD
COMMAND "${CMAKE_SOURCE_DIR}/scripts/make-dist.sh"
)
add_custom_target(daemon)
add_custom_command(
TARGET daemon
POST_BUILD
COMMAND env nDPIsrvd_ARGS='-C 1024' "${CMAKE_SOURCE_DIR}/scripts/daemon.sh" "$<TARGET_FILE:nDPId>" "$<TARGET_FILE:nDPIsrvd>"
TARGET daemon
COMMAND "${CMAKE_SOURCE_DIR}/scripts/daemon.sh" "$<TARGET_FILE:nDPId>" "$<TARGET_FILE:nDPIsrvd>"
DEPENDS nDPId nDPIsrvd
)
add_dependencies(daemon nDPId nDPIsrvd)
if(CMAKE_CROSSCOMPILING)
if(NOT CROSS_COMPILE_TRIPLET STREQUAL "")
set(CMAKE_C_COMPILER_TARGET ${CROSS_COMPILE_TRIPLET})
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
@@ -252,29 +115,14 @@ if(ENABLE_COVERAGE)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --coverage")
set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} --coverage")
add_custom_target(coverage DEPENDS "${CMAKE_BINARY_DIR}/coverage_report/nDPId/index.html")
add_custom_target(coverage)
add_custom_command(
OUTPUT "${CMAKE_BINARY_DIR}/coverage_report/nDPId/index.html"
COMMAND lcov --directory "${CMAKE_BINARY_DIR}" --directory "${CMAKE_SOURCE_DIR}/libnDPI" --capture --output-file "${CMAKE_BINARY_DIR}/lcov.info"
COMMAND genhtml -o "${CMAKE_BINARY_DIR}/coverage_report" "${CMAKE_BINARY_DIR}/lcov.info"
DEPENDS nDPId nDPId-test nDPIsrvd
TARGET coverage
COMMAND "${CMAKE_SOURCE_DIR}/scripts/code-coverage.sh"
DEPENDS nDPId nDPIsrvd nDPId-test
)
add_custom_target(coverage-clean)
add_custom_command(
TARGET coverage-clean
COMMAND find "${CMAKE_BINARY_DIR}" "${CMAKE_SOURCE_DIR}/libnDPI" -name "*.gcda" -delete
POST_BUILD
)
add_custom_target(coverage-view)
add_custom_command(
TARGET coverage-view
COMMAND cd "${CMAKE_BINARY_DIR}/coverage_report" && python3 -m http.server
POST_BUILD
)
add_dependencies(coverage-view coverage)
endif()
if(ENABLE_SANITIZER)
# TODO: Check for `-fsanitize-memory-track-origins` and add if available?
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address -fsanitize=undefined -fno-sanitize=alignment -fsanitize=enum -fsanitize=leak")
endif()
if(ENABLE_SANITIZER_THREAD)
@@ -284,35 +132,17 @@ if(ENABLE_ZLIB)
set(ZLIB_DEFS "-DENABLE_ZLIB=1")
pkg_check_modules(ZLIB REQUIRED zlib)
endif()
if(BUILD_EXAMPLES)
if(ENABLE_DBUS)
pkg_check_modules(DBUS REQUIRED dbus-1)
endif()
if(ENABLE_CURL)
pkg_check_modules(CURL REQUIRED libcurl)
endif()
if(NDPI_WITH_GCRYPT)
message(STATUS "Enable GCRYPT")
set(NDPI_ADDITIONAL_ARGS "${NDPI_ADDITIONAL_ARGS} --with-local-libgcrypt")
endif()
if(BUILD_NDPI)
if(NDPI_WITH_GCRYPT)
message(STATUS "nDPI: Enable GCRYPT")
set(NDPI_ADDITIONAL_ARGS "${NDPI_ADDITIONAL_ARGS} --with-local-libgcrypt")
endif()
if(NDPI_WITH_PCRE)
message(STATUS "nDPI: Enable PCRE")
set(NDPI_ADDITIONAL_ARGS "${NDPI_ADDITIONAL_ARGS} --with-pcre2")
endif()
if(NDPI_WITH_MAXMINDDB)
message(STATUS "nDPI: Enable MAXMINDDB")
set(NDPI_ADDITIONAL_ARGS "${NDPI_ADDITIONAL_ARGS} --with-maxminddb")
endif()
if(ENABLE_COVERAGE)
message(STATUS "nDPI: Enable Coverage")
set(NDPI_ADDITIONAL_ARGS "${NDPI_ADDITIONAL_ARGS} --enable-code-coverage")
endif()
if(CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "")
message(STATUS "nDPI: Enable Debug Build")
set(NDPI_ADDITIONAL_ARGS "${NDPI_ADDITIONAL_ARGS} --enable-debug-build --enable-debug-messages")
endif()
if(NDPI_WITH_PCRE)
message(STATUS "Enable PCRE")
set(NDPI_ADDITIONAL_ARGS "${NDPI_ADDITIONAL_ARGS} --with-pcre")
endif()
if(NDPI_WITH_MAXMINDDB)
message(STATUS "Enable MAXMINDDB")
set(NDPI_ADDITIONAL_ARGS "${NDPI_ADDITIONAL_ARGS} --with-maxminddb")
endif()
execute_process(
@@ -327,7 +157,6 @@ if(GIT_VERSION STREQUAL "" OR NOT IS_DIRECTORY "${CMAKE_SOURCE_DIR}/.git")
set(GIT_VERSION "${CPACK_PACKAGE_VERSION}-release")
endif()
endif()
set(PKG_VERSION "${CPACK_PACKAGE_VERSION}")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra")
set(NDPID_DEFS -DJSMN_STATIC=1 -DJSMN_STRICT=1 -DJSMN_PARENT_LINKS=1)
@@ -335,13 +164,8 @@ set(NDPID_DEPS_INC "${CMAKE_SOURCE_DIR}"
"${CMAKE_SOURCE_DIR}/dependencies"
"${CMAKE_SOURCE_DIR}/dependencies/jsmn"
"${CMAKE_SOURCE_DIR}/dependencies/uthash/src")
if(CMAKE_CROSSCOMPILING)
add_definitions("-DCROSS_COMPILATION=1")
endif()
if(ENABLE_MEMORY_PROFILING)
if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug" AND NOT CMAKE_BUILD_TYPE STREQUAL "")
message(WARNING "ENABLE_MEMORY_PROFILING should not be used in production environments.")
endif()
message(WARNING "ENABLE_MEMORY_PROFILING should not be used in production environments.")
add_definitions("-DENABLE_MEMORY_PROFILING=1"
"-Duthash_malloc=nDPIsrvd_uthash_malloc"
"-Duthash_free=nDPIsrvd_uthash_free")
@@ -366,15 +190,13 @@ if(BUILD_NDPI)
CONFIGURE_COMMAND env
CC=${CMAKE_C_COMPILER}
CXX=false
AR=${CMAKE_AR}
RANLIB=${CMAKE_RANLIB}
PKG_CONFIG=${PKG_CONFIG_EXECUTABLE}
CFLAGS=${CMAKE_C_FLAGS}
LDFLAGS=${CMAKE_MODULE_LINKER_FLAGS}
CROSS_COMPILE_TRIPLET=${CROSS_COMPILE_TRIPLET}
ADDITIONAL_ARGS=${NDPI_ADDITIONAL_ARGS}
MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}
DEST_INSTALL=${CMAKE_BINARY_DIR}/libnDPI
FORCE_GIT_UPDATE=${BUILD_NDPI_FORCE_GIT_UPDATE}
${CMAKE_CURRENT_SOURCE_DIR}/scripts/get-and-build-libndpi.sh
BUILD_BYPRODUCTS ${CMAKE_BINARY_DIR}/libnDPI/lib/libndpi.a
BUILD_COMMAND ""
@@ -382,9 +204,7 @@ if(BUILD_NDPI)
BUILD_IN_SOURCE 1)
add_custom_target(clean-libnDPI
COMMAND ${CMAKE_BUILD_TOOL} clean
COMMAND rm -rf ${CMAKE_BINARY_DIR}/libnDPI
COMMAND rm -f ${CMAKE_BINARY_DIR}/libnDPI-prefix/src/libnDPI-stamp/libnDPI-configure
COMMAND rm -rf ${CMAKE_BINARY_DIR}/libnDPI ${CMAKE_BINARY_DIR}/libnDPI-prefix
)
set(STATIC_LIBNDPI_INSTALLDIR "${CMAKE_BINARY_DIR}/libnDPI")
@@ -392,19 +212,13 @@ if(BUILD_NDPI)
add_dependencies(nDPId-test libnDPI)
endif()
if(ENABLE_CRYPTO)
find_package(OpenSSL REQUIRED)
set(OSSL_DEFS "-DENABLE_CRYPTO=1")
set(OSSL_LIBRARY "${OPENSSL_SSL_LIBRARY}" "${OPENSSL_CRYPTO_LIBRARY}")
endif()
if(STATIC_LIBNDPI_INSTALLDIR OR BUILD_NDPI OR NDPI_NO_PKGCONFIG)
if(NDPI_WITH_GCRYPT)
find_package(GCRYPT "1.4.2" REQUIRED)
endif()
if(NDPI_WITH_PCRE)
pkg_check_modules(PCRE REQUIRED libpcre2-8)
pkg_check_modules(PCRE REQUIRED libpcre>=8.39)
endif()
if(NDPI_WITH_MAXMINDDB)
@@ -415,19 +229,7 @@ endif()
if(STATIC_LIBNDPI_INSTALLDIR OR BUILD_NDPI)
add_definitions("-DLIBNDPI_STATIC=1")
set(STATIC_LIBNDPI_INC "${STATIC_LIBNDPI_INSTALLDIR}/include/ndpi")
if(CMAKE_SIZEOF_VOID_P EQUAL 8)
if(EXISTS "${STATIC_LIBNDPI_INSTALLDIR}/lib64/libndpi.a")
set(STATIC_LIBNDPI_LIB "${STATIC_LIBNDPI_INSTALLDIR}/lib64/libndpi.a")
else()
set(STATIC_LIBNDPI_LIB "${STATIC_LIBNDPI_INSTALLDIR}/lib/libndpi.a")
endif()
else()
if(EXISTS "${STATIC_LIBNDPI_INSTALLDIR}/lib32/libndpi.a")
set(STATIC_LIBNDPI_LIB "${STATIC_LIBNDPI_INSTALLDIR}/lib32/libndpi.a")
else()
set(STATIC_LIBNDPI_LIB "${STATIC_LIBNDPI_INSTALLDIR}/lib/libndpi.a")
endif()
endif()
set(STATIC_LIBNDPI_LIB "${STATIC_LIBNDPI_INSTALLDIR}/lib/libndpi.a")
if(STATIC_LIBNDPI_INSTALLDIR AND NOT BUILD_NDPI)
if(NOT EXISTS "${STATIC_LIBNDPI_INC}" OR NOT EXISTS "${STATIC_LIBNDPI_LIB}")
@@ -437,13 +239,9 @@ if(STATIC_LIBNDPI_INSTALLDIR OR BUILD_NDPI)
endif()
unset(DEFAULT_NDPI_INCLUDE CACHE)
unset(pkgcfg_lib_NDPI_ndpi CACHE)
else()
if(NOT NDPI_NO_PKGCONFIG)
pkg_check_modules(NDPI REQUIRED libndpi>=5.0.0)
if(NOT pkgcfg_lib_NDPI_ndpi)
find_package(NDPI "5.0.0" REQUIRED)
endif()
pkg_check_modules(NDPI REQUIRED libndpi>=4.3.0)
unset(STATIC_LIBNDPI_INC CACHE)
unset(STATIC_LIBNDPI_LIB CACHE)
@@ -452,62 +250,34 @@ else()
set(DEFAULT_NDPI_INCLUDE ${NDPI_INCLUDE_DIRS})
endif()
pkg_check_modules(PCAP libpcap>=1.9.0) # no *.pc file before 1.9.0
if(NOT pkgcfg_lib_PCAP_pcap)
pkg_check_modules(PCAP libpcap>=1.8.1) # seems like some distributions provide their own *.pc file for 1.8.1 (e.g. Ubuntu-18.04)
endif()
if(NOT pkgcfg_lib_PCAP_pcap)
find_package(PCAP "1.9.0" REQUIRED)
endif()
find_package(PCAP "1.8.1" REQUIRED)
target_compile_options(nDPId PRIVATE "-pthread")
target_compile_definitions(nDPId PRIVATE -D_GNU_SOURCE=1 -DPKG_VERSION=\"${PKG_VERSION}\" -DGIT_VERSION=\"${GIT_VERSION}\"
${NDPID_DEFS} ${EPOLL_DEFS} ${ZLIB_DEFS} ${PFRING_DEFS} ${OSSL_DEFS})
target_include_directories(nDPId PRIVATE "${STATIC_LIBNDPI_INC}" "${DEFAULT_NDPI_INCLUDE}" ${NDPID_DEPS_INC} ${PFRING_KERNEL_INC} ${PFRING_INC})
target_link_libraries(nDPId "${STATIC_LIBNDPI_LIB}" "${STATIC_PFRING_LIB}" "${pkgcfg_lib_PCAP_pcap}" "${pkgcfg_lib_NDPI_ndpi}"
"${pkgcfg_lib_PCRE_pcre2-8}" "${pkgcfg_lib_MAXMINDDB_maxminddb}" "${pkgcfg_lib_ZLIB_z}"
"${GCRYPT_LIBRARY}" "${GCRYPT_ERROR_LIBRARY}" "${PCAP_LIBRARY}" "${LIBM_LIB}" "${PF_RING_LIB}"
"${OSSL_LIBRARY}" "-pthread")
target_compile_definitions(nDPId PRIVATE -D_GNU_SOURCE=1 -DGIT_VERSION=\"${GIT_VERSION}\" ${NDPID_DEFS} ${ZLIB_DEFS})
target_include_directories(nDPId PRIVATE "${STATIC_LIBNDPI_INC}" "${DEFAULT_NDPI_INCLUDE}" ${NDPID_DEPS_INC})
target_link_libraries(nDPId "${STATIC_LIBNDPI_LIB}" "${pkgcfg_lib_NDPI_ndpi}"
"${pkgcfg_lib_PCRE_pcre}" "${pkgcfg_lib_MAXMINDDB_maxminddb}" "${pkgcfg_lib_ZLIB_z}"
"${GCRYPT_LIBRARY}" "${GCRYPT_ERROR_LIBRARY}" "${PCAP_LIBRARY}" "${LIBM_LIB}"
"-pthread")
target_compile_definitions(nDPIsrvd PRIVATE -D_GNU_SOURCE=1 -DPKG_VERSION=\"${PKG_VERSION}\" -DGIT_VERSION=\"${GIT_VERSION}\" ${NDPID_DEFS} ${EPOLL_DEFS})
target_compile_definitions(nDPIsrvd PRIVATE -D_GNU_SOURCE=1 -DGIT_VERSION=\"${GIT_VERSION}\" ${NDPID_DEFS})
target_include_directories(nDPIsrvd PRIVATE ${NDPID_DEPS_INC})
target_include_directories(nDPId-test PRIVATE ${NDPID_DEPS_INC})
target_compile_options(nDPId-test PRIVATE "-Wno-unused-function" "-pthread")
target_compile_definitions(nDPId-test PRIVATE -D_GNU_SOURCE=1 -DNO_MAIN=1 -DPKG_VERSION=\"${PKG_VERSION}\" -DGIT_VERSION=\"${GIT_VERSION}\"
${NDPID_DEFS} ${EPOLL_DEFS} ${ZLIB_DEFS} ${PFRING_DEFS} ${OSSL_DEFS} ${NDPID_TEST_MPROF_DEFS})
target_compile_definitions(nDPId-test PRIVATE -D_GNU_SOURCE=1 -DNO_MAIN=1 -DGIT_VERSION=\"${GIT_VERSION}\"
${NDPID_DEFS} ${ZLIB_DEFS} ${NDPID_TEST_MPROF_DEFS})
target_include_directories(nDPId-test PRIVATE
"${STATIC_LIBNDPI_INC}" "${DEFAULT_NDPI_INCLUDE}" ${NDPID_DEPS_INC} ${PFRING_KERNEL_INC} ${PFRING_INC})
target_link_libraries(nDPId-test "${STATIC_LIBNDPI_LIB}" "${STATIC_PFRING_LIB}" "${pkgcfg_lib_PCAP_pcap}" "${pkgcfg_lib_NDPI_ndpi}"
"${pkgcfg_lib_PCRE_pcre2-8}" "${pkgcfg_lib_MAXMINDDB_maxminddb}" "${pkgcfg_lib_ZLIB_z}"
"${GCRYPT_LIBRARY}" "${GCRYPT_ERROR_LIBRARY}" "${PCAP_LIBRARY}" "${LIBM_LIB}" "${PF_RING_LIB}"
"${OSSL_LIBRARY}" "-pthread")
if(CMAKE_C_COMPILER_ID STREQUAL "Clang")
add_executable(fuzz_ndpi_process_packet test/fuzz_ndpi_process_packet.c)
if(BUILD_NDPI)
add_dependencies(fuzz_ndpi_process_packet libnDPI)
endif()
target_compile_options(fuzz_ndpi_process_packet PRIVATE "-Wno-unused-function" "-fsanitize=fuzzer" "-pthread")
target_compile_definitions(fuzz_ndpi_process_packet PRIVATE -D_GNU_SOURCE=1
-DPKG_VERSION=\"${PKG_VERSION}\" -DGIT_VERSION=\"${GIT_VERSION}\"
${NDPID_DEFS} ${EPOLL_DEFS} ${ZLIB_DEFS} ${PFRING_DEFS})
target_include_directories(fuzz_ndpi_process_packet PRIVATE "${STATIC_LIBNDPI_INC}" "${DEFAULT_NDPI_INCLUDE}"
${NDPID_DEPS_INC} ${PFRING_KERNEL_INC} ${PFRING_INC})
target_link_libraries(fuzz_ndpi_process_packet "${STATIC_LIBNDPI_LIB}" "${STATIC_PFRING_LIB}" "${pkgcfg_lib_PCAP_pcap}" "${pkgcfg_lib_NDPI_ndpi}"
"${pkgcfg_lib_PCRE_pcre2-8}" "${pkgcfg_lib_MAXMINDDB_maxminddb}" "${pkgcfg_lib_ZLIB_z}"
"${GCRYPT_LIBRARY}" "${GCRYPT_ERROR_LIBRARY}" "${PCAP_LIBRARY}" "${LIBM_LIB}" "${PF_RING_LIB}"
"-pthread")
target_link_options(fuzz_ndpi_process_packet PRIVATE "-fsanitize=fuzzer")
endif()
"${STATIC_LIBNDPI_INC}" "${DEFAULT_NDPI_INCLUDE}" ${NDPID_DEPS_INC})
target_link_libraries(nDPId-test "${STATIC_LIBNDPI_LIB}" "${pkgcfg_lib_NDPI_ndpi}"
"${pkgcfg_lib_PCRE_pcre}" "${pkgcfg_lib_MAXMINDDB_maxminddb}" "${pkgcfg_lib_ZLIB_z}"
"${GCRYPT_LIBRARY}" "${GCRYPT_ERROR_LIBRARY}" "${PCAP_LIBRARY}" "${LIBM_LIB}"
"-pthread")
if(BUILD_EXAMPLES)
add_executable(nDPIsrvd-collectd examples/c-collectd/c-collectd.c utils.c)
if(BUILD_NDPI)
add_dependencies(nDPIsrvd-collectd libnDPI)
endif()
add_executable(nDPIsrvd-collectd examples/c-collectd/c-collectd.c)
target_compile_definitions(nDPIsrvd-collectd PRIVATE ${NDPID_DEFS})
target_include_directories(nDPIsrvd-collectd PRIVATE
"${STATIC_LIBNDPI_INC}" "${DEFAULT_NDPI_INCLUDE}" "${CMAKE_SOURCE_DIR}" ${NDPID_DEPS_INC})
target_include_directories(nDPIsrvd-collectd PRIVATE ${NDPID_DEPS_INC})
add_executable(nDPIsrvd-captured examples/c-captured/c-captured.c utils.c)
if(BUILD_NDPI)
@@ -516,171 +286,81 @@ if(BUILD_EXAMPLES)
target_compile_definitions(nDPIsrvd-captured PRIVATE ${NDPID_DEFS})
target_include_directories(nDPIsrvd-captured PRIVATE
"${STATIC_LIBNDPI_INC}" "${DEFAULT_NDPI_INCLUDE}" "${CMAKE_SOURCE_DIR}" ${NDPID_DEPS_INC})
target_link_libraries(nDPIsrvd-captured "${pkgcfg_lib_PCAP_pcap}" "${pkgcfg_lib_NDPI_ndpi}"
"${pkgcfg_lib_PCRE_pcre2-8}" "${pkgcfg_lib_MAXMINDDB_maxminddb}"
target_link_libraries(nDPIsrvd-captured "${pkgcfg_lib_NDPI_ndpi}"
"${pkgcfg_lib_PCRE_pcre}" "${pkgcfg_lib_MAXMINDDB_maxminddb}"
"${GCRYPT_LIBRARY}" "${GCRYPT_ERROR_LIBRARY}" "${PCAP_LIBRARY}")
add_executable(nDPIsrvd-analysed examples/c-analysed/c-analysed.c utils.c)
if(BUILD_NDPI)
add_dependencies(nDPIsrvd-analysed libnDPI)
endif()
target_compile_definitions(nDPIsrvd-analysed PRIVATE ${NDPID_DEFS})
target_include_directories(nDPIsrvd-analysed PRIVATE
"${STATIC_LIBNDPI_INC}" "${DEFAULT_NDPI_INCLUDE}" "${CMAKE_SOURCE_DIR}" ${NDPID_DEPS_INC})
add_executable(nDPIsrvd-json-dump examples/c-json-stdout/c-json-stdout.c)
target_compile_definitions(nDPIsrvd-json-dump PRIVATE ${NDPID_DEFS})
target_include_directories(nDPIsrvd-json-dump PRIVATE ${NDPID_DEPS_INC})
add_executable(nDPIsrvd-simple examples/c-simple/c-simple.c)
target_compile_definitions(nDPIsrvd-simple PRIVATE ${NDPID_DEFS})
target_include_directories(nDPIsrvd-simple PRIVATE ${NDPID_DEPS_INC})
target_link_libraries(nDPIsrvd-simple "${pkgcfg_lib_NDPI_ndpi}"
"${pkgcfg_lib_PCRE_pcre}" "${pkgcfg_lib_MAXMINDDB_maxminddb}"
"${GCRYPT_LIBRARY}" "${GCRYPT_ERROR_LIBRARY}" "${PCAP_LIBRARY}")
if(ENABLE_COVERAGE)
add_dependencies(coverage nDPIsrvd-analysed nDPIsrvd-collectd nDPIsrvd-captured nDPIsrvd-simple)
if(BUILD_NDPI)
add_dependencies(coverage libnDPI)
endif()
add_dependencies(coverage nDPIsrvd-collectd nDPIsrvd-captured nDPIsrvd-json-dump nDPIsrvd-simple)
endif()
if(ENABLE_DBUS)
add_executable(nDPIsrvd-notifyd examples/c-notifyd/c-notifyd.c utils.c)
if(BUILD_NDPI)
add_dependencies(nDPIsrvd-notifyd libnDPI)
endif()
target_compile_definitions(nDPIsrvd-notifyd PRIVATE ${NDPID_DEFS})
target_include_directories(nDPIsrvd-notifyd PRIVATE
"${STATIC_LIBNDPI_INC}" "${DEFAULT_NDPI_INCLUDE}" "${CMAKE_SOURCE_DIR}" "${NDPID_DEPS_INC}"
"${DBUS_INCLUDE_DIRS}")
target_link_libraries(nDPIsrvd-notifyd "${DBUS_LIBRARIES}")
install(TARGETS nDPIsrvd-notifyd DESTINATION bin)
endif()
if(ENABLE_CURL)
add_executable(nDPIsrvd-influxd examples/c-influxd/c-influxd.c utils.c)
if(BUILD_NDPI)
add_dependencies(nDPIsrvd-influxd libnDPI)
endif()
target_compile_definitions(nDPIsrvd-influxd PRIVATE ${NDPID_DEFS})
target_include_directories(nDPIsrvd-influxd PRIVATE
"${STATIC_LIBNDPI_INC}" "${DEFAULT_NDPI_INCLUDE}" "${CMAKE_SOURCE_DIR}" "${NDPID_DEPS_INC}"
"${CURL_INCLUDE_DIRS}")
target_link_libraries(nDPIsrvd-influxd "${CURL_LIBRARIES}")
install(TARGETS nDPIsrvd-influxd DESTINATION bin)
endif()
install(TARGETS nDPIsrvd-analysed nDPIsrvd-collectd nDPIsrvd-captured nDPIsrvd-simple DESTINATION bin)
install(FILES examples/c-collectd/plugin_nDPIsrvd.conf examples/c-collectd/rrdgraph.sh DESTINATION share/nDPId/nDPIsrvd-collectd)
install(DIRECTORY examples/c-collectd/www DESTINATION share/nDPId/nDPIsrvd-collectd)
install(TARGETS nDPIsrvd-collectd nDPIsrvd-captured nDPIsrvd-json-dump nDPIsrvd-simple DESTINATION bin)
endif()
if(BUILD_RUST_EXAMPLES)
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/target/release/rs-simple
COMMAND cargo build --release
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/examples/rs-simple
COMMENT "Build Rust executable with cargo: rs-simple"
)
add_custom_target(rs-simple ALL
DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/target/release/rs-simple
)
endif()
if(ENABLE_SYSTEMD)
configure_file(packages/systemd/ndpisrvd.service.in ndpisrvd.service @ONLY)
configure_file(packages/systemd/ndpid@.service.in ndpid@.service @ONLY)
install(DIRECTORY DESTINATION etc/nDPId)
install(FILES "ndpid.conf.example" DESTINATION share/nDPId)
install(FILES "ndpisrvd.conf.example" DESTINATION share/nDPId)
install(FILES "${CMAKE_BINARY_DIR}/ndpisrvd.service" DESTINATION lib/systemd/system)
install(FILES "${CMAKE_BINARY_DIR}/ndpid@.service" DESTINATION lib/systemd/system)
endif()
install(FILES config.h
dependencies/nDPIsrvd.h
dependencies/jsmn/jsmn.h
dependencies/uthash/src/utarray.h
dependencies/uthash/src/uthash.h
dependencies/uthash/src/utlist.h
dependencies/uthash/src/utringbuffer.h
dependencies/uthash/src/utstack.h
dependencies/uthash/src/utstring.h
DESTINATION include/nDPId)
install(TARGETS nDPId DESTINATION sbin)
install(TARGETS nDPIsrvd nDPId-test DESTINATION bin)
if(BUILD_EXAMPLES)
install(FILES dependencies/nDPIsrvd.py
DESTINATION share/nDPId)
install(FILES examples/py-flow-info/flow-info.py
DESTINATION bin RENAME nDPIsrvd-flow-info.py
PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
install(FILES examples/py-json-stdout/json-stdout.py
DESTINATION bin RENAME nDPIsrvd-json-stdout.py
PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
install(FILES examples/py-schema-validation/py-schema-validation.py
DESTINATION bin RENAME nDPIsrvd-schema-validation.py
PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
install(FILES examples/py-semantic-validation/py-semantic-validation.py
DESTINATION bin RENAME nDPIsrvd-semantic-validation.py
PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
install(FILES examples/py-machine-learning/sklearn-random-forest.py
DESTINATION bin RENAME nDPIsrvd-sklearn.py
PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
endif()
install(FILES dependencies/nDPIsrvd.py examples/py-flow-dashboard/plotly_dash.py
DESTINATION share/nDPId)
install(FILES examples/py-flow-info/flow-info.py
DESTINATION bin RENAME nDPIsrvd-flow-info.py
PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
install(FILES examples/py-flow-dashboard/flow-dash.py
DESTINATION bin RENAME nDPIsrvd-flow-dash.py
PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
install(FILES examples/py-ja3-checker/py-ja3-checker.py
DESTINATION bin RENAME nDPIsrvd-ja3-checker.py
PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
install(FILES examples/py-json-stdout/json-stdout.py
DESTINATION bin RENAME nDPIsrvd-json-stdout.py
PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
install(FILES examples/py-schema-validation/py-schema-validation.py
DESTINATION bin RENAME nDPIsrvd-schema-validation.py
PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
install(FILES examples/py-semantic-validation/py-semantic-validation.py
DESTINATION bin RENAME nDPIsrvd-semantic-validation.py
PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
install(FILES schema/error_event_schema.json schema/daemon_event_schema.json
schema/flow_event_schema.json schema/packet_event_schema.json DESTINATION share/nDPId/json-schema)
schema/flow_event_schema.json schema/packet_event_schema.json DESTINATION share/nDPId)
message(STATUS "--------------------------")
message(STATUS "nDPId GIT_VERSION........: ${GIT_VERSION}")
message(STATUS "Cross Compilation........: ${CMAKE_CROSSCOMPILING}")
message(STATUS "CROSS_COMPILE_TRIPLET....: ${CROSS_COMPILE_TRIPLET}")
message(STATUS "CMAKE_BUILD_TYPE.........: ${CMAKE_BUILD_TYPE}")
message(STATUS "CMAKE_C_FLAGS............: ${CMAKE_C_FLAGS}")
message(STATUS "NDPID_DEFS...............: ${NDPID_DEFS}")
message(STATUS "FORCE_POLL...............: ${FORCE_POLL}")
message(STATUS "ENABLE_PFRING............: ${ENABLE_PFRING}")
if(ENABLE_PFRING)
message(STATUS "PFRING_LINK_STATIC.......: ${PFRING_LINK_STATIC}")
endif()
message(STATUS "ENABLE_CRYPTO............: ${ENABLE_CRYPTO}")
message(STATUS "ENABLE_COVERAGE..........: ${ENABLE_COVERAGE}")
message(STATUS "ENABLE_SANITIZER.........: ${ENABLE_SANITIZER}")
message(STATUS "ENABLE_SANITIZER_THREAD..: ${ENABLE_SANITIZER_THREAD}")
message(STATUS "ENABLE_MEMORY_PROFILING..: ${ENABLE_MEMORY_PROFILING}")
message(STATUS "ENABLE_ZLIB..............: ${ENABLE_ZLIB}")
message(STATUS "BUILD_NDPI...............: ${BUILD_NDPI}")
message(STATUS "BUILD_EXAMPLES...........: ${BUILD_EXAMPLES}")
message(STATUS "BUILD_RUST_EXAMPLES......: ${BUILD_RUST_EXAMPLES}")
if(BUILD_EXAMPLES)
message(STATUS "ENABLE_DBUS..............: ${ENABLE_DBUS}")
message(STATUS "ENABLE_CURL..............: ${ENABLE_CURL}")
if(STATIC_LIBNDPI_INSTALLDIR)
message(STATUS "STATIC_LIBNDPI_INSTALLDIR: ${STATIC_LIBNDPI_INSTALLDIR}")
endif()
message(STATUS "BUILD_NDPI...............: ${BUILD_NDPI}")
if(BUILD_NDPI)
message(STATUS "NDPI_ADDITIONAL_ARGS.....: ${NDPI_ADDITIONAL_ARGS}")
endif()
message(STATUS "NDPI_NO_PKGCONFIG........: ${NDPI_NO_PKGCONFIG}")
message(STATUS "--------------------------")
if(PFRING_INSTALLDIR)
message(STATUS "PFRING_INSTALLDIR........: ${PFRING_INSTALLDIR}")
message(STATUS "- PFRING_INC.............: ${PFRING_INC}")
message(STATUS "- PFRING_KERNEL_INC......: ${PFRING_KERNEL_INC}")
message(STATUS "- STATIC_PFRING_LIB......: ${STATIC_PFRING_LIB}")
message(STATUS "- SHARED_PFRING_LIB......: ${PF_RING_LIB}")
message(STATUS "--------------------------")
endif()
if(STATIC_LIBNDPI_INSTALLDIR)
message(STATUS "STATIC_LIBNDPI_INSTALLDIR: ${STATIC_LIBNDPI_INSTALLDIR}")
endif()
if(STATIC_LIBNDPI_INSTALLDIR OR BUILD_NDPI OR NDPI_NO_PKGCONFIG)
message(STATUS "- STATIC_LIBNDPI_INC.....: ${STATIC_LIBNDPI_INC}")
message(STATUS "- STATIC_LIBNDPI_LIB.....: ${STATIC_LIBNDPI_LIB}")
message(STATUS "- NDPI_WITH_GCRYPT.......: ${NDPI_WITH_GCRYPT}")
message(STATUS "- NDPI_WITH_PCRE.........: ${NDPI_WITH_PCRE}")
message(STATUS "- NDPI_WITH_MAXMINDDB....: ${NDPI_WITH_MAXMINDDB}")
message(STATUS "--------------------------")
message(STATUS "- STATIC_LIBNDPI_INC....: ${STATIC_LIBNDPI_INC}")
message(STATUS "- STATIC_LIBNDPI_LIB....: ${STATIC_LIBNDPI_LIB}")
message(STATUS "- NDPI_WITH_GCRYPT......: ${NDPI_WITH_GCRYPT}")
message(STATUS "- NDPI_WITH_PCRE........: ${NDPI_WITH_PCRE}")
message(STATUS "- NDPI_WITH_MAXMINDDB...: ${NDPI_WITH_MAXMINDDB}")
endif()
if(NOT STATIC_LIBNDPI_INSTALLDIR AND NOT BUILD_NDPI)
message(STATUS "- DEFAULT_NDPI_INCLUDE...: ${DEFAULT_NDPI_INCLUDE}")
endif()
if(NOT NDPI_NO_PKGCONFIG)
message(STATUS "- pkgcfg_lib_NDPI_ndpi...: ${pkgcfg_lib_NDPI_ndpi}")
message(STATUS "- DEFAULT_NDPI_INCLUDE..: ${DEFAULT_NDPI_INCLUDE}")
endif()
message(STATUS "--------------------------")
if(CMAKE_C_COMPILER_ID STREQUAL "Clang")
message(STATUS "Fuzzing enabled")
endif()

46
Dockerfile
View File

@@ -1,46 +0,0 @@
FROM ubuntu:22.04 AS builder-ubuntu-2204
WORKDIR /root
RUN apt-get -y update \
&& apt-get install -y --no-install-recommends \
autoconf automake build-essential ca-certificates cmake git \
libpcap-dev libcurl4-openssl-dev libdbus-1-dev libtool make pkg-config unzip wget \
&& apt-get clean \
&& git clone https://github.com/utoni/nDPId.git
WORKDIR /root/nDPId
RUN cmake -S . -B build -DBUILD_NDPI=ON -DBUILD_EXAMPLES=ON \
-DENABLE_DBUS=ON -DENABLE_CURL=ON \
&& cmake --build build --verbose
FROM ubuntu:22.04
USER root
WORKDIR /
COPY --from=builder-ubuntu-2204 /root/nDPId/build/nDPId /usr/sbin/nDPId
COPY --from=builder-ubuntu-2204 /root/nDPId/build/nDPIsrvd /usr/bin/nDPIsrvd
COPY --from=builder-ubuntu-2204 /root/nDPId/build/nDPId-test /usr/bin/nDPId-test
COPY --from=builder-ubuntu-2204 /root/nDPId/build/nDPIsrvd-collectd /usr/bin/nDPIsrvd-collectd
COPY --from=builder-ubuntu-2204 /root/nDPId/build/nDPIsrvd-captured /usr/bin/nDPIsrvd-captured
COPY --from=builder-ubuntu-2204 /root/nDPId/build/nDPIsrvd-analysed /usr/bin/nDPIsrvd-anaylsed
COPY --from=builder-ubuntu-2204 /root/nDPId/build/nDPIsrvd-analysed /usr/bin/nDPIsrvd-anaylsed
COPY --from=builder-ubuntu-2204 /root/nDPId/build/nDPIsrvd-notifyd /usr/bin/nDPIsrvd-notifyd
COPY --from=builder-ubuntu-2204 /root/nDPId/build/nDPIsrvd-influxd /usr/bin/nDPIsrvd-influxd
COPY --from=builder-ubuntu-2204 /root/nDPId/build/nDPIsrvd-simple /usr/bin/nDPIsrvd-simple
RUN apt-get -y update \
&& apt-get install -y --no-install-recommends libpcap-dev \
&& apt-get clean
USER nobody
RUN /usr/bin/nDPIsrvd -h || { RC=$?; test ${RC} -eq 1; }; \
/usr/sbin/nDPId -h || { RC=$?; test ${RC} -eq 1; }
FROM archlinux:base-devel AS builder-archlinux
WORKDIR /root
RUN pacman --noconfirm -Sy cmake git unzip wget && mkdir /build && chown nobody /build && cd /build \
&& runuser -u nobody git clone https://github.com/utoni/nDPId.git
WORKDIR /build/nDPId/packages/archlinux
RUN runuser -u nobody makepkg

254
README.md
View File

@@ -1,41 +1,22 @@
[![Build](https://github.com/utoni/nDPId/actions/workflows/build.yml/badge.svg)](https://github.com/utoni/nDPId/actions/workflows/build.yml)
[![Gitlab-CI](https://gitlab.com/utoni/nDPId/badges/main/pipeline.svg)](https://gitlab.com/utoni/nDPId/-/pipelines)
[![Circle-CI](https://circleci.com/gh/utoni/nDPId.svg?style=shield "Circle-CI")](https://app.circleci.com/pipelines/github/utoni/nDPId)
[![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=lnslbrty_nDPId&metric=ncloc)](https://sonarcloud.io/summary/new_code?id=lnslbrty_nDPId)
[![Code Smells](https://sonarcloud.io/api/project_badges/measure?project=lnslbrty_nDPId&metric=code_smells)](https://sonarcloud.io/summary/new_code?id=lnslbrty_nDPId)
[![Bugs](https://sonarcloud.io/api/project_badges/measure?project=lnslbrty_nDPId&metric=bugs)](https://sonarcloud.io/summary/new_code?id=lnslbrty_nDPId)
[![Vulnerabilities](https://sonarcloud.io/api/project_badges/measure?project=lnslbrty_nDPId&metric=vulnerabilities)](https://sonarcloud.io/summary/new_code?id=lnslbrty_nDPId)
[![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=lnslbrty_nDPId&metric=reliability_rating)](https://sonarcloud.io/summary/new_code?id=lnslbrty_nDPId)
[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=lnslbrty_nDPId&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=lnslbrty_nDPId)
![Docker Automated build](https://img.shields.io/docker/automated/utoni/ndpid)
# References
[ntop Webinar 2022](https://www.ntop.org/webinar/ntop-webinar-on-dec-14th-community-meeting-and-future-plans/)
[ntopconf 2023](https://www.ntop.org/ntopconf2023/)
# Disclaimer
Please respect & protect the privacy of others.
The purpose of this software is not to spy on others, but to detect network anomalies and malicious traffic.
[![Gitlab-CI](https://gitlab.com/utoni/nDPId/badges/master/pipeline.svg)](https://gitlab.com/utoni/nDPId/-/pipelines)
# Abstract
nDPId is a set of daemons and tools to capture, process and classify network traffic.
Its minimal dependencies (besides a to some extent modern C library and POSIX threads) are libnDPI (>=5.0.0 or current github dev branch) and libpcap.
It's minimal dependencies (besides a half-way modern c library and POSIX threads) are libnDPI (>= 4.4.0 or current github dev branch) and libpcap.
The daemon `nDPId` is capable of multithreading for packet processing, but w/o mutexes for performance reasons.
Instead, synchronization is achieved by a packet distribution mechanism.
To balance the workload to all threads (more or less) equally, a unique identifier represented as hash value is calculated using a 3-tuple consisting of: IPv4/IPv6 src/dst address; IP header value of the layer4 protocol; and (for TCP/UDP) src/dst port. Other protocols e.g. ICMP/ICMPv6 lack relevance for DPI, thus nDPId does not distinguish between different ICMP/ICMPv6 flows coming from the same host. This saves memory and performance, but might change in the future.
Instead synchronization is achieved by a packet distribution mechanism.
To balance all workload to all threads (more or less) equally a unique identifier represented as hash value is calculated using a 3-tuple consisting of IPv4/IPv6 src/dst address, IP header value of the layer4 protocol and (for TCP/UDP) src/dst port. Other protocols e.g. ICMP/ICMPv6 are lacking relevance for DPI, thus nDPId does not distinguish between different ICMP/ICMPv6 flows coming from the same host. Saves memory and performance, but might change in the future.
`nDPId` uses libnDPI's JSON serialization interface to generate a JSON messages for each event it receives from the library and which it then sends out to a UNIX-socket (default: `/tmp/ndpid-collector.sock` ). From such a socket, `nDPIsrvd` (or other custom applications) can retrieve incoming JSON-messages and further proceed working/distributing messages to higher-level applications.
`nDPId` uses libnDPI's JSON serialization interface to generate a JSON strings for each event it receive from the library and which it then sends out to a UNIX-socket (default: /tmp/ndpid-collector.sock ). From such a socket, `nDPIsrvd` (or other custom applications) can retrieve incoming JSON-messages and further proceed working/distributing messages to higher-level applications.
Unfortunately, `nDPIsrvd` does not yet support any encryption/authentication for TCP connections (TODO!).
Unfortunately `nDPIsrvd` does currently not support any encryption/authentication for TCP connections (TODO!).
# Architecture
This project uses a kind of microservice architecture.
This project uses some kind of microservice architecture.
```text
connect to UNIX socket [1] connect to UNIX/TCP socket [2]
@@ -55,15 +36,12 @@ _______________________ | | ________________
```
where:
* `nDPId` capture traffic, extract traffic data (with libnDPI) and send a JSON-serialized output stream to an already existing UNIX-socket;
* `nDPIsrvd`:
* create and manage an "incoming" UNIX-socket (ref [1] above), to fetch data from a local `nDPId`;
* apply a buffering logic to received data;
* apply a filtering logic to received data to select "flow_event_id" related JSONs;
* create and manage an "outgoing" UNIX or TCP socket (ref [2] above) to relay matched events
to connected clients
to connected clients
* `consumers` are common/custom applications being able to receive selected flows/events, via both UNIX-socket or TCP-socket.
@@ -71,98 +49,26 @@ where:
JSON messages streamed by both `nDPId` and `nDPIsrvd` are presented with:
* a 5-digit-number describing (as decimal number) the **entire** JSON message including the newline `\n` at the end;
* a 5-digit-number describing (as decimal number) of the **entire** JSON string including the newline `\n` at the end;
* the JSON messages
```text
[5-digit-number][JSON message]
[5-digit-number][JSON string]
```
as with the following example:
```text
01223{"flow_event_id":7,"flow_event_name":"detection-update","thread_id":12,"packet_id":307,"source":"wlan0", ...snip...}
00458{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":11,"packet_id":324,"source":"wlan0", ...snip...}
00572{"flow_event_id":1,"flow_event_name":"new","thread_id":11,"packet_id":324,"source":"wlan0", ...snip...}
01223{"flow_event_id":7,"flow_event_name":"detection-update","thread_id":12,"packet_id":307,"source":"wlan0",[...]}
00458{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":11,"packet_id":324,"source":"wlan0",[...]]}
00572{"flow_event_id":1,"flow_event_name":"new","thread_id":11,"packet_id":324,"source":"wlan0",[...]}
```
The full stream of `nDPId` generated JSON-events can be retrieved directly from `nDPId`, without relying on `nDPIsrvd`, by providing a properly managed UNIX-socket.
Technical details about the JSON-message format can be obtained from the related `.schema` file included in the `schema` directory
Technical details about JSON-messages format can be obtained from related `.schema` file included in the `schema` directory
# Events
`nDPId` generates JSON messages whereby each string is assigned to a certain event.
Those events specify the contents (key-value-pairs) of the JSON message.
They are divided into four categories, each with a number of subevents.
## Error Events
They are 17 distinct events, indicating that layer2 or layer3 packet processing failed or not enough flow memory available:
1. Unknown datalink layer packet
2. Unknown L3 protocol
3. Unsupported datalink layer
4. Packet too short
5. Unknown packet type
6. Packet header invalid
7. IP4 packet too short
8. Packet smaller than IP4 header:
9. nDPI IPv4/L4 payload detection failed
10. IP6 packet too short
11. Packet smaller than IP6 header
12. nDPI IPv6/L4 payload detection failed
13. TCP packet smaller than expected
14. UDP packet smaller than expected
15. Captured packet size is smaller than expected packet size
16. Max flows to track reached
17. Flow memory allocation failed
Detailed JSON-schema is available [here](schema/error_event_schema.json)
## Daemon Events
There are 4 distinct events indicating startup/shutdown or status events as well as a reconnect event if there was a previous connection failure (collector):
1. init: `nDPId` startup
2. reconnect: (UNIX) socket connection lost previously and was established again
3. shutdown: `nDPId` terminates gracefully
4. status: statistics about the daemon itself e.g. memory consumption, zLib compressions (if enabled)
Detailed JSON-schema is available [here](schema/daemon_event_schema.json)
## Packet Events
There are 2 events containing base64 encoded packet payloads either belonging to a flow or not:
1. packet: does not belong to any flow
2. packet-flow: belongs to a flow e.g. TCP/UDP or ICMP
Detailed JSON-schema is available [here](schema/packet_event_schema.json)
## Flow Events
There are 9 distinct events related to a flow:
1. new: a new TCP/UDP/ICMP flow seen which will be tracked
2. end: a TCP connection terminates
3. idle: a flow timed out, because there was no packet on the wire for a certain amount of time
4. update: inform nDPIsrvd or other apps about a long-lasting flow, whose detection was finished a long time ago but is still active
5. analyse: provide some information about extracted features of a flow (Experimental; disabled per default, enable with `-A`)
6. guessed: `libnDPI` was not able to reliably detect a layer7 protocol and falls back to IP/Port based detection
7. detected: `libnDPI` sucessfully detected a layer7 protocol
8. detection-update: `libnDPI` dissected more layer7 protocol data (after detection already done)
9. not-detected: neither detected nor guessed
Detailed JSON-schema is available [here](schema/flow_event_schema.json). Also, a graphical representation of *Flow Events* timeline is available [here](schema/flow_events_diagram.png).
# Flow States
A flow can have three different states while it is been tracked by `nDPId`.
1. skipped: the flow will be tracked, but no detection will happen to reduce memory usage.
See command line argument `-I` and `-E`
2. finished: detection finished and the memory used for the detection is freed
3. info: detection is in progress and all flow memory required for `libnDPI` is allocated (this state consumes most memory)
# Build (CMake)
`nDPId` build system is based on [CMake](https://cmake.org/)
@@ -182,7 +88,7 @@ see below for a full/test live-session
![](examples/ndpid_install_and_run.gif)
Based on your build environment and/or desiderata, you could need:
Based on your building environment and/or desiderata, you could need:
```shell
mkdir build
@@ -193,62 +99,45 @@ ccmake ..
or to build with a staticially linked libnDPI:
```shell
cmake -S . -B ./build \
-DSTATIC_LIBNDPI_INSTALLDIR=[path/to/your/libnDPI/installdir] \
-DNDPI_NO_PKGCONFIG=ON
cmake --build ./build
mkdir build
cd build
cmake .. -DSTATIC_LIBNDPI_INSTALLDIR=[path/to/your/libnDPI/installdir]
```
If you use the latter, make sure that you've configured libnDPI with `./configure --prefix=[path/to/your/libnDPI/installdir]`
and remember to set the all-necessary CMake variables to link against shared libraries used by your nDPI build.
You'll also need to use `-DNDPI_NO_PKGCONFIG=ON` if `STATIC_LIBNDPI_INSTALLDIR` does not contain a pkg-config file.
If you're using the latter one, make sure that you've configured libnDPI with `./configure --prefix=[path/to/your/libnDPI/installdir]`
and do not forget to set the all necessary CMake variables to link against shared libraries used by your nDPI build.
e.g.:
```shell
cmake -S . -B ./build \
-DSTATIC_LIBNDPI_INSTALLDIR=[path/to/your/libnDPI/installdir] \
-DNDPI_NO_PKGCONFIG=ON \
-DNDPI_WITH_GCRYPT=ON -DNDPI_WITH_PCRE=OFF -DNDPI_WITH_MAXMINDDB=OFF
cmake --build ./build
mkdir build
cd build
cmake .. -DSTATIC_LIBNDPI_INSTALLDIR=[path/to/your/libnDPI/installdir] -DNDPI_WITH_GCRYPT=ON -DNDPI_WITH_PCRE=OFF -DNDPI_WITH_MAXMINDDB=OFF
```
Or let a shell script do the work for you:
```shell
cmake -S . -B ./build \
-DBUILD_NDPI=ON
cmake --build ./build
mkdir build
cd build
cmake .. -DBUILD_NDPI=ON
```
The CMake cache variable `-DBUILD_NDPI=ON` builds a version of `libnDPI` residing as a git submodule in this repository.
The CMake cache variable `-DBUILD_NDPI=ON` builds a version of `libnDPI` residing as git submodule in this repository.
# run
As mentioned above, in order to run `nDPId`, a UNIX-socket needs to be provided in order to stream our related JSON-data.
As mentioned above, in order to run `nDPId` a UNIX-socket need to be provided in order to stream our related JSON-data.
Such a UNIX-socket can be provided by both the included `nDPIsrvd` daemon, or, if you simply need a quick check, with the [ncat](https://nmap.org/book/ncat-man.html) utility, with a simple `ncat -U /tmp/listen.sock -l -k`. Remember that OpenBSD `netcat` is not able to handle multiple connections reliably.
Such a UNIX-socket can be provided by both the included `nDPIsrvd` daemon, or, if you simply need a quick check, with the [ncat](https://nmap.org/book/ncat-man.html) utility, with a simple `ncat -U /tmp/listen.sock -l -k`
Once the socket is ready, you can run `nDPId` capturing and analyzing your own traffic, with something similar to: `sudo nDPId -c /tmp/listen.sock`
If you're using OpenBSD `netcat`, you need to run: `sudo nDPId -c /tmp/listen.sock -o max-reader-threads=1`
Make sure that the UNIX socket is accessible by the user (see -u) to whom nDPId changes to, default: nobody.
Once the socket is ready, you can run `nDPId` capturing and analyzing your own traffic, with something similar to:
Of course, both `ncat` and `nDPId` need to point to the same UNIX-socket (`nDPId` provides the `-c` option, exactly for this. By default, `nDPId` refers to `/tmp/ndpid-collector.sock`, and the same default-path is also used by `nDPIsrvd` for the incoming socket).
Of course, both `ncat` and `nDPId` need to point to the same UNIX-socket (`nDPId` provides the `-c` option, exactly for this. As a default, `nDPId` refer to `/tmp/ndpid-collector.sock`, and the same default-path is also used by `nDPIsrvd` as for the incoming socket)
Give `nDPId` some real-traffic. You can capture your own traffic, with something similar to:
You also need to provide `nDPId` some real-traffic. You can capture your own traffic, with something similar to:
```shell
socat -u UNIX-Listen:/tmp/listen.sock,fork - # does the same as `ncat`
sudo chown nobody:nobody /tmp/listen.sock # default `nDPId` user/group, see `-u` and `-g`
sudo ./nDPId -c /tmp/listen.sock -l
```
`nDPId` supports also UDP collector endpoints:
```shell
nc -d -u 127.0.0.1 7000 -l -k
sudo ./nDPId -c 127.0.0.1:7000 -l
```
./nDPId -c /tmp/listen.sock -i wlan0 -l
or you can generate a nDPId-compatible JSON dump with:
@@ -263,7 +152,7 @@ Daemons:
make -C [path-to-a-build-dir] daemon
```
Or a manual approach with:
Or you can proceed with a manual approach with:
```shell
./nDPIsrvd -d
@@ -281,69 +170,34 @@ And why not a flow-info example?
./examples/py-flow-info/flow-info.py
```
or
```shell
./nDPIsrvd-json-dump
```
or anything below `./examples`.
# nDPId tuning
It is possible to change `nDPId` internals w/o recompiling by using `-o subopt=value`.
But be careful: changing the default values may render `nDPId` useless and is not well tested.
Suboptions for `-o`:
Format: `subopt` (unit, comment): description
* `max-flows-per-thread` (N, caution advised): affects max. memory usage
* `max-idle-flows-per-thread` (N, safe): max. allowed idle flows whose memory gets freed after `flow-scan-interval`
* `max-reader-threads` (N, safe): amount of packet processing threads, every thread can have a max. of `max-flows-per-thread` flows
* `daemon-status-interval` (ms, safe): specifies how often daemon event `status` is generated
* `compression-scan-interval` (ms, untested): specifies how often `nDPId` scans for inactive flows ready for compression
* `compression-flow-inactivity` (ms, untested): the shortest period of time elapsed before `nDPId` considers compressing a flow (e.g. nDPI flow struct) that neither sent nor received any data
* `flow-scan-interval` (ms, safe): min. amount of time after which `nDPId` scans for idle or long-lasting flows
* `generic-max-idle-time` (ms, untested): time after which a non TCP/UDP/ICMP flow times out
* `icmp-max-idle-time` (ms, untested): time after which an ICMP flow times out
* `udp-max-idle-time` (ms, caution advised): time after which an UDP flow times out
* `tcp-max-idle-time` (ms, caution advised): time after which a TCP flow times out
* `tcp-max-post-end-flow-time` (ms, caution advised): a TCP flow that received a FIN or RST waits this amount of time before flow tracking stops and the flow memory is freed
* `max-packets-per-flow-to-send` (N, safe): max. `packet-flow` events generated for the first N packets of each flow
* `max-packets-per-flow-to-process` (N, caution advised): max. amount of packets processed by `libnDPI`
* `max-packets-per-flow-to-analyze` (N, safe): max. packets to analyze before sending an `analyse` event, requires `-A`
* `error-event-threshold-n` (N, safe): max. error events to send until threshold time has passed
* `error-event-threshold-time` (N, safe): time after which the error event threshold resets
# test
The recommended way to run regression / diff tests:
The recommended way to run integration / diff tests:
```shell
cmake -S . -B ./build-like-ci \
-DBUILD_NDPI=ON -DENABLE_ZLIB=ON -DBUILD_EXAMPLES=ON
# optional: -DENABLE_CURL=ON -DENABLE_SANITIZER=ON
./test/run_tests.sh ./libnDPI ./build-like-ci/nDPId-test
# or: make -C ./build-like-ci test
mkdir build
cd build
cmake .. -DBUILD_NDPI=ON
make nDPId-test test
```
Run `./test/run_tests.sh` to see some usage information.
Alternatively you can run some integration tests manually:
`./test/run_tests.sh [/path/to/libnDPI/root/directory] [/path/to/nDPId-test]`
e.g.:
`./test/run_tests.sh [${HOME}/git/nDPI] [${HOME}/git/nDPId/build/nDPId-test]`
Remember that all test results are tied to a specific libnDPI commit hash
as part of the `git submodule`. Using `test/run_tests.sh` for other commit hashes
will most likely result in PCAP diffs.
will most likely result in PCAP diff's.
# Code Coverage
You may generate code coverage by using:
```shell
cmake -S . -B ./build-coverage \
-DENABLE_COVERAGE=ON -DENABLE_ZLIB=ON
# optional: -DBUILD_NDPI=ON
make -C ./build-coverage coverage-clean
make -C ./build-coverage clean
make -C ./build-coverage all
./test/run_tests.sh ./libnDPI ./build-coverage/nDPId-test
make -C ./build-coverage coverage
make -C ./build-coverage coverage-view
```
# Contributors
Special thanks to Damiano Verzulli ([@verzulli](https://github.com/verzulli)) from [GARRLab](https://www.garrlab.it) for providing server and test infrastructure.
Why not use `examples/py-flow-dashboard/flow-dash.py` to visualize nDPId's output.

18
SECURITY.md
View File

@@ -1,18 +0,0 @@
# Security Policy
I encourage you to submit a pull request if you have a solution or fix for anything even security vulnerabilities.
Your contributions help advance and enhance safety for all users :star:.
## Reporting a Bug :bug: :bug:
Simply use GitHub issues to report a bug with related information to debug the issue :pencil:.
## Reporting a Vulnerability :closed_lock_with_key: :eyes:
For sensitive security issues, please email <toni@impl.cc> including the following information:
- Description of the vulnerability
- Steps to reproduce the issue
- Affected versions i.e. release tags, git commit hashes or git branch
- If applicable, a data sample (preferably `pcap/pcapng`) to reproduce
- If known, any mitigations or fixes for the issue

21
TODO.md
View File

@@ -1,20 +1,5 @@
# TODOs
1.8:
* let nDPIsrvd (collector) connect to other nDPIsrvd instances (as distributor)
* nDPIsrvd GnuTLS support for TCP/IP distributor connections
* provide nDPId-exportd daemon which will only send captured packets to an nDPId instance running on a different machine
2.0.0:
* switch to semantic versioning for the greater good ;)
no release plan:
* merge flow end/idle event into idle event (end is not really useful..)
* provide a shared library for C / C++ for distributor application developers
* improve UDP/TCP timeout handling by reading netfilter conntrack timeouts from /proc (or just read conntrack table entries)
* detect interface / timeout changes and apply them to nDPId
* switch to MIT or BSD License
* libdaq integration
1. improve UDP/TCP timeout handling by reading netfilter conntrack timeouts from /proc (or just read conntrack table entries)
2. detect interface / timeout changes and apply them to nDPId
3. implement AEAD crypto via libsodium (at least for TCP communication)

28
cmake/CheckEpoll.cmake
View File

@@ -1,28 +0,0 @@
# - Check if the system supports epoll.
# CHECK_EPOLL(<var>)
# <var> - variable to store the result
# (1 for success, empty for failure)
#=============================================================================
# This software is in the public domain, furnished "as is", without technical
# support, and with no warranty, express or implied, as to its usefulness for
# any purpose.
#=============================================================================
macro(CHECK_EPOLL VARIABLE)
if(UNIX)
if("${VARIABLE}" MATCHES "^${VARIABLE}$")
message(STATUS "Check if the system supports epoll")
include(CheckSymbolExists)
check_symbol_exists(epoll_create "sys/epoll.h" EPOLL_PROTOTYPE_EXISTS)
if(EPOLL_PROTOTYPE_EXISTS)
message(STATUS "Check if the system supports epoll - yes")
set(${VARIABLE} 1 CACHE INTERNAL "Result of CHECK_EPOLL" FORCE)
else(EPOLL_PROTOTYPE_EXISTS)
message(STATUS "Check if the system supports epoll - no")
set(${VARIABLE} "" CACHE INTERNAL "Result of CHECK_EPOLL" FORCE)
endif(EPOLL_PROTOTYPE_EXISTS)
endif("${VARIABLE}" MATCHES "^${VARIABLE}$")
endif(UNIX)
endmacro(CHECK_EPOLL)

41
config.h
View File

@@ -2,7 +2,6 @@
#define CONFIG_H 1
/* macros shared across multiple executables */
#define DEFAULT_CHUSER "nobody"
#define COLLECTOR_UNIX_SOCKET "/tmp/ndpid-collector.sock"
#define DISTRIBUTOR_UNIX_SOCKET "/tmp/ndpid-distributor.sock"
#define DISTRIBUTOR_HOST "127.0.0.1"
@@ -12,44 +11,34 @@
* NOTE: Buffer size needs to keep in sync with other implementations
* e.g. dependencies/nDPIsrvd.py
*/
#define NETWORK_BUFFER_MAX_SIZE 33792u /* 8192 + 8192 + 8192 + 8192 + 1024 */
#define NETWORK_BUFFER_MAX_SIZE 16384u /* 8192 + 8192 */
#define NETWORK_BUFFER_LENGTH_DIGITS 5u
#define NETWORK_BUFFER_LENGTH_DIGITS_STR "5"
#define PFRING_BUFFER_SIZE 65536u
#define TIME_S_TO_US(s) (s * 1000llu * 1000llu)
/* nDPId default config options */
#define nDPId_PIDFILE "/tmp/ndpid.pid"
#define nDPId_MAX_FLOWS_PER_THREAD 65536u
#define nDPId_MAX_FLOWS_PER_THREAD 4096u
#define nDPId_MAX_IDLE_FLOWS_PER_THREAD (nDPId_MAX_FLOWS_PER_THREAD / 32u)
#define nDPId_TICK_RESOLUTION 1000u
#define nDPId_MAX_READER_THREADS 32u
#define nDPId_ERROR_EVENT_THRESHOLD_N 16u
#define nDPId_ERROR_EVENT_THRESHOLD_TIME TIME_S_TO_US(10u) /* 10 sec */
#define nDPId_DAEMON_STATUS_INTERVAL TIME_S_TO_US(600u) /* 600 sec */
#define nDPId_MEMORY_PROFILING_LOG_INTERVAL TIME_S_TO_US(5u) /* 5 sec */
#define nDPId_COMPRESSION_SCAN_INTERVAL TIME_S_TO_US(20u) /* 20 sec */
#define nDPId_COMPRESSION_FLOW_INACTIVITY TIME_S_TO_US(30u) /* 30 sec */
#define nDPId_FLOW_SCAN_INTERVAL TIME_S_TO_US(10u) /* 10 sec */
#define nDPId_GENERIC_IDLE_TIME TIME_S_TO_US(600u) /* 600 sec */
#define nDPId_ICMP_IDLE_TIME TIME_S_TO_US(120u) /* 120 sec */
#define nDPId_TCP_IDLE_TIME TIME_S_TO_US(7440u) /* 7440 sec */
#define nDPId_UDP_IDLE_TIME TIME_S_TO_US(180u) /* 180 sec */
#define nDPId_TCP_POST_END_FLOW_TIME TIME_S_TO_US(120u) /* 120 sec */
#define nDPId_DAEMON_STATUS_INTERVAL 600000u /* 600 sec */
#define nDPId_MEMORY_PROFILING_LOG_INTERVAL 5000u /* 5 sec */
#define nDPId_COMPRESSION_SCAN_INTERVAL 20000u /* 20 sec */
#define nDPId_COMPRESSION_FLOW_INACTIVITY 30000u /* 30 sec */
#define nDPId_FLOW_SCAN_INTERVAL 10000u /* 10 sec */
#define nDPId_GENERIC_IDLE_TIME 600000u /* 600 sec */
#define nDPId_ICMP_IDLE_TIME 120000u /* 120 sec */
#define nDPId_TCP_IDLE_TIME 7440000u /* 7440 sec */
#define nDPId_UDP_IDLE_TIME 180000u /* 180 sec */
#define nDPId_TCP_POST_END_FLOW_TIME 120000u /* 120 sec */
#define nDPId_THREAD_DISTRIBUTION_SEED 0x03dd018b
#define nDPId_PACKETS_PLEN_MAX 8192u /* 8kB */
#define nDPId_PACKETS_PER_FLOW_TO_SEND 15u
#define nDPId_PACKETS_PER_FLOW_TO_PROCESS 32u
#define nDPId_PACKETS_PER_FLOW_TO_ANALYZE 32u
#define nDPId_ANALYZE_PLEN_MAX 1504u
#define nDPId_ANALYZE_PLEN_BIN_LEN 32u
#define nDPId_ANALYZE_PLEN_NUM_BINS 48u
#define nDPId_PACKETS_PER_FLOW_TO_PROCESS NDPI_DEFAULT_MAX_NUM_PKTS_PER_FLOW_TO_DISSECT
#define nDPId_FLOW_STRUCT_SEED 0x5defc104
/* nDPIsrvd default config options */
#define nDPIsrvd_PIDFILE "/tmp/ndpisrvd.pid"
#define nDPIsrvd_MAX_REMOTE_DESCRIPTORS 128
#define nDPIsrvd_MAX_REMOTE_DESCRIPTORS 32
#define nDPIsrvd_MAX_WRITE_BUFFERS 1024
#endif

6
dependencies/jsmn/jsmn.h vendored
View File

@@ -196,10 +196,10 @@ static int jsmn_parse_string(jsmn_parser *parser, const char *js,
jsmntok_t *token;
int start = parser->pos;
/* Skip starting quote */
parser->pos++;
/* Skip starting quote */
for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++) {
char c = js[parser->pos];

811
dependencies/nDPIsrvd.h vendored
View File

File diff suppressed because it is too large Load Diff

150
dependencies/nDPIsrvd.py vendored
View File

@@ -21,20 +21,19 @@ DEFAULT_PORT = 7000
DEFAULT_UNIX = '/tmp/ndpid-distributor.sock'
NETWORK_BUFFER_MIN_SIZE = 6 # NETWORK_BUFFER_LENGTH_DIGITS + 1
NETWORK_BUFFER_MAX_SIZE = 33792 # Please keep this value in sync with the one in config.h
nDPId_PACKETS_PLEN_MAX = 8192 # Please keep this value in sync with the one in config.h
NETWORK_BUFFER_MAX_SIZE = 16384 # Please keep this value in sync with the one in config.h
PKT_TYPE_ETH_IP4 = 0x0800
PKT_TYPE_ETH_IP6 = 0x86DD
class TermColor:
HINT = '\033[33m'
HINT = '\033[33m'
WARNING = '\033[93m'
FAIL = '\033[91m'
BOLD = '\033[1m'
END = '\033[0m'
BLINK = '\x1b[5m'
FAIL = '\033[91m'
BOLD = '\033[1m'
END = '\033[0m'
BLINK = '\x1b[5m'
if USE_COLORAMA is True:
COLOR_TUPLES = [ (Fore.BLUE, [Back.RED, Back.MAGENTA, Back.WHITE]),
@@ -52,21 +51,6 @@ class TermColor:
(Fore.LIGHTWHITE_EX, [Back.LIGHTBLACK_EX, Back.BLACK]),
(Fore.LIGHTYELLOW_EX, [Back.LIGHTRED_EX, Back.RED]) ]
@staticmethod
def disableColor():
TermColor.HINT = ''
TermColor.WARNING = ''
TermColor.FAIL = ''
TermColor.BOLD = ''
TermColor.END = ''
TermColor.BLINK = ''
global USE_COLORAMA
USE_COLORAMA = False
@staticmethod
def disableBlink():
TermColor.BLINK = ''
@staticmethod
def calcColorHash(string):
h = 0
@@ -84,9 +68,9 @@ class TermColor:
@staticmethod
def setColorByString(string):
global USE_COLORAMA
if USE_COLORAMA is True:
fg_color, bg_color = TermColor.getColorsByHash(string)
color_hash = TermColor.calcColorHash(string)
return '{}{}{}{}{}'.format(Style.BRIGHT, fg_color, bg_color, string, Style.RESET_ALL)
else:
return '{}{}{}'.format(TermColor.BOLD, string, TermColor.END)
@@ -141,9 +125,9 @@ class Instance:
if 'thread_id' not in json_dict:
return
thread_id = json_dict['thread_id']
if 'thread_ts_usec' in json_dict:
if 'thread_ts_msec' in json_dict:
mrtf = self.getMostRecentFlowTime(thread_id) if thread_id in self.thread_data else 0
self.setMostRecentFlowTime(thread_id, max(json_dict['thread_ts_usec'], mrtf))
self.setMostRecentFlowTime(thread_id, max(json_dict['thread_ts_msec'], mrtf))
class Flow:
@@ -185,16 +169,13 @@ class FlowManager:
if alias not in self.instances:
self.instances[alias] = dict()
if source not in self.instances[alias]:
self.instances[alias][source] = dict()
self.instances[alias][source] = Instance(alias, source)
self.instances[alias][source].setMostRecentFlowTimeFromJSON(json_dict)
return self.instances[alias][source]
@staticmethod
def getLastPacketTime(instance, flow_id, json_dict):
return max(int(json_dict['flow_src_last_pkt_time']), int(json_dict['flow_dst_last_pkt_time']), instance.flows[flow_id].flow_last_seen)
def getFlow(self, instance, json_dict):
if 'flow_id' not in json_dict:
return None
@@ -202,13 +183,13 @@ class FlowManager:
flow_id = int(json_dict['flow_id'])
if flow_id in instance.flows:
instance.flows[flow_id].flow_last_seen = FlowManager.getLastPacketTime(instance, flow_id, json_dict)
instance.flows[flow_id].flow_last_seen = int(json_dict['flow_last_seen'])
instance.flows[flow_id].flow_idle_time = int(json_dict['flow_idle_time'])
return instance.flows[flow_id]
thread_id = int(json_dict['thread_id'])
instance.flows[flow_id] = Flow(flow_id, thread_id)
instance.flows[flow_id].flow_last_seen = FlowManager.getLastPacketTime(instance, flow_id, json_dict)
instance.flows[flow_id].flow_last_seen = int(json_dict['flow_last_seen'])
instance.flows[flow_id].flow_idle_time = int(json_dict['flow_idle_time'])
instance.flows[flow_id].cleanup_reason = FlowManager.CLEANUP_REASON_INVALID
@@ -298,7 +279,6 @@ class nDPIsrvdException(Exception):
INVALID_LINE_RECEIVED = 4
CALLBACK_RETURNED_FALSE = 5
SOCKET_TIMEOUT = 6
JSON_DECODE_ERROR = 7
def __init__(self, etype):
self.etype = etype
@@ -345,51 +325,11 @@ class SocketTimeout(nDPIsrvdException):
def __str__(self):
return 'Socket timeout.'
class JsonDecodeError(nDPIsrvdException):
def __init__(self, json_exception, failed_line):
super().__init__(nDPIsrvdException.JSON_DECODE_ERROR)
self.json_exception = json_exception
self.failed_line = failed_line
def __str__(self):
return '{}: {}'.format(self.json_exception, self.failed_line)
class JsonFilter():
def __init__(self, filter_string):
self.filter_string = filter_string
self.filter = compile(filter_string, '<string>', 'eval')
def evaluate(self, json_dict):
if type(json_dict) is not dict:
raise nDPIsrvdException('Could not evaluate JSON Filter: expected dictionary, got {}'.format(type(json_dict)))
return eval(self.filter, {'json_dict': json_dict})
class nDPIsrvdSocket:
def __init__(self):
self.sock_family = None
self.flow_mgr = FlowManager()
self.received_bytes = 0
self.json_filter = list()
def addFilter(self, filter_str):
self.json_filter.append(JsonFilter(filter_str))
def evalFilters(self, json_dict):
for jf in self.json_filter:
try:
json_filter_retval = jf.evaluate(json_dict)
except Exception as err:
print()
sys.stderr.write('Error while evaluating expression "{}"\n'.format(jf.filter_string))
raise err
if not isinstance(json_filter_retval, bool):
print()
sys.stderr.write('Error while evaluating expression "{}"\n'.format(jf.filter_string))
raise nDPIsrvdException('JSON Filter returned an invalid type: expected bool, got {}'.format(type(json_filter_retval)))
if json_filter_retval is False:
return False
return True
def connect(self, addr):
if type(addr) is tuple:
@@ -405,8 +345,6 @@ class nDPIsrvdSocket:
self.msglen = 0
self.digitlen = 0
self.lines = []
self.failed_lines = []
self.filtered_lines = 0
def timeout(self, timeout):
self.sock.settimeout(timeout)
@@ -460,45 +398,25 @@ class nDPIsrvdSocket:
def parse(self, callback_json, callback_flow_cleanup, global_user_data):
retval = True
index = 0
for received_line in self.lines:
try:
json_dict = json.loads(received_line[0].decode('ascii', errors='replace'), strict=True)
except json.decoder.JSONDecodeError as e:
json_dict = dict()
self.failed_lines += [received_line]
self.lines = self.lines[1:]
raise JsonDecodeError(e, received_line)
json_dict = json.loads(received_line[0].decode('ascii', errors='replace'), strict=True)
instance = self.flow_mgr.getInstance(json_dict)
if instance is None:
self.failed_lines += [received_line]
retval = False
continue
current_flow = self.flow_mgr.getFlow(instance, json_dict)
filter_eval = self.evalFilters(json_dict)
if filter_eval is True:
try:
if callback_json(json_dict, instance, current_flow, global_user_data) is not True:
self.failed_lines += [received_line]
retval = False
except Exception as e:
self.failed_lines += [received_line]
self.lines = self.lines[1:]
raise(e)
else:
self.filtered_lines += 1
if callback_json(json_dict, instance, self.flow_mgr.getFlow(instance, json_dict), global_user_data) is not True:
retval = False
for _, flow in self.flow_mgr.getFlowsToCleanup(instance, json_dict).items():
if callback_flow_cleanup is None:
pass
elif filter_eval is True and callback_flow_cleanup(instance, flow, global_user_data) is not True:
self.failed_lines += [received_line]
self.lines = self.lines[1:]
elif callback_flow_cleanup(instance, flow, global_user_data) is not True:
retval = False
index += 1
self.lines = self.lines[1:]
self.lines = self.lines[index:]
return retval
@@ -522,31 +440,22 @@ class nDPIsrvdSocket:
return self.flow_mgr.doShutdown().items()
def verify(self):
if len(self.failed_lines) > 0:
raise nDPIsrvdException('Failed lines > 0: {}'.format(len(self.failed_lines)))
return self.flow_mgr.verifyFlows()
def defaultArgumentParser(desc='nDPIsrvd Python Interface', enable_json_filter=False,
def defaultArgumentParser(desc='nDPIsrvd Python Interface',
help_formatter=argparse.ArgumentDefaultsHelpFormatter):
parser = argparse.ArgumentParser(description=desc, formatter_class=help_formatter)
parser.add_argument('--host', type=str, help='nDPIsrvd host IP')
parser.add_argument('--port', type=int, default=DEFAULT_PORT, help='nDPIsrvd TCP port')
parser.add_argument('--unix', type=str, help='nDPIsrvd unix socket path')
if enable_json_filter is True:
parser.add_argument('--filter', type=str, action='append',
help='Set a filter string which if evaluates to True will invoke the JSON callback.\n'
'Example: json_dict[\'flow_event_name\'] == \'detected\' will only process \'detected\' events.')
return parser
def toSeconds(usec):
return usec / (1000 * 1000)
def validateAddress(args):
tcp_addr_set = False
address = None
if args.host is None:
address_tcpip = (DEFAULT_HOST, args.port)
address_tcpip = (DEFAULT_HOST, DEFAULT_PORT)
else:
address_tcpip = (args.host, args.port)
tcp_addr_set = True
@@ -568,23 +477,6 @@ def validateAddress(args):
return address
def prepareJsonFilter(args, nsock):
# HowTo use JSON Filters:
# Add `--filter [FILTER_STRING]` to the Python scripts that support JSON filtering.
# Examples:
# ./examples/py-json-stdout/json-stdout.py --filter '"ndpi" in json_dict and "proto" in json_dict["ndpi"]'
# The command above will print only JSONs that have the subobjects json_dict["ndpi"] and json_dict["ndpi"]["proto"] available.
# ./examples/py-flow-info/flow-info.py --filter 'json_dict["source"] == "eth0"' --filter '"flow_event_name" in json_dict and json_dict["flow_event_name"] == "analyse"'
# Multiple JSON filter will be ANDed together.
# Note: You may *only* use the global "json_dict" in your expressions.
try:
json_filter = args.filter
if json_filter is not None:
for jf in json_filter:
nsock.addFilter(jf)
except AttributeError:
pass
global schema
schema = {'packet_event_schema' : None, 'error_event_schema' : None, 'daemon_event_schema' : None, 'flow_event_schema' : None}

41
dependencies/uthash/.github/workflows/build.yml vendored
View File

@@ -1,41 +0,0 @@
name: build # This name shows up in badge.svg
on:
push: # any branch
pull_request:
branches: [ "master" ]
jobs:
build-gcc:
strategy:
matrix:
os: [ubuntu-latest, macos-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- run: make -C tests EXTRA_CFLAGS="-W -Wall -Wextra -Wswitch-default"
- run: make -C tests clean ; make -C tests pedantic
- run: make -C tests clean ; make -C tests pedantic EXTRA_CFLAGS=-DNO_DECLTYPE
- run: make -C tests clean ; make -C tests cplusplus
- run: make -C tests clean ; make -C tests cplusplus EXTRA_CFLAGS=-DNO_DECLTYPE
build-clang:
strategy:
matrix:
os: [ubuntu-latest, macos-latest]
runs-on: ${{ matrix.os }}
env:
CC: clang
CXX: clang++
steps:
- uses: actions/checkout@v3
- run: make -C tests EXTRA_CFLAGS="-W -Wall -Wextra -Wswitch-default"
- run: make -C tests clean ; make -C tests pedantic
- run: make -C tests clean ; make -C tests pedantic EXTRA_CFLAGS=-DNO_DECLTYPE
- run: make -C tests clean ; make -C tests cplusplus
- run: make -C tests clean ; make -C tests cplusplus EXTRA_CFLAGS=-DNO_DECLTYPE
build-asciidoc:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- run: sudo apt-get update && sudo apt-get install asciidoc -y
- run: make -C doc

2
dependencies/uthash/LICENSE vendored
View File

@@ -1,4 +1,4 @@
Copyright (c) 2005-2022, Troy D. Hanson https://troydhanson.github.io/uthash/
Copyright (c) 2005-2021, Troy D. Hanson http://troydhanson.github.io/uthash/
All rights reserved.
Redistribution and use in source and binary forms, with or without

1
dependencies/uthash/README.md vendored
View File

@@ -1,6 +1,5 @@
[![Build status](https://api.travis-ci.org/troydhanson/uthash.svg?branch=master)](https://travis-ci.org/troydhanson/uthash)
[![GitHub CI status](https://github.com/troydhanson/uthash/actions/workflows/build.yml/badge.svg)](https://github.com/troydhanson/uthash/actions/workflows/build.yml)
Documentation for uthash is available at:

6
dependencies/uthash/doc/index.html vendored
View File

@@ -13,8 +13,8 @@
</div> <!-- banner -->
<div id="topnav">
<a href="https://github.com/troydhanson/uthash">GitHub page</a> &gt;
uthash home <!-- https://troydhanson.github.io/uthash/ -->
<a href="http://github.com/troydhanson/uthash">GitHub page</a> &gt;
uthash home <!-- http://troydhanson.github.io/uthash/ -->
<a href="https://twitter.com/share" class="twitter-share-button" data-via="troydhanson">Tweet</a>
<script>!function(d,s,id){var js,fjs=d.getElementsByTagName(s)[0];if(!d.getElementById(id)){js=d.createElement(s);js.id=id;js.src="//platform.twitter.com/widgets.js";fjs.parentNode.insertBefore(js,fjs);}}(document,"script","twitter-wjs");</script>
@@ -43,7 +43,7 @@
<h2>developer</h2>
<div><a href="https://troydhanson.github.io/">Troy D. Hanson</a></div>
<div><a href="http://troydhanson.github.io/">Troy D. Hanson</a></div>
<h2>maintainer</h2>
<div><a href="https://github.com/Quuxplusone">Arthur O'Dwyer</a></div>

4
dependencies/uthash/doc/license.html vendored
View File

@@ -13,7 +13,7 @@
</div> <!-- banner -->
<div id="topnav">
<a href="https://troydhanson.github.io/uthash/">uthash home</a> &gt;
<a href="http://troydhanson.github.io/uthash/">uthash home</a> &gt;
BSD license
</div>
@@ -21,7 +21,7 @@
<div id="mid">
<div id="main">
<pre>
Copyright (c) 2005-2022, Troy D. Hanson https://troydhanson.github.io/uthash/
Copyright (c) 2005-2021, Troy D. Hanson http://troydhanson.github.io/uthash/
All rights reserved.
Redistribution and use in source and binary forms, with or without

1
dependencies/uthash/doc/styles.css vendored
View File

@@ -29,6 +29,7 @@ h1,p { margin: 0; } /* non-0 margin on firefox */
background-repeat: repeat-y;
/* background-color: #ffddaa; */
padding-top: 20px;
padding-top: 20px;
margin-bottom: 10px;
}

44
dependencies/uthash/doc/userguide.txt vendored
View File

@@ -5,7 +5,7 @@ v2.3.0, February 2021
To download uthash, follow this link back to the
https://github.com/troydhanson/uthash[GitHub project page].
Back to my https://troydhanson.github.io/[other projects].
Back to my http://troydhanson.github.io/[other projects].
A hash in C
-----------
@@ -805,7 +805,7 @@ Here is a simple example where a structure has a pointer member, called `key`.
.A pointer key
----------------------------------------------------------------------
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include "uthash.h"
@@ -816,16 +816,17 @@ typedef struct {
} el_t;
el_t *hash = NULL;
void *someaddr = &hash;
char *someaddr = NULL;
int main() {
el_t *d;
el_t *e = (el_t *)malloc(sizeof *e);
e->key = someaddr;
if (!e) return -1;
e->key = (void*)someaddr;
e->i = 1;
HASH_ADD_PTR(hash, key, e);
HASH_FIND_PTR(hash, &someaddr, d);
assert(d == e);
if (d) printf("found\n");
/* release memory */
HASH_DEL(hash, e);
@@ -834,7 +835,9 @@ int main() {
}
----------------------------------------------------------------------
This example is included in `tests/test57.c`.
This example is included in `tests/test57.c`. Note that the end of the program
deletes the element out of the hash, (and since no more elements remain in the
hash), uthash releases its internal memory.
Structure keys
~~~~~~~~~~~~~~
@@ -890,7 +893,7 @@ int main(int argc, char *argv[]) {
----------------------------------------------------------------------
This usage is nearly the same as the usage of a compound key explained below.
This usage is nearly the same as use of a compound key explained below.
Note that the general macros require the name of the `UT_hash_handle` to be
passed as the first argument (here, this is `hh`). The general macros are
@@ -1150,16 +1153,17 @@ always used with the `users_by_name` hash table).
Sorted insertion of new items
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
To maintain a sorted hash, you have two options. Your first
option is to use the `HASH_SRT` macro, which will sort any unordered list in
If you would like to maintain a sorted hash you have two options. The first
option is to use the HASH_SRT() macro, which will sort any unordered list in
'O(n log(n))'. This is the best strategy if you're just filling up a hash
table with items in random order with a single final `HASH_SRT` operation
when all is done. If you need the table to remain sorted as you add and remove
items, you can use `HASH_SRT` after every insertion operation, but that gives
a computational complexity of 'O(n^2 log n)' to insert 'n' items.
table with items in random order with a single final HASH_SRT() operation
when all is done. Obviously, this won't do what you want if you need
the list to be in an ordered state at times between insertion of
items. You can use HASH_SRT() after every insertion operation, but that will
yield a computational complexity of 'O(n^2 log n)'.
Your second option is to use the in-order add and replace macros.
The `HASH_ADD_*_INORDER` macros work just like their `HASH_ADD_*` counterparts, but
The second route you can take is via the in-order add and replace macros.
The `HASH_ADD_INORDER*` macros work just like their `HASH_ADD*` counterparts, but
with an additional comparison-function argument:
int name_sort(struct my_struct *a, struct my_struct *b) {
@@ -1168,11 +1172,11 @@ with an additional comparison-function argument:
HASH_ADD_KEYPTR_INORDER(hh, items, &item->name, strlen(item->name), item, name_sort);
These macros assume that the hash is already sorted according to the
comparison function, and insert the new item in its proper place.
A single insertion takes 'O(n)', resulting in a total computational
complexity of 'O(n^2)' to insert all 'n' items: slower than a single
`HASH_SRT`, but faster than doing a `HASH_SRT` after every insertion.
New items are sorted at insertion time in 'O(n)', thus resulting in a
total computational complexity of 'O(n^2)' for the creation of the hash
table with all items.
For in-order add to work, the list must be in an ordered state before
insertion of the new item.
Several sort orders
~~~~~~~~~~~~~~~~~~~

2
dependencies/uthash/doc/utarray.txt vendored
View File

@@ -139,7 +139,7 @@ a copy of the source string and pushes that copy into the array.
About UT_icd
~~~~~~~~~~~~
Arrays can be made of any type of element, not just integers and strings. The
Arrays be made of any type of element, not just integers and strings. The
elements can be basic types or structures. Unless you're dealing with integers
and strings (which use pre-defined `ut_int_icd` and `ut_str_icd`), you'll need
to define a `UT_icd` helper structure. This structure contains everything that

18
dependencies/uthash/src/utarray.h vendored
View File

@@ -1,5 +1,5 @@
/*
Copyright (c) 2008-2022, Troy D. Hanson https://troydhanson.github.io/uthash/
Copyright (c) 2008-2021, Troy D. Hanson http://troydhanson.github.io/uthash/
All rights reserved.
Redistribution and use in source and binary forms, with or without
@@ -38,6 +38,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define UTARRAY_UNUSED
#endif
#ifdef oom
#error "The name of macro 'oom' has been changed to 'utarray_oom'. Please update your code."
#define utarray_oom() oom()
#endif
#ifndef utarray_oom
#define utarray_oom() exit(-1)
#endif
@@ -229,16 +234,7 @@ typedef struct {
static void utarray_str_cpy(void *dst, const void *src) {
char *const *srcc = (char *const *)src;
char **dstc = (char**)dst;
if (*srcc == NULL) {
*dstc = NULL;
} else {
*dstc = (char*)malloc(strlen(*srcc) + 1);
if (*dstc == NULL) {
utarray_oom();
} else {
strcpy(*dstc, *srcc);
}
}
*dstc = (*srcc == NULL) ? NULL : strdup(*srcc);
}
static void utarray_str_dtor(void *elt) {
char **eltc = (char**)elt;

16
dependencies/uthash/src/uthash.h vendored
View File

@@ -1,5 +1,5 @@
/*
Copyright (c) 2003-2022, Troy D. Hanson https://troydhanson.github.io/uthash/
Copyright (c) 2003-2021, Troy D. Hanson http://troydhanson.github.io/uthash/
All rights reserved.
Redistribution and use in source and binary forms, with or without
@@ -51,8 +51,6 @@ typedef unsigned char uint8_t;
#else /* VS2008 or older (or VS2010 in C mode) */
#define NO_DECLTYPE
#endif
#elif defined(__MCST__) /* Elbrus C Compiler */
#define DECLTYPE(x) (__typeof(x))
#elif defined(__BORLANDC__) || defined(__ICCARM__) || defined(__LCC__) || defined(__WATCOMC__)
#define NO_DECLTYPE
#else /* GNU, Sun and other compilers */
@@ -159,7 +157,7 @@ do {
if (head) { \
unsigned _hf_bkt; \
HASH_TO_BKT(hashval, (head)->hh.tbl->num_buckets, _hf_bkt); \
if (HASH_BLOOM_TEST((head)->hh.tbl, hashval)) { \
if (HASH_BLOOM_TEST((head)->hh.tbl, hashval) != 0) { \
HASH_FIND_IN_BKT((head)->hh.tbl, hh, (head)->hh.tbl->buckets[ _hf_bkt ], keyptr, keylen, hashval, out); \
} \
} \
@@ -196,7 +194,7 @@ do {
} while (0)
#define HASH_BLOOM_BITSET(bv,idx) (bv[(idx)/8U] |= (1U << ((idx)%8U)))
#define HASH_BLOOM_BITTEST(bv,idx) ((bv[(idx)/8U] & (1U << ((idx)%8U))) != 0)
#define HASH_BLOOM_BITTEST(bv,idx) (bv[(idx)/8U] & (1U << ((idx)%8U)))
#define HASH_BLOOM_ADD(tbl,hashv) \
HASH_BLOOM_BITSET((tbl)->bloom_bv, ((hashv) & (uint32_t)((1UL << (tbl)->bloom_nbits) - 1U)))
@@ -208,7 +206,7 @@ do {
#define HASH_BLOOM_MAKE(tbl,oomed)
#define HASH_BLOOM_FREE(tbl)
#define HASH_BLOOM_ADD(tbl,hashv)
#define HASH_BLOOM_TEST(tbl,hashv) 1
#define HASH_BLOOM_TEST(tbl,hashv) (1)
#define HASH_BLOOM_BYTELEN 0U
#endif
@@ -452,7 +450,7 @@ do {
#define HASH_DELETE_HH(hh,head,delptrhh) \
do { \
const struct UT_hash_handle *_hd_hh_del = (delptrhh); \
struct UT_hash_handle *_hd_hh_del = (delptrhh); \
if ((_hd_hh_del->prev == NULL) && (_hd_hh_del->next == NULL)) { \
HASH_BLOOM_FREE((head)->hh.tbl); \
uthash_free((head)->hh.tbl->buckets, \
@@ -595,9 +593,7 @@ do {
/* SAX/FNV/OAT/JEN hash functions are macro variants of those listed at
* http://eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx
* (archive link: https://archive.is/Ivcan )
*/
* http://eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx */
#define HASH_SAX(key,keylen,hashv) \
do { \
unsigned _sx_i; \

7
dependencies/uthash/src/utlist.h vendored
View File

@@ -1,5 +1,5 @@
/*
Copyright (c) 2007-2022, Troy D. Hanson https://troydhanson.github.io/uthash/
Copyright (c) 2007-2021, Troy D. Hanson http://troydhanson.github.io/uthash/
All rights reserved.
Redistribution and use in source and binary forms, with or without
@@ -70,8 +70,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#else /* VS2008 or older (or VS2010 in C mode) */
#define NO_DECLTYPE
#endif
#elif defined(__MCST__) /* Elbrus C Compiler */
#define LDECLTYPE(x) __typeof(x)
#elif defined(__BORLANDC__) || defined(__ICCARM__) || defined(__LCC__) || defined(__WATCOMC__)
#define NO_DECLTYPE
#else /* GNU, Sun and other compilers */
@@ -711,8 +709,7 @@ do {
assert((del)->prev != NULL); \
if ((del)->prev == (del)) { \
(head)=NULL; \
} else if ((del) == (head)) { \
assert((del)->next != NULL); \
} else if ((del)==(head)) { \
(del)->next->prev = (del)->prev; \
(head) = (del)->next; \
} else { \

2
dependencies/uthash/src/utringbuffer.h vendored
View File

@@ -1,5 +1,5 @@
/*
Copyright (c) 2015-2022, Troy D. Hanson https://troydhanson.github.io/uthash/
Copyright (c) 2015-2021, Troy D. Hanson http://troydhanson.github.io/uthash/
All rights reserved.
Redistribution and use in source and binary forms, with or without

2
dependencies/uthash/src/utstack.h vendored
View File

@@ -1,5 +1,5 @@
/*
Copyright (c) 2018-2022, Troy D. Hanson https://troydhanson.github.io/uthash/
Copyright (c) 2018-2021, Troy D. Hanson http://troydhanson.github.io/uthash/
All rights reserved.
Redistribution and use in source and binary forms, with or without

2
dependencies/uthash/src/utstring.h vendored
View File

@@ -1,5 +1,5 @@
/*
Copyright (c) 2008-2022, Troy D. Hanson https://troydhanson.github.io/uthash/
Copyright (c) 2008-2021, Troy D. Hanson http://troydhanson.github.io/uthash/
All rights reserved.
Redistribution and use in source and binary forms, with or without

2
dependencies/uthash/tests/Makefile vendored
View File

@@ -12,7 +12,7 @@ PROGS = test1 test2 test3 test4 test5 test6 test7 test8 test9 \
test66 test67 test68 test69 test70 test71 test72 test73 \
test74 test75 test76 test77 test78 test79 test80 test81 \
test82 test83 test84 test85 test86 test87 test88 test89 \
test90 test91 test92 test93 test94 test95 test96 test97
test90 test91 test92 test93 test94 test95 test96
CFLAGS += -I$(HASHDIR)
#CFLAGS += -DHASH_BLOOM=16
#CFLAGS += -O2

1
dependencies/uthash/tests/README vendored
View File

@@ -98,7 +98,6 @@ test93: alt_fatal
test94: utlist with fields named other than 'next' and 'prev'
test95: utstack
test96: HASH_FUNCTION + HASH_KEYCMP
test97: deleting a const-qualified node from a hash
Other Make targets
================================================================================

2
dependencies/uthash/tests/hashscan.c vendored
View File

@@ -1,5 +1,5 @@
/*
Copyright (c) 2005-2022, Troy D. Hanson https://troydhanson.github.io/uthash/
Copyright (c) 2005-2021, Troy D. Hanson http://troydhanson.github.io/uthash/
All rights reserved.
Redistribution and use in source and binary forms, with or without

1
dependencies/uthash/tests/test57.ans vendored
View File

@@ -0,0 +1 @@
found

57
dependencies/uthash/tests/test57.c vendored
View File

@@ -1,5 +1,5 @@
#include <assert.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include "uthash.h"
typedef struct {
@@ -8,46 +8,25 @@ typedef struct {
UT_hash_handle hh;
} el_t;
el_t *findit(el_t *hash, void *keytofind)
{
el_t *found;
HASH_FIND_PTR(hash, &keytofind, found);
return found;
}
int main()
{
el_t *d;
el_t *hash = NULL;
el_t e1;
el_t e2;
e1.key = NULL;
e1.i = 1;
e2.key = &e2;
e2.i = 2;
assert(findit(hash, NULL) == NULL);
assert(findit(hash, &e1) == NULL);
assert(findit(hash, &e2) == NULL);
HASH_ADD_PTR(hash, key, &e1);
assert(findit(hash, NULL) == &e1);
assert(findit(hash, &e1) == NULL);
assert(findit(hash, &e2) == NULL);
HASH_ADD_PTR(hash, key, &e2);
assert(findit(hash, NULL) == &e1);
assert(findit(hash, &e1) == NULL);
assert(findit(hash, &e2) == &e2);
HASH_DEL(hash, &e1);
assert(findit(hash, NULL) == NULL);
assert(findit(hash, &e1) == NULL);
assert(findit(hash, &e2) == &e2);
HASH_CLEAR(hh, hash);
assert(hash == NULL);
char *someaddr = NULL;
el_t *e = (el_t*)malloc(sizeof(el_t));
if (!e) {
return -1;
}
e->key = (void*)someaddr;
e->i = 1;
HASH_ADD_PTR(hash,key,e);
HASH_FIND_PTR(hash, &someaddr, d);
if (d != NULL) {
printf("found\n");
}
/* release memory */
HASH_DEL(hash,e);
free(e);
return 0;
}

2
dependencies/uthash/tests/test65.c vendored
View File

@@ -3,7 +3,7 @@
#include "uthash.h"
// this is an example of how to do a LRU cache in C using uthash
// https://troydhanson.github.io/uthash/
// http://troydhanson.github.io/uthash/
// by Jehiah Czebotar 2011 - jehiah@gmail.com
// this code is in the public domain http://unlicense.org/

0
dependencies/uthash/tests/test97.ans vendored
View File

57
dependencies/uthash/tests/test97.c vendored
View File

@@ -1,57 +0,0 @@
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include "uthash.h"
struct item {
int payload;
UT_hash_handle hh;
};
void delete_without_modifying(struct item *head, const struct item *p)
{
struct item old;
memcpy(&old, p, sizeof(struct item)); // also copy the padding bits
assert(memcmp(&old, p, sizeof(struct item)) == 0);
assert(p->hh.tbl == head->hh.tbl); // class invariant
HASH_DEL(head, p);
assert(memcmp(&old, p, sizeof(struct item)) == 0); // unmodified by HASH_DEL
}
int main()
{
struct item *items = NULL;
struct item *found = NULL;
int fortytwo = 42;
int i;
for (i=0; i < 100; i++) {
struct item *p = (struct item *)malloc(sizeof *p);
p->payload = i;
HASH_ADD_INT(items, payload, p);
}
assert(HASH_COUNT(items) == 100);
// Delete item "42" from the hash, wherever it is.
HASH_FIND_INT(items, &fortytwo, found);
assert(found != NULL);
assert(found->payload == 42);
delete_without_modifying(items, found);
assert(HASH_COUNT(items) == 99);
HASH_FIND_INT(items, &fortytwo, found);
assert(found == NULL);
// Delete the very first item in the hash.
assert(items != NULL);
i = items->payload;
delete_without_modifying(items, items);
assert(HASH_COUNT(items) == 98);
HASH_FIND_INT(items, &i, found);
assert(found == NULL);
// leak the items, we don't care
return 0;
}

79
examples/README.md
View File

@@ -3,96 +3,65 @@
Some ready-2-use/ready-2-extend examples/utils.
All examples are prefixed with their used LANG.
## c-analysed
A feature extractor useful for ML/DL use cases.
It generates CSV files from flow "analyse" events.
Used also by `tests/run_tests.sh` if available.
## c-captured
A capture daemon suitable for low-resource devices.
It saves flows that were guessed/undetected/risky/midstream to a PCAP file for manual analysis.
Used also by `tests/run_tests.sh` if available.
Basicially a combination of `py-flow-undetected-to-pcap` and `py-risky-flow-to-pcap`.
## c-collectd
A collecd-exec compatible middleware that gathers statistic values from nDPId.
Used also by `tests/run_tests.sh` if available.
## c-influxd
## c-json-stdout
An InfluxDB push daemon. It aggregates various statistics gathered from nDPId.
The results are sent to a specified InfluxDB endpoint.
![](ndpid_grafana_example.png)
## c-notifyd
A notification daemon that sends information about suspicious flow events to DBUS.
Tiny nDPId json dumper. Does not provide any useful funcationality besides dumping parsed JSON objects.
## c-simple
Integration example that verifies flow timeouts on SIGUSR1.
Very tiny integration example.
## cxx-graph
## ~~go-dashboard~~ (DISCONTINUED!)
A standalone GLFW/OpenGL application that draws statistical data using ImWeb/ImPlot/ImGui.
## js-rt-analyzer
[nDPId-rt-analyzer](https://gitlab.com/verzulli/ndpid-rt-analyzer.git)
## js-rt-analyzer-frontend
[nDPId-rt-analyzer-frontend](https://gitlab.com/verzulli/ndpid-rt-analyzer-frontend.git)
A discontinued tty UI nDPId dashboard.
Removed with commit 29c72fb30bb7d5614c0a8ebb73bee2ac7eca6608.
## py-flow-info
Console friendly, colorful, prettyfied event printer.
Required by `tests/run_tests.sh`
Prints prettyfied information about flow events.
## py-machine-learning
## py-flow-dashboard
Contains:
1. Classification via Random Forests and SciLearn
2. Anomaly Detection via Autoencoder and Keras (Work-In-Progress!)
Use sklearn together with CSVs created with **c-analysed** to train and predict DPI detections.
Try it with: `./examples/py-machine-learning/sklearn_random_forest.py --csv ./ndpi-analysed.csv --proto-class tls.youtube --proto-class tls.github --proto-class tls.spotify --proto-class tls.facebook --proto-class tls.instagram --proto-class tls.doh_dot --proto-class quic --proto-class icmp`
This way you should get 9 different classification classes.
You may notice that some classes e.g. TLS protocol classifications have a higher false-negative/false-positive rate.
Unfortunately, I can not provide any datasets due to some privacy concerns.
But you may use a [pre-trained model](https://drive.google.com/file/d/1KEwbP-Gx7KJr54wNoa63I56VI4USCAPL/view?usp=sharing) with `--load-model`.
A realtime web based graph using Plotly/Dash.
Probably the most informative example.
## py-flow-multiprocess
Simple Python Multiprocess example spawning two worker processes, one connecting to nDPIsrvd and one printing flow id's to STDOUT.
## py-flow-undetected-to-pcap
Captures and saves undetected flows to a PCAP file.
## py-json-stdout
Dump received and parsed JSON objects.
Dump received and parsed JSON strings.
## py-risky-flow-to-pcap
Captures and saves risky flows to a PCAP file.
## py-schema-validation
Validate nDPId JSON messages against pre-defined JSON schema's.
Validate nDPId JSON strings against pre-defined JSON schema's.
See `schema/`.
Required by `tests/run_tests.sh`
## py-semantic-validation
Validate nDPId JSON messages against internal event semantics.
Validate nDPId JSON strings against internal event semantics.
Required by `tests/run_tests.sh`
## rs-simple
## py-ja3-checker
A straight forward Rust deserialization/parsing example.
## yaml-filebeat
An example filebeat configuration to parse and send nDPId JSON
messages to Elasticsearch. Allowing long term storage and data visualization with kibana
and various other tools that interact with Elasticsearch (No logstash required).
Captures JA3 hashes from nDPIsrvd and checks them against known hashes from [ja3er.com](https://ja3er.com).

2320
examples/c-analysed/c-analysed.c
View File

File diff suppressed because it is too large Load Diff

944
examples/c-captured/c-captured.c
View File

File diff suppressed because it is too large Load Diff

14
examples/c-collectd/README.md
View File

@@ -1,14 +0,0 @@
HowTo use this
==============
This HowTo assumes that the examples were sucessfully compiled and installed within the prefix `/usr` on your target machine.
1. Make sure nDPId and Collectd is running.
2. Edit `collectd.conf` usually in `/etc`.
3. Add the lines in `plugin_nDPIsrvd.conf` to your `collectd.conf`.
You may adapt this file depending what command line arguments you'd supplied to `nDPId`.
4. Reload your Collectd instance.
5. Optional: Install a http server of your choice.
Place the files in `/usr/share/nDPId/nDPIsrvd-collectd/www` somewhere in your www root.
6. Optional: Add `rrdgraph.sh` as cron job e.g. `0 * * * * /usr/share/nDPId/nDPIsrvd-collectd/rrdgraph.sh [path-to-the-collectd-rrd-directory] [path-to-your-dpi-wwwroot]`.
This will run `rrdgraph.sh` once per hour. You can adjust this until it fit your needs.

1840
examples/c-collectd/c-collectd.c
View File

File diff suppressed because it is too large Load Diff

12
examples/c-collectd/plugin_nDPIsrvd.conf
View File

@@ -3,12 +3,12 @@ LoadPlugin exec
<Plugin exec>
Exec "ndpi" "/usr/bin/nDPIsrvd-collectd"
# Exec "ndpi" "/usr/bin/nDPIsrvd-collectd" "-s" "/tmp/ndpid-distributor.sock"
# Exec "ndpi" "/tmp/nDPIsrvd-collectd" "-s" "127.0.0.1:7000"
# Exec "ndpi" "/usr/bin/nDPIsrvd-collectd" "-s" "127.0.0.1:7000"
</Plugin>
# Uncomment for testing
#LoadPlugin write_log
#LoadPlugin rrdtool
#<Plugin rrdtool>
# DataDir "nDPIsrvd-collectd"
#</Plugin>
LoadPlugin write_log
LoadPlugin rrdtool
<Plugin rrdtool>
DataDir "nDPIsrvd-collectd"
</Plugin>

65
examples/c-collectd/plugin_nDPIsrvd_types.db Normal file
View File

@@ -0,0 +1,65 @@
# Add those types to collectd types.db
# e.g. `cat plugin_nDPIsrvd_types.db >>/usr/share/collectd/types.db'
# flow event counters
flow_new_count value:GAUGE:0:U
flow_end_count value:GAUGE:0:U
flow_idle_count value:GAUGE:0:U
flow_guessed_count value:GAUGE:0:U
flow_detected_count value:GAUGE:0:U
flow_detection_update_count value:GAUGE:0:U
flow_not_detected_count value:GAUGE:0:U
# flow additional counters
flow_total_bytes value:GAUGE:0:U
flow_risky_count value:GAUGE:0:U
# flow breed counters
flow_breed_safe_count value:GAUGE:0:U
flow_breed_acceptable_count value:GAUGE:0:U
flow_breed_fun_count value:GAUGE:0:U
flow_breed_unsafe_count value:GAUGE:0:U
flow_breed_potentially_dangerous_count value:GAUGE:0:U
flow_breed_dangerous_count value:GAUGE:0:U
flow_breed_unrated_count value:GAUGE:0:U
flow_breed_unknown_count value:GAUGE:0:U
# flow category counters
flow_category_media_count value:GAUGE:0:U
flow_category_vpn_count value:GAUGE:0:U
flow_category_email_count value:GAUGE:0:U
flow_category_data_transfer_count value:GAUGE:0:U
flow_category_web_count value:GAUGE:0:U
flow_category_social_network_count value:GAUGE:0:U
flow_category_download_count value:GAUGE:0:U
flow_category_game_count value:GAUGE:0:U
flow_category_chat_count value:GAUGE:0:U
flow_category_voip_count value:GAUGE:0:U
flow_category_database_count value:GAUGE:0:U
flow_category_remote_access_count value:GAUGE:0:U
flow_category_cloud_count value:GAUGE:0:U
flow_category_network_count value:GAUGE:0:U
flow_category_collaborative_count value:GAUGE:0:U
flow_category_rpc_count value:GAUGE:0:U
flow_category_streaming_count value:GAUGE:0:U
flow_category_system_count value:GAUGE:0:U
flow_category_software_update_count value:GAUGE:0:U
flow_category_music_count value:GAUGE:0:U
flow_category_video_count value:GAUGE:0:U
flow_category_shopping_count value:GAUGE:0:U
flow_category_productivity_count value:GAUGE:0:U
flow_category_file_sharing_count value:GAUGE:0:U
flow_category_mining_count value:GAUGE:0:U
flow_category_malware_count value:GAUGE:0:U
flow_category_advertisment_count value:GAUGE:0:U
flow_category_other_count value:GAUGE:0:U
flow_category_unknown_count value:GAUGE:0:U
# flow l3 / l4 counters
flow_l3_ip4_count value:GAUGE:0:U
flow_l3_ip6_count value:GAUGE:0:U
flow_l3_other_count value:GAUGE:0:U
flow_l4_icmp_count value:GAUGE:0:U
flow_l4_tcp_count value:GAUGE:0:U
flow_l4_udp_count value:GAUGE:0:U
flow_l4_other_count value:GAUGE:0:U

631
examples/c-collectd/rrdgraph.sh
View File

@@ -1,631 +0,0 @@
#!/usr/bin/env sh
RRDDIR="${1}"
OUTDIR="${2}"
RRDARGS="--width=800 --height=400"
REQUIRED_RRDCNT=130
if [ -z "${RRDDIR}" ]; then
printf '%s: Missing RRD directory which contains nDPIsrvd/Collectd files.\n' "${0}"
exit 1
fi
if [ -z "${OUTDIR}" ]; then
printf '%s: Missing Output directory which contains HTML files.\n' "${0}"
exit 1
fi
if [ $(ls -al ${RRDDIR}/gauge-flow_*.rrd | wc -l) -ne ${REQUIRED_RRDCNT} ]; then
printf '%s: Missing some *.rrd files. Expected: %s, Got: %s\n' "${0}" "${REQUIRED_RRDCNT}" "$(ls -al ${RRDDIR}/gauge-flow_*.rrd | wc -l)"
exit 1
fi
if [ ! -r "${OUTDIR}/index.html" -o ! -r "${OUTDIR}/flows.html" -o ! -r "${OUTDIR}/other.html" -o ! -r "${OUTDIR}/detections.html" -o ! -r "${OUTDIR}/categories.html" ]; then
printf '%s: Missing some *.html files.\n' "${0}"
exit 1
fi
TIME_PAST_HOUR="--start=-3600 --end=-0"
TIME_PAST_12HOURS="--start=-43200 --end=-0"
TIME_PAST_DAY="--start=-86400 --end=-0"
TIME_PAST_WEEK="--start=-604800 --end=-0"
TIME_PAST_MONTH="--start=-2419200 --end=-0"
TIME_PAST_3MONTHS="--start=-8035200 --end=-0"
TIME_PAST_YEAR="--start=-31536000 --end=-0"
rrdtool_graph_colorize_missing_data() {
printf 'CDEF:offline=%s,UN,INF,* AREA:offline#B3B3B311:' "${1}"
}
rrdtool_graph_print_cur_min_max_avg() {
printf 'GPRINT:%s:LAST:Current\:%%8.2lf ' "${1}"
printf 'GPRINT:%s:MIN:Minimum\:%%8.2lf ' "${1}"
printf 'GPRINT:%s:MAX:Maximum\:%%8.2lf ' "${1}"
printf 'GPRINT:%s:AVERAGE:Average\:%%8.2lf\\n' "${1}"
}
rrdtool_graph() {
TITLE="${1}"
shift
YAXIS_NAME="${1}"
shift
OUTPNG="${1}"
shift
rrdtool graph ${RRDARGS} -t "${TITLE} (past hour)" -v ${YAXIS_NAME} -Y ${TIME_PAST_HOUR} "${OUTPNG}_past_hour.png" ${*}
rrdtool graph ${RRDARGS} -t "${TITLE} (past 12 hours)" -v ${YAXIS_NAME} -Y ${TIME_PAST_12HOURS} "${OUTPNG}_past_12hours.png" ${*}
rrdtool graph ${RRDARGS} -t "${TITLE} (past day)" -v ${YAXIS_NAME} -Y ${TIME_PAST_DAY} "${OUTPNG}_past_day.png" ${*}
rrdtool graph ${RRDARGS} -t "${TITLE} (past week)" -v ${YAXIS_NAME} -Y ${TIME_PAST_WEEK} "${OUTPNG}_past_week.png" ${*}
rrdtool graph ${RRDARGS} -t "${TITLE} (past month)" -v ${YAXIS_NAME} -Y ${TIME_PAST_MONTH} "${OUTPNG}_past_month.png" ${*}
rrdtool graph ${RRDARGS} -t "${TITLE} (past 3 months)" -v ${YAXIS_NAME} -Y ${TIME_PAST_3MONTHS} "${OUTPNG}_past_month.png" ${*}
rrdtool graph ${RRDARGS} -t "${TITLE} (past year)" -v ${YAXIS_NAME} -Y ${TIME_PAST_YEAR} "${OUTPNG}_past_year.png" ${*}
}
rrdtool_graph Flows Amount "${OUTDIR}/flows" \
DEF:flows_active=${RRDDIR}/gauge-flow_active_count.rrd:value:AVERAGE \
$(rrdtool_graph_colorize_missing_data flows_active) \
AREA:flows_active#54EC48::STACK \
LINE2:flows_active#24BC14:"Active" \
$(rrdtool_graph_print_cur_min_max_avg flows_active)
rrdtool_graph Detections Amount "${OUTDIR}/detections" \
DEF:flows_detected=${RRDDIR}/gauge-flow_detected_count.rrd:value:AVERAGE \
DEF:flows_guessed=${RRDDIR}/gauge-flow_guessed_count.rrd:value:AVERAGE \
DEF:flows_not_detected=${RRDDIR}/gauge-flow_not_detected_count.rrd:value:AVERAGE \
DEF:flows_detection_update=${RRDDIR}/counter-flow_detection_update_count.rrd:value:AVERAGE \
DEF:flows_risky=${RRDDIR}/counter-flow_risky_count.rrd:value:AVERAGE \
$(rrdtool_graph_colorize_missing_data flows_detected) \
AREA:flows_detected#00bfff::STACK \
AREA:flows_detection_update#a1b8c4::STACK \
AREA:flows_guessed#ffff4d::STACK \
AREA:flows_not_detected#ffa64d::STACK \
AREA:flows_risky#ff4000::STACK \
LINE2:flows_detected#0000ff:"Detected........" \
$(rrdtool_graph_print_cur_min_max_avg flows_detected) \
LINE2:flows_guessed#cccc00:"Guessed........." \
$(rrdtool_graph_print_cur_min_max_avg flows_guessed) \
LINE2:flows_not_detected#ff8000:"Not-Detected...." \
$(rrdtool_graph_print_cur_min_max_avg flows_not_detected) \
LINE2:flows_detection_update#4f6e7d:"Detection-Update" \
$(rrdtool_graph_print_cur_min_max_avg flows_detection_update) \
LINE2:flows_risky#b32d00:"Risky..........." \
$(rrdtool_graph_print_cur_min_max_avg flows_risky)
rrdtool_graph "Traffic (IN/OUT)" Bytes "${OUTDIR}/traffic" \
DEF:total_src_bytes=${RRDDIR}/counter-flow_src_total_bytes.rrd:value:AVERAGE \
DEF:total_dst_bytes=${RRDDIR}/counter-flow_dst_total_bytes.rrd:value:AVERAGE \
$(rrdtool_graph_colorize_missing_data total_src_bytes) \
AREA:total_src_bytes#00cc99:"Total-Bytes-Source2Dest":STACK \
$(rrdtool_graph_print_cur_min_max_avg total_src_bytes) \
STACK:total_dst_bytes#669999:"Total-Bytes-Dest2Source" \
$(rrdtool_graph_print_cur_min_max_avg total_dst_bytes)
rrdtool_graph Layer3-Flows Amount "${OUTDIR}/layer3" \
DEF:layer3_ip4=${RRDDIR}/gauge-flow_l3_ip4_count.rrd:value:AVERAGE \
DEF:layer3_ip6=${RRDDIR}/gauge-flow_l3_ip6_count.rrd:value:AVERAGE \
DEF:layer3_other=${RRDDIR}/gauge-flow_l3_other_count.rrd:value:AVERAGE \
$(rrdtool_graph_colorize_missing_data layer3_ip4) \
AREA:layer3_ip4#73d97d::STACK \
AREA:layer3_ip6#66b3ff::STACK \
AREA:layer3_other#bea1c4::STACK \
LINE2:layer3_ip4#21772a:"IPv4." \
$(rrdtool_graph_print_cur_min_max_avg layer3_ip4) \
LINE2:layer3_ip6#0066cc:"IPv6." \
$(rrdtool_graph_print_cur_min_max_avg layer3_ip6) \
LINE2:layer3_other#92629d:"Other" \
$(rrdtool_graph_print_cur_min_max_avg layer3_other)
rrdtool_graph Layer4-Flows Amount "${OUTDIR}/layer4" \
DEF:layer4_tcp=${RRDDIR}/gauge-flow_l4_tcp_count.rrd:value:AVERAGE \
DEF:layer4_udp=${RRDDIR}/gauge-flow_l4_udp_count.rrd:value:AVERAGE \
DEF:layer4_icmp=${RRDDIR}/gauge-flow_l4_icmp_count.rrd:value:AVERAGE \
DEF:layer4_other=${RRDDIR}/gauge-flow_l4_other_count.rrd:value:AVERAGE \
$(rrdtool_graph_colorize_missing_data layer4_tcp) \
AREA:layer4_tcp#73d97d::STACK \
AREA:layer4_udp#66b3ff::STACK \
AREA:layer4_icmp#ee5d9a::STACK \
AREA:layer4_other#bea1c4::STACK \
LINE2:layer4_tcp#21772a:"TCP.." \
$(rrdtool_graph_print_cur_min_max_avg layer4_tcp) \
LINE2:layer4_udp#0066cc:"UDP.." \
$(rrdtool_graph_print_cur_min_max_avg layer4_udp) \
LINE2:layer4_icmp#d01663:"ICMP." \
$(rrdtool_graph_print_cur_min_max_avg layer4_icmp) \
LINE2:layer4_other#83588d:"Other" \
$(rrdtool_graph_print_cur_min_max_avg layer4_other)
rrdtool_graph Confidence Amount "${OUTDIR}/confidence" \
DEF:conf_ip=${RRDDIR}/gauge-flow_confidence_by_ip.rrd:value:AVERAGE \
DEF:conf_port=${RRDDIR}/gauge-flow_confidence_by_port.rrd:value:AVERAGE \
DEF:conf_aggr=${RRDDIR}/gauge-flow_confidence_dpi_aggressive.rrd:value:AVERAGE \
DEF:conf_cache=${RRDDIR}/gauge-flow_confidence_dpi_cache.rrd:value:AVERAGE \
DEF:conf_pcache=${RRDDIR}/gauge-flow_confidence_dpi_partial_cache.rrd:value:AVERAGE \
DEF:conf_part=${RRDDIR}/gauge-flow_confidence_dpi_partial.rrd:value:AVERAGE \
DEF:conf_dpi=${RRDDIR}/gauge-flow_confidence_dpi.rrd:value:AVERAGE \
DEF:conf_nbpf=${RRDDIR}/gauge-flow_confidence_nbpf.rrd:value:AVERAGE \
DEF:conf_ukn=${RRDDIR}/gauge-flow_confidence_unknown.rrd:value:AVERAGE \
$(rrdtool_graph_colorize_missing_data conf_ip) \
AREA:conf_ip#4dff4d::STACK \
AREA:conf_port#c2ff33::STACK \
AREA:conf_aggr#ffe433::STACK \
AREA:conf_cache#ffb133::STACK \
AREA:conf_pcache#ff5f33::STACK \
AREA:conf_part#e74b5b::STACK \
AREA:conf_dpi#a5aca0::STACK \
AREA:conf_nbpf#d7c1cc::STACK \
AREA:conf_ukn#ddccbb::STACK \
LINE2:conf_ip#00e600:"By-IP................" \
$(rrdtool_graph_print_cur_min_max_avg conf_ip) \
LINE2:conf_port#8fce00:"By-Port.............." \
$(rrdtool_graph_print_cur_min_max_avg conf_port) \
LINE2:conf_aggr#e6c700:"DPI-Aggressive......." \
$(rrdtool_graph_print_cur_min_max_avg conf_aggr) \
LINE2:conf_cache#e68e00:"DPI-Cache............" \
$(rrdtool_graph_print_cur_min_max_avg conf_cache) \
LINE2:conf_pcache#e63200:"DPI-Partial-Cache...." \
$(rrdtool_graph_print_cur_min_max_avg conf_pcache) \
LINE2:conf_part#c61b2b:"DPI-Partial.........." \
$(rrdtool_graph_print_cur_min_max_avg conf_part) \
LINE2:conf_dpi#7e8877:"DPI.................." \
$(rrdtool_graph_print_cur_min_max_avg conf_dpi) \
LINE2:conf_nbpf#ae849a:"nBPF................." \
$(rrdtool_graph_print_cur_min_max_avg conf_nbpf) \
LINE2:conf_ukn#aa9988:"Unknown.............." \
$(rrdtool_graph_print_cur_min_max_avg conf_ukn)
rrdtool_graph Breeds Amount "${OUTDIR}/breed" \
DEF:breed_safe=${RRDDIR}/gauge-flow_breed_safe_count.rrd:value:AVERAGE \
DEF:breed_acceptable=${RRDDIR}/gauge-flow_breed_acceptable_count.rrd:value:AVERAGE \
DEF:breed_fun=${RRDDIR}/gauge-flow_breed_fun_count.rrd:value:AVERAGE \
DEF:breed_unsafe=${RRDDIR}/gauge-flow_breed_unsafe_count.rrd:value:AVERAGE \
DEF:breed_potentially_dangerous=${RRDDIR}/gauge-flow_breed_potentially_dangerous_count.rrd:value:AVERAGE \
DEF:breed_dangerous=${RRDDIR}/gauge-flow_breed_dangerous_count.rrd:value:AVERAGE \
DEF:breed_unrated=${RRDDIR}/gauge-flow_breed_unrated_count.rrd:value:AVERAGE \
DEF:breed_unknown=${RRDDIR}/gauge-flow_breed_unknown_count.rrd:value:AVERAGE \
$(rrdtool_graph_colorize_missing_data breed_safe) \
AREA:breed_safe#4dff4d::STACK \
AREA:breed_acceptable#c2ff33::STACK \
AREA:breed_fun#ffe433::STACK \
AREA:breed_unsafe#ffb133::STACK \
AREA:breed_potentially_dangerous#ff5f33::STACK \
AREA:breed_dangerous#e74b5b::STACK \
AREA:breed_unrated#a5aca0::STACK \
AREA:breed_unknown#d7c1cc::STACK \
LINE2:breed_safe#00e600:"Safe................." \
$(rrdtool_graph_print_cur_min_max_avg breed_safe) \
LINE2:breed_acceptable#8fce00:"Acceptable..........." \
$(rrdtool_graph_print_cur_min_max_avg breed_acceptable) \
LINE2:breed_fun#e6c700:"Fun.................." \
$(rrdtool_graph_print_cur_min_max_avg breed_fun) \
LINE2:breed_unsafe#e68e00:"Unsafe..............." \
$(rrdtool_graph_print_cur_min_max_avg breed_unsafe) \
LINE2:breed_potentially_dangerous#e63200:"Potentially-Dangerous" \
$(rrdtool_graph_print_cur_min_max_avg breed_potentially_dangerous) \
LINE2:breed_dangerous#c61b2b:"Dangerous............" \
$(rrdtool_graph_print_cur_min_max_avg breed_dangerous) \
LINE2:breed_unrated#7e8877:"Unrated.............." \
$(rrdtool_graph_print_cur_min_max_avg breed_unrated) \
LINE2:breed_unknown#ae849a:"Unknown.............." \
$(rrdtool_graph_print_cur_min_max_avg breed_unknown)
rrdtool_graph Categories 'Amount(SUM)' "${OUTDIR}/categories" \
DEF:cat_adlt=${RRDDIR}/gauge-flow_category_adult_content_count.rrd:value:AVERAGE \
DEF:cat_ads=${RRDDIR}/gauge-flow_category_advertisment_count.rrd:value:AVERAGE \
DEF:cat_chat=${RRDDIR}/gauge-flow_category_chat_count.rrd:value:AVERAGE \
DEF:cat_cloud=${RRDDIR}/gauge-flow_category_cloud_count.rrd:value:AVERAGE \
DEF:cat_collab=${RRDDIR}/gauge-flow_category_collaborative_count.rrd:value:AVERAGE \
DEF:cat_conn=${RRDDIR}/gauge-flow_category_conn_check_count.rrd:value:AVERAGE \
DEF:cat_cybr=${RRDDIR}/gauge-flow_category_cybersecurity_count.rrd:value:AVERAGE \
DEF:cat_xfer=${RRDDIR}/gauge-flow_category_data_transfer_count.rrd:value:AVERAGE \
DEF:cat_db=${RRDDIR}/gauge-flow_category_database_count.rrd:value:AVERAGE \
DEF:cat_dl=${RRDDIR}/gauge-flow_category_download_count.rrd:value:AVERAGE \
DEF:cat_mail=${RRDDIR}/gauge-flow_category_email_count.rrd:value:AVERAGE \
DEF:cat_fs=${RRDDIR}/gauge-flow_category_file_sharing_count.rrd:value:AVERAGE \
DEF:cat_game=${RRDDIR}/gauge-flow_category_game_count.rrd:value:AVERAGE \
DEF:cat_gamb=${RRDDIR}/gauge-flow_category_gambling_count.rrd:value:AVERAGE \
DEF:cat_iot=${RRDDIR}/gauge-flow_category_iot_scada_count.rrd:value:AVERAGE \
DEF:cat_mal=${RRDDIR}/gauge-flow_category_malware_count.rrd:value:AVERAGE \
DEF:cat_med=${RRDDIR}/gauge-flow_category_media_count.rrd:value:AVERAGE \
DEF:cat_min=${RRDDIR}/gauge-flow_category_mining_count.rrd:value:AVERAGE \
DEF:cat_mus=${RRDDIR}/gauge-flow_category_music_count.rrd:value:AVERAGE \
DEF:cat_net=${RRDDIR}/gauge-flow_category_network_count.rrd:value:AVERAGE \
DEF:cat_prod=${RRDDIR}/gauge-flow_category_productivity_count.rrd:value:AVERAGE \
DEF:cat_rem=${RRDDIR}/gauge-flow_category_remote_access_count.rrd:value:AVERAGE \
DEF:cat_rpc=${RRDDIR}/gauge-flow_category_rpc_count.rrd:value:AVERAGE \
DEF:cat_shop=${RRDDIR}/gauge-flow_category_shopping_count.rrd:value:AVERAGE \
DEF:cat_soc=${RRDDIR}/gauge-flow_category_social_network_count.rrd:value:AVERAGE \
DEF:cat_soft=${RRDDIR}/gauge-flow_category_software_update_count.rrd:value:AVERAGE \
DEF:cat_str=${RRDDIR}/gauge-flow_category_streaming_count.rrd:value:AVERAGE \
DEF:cat_sys=${RRDDIR}/gauge-flow_category_system_count.rrd:value:AVERAGE \
DEF:cat_ukn=${RRDDIR}/gauge-flow_category_unknown_count.rrd:value:AVERAGE \
DEF:cat_uns=${RRDDIR}/gauge-flow_category_unspecified_count.rrd:value:AVERAGE \
DEF:cat_vid=${RRDDIR}/gauge-flow_category_video_count.rrd:value:AVERAGE \
DEF:cat_vrt=${RRDDIR}/gauge-flow_category_virt_assistant_count.rrd:value:AVERAGE \
DEF:cat_voip=${RRDDIR}/gauge-flow_category_voip_count.rrd:value:AVERAGE \
DEF:cat_vpn=${RRDDIR}/gauge-flow_category_vpn_count.rrd:value:AVERAGE \
DEF:cat_web=${RRDDIR}/gauge-flow_category_web_count.rrd:value:AVERAGE \
DEF:cat_banned=${RRDDIR}/gauge-flow_category_banned_site_count.rrd:value:AVERAGE \
DEF:cat_unavail=${RRDDIR}/gauge-flow_category_site_unavail_count.rrd:value:AVERAGE \
DEF:cat_allowed=${RRDDIR}/gauge-flow_category_allowed_site_count.rrd:value:AVERAGE \
DEF:cat_antimal=${RRDDIR}/gauge-flow_category_antimalware_count.rrd:value:AVERAGE \
DEF:cat_crypto=${RRDDIR}/gauge-flow_category_crypto_currency_count.rrd:value:AVERAGE \
$(rrdtool_graph_colorize_missing_data cat_adlt) \
AREA:cat_adlt#f0c032:"Adult.................." \
$(rrdtool_graph_print_cur_min_max_avg cat_adlt) \
STACK:cat_ads#f1c232:"Advertisment..........." \
$(rrdtool_graph_print_cur_min_max_avg cat_ads) \
STACK:cat_chat#6fa8dc:"Chat..................." \
$(rrdtool_graph_print_cur_min_max_avg cat_chat) \
STACK:cat_cloud#2986cc:"Cloud.................." \
$(rrdtool_graph_print_cur_min_max_avg cat_cloud) \
STACK:cat_collab#3212aa:"Collaborative.........." \
$(rrdtool_graph_print_cur_min_max_avg cat_collab) \
STACK:cat_conn#22aa11:"Connection-Check......." \
$(rrdtool_graph_print_cur_min_max_avg cat_conn) \
STACK:cat_cybr#00ff00:"Cybersecurity.........." \
$(rrdtool_graph_print_cur_min_max_avg cat_cybr) \
STACK:cat_xfer#16537e:"Data-Transfer.........." \
$(rrdtool_graph_print_cur_min_max_avg cat_xfer) \
STACK:cat_db#cc0000:"Database..............." \
$(rrdtool_graph_print_cur_min_max_avg cat_db) \
STACK:cat_dl#6a329f:"Download..............." \
$(rrdtool_graph_print_cur_min_max_avg cat_dl) \
STACK:cat_mail#3600cc:"Mail..................." \
$(rrdtool_graph_print_cur_min_max_avg cat_mail) \
STACK:cat_fs#c90076:"File-Sharing..........." \
$(rrdtool_graph_print_cur_min_max_avg cat_fs) \
STACK:cat_game#00ff26:"Game..................." \
$(rrdtool_graph_print_cur_min_max_avg cat_game) \
STACK:cat_gamb#aa0026:"Gambling..............." \
$(rrdtool_graph_print_cur_min_max_avg cat_gamb) \
STACK:cat_iot#227867:"IoT-Scada.............." \
$(rrdtool_graph_print_cur_min_max_avg cat_iot) \
STACK:cat_mal#f44336:"Malware................" \
$(rrdtool_graph_print_cur_min_max_avg cat_mal) \
STACK:cat_med#ff8300:"Media.................." \
$(rrdtool_graph_print_cur_min_max_avg cat_med) \
STACK:cat_min#ff0000:"Mining................." \
$(rrdtool_graph_print_cur_min_max_avg cat_min) \
STACK:cat_mus#00fff0:"Music.................." \
$(rrdtool_graph_print_cur_min_max_avg cat_mus) \
STACK:cat_net#ddff00:"Network................" \
$(rrdtool_graph_print_cur_min_max_avg cat_net) \
STACK:cat_prod#29ff00:"Productivity..........." \
$(rrdtool_graph_print_cur_min_max_avg cat_prod) \
STACK:cat_rem#b52c2c:"Remote-Access.........." \
$(rrdtool_graph_print_cur_min_max_avg cat_rem) \
STACK:cat_rpc#e15a5a:"Remote-Procedure-Call.." \
$(rrdtool_graph_print_cur_min_max_avg cat_rpc) \
STACK:cat_shop#0065ff:"Shopping..............." \
$(rrdtool_graph_print_cur_min_max_avg cat_shop) \
STACK:cat_soc#8fce00:"Social-Network........." \
$(rrdtool_graph_print_cur_min_max_avg cat_soc) \
STACK:cat_soft#007a0d:"Software-Update........" \
$(rrdtool_graph_print_cur_min_max_avg cat_soft) \
STACK:cat_str#ff00b8:"Streaming.............." \
$(rrdtool_graph_print_cur_min_max_avg cat_str) \
STACK:cat_sys#f4ff00:"System................." \
$(rrdtool_graph_print_cur_min_max_avg cat_sys) \
STACK:cat_ukn#999999:"Unknown................" \
$(rrdtool_graph_print_cur_min_max_avg cat_ukn) \
STACK:cat_uns#999999:"Unspecified............" \
$(rrdtool_graph_print_cur_min_max_avg cat_uns) \
STACK:cat_vid#518820:"Video.................." \
$(rrdtool_graph_print_cur_min_max_avg cat_vid) \
STACK:cat_vrt#216820:"Virtual-Assistant......" \
$(rrdtool_graph_print_cur_min_max_avg cat_vrt) \
STACK:cat_voip#ffc700:"Voice-Over-IP.........." \
$(rrdtool_graph_print_cur_min_max_avg cat_voip) \
STACK:cat_vpn#378035:"Virtual-Private-Network" \
$(rrdtool_graph_print_cur_min_max_avg cat_vpn) \
STACK:cat_web#00fffb:"Web...................." \
$(rrdtool_graph_print_cur_min_max_avg cat_web) \
STACK:cat_banned#ff1010:"Banned-Sites..........." \
$(rrdtool_graph_print_cur_min_max_avg cat_banned) \
STACK:cat_unavail#ff1010:"Sites-Unavailable......" \
$(rrdtool_graph_print_cur_min_max_avg cat_unavail) \
STACK:cat_allowed#ff1010:"Allowed-Sites.........." \
$(rrdtool_graph_print_cur_min_max_avg cat_allowed) \
STACK:cat_antimal#ff1010:"Antimalware............" \
$(rrdtool_graph_print_cur_min_max_avg cat_antimal) \
STACK:cat_crypto#afaf00:"Crypto-Currency........" \
$(rrdtool_graph_print_cur_min_max_avg cat_crypto)
rrdtool_graph JSON 'Lines' "${OUTDIR}/json_lines" \
DEF:json_lines=${RRDDIR}/counter-json_lines.rrd:value:AVERAGE \
$(rrdtool_graph_colorize_missing_data json_lines) \
AREA:json_lines#4dff4d::STACK \
LINE2:json_lines#00e600:"JSON-lines" \
$(rrdtool_graph_print_cur_min_max_avg json_lines)
rrdtool_graph JSON 'Bytes' "${OUTDIR}/json_bytes" \
DEF:json_bytes=${RRDDIR}/counter-json_bytes.rrd:value:AVERAGE \
$(rrdtool_graph_colorize_missing_data json_bytes) \
AREA:json_bytes#4dff4d::STACK \
LINE2:json_bytes#00e600:"JSON-bytes" \
$(rrdtool_graph_print_cur_min_max_avg json_bytes)
rrdtool_graph Events 'Amount' "${OUTDIR}/events" \
DEF:init=${RRDDIR}/counter-init_count.rrd:value:AVERAGE \
DEF:reconnect=${RRDDIR}/counter-reconnect_count.rrd:value:AVERAGE \
DEF:shutdown=${RRDDIR}/counter-shutdown_count.rrd:value:AVERAGE \
DEF:status=${RRDDIR}/counter-status_count.rrd:value:AVERAGE \
DEF:packet=${RRDDIR}/counter-packet_count.rrd:value:AVERAGE \
DEF:packet_flow=${RRDDIR}/counter-packet_flow_count.rrd:value:AVERAGE \
DEF:new=${RRDDIR}/counter-flow_new_count.rrd:value:AVERAGE \
DEF:ewd=${RRDDIR}/counter-flow_end_count.rrd:value:AVERAGE \
DEF:idle=${RRDDIR}/counter-flow_idle_count.rrd:value:AVERAGE \
DEF:update=${RRDDIR}/counter-flow_update_count.rrd:value:AVERAGE \
DEF:detection_update=${RRDDIR}/counter-flow_detection_update_count.rrd:value:AVERAGE \
DEF:guessed=${RRDDIR}/counter-flow_guessed_count.rrd:value:AVERAGE \
DEF:detected=${RRDDIR}/counter-flow_detected_count.rrd:value:AVERAGE \
DEF:not_detected=${RRDDIR}/counter-flow_not_detected_count.rrd:value:AVERAGE \
DEF:analyse=${RRDDIR}/counter-flow_analyse_count.rrd:value:AVERAGE \
$(rrdtool_graph_colorize_missing_data init) \
AREA:init#f1c232:"Init..................." \
$(rrdtool_graph_print_cur_min_max_avg init) \
STACK:reconnect#63bad9:"Reconnect.............." \
$(rrdtool_graph_print_cur_min_max_avg reconnect) \
STACK:shutdown#3a6f82:"Shutdown..............." \
$(rrdtool_graph_print_cur_min_max_avg shutdown) \
STACK:status#b7cbd1:"Status................." \
$(rrdtool_graph_print_cur_min_max_avg status) \
STACK:packet#0aff3f:"Packet................." \
$(rrdtool_graph_print_cur_min_max_avg packet) \
STACK:packet_flow#00c72b:"Packet-Flow............" \
$(rrdtool_graph_print_cur_min_max_avg packet_flow) \
STACK:new#c76700:"New...................." \
$(rrdtool_graph_print_cur_min_max_avg new) \
STACK:ewd#c78500:"End...................." \
$(rrdtool_graph_print_cur_min_max_avg ewd) \
STACK:idle#c7a900:"Idle..................." \
$(rrdtool_graph_print_cur_min_max_avg idle) \
STACK:update#c7c400:"Updates................" \
$(rrdtool_graph_print_cur_min_max_avg update) \
STACK:detection_update#a2c700:"Detection-Updates......" \
$(rrdtool_graph_print_cur_min_max_avg detection_update) \
STACK:guessed#7bc700:"Guessed................" \
$(rrdtool_graph_print_cur_min_max_avg guessed) \
STACK:detected#00c781:"Detected..............." \
$(rrdtool_graph_print_cur_min_max_avg detected) \
STACK:not_detected#00bdc7:"Not-Detected..........." \
$(rrdtool_graph_print_cur_min_max_avg not_detected) \
STACK:analyse#1400c7:"Analyse................" \
$(rrdtool_graph_print_cur_min_max_avg analyse)
rrdtool_graph Error-Events 'Amount' "${OUTDIR}/error_events" \
DEF:error_0=${RRDDIR}/counter-error_unknown_datalink.rrd:value:AVERAGE \
DEF:error_1=${RRDDIR}/counter-error_unknown_l3_protocol.rrd:value:AVERAGE \
DEF:error_2=${RRDDIR}/counter-error_unsupported_datalink.rrd:value:AVERAGE \
DEF:error_3=${RRDDIR}/counter-error_packet_too_short.rrd:value:AVERAGE \
DEF:error_4=${RRDDIR}/counter-error_packet_type_unknown.rrd:value:AVERAGE \
DEF:error_5=${RRDDIR}/counter-error_packet_header_invalid.rrd:value:AVERAGE \
DEF:error_6=${RRDDIR}/counter-error_ip4_packet_too_short.rrd:value:AVERAGE \
DEF:error_7=${RRDDIR}/counter-error_ip4_size_smaller_than_header.rrd:value:AVERAGE \
DEF:error_8=${RRDDIR}/counter-error_ip4_l4_payload_detection.rrd:value:AVERAGE \
DEF:error_9=${RRDDIR}/counter-error_ip6_packet_too_short.rrd:value:AVERAGE \
DEF:error_10=${RRDDIR}/counter-error_ip6_size_smaller_than_header.rrd:value:AVERAGE \
DEF:error_11=${RRDDIR}/counter-error_ip6_l4_payload_detection.rrd:value:AVERAGE \
DEF:error_12=${RRDDIR}/counter-error_tcp_packet_too_short.rrd:value:AVERAGE \
DEF:error_13=${RRDDIR}/counter-error_udp_packet_too_short.rrd:value:AVERAGE \
DEF:error_14=${RRDDIR}/counter-error_capture_size_smaller_than_packet.rrd:value:AVERAGE \
DEF:error_15=${RRDDIR}/counter-error_max_flows_to_track.rrd:value:AVERAGE \
DEF:error_16=${RRDDIR}/counter-error_flow_memory_alloc.rrd:value:AVERAGE \
$(rrdtool_graph_colorize_missing_data error_0) \
AREA:error_0#ff6a00:"Unknown-datalink-layer-packet............................" \
$(rrdtool_graph_print_cur_min_max_avg error_0) \
STACK:error_1#bf7540:"Unknown-L3-protocol......................................" \
$(rrdtool_graph_print_cur_min_max_avg error_1) \
STACK:error_2#ffd500:"Unsupported-datalink-layer..............................." \
$(rrdtool_graph_print_cur_min_max_avg error_2) \
STACK:error_3#bfaa40:"Packet-too-short........................................." \
$(rrdtool_graph_print_cur_min_max_avg error_3) \
STACK:error_4#bfff00:"Unknown-packet-type......................................" \
$(rrdtool_graph_print_cur_min_max_avg error_4) \
STACK:error_5#9fbf40:"Packet-header-invalid...................................." \
$(rrdtool_graph_print_cur_min_max_avg error_5) \
STACK:error_6#55ff00:"IP4-packet-too-short....................................." \
$(rrdtool_graph_print_cur_min_max_avg error_6) \
STACK:error_7#6abf40:"Packet-smaller-than-IP4-header..........................." \
$(rrdtool_graph_print_cur_min_max_avg error_7) \
STACK:error_8#00ff15:"nDPI-IPv4/L4-payload-detection-failed...................." \
$(rrdtool_graph_print_cur_min_max_avg error_8) \
STACK:error_9#40bf4a:"IP6-packet-too-short....................................." \
$(rrdtool_graph_print_cur_min_max_avg error_9) \
STACK:error_10#00ff80:"Packet-smaller-than-IP6-header..........................." \
$(rrdtool_graph_print_cur_min_max_avg error_10) \
STACK:error_11#40bf80:"nDPI-IPv6/L4-payload-detection-failed...................." \
$(rrdtool_graph_print_cur_min_max_avg error_11) \
STACK:error_12#00ffea:"TCP-packet-smaller-than-expected........................." \
$(rrdtool_graph_print_cur_min_max_avg error_12) \
STACK:error_13#40bfb5:"UDP-packet-smaller-than-expected........................." \
$(rrdtool_graph_print_cur_min_max_avg error_13) \
STACK:error_14#00aaff:"Captured-packet-size-is-smaller-than-expected-packet-size" \
$(rrdtool_graph_print_cur_min_max_avg error_14) \
STACK:error_15#4095bf:"Max-flows-to-track-reached..............................." \
$(rrdtool_graph_print_cur_min_max_avg error_15) \
STACK:error_16#0040ff:"Flow-memory-allocation-failed............................" \
$(rrdtool_graph_print_cur_min_max_avg error_16)
rrdtool_graph Risk-Severites Amount "${OUTDIR}/severities" \
DEF:sever_crit=${RRDDIR}/gauge-flow_severity_critical.rrd:value:AVERAGE \
DEF:sever_emer=${RRDDIR}/gauge-flow_severity_emergency.rrd:value:AVERAGE \
DEF:sever_high=${RRDDIR}/gauge-flow_severity_high.rrd:value:AVERAGE \
DEF:sever_low=${RRDDIR}/gauge-flow_severity_low.rrd:value:AVERAGE \
DEF:sever_med=${RRDDIR}/gauge-flow_severity_medium.rrd:value:AVERAGE \
DEF:sever_sev=${RRDDIR}/gauge-flow_severity_severe.rrd:value:AVERAGE \
DEF:sever_ukn=${RRDDIR}/gauge-flow_severity_unknown.rrd:value:AVERAGE \
$(rrdtool_graph_colorize_missing_data sever_crit) \
AREA:sever_crit#e68e00::STACK \
AREA:sever_emer#e63200::STACK \
AREA:sever_high#e6c700::STACK \
AREA:sever_low#00e600::STACK \
AREA:sever_med#8fce00::STACK \
AREA:sever_sev#c61b2b::STACK \
AREA:sever_ukn#7e8877::STACK \
LINE2:sever_crit#e68e00:"Critical............." \
$(rrdtool_graph_print_cur_min_max_avg sever_crit) \
LINE2:sever_emer#e63200:"Emergency............" \
$(rrdtool_graph_print_cur_min_max_avg sever_emer) \
LINE2:sever_high#e6c700:"High................." \
$(rrdtool_graph_print_cur_min_max_avg sever_high) \
LINE2:sever_low#00e600:"Low.................." \
$(rrdtool_graph_print_cur_min_max_avg sever_low) \
LINE2:sever_med#8fce00:"Medium..............." \
$(rrdtool_graph_print_cur_min_max_avg sever_med) \
LINE2:sever_sev#c61b2b:"Severe..............." \
$(rrdtool_graph_print_cur_min_max_avg sever_sev) \
LINE2:sever_ukn#7e8877:"Unknown.............." \
$(rrdtool_graph_print_cur_min_max_avg sever_ukn)
rrdtool_graph Risks 'Amount' "${OUTDIR}/risky_events" \
DEF:risk_1=${RRDDIR}/gauge-flow_risk_1_count.rrd:value:AVERAGE \
DEF:risk_2=${RRDDIR}/gauge-flow_risk_2_count.rrd:value:AVERAGE \
DEF:risk_3=${RRDDIR}/gauge-flow_risk_3_count.rrd:value:AVERAGE \
DEF:risk_4=${RRDDIR}/gauge-flow_risk_4_count.rrd:value:AVERAGE \
DEF:risk_5=${RRDDIR}/gauge-flow_risk_5_count.rrd:value:AVERAGE \
DEF:risk_6=${RRDDIR}/gauge-flow_risk_6_count.rrd:value:AVERAGE \
DEF:risk_7=${RRDDIR}/gauge-flow_risk_7_count.rrd:value:AVERAGE \
DEF:risk_8=${RRDDIR}/gauge-flow_risk_8_count.rrd:value:AVERAGE \
DEF:risk_9=${RRDDIR}/gauge-flow_risk_9_count.rrd:value:AVERAGE \
DEF:risk_10=${RRDDIR}/gauge-flow_risk_10_count.rrd:value:AVERAGE \
DEF:risk_11=${RRDDIR}/gauge-flow_risk_11_count.rrd:value:AVERAGE \
DEF:risk_12=${RRDDIR}/gauge-flow_risk_12_count.rrd:value:AVERAGE \
DEF:risk_13=${RRDDIR}/gauge-flow_risk_13_count.rrd:value:AVERAGE \
DEF:risk_14=${RRDDIR}/gauge-flow_risk_14_count.rrd:value:AVERAGE \
DEF:risk_15=${RRDDIR}/gauge-flow_risk_15_count.rrd:value:AVERAGE \
DEF:risk_16=${RRDDIR}/gauge-flow_risk_16_count.rrd:value:AVERAGE \
DEF:risk_17=${RRDDIR}/gauge-flow_risk_17_count.rrd:value:AVERAGE \
DEF:risk_18=${RRDDIR}/gauge-flow_risk_18_count.rrd:value:AVERAGE \
DEF:risk_19=${RRDDIR}/gauge-flow_risk_19_count.rrd:value:AVERAGE \
DEF:risk_20=${RRDDIR}/gauge-flow_risk_20_count.rrd:value:AVERAGE \
DEF:risk_21=${RRDDIR}/gauge-flow_risk_21_count.rrd:value:AVERAGE \
DEF:risk_22=${RRDDIR}/gauge-flow_risk_22_count.rrd:value:AVERAGE \
DEF:risk_23=${RRDDIR}/gauge-flow_risk_23_count.rrd:value:AVERAGE \
DEF:risk_24=${RRDDIR}/gauge-flow_risk_24_count.rrd:value:AVERAGE \
DEF:risk_25=${RRDDIR}/gauge-flow_risk_25_count.rrd:value:AVERAGE \
DEF:risk_26=${RRDDIR}/gauge-flow_risk_26_count.rrd:value:AVERAGE \
DEF:risk_27=${RRDDIR}/gauge-flow_risk_27_count.rrd:value:AVERAGE \
DEF:risk_28=${RRDDIR}/gauge-flow_risk_28_count.rrd:value:AVERAGE \
DEF:risk_29=${RRDDIR}/gauge-flow_risk_29_count.rrd:value:AVERAGE \
DEF:risk_30=${RRDDIR}/gauge-flow_risk_30_count.rrd:value:AVERAGE \
DEF:risk_31=${RRDDIR}/gauge-flow_risk_31_count.rrd:value:AVERAGE \
DEF:risk_32=${RRDDIR}/gauge-flow_risk_32_count.rrd:value:AVERAGE \
DEF:risk_33=${RRDDIR}/gauge-flow_risk_33_count.rrd:value:AVERAGE \
DEF:risk_34=${RRDDIR}/gauge-flow_risk_34_count.rrd:value:AVERAGE \
DEF:risk_35=${RRDDIR}/gauge-flow_risk_35_count.rrd:value:AVERAGE \
DEF:risk_36=${RRDDIR}/gauge-flow_risk_36_count.rrd:value:AVERAGE \
DEF:risk_37=${RRDDIR}/gauge-flow_risk_37_count.rrd:value:AVERAGE \
DEF:risk_38=${RRDDIR}/gauge-flow_risk_38_count.rrd:value:AVERAGE \
DEF:risk_39=${RRDDIR}/gauge-flow_risk_39_count.rrd:value:AVERAGE \
DEF:risk_40=${RRDDIR}/gauge-flow_risk_40_count.rrd:value:AVERAGE \
DEF:risk_41=${RRDDIR}/gauge-flow_risk_41_count.rrd:value:AVERAGE \
DEF:risk_42=${RRDDIR}/gauge-flow_risk_42_count.rrd:value:AVERAGE \
DEF:risk_43=${RRDDIR}/gauge-flow_risk_43_count.rrd:value:AVERAGE \
DEF:risk_44=${RRDDIR}/gauge-flow_risk_44_count.rrd:value:AVERAGE \
DEF:risk_45=${RRDDIR}/gauge-flow_risk_45_count.rrd:value:AVERAGE \
DEF:risk_46=${RRDDIR}/gauge-flow_risk_46_count.rrd:value:AVERAGE \
DEF:risk_47=${RRDDIR}/gauge-flow_risk_47_count.rrd:value:AVERAGE \
DEF:risk_48=${RRDDIR}/gauge-flow_risk_48_count.rrd:value:AVERAGE \
DEF:risk_49=${RRDDIR}/gauge-flow_risk_49_count.rrd:value:AVERAGE \
DEF:risk_50=${RRDDIR}/gauge-flow_risk_50_count.rrd:value:AVERAGE \
DEF:risk_51=${RRDDIR}/gauge-flow_risk_51_count.rrd:value:AVERAGE \
DEF:risk_52=${RRDDIR}/gauge-flow_risk_52_count.rrd:value:AVERAGE \
DEF:risk_53=${RRDDIR}/gauge-flow_risk_53_count.rrd:value:AVERAGE \
DEF:risk_unknown=${RRDDIR}/gauge-flow_risk_unknown_count.rrd:value:AVERAGE \
$(rrdtool_graph_colorize_missing_data risk_1) \
AREA:risk_1#ff0000:"XSS-Attack..............................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_1) \
STACK:risk_2#ff5500:"SQL-Injection............................................" \
$(rrdtool_graph_print_cur_min_max_avg risk_2) \
STACK:risk_3#ffaa00:"RCE-Injection............................................" \
$(rrdtool_graph_print_cur_min_max_avg risk_3) \
STACK:risk_4#ffff00:"Binary-App-Transfer......................................" \
$(rrdtool_graph_print_cur_min_max_avg risk_4) \
STACK:risk_5#aaff00:"Known-Proto-on-Non-Std-Port.............................." \
$(rrdtool_graph_print_cur_min_max_avg risk_5) \
STACK:risk_6#55ff00:"Self-signed-Cert........................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_6) \
STACK:risk_7#00ff55:"Obsolete-TLS-v1.1-or-older..............................." \
$(rrdtool_graph_print_cur_min_max_avg risk_7) \
STACK:risk_8#00ffaa:"Weak-TLS-Cipher.........................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_8) \
STACK:risk_9#00ffff:"TLS-Cert-Expired........................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_9) \
STACK:risk_10#00aaff:"TLS-Cert-Mismatch........................................" \
$(rrdtool_graph_print_cur_min_max_avg risk_10) \
STACK:risk_11#0055ff:"HTTP-Suspicious-User-Agent..............................." \
$(rrdtool_graph_print_cur_min_max_avg risk_11) \
STACK:risk_12#0000ff:"HTTP-Numeric-IP-Address.................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_12) \
STACK:risk_13#5500ff:"HTTP-Suspicious-URL......................................" \
$(rrdtool_graph_print_cur_min_max_avg risk_13) \
STACK:risk_14#aa00ff:"HTTP-Suspicious-Header..................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_14) \
STACK:risk_15#ff00ff:"TLS-probably-Not-Carrying-HTTPS.........................." \
$(rrdtool_graph_print_cur_min_max_avg risk_15) \
STACK:risk_16#ff00aa:"Suspicious-DGA-Domain-name..............................." \
$(rrdtool_graph_print_cur_min_max_avg risk_16) \
STACK:risk_17#ff0055:"Malformed-Packet........................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_17) \
STACK:risk_18#602020:"SSH-Obsolete-Client-Version/Cipher......................." \
$(rrdtool_graph_print_cur_min_max_avg risk_18) \
STACK:risk_19#603a20:"SSH-Obsolete-Server-Version/Cipher......................." \
$(rrdtool_graph_print_cur_min_max_avg risk_19) \
STACK:risk_20#605520:"SMB-Insecure-Version....................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_20) \
STACK:risk_21#506020:"TLS-Suspicious-ESNI-Usage................................" \
$(rrdtool_graph_print_cur_min_max_avg risk_21) \
STACK:risk_22#356020:"Unsafe-Protocol.........................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_22) \
STACK:risk_23#206025:"Suspicious-DNS-Traffic..................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_23) \
STACK:risk_24#206040:"Missing-SNI-TLS-Extension................................" \
$(rrdtool_graph_print_cur_min_max_avg risk_24) \
STACK:risk_25#20605a:"HTTP-Suspicious-Content.................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_25) \
STACK:risk_26#204a60:"Risky-ASN................................................" \
$(rrdtool_graph_print_cur_min_max_avg risk_26) \
STACK:risk_27#203060:"Risky-Domain-Name........................................" \
$(rrdtool_graph_print_cur_min_max_avg risk_27) \
STACK:risk_28#2a2060:"Malicious-JA3-Fingerprint................................" \
$(rrdtool_graph_print_cur_min_max_avg risk_28) \
STACK:risk_29#452060:"Malicious-SSL-Cert/SHA1-Fingerprint......................" \
$(rrdtool_graph_print_cur_min_max_avg risk_29) \
STACK:risk_30#602060:"Desktop/File-Sharing....................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_30) \
STACK:risk_31#602045:"Uncommon-TLS-ALPN........................................" \
$(rrdtool_graph_print_cur_min_max_avg risk_31) \
STACK:risk_32#df2020:"TLS-Cert-Validity-Too-Long..............................." \
$(rrdtool_graph_print_cur_min_max_avg risk_32) \
STACK:risk_33#df6020:"TLS-Suspicious-Extension................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_33) \
STACK:risk_34#df9f20:"TLS-Fatal-Alert.........................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_34) \
STACK:risk_35#dfdf20:"Suspicious-Entropy......................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_35) \
STACK:risk_36#9fdf20:"Clear-Text-Credentials..................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_36) \
STACK:risk_37#60df20:"Large-DNS-Packet........................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_37) \
STACK:risk_38#20df20:"Fragmented-DNS-Message..................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_38) \
STACK:risk_39#20df60:"Text-With-Non-Printable-Chars............................" \
$(rrdtool_graph_print_cur_min_max_avg risk_39) \
STACK:risk_40#20df9f:"Possible-Exploit........................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_40) \
STACK:risk_41#20dfdf:"TLS-Cert-About-To-Expire................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_41) \
STACK:risk_42#209fdf:"IDN-Domain-Name.........................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_42) \
STACK:risk_43#2060df:"Error-Code..............................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_43) \
STACK:risk_44#2020df:"Crawler/Bot.............................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_44) \
STACK:risk_45#6020df:"Anonymous-Subscriber....................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_45) \
STACK:risk_46#9f20df:"Unidirectional-Traffic..................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_46) \
STACK:risk_47#df20df:"HTTP-Obsolete-Server....................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_47) \
STACK:risk_48#df68df:"Periodic-Flow............................................" \
$(rrdtool_graph_print_cur_min_max_avg risk_48) \
STACK:risk_49#dfffdf:"Minor-Issues............................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_49) \
STACK:risk_50#ef20df:"TCP-Connection-Issues...................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_50) \
STACK:risk_51#ef60df:"Fully-Encrypted.........................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_51) \
STACK:risk_52#efa0df:"Invalid-ALPN/SNI-combination............................." \
$(rrdtool_graph_print_cur_min_max_avg risk_52) \
STACK:risk_53#efffdf:"Malware-Host-Contacted..................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_53) \
STACK:risk_unknown#df2060:"Unknown.................................................." \
$(rrdtool_graph_print_cur_min_max_avg risk_unknown)

7
examples/c-collectd/www/dpi/bootstrap.css vendored
View File

File diff suppressed because one or more lines are too long

7
examples/c-collectd/www/dpi/bootstrap.js vendored
View File

File diff suppressed because one or more lines are too long

198
examples/c-collectd/www/dpi/categories.html
View File

@@ -1,198 +0,0 @@
<!DOCTYPE html>
<html lang="en"><head>
<meta http-equiv="cache-control" content="max-age=0" />
<meta http-equiv="cache-control" content="no-cache" />
<meta http-equiv="expires" content="0" />
<meta http-equiv="expires" content="Tue, 01 Jan 1980 1:00:00 GMT" />
<meta http-equiv="pragma" content="no-cache" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<meta name="description" content="nDPId RRD Graph">
<meta name="author" content="Toni Uhlig">
<link rel="icon" href="https://getbootstrap.com/docs/4.0/assets/img/favicons/favicon.ico">
<title>nDPId Dashboard</title>
<link rel="canonical" href="https://getbootstrap.com/docs/4.0/examples/dashboard/">
<!-- Bootstrap core CSS -->
<link href="bootstrap.css" rel="stylesheet">
<!-- Custom styles for this template -->
<link href="dashboard.css" rel="stylesheet">
</head>
<body>
<nav class="navbar navbar-dark sticky-top bg-dark flex-md-nowrap p-0">
<a class="navbar-brand col-sm-3 col-md-2 mr-0" href="https://github.com/utoni/nDPId">nDPId Collectd RRD Graph</a>
</nav>
<div class="container-fluid">
<div class="row">
<nav class="col-md-2 d-none d-md-block bg-light sidebar">
<div class="sidebar-sticky">
<h6 class="sidebar-heading d-flex justify-content-between align-items-center px-3 mt-4 mb-1 text-muted">
<span>Graphs</span>
</h6>
<ul class="nav flex-column mb-2">
<li class="nav-item">
<a class="nav-link" href="index.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Home
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="flows.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line><polyline points="10 9 9 9 8 9"></polyline>
</svg>
Flows
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="other.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Other
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="detections.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Detections
</a>
</li>
<li class="nav-item">
<a class="nav-link active" href="categories.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Categories
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="risks.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Risks
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="jsons.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
JSONs
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="events.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Events
</a>
</li>
</ul>
</div>
</nav>
<main role="main" class="col-md-9 ml-sm-auto col-lg-10 pt-3 px-4">
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="breed_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="breed_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="breed_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="breed_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="breed_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="breed_past_year.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="categories_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="categories_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="categories_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="categories_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="categories_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="categories_past_year.png" class="img-fluid" alt="Responsive image">
</div>
</main>
</div>
</div>
<!-- Bootstrap core JavaScript
================================================== -->
<!-- Placed at the end of the document so the pages load faster -->
<script src="jquery-3.js" integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN" crossorigin="anonymous"></script>
<script>window.jQuery || document.write('<script src="../../assets/js/vendor/jquery-slim.min.js"><\/script>')</script>
<script src="popper.js"></script>
<script src="bootstrap.js"></script>
<!-- Icons -->
<script src="feather.js"></script>
<script>
feather.replace()
</script>
</body></html>

93
examples/c-collectd/www/dpi/dashboard.css
View File

@@ -1,93 +0,0 @@
body {
font-size: .875rem;
}
.feather {
width: 16px;
height: 16px;
vertical-align: text-bottom;
}
/*
* Sidebar
*/
.sidebar {
position: fixed;
top: 0;
bottom: 0;
left: 0;
z-index: 100; /* Behind the navbar */
padding: 0;
box-shadow: inset -1px 0 0 rgba(0, 0, 0, .1);
}
.sidebar-sticky {
position: -webkit-sticky;
position: sticky;
top: 48px; /* Height of navbar */
height: calc(100vh - 48px);
padding-top: .5rem;
overflow-x: hidden;
overflow-y: auto; /* Scrollable contents if viewport is shorter than content. */
}
.sidebar .nav-link {
font-weight: 500;
color: #333;
}
.sidebar .nav-link .feather {
margin-right: 4px;
color: #999;
}
.sidebar .nav-link.active {
color: #007bff;
}
.sidebar .nav-link:hover .feather,
.sidebar .nav-link.active .feather {
color: inherit;
}
.sidebar-heading {
font-size: .75rem;
text-transform: uppercase;
}
/*
* Navbar
*/
.navbar-brand {
padding-top: .75rem;
padding-bottom: .75rem;
font-size: 1rem;
background-color: rgba(0, 0, 0, .25);
box-shadow: inset -1px 0 0 rgba(0, 0, 0, .25);
}
.navbar .form-control {
padding: .75rem 1rem;
border-width: 0;
border-radius: 0;
}
.form-control-dark {
color: #fff;
background-color: rgba(255, 255, 255, .1);
border-color: rgba(255, 255, 255, .1);
}
.form-control-dark:focus {
border-color: transparent;
box-shadow: 0 0 0 3px rgba(255, 255, 255, .25);
}
/*
* Utilities
*/
.border-top { border-top: 1px solid #e5e5e5; }
.border-bottom { border-bottom: 1px solid #e5e5e5; }

198
examples/c-collectd/www/dpi/detections.html
View File

@@ -1,198 +0,0 @@
<!DOCTYPE html>
<html lang="en"><head>
<meta http-equiv="cache-control" content="max-age=0" />
<meta http-equiv="cache-control" content="no-cache" />
<meta http-equiv="expires" content="0" />
<meta http-equiv="expires" content="Tue, 01 Jan 1980 1:00:00 GMT" />
<meta http-equiv="pragma" content="no-cache" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<meta name="description" content="nDPId RRD Graph">
<meta name="author" content="Toni Uhlig">
<link rel="icon" href="https://getbootstrap.com/docs/4.0/assets/img/favicons/favicon.ico">
<title>nDPId Dashboard</title>
<link rel="canonical" href="https://getbootstrap.com/docs/4.0/examples/dashboard/">
<!-- Bootstrap core CSS -->
<link href="bootstrap.css" rel="stylesheet">
<!-- Custom styles for this template -->
<link href="dashboard.css" rel="stylesheet">
</head>
<body>
<nav class="navbar navbar-dark sticky-top bg-dark flex-md-nowrap p-0">
<a class="navbar-brand col-sm-3 col-md-2 mr-0" href="https://github.com/utoni/nDPId">nDPId Collectd RRD Graph</a>
</nav>
<div class="container-fluid">
<div class="row">
<nav class="col-md-2 d-none d-md-block bg-light sidebar">
<div class="sidebar-sticky">
<h6 class="sidebar-heading d-flex justify-content-between align-items-center px-3 mt-4 mb-1 text-muted">
<span>Graphs</span>
</h6>
<ul class="nav flex-column mb-2">
<li class="nav-item">
<a class="nav-link" href="index.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Home
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="flows.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line><polyline points="10 9 9 9 8 9"></polyline>
</svg>
Flows
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="other.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Other
</a>
</li>
<li class="nav-item">
<a class="nav-link active" href="detections.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Detections
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="categories.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Categories
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="risks.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Risks
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="jsons.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
JSONs
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="events.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Events
</a>
</li>
</ul>
</div>
</nav>
<main role="main" class="col-md-9 ml-sm-auto col-lg-10 pt-3 px-4">
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="detections_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="detections_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="detections_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="detections_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="detections_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="detections_past_year.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="confidence_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="confidence_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="confidence_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="confidence_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="confidence_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="confidence_past_year.png" class="img-fluid" alt="Responsive image">
</div>
</main>
</div>
</div>
<!-- Bootstrap core JavaScript
================================================== -->
<!-- Placed at the end of the document so the pages load faster -->
<script src="jquery-3.js" integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN" crossorigin="anonymous"></script>
<script>window.jQuery || document.write('<script src="../../assets/js/vendor/jquery-slim.min.js"><\/script>')</script>
<script src="popper.js"></script>
<script src="bootstrap.js"></script>
<!-- Icons -->
<script src="feather.js"></script>
<script>
feather.replace()
</script>
</body></html>

198
examples/c-collectd/www/dpi/events.html
View File

@@ -1,198 +0,0 @@
<!DOCTYPE html>
<html lang="en"><head>
<meta http-equiv="cache-control" content="max-age=0" />
<meta http-equiv="cache-control" content="no-cache" />
<meta http-equiv="expires" content="0" />
<meta http-equiv="expires" content="Tue, 01 Jan 1980 1:00:00 GMT" />
<meta http-equiv="pragma" content="no-cache" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<meta name="description" content="nDPId RRD Graph">
<meta name="author" content="Toni Uhlig">
<link rel="icon" href="https://getbootstrap.com/docs/4.0/assets/img/favicons/favicon.ico">
<title>nDPId Dashboard</title>
<link rel="canonical" href="https://getbootstrap.com/docs/4.0/examples/dashboard/">
<!-- Bootstrap core CSS -->
<link href="bootstrap.css" rel="stylesheet">
<!-- Custom styles for this template -->
<link href="dashboard.css" rel="stylesheet">
</head>
<body>
<nav class="navbar navbar-dark sticky-top bg-dark flex-md-nowrap p-0">
<a class="navbar-brand col-sm-3 col-md-2 mr-0" href="https://github.com/utoni/nDPId">nDPId Collectd RRD Graph</a>
</nav>
<div class="container-fluid">
<div class="row">
<nav class="col-md-2 d-none d-md-block bg-light sidebar">
<div class="sidebar-sticky">
<h6 class="sidebar-heading d-flex justify-content-between align-items-center px-3 mt-4 mb-1 text-muted">
<span>Graphs</span>
</h6>
<ul class="nav flex-column mb-2">
<li class="nav-item">
<a class="nav-link" href="index.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Home
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="flows.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line><polyline points="10 9 9 9 8 9"></polyline>
</svg>
Flows
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="other.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Other
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="detections.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Detections
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="categories.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Categories
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="risks.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Risks
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="jsons.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
JSONs
</a>
</li>
<li class="nav-item">
<a class="nav-link active" href="events.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Events
</a>
</li>
</ul>
</div>
</nav>
<main role="main" class="col-md-9 ml-sm-auto col-lg-10 pt-3 px-4">
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="events_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="events_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="events_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="events_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="events_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="events_past_year.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="error_events_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="error_events_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="error_events_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="error_events_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="error_events_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="error_events_past_year.png" class="img-fluid" alt="Responsive image">
</div>
</main>
</div>
</div>
<!-- Bootstrap core JavaScript
================================================== -->
<!-- Placed at the end of the document so the pages load faster -->
<script src="jquery-3.js" integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN" crossorigin="anonymous"></script>
<script>window.jQuery || document.write('<script src="../../assets/js/vendor/jquery-slim.min.js"><\/script>')</script>
<script src="popper.js"></script>
<script src="bootstrap.js"></script>
<!-- Icons -->
<script src="feather.js"></script>
<script>
feather.replace()
</script>
</body></html>

13
examples/c-collectd/www/dpi/feather.js
View File

File diff suppressed because one or more lines are too long

179
examples/c-collectd/www/dpi/flows.html
View File

@@ -1,179 +0,0 @@
<!DOCTYPE html>
<html lang="en"><head>
<meta http-equiv="cache-control" content="max-age=0" />
<meta http-equiv="cache-control" content="no-cache" />
<meta http-equiv="expires" content="0" />
<meta http-equiv="expires" content="Tue, 01 Jan 1980 1:00:00 GMT" />
<meta http-equiv="pragma" content="no-cache" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<meta name="description" content="nDPId RRD Graph">
<meta name="author" content="Toni Uhlig">
<link rel="icon" href="https://getbootstrap.com/docs/4.0/assets/img/favicons/favicon.ico">
<title>nDPId Dashboard</title>
<link rel="canonical" href="https://getbootstrap.com/docs/4.0/examples/dashboard/">
<!-- Bootstrap core CSS -->
<link href="bootstrap.css" rel="stylesheet">
<!-- Custom styles for this template -->
<link href="dashboard.css" rel="stylesheet">
</head>
<body>
<nav class="navbar navbar-dark sticky-top bg-dark flex-md-nowrap p-0">
<a class="navbar-brand col-sm-3 col-md-2 mr-0" href="https://github.com/utoni/nDPId">nDPId Collectd RRD Graph</a>
</nav>
<div class="container-fluid">
<div class="row">
<nav class="col-md-2 d-none d-md-block bg-light sidebar">
<div class="sidebar-sticky">
<h6 class="sidebar-heading d-flex justify-content-between align-items-center px-3 mt-4 mb-1 text-muted">
<span>Graphs</span>
</h6>
<ul class="nav flex-column mb-2">
<li class="nav-item">
<a class="nav-link" href="index.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Home
</a>
</li>
<li class="nav-item">
<a class="nav-link active" href="flows.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line><polyline points="10 9 9 9 8 9"></polyline>
</svg>
Flows
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="other.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Other
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="detections.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Detections
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="categories.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Categories
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="risks.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Risks
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="jsons.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
JSONs
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="events.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Events
</a>
</li>
</ul>
</div>
</nav>
<main role="main" class="col-md-9 ml-sm-auto col-lg-10 pt-3 px-4">
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="flows_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="flows_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="flows_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="flows_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="flows_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="flows_past_year.png" class="img-fluid" alt="Responsive image">
</div>
</main>
</div>
</div>
<!-- Bootstrap core JavaScript
================================================== -->
<!-- Placed at the end of the document so the pages load faster -->
<script src="jquery-3.js" integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN" crossorigin="anonymous"></script>
<script>window.jQuery || document.write('<script src="../../assets/js/vendor/jquery-slim.min.js"><\/script>')</script>
<script src="popper.js"></script>
<script src="bootstrap.js"></script>
<!-- Icons -->
<script src="feather.js"></script>
<script>
feather.replace()
</script>
</body></html>

425
examples/c-collectd/www/dpi/index.html
View File

@@ -1,425 +0,0 @@
<!DOCTYPE html>
<html lang="en"><head>
<meta http-equiv="cache-control" content="max-age=0" />
<meta http-equiv="cache-control" content="no-cache" />
<meta http-equiv="expires" content="0" />
<meta http-equiv="expires" content="Tue, 01 Jan 1980 1:00:00 GMT" />
<meta http-equiv="pragma" content="no-cache" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<meta name="description" content="nDPId RRD Graph">
<meta name="author" content="Toni Uhlig">
<link rel="icon" href="https://getbootstrap.com/docs/4.0/assets/img/favicons/favicon.ico">
<title>nDPId Dashboard</title>
<link rel="canonical" href="https://getbootstrap.com/docs/4.0/examples/dashboard/">
<!-- Bootstrap core CSS -->
<link href="bootstrap.css" rel="stylesheet">
<!-- Custom styles for this template -->
<link href="dashboard.css" rel="stylesheet">
</head>
<body>
<nav class="navbar navbar-dark sticky-top bg-dark flex-md-nowrap p-0">
<a class="navbar-brand col-sm-3 col-md-2 mr-0" href="https://github.com/utoni/nDPId">nDPId Collectd RRD Graph</a>
</nav>
<div class="container-fluid">
<div class="row">
<nav class="col-md-2 d-none d-md-block bg-light sidebar">
<div class="sidebar-sticky">
<h6 class="sidebar-heading d-flex justify-content-between align-items-center px-3 mt-4 mb-1 text-muted">
<span>Graphs</span>
</h6>
<ul class="nav flex-column mb-2">
<li class="nav-item">
<a class="nav-link active" href="index.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Home
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="flows.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line><polyline points="10 9 9 9 8 9"></polyline>
</svg>
Flows
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="other.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Other
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="detections.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Detections
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="categories.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Categories
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="risks.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Risks
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="jsons.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
JSONs
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="events.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Events
</a>
</li>
</ul>
</div>
</nav>
<main role="main" class="col-md-9 ml-sm-auto col-lg-10 pt-3 px-4">
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="flows_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="flows_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="flows_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="flows_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="flows_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="flows_past_year.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="detections_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="detections_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="detections_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="detections_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="detections_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="detections_past_year.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="confidence_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="confidence_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="confidence_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="confidence_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="confidence_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="confidence_past_year.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="traffic_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="traffic_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="traffic_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="traffic_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="traffic_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="traffic_past_year.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer3_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer3_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer3_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer3_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer3_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer3_past_year.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer4_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer4_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer4_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer4_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer4_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer4_past_year.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="breed_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="breed_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="breed_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="breed_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="breed_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="breed_past_year.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="categories_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="categories_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="categories_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="categories_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="categories_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="categories_past_year.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="events_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="events_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="events_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="events_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="events_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="events_past_year.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="error_events_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="error_events_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="error_events_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="error_events_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="error_events_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="error_events_past_year.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_lines_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_lines_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_lines_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_lines_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_lines_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_lines_past_year.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_bytes_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_bytes_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_bytes_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_bytes_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_bytes_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_bytes_past_year.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="severities_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="severities_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="severities_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="severities_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="severities_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="severities_past_year.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="risky_events_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="risky_events_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="risky_events_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="risky_events_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="risky_events_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="risky_events_past_year.png" class="img-fluid" alt="Responsive image">
</div>
</main>
</div>
</div>
<!-- Bootstrap core JavaScript
================================================== -->
<!-- Placed at the end of the document so the pages load faster -->
<script src="jquery-3.js" integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN" crossorigin="anonymous"></script>
<script>window.jQuery || document.write('<script src="../../assets/js/vendor/jquery-slim.min.js"><\/script>')</script>
<script src="popper.js"></script>
<script src="bootstrap.js"></script>
<!-- Icons -->
<script src="feather.js"></script>
<script>
feather.replace()
</script>
</body></html>

4
examples/c-collectd/www/dpi/jquery-3.js vendored
View File

File diff suppressed because one or more lines are too long

198
examples/c-collectd/www/dpi/jsons.html
View File

@@ -1,198 +0,0 @@
<!DOCTYPE html>
<html lang="en"><head>
<meta http-equiv="cache-control" content="max-age=0" />
<meta http-equiv="cache-control" content="no-cache" />
<meta http-equiv="expires" content="0" />
<meta http-equiv="expires" content="Tue, 01 Jan 1980 1:00:00 GMT" />
<meta http-equiv="pragma" content="no-cache" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<meta name="description" content="nDPId RRD Graph">
<meta name="author" content="Toni Uhlig">
<link rel="icon" href="https://getbootstrap.com/docs/4.0/assets/img/favicons/favicon.ico">
<title>nDPId Dashboard</title>
<link rel="canonical" href="https://getbootstrap.com/docs/4.0/examples/dashboard/">
<!-- Bootstrap core CSS -->
<link href="bootstrap.css" rel="stylesheet">
<!-- Custom styles for this template -->
<link href="dashboard.css" rel="stylesheet">
</head>
<body>
<nav class="navbar navbar-dark sticky-top bg-dark flex-md-nowrap p-0">
<a class="navbar-brand col-sm-3 col-md-2 mr-0" href="https://github.com/utoni/nDPId">nDPId Collectd RRD Graph</a>
</nav>
<div class="container-fluid">
<div class="row">
<nav class="col-md-2 d-none d-md-block bg-light sidebar">
<div class="sidebar-sticky">
<h6 class="sidebar-heading d-flex justify-content-between align-items-center px-3 mt-4 mb-1 text-muted">
<span>Graphs</span>
</h6>
<ul class="nav flex-column mb-2">
<li class="nav-item">
<a class="nav-link" href="index.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Home
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="flows.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line><polyline points="10 9 9 9 8 9"></polyline>
</svg>
Flows
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="other.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Other
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="detections.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Detections
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="categories.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Categories
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="risks.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Risks
</a>
</li>
<li class="nav-item">
<a class="nav-link active" href="jsons.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
JSONs
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="events.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Events
</a>
</li>
</ul>
</div>
</nav>
<main role="main" class="col-md-9 ml-sm-auto col-lg-10 pt-3 px-4">
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_lines_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_lines_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_lines_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_lines_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_lines_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_lines_past_year.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_bytes_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_bytes_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_bytes_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_bytes_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_bytes_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="json_bytes_past_year.png" class="img-fluid" alt="Responsive image">
</div>
</main>
</div>
</div>
<!-- Bootstrap core JavaScript
================================================== -->
<!-- Placed at the end of the document so the pages load faster -->
<script src="jquery-3.js" integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN" crossorigin="anonymous"></script>
<script>window.jQuery || document.write('<script src="../../assets/js/vendor/jquery-slim.min.js"><\/script>')</script>
<script src="popper.js"></script>
<script src="bootstrap.js"></script>
<!-- Icons -->
<script src="feather.js"></script>
<script>
feather.replace()
</script>
</body></html>

217
examples/c-collectd/www/dpi/other.html
View File

@@ -1,217 +0,0 @@
<!DOCTYPE html>
<html lang="en"><head>
<meta http-equiv="cache-control" content="max-age=0" />
<meta http-equiv="cache-control" content="no-cache" />
<meta http-equiv="expires" content="0" />
<meta http-equiv="expires" content="Tue, 01 Jan 1980 1:00:00 GMT" />
<meta http-equiv="pragma" content="no-cache" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<meta name="description" content="nDPId RRD Graph">
<meta name="author" content="Toni Uhlig">
<link rel="icon" href="https://getbootstrap.com/docs/4.0/assets/img/favicons/favicon.ico">
<title>nDPId Dashboard</title>
<link rel="canonical" href="https://getbootstrap.com/docs/4.0/examples/dashboard/">
<!-- Bootstrap core CSS -->
<link href="bootstrap.css" rel="stylesheet">
<!-- Custom styles for this template -->
<link href="dashboard.css" rel="stylesheet">
</head>
<body>
<nav class="navbar navbar-dark sticky-top bg-dark flex-md-nowrap p-0">
<a class="navbar-brand col-sm-3 col-md-2 mr-0" href="https://github.com/utoni/nDPId">nDPId Collectd RRD Graph</a>
</nav>
<div class="container-fluid">
<div class="row">
<nav class="col-md-2 d-none d-md-block bg-light sidebar">
<div class="sidebar-sticky">
<h6 class="sidebar-heading d-flex justify-content-between align-items-center px-3 mt-4 mb-1 text-muted">
<span>Graphs</span>
</h6>
<ul class="nav flex-column mb-2">
<li class="nav-item">
<a class="nav-link" href="index.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Home
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="flows.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line><polyline points="10 9 9 9 8 9"></polyline>
</svg>
Flows
</a>
</li>
<li class="nav-item">
<a class="nav-link active" href="other.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Other
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="detections.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Detections
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="categories.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Categories
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="risks.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Risks
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="jsons.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
JSONs
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="events.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Events
</a>
</li>
</ul>
</div>
</nav>
<main role="main" class="col-md-9 ml-sm-auto col-lg-10 pt-3 px-4">
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="traffic_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="traffic_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="traffic_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="traffic_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="traffic_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="traffic_past_year.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer3_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer3_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer3_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer3_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer3_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer3_past_year.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer4_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer4_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer4_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer4_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer4_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="layer4_past_year.png" class="img-fluid" alt="Responsive image">
</div>
</main>
</div>
</div>
<!-- Bootstrap core JavaScript
================================================== -->
<!-- Placed at the end of the document so the pages load faster -->
<script src="jquery-3.js" integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN" crossorigin="anonymous"></script>
<script>window.jQuery || document.write('<script src="../../assets/js/vendor/jquery-slim.min.js"><\/script>')</script>
<script src="popper.js"></script>
<script src="bootstrap.js"></script>
<!-- Icons -->
<script src="feather.js"></script>
<script>
feather.replace()
</script>
</body></html>

5
examples/c-collectd/www/dpi/popper.js
View File

File diff suppressed because one or more lines are too long

198
examples/c-collectd/www/dpi/risks.html
View File

@@ -1,198 +0,0 @@
<!DOCTYPE html>
<html lang="en"><head>
<meta http-equiv="cache-control" content="max-age=0" />
<meta http-equiv="cache-control" content="no-cache" />
<meta http-equiv="expires" content="0" />
<meta http-equiv="expires" content="Tue, 01 Jan 1980 1:00:00 GMT" />
<meta http-equiv="pragma" content="no-cache" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<meta name="description" content="nDPId RRD Graph">
<meta name="author" content="Toni Uhlig">
<link rel="icon" href="https://getbootstrap.com/docs/4.0/assets/img/favicons/favicon.ico">
<title>nDPId Dashboard</title>
<link rel="canonical" href="https://getbootstrap.com/docs/4.0/examples/dashboard/">
<!-- Bootstrap core CSS -->
<link href="bootstrap.css" rel="stylesheet">
<!-- Custom styles for this template -->
<link href="dashboard.css" rel="stylesheet">
</head>
<body>
<nav class="navbar navbar-dark sticky-top bg-dark flex-md-nowrap p-0">
<a class="navbar-brand col-sm-3 col-md-2 mr-0" href="https://github.com/utoni/nDPId">nDPId Collectd RRD Graph</a>
</nav>
<div class="container-fluid">
<div class="row">
<nav class="col-md-2 d-none d-md-block bg-light sidebar">
<div class="sidebar-sticky">
<h6 class="sidebar-heading d-flex justify-content-between align-items-center px-3 mt-4 mb-1 text-muted">
<span>Graphs</span>
</h6>
<ul class="nav flex-column mb-2">
<li class="nav-item">
<a class="nav-link" href="index.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Home
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="flows.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line><polyline points="10 9 9 9 8 9"></polyline>
</svg>
Flows
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="other.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Other
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="detections.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Detections
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="categories.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Categories
</a>
</li>
<li class="nav-item">
<a class="nav-link active" href="risks.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Risks
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="jsons.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
JSONs
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="events.html">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-file-text">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"></path>
<polyline points="14 2 14 8 20 8"></polyline>
<line x1="16" y1="13" x2="8" y2="13"></line>
<line x1="16" y1="17" x2="8" y2="17"></line>
<polyline points="10 9 9 9 8 9"></polyline>
</svg>
Events
</a>
</li>
</ul>
</div>
</nav>
<main role="main" class="col-md-9 ml-sm-auto col-lg-10 pt-3 px-4">
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="severities_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="severities_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="severities_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="severities_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="severities_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="severities_past_year.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="risky_events_past_hour.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="risky_events_past_12hours.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="risky_events_past_day.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="risky_events_past_week.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="risky_events_past_month.png" class="img-fluid" alt="Responsive image">
</div>
<div class="d-flex justify-content-center flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">
<img src="risky_events_past_year.png" class="img-fluid" alt="Responsive image">
</div>
</main>
</div>
</div>
<!-- Bootstrap core JavaScript
================================================== -->
<!-- Placed at the end of the document so the pages load faster -->
<script src="jquery-3.js" integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN" crossorigin="anonymous"></script>
<script>window.jQuery || document.write('<script src="../../assets/js/vendor/jquery-slim.min.js"><\/script>')</script>
<script src="popper.js"></script>
<script src="bootstrap.js"></script>
<!-- Icons -->
<script src="feather.js"></script>
<script>
feather.replace()
</script>
</body></html>

1921
examples/c-influxd/c-influxd.c
View File

File diff suppressed because it is too large Load Diff

6021
examples/c-influxd/grafana-dashboard-simple.json
View File

File diff suppressed because it is too large Load Diff

138
examples/c-json-stdout/c-json-stdout.c Normal file
View File

@@ -0,0 +1,138 @@
#include <arpa/inet.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <unistd.h>
#include "config.h"
#include "jsmn.h"
static char serv_listen_addr[INET_ADDRSTRLEN] = DISTRIBUTOR_HOST;
static uint16_t serv_listen_port = DISTRIBUTOR_PORT;
int main(void)
{
int sockfd = socket(AF_INET, SOCK_STREAM, 0);
struct sockaddr_in remote_addr = {};
socklen_t remote_addrlen = sizeof(remote_addr);
uint8_t buf[NETWORK_BUFFER_MAX_SIZE];
size_t buf_used = 0;
size_t json_start = 0;
unsigned long long int json_bytes = 0;
jsmn_parser parser;
jsmntok_t tokens[128];
if (sockfd < 0)
{
perror("socket");
return 1;
}
remote_addr.sin_family = AF_INET;
if (inet_pton(AF_INET, &serv_listen_addr[0], &remote_addr.sin_addr) != 1)
{
perror("inet_pton");
return 1;
}
remote_addr.sin_port = htons(serv_listen_port);
if (connect(sockfd, (struct sockaddr *)&remote_addr, remote_addrlen) != 0)
{
perror("connect");
return 1;
}
while (1)
{
errno = 0;
ssize_t bytes_read = read(sockfd, buf + buf_used, sizeof(buf) - buf_used);
if (bytes_read <= 0 || errno != 0)
{
fprintf(stderr, "Remote end disconnected.\n");
break;
}
buf_used += bytes_read;
while (buf_used >= NETWORK_BUFFER_LENGTH_DIGITS + 1)
{
if (buf[NETWORK_BUFFER_LENGTH_DIGITS] != '{')
{
fprintf(stderr, "BUG: JSON invalid opening character: '%c'\n", buf[NETWORK_BUFFER_LENGTH_DIGITS]);
exit(1);
}
char * json_str_start = NULL;
json_bytes = strtoull((char *)buf, &json_str_start, 10);
json_bytes += (uint8_t *)json_str_start - buf;
json_start = (uint8_t *)json_str_start - buf;
if (errno == ERANGE)
{
fprintf(stderr, "BUG: Size of JSON exceeds limit\n");
exit(1);
}
if ((uint8_t *)json_str_start == buf)
{
fprintf(stderr, "BUG: Missing size before JSON string: \"%.*s\"\n", NETWORK_BUFFER_LENGTH_DIGITS, buf);
exit(1);
}
if (json_bytes > sizeof(buf))
{
fprintf(stderr, "BUG: JSON string too big: %llu > %zu\n", json_bytes, sizeof(buf));
exit(1);
}
if (json_bytes > buf_used)
{
break;
}
if (buf[json_bytes - 2] != '}' ||
buf[json_bytes - 1] != '\n')
{
fprintf(stderr, "BUG: Invalid JSON string: \"%.*s\"\n", (int)json_bytes, buf);
exit(1);
}
int r;
jsmn_init(&parser);
r = jsmn_parse(&parser,
(char *)(buf + json_start),
json_bytes - json_start,
tokens,
sizeof(tokens) / sizeof(tokens[0]));
if (r < 0 || tokens[0].type != JSMN_OBJECT)
{
fprintf(stderr, "JSON parsing failed with return value %d at position %u\n", r, parser.pos);
fprintf(stderr, "JSON string: '%.*s'\n", (int)(json_bytes - json_start), (char *)(buf + json_start));
exit(1);
}
for (int i = 1; i < r; i++)
{
if (i % 2 == 1)
{
#ifdef JSMN_PARENT_LINKS
printf("[%d][%d]", i, tokens[i].parent);
#endif
printf("[%.*s : ", tokens[i].end - tokens[i].start, (char *)(buf + json_start) + tokens[i].start);
}
else
{
printf("%.*s] ", tokens[i].end - tokens[i].start, (char *)(buf + json_start) + tokens[i].start);
}
}
printf("EoF\n");
memmove(buf, buf + json_bytes, buf_used - json_bytes);
buf_used -= json_bytes;
json_bytes = 0;
json_start = 0;
}
}
return 0;
}

714
examples/c-notifyd/c-notifyd.c
View File

@@ -1,714 +0,0 @@
#include <dbus-1.0/dbus/dbus.h>
#include <signal.h>
#include <stdint.h>
#include "nDPIsrvd.h"
#include "utstring.h"
#include "utils.h"
#define SLEEP_TIME_IN_S (3)
struct flow_user_data
{
nDPIsrvd_ull detected_risks;
};
enum dbus_level
{
DBUS_LOW = 0,
DBUS_NORMAL,
DBUS_CRITICAL
};
static char const * const flow_severities[] = {"Low", "Medium", "High", "Severe", "Critical", "Emergency"};
static char const * const flow_breeds[] = {
"Safe", "Acceptable", "Fun", "Unsafe", "Potentially_Dangerous", "Tracker_Ads", "Dangerous", "Unrated", "???"};
static char const * const flow_categories[] = {"Unspecified",
"Media",
"VPN",
"Email",
"DataTransfer",
"Web",
"SocialNetwork",
"Download",
"Game",
"Chat",
"VoIP",
"Database",
"RemoteAccess",
"Cloud",
"Network",
"Collaborative",
"RPC",
"Streaming",
"System",
"SoftwareUpdate",
"Music",
"Video",
"Shopping",
"Productivity",
"FileSharing",
"ConnCheck",
"IoT-Scada",
"VirtAssistant",
"Cybersecurity",
"AdultContent",
"Mining",
"Malware",
"Advertisement",
"Banned_Site",
"Site_Unavailable",
"Allowed_Site",
"Antimalware",
"Crypto_Currency",
"Gambling",
"Health",
"ArtifIntelligence",
"Finance",
"News",
"Sport",
"Business",
"Internet",
"Blockchain_Crypto",
"Blog_Forum",
"Government",
"Education",
"CDN_Proxy",
"Hw_Sw",
"Dating",
"Travel",
"Food",
"Bots",
"Scanners",
"Hosting",
"Art",
"Fashion",
"Books",
"Science",
"Maps_Navigation",
"Login_Portal",
"Legal",
"Environmental_Services",
"Culture",
"Housing",
"Telecommunication",
"Transportation",
"Design",
"Employment",
"Events",
"Weather",
"Lifestyle",
"Real_Estate",
"Security",
"Environment",
"Hobby",
"Computer_Science",
"Construction",
"Engineering",
"Religion",
"Entertainment",
"Agriculture",
"Technology",
"Beauty",
"History",
"Politics",
"Vehicles"};
static uint8_t desired_flow_severities[nDPIsrvd_ARRAY_LENGTH(flow_severities)] = {};
static uint8_t desired_flow_breeds[nDPIsrvd_ARRAY_LENGTH(flow_breeds)] = {};
static uint8_t desired_flow_categories[nDPIsrvd_ARRAY_LENGTH(flow_categories)] = {};
static unsigned int id = 0;
static char const * const application = "nDPIsrvd.notifyd";
static int main_thread_shutdown = 0;
static char * pidfile = NULL;
static char * serv_optarg = NULL;
#ifdef ENABLE_MEMORY_PROFILING
void nDPIsrvd_memprof_log_alloc(size_t alloc_size)
{
(void)alloc_size;
}
void nDPIsrvd_memprof_log_free(size_t free_size)
{
(void)free_size;
}
void nDPIsrvd_memprof_log(char const * const format, ...)
{
va_list ap;
va_start(ap, format);
fprintf(stderr, "%s", "nDPIsrvd MemoryProfiler: ");
vfprintf(stderr, format, ap);
fprintf(stderr, "%s\n", "");
va_end(ap);
}
#endif
static void send_to_dbus(char const * const icon,
char const * const urgency,
enum dbus_level level,
char const * const summary,
char const * const body,
int timeout)
{
DBusConnection * connection = dbus_bus_get(DBUS_BUS_SESSION, 0);
DBusMessage * message = dbus_message_new_method_call("org.freedesktop.Notifications",
"/org/freedesktop/Notifications",
"org.freedesktop.Notifications",
"Notify");
DBusMessageIter iter[4];
dbus_message_iter_init_append(message, iter);
dbus_message_iter_append_basic(iter, 's', &application);
dbus_message_iter_append_basic(iter, 'u', &id);
dbus_message_iter_append_basic(iter, 's', &icon);
dbus_message_iter_append_basic(iter, 's', &summary);
dbus_message_iter_append_basic(iter, 's', &body);
dbus_message_iter_open_container(iter, 'a', "s", iter + 1);
dbus_message_iter_close_container(iter, iter + 1);
dbus_message_iter_open_container(iter, 'a', "{sv}", iter + 1);
dbus_message_iter_open_container(iter + 1, 'e', 0, iter + 2);
dbus_message_iter_append_basic(iter + 2, 's', &urgency);
dbus_message_iter_open_container(iter + 2, 'v', "y", iter + 3);
dbus_message_iter_append_basic(iter + 3, 'y', &level);
dbus_message_iter_close_container(iter + 2, iter + 3);
dbus_message_iter_close_container(iter + 1, iter + 2);
dbus_message_iter_close_container(iter, iter + 1);
dbus_message_iter_append_basic(iter, 'i', &timeout);
dbus_connection_send(connection, message, 0);
dbus_connection_flush(connection);
dbus_message_unref(message);
dbus_connection_unref(connection);
id++;
}
static void notify(enum dbus_level level, char const * const summary, int timeout, char const * const body)
{
send_to_dbus("dialog-information", "urgency", level, summary, body, timeout);
}
__attribute__((format(printf, 4, 5))) static void notifyf(
enum dbus_level level, char const * const summary, int timeout, char const * const body_fmt, ...)
{
va_list ap;
char buf[BUFSIZ];
va_start(ap, body_fmt);
if (vsnprintf(buf, sizeof(buf), body_fmt, ap) > 0)
{
notify(level, summary, timeout, buf);
}
va_end(ap);
}
static ssize_t get_value_index(char const * const possible_values[],
size_t possible_values_size,
char const * const needle,
size_t needle_len)
{
size_t i;
for (i = 0; i < possible_values_size; ++i)
{
if (strncmp(needle, possible_values[i], needle_len) == 0)
{
break;
}
}
if (i == possible_values_size)
{
return -1;
}
return i;
}
static void check_value(char const * const possible_values[],
size_t possible_values_size,
char const * const needle,
size_t needle_len)
{
if (get_value_index(possible_values, possible_values_size, needle, needle_len) == -1)
{
notifyf(DBUS_CRITICAL, "nDPIsrvd-notifyd", 5000, "BUG: Unknown value: %.*s", (int)needle_len, needle);
}
}
static enum nDPIsrvd_callback_return notifyd_json_callback(struct nDPIsrvd_socket * const sock,
struct nDPIsrvd_instance * const instance,
struct nDPIsrvd_thread_data * const thread_data,
struct nDPIsrvd_flow * const flow)
{
(void)instance;
(void)thread_data;
struct nDPIsrvd_json_token const * const flow_event_name = TOKEN_GET_SZ(sock, "flow_event_name");
struct flow_user_data * flow_user_data = NULL;
if (flow != NULL)
{
flow_user_data = (struct flow_user_data *)flow->flow_user_data;
}
if (TOKEN_VALUE_EQUALS_SZ(sock, flow_event_name, "detected") != 0 ||
TOKEN_VALUE_EQUALS_SZ(sock, flow_event_name, "detection-update") != 0 ||
TOKEN_VALUE_EQUALS_SZ(sock, flow_event_name, "update") != 0)
{
struct nDPIsrvd_json_token const * const flow_risks = TOKEN_GET_SZ(sock, "ndpi", "flow_risk");
struct nDPIsrvd_json_token const * current = NULL;
int next_child_index = -1, desired_severity_found = 0;
UT_string risks;
utstring_init(&risks);
if (flow_risks != NULL)
{
while ((current = nDPIsrvd_get_next_token(sock, flow_risks, &next_child_index)) != NULL)
{
nDPIsrvd_ull numeric_risk_value = (nDPIsrvd_ull)-1;
size_t flow_risk_key_len = 0;
char const * const flow_risk_key = TOKEN_GET_KEY(sock, current, &flow_risk_key_len);
if (flow_risk_key == NULL || flow_risk_key_len == 0)
{
continue;
}
if (str_value_to_ull(flow_risk_key, &numeric_risk_value) == CONVERSION_OK && flow_user_data != NULL &&
(flow_user_data->detected_risks & (1ull << numeric_risk_value)) == 0)
{
flow_user_data->detected_risks |= (1ull << (numeric_risk_value - 1));
char flow_risk_sz[flow_risk_key_len + 1];
snprintf(flow_risk_sz, sizeof(flow_risk_sz), "%llu", numeric_risk_value);
size_t flow_risk_len = 0;
size_t flow_severity_len = 0;
char const * const flow_risk_str =
TOKEN_GET_VALUE(sock,
TOKEN_GET_SZ(sock, "ndpi", "flow_risk", flow_risk_sz, "risk"),
&flow_risk_len);
char const * const flow_severity_str =
TOKEN_GET_VALUE(sock,
TOKEN_GET_SZ(sock, "ndpi", "flow_risk", flow_risk_sz, "severity"),
&flow_severity_len);
if (flow_risk_str == NULL || flow_risk_len == 0 || flow_severity_str == NULL ||
flow_severity_len == 0)
{
continue;
}
ssize_t severity_index = get_value_index(flow_severities,
nDPIsrvd_ARRAY_LENGTH(flow_severities),
flow_severity_str,
flow_severity_len);
if (severity_index != -1 && desired_flow_severities[severity_index] != 0)
{
desired_severity_found = 1;
}
utstring_printf(&risks,
"Risk: '%.*s'\n"
"Severity: '%.*s'\n",
(int)flow_risk_len,
flow_risk_str,
(int)flow_severity_len,
flow_severity_str);
check_value(flow_severities,
nDPIsrvd_ARRAY_LENGTH(flow_severities),
flow_severity_str,
flow_severity_len);
}
}
}
{
size_t flow_srcip_len = 0;
size_t flow_dstip_len = 0;
size_t flow_breed_len = 0;
size_t flow_category_len = 0;
size_t flow_hostname_len = 0;
char const * const flow_srcip = TOKEN_GET_VALUE(sock, TOKEN_GET_SZ(sock, "src_ip"), &flow_srcip_len);
char const * const flow_dstip = TOKEN_GET_VALUE(sock, TOKEN_GET_SZ(sock, "dst_ip"), &flow_dstip_len);
char const * const flow_breed_str =
TOKEN_GET_VALUE(sock, TOKEN_GET_SZ(sock, "ndpi", "breed"), &flow_breed_len);
char const * const flow_category_str =
TOKEN_GET_VALUE(sock, TOKEN_GET_SZ(sock, "ndpi", "category"), &flow_category_len);
char const * const flow_hostname =
TOKEN_GET_VALUE(sock, TOKEN_GET_SZ(sock, "ndpi", "hostname"), &flow_hostname_len);
if (flow_breed_str != NULL && flow_breed_len != 0 && flow_category_str != NULL && flow_category_len != 0)
{
ssize_t breed_index =
get_value_index(flow_breeds, nDPIsrvd_ARRAY_LENGTH(flow_breeds), flow_breed_str, flow_breed_len);
ssize_t category_index = get_value_index(flow_categories,
nDPIsrvd_ARRAY_LENGTH(flow_categories),
flow_category_str,
flow_category_len);
if ((breed_index != -1 && desired_flow_breeds[breed_index] != 0) ||
(category_index != -1 && desired_flow_categories[category_index] != 0) ||
desired_severity_found != 0)
{
notifyf(DBUS_CRITICAL,
"Flow Notification",
5000,
"%.*s -> %.*s (%.*s)\nBreed: '%.*s', Category: '%.*s'\n%s",
(int)flow_srcip_len,
flow_srcip,
(int)flow_dstip_len,
flow_dstip,
(flow_hostname_len > 0 ? (int)flow_hostname_len : 1),
(flow_hostname_len > 0 ? flow_hostname : "-"),
(int)flow_breed_len,
flow_breed_str,
(int)flow_category_len,
flow_category_str,
(utstring_len(&risks) > 0 ? utstring_body(&risks) : "No flow risks detected\n"));
}
check_value(flow_breeds, nDPIsrvd_ARRAY_LENGTH(flow_breeds), flow_breed_str, flow_breed_len);
check_value(flow_categories,
nDPIsrvd_ARRAY_LENGTH(flow_categories),
flow_category_str,
flow_category_len);
}
else if (desired_severity_found != 0)
{
notifyf(DBUS_CRITICAL,
"Risky Flow",
5000,
"%.*s -> %.*s (%.*s)\n%s",
(int)flow_srcip_len,
flow_srcip,
(int)flow_dstip_len,
flow_dstip,
(flow_hostname_len > 0 ? (int)flow_hostname_len : 1),
(flow_hostname_len > 0 ? flow_hostname : "-"),
utstring_body(&risks));
}
}
utstring_done(&risks);
}
return CALLBACK_OK;
}
static void print_usage(char const * const arg0)
{
static char const usage[] =
"Usage: %s "
"[-s host] [-C category...] [-B breed...] [-S severity...]\n\n"
"\t-s\tDestination where nDPIsrvd is listening on.\n"
"\t-C\tDesired nDPI category which fires a notificiation.\n"
"\t \tCan be specified multiple times.\n"
"\t-B\tDesired nDPI breed which fires a notification.\n"
"\t \tCan be specified multiple times.\n"
"\t-S\tDesired nDPI risk severity which fires a notification.\n"
"\t \tCan be specified multiple times.\n"
"\n"
"Possible values for `-C': %s\n"
"Possible values for `-B': %s\n"
"Possible values for `-S': %s\n"
"\n";
UT_string flow_categories_str, flow_breeds_str, flow_severities_str;
utstring_init(&flow_categories_str);
utstring_init(&flow_breeds_str);
utstring_init(&flow_severities_str);
for (size_t i = 0; i < nDPIsrvd_ARRAY_LENGTH(flow_categories); ++i)
{
utstring_printf(&flow_categories_str, "%s, ", flow_categories[i]);
}
flow_categories_str.d[flow_categories_str.i - 2] = '\0';
for (size_t i = 0; i < nDPIsrvd_ARRAY_LENGTH(flow_breeds); ++i)
{
utstring_printf(&flow_breeds_str, "%s, ", flow_breeds[i]);
}
flow_breeds_str.d[flow_breeds_str.i - 2] = '\0';
for (size_t i = 0; i < nDPIsrvd_ARRAY_LENGTH(flow_severities); ++i)
{
utstring_printf(&flow_severities_str, "%s, ", flow_severities[i]);
}
flow_severities_str.d[flow_severities_str.i - 2] = '\0';
fprintf(stderr,
usage,
arg0,
utstring_body(&flow_categories_str),
utstring_body(&flow_breeds_str),
utstring_body(&flow_severities_str));
utstring_done(&flow_severities_str);
utstring_done(&flow_breeds_str);
utstring_done(&flow_categories_str);
}
static int set_defaults(void)
{
char const * const default_severities[] = {"High", "Severe", "Critical", "Emergency"};
char const * const default_breeds[] = {"Unsafe", "Potentially_Dangerous", "Dangerous", "Unrated"};
char const * const default_categories[] = {"Mining", "Malware", "Banned_Site", "Crypto_Currency"};
for (size_t i = 0; i < nDPIsrvd_ARRAY_LENGTH(default_severities); ++i)
{
ssize_t index = get_value_index(flow_severities,
nDPIsrvd_ARRAY_LENGTH(flow_severities),
default_severities[i],
strlen(default_severities[i]));
if (index == -1)
{
return 1;
}
desired_flow_severities[index] = 1;
}
for (size_t i = 0; i < nDPIsrvd_ARRAY_LENGTH(default_breeds); ++i)
{
ssize_t index = get_value_index(flow_breeds,
nDPIsrvd_ARRAY_LENGTH(flow_breeds),
default_breeds[i],
strlen(default_breeds[i]));
if (index == -1)
{
return 1;
}
desired_flow_breeds[index] = 1;
}
for (size_t i = 0; i < nDPIsrvd_ARRAY_LENGTH(default_categories); ++i)
{
ssize_t index = get_value_index(flow_categories,
nDPIsrvd_ARRAY_LENGTH(flow_categories),
default_categories[i],
strlen(default_categories[i]));
if (index == -1)
{
return 1;
}
desired_flow_categories[index] = 1;
}
return 0;
}
static int parse_options(int argc, char ** argv, struct nDPIsrvd_socket * const sock)
{
int opt, force_defaults = 1;
while ((opt = getopt(argc, argv, "hldp:s:C:B:S:")) != -1)
{
switch (opt)
{
case 'd':
daemonize_enable();
break;
case 'p':
free(pidfile);
pidfile = strdup(optarg);
break;
case 's':
free(serv_optarg);
serv_optarg = strdup(optarg);
break;
case 'C':
{
ssize_t index =
get_value_index(flow_categories, nDPIsrvd_ARRAY_LENGTH(flow_categories), optarg, strlen(optarg));
if (index == -1)
{
fprintf(stderr, "Invalid argument for `-C': %s\n", optarg);
return 1;
}
else
{
desired_flow_categories[index] = 1;
}
force_defaults = 0;
break;
}
case 'B':
{
ssize_t index =
get_value_index(flow_breeds, nDPIsrvd_ARRAY_LENGTH(flow_breeds), optarg, strlen(optarg));
if (index == -1)
{
fprintf(stderr, "Invalid argument for `-B': %s\n", optarg);
return 1;
}
else
{
desired_flow_breeds[index] = 1;
}
force_defaults = 0;
break;
}
case 'S':
{
ssize_t index =
get_value_index(flow_severities, nDPIsrvd_ARRAY_LENGTH(flow_severities), optarg, strlen(optarg));
if (index == -1)
{
fprintf(stderr, "Invalid argument for `-S': %s\n", optarg);
return 1;
}
else
{
desired_flow_severities[index] = 1;
}
force_defaults = 0;
break;
}
default:
print_usage(argv[0]);
return 1;
}
}
if (force_defaults != 0 && set_defaults() != 0)
{
notifyf(DBUS_CRITICAL, "nDPIsrvd-notifyd", 5000, "%s\n", "BUG: Could not set default values.");
return 1;
}
if (serv_optarg == NULL)
{
serv_optarg = strdup(DISTRIBUTOR_UNIX_SOCKET);
}
if (nDPIsrvd_setup_address(&sock->address, serv_optarg) != 0)
{
notifyf(DBUS_CRITICAL, "nDPIsrvd-notifyd", 3000, "Could not parse address `%s'\n", serv_optarg);
return 1;
}
if (optind < argc)
{
notifyf(DBUS_CRITICAL, "nDPIsrvd-notifyd", 3000, "%s\n", "Unexpected argument after options");
return 1;
}
return 0;
}
static void sighandler(int signum)
{
switch (signum)
{
case SIGINT:
notify(DBUS_LOW, "nDPIsrvd-notifyd", 3000, "Received SIGINT, shutdown.");
break;
case SIGTERM:
notify(DBUS_LOW, "nDPIsrvd-notifyd", 3000, "Received SIGTERM, shutdown.");
break;
default:
notify(DBUS_LOW, "nDPIsrvd-notifyd", 3000, "Received unknown signal, shutdown.");
break;
}
main_thread_shutdown++;
}
int main(int argc, char ** argv)
{
signal(SIGINT, sighandler);
signal(SIGTERM, sighandler);
signal(SIGPIPE, SIG_IGN);
struct nDPIsrvd_socket * sock =
nDPIsrvd_socket_init(0, 0, 0, sizeof(struct flow_user_data), notifyd_json_callback, NULL, NULL);
if (sock == NULL)
{
notifyf(DBUS_CRITICAL, "nDPIsrvd-notifyd", 5000, "%s\n", "BUG: nDPIsrvd socket memory allocation failed!");
return 1;
}
if (parse_options(argc, argv, sock) != 0)
{
goto failure;
}
if (daemonize_with_pidfile(pidfile) != 0)
{
return 1;
}
int previous_connect_succeeded = 1;
do
{
if (nDPIsrvd_connect(sock) != CONNECT_OK)
{
if (main_thread_shutdown != 0)
{
break;
}
if (previous_connect_succeeded != 0)
{
notifyf(
DBUS_CRITICAL, "nDPIsrvd-notifyd", 3000, "nDPIsrvd socket connect to %s failed!\n", serv_optarg);
previous_connect_succeeded = 0;
}
nDPIsrvd_socket_close(sock);
sleep(SLEEP_TIME_IN_S);
continue;
}
previous_connect_succeeded = 1;
if (nDPIsrvd_set_read_timeout(sock, 3, 0) != 0)
{
notifyf(DBUS_CRITICAL, "nDPIsrvd-notifyd", 3000, "nDPIsrvd set read timeout failed: %s\n", strerror(errno));
goto failure;
}
notifyf(DBUS_NORMAL, "nDPIsrvd-notifyd", 3000, "Connected to '%s'\n", serv_optarg);
while (main_thread_shutdown == 0)
{
enum nDPIsrvd_read_return read_ret = nDPIsrvd_read(sock);
if (errno == EINTR)
{
continue;
}
if (read_ret == READ_TIMEOUT)
{
continue;
}
if (read_ret != READ_OK)
{
notifyf(DBUS_CRITICAL, "nDPIsrvd-notifyd", 3000, "nDPIsrvd socket read from %s failed!\n", serv_optarg);
break;
}
enum nDPIsrvd_parse_return parse_ret = nDPIsrvd_parse_all(sock);
if (parse_ret != PARSE_NEED_MORE_DATA)
{
notifyf(DBUS_CRITICAL,
"nDPIsrvd-notifyd",
3000,
"Could not parse JSON message %s: %.*s\n",
nDPIsrvd_enum_to_string(parse_ret),
nDPIsrvd_json_buffer_length(sock),
nDPIsrvd_json_buffer_string(sock));
break;
}
}
nDPIsrvd_socket_close(sock);
notifyf(DBUS_NORMAL, "nDPIsrvd-notifyd", 3000, "Disconnected from '%s'.\n", serv_optarg);
if (main_thread_shutdown == 0)
{
sleep(SLEEP_TIME_IN_S);
}
} while (main_thread_shutdown == 0);
failure:
nDPIsrvd_socket_free(&sock);
daemonize_shutdown(pidfile);
return 0;
}

61
examples/c-simple/c-simple.c
View File

@@ -9,16 +9,6 @@ static int main_thread_shutdown = 0;
static struct nDPIsrvd_socket * sock = NULL;
#ifdef ENABLE_MEMORY_PROFILING
void nDPIsrvd_memprof_log_alloc(size_t alloc_size)
{
(void)alloc_size;
}
void nDPIsrvd_memprof_log_free(size_t free_size)
{
(void)free_size;
}
void nDPIsrvd_memprof_log(char const * const format, ...)
{
va_list ap;
@@ -58,7 +48,7 @@ static void nDPIsrvd_write_flow_info_cb(struct nDPIsrvd_socket const * sock,
#endif
flow->last_seen,
flow->idle_time,
(thread_data != NULL && flow->last_seen + flow->idle_time >= thread_data->most_recent_flow_time
(flow->last_seen + flow->idle_time >= thread_data->most_recent_flow_time
? flow->last_seen + flow->idle_time - thread_data->most_recent_flow_time
: 0));
}
@@ -93,6 +83,8 @@ static void nDPIsrvd_verify_flows_cb(struct nDPIsrvd_thread_data const * const t
{
fprintf(stderr, "Thread [UNKNOWN], Flow %llu verification failed\n", flow->id_as_ull);
}
exit(1);
}
static void sighandler(int signum)
@@ -117,11 +109,6 @@ static void sighandler(int signum)
{
fprintf(stderr, "%s\n", "Flow verification succeeded.");
}
else
{
/* FATAL! */
exit(EXIT_FAILURE);
}
}
else if (main_thread_shutdown == 0)
{
@@ -142,21 +129,10 @@ static enum nDPIsrvd_callback_return simple_json_callback(struct nDPIsrvd_socket
return CALLBACK_OK;
}
struct nDPIsrvd_json_token const * const alias = TOKEN_GET_SZ(sock, "alias");
struct nDPIsrvd_json_token const * const source = TOKEN_GET_SZ(sock, "source");
if (alias == NULL || source == NULL)
{
return CALLBACK_ERROR;
}
struct nDPIsrvd_json_token const * const flow_event_name = TOKEN_GET_SZ(sock, "flow_event_name");
if (TOKEN_VALUE_EQUALS_SZ(sock, flow_event_name, "new") != 0)
if (TOKEN_VALUE_EQUALS_SZ(flow_event_name, "new") != 0)
{
printf("Instance %.*s/%.*s (HT-Key: 0x%x), Thread %d, Flow %llu new\n",
nDPIsrvd_get_token_size(sock, alias),
nDPIsrvd_get_token_value(sock, alias),
nDPIsrvd_get_token_size(sock, source),
nDPIsrvd_get_token_value(sock, source),
printf("Instance 0x%x, Thread %d, Flow %llu new\n",
instance->alias_source_key,
flow->thread_id,
flow->id_as_ull);
@@ -174,21 +150,8 @@ static void simple_flow_cleanup_callback(struct nDPIsrvd_socket * const sock,
(void)sock;
(void)thread_data;
struct nDPIsrvd_json_token const * const alias = TOKEN_GET_SZ(sock, "alias");
struct nDPIsrvd_json_token const * const source = TOKEN_GET_SZ(sock, "source");
if (alias == NULL || source == NULL)
{
/* FATAL! */
fprintf(stderr, "BUG: Missing JSON token alias/source.\n");
exit(EXIT_FAILURE);
}
char const * const reason_str = nDPIsrvd_enum_to_string(reason);
printf("Instance %.*s/%.*s (HT-Key: 0x%x), Thread %d, Flow %llu cleanup, reason: %s\n",
nDPIsrvd_get_token_size(sock, alias),
nDPIsrvd_get_token_value(sock, alias),
nDPIsrvd_get_token_size(sock, source),
nDPIsrvd_get_token_value(sock, source),
printf("Instance 0x%x, Thread %d, Flow %llu cleanup, reason: %s\n",
instance->alias_source_key,
flow->thread_id,
flow->id_as_ull,
@@ -196,9 +159,7 @@ static void simple_flow_cleanup_callback(struct nDPIsrvd_socket * const sock,
if (reason == CLEANUP_REASON_FLOW_TIMEOUT)
{
/* FATAL! */
fprintf(stderr, "Flow %llu timeouted.\n", flow->id_as_ull);
exit(EXIT_FAILURE);
}
}
@@ -253,20 +214,14 @@ int main(int argc, char ** argv)
enum nDPIsrvd_parse_return parse_ret = nDPIsrvd_parse_all(sock);
if (parse_ret != PARSE_NEED_MORE_DATA)
{
printf("Could not parse JSON message %s: %.*s\n",
nDPIsrvd_enum_to_string(parse_ret),
nDPIsrvd_json_buffer_length(sock),
nDPIsrvd_json_buffer_string(sock));
printf("Could not parse json string: %s\n", nDPIsrvd_enum_to_string(parse_ret));
break;
}
}
if (main_thread_shutdown == 0 && read_ret != READ_OK)
{
printf("Parse read %s at JSON: %.*s\n",
nDPIsrvd_enum_to_string(read_ret),
nDPIsrvd_json_buffer_length(sock),
nDPIsrvd_json_buffer_string(sock));
printf("Parse read %s\n", nDPIsrvd_enum_to_string(read_ret));
}
return 1;

1
examples/cxx-graph

Submodule examples/cxx-graph deleted from 68eb1b105d

1
examples/js-rt-analyzer

Submodule examples/js-rt-analyzer deleted from 87cb7a0af5

1
examples/js-rt-analyzer-frontend

Submodule examples/js-rt-analyzer-frontend deleted from 6806ef7d13

BIN
examples/ndpid_grafana_example.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 62 KiB

3
examples/py-flow-dashboard/assets/flow-dash.css Normal file
View File

@@ -0,0 +1,3 @@
body {
background: black;
}

291
examples/py-flow-dashboard/flow-dash.py Executable file
View File

@@ -0,0 +1,291 @@
#!/usr/bin/env python3
import multiprocessing
import os
import sys
sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId')
sys.path.append(sys.base_prefix + '/share/nDPId')
import nDPIsrvd
from nDPIsrvd import nDPIsrvdSocket
import plotly_dash
FLOW_RISK_SEVERE = 4
FLOW_RISK_HIGH = 3
FLOW_RISK_MEDIUM = 2
FLOW_RISK_LOW = 1
def nDPIsrvd_worker_onFlowCleanup(instance, current_flow, global_user_data):
_, shared_flow_dict = global_user_data
flow_key = current_flow.flow_key
shared_flow_dict['current-flows'] -= 1
if flow_key not in shared_flow_dict:
return True
shared_flow_dict['total-l4-bytes'] += shared_flow_dict[flow_key]['total-l4-bytes']
if shared_flow_dict[flow_key]['is_detected'] is True:
shared_flow_dict['current-detected-flows'] -= 1
if shared_flow_dict[flow_key]['is_guessed'] is True:
shared_flow_dict['current-guessed-flows'] -= 1
if shared_flow_dict[flow_key]['is_not_detected'] is True:
shared_flow_dict['current-not-detected-flows'] -= 1
if shared_flow_dict[flow_key]['is_midstream'] is True:
shared_flow_dict['current-midstream-flows'] -= 1
if shared_flow_dict[flow_key]['is_risky'] > 0:
shared_flow_dict['current-risky-flows'] -= 1
if shared_flow_dict[flow_key]['is_risky'] == FLOW_RISK_LOW:
shared_flow_dict['current-risky-flows-low'] -= 1
elif shared_flow_dict[flow_key]['is_risky'] == FLOW_RISK_MEDIUM:
shared_flow_dict['current-risky-flows-medium'] -= 1
elif shared_flow_dict[flow_key]['is_risky'] == FLOW_RISK_HIGH:
shared_flow_dict['current-risky-flows-high'] -= 1
elif shared_flow_dict[flow_key]['is_risky'] == FLOW_RISK_SEVERE:
shared_flow_dict['current-risky-flows-severe'] -= 1
del shared_flow_dict[current_flow.flow_key]
return True
def nDPIsrvd_worker_onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
nsock, shared_flow_dict = global_user_data
shared_flow_dict['total-events'] += 1
shared_flow_dict['total-json-bytes'] = nsock.received_bytes
if 'error_event_name' in json_dict:
shared_flow_dict['total-base-events'] += 1
if 'daemon_event_name' in json_dict:
shared_flow_dict['total-daemon-events'] += 1
if 'packet_event_name' in json_dict and \
(json_dict['packet_event_name'] == 'packet' or \
json_dict['packet_event_name'] == 'packet-flow'):
shared_flow_dict['total-packet-events'] += 1
if 'flow_id' not in json_dict:
return True
else:
flow_key = json_dict['alias'] + '-' + json_dict['source'] + '-' + str(json_dict['flow_id'])
if flow_key not in shared_flow_dict:
current_flow.flow_key = flow_key
shared_flow_dict[flow_key] = mgr.dict()
shared_flow_dict[flow_key]['is_detected'] = False
shared_flow_dict[flow_key]['is_guessed'] = False
shared_flow_dict[flow_key]['is_not_detected'] = False
shared_flow_dict[flow_key]['is_midstream'] = False
shared_flow_dict[flow_key]['is_risky'] = 0
shared_flow_dict[flow_key]['total-l4-bytes'] = 0
shared_flow_dict[flow_key]['json'] = mgr.dict()
shared_flow_dict['total-flows'] += 1
shared_flow_dict['current-flows'] += 1
if current_flow.flow_key != flow_key:
return False
if 'flow_tot_l4_payload_len' in json_dict:
shared_flow_dict[flow_key]['total-l4-bytes'] = json_dict['flow_tot_l4_payload_len']
if 'midstream' in json_dict and json_dict['midstream'] != 0:
if shared_flow_dict[flow_key]['is_midstream'] is False:
shared_flow_dict['total-midstream-flows'] += 1
shared_flow_dict['current-midstream-flows'] += 1
shared_flow_dict[flow_key]['is_midstream'] = True
if 'ndpi' in json_dict:
shared_flow_dict[flow_key]['json']['ndpi'] = json_dict['ndpi']
if 'flow_risk' in json_dict['ndpi']:
if shared_flow_dict[flow_key]['is_risky'] == 0:
shared_flow_dict['total-risky-flows'] += 1
shared_flow_dict['current-risky-flows'] += 1
severity = shared_flow_dict[flow_key]['is_risky']
if severity == FLOW_RISK_LOW:
shared_flow_dict['current-risky-flows-low'] -= 1
elif severity == FLOW_RISK_MEDIUM:
shared_flow_dict['current-risky-flows-medium'] -= 1
elif severity == FLOW_RISK_HIGH:
shared_flow_dict['current-risky-flows-high'] -= 1
elif severity == FLOW_RISK_SEVERE:
shared_flow_dict['current-risky-flows-severe'] -= 1
for key in json_dict['ndpi']['flow_risk']:
if json_dict['ndpi']['flow_risk'][key]['severity'] == 'Low':
severity = max(severity, FLOW_RISK_LOW)
elif json_dict['ndpi']['flow_risk'][key]['severity'] == 'Medium':
severity = max(severity, FLOW_RISK_MEDIUM)
elif json_dict['ndpi']['flow_risk'][key]['severity'] == 'High':
severity = max(severity, FLOW_RISK_HIGH)
elif json_dict['ndpi']['flow_risk'][key]['severity'] == 'Severe':
severity = max(severity, FLOW_RISK_SEVERE)
else:
raise RuntimeError('Invalid flow risk severity: {}'.format(
json_dict['ndpi']['flow_risk'][key]['severity']))
shared_flow_dict[flow_key]['is_risky'] = severity
if severity == FLOW_RISK_LOW:
shared_flow_dict['current-risky-flows-low'] += 1
elif severity == FLOW_RISK_MEDIUM:
shared_flow_dict['current-risky-flows-medium'] += 1
elif severity == FLOW_RISK_HIGH:
shared_flow_dict['current-risky-flows-high'] += 1
elif severity == FLOW_RISK_SEVERE:
shared_flow_dict['current-risky-flows-severe'] += 1
if 'flow_event_name' not in json_dict:
return True
if json_dict['flow_state'] == 'finished' and \
json_dict['ndpi']['proto'] != 'Unknown' and \
shared_flow_dict[flow_key]['is_detected'] is False:
shared_flow_dict['total-detected-flows'] += 1
shared_flow_dict['current-detected-flows'] += 1
shared_flow_dict[flow_key]['is_detected'] = True
if json_dict['flow_event_name'] == 'new':
shared_flow_dict['total-flow-new-events'] += 1
elif json_dict['flow_event_name'] == 'update':
shared_flow_dict['total-flow-update-events'] += 1
elif json_dict['flow_event_name'] == 'end':
shared_flow_dict['total-flow-end-events'] += 1
elif json_dict['flow_event_name'] == 'idle':
shared_flow_dict['total-flow-idle-events'] += 1
elif json_dict['flow_event_name'] == 'guessed':
shared_flow_dict['total-flow-guessed-events'] += 1
if shared_flow_dict[flow_key]['is_guessed'] is False:
shared_flow_dict['total-guessed-flows'] += 1
shared_flow_dict['current-guessed-flows'] += 1
shared_flow_dict[flow_key]['is_guessed'] = True
elif json_dict['flow_event_name'] == 'not-detected':
shared_flow_dict['total-flow-not-detected-events'] += 1
if shared_flow_dict[flow_key]['is_not_detected'] is False:
shared_flow_dict['total-not-detected-flows'] += 1
shared_flow_dict['current-not-detected-flows'] += 1
shared_flow_dict[flow_key]['is_not_detected'] = True
elif json_dict['flow_event_name'] == 'detected' or \
json_dict['flow_event_name'] == 'detection-update':
if json_dict['flow_event_name'] == 'detection-update':
shared_flow_dict['total-flow-detection-update-events'] += 1
else:
shared_flow_dict['total-flow-detected-events'] += 1
if shared_flow_dict[flow_key]['is_detected'] is False:
shared_flow_dict['total-detected-flows'] += 1
shared_flow_dict['current-detected-flows'] += 1
shared_flow_dict[flow_key]['is_detected'] = True
if shared_flow_dict[flow_key]['is_guessed'] is True:
shared_flow_dict['total-guessed-flows'] -= 1
shared_flow_dict['current-guessed-flows'] -= 1
shared_flow_dict[flow_key]['is_guessed'] = False
return True
def nDPIsrvd_worker(address, shared_flow_dict):
sys.stderr.write('Recv buffer size: {}\n'
.format(nDPIsrvd.NETWORK_BUFFER_MAX_SIZE))
sys.stderr.write('Connecting to {} ..\n'
.format(address[0]+':'+str(address[1])
if type(address) is tuple else address))
try:
while True:
try:
nsock = nDPIsrvdSocket()
nsock.connect(address)
nsock.loop(nDPIsrvd_worker_onJsonLineRecvd,
nDPIsrvd_worker_onFlowCleanup,
(nsock, shared_flow_dict))
except nDPIsrvd.SocketConnectionBroken:
sys.stderr.write('Lost connection to {} .. reconnecting\n'
.format(address[0]+':'+str(address[1])
if type(address) is tuple else address))
except KeyboardInterrupt:
pass
if __name__ == '__main__':
argparser = nDPIsrvd.defaultArgumentParser()
argparser.add_argument('--listen-address', type=str, default='127.0.0.1', help='Plotly listen address')
argparser.add_argument('--listen-port', type=str, default=8050, help='Plotly listen port')
args = argparser.parse_args()
address = nDPIsrvd.validateAddress(args)
mgr = multiprocessing.Manager()
shared_flow_dict = mgr.dict()
shared_flow_dict['total-events'] = 0
shared_flow_dict['total-flow-new-events'] = 0
shared_flow_dict['total-flow-update-events'] = 0
shared_flow_dict['total-flow-end-events'] = 0
shared_flow_dict['total-flow-idle-events'] = 0
shared_flow_dict['total-flow-detected-events'] = 0
shared_flow_dict['total-flow-detection-update-events'] = 0
shared_flow_dict['total-flow-guessed-events'] = 0
shared_flow_dict['total-flow-not-detected-events'] = 0
shared_flow_dict['total-packet-events'] = 0
shared_flow_dict['total-base-events'] = 0
shared_flow_dict['total-daemon-events'] = 0
shared_flow_dict['total-json-bytes'] = 0
shared_flow_dict['total-l4-bytes'] = 0
shared_flow_dict['total-flows'] = 0
shared_flow_dict['total-detected-flows'] = 0
shared_flow_dict['total-risky-flows'] = 0
shared_flow_dict['total-midstream-flows'] = 0
shared_flow_dict['total-guessed-flows'] = 0
shared_flow_dict['total-not-detected-flows'] = 0
shared_flow_dict['current-flows'] = 0
shared_flow_dict['current-detected-flows'] = 0
shared_flow_dict['current-midstream-flows'] = 0
shared_flow_dict['current-guessed-flows'] = 0
shared_flow_dict['current-not-detected-flows'] = 0
shared_flow_dict['current-risky-flows'] = 0
shared_flow_dict['current-risky-flows-severe'] = 0
shared_flow_dict['current-risky-flows-high'] = 0
shared_flow_dict['current-risky-flows-medium'] = 0
shared_flow_dict['current-risky-flows-low'] = 0
nDPIsrvd_job = multiprocessing.Process(target=nDPIsrvd_worker,
args=(address, shared_flow_dict))
nDPIsrvd_job.start()
web_job = multiprocessing.Process(target=plotly_dash.web_worker,
args=(shared_flow_dict, args.listen_address, args.listen_port))
web_job.start()
nDPIsrvd_job.join()
web_job.terminate()
web_job.join()

414
examples/py-flow-dashboard/plotly_dash.py Normal file
View File

@@ -0,0 +1,414 @@
import math
import dash
try:
from dash import dcc
except ImportError:
import dash_core_components as dcc
try:
from dash import html
except ImportError:
import dash_html_components as html
try:
from dash import dash_table as dt
except ImportError:
import dash_table as dt
from dash.dependencies import Input, Output, State
import dash_daq as daq
import plotly.graph_objects as go
global shared_flow_dict
app = dash.Dash(__name__)
def generate_box():
return {
'display': 'flex', 'flex-direction': 'row',
'background-color': '#082255'
}
def generate_led_display(div_id, label_name):
return daq.LEDDisplay(
id=div_id,
label={'label': label_name, 'style': {'color': '#C4CDD5'}},
labelPosition='bottom',
value='0',
backgroundColor='#082255',
color='#C4CDD5',
)
def generate_gauge(div_id, label_name, max_value=10):
return daq.Gauge(
id=div_id,
value=0,
label={'label': label_name, 'style': {'color': '#C4CDD5'}},
max=max_value,
min=0,
)
def build_gauge(key, max_value=100):
gauge_max = int(max(max_value,
shared_flow_dict[key]))
grad_green = [0, int(gauge_max * 1/3)]
grad_yellow = [int(gauge_max * 1/3), int(gauge_max * 2/3)]
grad_red = [int(gauge_max * 2/3), gauge_max]
grad_dict = {
"gradient":True,
"ranges":{
"green":grad_green,
"yellow":grad_yellow,
"red":grad_red
}
}
return shared_flow_dict[key], gauge_max, grad_dict
def build_piechart(labels, values, color_map=None):
lay = dict(
plot_bgcolor = '#082255',
paper_bgcolor = '#082255',
font={"color": "#fff"},
uirevision=True,
autosize=True,
height=250,
margin = {'autoexpand': True, 'b': 0, 'l': 0, 'r': 0, 't': 0, 'pad': 0},
width = 500,
uniformtext_minsize = 12,
uniformtext_mode = 'hide',
)
return go.Figure(layout=lay, data=[go.Pie(labels=labels, values=values, sort=False, marker_colors=color_map, textinfo='percent', textposition='inside')])
COLOR_MAP = {
'piechart-flows': ['rgb(153, 153, 255)', 'rgb(153, 204, 255)', 'rgb(255, 204, 153)', 'rgb(255, 255, 255)'],
'piechart-midstream-flows': ['rgb(255, 255, 153)', 'rgb(153, 153, 255)'],
'piechart-risky-flows': ['rgb(255, 0, 0)', 'rgb(255, 128, 0)', 'rgb(255, 255, 0)', 'rgb(128, 255, 0)', 'rgb(153, 153, 255)'],
'graph-flows': {'Current Active Flows': {'color': 'rgb(153, 153, 255)', 'width': 1},
'Current Risky Flows': {'color': 'rgb(255, 153, 153)', 'width': 3},
'Current Midstream Flows': {'color': 'rgb(255, 255, 153)', 'width': 3},
'Current Guessed Flows': {'color': 'rgb(153, 204, 255)', 'width': 1},
'Current Not-Detected Flows': {'color': 'rgb(255, 204, 153)', 'width': 1},
'Current Unclassified Flows': {'color': 'rgb(255, 255, 255)', 'width': 1},
},
}
def generate_tab_flow():
return html.Div([
html.Div(children=[
dcc.Interval(id="tab-flow-default-interval", interval=1 * 2000, n_intervals=0),
html.Div(children=[
dt.DataTable(
id='table-info',
columns=[{'id': c.lower(), 'name': c, 'editable': False}
for c in ['Name', 'Total']],
style_header={
'backgroundColor': '#082233',
'color': 'white'
},
style_data={
'backgroundColor': '#082244',
'color': 'white'
},
)
], style={'display': 'flex', 'flex-direction': 'row'}),
html.Div(children=[
dcc.Graph(
id='piechart-flows',
config={
'displayModeBar': False,
},
figure=build_piechart(['Detected', 'Guessed', 'Not-Detected', 'Unclassified'],
[0, 0, 0, 0], COLOR_MAP['piechart-flows']),
),
], style={'padding': 10, 'flex': 1}),
html.Div(children=[
dcc.Graph(
id='piechart-midstream-flows',
config={
'displayModeBar': False,
},
figure=build_piechart(['Midstream', 'Not Midstream'],
[0, 0], COLOR_MAP['piechart-midstream-flows']),
),
], style={'padding': 10, 'flex': 1}),
html.Div(children=[
dcc.Graph(
id='piechart-risky-flows',
config={
'displayModeBar': False,
},
figure=build_piechart(['Severy Risk', 'High Risk', 'Medium Risk', 'Low Risk', 'No Risk'],
[0, 0], COLOR_MAP['piechart-risky-flows']),
),
], style={'padding': 10, 'flex': 1}),
], style=generate_box()),
html.Div(children=[
dcc.Interval(id="tab-flow-graph-interval", interval=4 * 1000, n_intervals=0),
dcc.Store(id="graph-traces"),
html.Div(children=[
dcc.Graph(
id="graph-flows",
config={
'displayModeBar': True,
'displaylogo': False,
},
style={'height':'60vh'},
),
], style={'padding': 10, 'flex': 1})
], style=generate_box())
])
def generate_tab_other():
return html.Div([
html.Div(children=[
dcc.Interval(id="tab-other-default-interval", interval=1 * 2000, n_intervals=0),
html.Div(children=[
dcc.Graph(
id='piechart-events',
config={
'displayModeBar': False,
},
),
], style={'padding': 10, 'flex': 1}),
], style=generate_box())
])
TABS_STYLES = {
'height': '34px'
}
TAB_STYLE = {
'borderBottom': '1px solid #d6d6d6',
'backgroundColor': '#385285',
'padding': '6px',
'fontWeight': 'bold',
}
TAB_SELECTED_STYLE = {
'borderTop': '1px solid #d6d6d6',
'borderBottom': '1px solid #d6d6d6',
'backgroundColor': '#119DFF',
'color': 'white',
'padding': '6px'
}
app.layout = html.Div([
dcc.Tabs(id="tabs-flow-dash", value="tab-flows", children=[
dcc.Tab(label="Flow", value="tab-flows", style=TAB_STYLE,
selected_style=TAB_SELECTED_STYLE,
children=generate_tab_flow()),
dcc.Tab(label="Other", value="tab-other", style=TAB_STYLE,
selected_style=TAB_SELECTED_STYLE,
children=generate_tab_other()),
], style=TABS_STYLES),
html.Div(id="tabs-content")
])
def prettifyBytes(bytes_received):
size_names = ['B', 'KB', 'MB', 'GB', 'TB']
if bytes_received == 0:
i = 0
else:
i = min(int(math.floor(math.log(bytes_received, 1024))), len(size_names) - 1)
p = math.pow(1024, i)
s = round(bytes_received / p, 2)
return '{:.2f} {}'.format(s, size_names[i])
@app.callback(output=[Output('table-info', 'data'),
Output('piechart-flows', 'figure'),
Output('piechart-midstream-flows', 'figure'),
Output('piechart-risky-flows', 'figure')],
inputs=[Input('tab-flow-default-interval', 'n_intervals')])
def tab_flow_update_components(n):
return [[{'name': 'JSON Events', 'total': shared_flow_dict['total-events']},
{'name': 'JSON Bytes', 'total': prettifyBytes(shared_flow_dict['total-json-bytes'])},
{'name': 'Layer4 Bytes', 'total': prettifyBytes(shared_flow_dict['total-l4-bytes'])},
{'name': 'Flows', 'total': shared_flow_dict['total-flows']},
{'name': 'Risky Flows', 'total': shared_flow_dict['total-risky-flows']},
{'name': 'Midstream Flows', 'total': shared_flow_dict['total-midstream-flows']},
{'name': 'Guessed Flows', 'total': shared_flow_dict['total-guessed-flows']},
{'name': 'Not Detected Flows', 'total': shared_flow_dict['total-not-detected-flows']}],
build_piechart(['Detected', 'Guessed', 'Not-Detected', 'Unclassified'],
[shared_flow_dict['current-detected-flows'],
shared_flow_dict['current-guessed-flows'],
shared_flow_dict['current-not-detected-flows'],
shared_flow_dict['current-flows']
- shared_flow_dict['current-detected-flows']
- shared_flow_dict['current-guessed-flows']
- shared_flow_dict['current-not-detected-flows']],
COLOR_MAP['piechart-flows']),
build_piechart(['Midstream', 'Not Midstream'],
[shared_flow_dict['current-midstream-flows'],
shared_flow_dict['current-flows'] -
shared_flow_dict['current-midstream-flows']],
COLOR_MAP['piechart-midstream-flows']),
build_piechart(['Severe', 'High', 'Medium', 'Low', 'No Risk'],
[shared_flow_dict['current-risky-flows-severe'],
shared_flow_dict['current-risky-flows-high'],
shared_flow_dict['current-risky-flows-medium'],
shared_flow_dict['current-risky-flows-low'],
shared_flow_dict['current-flows'] -
shared_flow_dict['current-risky-flows']],
COLOR_MAP['piechart-risky-flows'])]
@app.callback(output=[Output('graph-flows', 'figure'),
Output('graph-traces', 'data')],
inputs=[Input('tab-flow-graph-interval', 'n_intervals'),
Input('tab-flow-graph-interval', 'interval')],
state=[State('graph-traces', 'data')])
def tab_flow_update_graph(n, i, traces):
if traces is None:
traces = ([], [], [], [], [], [])
max_bins = 75
traces[0].append(shared_flow_dict['current-flows'])
traces[1].append(shared_flow_dict['current-risky-flows'])
traces[2].append(shared_flow_dict['current-midstream-flows'])
traces[3].append(shared_flow_dict['current-guessed-flows'])
traces[4].append(shared_flow_dict['current-not-detected-flows'])
traces[5].append(shared_flow_dict['current-flows']
- shared_flow_dict['current-detected-flows']
- shared_flow_dict['current-guessed-flows']
- shared_flow_dict['current-not-detected-flows'])
if len(traces[0]) > max_bins:
traces[0] = traces[0][1:]
traces[1] = traces[1][1:]
traces[2] = traces[2][1:]
traces[3] = traces[3][1:]
traces[4] = traces[4][1:]
traces[5] = traces[5][1:]
i /= 1000.0
x = list(range(max(n - max_bins, 0) * int(i), n * int(i), max(int(i), 0)))
if len(x) > 0 and x[0] > 60:
x = [round(t / 60, 2) for t in x]
x_div = 60
x_axis_title = 'Time (min)'
else:
x_div = 1
x_axis_title = 'Time (sec)'
min_x = max(0, x[0] if len(x) >= max_bins else 0)
max_x = max((max_bins * i) / x_div, x[max_bins - 1] if len(x) >= max_bins else 0)
lay = dict(
plot_bgcolor = '#082255',
paper_bgcolor = '#082255',
font={"color": "#fff"},
xaxis = {
'title': x_axis_title,
"showgrid": False,
"showline": False,
"fixedrange": True,
"tickmode": 'linear',
"tick0": round(max_bins / x_div, 2),
"dtick": round(max_bins / x_div, 2),
},
yaxis = {
'title': 'Flow Count',
"showgrid": False,
"showline": False,
"zeroline": False,
"fixedrange": True,
"tickmode": 'linear',
"dtick": 10,
},
uirevision=True,
autosize=True,
bargap=0.01,
bargroupgap=0,
hovermode="closest",
margin = {'b': 0, 'l': 0, 'r': 0, 't': 30, 'pad': 0},
legend = {'borderwidth': 0},
)
fig = go.Figure(layout=lay)
fig.update_xaxes(showgrid=True, gridwidth=1, gridcolor='#004D80', zeroline=True, zerolinewidth=1, range=[min_x, max_x])
fig.update_yaxes(showgrid=True, gridwidth=1, gridcolor='#004D80', zeroline=True, zerolinewidth=1)
fig.add_trace(go.Scatter(
x=x,
y=traces[0],
name='Current Active Flows',
mode='lines+markers',
line=COLOR_MAP['graph-flows']['Current Active Flows'],
))
fig.add_trace(go.Scatter(
x=x,
y=traces[1],
name='Current Risky Flows',
mode='lines+markers',
line=COLOR_MAP['graph-flows']['Current Risky Flows'],
))
fig.add_trace(go.Scatter(
x=x,
y=traces[2],
name='Current Midstream Flows',
mode='lines+markers',
line=COLOR_MAP['graph-flows']['Current Midstream Flows'],
))
fig.add_trace(go.Scatter(
x=x,
y=traces[3],
name='Current Guessed Flows',
mode='lines+markers',
line=COLOR_MAP['graph-flows']['Current Guessed Flows'],
))
fig.add_trace(go.Scatter(
x=x,
y=traces[4],
name='Current Not-Detected Flows',
mode='lines+markers',
line=COLOR_MAP['graph-flows']['Current Not-Detected Flows'],
))
fig.add_trace(go.Scatter(
x=x,
y=traces[5],
name='Current Unclassified Flows',
mode='lines+markers',
line=COLOR_MAP['graph-flows']['Current Unclassified Flows'],
))
return [fig, traces]
@app.callback(output=[Output('piechart-events', 'figure')],
inputs=[Input('tab-other-default-interval', 'n_intervals')])
def tab_other_update_components(n):
return [build_piechart(['Base', 'Daemon', 'Packet',
'Flow New', 'Flow Update', 'Flow End', 'Flow Idle',
'Flow Detection', 'Flow Detection-Updates', 'Flow Guessed', 'Flow Not-Detected'],
[shared_flow_dict['total-base-events'],
shared_flow_dict['total-daemon-events'],
shared_flow_dict['total-packet-events'],
shared_flow_dict['total-flow-new-events'],
shared_flow_dict['total-flow-update-events'],
shared_flow_dict['total-flow-end-events'],
shared_flow_dict['total-flow-idle-events'],
shared_flow_dict['total-flow-detected-events'],
shared_flow_dict['total-flow-detection-update-events'],
shared_flow_dict['total-flow-guessed-events'],
shared_flow_dict['total-flow-not-detected-events']])]
def web_worker(mp_shared_flow_dict, listen_host, listen_port):
global shared_flow_dict
shared_flow_dict = mp_shared_flow_dict
try:
app.run_server(debug=False, host=listen_host, port=listen_port)
except KeyboardInterrupt:
pass

3
examples/py-flow-dashboard/requirements.txt Normal file
View File

@@ -0,0 +1,3 @@
dash
dash_daq
Werkzeug==2.0

304
examples/py-flow-info/flow-info.py
View File

@@ -8,7 +8,6 @@ import datetime
sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId')
sys.path.append(os.path.dirname(sys.argv[0]))
sys.path.append(sys.base_prefix + '/share/nDPId')
import nDPIsrvd
from nDPIsrvd import nDPIsrvdSocket, TermColor
@@ -32,12 +31,12 @@ def set_attr_if_not_set(some_object, attr_name, value):
class Stats:
def __init__(self, nDPIsrvd_sock):
self.statusbar_enabled = True
self.start_time = time.time()
self.nsock = nDPIsrvd_sock
self.last_status_length = 0
self.avg_xfer_json_bytes = 0.0
self.expired_tot_l4_payload_len = 0
self.expired_avg_l4_payload_len = 0
self.total_flows = 0
self.risky_flows = 0
self.midstream_flows = 0
@@ -47,14 +46,11 @@ class Stats:
self.json_lines = 0
self.spinner_state = 0
def disableStatusbar(self):
self.statusbar_enabled = False
def updateSpinner(self):
if self.current_time + 0.25 <= time.time():
self.spinner_state += 1
def __getSpinner(self):
def getSpinner(self):
#spinner_states = ['-', '\\', '|', '/']
#spinner_states = ['▉', '▊', '▋', '▌', '▍', '▎', '▏', '▎', '▍', '▌', '▋', '▊', '▉']
spinner_states = ['←', '↖', '↑', '↗', '→', '↘', '↓', '↙']
@@ -63,12 +59,12 @@ class Stats:
#spinner_states = ['┤', '┘', '┴', '└', '├', '┌', '┬', '┐']
return spinner_states[self.spinner_state % len(spinner_states)]
def __getDataFromJson(self, json_dict, current_flow):
def getDataFromJson(self, json_dict, current_flow):
if current_flow is None:
return
set_attr_from_dict(current_flow, json_dict, 'flow_src_tot_l4_payload_len', 0)
set_attr_from_dict(current_flow, json_dict, 'flow_dst_tot_l4_payload_len', 0)
set_attr_from_dict(current_flow, json_dict, 'flow_tot_l4_payload_len', 0)
set_attr_from_dict(current_flow, json_dict, 'flow_avg_l4_payload_len', 0)
if 'ndpi' in json_dict:
set_attr_from_dict(current_flow, json_dict['ndpi'], 'flow_risk', {})
else:
@@ -91,21 +87,23 @@ class Stats:
self.json_lines += 1
self.current_time = time.time()
self.avg_xfer_json_bytes = self.nsock.received_bytes / (self.current_time - self.start_time)
self.__getDataFromJson(json_dict, current_flow)
self.getDataFromJson(json_dict, current_flow)
def updateOnCleanup(self, current_flow):
self.total_flows += 1
self.expired_tot_l4_payload_len += current_flow.flow_src_tot_l4_payload_len + current_flow.flow_dst_tot_l4_payload_len
self.expired_tot_l4_payload_len += current_flow.flow_tot_l4_payload_len
self.expired_avg_l4_payload_len += current_flow.flow_avg_l4_payload_len
self.risky_flows += 1 if len(current_flow.flow_risk) > 0 else 0
self.midstream_flows += 1 if current_flow.midstream != 0 else 0
self.guessed_flows += 1 if current_flow.guessed != 0 else 0
self.not_detected_flows += 1 if current_flow.not_detected != 0 else 0
def __getStatsFromFlowMgr(self):
def getStatsFromFlowMgr(self):
alias_count = 0
source_count = 0
flow_count = 0
flow_tot_l4_payload_len = 0.0
flow_avg_l4_payload_len = 0.0
risky = 0
midstream = 0
guessed = 0
@@ -120,74 +118,45 @@ class Stats:
flow_count += 1
current_flow = instances[alias][source].flows[flow_id]
try:
flow_src_tot_l4_payload_len = current_flow.flow_src_tot_l4_payload_len
flow_dst_tot_l4_payload_len = current_flow.flow_dst_tot_l4_payload_len
flow_risk = current_flow.flow_risk
midstream = current_flow.midstream
guessed = current_flow.guessed
not_detected = current_flow.not_detected
except AttributeError:
flow_src_tot_l4_payload_len = 0
flow_dst_tot_l4_payload_len = 0
flow_risk = []
midstream = 0
guessed = 0
not_detected = 0
flow_tot_l4_payload_len += flow_src_tot_l4_payload_len + flow_dst_tot_l4_payload_len
risky += 1 if len(flow_risk) > 0 else 0
midstream += 1 if midstream != 0 else 0
guessed += 1 if guessed != 0 else 0
not_detected = 1 if not_detected != 0 else 0
flow_tot_l4_payload_len += current_flow.flow_tot_l4_payload_len
flow_avg_l4_payload_len += current_flow.flow_avg_l4_payload_len
risky += 1 if len(current_flow.flow_risk) > 0 else 0
midstream += 1 if current_flow.midstream != 0 else 0
guessed += 1 if current_flow.guessed != 0 else 0
not_detected = 1 if current_flow.not_detected != 0 else 0
return alias_count, source_count, flow_count, \
flow_tot_l4_payload_len, \
flow_tot_l4_payload_len, flow_avg_l4_payload_len, \
risky, midstream, guessed, not_detected
@staticmethod
def prettifyBytes(bytes_received, is_byte_unit = True):
if not is_byte_unit:
size_names = ['', 'K', 'M', 'G', 'T']
divisor = 1000
else:
size_names = ['B', 'KiB', 'MiB', 'GiB', 'TiB']
divisor = 1024
def prettifyBytes(bytes_received):
size_names = ['B', 'KB', 'MB', 'GB', 'TB']
if bytes_received == 0:
i = 0
else:
i = min(int(math.floor(math.log(bytes_received, divisor))), len(size_names) - 1)
p = math.pow(divisor, i)
i = min(int(math.floor(math.log(bytes_received, 1024))), len(size_names) - 1)
p = math.pow(1024, i)
s = round(bytes_received / p, 2)
if not is_byte_unit:
return '{:.0f}{}'.format(s, ' ' + size_names[i] if len(size_names[i]) > 0 else size_names[i])
else:
return '{:.2f} {}'.format(s, size_names[i])
return '{:.2f} {}'.format(s, size_names[i])
def resetStatus(self):
if self.statusbar_enabled is False:
return
sys.stdout.write('\r' + str(' ' * self.last_status_length) + '\r')
sys.stdout.flush()
def printStatus(self):
if self.statusbar_enabled is False:
return
alias_count, source_count, flow_count, \
tot_l4_payload_len, \
risky, midstream, guessed, not_detected = self.__getStatsFromFlowMgr()
tot_l4_payload_len, avg_l4_payload_len, \
risky, midstream, guessed, not_detected = self.getStatsFromFlowMgr()
out_str = '\r[n|tot|avg JSONs: {}|{}|{}/s] [tot l4: {}] ' \
out_str = '\r[n|tot|avg JSONs: {}|{}|{}/s] [tot|avg l4: {}|{}] ' \
'[lss|srcs: {}|{}] ' \
'[flws|rsky|mdstrm|!dtctd|gssd: {}|{}|{}|{}|{} / {}|{}|{}|{}|{}] [{}]' \
''.format(self.json_lines,
Stats.prettifyBytes(self.nsock.received_bytes),
Stats.prettifyBytes(self.avg_xfer_json_bytes),
Stats.prettifyBytes(tot_l4_payload_len + self.expired_tot_l4_payload_len),
Stats.prettifyBytes(avg_l4_payload_len + self.expired_avg_l4_payload_len),
alias_count, source_count,
flow_count, risky, midstream, not_detected, guessed,
flow_count + self.total_flows,
@@ -195,7 +164,7 @@ class Stats:
midstream + self.midstream_flows,
not_detected + self.not_detected_flows,
guessed + self.guessed_flows,
self.__getSpinner())
self.getSpinner())
self.last_status_length = len(out_str) - 1 # '\r'
sys.stdout.write(out_str)
@@ -219,7 +188,7 @@ def checkEventFilter(json_dict):
'guessed': args.guessed, 'detected': args.detected,
'detection-update': args.detection_update,
'not-detected': args.not_detected,
'update': args.update, 'analyse': args.analyse}
'update': args.update}
if flow_events[json_dict['flow_event_name']] is True:
return True
@@ -257,19 +226,9 @@ def onFlowCleanup(instance, current_flow, global_user_data):
return True
def limitFloatValue(value, fmt, limit):
if float(value) < float(limit) and float(value) > 0.0:
return '<' + str(fmt).format(limit)
else:
return ' ' + str(fmt).format(value)
def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
stats = global_user_data
stats.update(json_dict, current_flow)
if 'packet_event_id' in json_dict:
return True
stats.resetStatus()
instance_and_source = ''
@@ -284,27 +243,26 @@ def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
basic_daemon_event_prefix = ''
timestamp = ''
if args.print_timestamp is True:
if 'thread_ts_usec' in json_dict:
if 'thread_ts_msec' in json_dict:
timestamp += '[{}]'.format(time.strftime('%H:%M:%S',
time.localtime(nDPIsrvd.toSeconds(json_dict['thread_ts_usec']))))
elif 'global_ts_usec' in json_dict:
time.localtime(json_dict['thread_ts_msec'] / 1000)))
elif 'global_ts_msec' in json_dict:
timestamp += '[{}]'.format(time.strftime('%H:%M:%S',
time.localtime(nDPIsrvd.toSeconds(json_dict['global_ts_usec']))))
time.localtime(json_dict['global_ts_msec'] / 1000)))
first_seen = ''
if args.print_first_seen is True:
basic_daemon_event_prefix += ' ' * 11
if 'flow_first_seen' in json_dict:
first_seen = '[' + prettifyTimediff(nDPIsrvd.toSeconds(json_dict['flow_first_seen']),
nDPIsrvd.toSeconds(json_dict['thread_ts_usec'])) + ']'
first_seen = '[' + prettifyTimediff(json_dict['flow_first_seen'] / 1000,
json_dict['thread_ts_msec'] / 1000) + ']'
last_seen = ''
if args.print_last_seen is True:
basic_daemon_event_prefix += ' ' * 11
if current_flow is not None:
flow_last_seen = nDPIsrvd.FlowManager.getLastPacketTime(instance, current_flow.flow_id, json_dict)
last_seen = '[' + prettifyTimediff(nDPIsrvd.toSeconds(flow_last_seen),
nDPIsrvd.toSeconds(json_dict['thread_ts_usec'])) + ']'
if 'flow_last_seen' in json_dict:
last_seen = '[' + prettifyTimediff(json_dict['flow_last_seen'] / 1000,
json_dict['thread_ts_msec'] / 1000) + ']'
if 'daemon_event_id' in json_dict:
if json_dict['daemon_event_name'] == 'status':
@@ -329,9 +287,9 @@ def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
stats.printStatus()
return True
if 'error_event_id' in json_dict:
print('{}{}{} {}: {} [{}/{}]'.format(timestamp, basic_daemon_event_prefix, instance_and_source,
print('{}{}{} {}: {}'.format(timestamp, basic_daemon_event_prefix, instance_and_source,
prettifyEvent([TermColor.FAIL, TermColor.BLINK], 15, 'ERROR-EVENT'),
json_dict['error_event_name'], json_dict['threshold_n'], json_dict['threshold_n_max']))
json_dict['error_event_name']))
stats.printStatus()
return True
elif 'flow_event_id' not in json_dict:
@@ -343,50 +301,24 @@ def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
return True
ndpi_proto_categ_breed = ''
next_lines = []
ndpi_frisk = ''
if 'ndpi' in json_dict:
ndpi_proto_categ_breed += ' '
if 'proto' in json_dict['ndpi']:
if args.ignore_protocol is not None:
for proto in args.ignore_protocol:
if json_dict['ndpi']['proto'].lower().startswith(proto.lower()) is True:
stats.printStatus()
return True
ndpi_proto_categ_breed += '[' + str(json_dict['ndpi']['proto']) + ']'
if 'proto_by_ip' in json_dict['ndpi']:
if args.ignore_ip_protocol is not None:
for proto in args.ignore_ip_protocol:
if json_dict['ndpi']['proto_by_ip'].lower().startswith(proto.lower()) is True:
stats.printStatus()
return True
ndpi_proto_categ_breed += '[' + str(json_dict['ndpi']['proto_by_ip']) + ']'
if 'category' in json_dict['ndpi']:
if args.ignore_category is not None:
for cat in args.ignore_category:
if json_dict['ndpi']['category'].lower().startswith(cat.lower()) is True:
stats.printStatus()
return True
ndpi_proto_categ_breed += '[' + str(json_dict['ndpi']['category']) + ']'
if 'breed' in json_dict['ndpi']:
if args.ignore_breed is not None:
for breed in args.ignore_breed:
if json_dict['ndpi']['breed'].lower().startswith(breed.lower()) is True:
stats.printStatus()
return True
ndpi_proto_categ_breed += '[' + str(json_dict['ndpi']['breed']) + ']'
if 'flow_risk' in json_dict['ndpi'] and args.hide_risk_info == False:
if 'flow_risk' in json_dict['ndpi']:
severity = 0
cnt = 0
next_lines += ['']
for key in json_dict['ndpi']['flow_risk']:
next_lines[0] += str(json_dict['ndpi']['flow_risk'][key]['risk']) + ', '
ndpi_frisk += str(json_dict['ndpi']['flow_risk'][key]['risk']) + ', '
if json_dict['ndpi']['flow_risk'][key]['severity'] == 'Low':
severity = max(severity, 1)
elif json_dict['ndpi']['flow_risk'][key]['severity'] == 'Medium':
@@ -408,10 +340,7 @@ def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
else:
color = ''
if severity >= args.min_risk_severity:
next_lines[0] = '{}{}{}: {}'.format(color, 'RISK', TermColor.END, next_lines[0][:-2])
else:
del next_lines[0]
ndpi_frisk = '{}{}{}: {}'.format(color, 'RISK', TermColor.END, ndpi_frisk[:-2])
line_suffix = ''
flow_event_name = ''
@@ -422,65 +351,15 @@ def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
elif json_dict['flow_event_name'] == 'not-detected':
flow_event_name += '{}{:>16}{}'.format(TermColor.WARNING + TermColor.BOLD + TermColor.BLINK,
json_dict['flow_event_name'], TermColor.END)
elif json_dict['flow_event_name'] == 'analyse':
flow_event_name += '{}{:>16}{}'.format(TermColor.WARNING,
json_dict['flow_event_name'], TermColor.END)
if args.print_analyse_results is True:
next_lines = [' {:>10}|{:>10}|{:>10}|{:>10}|{:>17}|{:>9}'.format(
'min', 'max', 'avg', 'stddev', 'variance', 'entropy')]
next_lines += ['[IAT.........: {}|{}|{}|{}|{}|{}]'.format(
limitFloatValue(nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['min']),
'{:>9.3f}', 0.001),
limitFloatValue(nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['max']),
'{:>9.3f}', 0.001),
limitFloatValue(nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['avg']),
'{:>9.3f}', 0.001),
limitFloatValue(nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['stddev']),
'{:>9.3f}', 0.001),
limitFloatValue(nDPIsrvd.toSeconds(json_dict['data_analysis']['iat']['var']),
'{:>16.3f}', 0.001),
limitFloatValue(json_dict['data_analysis']['iat']['ent'],
'{:>8.3f}', 0.001)
)]
next_lines += ['']
next_lines[-1] += '[PKTLEN......: {}|{}|{}|{}|{}|{}]'.format(
limitFloatValue(json_dict['data_analysis']['pktlen']['min'], '{:>9.3f}', 0.001),
limitFloatValue(json_dict['data_analysis']['pktlen']['max'], '{:>9.3f}', 0.001),
limitFloatValue(json_dict['data_analysis']['pktlen']['avg'], '{:>9.3f}', 0.001),
limitFloatValue(json_dict['data_analysis']['pktlen']['stddev'],
'{:>9.3f}', 0.001),
limitFloatValue(json_dict['data_analysis']['pktlen']['var'], '{:>16.3f}', 0.001),
limitFloatValue(json_dict['data_analysis']['pktlen']['ent'], '{:>8.3f}', 0.001)
)
next_lines += ['']
next_lines[-1] += '[BINS(c->s)..: {}]'.format(','.join([str(n) for n in json_dict['data_analysis']['bins']['c_to_s']]))
next_lines += ['']
next_lines[-1] += '[BINS(s->c)..: {}]'.format(','.join([str(n) for n in json_dict['data_analysis']['bins']['s_to_c']]))
next_lines += ['']
next_lines[-1] += '[DIRECTIONS..: {}]'.format(','.join([str(n) for n in json_dict['data_analysis']['directions']]))
next_lines += ['']
iats = ''
for n in json_dict['data_analysis']['iat']['data']:
iats += '{:.1f},'.format(n / 1000.0)
iats = iats[:-1]
next_lines[-1] += '[IATS(ms)....: {}]'.format(iats)
next_lines += ['']
next_lines[-1] += '[PKTLENS.....: {}]'.format(','.join([str(n) for n in json_dict['data_analysis']['pktlen']['data']]))
next_lines += ['']
ents = ''
for n in json_dict['data_analysis']['entropies']:
ents += '{:.1f},'.format(n)
ents = ents[:-1]
next_lines[-1] += '[ENTROPIES...: {}]'.format(ents)
else:
if json_dict['flow_event_name'] == 'new':
line_suffix = ''
if json_dict['midstream'] != 0:
line_suffix += ' [{}]'.format(TermColor.WARNING + TermColor.BLINK + 'MIDSTREAM' + TermColor.END)
line_suffix += '[{}] '.format(TermColor.WARNING + TermColor.BLINK + 'MIDSTREAM' + TermColor.END)
if args.ipwhois is True:
src_whois = whois(json_dict['src_ip'].lower())
dst_whois = whois(json_dict['dst_ip'].lower())
line_suffix += ' ['
line_suffix += '['
if src_whois is not None:
line_suffix += '{}'.format(src_whois)
if dst_whois is not None:
@@ -492,54 +371,17 @@ def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
line_suffix += ']'
flow_event_name += '{}{:>16}{}'.format(flow_active_color, json_dict['flow_event_name'], TermColor.END)
if 'ndpi' in json_dict and 'hostname' in json_dict['ndpi']:
if args.ignore_hostname is not None:
for hostname in args.ignore_hostname:
if json_dict['ndpi']['hostname'].lower().endswith(hostname.lower()) is True:
stats.printStatus()
return True
if args.print_hostname is True:
line_suffix += '[{}]'.format(json_dict['ndpi']['hostname'])
if args.skip_empty is True:
if json_dict['flow_src_tot_l4_payload_len'] == 0 or json_dict['flow_dst_tot_l4_payload_len'] == 0:
stats.printStatus()
return True
if args.print_bytes is True:
src_color = ''
dst_color = ''
tot_color = ''
if json_dict['flow_src_tot_l4_payload_len'] >= 1 * 1024 * 1024:
tot_color = src_color = TermColor.HINT
if json_dict['flow_src_tot_l4_payload_len'] >= 1 * 1024 * 1024 * 1024:
src_color += TermColor.BOLD + TermColor.BLINK
if json_dict['flow_dst_tot_l4_payload_len'] >= 1 * 1024 * 1024:
tot_color = dst_color = TermColor.HINT
if json_dict['flow_dst_tot_l4_payload_len'] >= 1 * 1024 * 1024 * 1024:
dst_color += TermColor.BOLD + TermColor.BLINK
line_suffix += '[' + src_color + Stats.prettifyBytes(json_dict['flow_src_tot_l4_payload_len']) + TermColor.END + ']' \
'[' + dst_color + Stats.prettifyBytes(json_dict['flow_dst_tot_l4_payload_len']) + TermColor.END +']' \
'[' + tot_color + Stats.prettifyBytes(json_dict['flow_src_tot_l4_payload_len'] + \
json_dict['flow_dst_tot_l4_payload_len']) + TermColor.END + ']'
if args.print_packets is True:
line_suffix += '[' + Stats.prettifyBytes(json_dict['flow_src_packets_processed'], False) + ']' \
'[' + Stats.prettifyBytes(json_dict['flow_dst_packets_processed'], False) + ']'
if json_dict['l3_proto'] == 'ip4':
print('{}{}{}{}{}: [{:.>6}]{} [{}][{:.>5}] [{:.>15}]{} -> [{:.>15}]{}{}{}' \
print('{}{}{}{}{}: [{:.>6}] [{}][{:.>5}] [{:.>15}]{} -> [{:.>15}]{} {}{}' \
''.format(timestamp, first_seen, last_seen, instance_and_source, flow_event_name,
json_dict['flow_id'],
'[{:.>4}]'.format(json_dict['vlan_id']) if 'vlan_id' in json_dict else '',
json_dict['l3_proto'], json_dict['l4_proto'],
json_dict['flow_id'], json_dict['l3_proto'], json_dict['l4_proto'],
json_dict['src_ip'].lower(),
'[{:.>5}]'.format(json_dict['src_port']) if 'src_port' in json_dict else '',
json_dict['dst_ip'].lower(),
'[{:.>5}]'.format(json_dict['dst_port']) if 'dst_port' in json_dict else '',
ndpi_proto_categ_breed, line_suffix))
elif json_dict['l3_proto'] == 'ip6':
print('{}{}{}{}{}: [{:.>6}] [{}][{:.>5}] [{:.>39}]{} -> [{:.>39}]{}{}{}' \
print('{}{}{}{}{}: [{:.>6}] [{}][{:.>5}] [{:.>39}]{} -> [{:.>39}]{} {}{}' \
''.format(timestamp, first_seen, last_seen, instance_and_source, flow_event_name,
json_dict['flow_id'], json_dict['l3_proto'], json_dict['l4_proto'],
json_dict['src_ip'].lower(),
@@ -550,38 +392,24 @@ def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
else:
raise RuntimeError('unsupported l3 protocol: {}'.format(json_dict['l3_proto']))
for line in next_lines:
if len(ndpi_frisk) > 0:
print('{}{}{}{}{:>18}{}'.format(timestamp, first_seen, last_seen,
instance_and_source, '', line))
instance_and_source, '', ndpi_frisk))
stats.printStatus()
return True
if __name__ == '__main__':
argparser = nDPIsrvd.defaultArgumentParser('Prettify and print events using the nDPIsrvd Python interface.', True)
argparser.add_argument('--no-color', action='store_true', default=False,
help='Disable all terminal colors.')
argparser.add_argument('--no-blink', action='store_true', default=False,
help='Disable all blink effects.')
argparser.add_argument('--no-statusbar', action='store_true', default=False,
help='Disable informational status bar.')
argparser = nDPIsrvd.defaultArgumentParser('Prettify and print events using the nDPIsrvd Python interface.')
argparser.add_argument('--hide-instance-info', action='store_true', default=False,
help='Hide instance Alias/Source prefixed every line.')
argparser.add_argument('--hide-risk-info', action='store_true', default=False,
help='Skip printing risks.')
argparser.add_argument('--print-timestamp', action='store_true', default=False,
help='Print received event timestamps.')
argparser.add_argument('--print-first-seen', action='store_true', default=False,
help='Print first seen flow time diff.')
argparser.add_argument('--print-last-seen', action='store_true', default=False,
help='Print last seen flow time diff.')
argparser.add_argument('--print-bytes', action='store_true', default=False,
help='Print received/transmitted source/dest bytes for every flow.')
argparser.add_argument('--print-packets', action='store_true', default=False,
help='Print received/transmitted source/dest packets for every flow.')
argparser.add_argument('--skip-empty', action='store_true', default=False,
help='Do not print flows that did not carry any layer7 payload.')
argparser.add_argument('--guessed', action='store_true', default=False, help='Print only guessed flow events.')
argparser.add_argument('--not-detected', action='store_true', default=False, help='Print only undetected flow events.')
argparser.add_argument('--detected', action='store_true', default=False, help='Print only detected flow events.')
@@ -592,55 +420,27 @@ if __name__ == '__main__':
argparser.add_argument('--end', action='store_true', default=False, help='Print only end flow events.')
argparser.add_argument('--idle', action='store_true', default=False, help='Print only idle flow events.')
argparser.add_argument('--update', action='store_true', default=False, help='Print only update flow events.')
argparser.add_argument('--analyse', action='store_true', default=False, help='Print only analyse flow events.')
argparser.add_argument('--detection', action='store_true', default=False, help='Print only detected/guessed/not-detected flow events.')
argparser.add_argument('--detection', action='store_true', default=False, help='Print only detected/detection-update flow events.')
argparser.add_argument('--ipwhois', action='store_true', default=False, help='Use Python-IPWhois to print additional location information.')
argparser.add_argument('--print-hostname', action='store_true', default=False, help='Print detected hostnames if available.')
argparser.add_argument('--print-analyse-results', action='store_true', default=False,
help='Print detailed results of analyse events.')
argparser.add_argument('--ignore-protocol', action='append', help='Ignore printing lines with a certain protocol.')
argparser.add_argument('--ignore-ip-protocol', action='append', help='Ignore printing lines with a certain IP protocol.')
argparser.add_argument('--ignore-category', action='append', help='Ignore printing lines with a certain category.')
argparser.add_argument('--ignore-breed', action='append', help='Ignore printing lines with a certain breed.')
argparser.add_argument('--ignore-hostname', action='append', help='Ignore printing lines with a certain hostname.')
argparser.add_argument('--min-risk-severity', action='store', type=int, default=0, help='Print only risks with a risk severity greater or equal to the given argument')
args = argparser.parse_args()
if args.no_color is True:
TermColor.disableColor()
if args.no_blink is True:
TermColor.disableBlink()
if args.ipwhois is True:
import dns, ipwhois
whois_db = dict()
if args.detection is True:
args.detected = True
args.guessed = True
args.not_detected = True
address = nDPIsrvd.validateAddress(args)
sys.stderr.write('Recv buffer size: {}\n'.format(nDPIsrvd.NETWORK_BUFFER_MAX_SIZE))
sys.stderr.write('Connecting to {} ..\n'.format(address[0]+':'+str(address[1]) if type(address) is tuple else address))
nsock = nDPIsrvdSocket()
nDPIsrvd.prepareJsonFilter(args, nsock)
nsock.connect(address)
nsock.timeout(1.0)
stats = Stats(nsock)
if args.no_statusbar is True:
stats.disableStatusbar()
while True:
try:
nsock.loop(onJsonLineRecvd, onFlowCleanup, stats)
except nDPIsrvd.SocketConnectionBroken as err:
sys.stderr.write('\n{}\n'.format(err))
break
except KeyboardInterrupt:
print('\n\nKeyboard Interrupt: cleaned up {} flows.'.format(len(nsock.shutdown())))
break

3
examples/py-flow-muliprocess/py-flow-multiprocess.py
View File

@@ -6,8 +6,7 @@ import sys
sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId')
sys.path.append(os.path.dirname(sys.argv[0]))
sys.path.append(sys.base_prefix + '/share/nDPId')
sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId')
import nDPIsrvd
from nDPIsrvd import nDPIsrvdSocket

142
examples/py-ja3-checker/py-ja3-checker.py Executable file
View File

@@ -0,0 +1,142 @@
#!/usr/bin/env python3
import io
import json
import os
import pandas
import requests
import sys
import time
sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId')
sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId')
import nDPIsrvd
from nDPIsrvd import nDPIsrvdSocket
global ja3_fps
ja3_fps = dict()
# 1 hour = 3600 sec/hour = (60 minutes/hour) * (60 seconds/minute)
JA3_FP_MAX_AGE = 60 * 60
global ja3_bl
ja3_bl = None
global ja3_bl_printed
ja3_bl_printed = dict()
def downloadJA3Blacklist():
response = requests.get(
'https://sslbl.abuse.ch/blacklist/ja3_fingerprints.csv'
)
if response.status_code == 200:
global ja3_bl
ja3_bl = pandas.read_csv(io.StringIO(response.text), header=9)
return True
return False
def getBlacklisted(ja3_hash):
global ja3_bl
return ja3_bl[(ja3_bl['# ja3_md5'] == ja3_hash)]
def checkBlacklisted(ja3_hash):
if ja3_bl is None:
return
csv_entry = getBlacklisted(ja3_hash)
if not csv_entry.empty and ja3_hash not in ja3_bl_printed:
print('Found CSV JA3 blacklist entry:')
print(csv_entry)
ja3_bl_printed[ja3_hash] = True
class JA3ER(object):
def __init__(self, json_dict):
self.json = json_dict
self.last_checked = time.time()
def isTooOld(self):
current_time = time.time()
if current_time - self.last_checked >= JA3_FP_MAX_AGE:
return True
return False
def isJA3InfoTooOld(ja3_hash):
global ja3_fps
if ja3_hash in ja3_fps:
if ja3_fps[ja3_hash].isTooOld() is True:
print('Fingerprint {} too old, re-newing..'.format(ja3_hash))
return True
else:
return True
return False
def getInfoFromJA3ER(ja3_hash):
global ja3_fps
response = requests.get('https://ja3er.com/search/' + ja3_hash)
if response.status_code == 200:
ja3_fps[ja3_hash] = JA3ER(json.loads(response.text, strict=True))
if 'error' not in ja3_fps[ja3_hash].json:
print('Fingerprints for JA3 {}:'.format(ja3_hash))
for ua in ja3_fps[ja3_hash].json:
if 'User-Agent' in ua:
print('\tUser-Agent: {}\n'
'\t Last seen: {}, '
'Count: {}'.format(ua['User-Agent'],
ua['Last_seen'],
ua['Count']))
elif 'Comment' in ua:
print('\tComment...: {}\n'
'\t Reported: {}'
.format(ua['Comment'].replace('\r', '')
.replace('\n', ' '), ua['Reported']))
else:
print(ua)
else:
print('No fingerprint for JA3 {} found.'.format(ja3_hash))
def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
if 'tls' in json_dict and 'ja3' in json_dict['tls']:
if json_dict['tls']['client_requested_server_name'] == 'ja3er.com':
return True
if isJA3InfoTooOld(json_dict['tls']['ja3']) is True:
getInfoFromJA3ER(json_dict['tls']['ja3'])
if isJA3InfoTooOld(json_dict['tls']['ja3']) is True:
getInfoFromJA3ER(json_dict['tls']['ja3s'])
checkBlacklisted(json_dict['tls']['ja3'])
return True
if __name__ == '__main__':
argparser = nDPIsrvd.defaultArgumentParser()
args = argparser.parse_args()
address = nDPIsrvd.validateAddress(args)
sys.stderr.write('Recv buffer size: {}\n'
.format(nDPIsrvd.NETWORK_BUFFER_MAX_SIZE))
sys.stderr.write('Connecting to {} ..\n'
.format(address[0] + ':' +
str(address[1])
if type(address) is tuple else address))
if downloadJA3Blacklist() is False:
print('Could not download JA3 blacklist.')
nsock = nDPIsrvdSocket()
nsock.connect(address)
try:
nsock.loop(onJsonLineRecvd, None, None)
except nDPIsrvd.SocketConnectionBroken as err:
sys.stderr.write('\n{}\n'.format(err))
except KeyboardInterrupt:
print()

6
examples/py-json-stdout/json-stdout.py
View File

@@ -5,8 +5,7 @@ import sys
sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId')
sys.path.append(os.path.dirname(sys.argv[0]))
sys.path.append(sys.base_prefix + '/share/nDPId')
sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId')
import nDPIsrvd
from nDPIsrvd import nDPIsrvdSocket, TermColor
@@ -15,7 +14,7 @@ def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
return True
if __name__ == '__main__':
argparser = nDPIsrvd.defaultArgumentParser('Plain and simple nDPIsrvd JSON event printer with filter capabilities.', True)
argparser = nDPIsrvd.defaultArgumentParser()
args = argparser.parse_args()
address = nDPIsrvd.validateAddress(args)
@@ -23,6 +22,5 @@ if __name__ == '__main__':
sys.stderr.write('Connecting to {} ..\n'.format(address[0]+':'+str(address[1]) if type(address) is tuple else address))
nsock = nDPIsrvdSocket()
nDPIsrvd.prepareJsonFilter(args, nsock)
nsock.connect(address)
nsock.loop(onJsonLineRecvd, None, None)

384
examples/py-machine-learning/keras-autoencoder.py
View File

@@ -1,384 +0,0 @@
#!/usr/bin/env python3
import base64
import binascii
import datetime as dt
import math
import matplotlib.animation as ani
import matplotlib.pyplot as plt
import multiprocessing as mp
import numpy as np
import os
import queue
import sys
import tensorflow as tf
from tensorflow.keras import models, layers, preprocessing
from tensorflow.keras.layers import Embedding, Masking, Input, Dense
from tensorflow.keras.models import Model
from tensorflow.keras.utils import plot_model
from tensorflow.keras.losses import MeanSquaredError, KLDivergence
from tensorflow.keras.optimizers import Adam, SGD
from tensorflow.keras.callbacks import TensorBoard, EarlyStopping
sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId')
sys.path.append(os.path.dirname(sys.argv[0]))
sys.path.append(sys.base_prefix + '/share/nDPId')
import nDPIsrvd
from nDPIsrvd import nDPIsrvdSocket, TermColor
INPUT_SIZE = nDPIsrvd.nDPId_PACKETS_PLEN_MAX
LATENT_SIZE = 8
TRAINING_SIZE = 512
EPOCH_COUNT = 3
BATCH_SIZE = 16
LEARNING_RATE = 0.000001
ES_PATIENCE = 3
PLOT = False
PLOT_HISTORY = 100
TENSORBOARD = False
TB_LOGPATH = 'logs/' + dt.datetime.now().strftime("%Y%m%d-%H%M%S")
VAE_USE_KLDIV = False
VAE_USE_SGD = False
def generate_autoencoder():
# TODO: The current model does handle *each* packet separatly.
# But in fact, depending on the nDPId settings (nDPId_PACKETS_PER_FLOW_TO_SEND), packets can be in relation to each other.
# The accuracy may (or may not) improve significantly, but some of changes in the code are required.
input_i = Input(shape=(), name='input_i')
input_e = Embedding(input_dim=INPUT_SIZE, output_dim=INPUT_SIZE, mask_zero=True, name='input_e')(input_i)
masked_e = Masking(mask_value=0.0, name='masked_e')(input_e)
encoded_h1 = Dense(4096, activation='relu', name='encoded_h1')(masked_e)
encoded_h2 = Dense(2048, activation='relu', name='encoded_h2')(encoded_h1)
encoded_h3 = Dense(1024, activation='relu', name='encoded_h3')(encoded_h2)
encoded_h4 = Dense(512, activation='relu', name='encoded_h4')(encoded_h3)
encoded_h5 = Dense(128, activation='relu', name='encoded_h5')(encoded_h4)
encoded_h6 = Dense(64, activation='relu', name='encoded_h6')(encoded_h5)
encoded_h7 = Dense(32, activation='relu', name='encoded_h7')(encoded_h6)
latent = Dense(LATENT_SIZE, activation='relu', name='latent')(encoded_h7)
input_l = Input(shape=(LATENT_SIZE), name='input_l')
decoder_h1 = Dense(32, activation='relu', name='decoder_h1')(input_l)
decoder_h2 = Dense(64, activation='relu', name='decoder_h2')(decoder_h1)
decoder_h3 = Dense(128, activation='relu', name='decoder_h3')(decoder_h2)
decoder_h4 = Dense(512, activation='relu', name='decoder_h4')(decoder_h3)
decoder_h5 = Dense(1024, activation='relu', name='decoder_h5')(decoder_h4)
decoder_h6 = Dense(2048, activation='relu', name='decoder_h6')(decoder_h5)
decoder_h7 = Dense(4096, activation='relu', name='decoder_h7')(decoder_h6)
output_i = Dense(INPUT_SIZE, activation='sigmoid', name='output_i')(decoder_h7)
encoder = Model(input_e, latent, name='encoder')
decoder = Model(input_l, output_i, name='decoder')
return KLDivergence() if VAE_USE_KLDIV else MeanSquaredError(), \
SGD() if VAE_USE_SGD else Adam(learning_rate=LEARNING_RATE), \
Model(input_e, decoder(encoder(input_e)), name='VAE')
def compile_autoencoder():
loss, optimizer, autoencoder = generate_autoencoder()
autoencoder.compile(loss=loss, optimizer=optimizer, metrics=[])
return autoencoder
def get_autoencoder(load_from_file=None):
if load_from_file is None:
autoencoder = compile_autoencoder()
else:
autoencoder = models.load_model(load_from_file)
encoder_submodel = autoencoder.layers[1]
decoder_submodel = autoencoder.layers[2]
return encoder_submodel, decoder_submodel, autoencoder
def on_json_line(json_dict, instance, current_flow, global_user_data):
if 'packet_event_name' not in json_dict:
return True
if json_dict['packet_event_name'] != 'packet' and \
json_dict['packet_event_name'] != 'packet-flow':
return True
shutdown_event, training_event, padded_pkts, print_dots = global_user_data
if shutdown_event.is_set():
return False
try:
buf = base64.b64decode(json_dict['pkt'], validate=True)
except binascii.Error as err:
sys.stderr.write('\nBase64 Exception: {}\n'.format(str(err)))
sys.stderr.write('Affected JSON: {}\n'.format(str(json_dict)))
sys.stderr.flush()
return False
# Generate decimal byte buffer with valus from 0-255
int_buf = []
for v in buf:
int_buf.append(int(v))
mat = np.array([int_buf], dtype='float64')
# Normalize the values
mat = mat.astype('float64') / 255.0
# Mean removal
matmean = np.mean(mat, dtype='float64')
mat -= matmean
# Pad resulting matrice
buf = preprocessing.sequence.pad_sequences(mat, padding="post", maxlen=INPUT_SIZE, truncating='post', dtype='float64')
padded_pkts.put(buf[0])
#print(list(buf[0]))
if not training_event.is_set():
sys.stdout.write('.' * print_dots)
sys.stdout.flush()
print_dots = 1
else:
print_dots += 1
return True
def ndpisrvd_worker(address, shared_shutdown_event, shared_training_event, shared_packet_list):
nsock = nDPIsrvdSocket()
try:
nsock.connect(address)
print_dots = 1
nsock.loop(on_json_line, None, (shared_shutdown_event, shared_training_event, shared_packet_list, print_dots))
except nDPIsrvd.SocketConnectionBroken as err:
sys.stderr.write('\nnDPIsrvd-Worker Socket Error: {}\n'.format(err))
except KeyboardInterrupt:
sys.stderr.write('\n')
except Exception as err:
sys.stderr.write('\nnDPIsrvd-Worker Exception: {}\n'.format(str(err)))
sys.stderr.flush()
shared_shutdown_event.set()
def keras_worker(load_model, save_model, shared_shutdown_event, shared_training_event, shared_packet_queue, shared_plot_queue):
shared_training_event.set()
try:
encoder, _, autoencoder = get_autoencoder(load_model)
except Exception as err:
sys.stderr.write('Could not load Keras model from file: {}\n'.format(str(err)))
sys.stderr.flush()
encoder, _, autoencoder = get_autoencoder()
autoencoder.summary()
additional_callbacks = []
if TENSORBOARD is True:
tensorboard = TensorBoard(log_dir=TB_LOGPATH, histogram_freq=1)
additional_callbacks += [tensorboard]
early_stopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=ES_PATIENCE, restore_best_weights=True, start_from_epoch=0, verbose=0, mode='auto')
additional_callbacks += [early_stopping]
shared_training_event.clear()
try:
packets = list()
while not shared_shutdown_event.is_set():
try:
packet = shared_packet_queue.get(timeout=1)
except queue.Empty:
packet = None
if packet is None:
continue
packets.append(packet)
if len(packets) % TRAINING_SIZE == 0:
shared_training_event.set()
print('\nGot {} packets, training..'.format(len(packets)))
tmp = np.array(packets)
history = autoencoder.fit(
tmp, tmp, epochs=EPOCH_COUNT, batch_size=BATCH_SIZE,
validation_split=0.2,
shuffle=True,
callbacks=[additional_callbacks]
)
reconstructed_data = autoencoder.predict(tmp)
mse = np.mean(np.square(tmp - reconstructed_data))
reconstruction_accuracy = (1.0 / mse)
encoded_data = encoder.predict(tmp)
latent_activations = encoder.predict(tmp)
shared_plot_queue.put((reconstruction_accuracy, history.history['val_loss'], encoded_data[:, 0], encoded_data[:, 1], latent_activations))
packets.clear()
shared_training_event.clear()
except KeyboardInterrupt:
sys.stderr.write('\n')
except Exception as err:
if len(str(err)) == 0:
err = 'Unknown'
sys.stderr.write('\nKeras-Worker Exception: {}\n'.format(str(err)))
sys.stderr.flush()
if save_model is not None:
sys.stderr.write('Saving model to {}\n'.format(save_model))
sys.stderr.flush()
autoencoder.save(save_model)
try:
shared_shutdown_event.set()
except Exception:
pass
def plot_animate(i, shared_plot_queue, ax, xs, ys):
if not shared_plot_queue.empty():
accuracy, loss, encoded_data0, encoded_data1, latent_activations = shared_plot_queue.get(timeout=1)
epochs = len(loss)
loss_mean = sum(loss) / epochs
else:
return
(ax1, ax2, ax3, ax4) = ax
(ys1, ys2, ys3, ys4) = ys
if len(xs) == 0:
xs.append(epochs)
else:
xs.append(xs[-1] + epochs)
ys1.append(accuracy)
ys2.append(loss_mean)
xs = xs[-PLOT_HISTORY:]
ys1 = ys1[-PLOT_HISTORY:]
ys2 = ys2[-PLOT_HISTORY:]
ax1.clear()
ax1.plot(xs, ys1, '-')
ax2.clear()
ax2.plot(xs, ys2, '-')
ax3.clear()
ax3.scatter(encoded_data0, encoded_data1, marker='.')
ax4.clear()
ax4.imshow(latent_activations, cmap='viridis', aspect='auto')
ax1.set_xlabel('Epoch Count')
ax1.set_ylabel('Accuracy')
ax2.set_xlabel('Epoch Count')
ax2.set_ylabel('Validation Loss')
ax3.set_title('Latent Space')
ax4.set_title('Latent Space Heatmap')
ax4.set_xlabel('Latent Dimensions')
ax4.set_ylabel('Datapoints')
def plot_worker(shared_shutdown_event, shared_plot_queue):
try:
fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2)
fig.tight_layout()
ax1.set_xlabel('Epoch Count')
ax1.set_ylabel('Accuracy')
ax2.set_xlabel('Epoch Count')
ax2.set_ylabel('Validation Loss')
ax3.set_title('Latent Space')
ax4.set_title('Latent Space Heatmap')
ax4.set_xlabel('Latent Dimensions')
ax4.set_ylabel('Datapoints')
xs = []
ys1 = []
ys2 = []
ys3 = []
ys4 = []
ani.FuncAnimation(fig, plot_animate, fargs=(shared_plot_queue, (ax1, ax2, ax3, ax4), xs, (ys1, ys2, ys3, ys4)), interval=1000, cache_frame_data=False)
plt.subplots_adjust(left=0.05, right=0.95, top=0.95, bottom=0.05)
plt.margins(x=0, y=0)
plt.show()
except Exception as err:
sys.stderr.write('\nPlot-Worker Exception: {}\n'.format(str(err)))
sys.stderr.flush()
shared_shutdown_event.set()
return
if __name__ == '__main__':
sys.stderr.write('\b\n***************\n')
sys.stderr.write('*** WARNING ***\n')
sys.stderr.write('***************\n')
sys.stderr.write('\nThis is an unmature Autoencoder example.\n')
sys.stderr.write('Please do not rely on any of it\'s output!\n\n')
argparser = nDPIsrvd.defaultArgumentParser()
argparser.add_argument('--load-model', action='store',
help='Load a pre-trained model file.')
argparser.add_argument('--save-model', action='store',
help='Save the trained model to a file.')
argparser.add_argument('--training-size', action='store', type=int, default=TRAINING_SIZE,
help='Set the amount of captured packets required to start the training phase.')
argparser.add_argument('--batch-size', action='store', type=int, default=BATCH_SIZE,
help='Set the batch size used for the training phase.')
argparser.add_argument('--learning-rate', action='store', type=float, default=LEARNING_RATE,
help='Set the (initial) learning rate for the optimizer.')
argparser.add_argument('--plot', action='store_true', default=PLOT,
help='Show some model metrics using pyplot.')
argparser.add_argument('--plot-history', action='store', type=int, default=PLOT_HISTORY,
help='Set the history size of Line plots. Requires --plot')
argparser.add_argument('--tensorboard', action='store_true', default=TENSORBOARD,
help='Enable TensorBoard compatible logging callback.')
argparser.add_argument('--tensorboard-logpath', action='store', default=TB_LOGPATH,
help='TensorBoard logging path.')
argparser.add_argument('--use-sgd', action='store_true', default=VAE_USE_SGD,
help='Use SGD optimizer instead of Adam.')
argparser.add_argument('--use-kldiv', action='store_true', default=VAE_USE_KLDIV,
help='Use Kullback-Leibler loss function instead of Mean-Squared-Error.')
argparser.add_argument('--patience', action='store', type=int, default=ES_PATIENCE,
help='Epoch value for EarlyStopping. This value forces VAE fitting to if no improvment achieved.')
args = argparser.parse_args()
address = nDPIsrvd.validateAddress(args)
LEARNING_RATE = args.learning_rate
TRAINING_SIZE = args.training_size
BATCH_SIZE = args.batch_size
PLOT = args.plot
PLOT_HISTORY = args.plot_history
TENSORBOARD = args.tensorboard
TB_LOGPATH = args.tensorboard_logpath if args.tensorboard_logpath is not None else TB_LOGPATH
VAE_USE_SGD = args.use_sgd
VAE_USE_KLDIV = args.use_kldiv
ES_PATIENCE = args.patience
sys.stderr.write('Recv buffer size: {}\n'.format(nDPIsrvd.NETWORK_BUFFER_MAX_SIZE))
sys.stderr.write('Connecting to {} ..\n'.format(address[0]+':'+str(address[1]) if type(address) is tuple else address))
sys.stderr.write('PLOT={}, PLOT_HISTORY={}, LEARNING_RATE={}, TRAINING_SIZE={}, BATCH_SIZE={}\n\n'.format(PLOT, PLOT_HISTORY, LEARNING_RATE, TRAINING_SIZE, BATCH_SIZE))
mgr = mp.Manager()
shared_training_event = mgr.Event()
shared_training_event.clear()
shared_shutdown_event = mgr.Event()
shared_shutdown_event.clear()
shared_packet_queue = mgr.JoinableQueue()
shared_plot_queue = mgr.JoinableQueue()
nDPIsrvd_job = mp.Process(target=ndpisrvd_worker, args=(
address,
shared_shutdown_event,
shared_training_event,
shared_packet_queue
))
nDPIsrvd_job.start()
keras_job = mp.Process(target=keras_worker, args=(
args.load_model,
args.save_model,
shared_shutdown_event,
shared_training_event,
shared_packet_queue,
shared_plot_queue
))
keras_job.start()
if PLOT is True:
plot_job = mp.Process(target=plot_worker, args=(shared_shutdown_event, shared_plot_queue))
plot_job.start()
try:
shared_shutdown_event.wait()
except KeyboardInterrupt:
print('\nShutting down worker processess..')
if PLOT is True:
plot_job.terminate()
plot_job.join()
nDPIsrvd_job.terminate()
nDPIsrvd_job.join()
keras_job.join(timeout=3)
keras_job.terminate()

7
examples/py-machine-learning/requirements.txt
View File

@@ -1,7 +0,0 @@
joblib
tensorflow
scikit-learn
scipy
matplotlib
numpy
pandas

352
examples/py-machine-learning/sklearn-random-forest.py
View File

@@ -1,352 +0,0 @@
#!/usr/bin/env python3
import csv
import joblib
import matplotlib.pyplot
import numpy
import os
import pandas
import sklearn
import sklearn.ensemble
import sklearn.inspection
import sys
import time
sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId')
sys.path.append(os.path.dirname(sys.argv[0]))
sys.path.append(sys.base_prefix + '/share/nDPId')
import nDPIsrvd
from nDPIsrvd import nDPIsrvdSocket, TermColor
N_DIRS = 0
N_BINS = 0
ENABLE_FEATURE_IAT = False
ENABLE_FEATURE_PKTLEN = False
ENABLE_FEATURE_DIRS = True
ENABLE_FEATURE_BINS = True
PROTO_CLASSES = None
def getFeatures(json):
return [json['flow_src_packets_processed'],
json['flow_dst_packets_processed'],
json['flow_src_tot_l4_payload_len'],
json['flow_dst_tot_l4_payload_len']]
def getFeaturesFromArray(json, expected_len=0):
if type(json) is str:
dirs = numpy.fromstring(json, sep=',', dtype=int)
dirs = numpy.asarray(dirs, dtype=int).tolist()
elif type(json) is list:
dirs = json
else:
raise TypeError('Invalid type: {}.'.format(type(json)))
if expected_len > 0 and len(dirs) != expected_len:
raise RuntimeError('Invalid array length; Expected {}, Got {}.'.format(expected_len, len(dirs)))
return dirs
def getRelevantFeaturesCSV(line):
ret = list()
ret.extend(getFeatures(line));
if ENABLE_FEATURE_IAT is True:
ret.extend(getFeaturesFromArray(line['iat_data'], N_DIRS - 1))
if ENABLE_FEATURE_PKTLEN is True:
ret.extend(getFeaturesFromArray(line['pktlen_data'], N_DIRS))
if ENABLE_FEATURE_DIRS is True:
ret.extend(getFeaturesFromArray(line['directions'], N_DIRS))
if ENABLE_FEATURE_BINS is True:
ret.extend(getFeaturesFromArray(line['bins_c_to_s'], N_BINS))
ret.extend(getFeaturesFromArray(line['bins_s_to_c'], N_BINS))
return [ret]
def getRelevantFeaturesJSON(line):
ret = list()
ret.extend(getFeatures(line))
if ENABLE_FEATURE_IAT is True:
ret.extend(getFeaturesFromArray(line['data_analysis']['iat']['data'], N_DIRS - 1))
if ENABLE_FEATURE_PKTLEN is True:
ret.extend(getFeaturesFromArray(line['data_analysis']['pktlen']['data'], N_DIRS))
if ENABLE_FEATURE_DIRS is True:
ret.extend(getFeaturesFromArray(line['data_analysis']['directions'], N_DIRS))
if ENABLE_FEATURE_BINS is True:
ret.extend(getFeaturesFromArray(line['data_analysis']['bins']['c_to_s'], N_BINS))
ret.extend(getFeaturesFromArray(line['data_analysis']['bins']['s_to_c'], N_BINS) )
return [ret]
def getRelevantFeatureNames():
names = list()
names.extend(['flow_src_packets_processed', 'flow_dst_packets_processed',
'flow_src_tot_l4_payload_len', 'flow_dst_tot_l4_payload_len'])
if ENABLE_FEATURE_IAT is True:
for x in range(N_DIRS - 1):
names.append('iat_{}'.format(x))
if ENABLE_FEATURE_PKTLEN is True:
for x in range(N_DIRS):
names.append('pktlen_{}'.format(x))
if ENABLE_FEATURE_DIRS is True:
for x in range(N_DIRS):
names.append('dirs_{}'.format(x))
if ENABLE_FEATURE_BINS is True:
for x in range(N_BINS):
names.append('bins_c_to_s_{}'.format(x))
for x in range(N_BINS):
names.append('bins_s_to_c_{}'.format(x))
return names
def plotPermutatedImportance(model, X, y):
result = sklearn.inspection.permutation_importance(model, X, y, n_repeats=10, random_state=42, n_jobs=-1)
forest_importances = pandas.Series(result.importances_mean, index=getRelevantFeatureNames())
fig, ax = matplotlib.pyplot.subplots()
forest_importances.plot.bar(yerr=result.importances_std, ax=ax)
ax.set_title("Feature importances using permutation on full model")
ax.set_ylabel("Mean accuracy decrease")
fig.tight_layout()
matplotlib.pyplot.show()
def isProtoClass(proto_class, line):
if type(proto_class) != list or type(line) != str:
raise TypeError('Invalid type: {}/{}.'.format(type(proto_class), type(line)))
s = line.lower()
for x in range(len(proto_class)):
if s.startswith(proto_class[x].lower()) is True:
return x + 1
return 0
def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
if 'flow_event_name' not in json_dict:
return True
if json_dict['flow_event_name'] != 'analyse':
return True
if 'ndpi' not in json_dict:
return True
if 'proto' not in json_dict['ndpi']:
return True
#print(json_dict)
model, proto_class, disable_colors = global_user_data
try:
X = getRelevantFeaturesJSON(json_dict)
y = model.predict(X)
p = model.predict_log_proba(X)
if y[0] <= 0:
y_text = 'n/a'
else:
y_text = proto_class[y[0] - 1]
color_start = ''
color_end = ''
pred_failed = False
if disable_colors is False:
if json_dict['ndpi']['proto'].lower().startswith(y_text) is True:
color_start = TermColor.BOLD
color_end = TermColor.END
elif y_text not in proto_class and \
json_dict['ndpi']['proto'].lower() not in proto_class:
pass
else:
pred_failed = True
color_start = TermColor.WARNING + TermColor.BOLD
color_end = TermColor.END
probs = str()
for i in range(len(p[0])):
if json_dict['ndpi']['proto'].lower().startswith(proto_class[i - 1]) and disable_colors is False:
probs += '{}{:>2.1f}{}, '.format(TermColor.BOLD + TermColor.BLINK if pred_failed is True else '',
p[0][i], TermColor.END)
elif i == y[0]:
probs += '{}{:>2.1f}{}, '.format(color_start, p[0][i], color_end)
else:
probs += '{:>2.1f}, '.format(p[0][i])
probs = probs[:-2]
print('DPI Engine detected: {}{:>24}{}, Predicted: {}{:>24}{}, Probabilities: {}'.format(
color_start, json_dict['ndpi']['proto'].lower(), color_end,
color_start, y_text, color_end, probs))
if pred_failed is True:
pclass = isProtoClass(args.proto_class, json_dict['ndpi']['proto'].lower())
if pclass == 0:
msg = 'false positive'
else:
msg = 'false negative'
print('{:>46} {}{}{}'.format('[-]', TermColor.FAIL + TermColor.BOLD + TermColor.BLINK, msg, TermColor.END))
except Exception as err:
print('Got exception `{}\'\nfor json: {}'.format(err, json_dict))
return True
if __name__ == '__main__':
argparser = nDPIsrvd.defaultArgumentParser()
argparser.add_argument('--load-model', action='store',
help='Load a pre-trained model file.')
argparser.add_argument('--save-model', action='store',
help='Save the trained model to a file.')
argparser.add_argument('--csv', action='store',
help='Input CSV file generated with nDPIsrvd-analysed.')
argparser.add_argument('--proto-class', action='append', required=False,
help='nDPId protocol class of interest used for training and prediction. ' +
'Can be specified multiple times. Example: tls.youtube')
argparser.add_argument('--generate-feature-importance', action='store_true',
help='Generates the permutated feature importance with matplotlib.')
argparser.add_argument('--enable-iat', action='store_true', default=None,
help='Enable packet (I)nter (A)rrival (T)ime for learning and prediction.')
argparser.add_argument('--enable-pktlen', action='store_true', default=None,
help='Enable layer 4 packet lengths for learning and prediction.')
argparser.add_argument('--disable-dirs', action='store_true', default=None,
help='Disable packet directions for learning and prediction.')
argparser.add_argument('--disable-bins', action='store_true', default=None,
help='Disable packet length distribution for learning and prediction.')
argparser.add_argument('--disable-colors', action='store_true', default=False,
help='Disable any coloring.')
argparser.add_argument('--sklearn-jobs', action='store', type=int, default=1,
help='Number of sklearn processes during training.')
argparser.add_argument('--sklearn-estimators', action='store', type=int, default=1000,
help='Number of trees in the forest.')
argparser.add_argument('--sklearn-min-samples-leaf', action='store', type=int, default=0.0001,
help='The minimum number of samples required to be at a leaf node.')
argparser.add_argument('--sklearn-class-weight', default='balanced', const='balanced', nargs='?',
choices=['balanced', 'balanced_subsample'],
help='Weights associated with the protocol classes.')
argparser.add_argument('--sklearn-max-features', default='sqrt', const='sqrt', nargs='?',
choices=['sqrt', 'log2'],
help='The number of features to consider when looking for the best split.')
argparser.add_argument('--sklearn-max-depth', action='store', type=int, default=128,
help='The maximum depth of a tree.')
argparser.add_argument('--sklearn-verbosity', action='store', type=int, default=0,
help='Controls the verbosity of sklearn\'s random forest classifier.')
args = argparser.parse_args()
address = nDPIsrvd.validateAddress(args)
if args.csv is None and args.load_model is None:
sys.stderr.write('{}: Either `--csv` or `--load-model` required!\n'.format(sys.argv[0]))
sys.exit(1)
if args.csv is None and args.generate_feature_importance is True:
sys.stderr.write('{}: `--generate-feature-importance` requires `--csv`.\n'.format(sys.argv[0]))
sys.exit(1)
if args.proto_class is None or len(args.proto_class) == 0:
if args.csv is None and args.load_model is None:
sys.stderr.write('{}: `--proto-class` missing, no useful classification can be performed.\n'.format(sys.argv[0]))
else:
if args.load_model is not None:
sys.stderr.write('{}: `--proto-class` set, but you want to load an existing model.\n'.format(sys.argv[0]))
sys.exit(1)
if args.load_model is not None:
sys.stderr.write('{}: You are loading an existing model file. ' \
'Some --sklearn-* command line parameters won\'t have any effect!\n'.format(sys.argv[0]))
if args.enable_iat is not None:
sys.stderr.write('{}: `--enable-iat` set, but you want to load an existing model.\n'.format(sys.argv[0]))
sys.exit(1)
if args.enable_pktlen is not None:
sys.stderr.write('{}: `--enable-pktlen` set, but you want to load an existing model.\n'.format(sys.argv[0]))
sys.exit(1)
if args.disable_dirs is not None:
sys.stderr.write('{}: `--disable-dirs` set, but you want to load an existing model.\n'.format(sys.argv[0]))
sys.exit(1)
if args.disable_bins is not None:
sys.stderr.write('{}: `--disable-bins` set, but you want to load an existing model.\n'.format(sys.argv[0]))
sys.exit(1)
ENABLE_FEATURE_IAT = args.enable_iat if args.enable_iat is not None else ENABLE_FEATURE_IAT
ENABLE_FEATURE_PKTLEN = args.enable_pktlen if args.enable_pktlen is not None else ENABLE_FEATURE_PKTLEN
ENABLE_FEATURE_DIRS = args.disable_dirs if args.disable_dirs is not None else ENABLE_FEATURE_DIRS
ENABLE_FEATURE_BINS = args.disable_bins if args.disable_bins is not None else ENABLE_FEATURE_BINS
PROTO_CLASSES = args.proto_class
numpy.set_printoptions(formatter={'float_kind': "{:.1f}".format}, sign=' ')
numpy.seterr(divide = 'ignore')
if args.proto_class is not None:
for i in range(len(args.proto_class)):
args.proto_class[i] = args.proto_class[i].lower()
if args.load_model is not None:
sys.stderr.write('Loading model from {}\n'.format(args.load_model))
model, options = joblib.load(args.load_model)
ENABLE_FEATURE_IAT, ENABLE_FEATURE_PKTLEN, ENABLE_FEATURE_DIRS, ENABLE_FEATURE_BINS, args.proto_class = options
if args.csv is not None:
sys.stderr.write('Learning via CSV..\n')
with open(args.csv, newline='\n') as csvfile:
reader = csv.DictReader(csvfile, delimiter=',', quotechar='"')
X = list()
y = list()
for line in reader:
N_DIRS = len(getFeaturesFromArray(line['directions']))
N_BINS = len(getFeaturesFromArray(line['bins_c_to_s']))
break
for line in reader:
try:
X += getRelevantFeaturesCSV(line)
except RuntimeError as err:
print('Runtime Error: `{}\'\non line {}: {}'.format(err, reader.line_num - 1, line))
continue
except TypeError as err:
print('Type Error: `{}\'\non line {}: {}'.format(err, reader.line_num - 1, line))
continue
try:
y += [isProtoClass(args.proto_class, line['proto'])]
except TypeError as err:
X.pop()
print('Type Error: `{}\'\non line {}: {}'.format(err, reader.line_num - 1, line))
continue
sys.stderr.write('CSV data set contains {} entries.\n'.format(len(X)))
if args.load_model is None:
model = sklearn.ensemble.RandomForestClassifier(bootstrap=False,
class_weight = args.sklearn_class_weight,
n_jobs = args.sklearn_jobs,
n_estimators = args.sklearn_estimators,
verbose = args.sklearn_verbosity,
min_samples_leaf = args.sklearn_min_samples_leaf,
max_features = args.sklearn_max_features,
max_depth = args.sklearn_max_depth
)
options = (ENABLE_FEATURE_IAT, ENABLE_FEATURE_PKTLEN, ENABLE_FEATURE_DIRS, ENABLE_FEATURE_BINS, args.proto_class)
sys.stderr.write('Training model..\n')
model.fit(X, y)
if args.generate_feature_importance is True:
sys.stderr.write('Generating feature importance .. this may take some time\n')
plotPermutatedImportance(model, X, y)
if args.save_model is not None:
sys.stderr.write('Saving model to {}\n'.format(args.save_model))
joblib.dump([model, options], args.save_model)
print('ENABLE_FEATURE_PKTLEN: {}'.format(ENABLE_FEATURE_PKTLEN))
print('ENABLE_FEATURE_BINS..: {}'.format(ENABLE_FEATURE_BINS))
print('ENABLE_FEATURE_DIRS..: {}'.format(ENABLE_FEATURE_DIRS))
print('ENABLE_FEATURE_IAT...: {}'.format(ENABLE_FEATURE_IAT))
print('Map[*] -> [0]')
for x in range(len(args.proto_class)):
print('Map["{}"] -> [{}]'.format(args.proto_class[x], x + 1))
sys.stderr.write('Predicting realtime traffic..\n')
sys.stderr.write('Recv buffer size: {}\n'.format(nDPIsrvd.NETWORK_BUFFER_MAX_SIZE))
sys.stderr.write('Connecting to {} ..\n'.format(address[0]+':'+str(address[1]) if type(address) is tuple else address))
nsock = nDPIsrvdSocket()
nsock.connect(address)
nsock.loop(onJsonLineRecvd, None, (model, args.proto_class, args.disable_colors))

3
examples/py-schema-validation/py-schema-validation.py
View File

@@ -5,8 +5,7 @@ import sys
sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId')
sys.path.append(os.path.dirname(sys.argv[0]))
sys.path.append(sys.base_prefix + '/share/nDPId')
sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId')
import nDPIsrvd
from nDPIsrvd import nDPIsrvdSocket, TermColor

1
examples/py-schema-validation/requirements.txt
View File

@@ -1 +0,0 @@
jsonschema

113
examples/py-semantic-validation/py-semantic-validation.py
View File

@@ -1,23 +1,15 @@
#!/usr/bin/env python3
import base64
import os
import sys
sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId')
sys.path.append(os.path.dirname(sys.argv[0]))
sys.path.append(sys.base_prefix + '/share/nDPId')
sys.path.append(os.path.dirname(sys.argv[0]) + '/../usr/share/nDPId')
import nDPIsrvd
from nDPIsrvd import nDPIsrvdSocket, TermColor
class Stats:
KEYS = [ ['init','reconnect','shutdown','status' ], \
[ 'new','end','idle','update', ],
[ 'analyse' ], \
[ 'guessed','detected','detection-update','not-detected' ], \
[ 'packet', 'packet-flow'] ]
ALL_KEYS = KEYS[0] + KEYS[1] + KEYS[2] + KEYS[3] + KEYS[4]
def __init__(self, nDPIsrvd_sock):
self.nsock = nDPIsrvd_sock
@@ -28,7 +20,11 @@ class Stats:
self.print_nmb_every = self.print_dot_every * 5
def resetEventCounter(self):
for k in Stats.ALL_KEYS:
keys = ['init','reconnect','shutdown','status', \
'new','end','idle','update',
'guessed','detected','detection-update','not-detected', \
'packet', 'packet-flow']
for k in keys:
self.event_counter[k] = 0
def incrementEventCounter(self, json_dict):
@@ -56,9 +52,13 @@ class Stats:
return True
def getEventCounterStr(self):
keys = [ [ 'init','reconnect','shutdown','status' ], \
[ 'new','end','idle','update' ], \
[ 'guessed','detected','detection-update','not-detected' ], \
[ 'packet', 'packet-flow' ] ]
retval = str()
retval += '-' * 98 + '--\n'
for klist in Stats.KEYS:
for klist in keys:
for k in klist:
retval += '| {:<16}: {:<4} '.format(k, self.event_counter[k])
retval += '\n--' + '-' * 98 + '\n'
@@ -74,24 +74,6 @@ class SemanticValidationException(Exception):
else:
return 'Flow ID {}: {}'.format(self.current_flow.flow_id, self.text)
def verifyFlows(nsock, instance):
invalid_flows = nsock.verify()
if len(invalid_flows) > 0:
invalid_flows_str = ''
for flow_id in invalid_flows:
flow = instance.flows[flow_id]
try:
l4_proto = flow.l4_proto
except AttributeError:
l4_proto = 'n/a'
invalid_flows_str += '{} proto[{},{}] ts[{} + {} < {}] diff[{}], '.format(flow_id, l4_proto, flow.flow_idle_time,
flow.flow_last_seen, flow.flow_idle_time,
instance.getMostRecentFlowTime(flow.thread_id),
instance.getMostRecentFlowTime(flow.thread_id) -
(flow.flow_last_seen + flow.flow_idle_time))
raise SemanticValidationException(None, 'Flow Manager verification failed for: {}'.format(invalid_flows_str[:-2]))
def onFlowCleanup(instance, current_flow, global_user_data):
if type(instance) is not nDPIsrvd.Instance:
raise SemanticValidationException(current_flow,
@@ -119,14 +101,28 @@ def onFlowCleanup(instance, current_flow, global_user_data):
except AttributeError:
l4_proto = 'n/a'
verifyFlows(stats.nsock, instance)
invalid_flows = stats.nsock.verify()
if len(invalid_flows) > 0:
invalid_flows_str = ''
for flow_id in invalid_flows:
flow = instance.flows[flow_id]
try:
l4_proto = flow.l4_proto
except AttributeError:
l4_proto = 'n/a'
invalid_flows_str += '{} proto[{},{}] ts[{} + {} < {}] diff[{}], '.format(flow_id, l4_proto, flow.flow_idle_time,
flow.flow_last_seen, flow.flow_idle_time,
instance.most_recent_flow_time,
instance.most_recent_flow_time -
(flow.flow_last_seen + flow.flow_idle_time))
raise SemanticValidationException(None, 'Flow Manager verification failed for: {}'.format(invalid_flows_str[:-2]))
return True
def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
_, stats = global_user_data
stats.incrementEventCounter(json_dict)
verifyFlows(stats.nsock, instance)
if type(instance) is not nDPIsrvd.Instance:
raise SemanticValidationException(current_flow,
@@ -178,11 +174,9 @@ def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
elif json_dict['packet_event_name'] != 'packet-flow':
raise SemanticValidationException(current_flow, 'Layer4 protocol not found in JSON')
flow_last_seen = None
if 'flow_src_last_pkt_time' in json_dict or 'flow_dst_last_pkt_time' in json_dict:
flow_last_seen = max(json_dict['flow_src_last_pkt_time'], json_dict['flow_dst_last_pkt_time'])
if flow_last_seen != current_flow.flow_last_seen:
raise SemanticValidationException(current_flow, 'Flow last seen: {} != {}'.format(flow_last_seen,
if 'flow_last_seen' in json_dict:
if json_dict['flow_last_seen'] != current_flow.flow_last_seen:
raise SemanticValidationException(current_flow, 'Flow last seen: {} != {}'.format(json_dict['flow_last_seen'],
current_flow.flow_last_seen))
if 'flow_idle_time' in json_dict:
@@ -190,14 +184,15 @@ def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
raise SemanticValidationException(current_flow, 'Flow idle time mismatch: {} != {}'.format(json_dict['flow_idle_time'],
current_flow.flow_idle_time))
if (flow_last_seen is not None and 'flow_idle_time' not in json_dict) or \
(flow_last_seen is None and 'flow_idle_time' in json_dict):
if ('flow_last_seen' in json_dict and 'flow_idle_time' not in json_dict) or \
('flow_last_seen' not in json_dict and 'flow_idle_time' in json_dict):
raise SemanticValidationException(current_flow,
'Got a JSON message with only 2 of 3 keys, ' \
'required for timeout handling: flow_idle_time')
'Got a JSON string with only one of both keys, ' \
'both required for timeout handling:' \
'flow_last_seen, flow_idle_time')
if 'thread_ts_usec' in json_dict:
current_flow.thread_ts_usec = int(json_dict['thread_ts_usec'])
if 'thread_ts_msec' in json_dict:
current_flow.thread_ts_msec = int(json_dict['thread_ts_msec'])
if 'flow_packet_id' in json_dict:
try:
@@ -213,13 +208,11 @@ def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
try:
if current_flow.flow_ended == True:
raise SemanticValidationException(current_flow,
'Received JSON message for a flow that already ended/idled.')
'Received JSON string for a flow that already ended/idled.')
except AttributeError:
pass
if 'packet_event_name' in json_dict:
base64.b64decode(json_dict['pkt'], validate=True)
if json_dict['packet_event_name'] == 'packet-flow':
if lowest_possible_packet_id > json_dict['packet_id']:
raise SemanticValidationException(current_flow,
@@ -256,27 +249,11 @@ def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
except AttributeError:
pass
try:
if current_flow.flow_finished == True and \
json_dict['flow_event_name'] == 'detection-update':
raise SemanticValidationException(current_flow,
'Flow state already finished, but another detection-update received.')
except AttributeError:
pass
try:
if json_dict['flow_state'] == 'finished':
current_flow.flow_finished = True
elif json_dict['flow_state'] == 'info' and \
current_flow.flow_finished is True:
raise SemanticValidationException(current_flow,
'Flow state already finished, but switched back to info state.')
except AttributeError:
pass
try:
if current_flow.flow_finished == True and \
json_dict['flow_event_name'] != 'analyse' and \
json_dict['flow_event_name'] != 'update' and \
json_dict['flow_event_name'] != 'idle' and \
json_dict['flow_event_name'] != 'end':
@@ -287,14 +264,14 @@ def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
pass
try:
if json_dict['flow_first_seen'] > current_flow.thread_ts_usec or \
flow_last_seen > current_flow.thread_ts_usec or \
json_dict['flow_first_seen'] > flow_last_seen:
if json_dict['flow_first_seen'] > current_flow.thread_ts_msec or \
json_dict['flow_last_seen'] > current_flow.thread_ts_msec or \
json_dict['flow_first_seen'] > json_dict['flow_last_seen']:
raise SemanticValidationException(current_flow,
'Last packet timestamp is invalid: ' \
'first_seen({}) <= {} >= last_seen({})'.format(json_dict['flow_first_seen'],
current_flow.thread_ts_usec,
flow_last_seen))
current_flow.thread_ts_msec,
json_dict['flow_last_seen']))
except AttributeError:
if json_dict['flow_event_name'] == 'new':
pass
@@ -364,10 +341,6 @@ if __name__ == '__main__':
sys.stderr.write('\n{}\n'.format(err))
except KeyboardInterrupt:
print()
except Exception as e:
for failed_line in nsock.failed_lines:
sys.stderr.write('Affected JSON line: {}\n'.format(failed_line[0]))
raise(e)
sys.stderr.write('\nEvent counter:\n' + stats.getEventCounterStr() + '\n')
if args.strict is True:

21
examples/rs-simple/Cargo.toml
View File

@@ -1,21 +0,0 @@
[package]
name = "rs-simple"
version = "0.1.0"
authors = ["Toni Uhlig <toni@impl.cc>"]
edition = "2024"
[dependencies]
argh = "0.1"
bytes = "1"
crossterm = "0.29.0"
io = "0.0.2"
moka = { version = "0.12.10", features = ["future"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1.0"
tokio = { version = "1", features = ["full"] }
tui = "0.19.0"
[profile.release]
strip = true
lto = true
codegen-units = 1

860
examples/rs-simple/src/main.rs
View File

@@ -1,860 +0,0 @@
use argh::FromArgs;
use bytes::BytesMut;
use crossterm::{
cursor,
event::{self, KeyCode, KeyEvent},
ExecutableCommand,
terminal::{self, ClearType},
};
use moka::{future::Cache, Expiry};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::{
collections::HashMap,
fmt,
hash::{Hash, Hasher},
io::self,
sync::Arc,
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
};
use tokio::io::AsyncReadExt;
use tokio::sync::mpsc;
use tokio::sync::Mutex;
use tokio::sync::MutexGuard;
use tokio::net::TcpStream;
use tui::{
backend::CrosstermBackend,
layout::{Layout, Constraint, Direction},
style::{Style, Color, Modifier},
Terminal,
widgets::{Block, Borders, List, ListItem, Row, Table, TableState},
};
#[derive(FromArgs, Debug)]
/// Simple Rust nDPIsrvd Client Example
struct Args {
/// nDPIsrvd host(s) to connect to
#[argh(option)]
host: Vec<String>,
}
#[derive(Debug)]
enum ParseError {
Protocol(),
Json(),
Schema(),
}
impl From<serde_json::Error> for ParseError {
fn from(_: serde_json::Error) -> Self {
ParseError::Json()
}
}
#[derive(Serialize, Deserialize, Debug, PartialEq)]
#[serde(rename_all = "lowercase")]
enum EventName {
Invalid, New, End, Idle, Update, Analyse,
Guessed, Detected,
#[serde(rename = "detection-update")]
DetectionUpdate,
#[serde(rename = "not-detected")]
NotDetected,
}
#[derive(Serialize, Deserialize, Copy, Clone, Debug)]
#[serde(rename_all = "lowercase")]
enum State {
Unknown, Info, Finished,
}
#[derive(Serialize, Deserialize, Debug)]
struct FlowEventNdpiFlowRisk {
#[serde(rename = "risk")]
risk: String,
}
#[derive(Serialize, Deserialize, Debug)]
struct FlowEventNdpi {
#[serde(rename = "proto")]
proto: String,
#[serde(rename = "flow_risk")]
risks: Option<HashMap<String, FlowEventNdpiFlowRisk>>,
}
#[derive(Serialize, Deserialize, Debug)]
struct FlowEvent {
#[serde(rename = "flow_event_name")]
name: EventName,
#[serde(rename = "flow_id")]
id: u64,
#[serde(rename = "alias")]
alias: String,
#[serde(rename = "source")]
source: String,
#[serde(rename = "thread_id")]
thread_id: u64,
#[serde(rename = "flow_state")]
state: State,
#[serde(rename = "flow_first_seen")]
first_seen: u64,
#[serde(rename = "flow_src_last_pkt_time")]
src_last_pkt_time: u64,
#[serde(rename = "flow_dst_last_pkt_time")]
dst_last_pkt_time: u64,
#[serde(rename = "flow_idle_time")]
idle_time: u64,
#[serde(rename = "flow_src_packets_processed")]
src_packets_processed: u64,
#[serde(rename = "flow_dst_packets_processed")]
dst_packets_processed: u64,
#[serde(rename = "flow_src_tot_l4_payload_len")]
src_tot_l4_payload_len: u64,
#[serde(rename = "flow_dst_tot_l4_payload_len")]
dst_tot_l4_payload_len: u64,
#[serde(rename = "l3_proto")]
l3_proto: String,
#[serde(rename = "l4_proto")]
l4_proto: String,
#[serde(rename = "ndpi")]
ndpi: Option<FlowEventNdpi>,
}
#[derive(Serialize, Deserialize, Debug)]
struct PacketEvent {
pkt_datalink: u16,
pkt_caplen: u64,
pkt_len: u64,
pkt_l4_len: u64,
}
#[derive(Serialize, Deserialize, Clone, Debug)]
struct DaemonEventStatus {
#[serde(rename = "alias")]
alias: String,
#[serde(rename = "source")]
source: String,
#[serde(rename = "thread_id")]
thread_id: u64,
#[serde(rename = "packets-captured")]
packets_captured: u64,
#[serde(rename = "packets-processed")]
packets_processed: u64,
#[serde(rename = "total-skipped-flows")]
total_skipped_flows: u64,
#[serde(rename = "total-l4-payload-len")]
total_l4_payload_len: u64,
#[serde(rename = "total-not-detected-flows")]
total_not_detected_flows: u64,
#[serde(rename = "total-guessed-flows")]
total_guessed_flows: u64,
#[serde(rename = "total-detected-flows")]
total_detected_flows: u64,
#[serde(rename = "total-detection-updates")]
total_detection_updates: u64,
#[serde(rename = "total-updates")]
total_updates: u64,
#[serde(rename = "current-active-flows")]
current_active_flows: u64,
#[serde(rename = "total-active-flows")]
total_active_flows: u64,
#[serde(rename = "total-idle-flows")]
total_idle_flows: u64,
#[serde(rename = "total-compressions")]
total_compressions: u64,
#[serde(rename = "total-compression-diff")]
total_compression_diff: u64,
#[serde(rename = "current-compression-diff")]
current_compression_diff: u64,
#[serde(rename = "global-alloc-bytes")]
global_alloc_bytes: u64,
#[serde(rename = "global-alloc-count")]
global_alloc_count: u64,
#[serde(rename = "global-free-bytes")]
global_free_bytes: u64,
#[serde(rename = "global-free-count")]
global_free_count: u64,
#[serde(rename = "total-events-serialized")]
total_events_serialized: u64,
}
#[derive(Debug)]
enum EventType {
Flow(FlowEvent),
Packet(PacketEvent),
DaemonStatus(DaemonEventStatus),
Other(),
}
#[derive(Default)]
struct Stats {
ui_updates: u64,
flow_count: u64,
parse_errors: u64,
events: u64,
flow_events: u64,
packet_events: u64,
daemon_events: u64,
packet_events_total_caplen: u64,
packet_events_total_len: u64,
packet_events_total_l4_len: u64,
packets_captured: u64,
packets_processed: u64,
flows_total_skipped: u64,
flows_total_l4_payload_len: u64,
flows_total_not_detected: u64,
flows_total_guessed: u64,
flows_current_active: u64,
flows_total_compressions: u64,
flows_total_compression_diff: u64,
flows_current_compression_diff: u64,
global_alloc_bytes: u64,
global_alloc_count: u64,
global_free_bytes: u64,
global_free_count: u64,
total_events_serialized: u64,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum FlowExpiration {
IdleTime(u64),
}
struct FlowExpiry;
#[derive(Clone, Eq, Default, Debug)]
struct FlowKey {
id: u64,
alias: String,
source: String,
thread_id: u64,
}
#[derive(Clone, Debug)]
struct FlowValue {
state: State,
total_src_packets: u64,
total_dst_packets: u64,
total_src_bytes: u64,
total_dst_bytes: u64,
first_seen: std::time::SystemTime,
last_seen: std::time::SystemTime,
timeout_in: std::time::SystemTime,
risks: usize,
proto: String,
app_proto: String,
}
#[derive(Clone, Eq, Default, Debug)]
struct DaemonKey {
alias: String,
source: String,
thread_id: u64,
}
impl Default for State {
fn default() -> State {
State::Unknown
}
}
impl fmt::Display for State {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
State::Unknown => write!(f, "N/A"),
State::Info => write!(f, "Info"),
State::Finished => write!(f, "Finished"),
}
}
}
impl FlowExpiration {
fn as_duration(&self) -> Option<Duration> {
match self {
FlowExpiration::IdleTime(value) => Some(Duration::from_micros(*value)),
}
}
}
impl fmt::Display for FlowExpiration {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self.as_duration() {
Some(duration) => {
let secs = duration.as_secs();
write!(f, "{} s", secs)
}
None => write!(f, "N/A"),
}
}
}
impl Expiry<FlowKey, (FlowExpiration, FlowValue)> for FlowExpiry {
fn expire_after_create(
&self,
_key: &FlowKey,
value: &(FlowExpiration, FlowValue),
_current_time: Instant,
) -> Option<Duration> {
value.0.as_duration()
}
}
impl Hash for FlowKey {
fn hash<H: Hasher>(&self, state: &mut H) {
self.id.hash(state);
self.alias.hash(state);
self.source.hash(state);
self.thread_id.hash(state);
}
}
impl PartialEq for FlowKey {
fn eq(&self, other: &Self) -> bool {
self.id == other.id &&
self.alias == other.alias &&
self.source == other.source &&
self.thread_id == other.thread_id
}
}
impl Hash for DaemonKey {
fn hash<H: Hasher>(&self, state: &mut H) {
self.alias.hash(state);
self.source.hash(state);
self.thread_id.hash(state);
}
}
impl PartialEq for DaemonKey {
fn eq(&self, other: &Self) -> bool {
self.alias == other.alias &&
self.source == other.source &&
self.thread_id == other.thread_id
}
}
#[tokio::main]
async fn main() {
let args: Args = argh::from_env();
if args.host.len() == 0 {
eprintln!("At least one --host required");
return;
}
let mut connections: Vec<TcpStream> = Vec::new();
for host in args.host {
match TcpStream::connect(host.clone()).await {
Ok(stream) => {
connections.push(stream);
}
Err(e) => {
eprintln!("Fehler bei Verbindung zu {}: {}", host, e);
}
}
}
if let Err(e) = terminal::enable_raw_mode() {
eprintln!("Could not enable terminal raw mode: {}", e);
return;
}
let mut stdout = io::stdout();
if let Err(e) = stdout.execute(terminal::Clear(ClearType::All)) {
eprintln!("Could not clear your terminal: {}", e);
return;
}
if let Err(e) = stdout.execute(cursor::Hide) {
eprintln!("Could not hide your cursor: {}", e);
return;
}
let backend = CrosstermBackend::new(stdout);
let mut terminal = Terminal::new(backend);
let (tx, mut rx): (mpsc::Sender<String>, mpsc::Receiver<String>) = mpsc::channel(1024);
let data = Arc::new(Mutex::new(Stats::default()));
let data_tx = Arc::clone(&data);
let data_rx = Arc::clone(&data);
let flow_cache: Arc<Cache<FlowKey, (FlowExpiration, FlowValue)>> = Arc::new(Cache::builder()
.expire_after(FlowExpiry)
.build());
let flow_cache_rx = Arc::clone(&flow_cache);
let daemon_cache: Arc<Cache<DaemonKey, DaemonEventStatus>> = Arc::new(Cache::builder()
.time_to_live(Duration::from_secs(1800))
.build());
tokio::spawn(async move {
while let Some(msg) = rx.recv().await {
match parse_json(&msg) {
Ok(message) => {
let mut data_lock = data_tx.lock().await;
data_lock.events += 1;
update_stats(&message, &mut data_lock, &flow_cache, &daemon_cache).await;
}
Err(_message) => {
let mut data_lock = data_tx.lock().await;
data_lock.parse_errors += 1;
}
}
}
});
for mut stream in connections {
let cloned_tx = tx.clone();
tokio::spawn(async move {
let mut buffer = BytesMut::with_capacity(33792usize);
loop {
let n = match stream.read_buf(&mut buffer).await {
Ok(len) => len,
Err(_) => {
continue; // Versuche es erneut, wenn ein Fehler auftritt
}
};
if n == 0 {
break;
}
while let Some(message) = parse_message(&mut buffer) {
match cloned_tx.send(message).await {
Ok(_) => (),
Err(_) => return
}
}
}
});
}
let mut table_state = TableState::default();
let mut old_selected: Option<FlowKey> = None;
loop {
let flows: Vec<(FlowKey, (FlowExpiration, FlowValue))> = flow_cache_rx.iter().map(|(k, v)| (k.as_ref().clone(), v.clone()))
.take(128)
.collect();
let mut table_selected = match table_state.selected() {
Some(mut table_index) => {
if table_index >= flows.len() {
flows.len().saturating_sub(1)
} else {
if let Some(ref old_flow_key_selected) = old_selected {
if let Some(old_index) = flows.iter().position(|x| x.0 == *old_flow_key_selected) {
if old_index != table_index {
table_index = old_index;
}
} else {
old_selected = Some(flows.get(table_index).unwrap().0.clone());
}
}
table_index
}
}
None => 0,
};
match read_keypress() {
Some(KeyCode::Esc) => break,
Some(KeyCode::Char('q')) => break,
Some(KeyCode::Up) => {
table_selected = match table_selected {
i if i == 0 => flows.len().saturating_sub(1),
i => i - 1,
};
if let Some(new_selected) = flows.get(table_selected) {
old_selected = Some(new_selected.0.clone());
}
},
Some(KeyCode::Down) => {
table_selected = match table_selected {
i if i >= flows.len().saturating_sub(1) => 0,
i => i + 1,
};
if let Some(new_selected) = flows.get(table_selected) {
old_selected = Some(new_selected.0.clone());
}
},
Some(KeyCode::PageUp) => {
table_selected = match table_selected {
i if i == 0 => flows.len().saturating_sub(1),
i if i < 25 => 0,
i => i - 25,
};
if let Some(new_selected) = flows.get(table_selected) {
old_selected = Some(new_selected.0.clone());
}
},
Some(KeyCode::PageDown) => {
table_selected = match table_selected {
i if i >= flows.len().saturating_sub(1) => 0,
i if i >= flows.len().saturating_sub(25) => flows.len().saturating_sub(1),
i => i + 25,
};
if let Some(new_selected) = flows.get(table_selected) {
old_selected = Some(new_selected.0.clone());
}
},
Some(KeyCode::Home) => {
table_selected = 0;
if let Some(new_selected) = flows.get(table_selected) {
old_selected = Some(new_selected.0.clone());
}
},
Some(KeyCode::End) => {
table_selected = match table_selected {
_ => flows.len().saturating_sub(1),
};
if let Some(new_selected) = flows.get(table_selected) {
old_selected = Some(new_selected.0.clone());
}
},
Some(_) => (),
None => ()
};
let mut data_lock = data_rx.lock().await;
data_lock.ui_updates += 1;
draw_ui(terminal.as_mut().unwrap(), &mut table_state, table_selected, &data_lock, &flows);
}
if let Err(e) = terminal.unwrap().backend_mut().execute(cursor::Show) {
eprintln!("Could not show your cursor: {}", e);
return;
}
let mut stdout = io::stdout();
if let Err(e) = stdout.execute(terminal::Clear(ClearType::All)) {
eprintln!("Could not clear your terminal: {}", e);
return;
}
if let Err(e) = terminal::disable_raw_mode() {
eprintln!("Could not disable raw mode: {}", e);
return;
}
println!("\nDone.");
}
fn read_keypress() -> Option<KeyCode> {
if event::poll(Duration::from_millis(1000)).unwrap() {
if let event::Event::Key(KeyEvent { code, .. }) = event::read().unwrap() {
return Some(code);
}
}
None
}
fn parse_message(buffer: &mut BytesMut) -> Option<String> {
if let Some(pos) = buffer.iter().position(|&b| b == b'\n') {
let message = buffer.split_to(pos + 1);
return Some(String::from_utf8_lossy(&message).to_string());
}
None
}
fn parse_json(data: &str) -> Result<EventType, ParseError> {
let first_non_digit = data.find(|c: char| !c.is_ascii_digit()).unwrap_or(0);
let length_str = &data[0..first_non_digit];
let length: usize = length_str.parse().unwrap_or(0);
if length == 0 {
return Err(ParseError::Protocol());
}
let json_str = &data[first_non_digit..first_non_digit + length];
let value: Value = serde_json::from_str(json_str).map_err(|_| ParseError::Json()).unwrap();
if value.get("flow_event_name").is_some() {
let flow_event: FlowEvent = serde_json::from_value(value)?;
return Ok(EventType::Flow(flow_event));
} else if value.get("packet_event_name").is_some() {
let packet_event: PacketEvent = serde_json::from_value(value)?;
return Ok(EventType::Packet(packet_event));
} else if value.get("daemon_event_name").is_some() {
if value.get("daemon_event_name").unwrap() == "status" ||
value.get("daemon_event_name").unwrap() == "shutdown"
{
let daemon_status_event: DaemonEventStatus = serde_json::from_value(value)?;
return Ok(EventType::DaemonStatus(daemon_status_event));
}
return Ok(EventType::Other());
} else if value.get("error_event_name").is_some() {
return Ok(EventType::Other());
}
Err(ParseError::Schema())
}
async fn update_stats(event: &EventType, stats: &mut MutexGuard<'_, Stats>, cache: &Cache<FlowKey, (FlowExpiration, FlowValue)>, daemon_cache: &Cache<DaemonKey, DaemonEventStatus>) {
match &event {
EventType::Flow(flow_event) => {
stats.flow_events += 1;
stats.flow_count = cache.entry_count();
let key = FlowKey { id: flow_event.id, alias: flow_event.alias.to_string(),
source: flow_event.source.to_string(), thread_id: flow_event.thread_id };
if flow_event.name == EventName::End ||
flow_event.name == EventName::Idle
{
cache.remove(&key).await;
return;
}
let first_seen_seconds = flow_event.first_seen / 1_000_000;
let first_seen_nanos = (flow_event.first_seen % 1_000_000) * 1_000;
let first_seen_epoch = std::time::Duration::new(first_seen_seconds, first_seen_nanos as u32);
let first_seen_system = UNIX_EPOCH + first_seen_epoch;
let last_seen = std::cmp::max(flow_event.src_last_pkt_time,
flow_event.dst_last_pkt_time);
let last_seen_seconds = last_seen / 1_000_000;
let last_seen_nanos = (last_seen % 1_000_000) * 1_000;
let last_seen_epoch = std::time::Duration::new(last_seen_seconds, last_seen_nanos as u32);
let last_seen_system = UNIX_EPOCH + last_seen_epoch;
let timeout_seconds = (last_seen + flow_event.idle_time) / 1_000_000;
let timeout_nanos = ((last_seen + flow_event.idle_time) % 1_000_000) * 1_000;
let timeout_epoch = std::time::Duration::new(timeout_seconds, timeout_nanos as u32);
let timeout_system = UNIX_EPOCH + timeout_epoch;
let risks = match &flow_event.ndpi {
None => 0,
Some(ndpi) => match &ndpi.risks {
None => 0,
Some(risks) => risks.len(),
},
};
let app_proto = match &flow_event.ndpi {
None => "-",
Some(ndpi) => &ndpi.proto,
};
let value = FlowValue {
state: flow_event.state,
total_src_packets: flow_event.src_packets_processed,
total_dst_packets: flow_event.dst_packets_processed,
total_src_bytes: flow_event.src_tot_l4_payload_len,
total_dst_bytes: flow_event.dst_tot_l4_payload_len,
first_seen: first_seen_system,
last_seen: last_seen_system,
timeout_in: timeout_system,
risks: risks,
proto: flow_event.l3_proto.to_string() + "/" + &flow_event.l4_proto,
app_proto: app_proto.to_string(),
};
cache.insert(key, (FlowExpiration::IdleTime(flow_event.idle_time), value)).await;
}
EventType::Packet(packet_event) => {
stats.packet_events += 1;
stats.packet_events_total_caplen += packet_event.pkt_caplen;
stats.packet_events_total_len += packet_event.pkt_len;
stats.packet_events_total_l4_len += packet_event.pkt_l4_len;
}
EventType::DaemonStatus(daemon_status_event) => {
let key = DaemonKey { alias: daemon_status_event.alias.to_string(),
source: daemon_status_event.source.to_string(),
thread_id: daemon_status_event.thread_id };
stats.daemon_events += 1;
daemon_cache.insert(key, daemon_status_event.clone()).await;
stats.packets_captured = 0;
stats.packets_processed = 0;
stats.flows_total_skipped = 0;
stats.flows_total_l4_payload_len = 0;
stats.flows_total_not_detected = 0;
stats.flows_total_guessed = 0;
stats.flows_current_active = 0;
stats.flows_total_compressions = 0;
stats.flows_total_compression_diff = 0;
stats.flows_current_compression_diff = 0;
stats.global_alloc_bytes = 0;
stats.global_alloc_count = 0;
stats.global_free_bytes = 0;
stats.global_free_count = 0;
stats.total_events_serialized = 0;
let daemons: Vec<DaemonEventStatus> = daemon_cache.iter().map(|(_, v)| (v.clone())).collect();
for daemon in daemons {
stats.packets_captured += daemon.packets_captured;
stats.packets_processed += daemon.packets_processed;
stats.flows_total_skipped += daemon.total_skipped_flows;
stats.flows_total_l4_payload_len += daemon.total_l4_payload_len;
stats.flows_total_not_detected += daemon.total_not_detected_flows;
stats.flows_total_guessed += daemon.total_guessed_flows;
stats.flows_current_active += daemon.current_active_flows;
stats.flows_total_compressions += daemon.total_compressions;
stats.flows_total_compression_diff += daemon.total_compression_diff;
stats.flows_current_compression_diff += daemon.current_compression_diff;
stats.global_alloc_bytes += daemon.global_alloc_bytes;
stats.global_alloc_count += daemon.global_alloc_count;
stats.global_free_bytes += daemon.global_free_bytes;
stats.global_free_count += daemon.global_free_count;
stats.total_events_serialized += daemon.total_events_serialized;
}
}
EventType::Other() => {}
}
}
fn format_bytes(bytes: u64) -> String {
const KB: u64 = 1024;
const MB: u64 = KB * 1024;
const GB: u64 = MB * 1024;
if bytes >= GB {
format!("{} GB", bytes / GB)
} else if bytes >= MB {
format!("{} MB", bytes / MB)
} else if bytes >= KB {
format!("{} kB", bytes / KB)
} else {
format!("{} B", bytes)
}
}
fn draw_ui<B: tui::backend::Backend>(terminal: &mut Terminal<B>, table_state: &mut TableState, table_selected: usize, data: &MutexGuard<Stats>, flows: &Vec<(FlowKey, (FlowExpiration, FlowValue))>) {
let general_items = vec![
ListItem::new("TUI Updates..: ".to_owned() + &data.ui_updates.to_string()),
ListItem::new("Flows Cached.: ".to_owned() + &data.flow_count.to_string()),
ListItem::new("Total Events.: ".to_owned() + &data.events.to_string()),
ListItem::new("Parse Errors.: ".to_owned() + &data.parse_errors.to_string()),
ListItem::new("Flow Events..: ".to_owned() + &data.flow_events.to_string()),
];
let packet_items = vec![
ListItem::new("Total Events........: ".to_owned() + &data.packet_events.to_string()),
ListItem::new("Total Capture Length: ".to_owned() + &format_bytes(data.packet_events_total_caplen)),
ListItem::new("Total Length........: ".to_owned() + &format_bytes(data.packet_events_total_len)),
ListItem::new("Total L4 Length.....: ".to_owned() + &format_bytes(data.packet_events_total_l4_len)),
];
let daemon_items = vec![
ListItem::new("Total Events.............: ".to_owned() + &data.daemon_events.to_string()),
ListItem::new("Total Packets Captured...: ".to_owned() + &data.packets_captured.to_string()),
ListItem::new("Total Packets Processed..: ".to_owned() + &data.packets_processed.to_string()),
ListItem::new("Total Flows Skipped......: ".to_owned() + &data.flows_total_skipped.to_string()),
ListItem::new("Total Flows Not-Detected.: ".to_owned() + &data.flows_total_not_detected.to_string()),
ListItem::new("Total Compressions/Memory: ".to_owned() + &data.flows_total_compressions.to_string()
+ " / " + &format_bytes(data.flows_total_compression_diff) + " deflate"),
ListItem::new("Total Memory in Use......: ".to_owned() + &format_bytes(data.global_alloc_bytes - data.global_free_bytes)
+ " (" + &format_bytes(data.flows_current_compression_diff) + " deflate)"),
ListItem::new("Total Events Serialized..: ".to_owned() + &data.total_events_serialized.to_string()),
ListItem::new("Current Flows Active.....: ".to_owned() + &data.flows_current_active.to_string()),
];
let table_rows: Vec<Row> = flows
.into_iter()
.map(|(key, (_exp, val))| {
let first_seen_display = match val.first_seen.elapsed() {
Ok(elapsed) => {
match elapsed.as_secs() {
t if t > (3_600 * 24) => format!("{} d ago", t / (3_600 * 24)),
t if t > 3_600 => format!("{} h ago", t / 3_600),
t if t > 60 => format!("{} min ago", t / 60),
t if t > 0 => format!("{} s ago", t),
t if t == 0 => "< 1 s ago".to_string(),
t => format!("INVALID: {}", t),
}
}
Err(err) => format!("ERROR: {}", err)
};
let last_seen_display = match val.last_seen.elapsed() {
Ok(elapsed) => {
match elapsed.as_secs() {
t if t > (3_600 * 24) => format!("{} d ago", t / (3_600 * 24)),
t if t > 3_600 => format!("{} h ago", t / 3_600),
t if t > 60 => format!("{} min ago", t / 60),
t if t > 0 => format!("{} s ago", t),
t if t == 0 => "< 1 s ago".to_string(),
t => format!("INVALID: {}", t),
}
}
Err(_err) => "ERROR".to_string()
};
let timeout_display = match val.timeout_in.duration_since(SystemTime::now()) {
Ok(elapsed) => {
match elapsed.as_secs() {
t if t > (3_600 * 24) => format!("in {} d", t / (3_600 * 24)),
t if t > 3_600 => format!("in {} h", t / 3_600),
t if t > 60 => format!("in {} min", t / 60),
t if t > 0 => format!("in {} s", t),
t if t == 0 => "in < 1 s".to_string(),
t => format!("INVALID: {}", t),
}
}
Err(_err) => "EXPIRED".to_string()
};
Row::new(vec![
key.id.to_string(),
val.state.to_string(),
first_seen_display,
last_seen_display,
timeout_display,
(val.total_src_packets + val.total_dst_packets).to_string(),
format_bytes(val.total_src_bytes + val.total_dst_bytes),
val.risks.to_string(),
val.proto.to_string(),
val.app_proto.to_string(),
])
})
.collect();
terminal.draw(|f| {
let size = f.size();
let chunks = Layout::default()
.direction(Direction::Vertical)
.constraints(
[
Constraint::Length(11),
Constraint::Percentage(100),
].as_ref()
)
.split(size);
let top_chunks = Layout::default()
.direction(Direction::Horizontal)
.constraints(
[
Constraint::Percentage(25),
Constraint::Percentage(30),
Constraint::Percentage(55),
].as_ref()
)
.split(chunks[0]);
let table_selected_abs = match table_selected {
_ if flows.len() == 0 => 0,
i => i + 1,
};
let table = Table::new(table_rows)
.header(Row::new(vec!["Flow ID", "State", "First Seen", "Last Seen", "Timeout", "Total Packets", "Total Bytes", "Risks", "L3/L4", "L7"])
.style(Style::default().fg(Color::Yellow).add_modifier(Modifier::BOLD)))
.block(Block::default().title("Flow Table (selected: ".to_string() +
&table_selected_abs.to_string() +
"): " +
&flows.len().to_string() +
" item(s)").borders(Borders::ALL))
.highlight_style(Style::default().bg(Color::Blue))
.widths(&[
Constraint::Length(10),
Constraint::Length(10),
Constraint::Length(12),
Constraint::Length(12),
Constraint::Length(10),
Constraint::Length(13),
Constraint::Length(12),
Constraint::Length(6),
Constraint::Length(12),
Constraint::Length(15),
]);
let general_list = List::new(general_items)
.block(Block::default().title("General").borders(Borders::ALL));
let packet_list = List::new(packet_items)
.block(Block::default().title("Packet Events").borders(Borders::ALL));
let daemon_list = List::new(daemon_items)
.block(Block::default().title("Daemon Events").borders(Borders::ALL));
table_state.select(Some(table_selected));
f.render_widget(general_list, top_chunks[0]);
f.render_widget(packet_list, top_chunks[1]);
f.render_widget(daemon_list, top_chunks[2]);
f.render_stateful_widget(table, chunks[1], table_state);
}).unwrap();
}

28
examples/yaml-filebeat/filebeat.yml
View File

@@ -1,28 +0,0 @@
filebeat.inputs:
- type: unix
id: "NDPId-logs" # replace this index to your preference
max_message_size: 100MiB
index: "index-name" # Replace this with your desired index name in Elasticsearch
enabled: true
path: "/var/run/nDPId.sock" # point nDPId to this Unix Socket (Collector)
processors:
- script: # execute javascript to remove the first 5-digit-number and also the Newline at the end
lang: javascript
id: trim
source: >
function process(event) {
event.Put("message", event.Get("message").trim().slice(5));
}
- decode_json_fields: # Decode the Json output
fields: ["message"]
process_array: true
max_depth: 10
target: ""
overwrite_keys: true
add_error_key: false
- drop_fields: # Deletes the Message field, which is the undecoded json (You may comment this out if you need the original message)
fields: ["message"]
- rename:
fields:
- from: "source" # Prevents a conflict in Elasticsearch and renames the field
to: "Source_Interface"

2
libnDPI

Submodule libnDPI updated: 75db1a8a66...7c19de4904

1463
nDPId-test.c
View File

File diff suppressed because it is too large Load Diff

4203
nDPId.c
View File

File diff suppressed because it is too large Load Diff

843
nDPIsrvd.c
View File

File diff suppressed because it is too large Load Diff

207
ncrypt.c
View File

@@ -1,207 +0,0 @@
#include "ncrypt.h"
#include <endian.h>
#include <openssl/conf.h>
#include <openssl/core_names.h>
#include <openssl/err.h>
#include <openssl/pem.h>
#include <openssl/ssl.h>
#include <unistd.h>
int ncrypt_init(void)
{
SSL_load_error_strings();
OpenSSL_add_all_algorithms();
return NCRYPT_SUCCESS;
}
static int ncrypt_init_ctx(struct ncrypt_ctx * const ctx, SSL_METHOD const * const meth)
{
if (meth == NULL)
{
return NCRYPT_NULL_PTR;
}
if (ctx->ssl_ctx != NULL)
{
return NCRYPT_ALREADY_INITIALIZED;
}
ctx->ssl_ctx = SSL_CTX_new(meth);
if (ctx->ssl_ctx == NULL)
{
return NCRYPT_NOT_INITIALIZED;
}
SSL_CTX_set_min_proto_version(ctx->ssl_ctx, TLS1_3_VERSION);
SSL_CTX_set_max_proto_version(ctx->ssl_ctx, TLS1_3_VERSION);
SSL_CTX_set_ciphersuites(ctx->ssl_ctx, "TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256");
return NCRYPT_SUCCESS;
}
static int ncrypt_load_pems(struct ncrypt_ctx * const ctx,
char const * const ca_path,
char const * const privkey_pem_path,
char const * const pubkey_pem_path)
{
if (SSL_CTX_use_certificate_file(ctx->ssl_ctx, pubkey_pem_path, SSL_FILETYPE_PEM) <= 0 ||
SSL_CTX_use_PrivateKey_file(ctx->ssl_ctx, privkey_pem_path, SSL_FILETYPE_PEM) <= 0 ||
SSL_CTX_load_verify_locations(ctx->ssl_ctx, ca_path, NULL) <= 0)
{
return NCRYPT_PEM_LOAD_FAILED;
}
SSL_CTX_set_verify(ctx->ssl_ctx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL);
SSL_CTX_set_verify_depth(ctx->ssl_ctx, 4);
return NCRYPT_SUCCESS;
}
int ncrypt_init_client(struct ncrypt_ctx * const ctx,
char const * const ca_path,
char const * const privkey_pem_path,
char const * const pubkey_pem_path)
{
if (ca_path == NULL || privkey_pem_path == NULL || pubkey_pem_path == NULL)
{
return NCRYPT_NULL_PTR;
}
int rv = ncrypt_init_ctx(ctx, TLS_client_method());
if (rv != NCRYPT_SUCCESS)
{
return rv;
}
return ncrypt_load_pems(ctx, ca_path, privkey_pem_path, pubkey_pem_path);
}
int ncrypt_init_server(struct ncrypt_ctx * const ctx,
char const * const ca_path,
char const * const privkey_pem_path,
char const * const pubkey_pem_path)
{
if (ca_path == NULL || privkey_pem_path == NULL || pubkey_pem_path == NULL)
{
return NCRYPT_NULL_PTR;
}
int rv = ncrypt_init_ctx(ctx, TLS_server_method());
if (rv != NCRYPT_SUCCESS)
{
return rv;
}
return ncrypt_load_pems(ctx, ca_path, privkey_pem_path, pubkey_pem_path);
}
int ncrypt_on_connect(struct ncrypt_ctx * const ctx, int connect_fd, struct ncrypt_entity * const ent)
{
if (ent->ssl == NULL)
{
ent->ssl = SSL_new(ctx->ssl_ctx);
if (ent->ssl == NULL)
{
return NCRYPT_NOT_INITIALIZED;
}
SSL_set_fd(ent->ssl, connect_fd);
SSL_set_connect_state(ent->ssl);
}
int rv = SSL_do_handshake(ent->ssl);
if (rv != 1)
{
return SSL_get_error(ent->ssl, rv);
}
return NCRYPT_SUCCESS;
}
int ncrypt_on_accept(struct ncrypt_ctx * const ctx, int accept_fd, struct ncrypt_entity * const ent)
{
if (ent->ssl == NULL)
{
ent->ssl = SSL_new(ctx->ssl_ctx);
if (ent->ssl == NULL)
{
return NCRYPT_NOT_INITIALIZED;
}
SSL_set_fd(ent->ssl, accept_fd);
SSL_set_accept_state(ent->ssl);
}
int rv = SSL_accept(ent->ssl);
if (rv != 1)
{
return SSL_get_error(ent->ssl, rv);
}
return NCRYPT_SUCCESS;
}
ssize_t ncrypt_read(struct ncrypt_entity * const ent, char * const json_msg, size_t json_msg_len)
{
if (ent->ssl == NULL)
{
errno = EPROTO;
return -1;
}
int rv = SSL_read(ent->ssl, json_msg, json_msg_len);
if (rv <= 0)
{
int err = SSL_get_error(ent->ssl, rv);
if (err == SSL_ERROR_WANT_WRITE || err == SSL_ERROR_WANT_READ)
{
errno = EAGAIN;
}
else if (err != SSL_ERROR_SYSCALL)
{
errno = EPROTO;
}
return -1;
}
return rv;
}
ssize_t ncrypt_write(struct ncrypt_entity * const ent, char const * const json_msg, size_t json_msg_len)
{
if (ent->ssl == NULL)
{
errno = EPROTO;
return -1;
}
int rv = SSL_write(ent->ssl, json_msg, json_msg_len);
if (rv <= 0)
{
int err = SSL_get_error(ent->ssl, rv);
if (err == SSL_ERROR_WANT_WRITE || err == SSL_ERROR_WANT_READ)
{
errno = EAGAIN;
}
else if (err != SSL_ERROR_SYSCALL)
{
errno = EPROTO;
}
return -1;
}
return rv;
}
void ncrypt_free_entity(struct ncrypt_entity * const ent)
{
SSL_free(ent->ssl);
ent->ssl = NULL;
}
void ncrypt_free_ctx(struct ncrypt_ctx * const ctx)
{
SSL_CTX_free(ctx->ssl_ctx);
ctx->ssl_ctx = NULL;
EVP_cleanup();
}

73
ncrypt.h
View File

@@ -1,73 +0,0 @@
#ifndef NCRYPT_H
#define NCRYPT_H 1
#include <stdlib.h>
#define ncrypt_ctx(x) \
do \
{ \
(x)->ssl_ctx = NULL; \
} while (0);
#define ncrypt_entity(x) \
do \
{ \
(x)->ssl = NULL; \
(x)->handshake_done = 0; \
} while (0);
#define ncrypt_handshake_done(x) ((x)->handshake_done)
#define ncrypt_set_handshake(x) \
do \
{ \
(x)->handshake_done = 1; \
} while (0)
#define ncrypt_clear_handshake(x) \
do \
{ \
(x)->handshake_done = 0; \
} while (0)
enum
{
NCRYPT_SUCCESS = 0,
NCRYPT_NOT_INITIALIZED = -1,
NCRYPT_ALREADY_INITIALIZED = -2,
NCRYPT_NULL_PTR = -3,
NCRYPT_PEM_LOAD_FAILED = -4
};
struct ncrypt_ctx
{
void * ssl_ctx;
};
struct ncrypt_entity
{
void * ssl;
int handshake_done;
};
int ncrypt_init(void);
int ncrypt_init_client(struct ncrypt_ctx * const ctx,
char const * const ca_path,
char const * const privkey_pem_path,
char const * const pubkey_pem_path);
int ncrypt_init_server(struct ncrypt_ctx * const ctx,
char const * const ca_path,
char const * const privkey_pem_path,
char const * const pubkey_pem_path);
int ncrypt_on_connect(struct ncrypt_ctx * const ctx, int connect_fd, struct ncrypt_entity * const ent);
int ncrypt_on_accept(struct ncrypt_ctx * const ctx, int accept_fd, struct ncrypt_entity * const ent);
ssize_t ncrypt_read(struct ncrypt_entity * const ent, char * const json_msg, size_t json_msg_len);
ssize_t ncrypt_write(struct ncrypt_entity * const ent, char const * const json_msg, size_t json_msg_len);
void ncrypt_free_entity(struct ncrypt_entity * const ent);
void ncrypt_free_ctx(struct ncrypt_ctx * const ctx);
#endif

102
ndpid.conf.example
View File

@@ -1,102 +0,0 @@
[general]
# Set the network interface from which packets are captured and processed.
# Leave it empty to let nDPId choose the default network interface.
#netif = eth0
# Set a Berkeley Packet Filter.
# This will work for libpcap as well as with PF_RING.
#bpf = udp or tcp
# Decapsulate Layer4 tunnel protocols.
# Supported protocols: GRE
#decode-tunnel = true
#pidfile = /tmp/ndpid.pid
#user = nobody
#group = daemon
#riskdomains = /path/to/libnDPI/example/risky_domains.txt
#protocols = /path/to/libnDPI/example/protos.txt
#categories = /path/to/libnDPI/example/categories.txt
#ja4 = /path/to/libnDPI/example/ja4_fingerprints.csv
#sha1 = /path/to/libnDPI/example/sha1_fingerprints.csv
# Collector endpoint as UNIX socket (usually nDPIsrvd)
#collector = /run/nDPIsrvd/collector
# Collector endpoint as UDP socket (usually a custom application)
#collector = 127.0.0.1:7777
# Set a name for this nDPId instance
#alias = myhostname
# Set an optional UUID for this instance
# If the value starts with a '/' or '.', it is interpreted as a path
# from which the uuid is read from.
#uuid = 00000000-dead-c0de-0000-123456789abc
#uuid = ./path/to/some/file
#uuid = /proc/sys/kernel/random/uuid
#uuid = /sys/class/dmi/id/product_uuid
# Process only internal initial connections (src->dst)
#internal = true
# Process only external initial connections (dst->src)
#external = true
# Enable zLib compression of flow memory for long lasting flows
compression = true
# Enable "analyse" events, which can be used for machine learning.
# The daemon will generate some statistical values for every single flow.
# An "analyse" event is thrown after "max-packets-per-flow-to-analyse".
# Please note that the daemon will require a lot more heap memory for every flow.
#analysis = true
# Force poll() on systems that support epoll() as well
#poll = false
# Enable PF_RING packet capture instead of libpcap
#pfring = false
[tuning]
max-flows-per-thread = 2048
max-idle-flows-per-thread = 64
max-reader-threads = 10
daemon-status-interval = 600000000
#memory-profiling-log-interval = 5
compression-scan-interval = 20000000
compression-flow-inactivity = 30000000
flow-scan-interval = 10000000
generic-max-idle-time = 600000000
icmp-max-idle-time = 120000000
tcp-max-idle-time = 180000000
udp-max-idle-time = 7440000000
tcp-max-post-end-flow-time = 120000000
max-packets-per-flow-to-send = 15
max-packets-per-flow-to-process = 32
max-packets-per-flow-to-analyse = 32
error-event-threshold-n = 16
error-event-threshold-time = 10000000
# Please note that the following options are libnDPI related and can only be set via config file,
# not as commnand line parameter.
# See libnDPI/doc/configuration_parameters.md for detailed information.
[ndpi]
packets_limit_per_flow = 32
flow.direction_detection = enable
flow.track_payload = disable
tcp_ack_payload_heuristic = disable
fully_encrypted_heuristic = enable
libgcrypt.init = 1
dpi.compute_entropy = 1
fpc = disable
dpi.guess_on_giveup = 0x03
flow_risk_lists.load = 1
# Currently broken (upstream)
#flow_risk.crawler_bot.list.load = 1
log.level = 0
[protos]
tls.certificate_expiration_threshold = 7
tls.application_blocks_tracking = enable
stun.max_packets_extra_dissection = 8

31
ndpisrvd.conf.example
View File

@@ -1,31 +0,0 @@
[general]
#pidfile = /tmp/ndpisrvd.pid
#user = nobody
#group = nogroup
# Collector listener as UNIX socket
#collector = /run/nDPIsrvd/collector
# Distributor listener as UNIX socket
#distributor-unix = /run/nDPIsrvd/distributor
# Distributor listener as IP socket
#distributor-in = 127.0.0.1:7000
# Change group of the collector socket
#collector-group = daemon
# Change group of the distirbutor socket
#distirbutor-group = staff
# Max (distributor) clients allowed to connect to nDPIsrvd
max-remote-descriptors = 128
# Additional output buffers useful if a distributor sink speed unstable
max-write-buffers = 1024
# Fallback to blocking I/O if output buffers full
blocking-io-fallback = true
# Force poll() on systems that support epoll() as well
#poll = false

Some files were not shown because too many files have changed in this diff Show More