nDPId event I/O fixes.

* forcibly disable epoll even if available
 * nDPId-test event I/O selftest
 * CI event I/O tests

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

.github/workflows/build.yml

@@ -29,6 +29,7 @@ jobs:
             ndpid_zlib: "-DENABLE_ZLIB=ON"
             sanitizer: "-DENABLE_SANITIZER=OFF -DENABLE_SANITIZER_THREAD=OFF"
             coverage: "-DENABLE_COVERAGE=OFF"
+            poll: "-DFORCE_POLL=OFF"
             upload: true
             upload_suffix: ""
             ndpi_min_version: "4.8"
@@ -38,6 +39,7 @@ jobs:
             ndpid_zlib: "-DENABLE_ZLIB=ON"
             sanitizer: "-DENABLE_SANITIZER=OFF -DENABLE_SANITIZER_THREAD=OFF"
             coverage: "-DENABLE_COVERAGE=OFF"
+            poll: "-DFORCE_POLL=OFF"
             upload: true
             upload_suffix: "-host-gcrypt"
             ndpi_min_version: "4.8"
@@ -47,6 +49,7 @@ jobs:
             ndpid_zlib: "-DENABLE_ZLIB=OFF"
             sanitizer: "-DENABLE_SANITIZER=OFF -DENABLE_SANITIZER_THREAD=OFF"
             coverage: "-DENABLE_COVERAGE=OFF"
+            poll: "-DFORCE_POLL=OFF"
             upload: true
             upload_suffix: "-no-zlib"
             ndpi_min_version: "4.8"
@@ -56,6 +59,7 @@ jobs:
             ndpid_zlib: "-DENABLE_ZLIB=ON"
             sanitizer: "-DENABLE_SANITIZER=ON"
             coverage: "-DENABLE_COVERAGE=ON"
+            poll: "-DFORCE_POLL=ON"
             upload: false
             ndpi_min_version: "4.8"
           - compiler: "clang"
@@ -64,6 +68,7 @@ jobs:
             ndpid_zlib: "-DENABLE_ZLIB=ON"
             sanitizer: "-DENABLE_SANITIZER=ON" 
             coverage: "-DENABLE_COVERAGE=ON"
+            poll: "-DFORCE_POLL=OFF"
             upload: false
             ndpi_min_version: "4.8"
           - compiler: "clang-12"
@@ -72,6 +77,7 @@ jobs:
             ndpid_zlib: "-DENABLE_ZLIB=ON"
             sanitizer: "-DENABLE_SANITIZER_THREAD=ON"
             coverage: "-DENABLE_COVERAGE=ON"
+            poll: "-DFORCE_POLL=OFF"
             upload: false
             ndpi_min_version: "4.8"
           - compiler: "gcc-10"
@@ -80,6 +86,7 @@ jobs:
             ndpid_zlib: "-DENABLE_ZLIB=OFF"
             sanitizer: "-DENABLE_SANITIZER=ON"
             coverage: "-DENABLE_COVERAGE=OFF"
+            poll: "-DFORCE_POLL=ON"
             upload: false
             ndpi_min_version: "4.8"
           - compiler: "gcc-7"
@@ -88,6 +95,7 @@ jobs:
             ndpid_zlib: "-DENABLE_ZLIB=ON"
             sanitizer: "-DENABLE_SANITIZER=ON"
             coverage: "-DENABLE_COVERAGE=OFF"
+            poll: "-DFORCE_POLL=OFF"
             upload: false
             ndpi_min_version: "4.8"
 
@@ -132,7 +140,7 @@ jobs:
       - name: Configure nDPId
         run: |
           mkdir build && cd build
-          cmake .. -DENABLE_SYSTEMD=ON -DBUILD_EXAMPLES=ON -DBUILD_NDPI=ON ${{ matrix.coverage }} ${{ matrix.sanitizer }} ${{ matrix.ndpid_zlib }} ${{ matrix.ndpid_gcrypt }}
+          cmake .. -DENABLE_SYSTEMD=ON -DBUILD_EXAMPLES=ON -DBUILD_NDPI=ON ${{ matrix.poll }} ${{ matrix.coverage }} ${{ matrix.sanitizer }} ${{ matrix.ndpid_zlib }} ${{ matrix.ndpid_gcrypt }}
       - name: Build nDPId
         run: |
           make -C build all VERBOSE=1
@@ -142,7 +150,7 @@ jobs:
           cc -fsanitize=address -fsanitize=undefined -fno-sanitize=alignment -fsanitize=enum -fsanitize=leak nDPId.c utils.c -I./build/libnDPI/include/ndpi -I. -I./dependencies -I./dependencies/jsmn -I./dependencies/uthash/include -o /tmp/a.out -lpcap ./build/libnDPI/lib/libndpi.a -pthread -lm -lz
       - name: Test EXEC
         run: |
-          ./build/nDPId-test || test $? -eq 1
+          ./build/nDPId-test
           ./build/nDPId -h || test $? -eq 1
           ./build/nDPIsrvd -h || test $? -eq 1
       - name: Test DIFF
@@ -190,6 +198,6 @@ jobs:
           echo "wget returned: ${WGET_RET}"
           test $WGET_RET -ne 8 || echo "::warning file=nDPId.c::New libnDPI release required to build against release tarball."
           test $WGET_RET -ne 0 || { tar -xzvf ${{ matrix.ndpi_min_version }}.tar.gz && cd nDPI-${{ matrix.ndpi_min_version }} && ./autogen.sh --prefix=/usr --with-only-libndpi CC=${{ matrix.compiler }} CXX=false CFLAGS='-Werror' && sudo make install && cd .. ; }
-          test $WGET_RET -ne 0 || { echo "running cmake .."; cmake .. -DBUILD_EXAMPLES=ON -DBUILD_NDPI=OFF -DENABLE_SANITIZER=OFF ${{ matrix.coverage }} ${{ matrix.ndpi_min_version }} ; }
+          test $WGET_RET -ne 0 || { echo "running cmake .."; cmake .. -DBUILD_EXAMPLES=ON -DBUILD_NDPI=OFF -DENABLE_SANITIZER=OFF ${{ matrix.poll }} ${{ matrix.coverage }} ${{ matrix.ndpi_min_version }} ; }
           test $WGET_RET -ne 0 || { echo "running make .."; make all VERBOSE=1 ; }
           test $WGET_RET -eq 0 -o $WGET_RET -eq 8

.gitlab-ci.yml

@@ -37,6 +37,7 @@ build_and_test_static_libndpi_tsan:
     - tree libnDPI
     - make install VERBOSE=1 DESTDIR="$(realpath ../_install)"
     - cd ..
+    - ./_install/usr/local/bin/nDPId-test
     - ./test/run_tests.sh ./libnDPI ./_install/usr/local/bin/nDPId-test
   artifacts:
     expire_in: 1 week
@@ -61,6 +62,7 @@ build_and_test_static_libndpi:
     - test -x /bin/systemctl && sudo systemctl start ndpid@lo
     - test -x /bin/systemctl && sudo systemctl status ndpisrvd.service ndpid@lo.service
     - test -x /bin/systemctl && sudo systemctl stop ndpid@lo
+    - ./build-cmake-submodule/nDPId-test
     - ./test/run_tests.sh ./libnDPI ./build-cmake-submodule/nDPId-test
     - >
       if ldd ./build-cmake-submodule/nDPId | grep -qoEi libndpi; then \
@@ -83,6 +85,7 @@ build_and_test_static_libndpi_coverage:
     - tree libnDPI
     - make install VERBOSE=1 DESTDIR="$(realpath ../_install)"
     - cd ..
+    - ./build-cmake-submodule/nDPId-test
     - ./test/run_tests.sh ./libnDPI ./build-cmake-submodule/nDPId-test
     # generate coverage report
     - make -C ./build-cmake-submodule coverage || true
@@ -113,7 +116,7 @@ build_dynamic_libndpi:
     - make install VERBOSE=1 DESTDIR="$(realpath ../_install)"
     - cd ..
     - tree ./_install
-    - ./build/nDPId-test || test $? -eq 1
+    - ./build/nDPId-test
     - ./build/nDPId -h || test $? -eq 1
     - ./build/nDPIsrvd -h || test $? -eq 1
     # dameon start/stop test

CMakeLists.txt

@@ -35,8 +35,11 @@ include(CheckEpoll)
 
 check_epoll(HAS_EPOLL)
 if(HAS_EPOLL)
-    set(EPOLL_DEFS "-DENABLE_EPOLL=1")
+    option(FORCE_POLL "Force the use of poll() instead of epoll()." OFF)
-    set(EPOLL_SRCS "nio.c")
+    if(NOT FORCE_POLL)
+        set(EPOLL_DEFS "-DENABLE_EPOLL=1")
+        set(EPOLL_SRCS "nio.c")
+    endif()
 endif()
 
 if(NOT MATH_FUNCTION_EXISTS AND NOT NEED_LINKING_AGAINST_LIBM)
@@ -416,6 +419,7 @@ message(STATUS "Cross Compilation........: ${CMAKE_CROSSCOMPILING}")
 message(STATUS "CMAKE_BUILD_TYPE.........: ${CMAKE_BUILD_TYPE}")
 message(STATUS "CMAKE_C_FLAGS............: ${CMAKE_C_FLAGS}")
 message(STATUS "NDPID_DEFS...............: ${NDPID_DEFS}")
+message(STATUS "FORCE_POLL...............: ${FORCE_POLL}")
 message(STATUS "ENABLE_COVERAGE..........: ${ENABLE_COVERAGE}")
 message(STATUS "ENABLE_SANITIZER.........: ${ENABLE_SANITIZER}")
 message(STATUS "ENABLE_SANITIZER_THREAD..: ${ENABLE_SANITIZER_THREAD}")

nDPId-test.c

@@ -1477,12 +1477,123 @@ static int base64_selftest()
     return strncmp(base64_data, encoded_buf, base64_data_len) != 0;
 }
 
+static int nio_selftest()
+{
+    struct nio io;
+
+    nio_init(&io);
+
+#ifdef ENABLE_EPOLL
+    logger(0, "%s", "Using epoll for nio");
+#else
+    logger(0, "%s", "Using poll for nio");
+#endif
+
+#ifdef ENABLE_EPOLL
+    if (nio_use_epoll(&io, 5) != NIO_ERROR_SUCCESS)
+#else
+    if (nio_use_poll(&io, 3) != NIO_ERROR_SUCCESS)
+#endif
+    {
+        logger(1, "%s", "Could not use poll/epoll for nio");
+        goto error;
+    }
+
+    int pipefds[2];
+    int rv = pipe(pipefds);
+    if (rv < 0)
+    {
+        logger(1, "Could not create a pipe: %s", strerror(errno));
+        goto error;
+    }
+
+    if (nio_add_fd(&io, pipefds[1], NIO_EVENT_OUTPUT, NULL) != NIO_ERROR_SUCCESS ||
+        nio_add_fd(&io, pipefds[0], NIO_EVENT_INPUT, NULL) != NIO_ERROR_SUCCESS)
+    {
+        logger(1, "%s", "Could not add pipe fds to nio");
+        goto error;
+    }
+
+    if (fcntl_add_flags(pipefds[1], O_NONBLOCK) != 0 || fcntl_add_flags(pipefds[0], O_NONBLOCK) != 0)
+    {
+        logger(1, "%s", "Could not set pipe fds to O_NONBLOCK");
+        goto error;
+    }
+
+    char const wbuf[] = "AAAA";
+    size_t const wlen = strnlen(wbuf, sizeof(wbuf));
+    write(pipefds[1], wbuf, wlen);
+
+    if (nio_run(&io, 1000) != NIO_ERROR_SUCCESS)
+    {
+        logger(1, "%s", "Event notification failed");
+        goto error;
+    }
+
+    if (nio_can_output(&io, 0) != NIO_ERROR_SUCCESS)
+    {
+        logger(1, "%s", "Pipe fd (write) can not output");
+        goto error;
+    }
+
+    if (nio_has_input(&io, 1) != NIO_ERROR_SUCCESS)
+    {
+        logger(1, "%s", "Pipe fd (read) has no input");
+        goto error;
+    }
+
+    if (nio_is_valid(&io, 0) != NIO_ERROR_SUCCESS || nio_is_valid(&io, 1) != NIO_ERROR_SUCCESS ||
+        nio_has_error(&io, 0) == NIO_ERROR_SUCCESS || nio_has_error(&io, 1) == NIO_ERROR_SUCCESS)
+    {
+        logger(1, "%s", "Event validation failed");
+        goto error;
+    }
+
+    char rbuf[4];
+    if (read(pipefds[0], rbuf, sizeof(rbuf)) != sizeof(rbuf) || strncmp(rbuf, wbuf, wlen) != 0)
+    {
+        logger(1, "%s", "Buffer receive failed");
+        goto error;
+    }
+
+    if (nio_run(&io, 1000) != NIO_ERROR_SUCCESS)
+    {
+        logger(1, "%s", "Event notification failed");
+        goto error;
+    }
+
+    if (nio_can_output(&io, 0) != NIO_ERROR_SUCCESS)
+    {
+        logger(1, "%s", "Pipe fd (write) can not output");
+        goto error;
+    }
+
+    if (nio_has_input(&io, 1) == NIO_ERROR_SUCCESS)
+    {
+        logger(1, "%s", "Pipe fd (read) has input");
+        goto error;
+    }
+
+    if (nio_is_valid(&io, 0) != NIO_ERROR_SUCCESS || nio_is_valid(&io, 1) == NIO_ERROR_SUCCESS ||
+        nio_has_error(&io, 0) == NIO_ERROR_SUCCESS || nio_has_error(&io, 1) == NIO_ERROR_SUCCESS)
+    {
+        logger(1, "%s", "Event validation failed");
+        goto error;
+    }
+
+    nio_free(&io);
+    return 0;
+error:
+    nio_free(&io);
+    return 1;
+}
+
 #define THREADS_RETURNED_ERROR()                                                                                       \
     (nDPId_return.thread_return_value.val != 0 || nDPIsrvd_return.val != 0 ||                                          \
      distributor_return.thread_return_value.val != 0)
 int main(int argc, char ** argv)
 {
-    if (argc != 2)
+    if (argc != 1 && argc != 2)
     {
         usage(argv[0]);
         return 1;
@@ -1491,14 +1602,23 @@ int main(int argc, char ** argv)
     init_logging("nDPId-test");
     log_app_info();
 
-    if (base64_selftest() != 0)
+    if (signal(SIGPIPE, SIG_IGN) == SIG_ERR)
     {
         return 1;
     }
 
-    if (signal(SIGPIPE, SIG_IGN) == SIG_ERR)
+    if (argc == 1)
     {
-        return 1;
+        int retval = 0;
+
+        usage(argv[0]);
+        logger(1, "%s", "No pcap file provided. Running selftest mode.");
+
+        retval += base64_selftest();
+        retval += nio_selftest();
+
+        logger(1, "Selftest returned: %d", retval);
+        return retval;
     }
 
     nDPIsrvd_options.max_write_buffers = 32;

nio.c

@@ -5,11 +5,12 @@
 #ifdef ENABLE_EPOLL
 #include <sys/epoll.h>
 #endif
+#include <unistd.h>
 
 void nio_init(struct nio * io)
 {
     io->nready = -1;
-    io->poll_max_fds = -1;
+    io->poll_max_fds = 0;
     io->poll_cur_fds = 0;
     io->poll_fds = NULL;
     io->poll_ptrs = NULL;
@@ -27,6 +28,11 @@ int nio_use_poll(struct nio * io, nfds_t max_fds)
     io->poll_fds = (struct pollfd *)calloc(max_fds, sizeof(*io->poll_fds));
     io->poll_ptrs = calloc(max_fds, sizeof(*io->poll_ptrs));
 
+    for (size_t i = 0; i < max_fds; ++i)
+    {
+        io->poll_fds[i].fd = -1;
+    }
+
     return io->poll_fds == NULL || io->poll_ptrs == NULL; // return NIO_ERROR_INTERNAL on error
 }
 
@@ -40,7 +46,7 @@ int nio_use_epoll(struct nio * io, int max_events)
     io->max_events = max_events;
     io->events = calloc(max_events, sizeof(struct epoll_event));
 
-    return io->epoll_fd;
+    return io->events == NULL || io->epoll_fd < 0;
 #else
     (void)io;
     (void)max_events;
@@ -76,7 +82,7 @@ int nio_add_fd(struct nio * io, int fd, int event_flags, void * ptr)
     }
     else
 #endif
-    if (io->poll_max_fds > 0)
+        if (io->poll_max_fds > 0)
     {
         struct pollfd * unused_pollfd = NULL;
         void ** unused_ptr = NULL;
@@ -247,7 +253,7 @@ int nio_check(struct nio * io, int index, int events)
         return NIO_ERROR_INTERNAL;
 
 #ifdef ENABLE_EPOLL
-    if (io->epoll_fd >= 0 && index >= 0 && index < io->max_events)
+    if (io->epoll_fd >= 0 && index < io->max_events)
     {
         uint32_t epoll_events = 0;
 
@@ -255,18 +261,20 @@ int nio_check(struct nio * io, int index, int events)
             epoll_events |= EPOLLIN;
         if ((events & NIO_EVENT_OUTPUT) != 0)
             epoll_events |= EPOLLOUT;
+        if ((events & NIO_EVENT_ERROR) != 0)
+            epoll_events |= EPOLLERR | EPOLLHUP;
         if (epoll_events == 0)
             return NIO_ERROR_INTERNAL;
 
         struct epoll_event * ee = (struct epoll_event *)io->events;
-        if ((ee[index].events & epoll_events) != epoll_events)
+        if ((ee[index].events & epoll_events) == 0)
             return NIO_ERROR_INTERNAL;
 
         return NIO_ERROR_SUCCESS;
     }
     else
 #endif
-        if (io->poll_max_fds > 0 && index >= 0 && index < (int)io->poll_max_fds)
+        if (io->poll_max_fds > 0 && index < (int)io->poll_max_fds)
     {
         short int poll_events = 0;
 
@@ -274,10 +282,12 @@ int nio_check(struct nio * io, int index, int events)
             poll_events |= POLLIN;
         if ((events & NIO_EVENT_OUTPUT) != 0)
             poll_events |= POLLOUT;
+        if ((events & NIO_EVENT_ERROR) != 0)
+            poll_events |= POLLERR | POLLHUP;
         if (poll_events == 0)
             return NIO_ERROR_INTERNAL;
 
-        if (io->poll_fds[index].revents != poll_events)
+        if ((io->poll_fds[index].revents & poll_events) == 0)
             return NIO_ERROR_INTERNAL;
 
         return NIO_ERROR_SUCCESS;
@@ -292,13 +302,13 @@ int nio_is_valid(struct nio * io, int index)
         return NIO_ERROR_INTERNAL;
 
 #ifdef ENABLE_EPOLL
-    if (io->epoll_fd >= 0 && index >= 0 && index <= io->max_events)
+    if (io->epoll_fd >= 0 && index <= io->max_events)
     {
         return NIO_ERROR_SUCCESS;
     }
     else
 #endif
-        if (io->poll_max_fds > 0 && index >= 0 && index < (int)io->poll_max_fds)
+        if (io->poll_max_fds > 0 && index < (int)io->poll_max_fds)
     {
         if (io->poll_fds[index].revents != 0)
             return NIO_ERROR_SUCCESS;
@@ -324,6 +334,21 @@ int nio_has_error(struct nio * io, int index)
 
 void nio_free(struct nio * io)
 {
+    for (size_t i = 0; i < io->poll_max_fds; ++i)
+    {
+        if (io->poll_fds[i].fd >= 0)
+        {
+            close(io->poll_fds[i].fd);
+            io->poll_fds[i].fd = -1;
+        }
+    }
+#ifdef ENABLE_EPOLL
+    if (io->epoll_fd >= 0)
+    {
+        close(io->epoll_fd);
+        io->epoll_fd = -1;
+    }
+#endif
     free(io->poll_fds);
     free(io->poll_ptrs);
     free(io->events);

nio.h

@@ -15,7 +15,7 @@ enum
     NIO_EVENT_INVALID = 0,
     NIO_EVENT_INPUT = 1,
     NIO_EVENT_OUTPUT = 2,
-    NIO_EVENT_ERROR = 3,
+    NIO_EVENT_ERROR = 4,
 };
 
 struct nio