scottk/nDPId
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/nDPId.git synced 2025-10-29 17:32:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

nDPId event I/O fixes.

Browse Source 
* forcibly disable epoll even if available
 * nDPId-test event I/O selftest
 * CI event I/O tests

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>
This commit is contained in:
Toni Uhlig
2023-10-04 15:21:55 +02:00
parent a0e0611c56
commit 73b8c378f2
6 changed files with 180 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
.github/workflows/build.yml vendored
View File

@@ -29,6 +29,7 @@ jobs:
ndpid_zlib: "-DENABLE_ZLIB=ON"
sanitizer: "-DENABLE_SANITIZER=OFF -DENABLE_SANITIZER_THREAD=OFF"
coverage: "-DENABLE_COVERAGE=OFF"
poll: "-DFORCE_POLL=OFF"
upload: true
upload_suffix: ""
ndpi_min_version: "4.8"
@@ -38,6 +39,7 @@ jobs:
ndpid_zlib: "-DENABLE_ZLIB=ON"
sanitizer: "-DENABLE_SANITIZER=OFF -DENABLE_SANITIZER_THREAD=OFF"
coverage: "-DENABLE_COVERAGE=OFF"
poll: "-DFORCE_POLL=OFF"
upload: true
upload_suffix: "-host-gcrypt"
ndpi_min_version: "4.8"
@@ -47,6 +49,7 @@ jobs:
ndpid_zlib: "-DENABLE_ZLIB=OFF"
sanitizer: "-DENABLE_SANITIZER=OFF -DENABLE_SANITIZER_THREAD=OFF"
coverage: "-DENABLE_COVERAGE=OFF"
poll: "-DFORCE_POLL=OFF"
upload: true
upload_suffix: "-no-zlib"
ndpi_min_version: "4.8"
@@ -56,6 +59,7 @@ jobs:
ndpid_zlib: "-DENABLE_ZLIB=ON"
sanitizer: "-DENABLE_SANITIZER=ON"
coverage: "-DENABLE_COVERAGE=ON"
poll: "-DFORCE_POLL=ON"
upload: false
ndpi_min_version: "4.8"
- compiler: "clang"
@@ -64,6 +68,7 @@ jobs:
ndpid_zlib: "-DENABLE_ZLIB=ON"
sanitizer: "-DENABLE_SANITIZER=ON"
coverage: "-DENABLE_COVERAGE=ON"
poll: "-DFORCE_POLL=OFF"
upload: false
ndpi_min_version: "4.8"
- compiler: "clang-12"
@@ -72,6 +77,7 @@ jobs:
ndpid_zlib: "-DENABLE_ZLIB=ON"
sanitizer: "-DENABLE_SANITIZER_THREAD=ON"
coverage: "-DENABLE_COVERAGE=ON"
poll: "-DFORCE_POLL=OFF"
upload: false
ndpi_min_version: "4.8"
- compiler: "gcc-10"
@@ -80,6 +86,7 @@ jobs:
ndpid_zlib: "-DENABLE_ZLIB=OFF"
sanitizer: "-DENABLE_SANITIZER=ON"
coverage: "-DENABLE_COVERAGE=OFF"
poll: "-DFORCE_POLL=ON"
upload: false
ndpi_min_version: "4.8"
- compiler: "gcc-7"
@@ -88,6 +95,7 @@ jobs:
ndpid_zlib: "-DENABLE_ZLIB=ON"
sanitizer: "-DENABLE_SANITIZER=ON"
coverage: "-DENABLE_COVERAGE=OFF"
poll: "-DFORCE_POLL=OFF"
upload: false
ndpi_min_version: "4.8"
@@ -132,7 +140,7 @@ jobs:
- name: Configure nDPId
run: |
mkdir build && cd build
cmake .. -DENABLE_SYSTEMD=ON -DBUILD_EXAMPLES=ON -DBUILD_NDPI=ON ${{ matrix.coverage }} ${{ matrix.sanitizer }} ${{ matrix.ndpid_zlib }} ${{ matrix.ndpid_gcrypt }}
cmake .. -DENABLE_SYSTEMD=ON -DBUILD_EXAMPLES=ON -DBUILD_NDPI=ON ${{ matrix.poll }} ${{ matrix.coverage }} ${{ matrix.sanitizer }} ${{ matrix.ndpid_zlib }} ${{ matrix.ndpid_gcrypt }}
- name: Build nDPId
run: |
make -C build all VERBOSE=1
@@ -142,7 +150,7 @@ jobs:
cc -fsanitize=address -fsanitize=undefined -fno-sanitize=alignment -fsanitize=enum -fsanitize=leak nDPId.c utils.c -I./build/libnDPI/include/ndpi -I. -I./dependencies -I./dependencies/jsmn -I./dependencies/uthash/include -o /tmp/a.out -lpcap ./build/libnDPI/lib/libndpi.a -pthread -lm -lz
- name: Test EXEC
run: |
./build/nDPId-test || test $? -eq 1
./build/nDPId-test
./build/nDPId -h || test $? -eq 1
./build/nDPIsrvd -h || test $? -eq 1
- name: Test DIFF
@@ -190,6 +198,6 @@ jobs:
echo "wget returned: ${WGET_RET}"
test $WGET_RET -ne 8 || echo "::warning file=nDPId.c::New libnDPI release required to build against release tarball."
test $WGET_RET -ne 0 || { tar -xzvf ${{ matrix.ndpi_min_version }}.tar.gz && cd nDPI-${{ matrix.ndpi_min_version }} && ./autogen.sh --prefix=/usr --with-only-libndpi CC=${{ matrix.compiler }} CXX=false CFLAGS='-Werror' && sudo make install && cd .. ; }
test $WGET_RET -ne 0 || { echo "running cmake .."; cmake .. -DBUILD_EXAMPLES=ON -DBUILD_NDPI=OFF -DENABLE_SANITIZER=OFF ${{ matrix.coverage }} ${{ matrix.ndpi_min_version }} ; }
test $WGET_RET -ne 0 || { echo "running cmake .."; cmake .. -DBUILD_EXAMPLES=ON -DBUILD_NDPI=OFF -DENABLE_SANITIZER=OFF ${{ matrix.poll }} ${{ matrix.coverage }} ${{ matrix.ndpi_min_version }} ; }
test $WGET_RET -ne 0 || { echo "running make .."; make all VERBOSE=1 ; }
test $WGET_RET -eq 0 -o $WGET_RET -eq 8

5
.gitlab-ci.yml
View File

@@ -37,6 +37,7 @@ build_and_test_static_libndpi_tsan:
- tree libnDPI
- make install VERBOSE=1 DESTDIR="$(realpath ../_install)"
- cd ..
- ./_install/usr/local/bin/nDPId-test
- ./test/run_tests.sh ./libnDPI ./_install/usr/local/bin/nDPId-test
artifacts:
expire_in: 1 week
@@ -61,6 +62,7 @@ build_and_test_static_libndpi:
- test -x /bin/systemctl && sudo systemctl start ndpid@lo
- test -x /bin/systemctl && sudo systemctl status ndpisrvd.service ndpid@lo.service
- test -x /bin/systemctl && sudo systemctl stop ndpid@lo
- ./build-cmake-submodule/nDPId-test
- ./test/run_tests.sh ./libnDPI ./build-cmake-submodule/nDPId-test
- >
if ldd ./build-cmake-submodule/nDPId | grep -qoEi libndpi; then \
@@ -83,6 +85,7 @@ build_and_test_static_libndpi_coverage:
- tree libnDPI
- make install VERBOSE=1 DESTDIR="$(realpath ../_install)"
- cd ..
- ./build-cmake-submodule/nDPId-test
- ./test/run_tests.sh ./libnDPI ./build-cmake-submodule/nDPId-test
# generate coverage report
- make -C ./build-cmake-submodule coverage || true
@@ -113,7 +116,7 @@ build_dynamic_libndpi:
- make install VERBOSE=1 DESTDIR="$(realpath ../_install)"
- cd ..
- tree ./_install
- ./build/nDPId-test || test $? -eq 1
- ./build/nDPId-test
- ./build/nDPId -h || test $? -eq 1
- ./build/nDPIsrvd -h || test $? -eq 1
# dameon start/stop test

8
CMakeLists.txt
View File

@@ -35,8 +35,11 @@ include(CheckEpoll)
check_epoll(HAS_EPOLL)
if(HAS_EPOLL)
set(EPOLL_DEFS "-DENABLE_EPOLL=1")
set(EPOLL_SRCS "nio.c")
option(FORCE_POLL "Force the use of poll() instead of epoll()." OFF)
if(NOT FORCE_POLL)
set(EPOLL_DEFS "-DENABLE_EPOLL=1")
set(EPOLL_SRCS "nio.c")
endif()
endif()
if(NOT MATH_FUNCTION_EXISTS AND NOT NEED_LINKING_AGAINST_LIBM)
@@ -416,6 +419,7 @@ message(STATUS "Cross Compilation........: ${CMAKE_CROSSCOMPILING}")
message(STATUS "CMAKE_BUILD_TYPE.........: ${CMAKE_BUILD_TYPE}")
message(STATUS "CMAKE_C_FLAGS............: ${CMAKE_C_FLAGS}")
message(STATUS "NDPID_DEFS...............: ${NDPID_DEFS}")
message(STATUS "FORCE_POLL...............: ${FORCE_POLL}")
message(STATUS "ENABLE_COVERAGE..........: ${ENABLE_COVERAGE}")
message(STATUS "ENABLE_SANITIZER.........: ${ENABLE_SANITIZER}")
message(STATUS "ENABLE_SANITIZER_THREAD..: ${ENABLE_SANITIZER_THREAD}")

128
nDPId-test.c
View File

@@ -1477,12 +1477,123 @@ static int base64_selftest()
return strncmp(base64_data, encoded_buf, base64_data_len) != 0;
}
static int nio_selftest()
{
struct nio io;
nio_init(&io);
#ifdef ENABLE_EPOLL
logger(0, "%s", "Using epoll for nio");
#else
logger(0, "%s", "Using poll for nio");
#endif
#ifdef ENABLE_EPOLL
if (nio_use_epoll(&io, 5) != NIO_ERROR_SUCCESS)
#else
if (nio_use_poll(&io, 3) != NIO_ERROR_SUCCESS)
#endif
{
logger(1, "%s", "Could not use poll/epoll for nio");
goto error;
}
int pipefds[2];
int rv = pipe(pipefds);
if (rv < 0)
{
logger(1, "Could not create a pipe: %s", strerror(errno));
goto error;
}
if (nio_add_fd(&io, pipefds[1], NIO_EVENT_OUTPUT, NULL) != NIO_ERROR_SUCCESS ||
nio_add_fd(&io, pipefds[0], NIO_EVENT_INPUT, NULL) != NIO_ERROR_SUCCESS)
{
logger(1, "%s", "Could not add pipe fds to nio");
goto error;
}
if (fcntl_add_flags(pipefds[1], O_NONBLOCK) != 0 || fcntl_add_flags(pipefds[0], O_NONBLOCK) != 0)
{
logger(1, "%s", "Could not set pipe fds to O_NONBLOCK");
goto error;
}
char const wbuf[] = "AAAA";
size_t const wlen = strnlen(wbuf, sizeof(wbuf));
write(pipefds[1], wbuf, wlen);
if (nio_run(&io, 1000) != NIO_ERROR_SUCCESS)
{
logger(1, "%s", "Event notification failed");
goto error;
}
if (nio_can_output(&io, 0) != NIO_ERROR_SUCCESS)
{
logger(1, "%s", "Pipe fd (write) can not output");
goto error;
}
if (nio_has_input(&io, 1) != NIO_ERROR_SUCCESS)
{
logger(1, "%s", "Pipe fd (read) has no input");
goto error;
}
if (nio_is_valid(&io, 0) != NIO_ERROR_SUCCESS || nio_is_valid(&io, 1) != NIO_ERROR_SUCCESS ||
nio_has_error(&io, 0) == NIO_ERROR_SUCCESS || nio_has_error(&io, 1) == NIO_ERROR_SUCCESS)
{
logger(1, "%s", "Event validation failed");
goto error;
}
char rbuf[4];
if (read(pipefds[0], rbuf, sizeof(rbuf)) != sizeof(rbuf) || strncmp(rbuf, wbuf, wlen) != 0)
{
logger(1, "%s", "Buffer receive failed");
goto error;
}
if (nio_run(&io, 1000) != NIO_ERROR_SUCCESS)
{
logger(1, "%s", "Event notification failed");
goto error;
}
if (nio_can_output(&io, 0) != NIO_ERROR_SUCCESS)
{
logger(1, "%s", "Pipe fd (write) can not output");
goto error;
}
if (nio_has_input(&io, 1) == NIO_ERROR_SUCCESS)
{
logger(1, "%s", "Pipe fd (read) has input");
goto error;
}
if (nio_is_valid(&io, 0) != NIO_ERROR_SUCCESS || nio_is_valid(&io, 1) == NIO_ERROR_SUCCESS ||
nio_has_error(&io, 0) == NIO_ERROR_SUCCESS || nio_has_error(&io, 1) == NIO_ERROR_SUCCESS)
{
logger(1, "%s", "Event validation failed");
goto error;
}
nio_free(&io);
return 0;
error:
nio_free(&io);
return 1;
}
#define THREADS_RETURNED_ERROR() \
(nDPId_return.thread_return_value.val != 0 || nDPIsrvd_return.val != 0 || \
distributor_return.thread_return_value.val != 0)
int main(int argc, char ** argv)
{
if (argc != 2)
if (argc != 1 && argc != 2)
{
usage(argv[0]);
return 1;
@@ -1491,14 +1602,23 @@ int main(int argc, char ** argv)
init_logging("nDPId-test");
log_app_info();
if (base64_selftest() != 0)
if (signal(SIGPIPE, SIG_IGN) == SIG_ERR)
{
return 1;
}
if (signal(SIGPIPE, SIG_IGN) == SIG_ERR)
if (argc == 1)
{
return 1;
int retval = 0;
usage(argv[0]);
logger(1, "%s", "No pcap file provided. Running selftest mode.");
retval += base64_selftest();
retval += nio_selftest();
logger(1, "Selftest returned: %d", retval);
return retval;
}
nDPIsrvd_options.max_write_buffers = 32;

43
nio.c
View File

@@ -5,11 +5,12 @@
#ifdef ENABLE_EPOLL
#include <sys/epoll.h>
#endif
#include <unistd.h>
void nio_init(struct nio * io)
{
io->nready = -1;
io->poll_max_fds = -1;
io->poll_max_fds = 0;
io->poll_cur_fds = 0;
io->poll_fds = NULL;
io->poll_ptrs = NULL;
@@ -27,6 +28,11 @@ int nio_use_poll(struct nio * io, nfds_t max_fds)
io->poll_fds = (struct pollfd *)calloc(max_fds, sizeof(*io->poll_fds));
io->poll_ptrs = calloc(max_fds, sizeof(*io->poll_ptrs));
for (size_t i = 0; i < max_fds; ++i)
{
io->poll_fds[i].fd = -1;
}
return io->poll_fds == NULL || io->poll_ptrs == NULL; // return NIO_ERROR_INTERNAL on error
}
@@ -40,7 +46,7 @@ int nio_use_epoll(struct nio * io, int max_events)
io->max_events = max_events;
io->events = calloc(max_events, sizeof(struct epoll_event));
return io->epoll_fd;
return io->events == NULL || io->epoll_fd < 0;
#else
(void)io;
(void)max_events;
@@ -76,7 +82,7 @@ int nio_add_fd(struct nio * io, int fd, int event_flags, void * ptr)
}
else
#endif
if (io->poll_max_fds > 0)
if (io->poll_max_fds > 0)
{
struct pollfd * unused_pollfd = NULL;
void ** unused_ptr = NULL;
@@ -247,7 +253,7 @@ int nio_check(struct nio * io, int index, int events)
return NIO_ERROR_INTERNAL;
#ifdef ENABLE_EPOLL
if (io->epoll_fd >= 0 && index >= 0 && index < io->max_events)
if (io->epoll_fd >= 0 && index < io->max_events)
{
uint32_t epoll_events = 0;
@@ -255,18 +261,20 @@ int nio_check(struct nio * io, int index, int events)
epoll_events |= EPOLLIN;
if ((events & NIO_EVENT_OUTPUT) != 0)
epoll_events |= EPOLLOUT;
if ((events & NIO_EVENT_ERROR) != 0)
epoll_events |= EPOLLERR | EPOLLHUP;
if (epoll_events == 0)
return NIO_ERROR_INTERNAL;
struct epoll_event * ee = (struct epoll_event *)io->events;
if ((ee[index].events & epoll_events) != epoll_events)
if ((ee[index].events & epoll_events) == 0)
return NIO_ERROR_INTERNAL;
return NIO_ERROR_SUCCESS;
}
else
#endif
if (io->poll_max_fds > 0 && index >= 0 && index < (int)io->poll_max_fds)
if (io->poll_max_fds > 0 && index < (int)io->poll_max_fds)
{
short int poll_events = 0;
@@ -274,10 +282,12 @@ int nio_check(struct nio * io, int index, int events)
poll_events |= POLLIN;
if ((events & NIO_EVENT_OUTPUT) != 0)
poll_events |= POLLOUT;
if ((events & NIO_EVENT_ERROR) != 0)
poll_events |= POLLERR | POLLHUP;
if (poll_events == 0)
return NIO_ERROR_INTERNAL;
if (io->poll_fds[index].revents != poll_events)
if ((io->poll_fds[index].revents & poll_events) == 0)
return NIO_ERROR_INTERNAL;
return NIO_ERROR_SUCCESS;
@@ -292,13 +302,13 @@ int nio_is_valid(struct nio * io, int index)
return NIO_ERROR_INTERNAL;
#ifdef ENABLE_EPOLL
if (io->epoll_fd >= 0 && index >= 0 && index <= io->max_events)
if (io->epoll_fd >= 0 && index <= io->max_events)
{
return NIO_ERROR_SUCCESS;
}
else
#endif
if (io->poll_max_fds > 0 && index >= 0 && index < (int)io->poll_max_fds)
if (io->poll_max_fds > 0 && index < (int)io->poll_max_fds)
{
if (io->poll_fds[index].revents != 0)
return NIO_ERROR_SUCCESS;
@@ -324,6 +334,21 @@ int nio_has_error(struct nio * io, int index)
void nio_free(struct nio * io)
{
for (size_t i = 0; i < io->poll_max_fds; ++i)
{
if (io->poll_fds[i].fd >= 0)
{
close(io->poll_fds[i].fd);
io->poll_fds[i].fd = -1;
}
}
#ifdef ENABLE_EPOLL
if (io->epoll_fd >= 0)
{
close(io->epoll_fd);
io->epoll_fd = -1;
}
#endif
free(io->poll_fds);
free(io->poll_ptrs);
free(io->events);

2
nio.h
View File

@@ -15,7 +15,7 @@ enum
NIO_EVENT_INVALID = 0,
NIO_EVENT_INPUT = 1,
NIO_EVENT_OUTPUT = 2,
NIO_EVENT_ERROR = 3,
NIO_EVENT_ERROR = 4,
};
struct nio