scottk/nDPId
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/nDPId.git synced 2025-11-02 03:07:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Declared nDPI id and flow structs in nDPId flow info struct.

Browse Source 
Two reasons:
 * reduce heap memory allocations
 * nDPId flow info struct may be inflated in the future (more bytes to compress)

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>
This commit is contained in:
Toni Uhlig
2020-12-30 13:22:15 +01:00
parent e0be911633
commit 613e60ca2a
3 changed files with 39 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
dependencies/nDPIsrvd.py vendored
View File

@@ -28,8 +28,7 @@ BASIC_EVENTS = ['Invalid', 'Unknown-Datalink-Layer', 'Unknown-Layer3-Protocol',
'IP4-Size-Smaller-Than-Header', 'IP4-Layer4-Payload-Detection-Failed', 'IP6-Packet-Too-Short', 'IP4-Size-Smaller-Than-Header', 'IP4-Layer4-Payload-Detection-Failed', 'IP6-Packet-Too-Short',
'IP6-Size-Smaller-Than-Header', 'IP6-Layer4-Payload-Detection-Failed', 'TCP-Packet-Too-Short', 'IP6-Size-Smaller-Than-Header', 'IP6-Layer4-Payload-Detection-Failed', 'TCP-Packet-Too-Short',
'UDP-Packet-Too-Short', 'Capture-Size-Smaller-Than-Packet-Size', 'Max-Flow-To-Track', 'UDP-Packet-Too-Short', 'Capture-Size-Smaller-Than-Packet-Size', 'Max-Flow-To-Track',
'Flow-Memory-Allocation-Failed', 'NDPI-Flow-Memory-Allocation-Failed', 'Flow-Memory-Allocation-Failed']
'NDPI-ID-Memory-Allocation-Failed']
PACKET_EVENTS = [ ('Invalid','invalid'), ('Packet','packet'), ('Packet-Flow','packet-flow') ] PACKET_EVENTS = [ ('Invalid','invalid'), ('Packet','packet'), ('Packet-Flow','packet-flow') ]
FLOW_EVENTS = [ ('Invalid','invalid'), ('New','new'), ('End','end'), ('Idle','idle'), ('Guessed','guessed'), \ FLOW_EVENTS = [ ('Invalid','invalid'), ('New','new'), ('End','end'), ('Idle','idle'), ('Guessed','guessed'), \
('Detected','detected'), ('Detection-Update','detection-update'), ('Not-Detected','not-detected') ] ('Detected','detected'), ('Detection-Update','detection-update'), ('Not-Detected','not-detected') ]

2
examples/c-json-stdout/c-json-stdout.c
View File

@@ -93,7 +93,7 @@ int main(void)
if (buf[json_bytes - 2] != '}' || if (buf[json_bytes - 2] != '}' ||
buf[json_bytes - 1] != '\n') buf[json_bytes - 1] != '\n')
{ {
fprintf(stderr, "BUG: Invalid JSON string: %.*s\n", (int)json_bytes, buf); fprintf(stderr, "BUG: Invalid JSON string: \"%.*s\"\n", (int)json_bytes, buf);
exit(1); exit(1);
} }

109
nDPId.c
View File

@@ -69,9 +69,20 @@ struct nDPId_flow_info
struct ndpi_proto detected_l7_protocol; struct ndpi_proto detected_l7_protocol;
struct ndpi_proto guessed_l7_protocol; struct ndpi_proto guessed_l7_protocol;
struct ndpi_flow_struct * ndpi_flow; union {
struct ndpi_id_struct * ndpi_src; uint8_t ndpi_flow_raw[SIZEOF_FLOW_STRUCT];
struct ndpi_id_struct * ndpi_dst; struct ndpi_flow_struct ndpi_flow;
};
union {
uint8_t ndpi_src_raw[SIZEOF_ID_STRUCT];
struct ndpi_id_struct ndpi_src;
};
union {
uint8_t ndpi_dst_raw[SIZEOF_ID_STRUCT];
struct ndpi_id_struct ndpi_dst;
};
}; };
struct nDPId_workflow struct nDPId_workflow
@@ -159,8 +170,6 @@ enum basic_event
CAPTURE_SIZE_SMALLER_THAN_PACKET_SIZE, CAPTURE_SIZE_SMALLER_THAN_PACKET_SIZE,
MAX_FLOW_TO_TRACK, MAX_FLOW_TO_TRACK,
FLOW_MEMORY_ALLOCATION_FAILED, FLOW_MEMORY_ALLOCATION_FAILED,
NDPI_FLOW_MEMORY_ALLOCATION_FAILED,
NDPI_ID_MEMORY_ALLOCATION_FAILED,
BASIC_EVENT_COUNT BASIC_EVENT_COUNT
}; };
@@ -206,8 +215,6 @@ static char const * const basic_event_name_table[BASIC_EVENT_COUNT] = {
[CAPTURE_SIZE_SMALLER_THAN_PACKET_SIZE] = "Captured packet size is smaller than packet size", [CAPTURE_SIZE_SMALLER_THAN_PACKET_SIZE] = "Captured packet size is smaller than packet size",
[MAX_FLOW_TO_TRACK] = "Max flows to track reached", [MAX_FLOW_TO_TRACK] = "Max flows to track reached",
[FLOW_MEMORY_ALLOCATION_FAILED] = "Flow memory allocation failed", [FLOW_MEMORY_ALLOCATION_FAILED] = "Flow memory allocation failed",
[NDPI_FLOW_MEMORY_ALLOCATION_FAILED] = "nDPI Flow memory allocation failed",
[NDPI_ID_MEMORY_ALLOCATION_FAILED] = "Not enough memory for src id struct",
}; };
static char const * const daemon_event_name_table[DAEMON_EVENT_COUNT] = { static char const * const daemon_event_name_table[DAEMON_EVENT_COUNT] = {
@@ -264,7 +271,7 @@ static char * const subopt_token[] = {[MAX_FLOWS_PER_THREAD] = "max-flows-per-th
static void free_workflow(struct nDPId_workflow ** const workflow); static void free_workflow(struct nDPId_workflow ** const workflow);
static void serialize_and_send(struct nDPId_reader_thread * const reader_thread); static void serialize_and_send(struct nDPId_reader_thread * const reader_thread);
static void jsonize_flow_event(struct nDPId_reader_thread * const reader_thread, static void jsonize_flow_event(struct nDPId_reader_thread * const reader_thread,
struct nDPId_flow_info const * const flow, struct nDPId_flow_info * const flow,
enum flow_event event); enum flow_event event);
static struct nDPId_workflow * init_workflow(char const * const file_or_device) static struct nDPId_workflow * init_workflow(char const * const file_or_device)
@@ -364,9 +371,6 @@ static void ndpi_flow_info_freer(void * const node)
{ {
struct nDPId_flow_info * const flow = (struct nDPId_flow_info *)node; struct nDPId_flow_info * const flow = (struct nDPId_flow_info *)node;
ndpi_free(flow->ndpi_dst);
ndpi_free(flow->ndpi_src);
ndpi_flow_free(flow->ndpi_flow);
ndpi_free(flow); ndpi_free(flow);
} }
@@ -590,7 +594,7 @@ static void process_idle_flow(struct nDPId_reader_thread * const reader_thread,
if (ndpi_is_protocol_detected(workflow->ndpi_struct, f->guessed_l7_protocol) == 0) if (ndpi_is_protocol_detected(workflow->ndpi_struct, f->guessed_l7_protocol) == 0)
{ {
f->guessed_l7_protocol = f->guessed_l7_protocol =
ndpi_detection_giveup(workflow->ndpi_struct, f->ndpi_flow, 1, &protocol_was_guessed); ndpi_detection_giveup(workflow->ndpi_struct, &f->ndpi_flow, 1, &protocol_was_guessed);
} }
else else
{ {
@@ -1023,7 +1027,7 @@ static void jsonize_packet_event(struct nDPId_reader_thread * const reader_threa
/* I decided against ndpi_flow2json as does not fulfill my needs. */ /* I decided against ndpi_flow2json as does not fulfill my needs. */
static void jsonize_flow_event(struct nDPId_reader_thread * const reader_thread, static void jsonize_flow_event(struct nDPId_reader_thread * const reader_thread,
struct nDPId_flow_info const * const flow, struct nDPId_flow_info * const flow,
enum flow_event event) enum flow_event event)
{ {
struct nDPId_workflow * const workflow = reader_thread->workflow; struct nDPId_workflow * const workflow = reader_thread->workflow;
@@ -1054,7 +1058,7 @@ static void jsonize_flow_event(struct nDPId_reader_thread * const reader_thread,
case FLOW_EVENT_NOT_DETECTED: case FLOW_EVENT_NOT_DETECTED:
case FLOW_EVENT_GUESSED: case FLOW_EVENT_GUESSED:
if (ndpi_dpi2json( if (ndpi_dpi2json(
workflow->ndpi_struct, flow->ndpi_flow, flow->guessed_l7_protocol, &workflow->ndpi_serializer) != 0) workflow->ndpi_struct, &flow->ndpi_flow, flow->guessed_l7_protocol, &workflow->ndpi_serializer) != 0)
{ {
syslog(LOG_DAEMON | LOG_ERR, syslog(LOG_DAEMON | LOG_ERR,
"[%8llu, %4u] ndpi_dpi2json failed for not-detected/guessed flow", "[%8llu, %4u] ndpi_dpi2json failed for not-detected/guessed flow",
@@ -1066,7 +1070,7 @@ static void jsonize_flow_event(struct nDPId_reader_thread * const reader_thread,
case FLOW_EVENT_DETECTED: case FLOW_EVENT_DETECTED:
case FLOW_EVENT_DETECTION_UPDATE: case FLOW_EVENT_DETECTION_UPDATE:
if (ndpi_dpi2json(workflow->ndpi_struct, if (ndpi_dpi2json(workflow->ndpi_struct,
flow->ndpi_flow, &flow->ndpi_flow,
flow->detected_l7_protocol, flow->detected_l7_protocol,
&workflow->ndpi_serializer) != 0) &workflow->ndpi_serializer) != 0)
{ {
@@ -1742,60 +1746,21 @@ static void ndpi_process_packet(uint8_t * const args,
flow_to_process->flow_id = global_flow_id++; flow_to_process->flow_id = global_flow_id++;
#endif #endif
flow_to_process->ndpi_flow = (struct ndpi_flow_struct *)ndpi_flow_malloc(SIZEOF_FLOW_STRUCT); memset(&flow_to_process->ndpi_flow, 0, (SIZEOF_FLOW_STRUCT > sizeof(struct ndpi_flow_struct) ?
if (flow_to_process->ndpi_flow == NULL) SIZEOF_FLOW_STRUCT : sizeof(struct ndpi_flow_struct)));
{ memset(&flow_to_process->ndpi_src, 0, (SIZEOF_ID_STRUCT > sizeof(struct ndpi_id_struct) ?
jsonize_packet_event(reader_thread, header, packet, type, ip_offset, NULL, PACKET_EVENT_PAYLOAD); SIZEOF_ID_STRUCT : sizeof(struct ndpi_id_struct)));
jsonize_basic_eventf(reader_thread, memset(&flow_to_process->ndpi_dst, 0, (SIZEOF_ID_STRUCT > sizeof(struct ndpi_id_struct) ?
NDPI_FLOW_MEMORY_ALLOCATION_FAILED, SIZEOF_ID_STRUCT : sizeof(struct ndpi_id_struct)));
"%s%u %s%zu",
"flow_id",
flow_to_process->flow_id,
"size",
SIZEOF_FLOW_STRUCT);
return;
}
memset(flow_to_process->ndpi_flow, 0, SIZEOF_FLOW_STRUCT);
flow_to_process->ndpi_src = (struct ndpi_id_struct *)ndpi_calloc(1, SIZEOF_ID_STRUCT);
if (flow_to_process->ndpi_src == NULL)
{
jsonize_packet_event(reader_thread, header, packet, type, ip_offset, NULL, PACKET_EVENT_PAYLOAD);
jsonize_basic_eventf(reader_thread,
NDPI_ID_MEMORY_ALLOCATION_FAILED,
"%s%u %s%zu %s%s",
"flow_id",
flow_to_process->flow_id,
"size",
SIZEOF_ID_STRUCT,
"direction",
"src");
return;
}
flow_to_process->ndpi_dst = (struct ndpi_id_struct *)ndpi_calloc(1, SIZEOF_ID_STRUCT);
if (flow_to_process->ndpi_dst == NULL)
{
jsonize_packet_event(reader_thread, header, packet, type, ip_offset, NULL, PACKET_EVENT_PAYLOAD);
jsonize_basic_eventf(reader_thread,
NDPI_ID_MEMORY_ALLOCATION_FAILED,
"%s%u %s%zu %s%s",
"flow_id",
flow_to_process->flow_id,
"size",
SIZEOF_ID_STRUCT,
"direction",
"dst");
return;
}
if (ndpi_tsearch(flow_to_process, &workflow->ndpi_flows_active[hashed_index], ndpi_workflow_node_cmp) == NULL) if (ndpi_tsearch(flow_to_process, &workflow->ndpi_flows_active[hashed_index], ndpi_workflow_node_cmp) == NULL)
{ {
/* Possible Leak, but should not happen as we'd abort earlier. */ /* Possible Leak, but should not happen as we'd abort earlier. */
return; return;
} }
ndpi_src = flow_to_process->ndpi_src; ndpi_src = &flow_to_process->ndpi_src;
ndpi_dst = flow_to_process->ndpi_dst; ndpi_dst = &flow_to_process->ndpi_dst;
is_new_flow = 1; is_new_flow = 1;
} }
@@ -1805,13 +1770,13 @@ static void ndpi_process_packet(uint8_t * const args,
if (direction_changed != 0) if (direction_changed != 0)
{ {
ndpi_src = flow_to_process->ndpi_dst; ndpi_src = &flow_to_process->ndpi_dst;
ndpi_dst = flow_to_process->ndpi_src; ndpi_dst = &flow_to_process->ndpi_src;
} }
else else
{ {
ndpi_src = flow_to_process->ndpi_src; ndpi_src = &flow_to_process->ndpi_src;
ndpi_dst = flow_to_process->ndpi_dst; ndpi_dst = &flow_to_process->ndpi_dst;
} }
} }
@@ -1849,11 +1814,11 @@ static void ndpi_process_packet(uint8_t * const args,
} }
/* We currently process max. 254 packets per flow. TODO: The user should decide this! */ /* We currently process max. 254 packets per flow. TODO: The user should decide this! */
if (flow_to_process->ndpi_flow->num_processed_pkts == 0xFF) if (flow_to_process->ndpi_flow.num_processed_pkts == 0xFF)
{ {
return; return;
} }
else if (flow_to_process->ndpi_flow->num_processed_pkts == 0xFE) else if (flow_to_process->ndpi_flow.num_processed_pkts == 0xFE)
{ {
if (flow_to_process->detection_completed != 0) if (flow_to_process->detection_completed != 0)
{ {
@@ -1864,7 +1829,7 @@ static void ndpi_process_packet(uint8_t * const args,
/* last chance to guess something, better then nothing */ /* last chance to guess something, better then nothing */
uint8_t protocol_was_guessed = 0; uint8_t protocol_was_guessed = 0;
flow_to_process->guessed_l7_protocol = flow_to_process->guessed_l7_protocol =
ndpi_detection_giveup(workflow->ndpi_struct, flow_to_process->ndpi_flow, 1, &protocol_was_guessed); ndpi_detection_giveup(workflow->ndpi_struct, &flow_to_process->ndpi_flow, 1, &protocol_was_guessed);
if (protocol_was_guessed != 0) if (protocol_was_guessed != 0)
{ {
jsonize_flow_event(reader_thread, flow_to_process, FLOW_EVENT_GUESSED); jsonize_flow_event(reader_thread, flow_to_process, FLOW_EVENT_GUESSED);
@@ -1877,7 +1842,7 @@ static void ndpi_process_packet(uint8_t * const args,
} }
flow_to_process->detected_l7_protocol = ndpi_detection_process_packet(workflow->ndpi_struct, flow_to_process->detected_l7_protocol = ndpi_detection_process_packet(workflow->ndpi_struct,
flow_to_process->ndpi_flow, &flow_to_process->ndpi_flow,
ip != NULL ? (uint8_t *)ip : (uint8_t *)ip6, ip != NULL ? (uint8_t *)ip : (uint8_t *)ip6,
ip_size, ip_size,
time_ms, time_ms,
@@ -1890,11 +1855,11 @@ static void ndpi_process_packet(uint8_t * const args,
flow_to_process->detection_completed = 1; flow_to_process->detection_completed = 1;
workflow->detected_flow_protocols++; workflow->detected_flow_protocols++;
jsonize_flow_event(reader_thread, flow_to_process, FLOW_EVENT_DETECTED); jsonize_flow_event(reader_thread, flow_to_process, FLOW_EVENT_DETECTED);
flow_to_process->last_ndpi_flow_struct_hash = calculate_ndpi_flow_struct_hash(flow_to_process->ndpi_flow); flow_to_process->last_ndpi_flow_struct_hash = calculate_ndpi_flow_struct_hash(&flow_to_process->ndpi_flow);
} }
else if (flow_to_process->detection_completed == 1) else if (flow_to_process->detection_completed == 1)
{ {
uint32_t hash = calculate_ndpi_flow_struct_hash(flow_to_process->ndpi_flow); uint32_t hash = calculate_ndpi_flow_struct_hash(&flow_to_process->ndpi_flow);
if (hash != flow_to_process->last_ndpi_flow_struct_hash) if (hash != flow_to_process->last_ndpi_flow_struct_hash)
{ {
jsonize_flow_event(reader_thread, flow_to_process, FLOW_EVENT_DETECTION_UPDATE); jsonize_flow_event(reader_thread, flow_to_process, FLOW_EVENT_DETECTION_UPDATE);