Removed example py-ja3-checker.

* renamed sklearn-ml.py to sklearn-random-forest.py (there is more to come!)
 * force all protocol classes to lower case

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

CMakeLists.txt

@@ -334,9 +334,6 @@ install(FILES examples/py-flow-info/flow-info.py
 install(FILES examples/py-flow-dashboard/flow-dash.py
         DESTINATION bin RENAME nDPIsrvd-flow-dash.py
         PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
-install(FILES examples/py-ja3-checker/py-ja3-checker.py
-        DESTINATION bin RENAME nDPIsrvd-ja3-checker.py
-        PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
 install(FILES examples/py-json-stdout/json-stdout.py
         DESTINATION bin RENAME nDPIsrvd-json-stdout.py
         PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
@@ -346,6 +343,9 @@ install(FILES examples/py-schema-validation/py-schema-validation.py
 install(FILES examples/py-semantic-validation/py-semantic-validation.py
         DESTINATION bin RENAME nDPIsrvd-semantic-validation.py
         PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
+install(FILES examples/py-machine-learning/sklearn-random-forest.py
+        DESTINATION bin RENAME nDPIsrvd-sklearn.py
+        PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
 install(FILES schema/error_event_schema.json schema/daemon_event_schema.json
               schema/flow_event_schema.json schema/packet_event_schema.json DESTINATION share/nDPId/json-schema)
 

examples/README.md

@@ -80,7 +80,3 @@ Required by `tests/run_tests.sh`
 
 Validate nDPId JSON strings against internal event semantics.
 Required by `tests/run_tests.sh`
-
-## py-ja3-checker
-
-Captures JA3 hashes from nDPIsrvd and checks them against known hashes from [ja3er.com](https://ja3er.com).

examples/py-ja3-checker/py-ja3-checker.py

@@ -1,143 +0,0 @@
-#!/usr/bin/env python3
-
-import io
-import json
-import os
-import pandas
-import requests
-import sys
-import time
-
-sys.path.append(os.path.dirname(sys.argv[0]) + '/../../dependencies')
-sys.path.append(os.path.dirname(sys.argv[0]) + '/../share/nDPId')
-sys.path.append(os.path.dirname(sys.argv[0]))
-sys.path.append(sys.base_prefix + '/share/nDPId')
-import nDPIsrvd
-from nDPIsrvd import nDPIsrvdSocket
-
-global ja3_fps
-ja3_fps = dict()
-# 1 hour = 3600 sec/hour = (60 minutes/hour) * (60 seconds/minute)
-JA3_FP_MAX_AGE = 60 * 60
-
-global ja3_bl
-ja3_bl = None
-
-global ja3_bl_printed
-ja3_bl_printed = dict()
-
-
-def downloadJA3Blacklist():
-    response = requests.get(
-        'https://sslbl.abuse.ch/blacklist/ja3_fingerprints.csv'
-    )
-    if response.status_code == 200:
-        global ja3_bl
-        ja3_bl = pandas.read_csv(io.StringIO(response.text), header=9)
-        return True
-    return False
-
-
-def getBlacklisted(ja3_hash):
-    global ja3_bl
-    return ja3_bl[(ja3_bl['# ja3_md5'] == ja3_hash)]
-
-
-def checkBlacklisted(ja3_hash):
-    if ja3_bl is None:
-        return
-    csv_entry = getBlacklisted(ja3_hash)
-    if not csv_entry.empty and ja3_hash not in ja3_bl_printed:
-        print('Found CSV JA3 blacklist entry:')
-        print(csv_entry)
-        ja3_bl_printed[ja3_hash] = True
-
-
-class JA3ER(object):
-    def __init__(self, json_dict):
-        self.json = json_dict
-        self.last_checked = time.time()
-
-    def isTooOld(self):
-        current_time = time.time()
-        if current_time - self.last_checked >= JA3_FP_MAX_AGE:
-            return True
-        return False
-
-
-def isJA3InfoTooOld(ja3_hash):
-    global ja3_fps
-    if ja3_hash in ja3_fps:
-        if ja3_fps[ja3_hash].isTooOld() is True:
-            print('Fingerprint {} too old, re-newing..'.format(ja3_hash))
-            return True
-    else:
-        return True
-
-    return False
-
-
-def getInfoFromJA3ER(ja3_hash):
-    global ja3_fps
-    response = requests.get('https://ja3er.com/search/' + ja3_hash)
-    if response.status_code == 200:
-        ja3_fps[ja3_hash] = JA3ER(json.loads(response.text, strict=True))
-        if 'error' not in ja3_fps[ja3_hash].json:
-            print('Fingerprints for JA3 {}:'.format(ja3_hash))
-            for ua in ja3_fps[ja3_hash].json:
-                if 'User-Agent' in ua:
-                    print('\tUser-Agent: {}\n'
-                          '\t            Last seen: {}, '
-                          'Count: {}'.format(ua['User-Agent'],
-                                             ua['Last_seen'],
-                                             ua['Count']))
-                elif 'Comment' in ua:
-                    print('\tComment...: {}\n'
-                          '\t            Reported: {}'
-                          .format(ua['Comment'].replace('\r', '')
-                                  .replace('\n', ' '), ua['Reported']))
-                else:
-                    print(ua)
-        else:
-            print('No fingerprint for JA3 {} found.'.format(ja3_hash))
-
-
-def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
-    if 'tls' in json_dict and 'ja3' in json_dict['tls']:
-
-        if json_dict['tls']['client_requested_server_name'] == 'ja3er.com':
-            return True
-
-        if isJA3InfoTooOld(json_dict['tls']['ja3']) is True:
-            getInfoFromJA3ER(json_dict['tls']['ja3'])
-
-        if isJA3InfoTooOld(json_dict['tls']['ja3']) is True:
-            getInfoFromJA3ER(json_dict['tls']['ja3s'])
-
-        checkBlacklisted(json_dict['tls']['ja3'])
-
-    return True
-
-
-if __name__ == '__main__':
-    argparser = nDPIsrvd.defaultArgumentParser()
-    args = argparser.parse_args()
-    address = nDPIsrvd.validateAddress(args)
-
-    sys.stderr.write('Recv buffer size: {}\n'
-                     .format(nDPIsrvd.NETWORK_BUFFER_MAX_SIZE))
-    sys.stderr.write('Connecting to {} ..\n'
-                     .format(address[0] + ':' +
-                             str(address[1])
-                             if type(address) is tuple else address))
-
-    if downloadJA3Blacklist() is False:
-        print('Could not download JA3 blacklist.')
-    nsock = nDPIsrvdSocket()
-    nsock.connect(address)
-    try:
-        nsock.loop(onJsonLineRecvd, None, None)
-    except nDPIsrvd.SocketConnectionBroken as err:
-        sys.stderr.write('\n{}\n'.format(err))
-    except KeyboardInterrupt:
-        print()

examples/py-machine-learning/sklearn-random-forest.py

@@ -159,7 +159,7 @@ def onJsonLineRecvd(json_dict, instance, current_flow, global_user_data):
         probs = probs[:-2]
 
         print('DPI Engine detected: {}{:>24}{}, Predicted: {}{:>24}{}, Score: {}, Probabilities: {}'.format(
-              color_start, json_dict['ndpi']['proto'], color_end,
+              color_start, json_dict['ndpi']['proto'].lower(), color_end,
               color_start, y_text, color_end, s, probs))
     except Exception as err:
         print('Got exception `{}\'\nfor json: {}'.format(err, json_dict))
@@ -219,6 +219,9 @@ if __name__ == '__main__':
     numpy.set_printoptions(formatter={'float_kind': "{:.1f}".format}, sign=' ')
     numpy.seterr(divide = 'ignore')
 
+    for i in range(len(args.proto_class)):
+        args.proto_class[i] = args.proto_class[i].lower()
+
     sys.stderr.write('Learning via CSV..\n')
     with open(args.csv, newline='\n') as csvfile:
         reader = csv.DictReader(csvfile, delimiter=',', quotechar='"')
@@ -232,7 +235,6 @@ if __name__ == '__main__':
 
         for line in reader:
             try:
-                #if isProtoClass(args.proto_class, line['proto']) > 0:
                 X += getRelevantFeaturesCSV(line)
                 y += [isProtoClass(args.proto_class, line['proto'])]
             except RuntimeError as err: