Chg: update image tag in helm values to v2.7.0-RC2

Fixes for WIFI-10839 and others

.github/workflows/cleanup.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - main
+      - 'release/*'
     types: [ closed ]
 
 defaults:
@@ -16,4 +17,10 @@ jobs:
     steps:
       - run: |
           export PR_BRANCH_TAG=$(echo ${GITHUB_HEAD_REF#refs/heads/} | tr '/' '-')
-          curl -uucentral:${{ secrets.DOCKER_REGISTRY_PASSWORD }} -X DELETE "https://tip.jfrog.io/artifactory/tip-wlan-cloud-ucentral/owanalytics/$PR_BRANCH_TAG"
+
+          if [[ ! $PR_BRANCH_TAG =~ (main|master|release-*) ]]; then
+            echo "PR branch is $PR_BRANCH_TAG, deleting Docker image"
+            curl -s -uucentral:${{ secrets.DOCKER_REGISTRY_PASSWORD }} -X DELETE "https://tip.jfrog.io/artifactory/tip-wlan-cloud-ucentral/owanalytics/$PR_BRANCH_TAG"
+          else
+            echo "PR branch is $PR_BRANCH_TAG, not deleting Docker image"
+          fi

.gitignore

@@ -32,6 +32,7 @@
 *.app
 
 test_scripts/curl/token.json
+test_scripts/curl/result.json
 .vscode/c_cpp_properties.json
 test_scripts/curl/result.json
 *.swp

CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.13)
-project(owanalytics VERSION 2.6.0)
+project(owanalytics VERSION 2.7.0)
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_GENERATOR "Unix Makefiles")
@@ -63,6 +63,8 @@ include_directories(/usr/local/include  /usr/local/opt/openssl/include src inclu
 
 configure_file(src/ow_version.h.in ${PROJECT_SOURCE_DIR}/src/ow_version.h @ONLY)
 
+add_definitions(-DPOCO_LOG_DEBUG="1")
+
 add_executable(owanalytics
         build
         src/ow_version.h.in

Dockerfile

@@ -1,4 +1,10 @@
-FROM alpine:3.15 AS build-base
+ARG ALPINE_VERSION=3.16.2
+ARG POCO_VERSION=poco-tip-v1
+ARG FMTLIB_VERSION=9.0.0
+ARG CPPKAFKA_VERSION=tip-v1
+ARG JSON_VALIDATOR_VERSION=2.1.0
+
+FROM alpine:$ALPINE_VERSION AS build-base
 
 RUN apk add --update --no-cache \
     make cmake g++ git \
@@ -9,8 +15,10 @@ RUN apk add --update --no-cache \
 
 FROM build-base AS poco-build
 
-ADD https://api.github.com/repos/stephb9959/poco/git/refs/heads/master version.json
-RUN git clone https://github.com/stephb9959/poco /poco
+ARG POCO_VERSION
+
+ADD https://api.github.com/repos/AriliaWireless/poco/git/refs/tags/${POCO_VERSION} version.json
+RUN git clone https://github.com/AriliaWireless/poco --branch ${POCO_VERSION} /poco
 
 WORKDIR /poco
 RUN mkdir cmake-build
@@ -19,10 +27,26 @@ RUN cmake ..
 RUN cmake --build . --config Release -j8
 RUN cmake --build . --target install
 
+FROM build-base AS fmtlib-build
+
+ARG FMTLIB_VERSION
+
+ADD https://api.github.com/repos/fmtlib/fmt/git/refs/tags/${FMTLIB_VERSION} version.json
+RUN git clone https://github.com/fmtlib/fmt --branch ${FMTLIB_VERSION} /fmtlib
+
+WORKDIR /fmtlib
+RUN mkdir cmake-build
+WORKDIR cmake-build
+RUN cmake ..
+RUN make
+RUN make install
+
 FROM build-base AS cppkafka-build
 
-ADD https://api.github.com/repos/stephb9959/cppkafka/git/refs/heads/master version.json
-RUN git clone https://github.com/stephb9959/cppkafka /cppkafka
+ARG CPPKAFKA_VERSION
+
+ADD https://api.github.com/repos/AriliaWireless/cppkafka/git/refs/tags/${CPPKAFKA_VERSION} version.json
+RUN git clone https://github.com/AriliaWireless/cppkafka --branch ${CPPKAFKA_VERSION} /cppkafka
 
 WORKDIR /cppkafka
 RUN mkdir cmake-build
@@ -33,8 +57,10 @@ RUN cmake --build . --target install
 
 FROM build-base AS json-schema-validator-build
 
-ADD https://api.github.com/repos/pboettch/json-schema-validator/git/refs/heads/master version.json
-RUN git clone https://github.com/pboettch/json-schema-validator /json-schema-validator
+ARG JSON_VALIDATOR_VERSION
+
+ADD https://api.github.com/repos/pboettch/json-schema-validator/git/refs/tags/${JSON_VALIDATOR_VERSION} version.json
+RUN git clone https://github.com/pboettch/json-schema-validator --branch ${JSON_VALIDATOR_VERSION} /json-schema-validator
 
 WORKDIR /json-schema-validator
 RUN mkdir cmake-build
@@ -43,18 +69,6 @@ RUN cmake ..
 RUN make
 RUN make install
 
-FROM build-base AS fmtlib-build
-
-ADD https://api.github.com/repos/fmtlib/fmt/git/refs/heads/master version.json
-RUN git clone https://github.com/fmtlib/fmt /fmtlib
-
-WORKDIR /fmtlib
-RUN mkdir cmake-build
-WORKDIR cmake-build
-RUN cmake ..
-RUN make
-RUN make install
-
 FROM build-base AS owanalytics-build
 
 ADD CMakeLists.txt build /owanalytics/
@@ -77,7 +91,7 @@ WORKDIR /owanalytics/cmake-build
 RUN cmake ..
 RUN cmake --build . --config Release -j8
 
-FROM alpine:3.15
+FROM alpine:$ALPINE_VERSION
 
 ENV OWANALYTICS_USER=owanalytics \
     OWANALYTICS_ROOT=/owanalytics-data \

build

@@ -1 +1 @@
-68
+18

docker-entrypoint.sh

@@ -5,7 +5,7 @@ if [ "$SELFSIGNED_CERTS" = 'true' ]; then
     update-ca-certificates
 fi
 
-if [[ "$TEMPLATE_CONFIG" = 'true' && ! -f "$OWANALYTICS_CONFIG"/owanalytics.properties ]]; then
+if [[ "$TEMPLATE_CONFIG" = 'true' ]]; then
   RESTAPI_HOST_ROOTCA=${RESTAPI_HOST_ROOTCA:-"\$OWANALYTICS_ROOT/certs/restapi-ca.pem"} \
   RESTAPI_HOST_PORT=${RESTAPI_HOST_PORT:-"16009"} \
   RESTAPI_HOST_CERT=${RESTAPI_HOST_CERT:-"\$OWANALYTICS_ROOT/certs/restapi-cert.pem"} \
@@ -24,6 +24,7 @@ if [[ "$TEMPLATE_CONFIG" = 'true' && ! -f "$OWANALYTICS_CONFIG"/owanalytics.prop
   SYSTEM_URI_PRIVATE=${SYSTEM_URI_PRIVATE:-"https://localhost:17009"} \
   SYSTEM_URI_PUBLIC=${SYSTEM_URI_PUBLIC:-"https://localhost:16009"} \
   SYSTEM_URI_UI=${SYSTEM_URI_UI:-"http://localhost"} \
+  SECURITY_RESTAPI_DISABLE=${SECURITY_RESTAPI_DISABLE:-"false"} \
   KAFKA_ENABLE=${KAFKA_ENABLE:-"true"} \
   KAFKA_BROKERLIST=${KAFKA_BROKERLIST:-"localhost:9092"} \
   KAFKA_SSL_CA_LOCATION=${KAFKA_SSL_CA_LOCATION:-""} \

helm/.gitignore

@@ -1 +1,3 @@
 *.swp
+Chart.lock
+charts/

helm/README.md

@@ -70,8 +70,8 @@ The following table lists the configurable parameters of the chart and their def
 | persistence.size | string | Defines PV size | `'10Gi'` |
 | public_env_variables | hash | Defines list of environment variables to be passed to the Analytics | |
 | configProperties | hash | Configuration properties that should be passed to the application in `owanalytics.properties`. May be passed by key in set (i.e. `configProperties."rtty\.token"`) | |
-| certs | hash | Defines files (keys and certificates) that should be passed to the Analytics (PEM format is adviced to be used) (see `volumes.owanalytics` on where it is mounted) |  |
-
+| existingCertsSecret | string | Existing Kubernetes secret containing all required certificates and private keys for microservice operation. If set, certificates from `certs` key are ignored | `""` |
+| certs | hash | Defines files (keys and certificates) that should be passed to the Gateway (PEM format is adviced to be used) (see `volumes.owanalytics` on where it is mounted). If `existingCertsSecret` is set, certificates passed this way will not be used. |  |
 
 Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example,
 

helm/templates/deployment.yaml

@@ -1,4 +1,5 @@
 {{- $root := . -}}
+{{- $storageType := index .Values.configProperties "storage.type" -}}
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -46,6 +47,39 @@ spec:
             - -timeout
             - 600s
 
+{{- if eq $storageType "postgresql" }}
+        - name: wait-postgres
+          image: "{{ .Values.images.owanalytics.repository }}:{{ .Values.images.owanalytics.tag }}"
+          imagePullPolicy: {{ .Values.images.owanalytics.pullPolicy }}
+          command:
+            - /wait-for-postgres.sh
+            - {{ index .Values.configProperties "storage.type.postgresql.host" }}
+            - echo
+            - "PostgreSQL is ready"
+          env:
+            - name: KUBERNETES_DEPLOYED
+              value: "{{ now }}"
+          {{- range $key, $value := .Values.public_env_variables }}
+            - name: {{ $key }}
+              value: {{ $value | quote }}
+          {{- end }}
+          {{- range $key, $value := .Values.secret_env_variables }}
+            - name: {{ $key }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "owanalytics.fullname" $root }}-env
+                  key: {{ $key }}
+          {{- end }}
+          volumeMounts:
+          {{- range .Values.volumes.owanalytics }}
+          - name: {{ .name }}
+            mountPath: {{ .mountPath }}
+            {{- if .subPath }}
+            subPath: {{ .subPath }}
+            {{- end }}
+          {{- end }}
+{{- end }}
+
       containers:
 
         - name: owanalytics

helm/values.yaml

@@ -9,7 +9,7 @@ fullnameOverride: ""
 images:
   owanalytics:
     repository: tip-tip-wlan-cloud-ucentral.jfrog.io/owanalytics
-    tag: main
+    tag: v2.7.0-RC2
     pullPolicy: Always
 #    regcred:
 #      registry: tip-tip-wlan-cloud-ucentral.jfrog.io
@@ -71,7 +71,7 @@ volumes:
       mountPath: /owanalytics-data/certs
       volumeDefinition: |
         secret:
-          secretName: {{ include "owanalytics.fullname" . }}-certs
+          secretName: {{ if .Values.existingCertsSecret }}{{ .Values.existingCertsSecret }}{{ else }}{{ include "owanalytics.fullname" . }}-certs{{ end }}
     # Change this if you want to use another volume type
     - name: persist
       mountPath: /owanalytics-data/persist
@@ -198,6 +198,9 @@ configProperties:
   storage.type.mysql.username: stephb
   storage.type.mysql.password: snoopy99
 
+# NOTE: List of required certificates may be found in "certs" key. Alternative way to pass required certificates is to create external secret with all required certificates and set secret name in "existingCertsSecret" key. Details may be found in https://github.com/Telecominfraproject/wlan-cloud-ucentral-deploy/tree/main/chart#tldr
+existingCertsSecret: ""
+
 certs:
   # restapi-ca.pem: ""
   # restapi-cert.pem: ""

openapi/owanalytics.yaml

@@ -27,71 +27,13 @@ components:
 
   responses:
     NotFound:
-      description: The specified resource was not found.
-      content:
-        application/json:
-          schema:
-            properties:
-              ErrorCode:
-                type: integer
-              ErrorDetails:
-                type: string
-              ErrorDescription:
-                type: string
-
+      $ref: 'https://github.com/Telecominfraproject/wlan-cloud-ucentralsec/blob/main/openpapi/owsec.yaml#/components/responses/NotFound'
     Unauthorized:
-      description: The requested does not have sufficient rights to perform the operation.
-      content:
-        application/json:
-          schema:
-            properties:
-              ErrorCode:
-                type: integer
-                enum:
-                  - 0     # Success
-                  - 1     # PASSWORD_CHANGE_REQUIRED,
-                  - 2     # INVALID_CREDENTIALS,
-                  - 3     # PASSWORD_ALREADY_USED,
-                  - 4     # USERNAME_PENDING_VERIFICATION,
-                  - 5     # PASSWORD_INVALID,
-                  - 6     # INTERNAL_ERROR,
-                  - 7     # ACCESS_DENIED,
-                  - 8     # INVALID_TOKEN
-                  - 9     # EXPIRED_TOKEN
-                  - 10    # RATE_LIMIT_EXCEEDED
-                  - 11    # BAD_MFA_TRANSACTION
-                  - 12    # MFA_FAILURE
-                  - 13    # SECURITY_SERVICE_UNREACHABLE
-              ErrorDetails:
-                type: string
-              ErrorDescription:
-                type: string
-
+      $ref: 'https://github.com/Telecominfraproject/wlan-cloud-ucentralsec/blob/main/openpapi/owsec.yaml#/components/responses/Unauthorized'
     Success:
-      description: The requested operation was performed.
-      content:
-        application/json:
-          schema:
-            properties:
-              Operation:
-                type: string
-              Details:
-                type: string
-              Code:
-                type: integer
-
+      $ref: 'https://github.com/Telecominfraproject/wlan-cloud-ucentralsec/blob/main/openpapi/owsec.yaml#/components/responses/Success'
     BadRequest:
-      description: The requested operation failed.
-      content:
-        application/json:
-          schema:
-            properties:
-              ErrorCode:
-                type: integer
-              ErrorDetails:
-                type: string
-              ErrorDescription:
-                type: integer
+      $ref: 'https://github.com/Telecominfraproject/wlan-cloud-ucentralsec/blob/main/openpapi/owsec.yaml#/components/responses/BadRequest'
 
   schemas:
     ObjectInfo:
@@ -1134,42 +1076,6 @@ paths:
         404:
           $ref: '#/components/responses/NotFound'
 
-
-  /iptocountry:
-    get:
-      tags:
-        - Utility
-      summary: Get the country code for an IP address
-      operationId: getIpToCountry
-      parameters:
-        - in: query
-          name: iplist
-          schema:
-            type: string
-            example:
-              10.2.2.2,10.3.4.3
-          required: true
-      responses:
-        200:
-          description: List of country codes.
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  enabled:
-                    type: boolean
-                  countryCodes:
-                    type: array
-                    items:
-                      type: string
-        400:
-          $ref: '#/components/responses/BadRequest'
-        403:
-          $ref: '#/components/responses/Unauthorized'
-        404:
-          $ref: '#/components/responses/NotFound'
-
   /wifiClientHistory:
     get:
       tags:

owanalytics.properties

@@ -34,6 +34,7 @@ openwifi.system.uri.private = https://localhost:17009
 openwifi.system.uri.public = https://ucentral.dpaas.arilia.com:16009
 openwifi.system.commandchannel = /tmp/app.owanalytics
 openwifi.system.uri.ui = owprov-ui.arilia.com
+openwifi.security.restapi.disable = false
 
 #############################
 # Generic information for all micro services
@@ -101,4 +102,4 @@ storage.type.mysql.connectiontimeout = 60
 ########################################################################
 logging.type = file
 logging.path = $OWANALYTICS_ROOT/logs
-logging.level = debug
+logging.level = debug

owanalytics.properties.tmpl

@@ -39,6 +39,7 @@ openwifi.system.uri.private = ${SYSTEM_URI_PRIVATE}
 openwifi.system.uri.public = ${SYSTEM_URI_PUBLIC}
 openwifi.system.commandchannel = /tmp/app.owanalytics
 openwifi.system.uri.ui = ${SYSTEM_URI_UI}
+openwifi.security.restapi.disable = ${SECURITY_RESTAPI_DISABLE}
 
 #############################
 # Generic information for all micro services

src/APStats.cpp

@@ -93,7 +93,7 @@ namespace OpenWifi {
         DI_.connected =true;
 
         AnalyticsObjects::DeviceTimePoint DTP;
-        Logger().information(fmt::format("{}: stats message.", DI_.serialNumber));
+        poco_information(Logger(),fmt::format("{}: stats message.", DI_.serialNumber));
 
         // find radios first to get associations.
         try {
@@ -158,6 +158,7 @@ namespace OpenWifi {
             auto interfaces = (*State)["interfaces"];
             DI_.associations_2g = DI_.associations_5g = DI_.associations_6g = 0;
             for(const auto &interface:interfaces) {
+                std::string InterfaceName = fmt::format("{}: {}", DI_.serialNumber, interface.contains("name") ? to_string(interface["name"]) : "unknown");
                 if(interface.contains("counters")) {
                     auto counters = interface["counters"];
                     GetJSON("collisions", counters, DTP.ap_data.collisions, (uint64_t) 0);
@@ -172,6 +173,34 @@ namespace OpenWifi {
                     GetJSON("tx_packets", counters, DTP.ap_data.tx_packets, (uint64_t) 0);
                 }
 
+                InterfaceClientEntryMap_t ICEM;
+                if(interface.contains("clients") && interface["clients"].is_array()) {
+                    try {
+                        auto Clients = interface["clients"];
+                        for(const auto & client: Clients) {
+                            if(client.contains("mac") && client["mac"].is_string()) {
+                                InterfaceClientEntry E;
+                                if(client.contains("ipv4_addresses") && client["ipv4_addresses"].is_array()) {
+                                    for(const auto &ip:client["ipv4_addresses"]) {
+                                        E.ipv4_addresses.push_back(ip);
+                                    }
+                                }
+                                if(client.contains("ipv6_addresses") && client["ipv6_addresses"].is_array()) {
+                                    for(const auto &ip:client["ipv6_addresses"]) {
+                                        E.ipv6_addresses.push_back(ip);
+                                    }
+                                }
+                                auto M = mac_filter(client["mac"]);
+                                ICEM[M] = E;
+                            }
+                        }
+                    } catch(...) {
+                        std::cout << "Exception will parsing clients: " << InterfaceName << std::endl;
+                    }
+                } else {
+                    // std::cout <<"Interface: No clients: " << InterfaceName << std::endl;
+                }
+
                 if(interface.contains("ssids")) {
                     auto ssids = interface["ssids"];
                     for (const auto &ssid: ssids) {
@@ -243,8 +272,19 @@ namespace OpenWifi {
                                 GetJSON("rx_packets",association,WFH.rx_packets,(uint64_t)0);
                                 GetJSON("tx_packets",association,WFH.tx_packets,(uint64_t)0);
 
-                                WFH.ipv4 = "---";
-                                WFH.ipv6 = "----";
+                                // try to locate the IP addresses
+                                auto ClientInfo = ICEM.find(WFH.station_id);
+                                if(ClientInfo!=end(ICEM)) {
+                                    if(!ClientInfo->second.ipv4_addresses.empty()) {
+                                        WFH.ipv4 = ClientInfo->second.ipv4_addresses[0];
+                                    }
+                                    if(!ClientInfo->second.ipv6_addresses.empty()) {
+                                        WFH.ipv6 = ClientInfo->second.ipv6_addresses[0];
+                                    }
+                                    std::cout << __LINE__ << ": " << InterfaceName << "   Mac Found: " << ICEM.size() << " entries. " << WFH.station_id << std::endl;
+                                } else {
+                                    std::cout << __LINE__ << ": " << InterfaceName << "   Mac NOT found: " << ICEM.size() << " entries. " << WFH.station_id << std::endl;
+                                }
 
                                 for(const auto &rd:DTP.radio_data) {
                                     if(rd.band == SSIDTP.band) {
@@ -311,7 +351,7 @@ namespace OpenWifi {
             }
             DTP.device_info = DI_;
         } catch (...) {
-            Logger().information(fmt::format("{}: stats failed parsing.", DI_.serialNumber));
+            poco_information(Logger(),fmt::format("{}: stats failed parsing.", DI_.serialNumber));
         }
 
         if(got_base) {
@@ -477,7 +517,7 @@ namespace OpenWifi {
                 GetJSON("locale", ConnectionData, DI_.locale, std::string{} );
             }
         } catch (...) {
-            Logger().information(fmt::format("{}: error parsing connection message.", DI_.serialNumber));
+            poco_information(Logger(),fmt::format("{}: error parsing connection message.", DI_.serialNumber));
         }
     }
 
@@ -488,7 +528,7 @@ namespace OpenWifi {
             GetJSON("sanity", *Health, DI_.health, (uint64_t)0 );
             poco_debug(Logger(),fmt::format("{}: health message.", DI_.serialNumber));
         } catch(...) {
-            Logger().information(fmt::format("{}: error parsing health message.", DI_.serialNumber));
+            poco_information(Logger(),fmt::format("{}: error parsing health message.", DI_.serialNumber));
         }
     }
 }

src/APStats.h

@@ -11,6 +11,13 @@
 
 namespace OpenWifi {
 
+    struct InterfaceClientEntry {
+        std::vector<std::string>        ipv4_addresses;
+        std::vector<std::string>        ipv6_addresses;
+    };
+
+    using InterfaceClientEntryMap_t = std::map<std::string,InterfaceClientEntry>;
+
     class AP {
     public:
         explicit AP(uint64_t mac, const std::string &venue_id, const std::string &BoardId, Poco::Logger &L) :

src/DeviceStatusReceiver.cpp

@@ -23,6 +23,7 @@ namespace OpenWifi {
     }
 
     void DeviceStatusReceiver::run() {
+        Utils::SetThreadName("dev-status");
         Poco::AutoPtr<Poco::Notification>	Note(Queue_.waitDequeueNotification());
         while(Note && Running_) {
             auto Msg = dynamic_cast<DeviceStatusMessage *>(Note.get());

src/HealthReceiver.cpp

@@ -23,6 +23,7 @@ namespace OpenWifi {
     }
 
     void HealthReceiver::run() {
+        Utils::SetThreadName("dev-health");
         Poco::AutoPtr<Poco::Notification>	Note(Queue_.waitDequeueNotification());
         while(Note && Running_) {
             auto Msg = dynamic_cast<HealthMessage *>(Note.get());

src/RESTAPI/RESTAPI_board_devices_handler.cpp

@@ -8,8 +8,13 @@
 
 namespace OpenWifi {
     void RESTAPI_board_devices_handler::DoGet() {
+
         auto id = GetBinding("id","");
 
+        if(id.empty()) {
+            return BadRequest(RESTAPI::Errors::MissingUUID);
+        }
+
         AnalyticsObjects::BoardInfo BI;
         if(!StorageService()->BoardsDB().GetRecord("id",id,BI)) {
             return NotFound();

src/RESTAPI/RESTAPI_board_handler.cpp

@@ -90,7 +90,7 @@ namespace OpenWifi {
         }
 
         if(StorageService()->BoardsDB().UpdateRecord("id",Existing.info.id,Existing)) {
-            VenueCoordinator()->ModifyBoard(Existing.info.id);
+            VenueCoordinator()->UpdateBoard(Existing.info.id);
             AnalyticsObjects::BoardInfo NewBoard;
             StorageService()->BoardsDB().GetRecord("id",Existing.info.id,NewBoard);
             Poco::JSON::Object  Answer;

src/RESTAPI/RESTAPI_board_timepoint_handler.cpp

@@ -5,54 +5,11 @@
 #include "RESTAPI_board_timepoint_handler.h"
 #include "StorageService.h"
 
+#include <algorithm>
+
 namespace OpenWifi {
-    static auto find_number_of_buckets(std::vector<AnalyticsObjects::DeviceTimePoint> &p) {
-        uint32_t buckets=0,cur_buckets=0;
-        std::string current_serialNumber;
-        for(const auto &i:p) {
-            if(current_serialNumber.empty()) {
-                current_serialNumber = i.device_info.serialNumber;
-                cur_buckets=0;
-            }
-            if(current_serialNumber==i.device_info.serialNumber) {
-                cur_buckets++;
-            } else {
-                buckets = std::max(buckets,cur_buckets);
-                current_serialNumber=i.device_info.serialNumber;
-                cur_buckets=1;
-            }
-        }
-        return std::max(buckets,cur_buckets);
-    }
-
-    typedef std::vector< std::vector<AnalyticsObjects::DeviceTimePoint>> split_points;
-
-    static void split_in_buckets([[maybe_unused]] uint32_t buckets,std::vector<AnalyticsObjects::DeviceTimePoint> &p,split_points &sp) {
-        std::string cur_sn;
-        uint32_t cur_bucket=0;
-        for(const auto &i:p) {
-            if(cur_sn.empty()) {
-                cur_bucket=1;
-                cur_sn=i.device_info.serialNumber;
-            }
-            if(cur_sn==i.device_info.serialNumber) {
-                if (cur_bucket>sp.size()) {
-                    std::vector<AnalyticsObjects::DeviceTimePoint>  tmp_p;
-                    tmp_p.push_back(i);
-                    sp.push_back(tmp_p);
-                    cur_bucket++;
-                } else {
-                    sp[cur_bucket-1].push_back(i);
-                    cur_bucket++;
-                }
-            } else {
-                cur_bucket=1;
-                sp[cur_bucket-1].push_back(i);
-                cur_bucket++;
-                cur_sn=i.device_info.serialNumber;
-            }
-        }
-    }
+    typedef std::vector<std::pair<std::uint64_t , std::uint64_t >>          bucket_timespans;
+    typedef std::vector< std::vector<AnalyticsObjects::DeviceTimePoint>>    split_points;
 
     template <typename X, typename M> void AverageAPData( X T, const std::vector<M> &Values, AnalyticsObjects::AveragePoint &P) {
         if(Values.empty())
@@ -89,6 +46,81 @@ namespace OpenWifi {
             P.avg = 0.0;
     }
 
+    static void NewSort(const AnalyticsObjects::DeviceTimePointList &l,split_points &sp) {
+
+        struct {
+            bool operator()(const AnalyticsObjects::DeviceTimePoint &lhs, const AnalyticsObjects::DeviceTimePoint &rhs) const {
+                if (lhs.device_info.serialNumber < rhs.device_info.serialNumber) return true;
+                if (lhs.device_info.serialNumber > rhs.device_info.serialNumber) return false;
+                return lhs.timestamp < rhs.timestamp;
+            }
+        } sort_serial_ts;
+
+        // attempt at finding an interval
+        AnalyticsObjects::DeviceTimePointList tmp{l};
+        std::sort(tmp.points.begin(),tmp.points.end(),sort_serial_ts);
+
+        std::string     cur_ser;
+        std::uint64_t   cur_int=0,start_val, last_val, first_val = 0;
+        for(const auto &point:tmp.points) {
+            if(cur_ser.empty()) {
+                start_val = point.timestamp;
+                cur_ser = point.serialNumber;
+                first_val = point.timestamp;
+                continue;
+            }
+
+            if(cur_ser==point.serialNumber) {
+                auto this_int = point.timestamp - start_val;
+                if(cur_int) {
+                    if(this_int<cur_int) {
+                        cur_int = this_int;
+                    }
+                } else {
+                    cur_int = this_int;
+                }
+                start_val = point.timestamp;
+            } else {
+                cur_ser = point.serialNumber;
+                start_val = point.timestamp;
+            }
+            last_val = point.timestamp;
+        }
+
+        // std::cout << "Intervals: " << cur_int << std::endl;
+
+        std::vector<std::pair<std::uint64_t,std::uint64_t>>     time_slots;         //  timeslot 0 has <t1,t2>
+        std::vector<std::set<std::string>>                      serial_numbers;     //  serial number already in a timeslot.
+
+
+        std::uint64_t cur_first=first_val,cur_end=0;
+        sp.clear();
+        while(cur_end<last_val) {
+            std::pair<std::uint64_t,std::uint64_t>  e;
+            e.first = cur_first;
+            e.second = e.first + cur_int-1;
+            cur_first = e.second+1;
+            cur_end = e.second;
+            time_slots.emplace_back(e);
+            std::set<std::string> q;
+            serial_numbers.emplace_back(q);
+            std::vector<AnalyticsObjects::DeviceTimePoint>  qq;
+            sp.emplace_back(qq);
+        }
+
+        for(const auto &point:tmp.points) {
+            std::uint64_t slot_index=0;
+            for(const auto &slot:time_slots) {
+                if(point.timestamp >= slot.first && point.timestamp <= slot.second) {
+                    serial_numbers[slot_index].insert(point.serialNumber);
+                    sp[slot_index].emplace_back(point);
+                }
+                slot_index++;
+            }
+        }
+
+    }
+
     void RESTAPI_board_timepoint_handler::DoGet() {
         auto id = GetBinding("id","");
         if(id.empty() || !Utils::ValidUUID(id)) {
@@ -102,7 +134,12 @@ namespace OpenWifi {
 
         auto fromDate = GetParameter("fromDate",0);
         auto endDate = GetParameter("endDate",0);
-        auto maxRecords = GetParameter("maxRecords",1000);
+        std::uint64_t maxRecords;
+        if(Request->has("limit"))
+            maxRecords = QB_.Limit;
+        else
+            maxRecords = GetParameter("maxRecords",1000);
+
         auto statsOnly = GetBoolParameter("statsOnly");
         auto pointsOnly = GetBoolParameter("pointsOnly");
         auto pointsStatsOnly = GetBoolParameter("pointsStatsOnly");
@@ -115,71 +152,39 @@ namespace OpenWifi {
             return ReturnObject(Answer);
         }
 
-        auto Points = std::make_unique<AnalyticsObjects::DeviceTimePointList>();
-        StorageService()->TimePointsDB().SelectRecords(id,fromDate, endDate, maxRecords, Points->points);
+        AnalyticsObjects::DeviceTimePointList Points;
+        StorageService()->TimePointsDB().SelectRecords(id,fromDate, endDate, maxRecords, Points.points);
+        std::cout << "1 MaxRecords=" << maxRecords << " retrieved=" << Points.points.size() << std::endl;
 
-        // sort by timestamp & serial number.
-        struct {
-            bool operator()(const AnalyticsObjects::DeviceTimePoint &lhs, const AnalyticsObjects::DeviceTimePoint &rhs) const {
-                if(lhs.device_info.serialNumber < rhs.device_info.serialNumber) return true;
-                if(lhs.device_info.serialNumber > rhs.device_info.serialNumber) return false;
-                return lhs.timestamp < rhs.timestamp;
-            }
-        } DeviceTimePoint_sort;
+        split_points sp;
 
-        struct {
-            bool operator()(const AnalyticsObjects::SSIDTimePoint &lhs, const AnalyticsObjects::SSIDTimePoint &rhs) const {
-                if(lhs.ssid < rhs.ssid) return true;
-                if(lhs.ssid > rhs.ssid) return false;
-                return lhs.bssid < rhs.bssid;
-            }
-        } SSID_sort;
+        NewSort(Points,sp);
+        std::cout << __LINE__ << std::endl;
 
-        struct {
-            bool operator()(const AnalyticsObjects::UETimePoint &lhs, const AnalyticsObjects::UETimePoint &rhs) const {
-                if(lhs.station < rhs.station) return true;
-                return false;
-            }
-        } Association_sort;
-
-        std::sort( Points->points.begin(), Points->points.end(), DeviceTimePoint_sort);
-        auto BucketsNeeded = find_number_of_buckets(Points->points);
-
-        auto sp = std::make_unique<split_points>();
-        split_in_buckets(BucketsNeeded,Points->points, *sp);
-        // must sort each bucket according to serial number.
-        for(auto &i: *sp) {
-            std::sort(i.begin(),i.end(),DeviceTimePoint_sort);
-            // now sort according to UEs within a block
-            for(auto &j:i) {
-                std::sort(j.ssid_data.begin(),j.ssid_data.end(),SSID_sort);
-                for(auto &k:j.ssid_data) {
-                    std::sort(k.associations.begin(),k.associations.end(),Association_sort);
-                }
-            }
-        }
-
-        auto Answer = std::make_unique<Poco::JSON::Object>();
+        Poco::JSON::Object  Answer;
         if(!pointsStatsOnly) {
-            auto Points_OuterArray = std::make_unique<Poco::JSON::Array>();
-            for (const auto &point_list:*sp) {
+            Poco::JSON::Array   Points_OuterArray;
+            for (const auto &point_list:sp) {
                 Poco::JSON::Array Points_InnerArray;
                 for (const auto &point: point_list) {
                     Poco::JSON::Object O;
                     point.to_json(O);
                     Points_InnerArray.add(O);
                 }
-                Points_OuterArray->add(Points_InnerArray);
+                Points_OuterArray.add(Points_InnerArray);
             }
-            Answer->set("points",*Points_OuterArray);
+            Answer.set("points",Points_OuterArray);
         }
 
         //  calculate the stats for each time slot
         if(!pointsOnly) {
-            auto Stats_Array = std::make_unique<Poco::JSON::Array>();
-            for (const auto &point_list:*sp) {
+            Poco::JSON::Array   Stats_Array;
+            for (const auto &point_list:sp) {
                 AnalyticsObjects::DeviceTimePointAnalysis DTPA;
 
+                if(point_list.empty())
+                    continue;
+
                 DTPA.timestamp = point_list[0].timestamp;
                 AverageAPData(&AnalyticsObjects::APTimePoint::tx_bytes_bw, point_list, DTPA.tx_bytes_bw);
                 AverageAPData(&AnalyticsObjects::APTimePoint::rx_bytes_bw, point_list, DTPA.rx_bytes_bw);
@@ -200,12 +205,12 @@ namespace OpenWifi {
 
                 Poco::JSON::Object Stats_point;
                 DTPA.to_json(Stats_point);
-                Stats_Array->add(Stats_point);
+                Stats_Array.add(Stats_point);
             }
-            Answer->set("stats", *Stats_Array);
+            Answer.set("stats", Stats_Array);
         }
 
-        return ReturnObject(*Answer);
+        return ReturnObject(Answer);
     }
 
     void RESTAPI_board_timepoint_handler::DoDelete() {

src/RESTAPI/RESTAPI_wificlienthistory_handler.cpp

@@ -20,10 +20,9 @@ namespace OpenWifi {
         }
 
         if(GetBoolParameter("macsOnly")) {
-
             auto macFilter = GetParameter("macFilter","");
             std::vector<uint64_t>    Macs;
-            WifiClientCache()->FindNumbers(venue,macFilter,500,Macs);
+            WifiClientCache()->FindNumbers(venue,macFilter,QB_.Offset, QB_.Limit, Macs);
             Poco::JSON::Array   Arr;
             for(const auto &mac: Macs)
                 Arr.add(Utils::IntToSerialNumber(mac));

src/RESTObjects/RESTAPI_GWobjects.cpp

@@ -203,6 +203,10 @@ namespace OpenWifi::GWObjects {
 		field_to_json(Obj,"kafkaClients", kafkaClients);
 		field_to_json(Obj,"kafkaPackets", kafkaPackets);
 		field_to_json(Obj,"locale", locale);
+		field_to_json(Obj,"started", started);
+		field_to_json(Obj,"sessionId", sessionId);
+		field_to_json(Obj,"connectionCompletionTime", connectionCompletionTime);
+		field_to_json(Obj,"totalConnectionTime", OpenWifi::Now() - started);
 
 		switch(VerifiedCertificate) {
 			case NO_CERTIFICATE:
@@ -218,6 +222,21 @@ namespace OpenWifi::GWObjects {
 		}
 	}
 
+	void DeviceConnectionStatistics::to_json(Poco::JSON::Object &Obj) const {
+		field_to_json(Obj,"averageConnectionTime", averageConnectionTime);
+		field_to_json(Obj,"connectedDevices", connectedDevices );
+	}
+
+	bool DeviceConnectionStatistics::from_json(const Poco::JSON::Object::Ptr &Obj) {
+		try {
+			field_from_json(Obj,"averageConnectionTime", averageConnectionTime);
+			field_from_json(Obj,"connectedDevices", connectedDevices );
+			return true;
+		} catch (const Poco::Exception &E) {
+		}
+		return false;
+	}
+
 	void RttySessionDetails::to_json(Poco::JSON::Object &Obj) const {
 		field_to_json(Obj,"serialNumber", SerialNumber);
 		field_to_json(Obj,"server", Server);
@@ -293,7 +312,6 @@ namespace OpenWifi::GWObjects {
 		} catch (const Poco::Exception &E) {
 		}
 		return false;
-
 	}
 
 	void RadiusProxyPoolList::to_json(Poco::JSON::Object &Obj) const {
@@ -314,6 +332,8 @@ namespace OpenWifi::GWObjects {
 		field_to_json(Obj,"description",description);
 		field_to_json(Obj,"authConfig",authConfig);
 		field_to_json(Obj,"acctConfig",acctConfig);
+		field_to_json(Obj,"coaConfig",coaConfig);
+		field_to_json(Obj,"useByDefault",useByDefault);
 	}
 
 	bool RadiusProxyPool::from_json(const Poco::JSON::Object::Ptr &Obj) {
@@ -322,6 +342,8 @@ namespace OpenWifi::GWObjects {
 			field_from_json(Obj,"description",description);
 			field_from_json(Obj,"authConfig",authConfig);
 			field_from_json(Obj,"acctConfig",acctConfig);
+			field_from_json(Obj,"coaConfig",coaConfig);
+			field_from_json(Obj,"useByDefault",useByDefault);
 			return true;
 		} catch (const Poco::Exception &E) {
 		}
@@ -329,7 +351,7 @@ namespace OpenWifi::GWObjects {
 	}
 
 	void RadiusProxyServerConfig::to_json(Poco::JSON::Object &Obj) const {
-		field_to_json(Obj,"policy",strategy);
+		field_to_json(Obj,"strategy",strategy);
 		field_to_json(Obj,"monitor",monitor);
 		field_to_json(Obj,"monitorMethod",monitorMethod);
 		field_to_json(Obj,"methodParameters",methodParameters);
@@ -338,7 +360,7 @@ namespace OpenWifi::GWObjects {
 
 	bool RadiusProxyServerConfig::from_json(const Poco::JSON::Object::Ptr &Obj) {
 		try {
-			field_from_json(Obj,"policy",strategy);
+			field_from_json(Obj,"strategy",strategy);
 			field_from_json(Obj,"monitor",monitor);
 			field_from_json(Obj,"monitorMethod",monitorMethod);
 			field_from_json(Obj,"methodParameters",methodParameters);
@@ -354,6 +376,16 @@ namespace OpenWifi::GWObjects {
 		field_to_json(Obj,"ip",ip);
 		field_to_json(Obj,"port",port);
 		field_to_json(Obj,"weight",weight);
+		field_to_json(Obj,"secret",secret);
+		field_to_json(Obj,"certificate",certificate);
+		field_to_json(Obj,"radsec",radsec);
+		field_to_json(Obj,"radsecPort",radsecPort);
+		field_to_json(Obj,"radsecSecret",radsecSecret);
+		field_to_json(Obj,"radsecCacerts",radsecCacerts);
+		field_to_json(Obj,"radsecCert",radsecCert);
+		field_to_json(Obj,"radsecKey",radsecKey);
+		field_to_json(Obj,"radsecRealms",radsecRealms);
+		field_to_json(Obj,"ignore",ignore);
 	}
 
 	bool RadiusProxyServerEntry::from_json(const Poco::JSON::Object::Ptr &Obj) {
@@ -362,6 +394,16 @@ namespace OpenWifi::GWObjects {
 			field_from_json(Obj,"ip",ip);
 			field_from_json(Obj,"port",port);
 			field_from_json(Obj,"weight",weight);
+			field_from_json(Obj,"secret",secret);
+			field_from_json(Obj,"certificate",certificate);
+			field_from_json(Obj,"radsec",radsec);
+			field_from_json(Obj,"radsecSecret",radsecSecret);
+			field_from_json(Obj,"radsecPort",radsecPort);
+			field_from_json(Obj,"radsecCacerts",radsecCacerts);
+			field_from_json(Obj,"radsecCert",radsecCert);
+			field_from_json(Obj,"radsecKey",radsecKey);
+			field_from_json(Obj,"radsecRealms",radsecRealms);
+			field_from_json(Obj,"ignore",ignore);
 			return true;
 		} catch (const Poco::Exception &E) {
 		}

src/RESTObjects/RESTAPI_GWobjects.h

@@ -38,6 +38,10 @@ namespace OpenWifi::GWObjects {
 		uint64_t 	kafkaPackets=0;
 		uint64_t 	websocketPackets=0;
 		std::string locale;
+		uint64_t 	started=0;
+		uint64_t 	sessionId=0;
+		double      connectionCompletionTime=0.0;
+
 		void to_json(Poco::JSON::Object &Obj) const;
 	};
 
@@ -71,6 +75,13 @@ namespace OpenWifi::GWObjects {
 		void Print() const;
 	};
 
+	struct DeviceConnectionStatistics {
+		std::uint64_t connectedDevices = 0;
+		std::uint64_t averageConnectionTime = 0;
+		void to_json(Poco::JSON::Object &Obj) const;
+		bool from_json(const Poco::JSON::Object::Ptr &Obj);
+	};
+
 	struct Statistics {
 		std::string SerialNumber;
 		uint64_t 	UUID = 0 ;
@@ -216,6 +227,16 @@ namespace OpenWifi::GWObjects {
 		std::string ip;
 		uint16_t 	port=0;
 		uint64_t 	weight=0;
+		std::string secret;
+		std::string certificate;
+		bool 		radsec=false;
+		uint16_t 	radsecPort=2083;
+		std::string radsecSecret;
+		std::string radsecKey;
+		std::string radsecCert;
+		std::vector<std::string> 	radsecCacerts;
+		std::vector<std::string>	radsecRealms;
+		bool 		ignore=false;
 
 		void to_json(Poco::JSON::Object &Obj) const;
 		bool from_json(const Poco::JSON::Object::Ptr &Obj);
@@ -237,6 +258,8 @@ namespace OpenWifi::GWObjects {
 		std::string description;
 		RadiusProxyServerConfig	authConfig;
 		RadiusProxyServerConfig	acctConfig;
+		RadiusProxyServerConfig	coaConfig;
+		bool 		useByDefault=false;
 
 		void to_json(Poco::JSON::Object &Obj) const;
 		bool from_json(const Poco::JSON::Object::Ptr &Obj);

src/StateReceiver.cpp

@@ -25,6 +25,7 @@ namespace OpenWifi {
 
     void StateReceiver::run() {
         Poco::AutoPtr<Poco::Notification>	Note(Queue_.waitDequeueNotification());
+        Utils::SetThreadName("dev-state");
         while(Note && Running_) {
             auto Msg = dynamic_cast<StateMessage *>(Note.get());
             if(Msg!= nullptr) {

src/StorageService.cpp

@@ -12,6 +12,7 @@
 namespace OpenWifi {
 
     int Storage::Start() {
+        poco_notice(Logger(),"Starting...");
 		std::lock_guard		Guard(Mutex_);
 
 		StorageClass::Start();
@@ -43,14 +44,14 @@ namespace OpenWifi {
         uint64_t start = 0 ;
         bool done = false;
         const uint64_t batch=100;
-        Logger().information("Starting cleanup of TimePoint Database");
+        poco_information(Logger(),"Starting cleanup of TimePoint Database");
         while(!done) {
             if(!BoardsDB().GetRecords(start,batch,BoardList)) {
                 for(const auto &board:BoardList) {
                     for(const auto &venue:board.venueList) {
                         auto now = OpenWifi::Now();
                         auto lower_bound = now - venue.retention;
-                        Logger().information(fmt::format("Removing old records for board '{}'",board.info.name));
+                        poco_information(Logger(),fmt::format("Removing old records for board '{}'",board.info.name));
                         BoardsDB().DeleteRecords(fmt::format(" boardId='{}' and timestamp<{}", board.info.id, lower_bound));
                     }
                 }
@@ -60,12 +61,13 @@ namespace OpenWifi {
 
         auto MaxDays = MicroService::instance().ConfigGetInt("wificlient.age.limit",14);
         auto LowerDate = OpenWifi::Now() - (MaxDays*60*60*24);
-        Logger().information(fmt::format("Removing WiFi Clients history older than {} days.", MaxDays));
+        poco_information(Logger(),fmt::format("Removing WiFi Clients history older than {} days.", MaxDays));
         StorageService()->WifiClientHistoryDB().DeleteRecords(fmt::format(" timestamp<{} ", LowerDate));
-        Logger().information(fmt::format("Done cleanup of databases. Next run in {} seconds.", PeriodicCleanup_));
+        poco_information(Logger(),fmt::format("Done cleanup of databases. Next run in {} seconds.", PeriodicCleanup_));
     }
 
     void Storage::run() {
+        Utils::SetThreadName("strg-updtr");
 	    Running_ = true ;
 	    bool FirstRun=true;
 	    long Retry = 2000;
@@ -80,11 +82,12 @@ namespace OpenWifi {
 	}
 
     void Storage::Stop() {
+        poco_notice(Logger(),"Stopping...");
 	    Running_=false;
         Timer_.stop();
 	    Updater_.wakeUp();
 	    Updater_.join();
-        Logger().notice("Stopping.");
+        poco_notice(Logger(),"Stopped...");
     }
 }
 

src/VenueCoordinator.cpp

@@ -11,29 +11,51 @@
 namespace OpenWifi {
 
     int VenueCoordinator::Start() {
+        poco_notice(Logger(),"Starting...");
         GetBoardList();
         Worker_.start(*this);
+
+        ReconcileTimerCallback_ = std::make_unique<Poco::TimerCallback<VenueCoordinator>>(*this,&VenueCoordinator::onReconcileTimer);
+        ReconcileTimerTimer_.setStartInterval( 3 * 60 * 1000 );
+        ReconcileTimerTimer_.setPeriodicInterval(3 * 60 * 1000); // 1 hours
+        ReconcileTimerTimer_.start(*ReconcileTimerCallback_, MicroService::instance().TimerPool());
+
         return 0;
     }
 
+    void VenueCoordinator::onReconcileTimer([[maybe_unused]] Poco::Timer &timer) {
+        std::lock_guard     G(Mutex_);
+        Utils::SetThreadName("brd-refresh");
+
+        poco_information(Logger(),"Starting to reconcile board information.");
+        for(const auto &[board_id, watcher]:Watchers_) {
+            poco_information(Logger(),fmt::format("Updating: {}", board_id));
+            UpdateBoard(board_id);
+        }
+        poco_information(Logger(),"Finished reconciling board information.");
+    }
+
     void VenueCoordinator::GetBoardList() {
         BoardsToWatch_.clear();
         auto F = [&](const AnalyticsObjects::BoardInfo &B) ->bool {
             BoardsToWatch_.insert(B);
-            // Logger().information(fmt::format("Starting watch for {}.", B.info.name));
+            // poco_information(Logger(),fmt::format("Starting watch for {}.", B.info.name));
             return true;
         };
         StorageService()->BoardsDB().Iterate(F);
     }
 
     void VenueCoordinator::Stop() {
+        poco_notice(Logger(),"Stopping...");
         Running_=false;
         Worker_.wakeUp();
         Worker_.wakeUp();
         Worker_.join();
+        poco_notice(Logger(),"Stopped...");
     }
 
     void VenueCoordinator::run() {
+        Utils::SetThreadName("venue-coord");
         Running_=true;
         while(Running_) {
             Poco::Thread::trySleep(20000);
@@ -58,9 +80,11 @@ namespace OpenWifi {
 
     void VenueCoordinator::RetireBoard(const AnalyticsObjects::BoardInfo &B) {
         Logger().error(fmt::format("Venue board '{}' is no longer in the system. Retiring its associated board.", B.venueList[0].name));
+        /*
         StopBoard(B.info.id);
         StorageService()->BoardsDB().DeleteRecord("id",B.info.id);
         StorageService()->TimePointsDB().DeleteRecords(fmt::format(" boardId='{}' ", B.info.id));
+         */
     }
 
     bool VenueCoordinator::GetDevicesForBoard(const AnalyticsObjects::BoardInfo &B, std::vector<uint64_t> & Devices, bool & VenueExists) {
@@ -95,7 +119,7 @@ namespace OpenWifi {
             ExistingBoards_[B.info.id] = Devices;
             Watchers_[B.info.id] = std::make_shared<VenueWatcher>(B.info.id, B.venueList[0].id, Logger(), Devices);
             Watchers_[B.info.id]->Start();
-            Logger().information(fmt::format("Started board {} for venue {}", B.info.name,B.venueList[0].id ));
+            poco_information(Logger(),fmt::format("Started board {} for venue {}", B.info.name,B.venueList[0].id ));
             return true;
         }
 
@@ -104,7 +128,7 @@ namespace OpenWifi {
             return false;
         }
 
-        Logger().information(fmt::format("Could not start board {}",B.info.name));
+        poco_information(Logger(),fmt::format("Could not start board {}",B.info.name));
         return false;
     }
 
@@ -118,7 +142,7 @@ namespace OpenWifi {
         }
     }
 
-    void VenueCoordinator::ModifyBoard(const std::string &id) {
+    void VenueCoordinator::UpdateBoard(const std::string &id) {
         AnalyticsObjects::BoardInfo B;
         if(StorageService()->BoardsDB().GetRecord("id",id,B)) {
             std::vector<uint64_t>   Devices;
@@ -129,15 +153,15 @@ namespace OpenWifi {
                 if(it!=ExistingBoards_.end()) {
                     if(it->second!=Devices) {
                         auto it2 = Watchers_.find(id);
-                        if(it2!=Watchers_.end())
+                        if(it2!=Watchers_.end()) {
                             it2->second->ModifySerialNumbers(Devices);
+                        }
                         ExistingBoards_[id] = Devices;
-                        Logger().information(fmt::format("Modified board {}",B.info.name));
+                        poco_information(Logger(),fmt::format("Modified board {}",B.info.name));
                     } else {
-                        Logger().information(fmt::format("No device changes in board {}",B.info.name));
+                        poco_information(Logger(),fmt::format("No device changes in board {}",B.info.name));
                     }
                 }
-                Logger().information(fmt::format("Modified board {}",B.info.name));
                 return;
             }
 
@@ -146,7 +170,7 @@ namespace OpenWifi {
                 return;
             }
 
-            Logger().information(fmt::format("Could not modify board {}",B.info.name));
+            poco_information(Logger(),fmt::format("Could not modify board {}",B.info.name));
         }
     }
 
@@ -162,7 +186,7 @@ namespace OpenWifi {
         if(StorageService()->BoardsDB().GetRecord("id",id,B))
             BoardsToWatch_.insert(B);
         else
-            Logger().information(fmt::format("Board {} does not seem to exist",id));
+            poco_information(Logger(),fmt::format("Board {} does not seem to exist",id));
     }
 
     void VenueCoordinator::GetDevices(std::string &id, AnalyticsObjects::DeviceInfoList &DIL) {

src/VenueCoordinator.h

@@ -3,9 +3,12 @@
 //
 
 #pragma once
+
 #include "framework/MicroService.h"
 #include "VenueWatcher.h"
 
+#include "Poco/Timer.h"
+
 namespace OpenWifi {
 
     class VenueCoordinator : public SubSystemServer, Poco::Runnable {
@@ -21,7 +24,7 @@ namespace OpenWifi {
         void run() override;
 
         void StopBoard(const std::string &id);
-        void ModifyBoard(const std::string &id);
+        void UpdateBoard(const std::string &id);
         void AddBoard(const std::string &id);
 
         bool GetDevicesForBoard(const AnalyticsObjects::BoardInfo &B, std::vector<uint64_t> & Devices, bool & VenueExists);
@@ -30,13 +33,17 @@ namespace OpenWifi {
         bool Watching(const std::string &id);
         void RetireBoard(const AnalyticsObjects::BoardInfo &B);
 
-    private:
-        Poco::Thread                                        Worker_;
-        std::atomic_bool                                    Running_=false;
-        std::set<AnalyticsObjects::BoardInfo>               BoardsToWatch_;
-        std::map<std::string,std::shared_ptr<VenueWatcher>> Watchers_;
+        void onReconcileTimer(Poco::Timer & timer);
 
-        std::map<std::string,std::vector<uint64_t>>            ExistingBoards_;
+    private:
+        Poco::Thread                                            Worker_;
+        std::atomic_bool                                        Running_=false;
+        std::set<AnalyticsObjects::BoardInfo>                   BoardsToWatch_;
+        std::map<std::string,std::shared_ptr<VenueWatcher>>     Watchers_;
+        std::unique_ptr<Poco::TimerCallback<VenueCoordinator>>  ReconcileTimerCallback_;
+        Poco::Timer                     		                ReconcileTimerTimer_;
+
+        std::map<std::string,std::vector<uint64_t>>             ExistingBoards_;
 
         VenueCoordinator() noexcept:
                 SubSystemServer("VenueCoordinator", "VENUE-COORD", "venue.coordinator")

src/VenueWatcher.cpp

@@ -10,6 +10,7 @@
 namespace OpenWifi {
 
     void VenueWatcher::Start() {
+        poco_notice(Logger(),"Starting...");
         for(const auto &mac:SerialNumbers_) {
             auto ap = std::make_shared<AP>(mac, venue_id_, boardId_, Logger());
             APs_[mac ] = ap;
@@ -24,6 +25,7 @@ namespace OpenWifi {
     }
 
     void VenueWatcher::Stop() {
+        poco_notice(Logger(),"Stopping...");
         Running_ = false;
         Queue_.wakeUpAll();
         Worker_.join();
@@ -31,9 +33,11 @@ namespace OpenWifi {
             StateReceiver()->DeRegister(i,this);
         DeviceStatusReceiver()->DeRegister(this);
         HealthReceiver()->DeRegister(this);
+        poco_notice(Logger(),"Stopped...");
     }
 
     void VenueWatcher::run() {
+        Utils::SetThreadName("venue-watch");
         Running_ = true;
         Poco::AutoPtr<Poco::Notification>	Msg(Queue_.waitDequeueNotification());
         while(Msg && Running_) {

src/VenueWatcher.h

@@ -69,9 +69,11 @@ namespace OpenWifi {
         void GetDevices(std::vector<AnalyticsObjects::DeviceInfo> & DI);
 
         void GetBandwidth(uint64_t start, uint64_t end, uint64_t interval , AnalyticsObjects::BandwidthAnalysis & BW);
-
+        inline std::string Venue() const {
+            return venue_id_;
+        }
     private:
-        std::recursive_mutex                        Mutex_;
+        std::mutex                                  Mutex_;
         std::string                                 boardId_;
         std::string                                 venue_id_;
         Poco::NotificationQueue                     Queue_;

src/WifiClientCache.cpp

@@ -10,6 +10,7 @@
 namespace OpenWifi {
 
 	int WifiClientCache::Start() {
+        poco_notice(Logger(),"Starting...");
         TimerCallback_ = std::make_unique<Poco::TimerCallback<WifiClientCache>>(*this,&WifiClientCache::onTimer);
         Timer_.setStartInterval( 30 * 1000);  // first run in 20 seconds
         Timer_.setPeriodicInterval( 60 * 60 * 1000); // 1 hours
@@ -18,7 +19,9 @@ namespace OpenWifi {
 	}
 
 	void WifiClientCache::Stop() {
+        poco_notice(Logger(),"Stopping...");
         Timer_.stop();
+        poco_notice(Logger(),"Stopped...");
 	}
 
     void WifiClientCache::onTimer([[maybe_unused]] Poco::Timer & timer) {
@@ -124,32 +127,36 @@ namespace OpenWifi {
 		}
 	}
 
-	void WifiClientCache::FindNumbers(const std::string &venue_id, const std::string &S, uint HowMany, std::vector<uint64_t> &A) {
+	void WifiClientCache::FindNumbers(const std::string &venueId, const std::string &SerialNumber, std::uint64_t StartingOffset, std::uint64_t HowMany, std::vector<uint64_t> &A) {
         std::lock_guard G(Mutex_);
 
         A.clear();
-        auto VenueIt = Cache_.find(venue_id);
+        auto VenueIt = Cache_.find(venueId);
         if(VenueIt==Cache_.end())
             return;
 
-		if(S.empty()) {
+		if(SerialNumber.empty()) {
             auto Start = VenueIt->second.SNs_.begin();
+            std::uint64_t Offset=0;
             while(HowMany && Start!=VenueIt->second.SNs_.end()) {
-                A.push_back(*Start);
+                if(Offset>=StartingOffset) {
+                    A.push_back(*Start);
+                    HowMany--;
+                }
                 Start++;
-                HowMany--;
+                Offset++;
             }
             return;
         }
 
-		if (S[0] == '*') {
+		if (SerialNumber[0] == '*') {
 			std::string Reversed;
-			std::copy(rbegin(S), rend(S)-1, std::back_inserter(Reversed));
+			std::copy(rbegin(SerialNumber), rend(SerialNumber)-1, std::back_inserter(Reversed));
 			if(Reversed.empty())
 				return;
 			return ReturnNumbers(Reversed, HowMany, VenueIt->second.Reverse_SNs_, A, true);
 		} else {
-			return ReturnNumbers(S, HowMany, VenueIt->second.SNs_, A, false);
+			return ReturnNumbers(SerialNumber, HowMany, VenueIt->second.SNs_, A, false);
 		}
 	}
 }

src/WifiClientCache.h

@@ -20,7 +20,7 @@ namespace OpenWifi {
 		void Stop() override;
 		void AddSerialNumber(const std::string &venueId, const std::string &SerialNumber);
 		void DeleteSerialNumber(const std::string &venueId, const std::string &SerialNumber);
-		void FindNumbers(const std::string &venueId, const std::string &SerialNumber, uint HowMany, std::vector<uint64_t> &A);
+		void FindNumbers(const std::string &venueId, const std::string &SerialNumber, std::uint64_t start, std::uint64_t HowMany, std::vector<uint64_t> &A);
 		inline bool NumberExists(const std::string &venueId, uint64_t SerialNumber) {
 			std::lock_guard		G(Mutex_);
             auto It = Cache_.find(venueId);

src/framework/ConfigurationValidator.cpp

@@ -44,7 +44,7 @@ static json DefaultUCentralSchema = R"(
 		"switch": {
 			"$ref": "#/$defs/switch"
 		},
-		"radios": {
+		"radiosgrep": {
 			"type": "array",
 			"items": {
 				"$ref": "#/$defs/radio"

src/framework/StorageClass.h

@@ -33,7 +33,6 @@ namespace OpenWifi {
         int Start() override {
             std::lock_guard		Guard(Mutex_);
 
-            Logger().setLevel(Poco::Message::PRIO_INFORMATION);
             Logger().notice("Starting.");
             std::string DBType = MicroService::instance().ConfigGetString("storage.type");
 

src/framework/WebSocketClientNotifications.h

@@ -146,6 +146,10 @@ namespace OpenWifi {
         WebSocketClientServer()->SendUserNotification(User,N);
     }
 
+    /////
+    /////
+    /////
+
     struct WebSocketNotificationRebootList {
         std::string                 title,
                 details,
@@ -189,5 +193,58 @@ namespace OpenWifi {
         WebSocketClientServer()->SendUserNotification(User,N);
     }
 
+    /////
+    /////
+    /////
+
+    struct WebSocketNotificationUpgradeList {
+        std::string                 title,
+                details,
+                jobId;
+        std::vector<std::string>    success,
+                                    skipped,
+                                    no_firmware,
+                                    not_connected;
+        uint64_t                    timeStamp=OpenWifi::Now();
+
+        void to_json(Poco::JSON::Object &Obj) const;
+        bool from_json(const Poco::JSON::Object::Ptr &Obj);
+    };
+
+    typedef WebSocketNotification<WebSocketNotificationUpgradeList> WebSocketClientNotificationVenueUpgradeList_t;
+
+    inline void WebSocketNotificationUpgradeList::to_json(Poco::JSON::Object &Obj) const {
+        RESTAPI_utils::field_to_json(Obj,"title",title);
+        RESTAPI_utils::field_to_json(Obj,"jobId",jobId);
+        RESTAPI_utils::field_to_json(Obj,"success",success);
+        RESTAPI_utils::field_to_json(Obj,"notConnected",not_connected);
+        RESTAPI_utils::field_to_json(Obj,"noFirmware",no_firmware);
+        RESTAPI_utils::field_to_json(Obj,"skipped",skipped);
+        RESTAPI_utils::field_to_json(Obj,"timeStamp",timeStamp);
+        RESTAPI_utils::field_to_json(Obj,"details",details);
+    }
+
+    inline bool WebSocketNotificationUpgradeList::from_json(const Poco::JSON::Object::Ptr &Obj) {
+        try {
+            RESTAPI_utils::field_from_json(Obj,"title",title);
+            RESTAPI_utils::field_from_json(Obj,"jobId",jobId);
+            RESTAPI_utils::field_from_json(Obj,"success",success);
+            RESTAPI_utils::field_from_json(Obj,"notConnected",not_connected);
+            RESTAPI_utils::field_from_json(Obj,"noFirmware",no_firmware);
+            RESTAPI_utils::field_from_json(Obj,"skipped",skipped);
+            RESTAPI_utils::field_from_json(Obj,"timeStamp",timeStamp);
+            RESTAPI_utils::field_from_json(Obj,"details",details);
+            return true;
+        } catch(...) {
+
+        }
+        return false;
+    }
+
+    inline void WebSocketClientNotificationVenueUpgradeCompletionToUser( const std::string & User, WebSocketClientNotificationVenueUpgradeList_t &N) {
+        N.type = "venue_upgrader";
+        WebSocketClientServer()->SendUserNotification(User,N);
+    }
+
 } // namespace OpenWifi
 

src/framework/orm.h

@@ -133,6 +133,37 @@ namespace ORM {
         return R;
     }
 
+    inline std::string WHERE_AND_(std::string Result) {
+        return Result;
+    }
+
+    template <typename T, typename... Args> std::string WHERE_AND_(std::string Result, const char *fieldName, const T &Value, Args... args) {
+        if constexpr(std::is_same_v<T,std::string>)
+        {
+            if(!Value.empty()) {
+                if(!Result.empty())
+                    Result += " and ";
+                Result += fieldName;
+                Result += '=';
+                Result += "'";
+                Result += Escape(Value);
+                Result += "'";
+            }
+        } else {
+            if(!Result.empty())
+                Result += " and ";
+            Result += fieldName ;
+            Result += '=';
+            Result += std::to_string(Value);
+        }
+        return WHERE_AND_(Result,args...);
+    }
+
+    template <typename... Args> std::string WHERE_AND(Args... args) {
+        std::string Result;
+        return WHERE_AND_(Result, args...);
+    }
+
     enum SqlComparison { EQ = 0, NEQ, LT, LTE, GT, GTE };
     enum SqlBinaryOp { AND = 0 , OR };
 

src/framework/ow_constants.h

@@ -196,6 +196,8 @@ namespace OpenWifi::RESTAPI::Errors {
 	static const struct msg InvalidRadiusServerEntry{1142,"RADIUS Server IP address invalid or port missing."};
 	static const struct msg InvalidRadiusServerWeigth{1143,"RADIUS Server IP weight cannot be 0."};
 
+	static const struct msg MaximumRTTYSessionsReached{1144,"Too many RTTY sessions currently active"};
+	static const struct msg DeviceIsAlreadyBusy{1145,"Device is already executing a command. Please try later."};
 }
 
 
@@ -428,6 +430,7 @@ namespace OpenWifi::uCentralProtocol {
 	static const char *RADIUSDATA = "data";
 	static const char *RADIUSACCT = "acct";
 	static const char *RADIUSAUTH = "auth";
+	static const char *RADIUSCOA = "coa";
 	static const char *RADIUSDST = "dst";
 	static const char *IES = "ies";
 	}
@@ -444,6 +447,7 @@ namespace OpenWifi::uCentralProtocol::Events {
     static const char *RECOVERY = "recovery";
     static const char *TELEMETRY = "telemetry";
     static const char *DEVICEUPDATE = "deviceupdate";
+	static const char *VENUE_BROADCAST = "venue_broadcast";
 
     enum EVENT_MSG {
 		ET_UNKNOWN,
@@ -456,7 +460,8 @@ namespace OpenWifi::uCentralProtocol::Events {
 		ET_CFGPENDING,
 		ET_RECOVERY,
 		ET_DEVICEUPDATE,
-		ET_TELEMETRY
+		ET_TELEMETRY,
+		ET_VENUEBROADCAST
 	};
 
 	inline EVENT_MSG EventFromString(const std::string & Method) {
@@ -480,6 +485,8 @@ namespace OpenWifi::uCentralProtocol::Events {
 			return ET_RECOVERY;
 		else if(strcmp(TELEMETRY,Method.c_str())==0)
 			return ET_TELEMETRY;
+		else if(strcmp(VENUE_BROADCAST,Method.c_str())==0)
+			return ET_VENUEBROADCAST;
 		return ET_UNKNOWN;
 	};
 }

src/storage/storage_timepoints.cpp

@@ -71,7 +71,7 @@ namespace OpenWifi {
         } else if (LastDate) {
             WhereClause = fmt::format(" boardId='{}' and (timestamp <= {}) ", boardId, LastDate);
         }
-        GetRecords(0,MaxRecords,Recs,WhereClause," order by timestamp ASC ");
+        GetRecords(0,MaxRecords,Recs,WhereClause," order by timestamp, serialNumber ASC ");
         return true;
     }
 

test_scripts/curl/cli

@@ -144,25 +144,9 @@ logout() {
   rm -rf token.json
 }
 
-venuecount() {
-    curl ${FLAGS} "https://${OWPROV}/api/v1/venue?countOnly=true" \
-        -H "Content-Type: application/json" \
-        -H "Authorization: Bearer ${token}" \
-        -H "accept: application/json" > ${result_file}
-    jq < ${result_file}
-}
-
-contactcount() {
-    curl ${FLAGS} "https://${OWPROV}/api/v1/contact?countOnly=true" \
-        -H "Content-Type: application/json" \
-        -H "Authorization: Bearer ${token}" \
-        -H "Accept: application/json" > ${result_file}
-    jq < ${result_file}
-}
-
 setloglevel() {
     payload="{ \"command\" : \"setloglevel\" , \"subsystems\" : [ { \"tag\" : \"$1\" , \"value\" : \"$2\" } ] }"
-    curl  ${FLAGS} -X POST "https://${OWPROV}/api/v1/system" \
+    curl  ${FLAGS} -X POST "https://${OWANALYTICS}/api/v1/system" \
         -H "Accept: application/json" \
         -H "Content-Type: application/json" \
         -H "Authorization: Bearer ${token}" \
@@ -171,7 +155,7 @@ setloglevel() {
 
 getloglevels() {
     payload="{ \"command\" : \"getloglevels\" }"
-    curl  ${FLAGS} -X POST "https://${OWPROV}/api/v1/system" \
+    curl  ${FLAGS} -X POST "https://${OWANALYTICS}/api/v1/system" \
         -H "Accept: application/json" \
         -H "Content-Type: application/json" \
         -H "Authorization: Bearer ${token}" \
@@ -180,7 +164,7 @@ getloglevels() {
 
 getloglevelnames() {
     payload="{ \"command\" : \"getloglevelnames\" }"
-    curl  ${FLAGS} -X POST "https://${OWPROV}/api/v1/system" \
+    curl  ${FLAGS} -X POST "https://${OWANALYTICS}/api/v1/system" \
         -H "Accept: application/json" \
         -H "Content-Type: application/json" \
         -H "Authorization: Bearer ${token}" \
@@ -189,7 +173,7 @@ getloglevelnames() {
 
 getsubsystemnames() {
     payload="{ \"command\" : \"getsubsystemnames\" }"
-    curl  ${FLAGS} -X POST "https://${OWPROV}/api/v1/system" \
+    curl  ${FLAGS} -X POST "https://${OWANALYTICS}/api/v1/system" \
         -H "Accept: application/json" \
         -H "Content-Type: application/json" \
         -H "Authorization: Bearer ${token}" \
@@ -197,7 +181,7 @@ getsubsystemnames() {
 }
 
 systeminfo() {
-    curl  ${FLAGS} -X GET "https://${OWPROV}/api/v1/system?command=info" \
+    curl  ${FLAGS} -X GET "https://${OWANALYTICS}/api/v1/system?command=info" \
     -H "Accept: application/json" \
     -H "Authorization: Bearer ${token}" > ${result_file}
     jq < ${result_file}
@@ -205,7 +189,7 @@ systeminfo() {
 
 reloadsubsystem() {
     payload="{ \"command\" : \"reload\", \"subsystems\" : [ \"$1\" ] }"
-    curl  ${FLAGS} -X POST "https://${OWPROV}/api/v1/system" \
+    curl  ${FLAGS} -X POST "https://${OWANALYTICS}/api/v1/system" \
         -H "Accept: application/json" \
         -H "Content-Type: application/json" \
         -H "Authorization: Bearer ${token}" \

test_scripts/curl/result.json

@@ -1 +0,0 @@
-{"children":[{"children":[{"children":[{"children":[],"name":"Ottawa Operations","type":"entity","uuid":"323054d0-3ff0-11ec-82ff-061b87871e04","venues":[{"children":[],"name":"Living Lab","type":"venue","uuid":"4159ea84-3ff0-11ec-9edb-061b87871e04"}]}],"name":"Ontario","type":"entity","uuid":"21dfa464-3ff0-11ec-9a86-061b87871e04","venues":[]},{"children":[{"children":[],"name":"Quebec","type":"entity","uuid":"27b74e1b-52d7-4414-a1b4-47574c799beb","venues":[]},{"children":[],"name":"Ontario","type":"entity","uuid":"3cb15a80-3550-11ec-9a22-061b87871e04","venues":[{"children":[],"name":"CN Tower","type":"venue","uuid":"3cb28f0e-3550-11ec-acfa-061b87871e04"},{"children":[],"name":"Rogers Center","type":"venue","uuid":"3cb3c798-3550-11ec-8c1a-061b87871e04"}]},{"children":[],"name":"BC","type":"entity","uuid":"3cb503c4-3550-11ec-b436-061b87871e04","venues":[{"children":[],"name":"BC Place","type":"venue","uuid":"3cb76452-3550-11ec-a5ab-061b87871e04"},{"children":[],"name":"Stanley Park","type":"venue","uuid":"3cb8b230-3550-11ec-bbe6-061b87871e04"},{"children":[],"name":"MegaLab","type":"venue","uuid":"a73a23c4-6f3d-4031-bc92-d28c205f2d85"},{"children":[],"name":"Bowen Development","type":"venue","uuid":"d656500b-9f00-4c09-9096-e209085c2a3e"}]}],"name":"Provinces","type":"entity","uuid":"3cac92ca-3550-11ec-b360-061b87871e04","venues":[]}],"name":"Canada","type":"entity","uuid":"3cab2c32-3550-11ec-8a03-061b87871e04","venues":[]},{"children":[],"name":"USA","type":"entity","uuid":"6392c6b3-ef48-4b81-8785-1b5b4888bbd2","venues":[]},{"children":[{"children":[],"name":"Hamburg","type":"entity","uuid":"5939f995-e059-439d-ab0c-29a6f7c0a495","venues":[]}],"name":"Germany","type":"entity","uuid":"6fb5662d-dc48-4fbd-b8f9-69d0308bb902","venues":[]},{"children":[],"name":"Test Land","type":"entity","uuid":"70161a6f-98af-45d7-b0ed-e6b15ba5d84e","venues":[{"children":[],"name":"Test","type":"venue","uuid":"1d06b289-449e-4669-abc9-1f2dcdd1fbbc"}]}],"name":"World","type":"entity","uuid":"0000-0000-0000","venues":[]}