Chg: update image tag in helm values to v4.1.0

WIFI-13821 version set to 4.1.0

.github/workflows/ci.yml

@@ -21,7 +21,7 @@ defaults:
 
 jobs:
   docker:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     env:
       DOCKER_REGISTRY_URL: tip-tip-wlan-cloud-ucentral.jfrog.io
       DOCKER_REGISTRY_USERNAME: ucentral

BUILDING.md

@@ -1,5 +1,5 @@
 # Building from source
-In order to build the OWPROV, you will need to install its dependencies, which includes the following:
+In order to build OWPROV, you will need to install its dependencies, which includes the following:
 - cmake
 - boost
 - POCO 1.10.1 or later
@@ -12,19 +12,19 @@ In order to build the OWPROV, you will need to install its dependencies, which i
 
 The build is done in 2 parts. The first part is to build a local copy of the framework tailored to your environment. This
 framework is called [Poco](https://github.com/pocoproject/poco). The version used in this project has a couple of fixes
-from the master copy needed for cmake. Please use the version of this [Poco fix](https://github.com/AriliaWireless/poco). Building
+from the master copy needed for cmake. Please use the version of this [Poco fix](https://github.com/Telecominfraproject/wlan-cloud-lib-poco). Building
 Poco may take several minutes depending on the platform you are building on.
 
 ## Ubuntu
 These instructions have proven to work on Ubuntu 20.4.
 ```bash
-sudo apt install git cmake g++ libssl-dev libmariadb-dev 
+sudo apt install git cmake g++ libssl-dev libmariadb-dev
 sudo apt install libpq-dev libaprutil1-dev apache2-dev libboost-all-dev
 sudo apt install librdkafka-dev // default-libmysqlclient-dev
 sudo apt install nlohmann-json-dev
 
 cd ~
-git clone https://github.com/AriliaWireless/poco --branch poco-tip-v1
+git clone https://github.com/Telecominfraproject/wlan-cloud-lib-poco --branch poco-tip-v1 poco
 cd poco
 mkdir cmake-build
 cd cmake-build
@@ -33,7 +33,7 @@ cmake --build . --config Release
 sudo cmake --build . --target install
 
 cd ~
-git clone https://github.com/AriliaWireless/cppkafka --branch tip-v1
+git clone https://github.com/Telecominfraproject/wlan-cloud-lib-cppkafka --branch tip-v1 cppkafka
 cd cppkafka
 mkdir cmake-build
 cd cmake-build
@@ -42,7 +42,7 @@ cmake --build . --config Release
 sudo cmake --build . --target install
 
 cd ~
-git clone https://github.com/AriliaWireless/valijson --branch tip-v1
+git clone https://github.com/Telecominfraproject/wlan-cloud-lib-valijson --branch tip-v1 valijson
 cd valijson
 mkdir cmake-build
 cd cmake-build
@@ -50,7 +50,8 @@ cmake ..
 cmake --build . --config Release
 sudo cmake --build . --target install
 
-git clone https://github.com/fmtlib/fmt --branch 9.0.0 /fmtlib
+cd ~
+git clone https://github.com/fmtlib/fmt --branch 9.0.0 fmtlib
 cd fmtlib
 mkdir cmake-build
 cd cmake-build
@@ -71,11 +72,11 @@ make -j 8
 The following instructions have proven to work on Fedora 33
 ```bash
 sudo yum install cmake g++ openssl-devel mysql-devel mysql apr-util-devel boost boost-devel
-sudo yum install yaml-cpp-devel lua-devel 
+sudo yum install yaml-cpp-devel lua-devel
 sudo dnf install postgresql.x86_64 librdkafka-devel
 sudo dnf install postgresql-devel json-devel
 
-git clone https://github.com/AriliaWireless/poco --branch poco-tip-v1
+git clone https://github.com/Telecominfraproject/wlan-cloud-lib-poco --branch poco-tip-v1 poco
 cd poco
 mkdir cmake-build
 cd cmake-build
@@ -83,7 +84,8 @@ cmake ..
 cmake --build . --config Release
 sudo cmake --build . --target install
 
-git clone https://github.com/AriliaWireless/cppkafka --branch tip-v1
+cd ~
+git clone https://github.com/Telecominfraproject/wlan-cloud-lib-cppkafka --branch tip-v1 cppkafka
 cd cppkafka
 mkdir cmake-build
 cd cmake-build
@@ -92,7 +94,7 @@ cmake --build . --config Release
 sudo cmake --build . --target install
 
 cd ~
-git clone https://github.com/AriliaWireless/valijson --branch tip-v1
+git clone https://github.com/Telecominfraproject/wlan-cloud-lib-valijson --branch tip-v1 valijson
 cd valijson
 mkdir cmake-build
 cd cmake-build
@@ -125,7 +127,7 @@ brew install openssl \
 	nlohmann-json \
 	fmt
 
-git clone https://github.com/AriliaWireless/poco --branch poco-tip-v1
+git clone https://github.com/Telecominfraproject/wlan-cloud-lib-poco --branch poco-tip-v1 poco
 pushd poco
 mkdir cmake-build
 push cmake-build
@@ -135,7 +137,7 @@ sudo cmake --build . --target install
 popd
 popd
 
-git clone https://github.com/AriliaWireless/cppkafka --branch tip-v1
+git clone https://github.com/Telecominfraproject/wlan-cloud-lib-cppkafka --branch tip-v1 cppkafka
 pushd cppkafka
 mkdir cmake-build
 pushd cmake-build
@@ -145,7 +147,7 @@ sudo cmake --build . --target install
 popd
 popd
 
-git clone https://github.com/AriliaWireless/valijson --branch tip-v1
+git clone https://github.com/Telecominfraproject/wlan-cloud-lib-valijson --branch tip-v1 valijson
 cd valijson
 mkdir cmake-build
 cd cmake-build
@@ -172,6 +174,8 @@ adding -DSMALL_BUILD=1 on the cmake build line.
 
 ```bash
 sudo apt install git cmake g++ libssl-dev libaprutil1-dev apache2-dev libboost-all-dev libyaml-cpp-dev
+
+cd ~
 git clone https://github.com/stephb9959/poco
 cd poco
 mkdir cmake-build

CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.13)
-project(owprov VERSION 3.0.0)
+project(owprov VERSION 4.1.0)
 
 set(CMAKE_CXX_STANDARD 17)
 
@@ -242,4 +242,3 @@ target_link_libraries(owprov PUBLIC
         CppKafka::cppkafka
         resolv
         fmt::fmt)
-

Dockerfile

@@ -15,8 +15,9 @@ FROM build-base AS poco-build
 
 ARG POCO_VERSION
 
-ADD https://api.github.com/repos/AriliaWireless/poco/git/refs/tags/${POCO_VERSION} version.json
-RUN git clone https://github.com/AriliaWireless/poco --branch ${POCO_VERSION} /poco
+ADD https://api.github.com/repos/Telecominfraproject/wlan-cloud-lib-poco/git/refs/tags/${POCO_VERSION} version.json
+RUN git clone https://github.com/Telecominfraproject/wlan-cloud-lib-poco --branch ${POCO_VERSION} /poco
+
 
 WORKDIR /poco
 RUN mkdir cmake-build
@@ -29,8 +30,8 @@ FROM build-base AS cppkafka-build
 
 ARG CPPKAFKA_VERSION
 
-ADD https://api.github.com/repos/AriliaWireless/cppkafka/git/refs/tags/${CPPKAFKA_VERSION} version.json
-RUN git clone https://github.com/AriliaWireless/cppkafka --branch ${CPPKAFKA_VERSION} /cppkafka
+ADD https://api.github.com/repos/Telecominfraproject/wlan-cloud-lib-cppkafka/git/refs/tags/${CPPKAFKA_VERSION} version.json
+RUN git clone https://github.com/Telecominfraproject/wlan-cloud-lib-cppkafka --branch ${CPPKAFKA_VERSION} /cppkafka
 
 WORKDIR /cppkafka
 RUN mkdir cmake-build
@@ -43,8 +44,8 @@ FROM build-base AS valijson-build
 
 ARG VALIJASON_VERSION
 
-ADD https://api.github.com/repos/AriliaWireless/valijson/git/refs/tags/${VALIJASON_VERSION} version.json
-RUN git clone https://github.com/AriliaWireless/valijson --branch ${VALIJASON_VERSION} /valijson
+ADD https://api.github.com/repos/Telecominfraproject/wlan-cloud-lib-valijson/git/refs/tags/${VALIJASON_VERSION} version.json
+RUN git clone https://github.com/Telecominfraproject/wlan-cloud-lib-valijson --branch ${VALIJASON_VERSION} /valijson
 
 WORKDIR /valijson
 RUN mkdir cmake-build

build

@@ -1 +1 @@
-2
+1

helm/values.yaml

@@ -9,7 +9,7 @@ fullnameOverride: ""
 images:
   owprov:
     repository: tip-tip-wlan-cloud-ucentral.jfrog.io/owprov
-    tag: main
+    tag: v4.1.0
     pullPolicy: Always
 #    regcred:
 #      registry: tip-tip-wlan-cloud-ucentral.jfrog.io

openapi/owprov.yaml

@@ -815,6 +815,17 @@ components:
           type: string
           minLength: 2
           maxLength: 2
+        imported:
+          type: integer
+          format: int64
+        connected:
+          type: integer
+          format: int64
+        platform:
+          type: string
+          enum:
+            - AP
+            - SWITCH
 
     VenueDeviceList:
       type: object
@@ -3240,6 +3251,15 @@ paths:
           schema:
             type: boolean
           required: false
+        - in: query
+          name: deviceType
+          schema:
+            type: string
+            enum:
+              - AP
+              - SWITCH
+          required: false
+          default: AP
       requestBody:
         description: Information used to create the new entity
         content:
@@ -3268,6 +3288,15 @@ paths:
             format: uuid
             example: When modifying the root entity, the uuid 0000-0000-0000 must be entered.
           required: true
+        - in: query
+          name: deviceType
+          schema:
+            type: string
+            enum:
+              - AP
+              - SWITCH
+          required: false
+          default: AP
       requestBody:
         description: Information used to modify the new entity
         content:

src/APConfig.cpp

@@ -79,6 +79,42 @@ namespace OpenWifi {
         return false;
     }
 
+	void APConfig::ReplaceNestedVariables(const std::string uuid, Poco::JSON::Object &Result) {
+		/*
+		Helper method contains code previously in ReplaceVariablesinObject.
+		Once the top-level variable is resolved, this will be called to resolve any
+		variables nested within the top-level variable.
+		*/
+		ProvObjects::VariableBlock VB;
+		if (StorageService()->VariablesDB().GetRecord("id", uuid, VB)) {
+			for (const auto &var: VB.variables) {
+				Poco::JSON::Parser P;
+				auto VariableBlockInfo =
+					P.parse(var.value).extract<Poco::JSON::Object::Ptr>();
+				auto VarNames = VariableBlockInfo->getNames();
+				for (const auto &j: VarNames) {
+					if(VariableBlockInfo->isArray(j)) {
+						auto Elements = VariableBlockInfo->getArray(j);
+						if(Elements->size()>0) {
+							Poco::JSON::Array InnerArray;
+							ReplaceVariablesInArray(*Elements, InnerArray);
+							Result.set(j, InnerArray);
+						} else {
+//                      	std::cout << "Empty Array!!!" << std::endl;
+						}
+					} else if(VariableBlockInfo->isObject(j)) {
+						Poco::JSON::Object  InnerEval;
+						auto O = VariableBlockInfo->getObject(j);
+						ReplaceVariablesInObject(*O,InnerEval);
+						Result.set(j, InnerEval);
+					} else {
+						Result.set(j, VariableBlockInfo->get(j));
+					}
+				}
+			}
+		}
+	}
+
     bool APConfig::ReplaceVariablesInObject(const Poco::JSON::Object &Original,
 											Poco::JSON::Object &Result) {
 		// get all the names and expand
@@ -86,41 +122,31 @@ namespace OpenWifi {
 		for (const auto &i : Names) {
             if (i == "__variableBlock") {
                 if (Original.isArray(i)) {
+					/*
+					E.g. of what the variable block would look like in an array:
+					"ssids": [
+						{
+							"__variableBlock": [
+								"79c083d2-d496-4de0-8600-76a63556851b"
+							]
+						}
+					]
+					*/
                     auto UUIDs = Original.getArray(i);
-                    for (const auto &uuid: *UUIDs) {
-                        ProvObjects::VariableBlock VB;
-                        if (StorageService()->VariablesDB().GetRecord("id", uuid, VB)) {
-                            for (const auto &var: VB.variables) {
-                                Poco::JSON::Parser P;
-                                auto VariableBlockInfo =
-                                        P.parse(var.value).extract<Poco::JSON::Object::Ptr>();
-                                auto VarNames = VariableBlockInfo->getNames();
-                                for (const auto &j: VarNames) {
-//                                    std::cout << "Name: " << j << std::endl;
-                                    if(VariableBlockInfo->isArray(j)) {
-                                        auto Elements = VariableBlockInfo->getArray(j);
-                                        if(Elements->size()>0) {
-                                            Poco::JSON::Array InnerArray;
-                                            ReplaceVariablesInArray(*Elements, InnerArray);
-                                            Result.set(j, InnerArray);
-//                                            std::cout << "Array!!!" << std::endl;
-                                        } else {
-//                                            std::cout << "Empty Array!!!" << std::endl;
-                                        }
-                                    } else if(VariableBlockInfo->isObject(j)) {
-                                        Poco::JSON::Object  InnerEval;
-//                                        std::cout << "Visiting object " << j << std::endl;
-                                        auto O = VariableBlockInfo->getObject(j);
-                                        ReplaceVariablesInObject(*O,InnerEval);
-                                        Result.set(j, InnerEval);
-                                    } else {
-                                        Result.set(j, VariableBlockInfo->get(j));
-                                    }
-                                }
-                            }
-                        }
-                    }
+                    for (const std::string &uuid: *UUIDs) {
+                        ReplaceNestedVariables(uuid, Result);
+					}
                 }
+				else {
+					/*
+					E.g. of what the variable block would look like replacing an entire json blob:
+					"services" : {
+						"__variableBlock": "ef8db4c0-f0ef-40d2-b676-c9c02ef39430"
+					}
+					*/
+					const std::string uuid = Original.get(i);
+					ReplaceNestedVariables(uuid, Result);
+				}
             } else if (i == "__radiusEndpoint") {
                 auto EndPointId = Original.get(i).toString();
                 ProvObjects::RADIUSEndPoint RE;
@@ -434,4 +460,4 @@ namespace OpenWifi {
 		} else {
 		}
 	}
-} // namespace OpenWifi
+} // namespace OpenWifi

src/APConfig.h

@@ -47,6 +47,7 @@ namespace OpenWifi {
 
 		bool ReplaceVariablesInArray(const Poco::JSON::Array &O,
 									 Poco::JSON::Array &Result);
+		void ReplaceNestedVariables(const std::string uuid, Poco::JSON::Object &Result);
 		bool ReplaceVariablesInObject(const Poco::JSON::Object &Original,
 									  Poco::JSON::Object &Result);
 

src/AutoDiscovery.cpp

@@ -5,6 +5,7 @@
 #include "AutoDiscovery.h"
 #include "Poco/JSON/Parser.h"
 #include "StorageService.h"
+#include "Tasks/VenueConfigUpdater.h"
 #include "framework/KafkaManager.h"
 #include "framework/KafkaTopics.h"
 #include "framework/ow_constants.h"
@@ -54,6 +55,8 @@ namespace OpenWifi {
             FW = P->get(uCentralProtocol::FIRMWARE).toString();
         if (P->has(uCentralProtocol::SERIALNUMBER))
             SN = P->get(uCentralProtocol::SERIALNUMBER).toString();
+        else if (P->has(uCentralProtocol::SERIAL))
+            SN = P->get(uCentralProtocol::SERIAL).toString();
         if (P->has("locale")) {
             locale = P->get("locale").toString();
         }
@@ -83,6 +86,7 @@ namespace OpenWifi {
 					Poco::JSON::Parser Parser;
 					auto Object = Parser.parse(Msg->Payload()).extract<Poco::JSON::Object::Ptr>();
                     bool    Connected=true;
+                    bool isConnection=false;
 
 					if (Object->has(uCentralProtocol::PAYLOAD)) {
                         auto PayloadObj = Object->getObject(uCentralProtocol::PAYLOAD);
@@ -91,6 +95,7 @@ namespace OpenWifi {
                             auto PingObj = PayloadObj->getObject("ping");
                             ProcessPing(PingObj, Firmware, SerialNumber, Compatible, ConnectedIP, Locale);
                         } else if(PayloadObj->has("capabilities")) {
+                            isConnection=true;
                             ProcessConnect(PayloadObj, Firmware, SerialNumber, Compatible, ConnectedIP, Locale);
                         } else if(PayloadObj->has("disconnection")) {
                             //  we ignore disconnection in provisioning
@@ -102,7 +107,12 @@ namespace OpenWifi {
 
                         if (!SerialNumber.empty() && Connected) {
                             StorageService()->InventoryDB().CreateFromConnection(
-                                    SerialNumber, ConnectedIP, Compatible, Locale);
+                                    SerialNumber, ConnectedIP, Compatible, Locale, isConnection);
+                            // Now that the entry has been created, we can try to push a config if
+                            // the connection was a capabilities message.
+                            if (isConnection){
+                                ComputeAndPushConfig(SerialNumber, Compatible, Logger());
+                            }
                         }
                     }
 				} catch (const Poco::Exception &E) {

src/RESTAPI/RESTAPI_configurations_handler.cpp

@@ -91,9 +91,10 @@ namespace OpenWifi {
 			}
 			auto Config = RawObject->get("configuration").toString();
 			Poco::JSON::Object Answer;
-			std::vector<std::string> Error;
+            auto deviceType = GetParameter("deviceType", "AP");
+            std::string Error;
 			auto Res =
-				ValidateUCentralConfiguration(Config, Error, GetBoolParameter("strict", true));
+				ValidateUCentralConfiguration(ConfigurationValidator::GetType(deviceType),Config, Error, GetBoolParameter("strict", true));
 			Answer.set("valid", Res);
 			Answer.set("error", Error);
 			return ReturnObject(Answer);
@@ -134,11 +135,27 @@ namespace OpenWifi {
 		}
 
 		std::vector<std::string> Errors;
-		if (!ValidateConfigBlock(NewObject, Errors)) {
-			return BadRequest(RESTAPI::Errors::ConfigBlockInvalid);
+        auto deviceType = GetParameter("deviceType", "AP");
+        if (!ValidateConfigBlock(ConfigurationValidator::GetType(deviceType), NewObject, Errors)) {
+            return BadRequest(RESTAPI::Errors::ConfigBlockInvalid);
+        }
+
+		Types::UUIDvec_t ToVariables;
+		if (RawObject->has("variables")) {
+			for (const auto &i : NewObject.variables) {
+				if (!i.empty() && !StorageService()->VariablesDB().Exists("id", i)) {
+					return BadRequest(RESTAPI::Errors::VariableMustExist);
+				}
+			}
+			for (const auto &i : NewObject.variables)
+				ToVariables.emplace_back(i);
+			
+			ToVariables = NewObject.variables;
 		}
 
 		if (DB_.CreateRecord(NewObject)) {
+			AddMembership(StorageService()->VariablesDB(),
+							 &ProvObjects::VariableBlock::configurations, ToVariables, NewObject.info.id);
 			MoveUsage(StorageService()->PolicyDB(), DB_, "", NewObject.managementPolicy,
 					  NewObject.info.id);
 			AddMembership(StorageService()->VenueDB(), &ProvObjects::Venue::configurations,
@@ -185,9 +202,10 @@ namespace OpenWifi {
 			Existing.deviceTypes = NewObject.deviceTypes;
 
 		std::vector<std::string> Errors;
-		if (!ValidateConfigBlock(NewObject, Errors)) {
-			return BadRequest(RESTAPI::Errors::ConfigBlockInvalid);
-		}
+        auto deviceType = GetParameter("deviceType", "AP");
+        if (!ValidateConfigBlock(ConfigurationValidator::GetType(deviceType), NewObject, Errors)) {
+            return BadRequest(RESTAPI::Errors::ConfigBlockInvalid);
+        }
 
 		if (RawObject->has("configuration")) {
 			Existing.configuration = NewObject.configuration;

src/RESTAPI/RESTAPI_db_helpers.h

@@ -389,6 +389,13 @@ namespace OpenWifi {
 			DB.ManipulateVectorMember(T, "id", Obj, Id, true);
 	}
 
+	template <typename db_type, typename Member>
+	void AddMembership(db_type &DB, Member T, const Types::UUIDvec_t &Obj, const std::string &Id) {
+		for (const auto &i : Obj) {
+			AddMembership(DB, T, i, Id);
+		}
+	}
+
 	template <typename db_type, typename Member>
 	void ManageMembership(db_type &DB, Member T, const std::string &From, const std::string &To,
 						  const std::string &Id) {
@@ -431,47 +438,49 @@ namespace OpenWifi {
 		return EntityDB::RootUUID();
 	}
 
-	inline bool ValidateConfigBlock(const ProvObjects::DeviceConfiguration &Config,
-									std::vector<std::string> &Errors) {
-		static const std::vector<std::string> SectionNames{
-			"globals",	   "interfaces", "metrics", "radios",	  "services",	"unit",
-			"definitions", "ethernet",	 "switch",	"config-raw", "third-party"};
+    inline bool ValidateConfigBlock(ConfigurationValidator::ConfigurationType Type, const ProvObjects::DeviceConfiguration &Config,
+                                    std::vector<std::string> &Errors) {
+        static const std::vector<std::string> SectionNames{
+                "globals",	   "interfaces", "metrics", "radios",	  "services",	"unit",
+                "definitions", "ethernet",	 "switch",	"config-raw", "third-party"};
 
-		for (const auto &i : Config.configuration) {
-			Poco::JSON::Parser P;
-			if (i.name.empty()) {
-				Errors.push_back("Name is empty");
-				return false;
-			}
+        for (const auto &i : Config.configuration) {
+            Poco::JSON::Parser P;
+            if (i.name.empty()) {
+                Errors.push_back("Name is empty");
+                return false;
+            }
 
-			try {
-				auto Blocks = P.parse(i.configuration).extract<Poco::JSON::Object::Ptr>();
-				auto N = Blocks->getNames();
-				for (const auto &j : N) {
-					if (std::find(SectionNames.cbegin(), SectionNames.cend(), j) ==
-						SectionNames.cend()) {
-						Errors.push_back("Unknown block name");
-						return false;
-					}
-				}
-			} catch (const Poco::JSON::JSONException &E) {
-				Errors.push_back("Invalid JSON document");
-				return false;
-			}
+            try {
+                auto Blocks = P.parse(i.configuration).extract<Poco::JSON::Object::Ptr>();
+                auto N = Blocks->getNames();
+                for (const auto &j : N) {
+                    if (std::find(SectionNames.cbegin(), SectionNames.cend(), j) ==
+                        SectionNames.cend()) {
+                        Errors.push_back("Unknown block name");
+                        return false;
+                    }
+                }
+            } catch (const Poco::JSON::JSONException &E) {
+                Errors.push_back("Invalid JSON document");
+                return false;
+            }
 
-			try {
-				if (ValidateUCentralConfiguration(i.configuration, Errors, true)) {
-					// std::cout << "Block: " << i.name << " is valid" << std::endl;
-				} else {
-					return false;
-				}
-			} catch (...) {
-				Errors.push_back("Invalid configuration caused an exception");
-				return false;
-			}
-		}
-		return true;
-	}
+            try {
+                std::string Error;
+                if (ValidateUCentralConfiguration(Type,i.configuration, Error, true)) {
+                    // std::cout << "Block: " << i.name << " is valid" << std::endl;
+                } else {
+                    Errors.push_back(Error);
+                    return false;
+                }
+            } catch (...) {
+                Errors.push_back("Invalid configuration caused an exception");
+                return false;
+            }
+        }
+        return true;
+    }
 
 	template <typename Type>
 	std::map<std::string, std::string> CreateObjects(Type &NewObject, RESTAPIHandler &R,
@@ -535,7 +544,7 @@ namespace OpenWifi {
 						ProvObjects::DeviceConfiguration DC;
 						if (DC.from_json(ConfigurationDetails)) {
 							if constexpr (std::is_same_v<Type, ProvObjects::InventoryTag>) {
-								if (!ValidateConfigBlock(DC, Errors)) {
+								if (!ValidateConfigBlock(ConfigurationValidator::ConfigurationType::AP,DC, Errors)) {
 									break;
 								}
 								ProvObjects::CreateObjectInfo(R.UserInfo_.userinfo, DC.info);

src/RESTAPI/RESTAPI_inventory_handler.cpp

@@ -14,28 +14,13 @@
 #include "RESTAPI/RESTAPI_db_helpers.h"
 #include "SerialNumberCache.h"
 #include "StorageService.h"
+#include "Tasks/VenueConfigUpdater.h"
 #include "framework/utils.h"
 #include "sdks/SDK_gw.h"
 #include "sdks/SDK_sec.h"
 
 namespace OpenWifi {
 
-	void GetRejectedLines(const Poco::JSON::Object::Ptr &Response, Types::StringVec &Warnings) {
-		try {
-			if (Response->has("results")) {
-				auto Results = Response->get("results").extract<Poco::JSON::Object::Ptr>();
-				auto Status = Results->get("status").extract<Poco::JSON::Object::Ptr>();
-				auto Rejected = Status->getArray("rejected");
-				std::transform(
-					Rejected->begin(), Rejected->end(), std::back_inserter(Warnings),
-					[](auto i) -> auto { return i.toString(); });
-				//                for(const auto &i:*Rejected)
-				//                  Warnings.push_back(i.toString());
-			}
-		} catch (...) {
-		}
-	}
-
 	void RESTAPI_inventory_handler::DoGet() {
 
 		ProvObjects::InventoryTag Existing;
@@ -314,6 +299,8 @@ namespace OpenWifi {
 			return NotFound();
 		}
 
+		std::string previous_venue = Existing.venue;
+
 		auto RemoveSubscriber = GetParameter("removeSubscriber");
 		if (!RemoveSubscriber.empty()) {
 			if (Existing.subscriber == RemoveSubscriber) {
@@ -471,6 +458,13 @@ namespace OpenWifi {
 			SDK::GW::Device::SetOwnerShip(this, SerialNumber, Existing.entity, Existing.venue,
 										  Existing.subscriber);
 
+			// Attempt an automatic config push when the venue is set and different than what is
+			// in DB.
+			poco_information(Logger(), fmt::format("New Venue {} Old Venue {}", NewObject.venue, previous_venue));
+			if (!NewObject.venue.empty() && NewObject.venue != previous_venue) {
+				ComputeAndPushConfig(SerialNumber, NewObject.deviceType, Logger());
+			}
+
 			ProvObjects::InventoryTag NewObjectCreated;
 			DB_.GetRecord("id", Existing.info.id, NewObjectCreated);
 			Poco::JSON::Object Answer;

src/RESTAPI/RESTAPI_inventory_list_handler.cpp

@@ -131,7 +131,9 @@ namespace OpenWifi {
 		} else {
 			ProvObjects::InventoryTagVec Tags;
 			DB_.GetRecords(QB_.Offset, QB_.Limit, Tags, "", OrderBy);
-			return MakeJSONObjectArray("taglist", Tags, *this);
+            return SendList(Tags, SerialOnly);
+
+//			return MakeJSONObjectArray("taglist", Tags, *this);
 		}
 	}
 } // namespace OpenWifi

src/RESTAPI/RESTAPI_venue_handler.cpp

@@ -187,7 +187,7 @@ namespace OpenWifi {
 			}
 		}
 
-		if (!NewObject.sourceIP.empty() && CIDR::ValidateIpRanges(NewObject.sourceIP)) {
+		if (!NewObject.sourceIP.empty() && !CIDR::ValidateIpRanges(NewObject.sourceIP)) {
 			return BadRequest(RESTAPI::Errors::InvalidIPRanges);
 		}
 
@@ -276,21 +276,19 @@ namespace OpenWifi {
 		auto testUpdateOnly = GetBoolParameter("testUpdateOnly");
 		if (testUpdateOnly) {
 			ProvObjects::SerialNumberList SNL;
-
+            StorageService()->InventoryDB().GetDevicesForVenue(UUID, SNL.serialNumbers);
 			Poco::JSON::Object Answer;
-			SNL.serialNumbers = Existing.devices;
 			SNL.to_json(Answer);
 			return ReturnObject(Answer);
 		}
 
 		if (GetBoolParameter("updateAllDevices")) {
 			ProvObjects::SerialNumberList SNL;
+            StorageService()->InventoryDB().GetDevicesForVenue(UUID, SNL.serialNumbers);
 
 			Poco::JSON::Object Answer;
-			SNL.serialNumbers = Existing.devices;
 			auto JobId = MicroServiceCreateUUID();
 			Types::StringVec Parameters{UUID};
-			;
 			auto NewJob = new VenueConfigUpdater(JobId, "VenueConfigurationUpdater", Parameters, 0,
 												 UserInfo_.userinfo, Logger());
 			JobController()->AddJob(dynamic_cast<Job *>(NewJob));
@@ -302,11 +300,10 @@ namespace OpenWifi {
 		if (GetBoolParameter("upgradeAllDevices")) {
 			if (GetBoolParameter("revisionsAvailable")) {
 				std::set<std::string> DeviceTypes;
-				for (const auto &serialNumber : Existing.devices) {
-					ProvObjects::InventoryTag Device;
-					if (StorageService()->InventoryDB().GetRecord("id", serialNumber, Device)) {
-						DeviceTypes.insert(Device.deviceType);
-					}
+                std::vector<ProvObjects::InventoryTag> ExistingDevices;
+                StorageService()->InventoryDB().GetDevicesForVenue(UUID, ExistingDevices);
+				for (const auto &device : ExistingDevices) {
+                    DeviceTypes.insert(device.deviceType);
 				}
 
 				//  Get all the revisions for all the device types
@@ -374,18 +371,17 @@ namespace OpenWifi {
 				return ReturnObject(Answer);
 			}
 
-			ProvObjects::SerialNumberList SNL;
-
 			auto Revision = GetParameter("revision", "");
 			if (Revision.empty()) {
 				return BadRequest(RESTAPI::Errors::MissingOrInvalidParameters);
 			}
 
+            ProvObjects::SerialNumberList SNL;
+            StorageService()->InventoryDB().GetDevicesForVenue(UUID, SNL.serialNumbers);
+
 			Poco::JSON::Object Answer;
-			SNL.serialNumbers = Existing.devices;
 			auto JobId = MicroServiceCreateUUID();
 			Types::StringVec Parameters{UUID, Revision};
-			;
 			auto NewJob = new VenueUpgrade(JobId, "VenueFirmwareUpgrade", Parameters, 0,
 										   UserInfo_.userinfo, Logger());
 			JobController()->AddJob(dynamic_cast<Job *>(NewJob));
@@ -396,9 +392,9 @@ namespace OpenWifi {
 
 		if (GetBoolParameter("rebootAllDevices")) {
 			ProvObjects::SerialNumberList SNL;
+            StorageService()->InventoryDB().GetDevicesForVenue(UUID, SNL.serialNumbers);
 
 			Poco::JSON::Object Answer;
-			SNL.serialNumbers = Existing.devices;
 			auto JobId = MicroServiceCreateUUID();
 			Types::StringVec Parameters{UUID};
 			;

src/RESTObjects/RESTAPI_ProvObjects.cpp

@@ -587,6 +587,9 @@ namespace OpenWifi::ProvObjects {
 		field_to_json(Obj, "locale", locale);
 		field_to_json(Obj, "realMacAddress", realMacAddress);
 		field_to_json(Obj, "doNotAllowOverrides", doNotAllowOverrides);
+        field_to_json(Obj, "imported", imported);
+        field_to_json(Obj, "connected", connected);
+        field_to_json(Obj, "platform", platform);
 	}
 
 	bool InventoryTag::from_json(const Poco::JSON::Object::Ptr &Obj) {
@@ -609,6 +612,9 @@ namespace OpenWifi::ProvObjects {
 			field_from_json(Obj, "locale", locale);
 			field_from_json(Obj, "realMacAddress", realMacAddress);
 			field_from_json(Obj, "doNotAllowOverrides", doNotAllowOverrides);
+            field_from_json(Obj, "imported", imported);
+            field_from_json(Obj, "connected", connected);
+            field_from_json(Obj, "platform", platform);
 			return true;
 		} catch (...) {
 		}

src/RESTObjects/RESTAPI_ProvObjects.h

@@ -490,9 +490,11 @@ namespace OpenWifi::ProvObjects {
         std::string locale;
         std::string realMacAddress;
         bool doNotAllowOverrides = false;
+        std::uint64_t imported=0;
+        std::uint64_t connected=0;
+        std::string platform{"AP"};
 
         void to_json(Poco::JSON::Object &Obj) const;
-
         bool from_json(const Poco::JSON::Object::Ptr &Obj);
     };
 

src/Tasks/VenueConfigUpdater.h

@@ -28,6 +28,42 @@ namespace OpenWifi {
 		}
 	}
 
+	[[maybe_unused]] static void ComputeAndPushConfig(const std::string &SerialNumber, const std::string &DeviceType, Poco::Logger &Logger) {
+		/*
+		Generic Helper to compute a device's config and push it down to the device.
+		*/
+		poco_information(Logger, fmt::format("Attempting to push venue config for device {}", SerialNumber));
+		auto DeviceConfig = std::make_shared<APConfig>(SerialNumber,
+														DeviceType, Logger, false);
+		auto Configuration = Poco::makeShared<Poco::JSON::Object>();
+		try {
+			if (DeviceConfig->Get(Configuration)) {
+				std::ostringstream OS;
+				Configuration->stringify(OS);
+				auto Response = Poco::makeShared<Poco::JSON::Object>();
+				poco_debug(Logger,
+							fmt::format("{}: Pushing configuration.", SerialNumber));
+				if (SDK::GW::Device::Configure(nullptr, SerialNumber, Configuration,
+												Response)) {
+					Logger.debug(
+						fmt::format("{}: Configuration pushed.", SerialNumber));
+					poco_information(Logger,
+										fmt::format("{}: Updated.", SerialNumber));
+				} else {
+					poco_information(Logger,
+										fmt::format("{}: Not updated.", SerialNumber));
+				}
+			} else {
+				poco_debug(Logger,
+							fmt::format("{}: Configuration is bad.", SerialNumber));
+			}
+		} catch (...) {
+			poco_debug(Logger,
+						fmt::format("{}: Configuration is bad (caused an exception).",
+									SerialNumber));
+		}
+	}
+
 	class VenueDeviceConfigUpdater : public Poco::Runnable {
 	  public:
 		VenueDeviceConfigUpdater(const std::string &UUID, const std::string &venue, Poco::Logger &L)
@@ -118,8 +154,9 @@ namespace OpenWifi {
 
 				Poco::ThreadPool Pool_;
 				std::list<VenueDeviceConfigUpdater *> JobList;
-
-				for (const auto &uuid : Venue.devices) {
+                std::vector<std::string> DeviceList;
+                StorageService()->InventoryDB().GetDevicesUUIDForVenue(Venue.info.id, DeviceList);
+				for (const auto &uuid : DeviceList) {
 					auto NewTask = new VenueDeviceConfigUpdater(uuid, Venue.info.name, Logger());
 					bool TaskAdded = false;
 					while (!TaskAdded) {

src/Tasks/VenueRebooter.h

@@ -68,8 +68,10 @@ namespace OpenWifi {
 
 				Poco::ThreadPool Pool_;
 				std::list<VenueDeviceRebooter *> JobList;
+                std::vector<std::string> DeviceList;
+                StorageService()->InventoryDB().GetDevicesUUIDForVenue(Venue.info.id, DeviceList);
 
-				for (const auto &uuid : Venue.devices) {
+				for (const auto &uuid : DeviceList) {
 					auto NewTask = new VenueDeviceRebooter(uuid, Venue.info.name, Logger());
 					bool TaskAdded = false;
 					while (!TaskAdded) {

src/Tasks/VenueUpgrade.h

@@ -28,7 +28,7 @@ namespace OpenWifi {
 
 				Storage::ApplyRules(rules_, Device.deviceRules);
 				if (Device.deviceRules.firmwareUpgrade == "no") {
-					poco_debug(Logger(), fmt::format("Skipped Upgrade: {}", Device.serialNumber));
+					poco_debug(Logger(), fmt::format("Skipped Upgrade: {} : Venue rules prevent upgrading", Device.serialNumber));
 					skipped_++;
 					done_ = true;
 					return;
@@ -36,10 +36,15 @@ namespace OpenWifi {
 
 				FMSObjects::Firmware F;
 				if (SDK::FMS::Firmware::GetFirmware(Device.deviceType, revision_, F)) {
-					if (SDK::GW::Device::Upgrade(nullptr, Device.serialNumber, 0, F.uri)) {
-						Logger().debug(
-							fmt::format("{}: Upgraded to {}.", Device.serialNumber, revision_));
-						upgraded_++;
+                    std::string Status;
+					if (SDK::GW::Device::Upgrade(nullptr, Device.serialNumber, 0, F.uri, Status)) {
+                        if(Status=="pending") {
+                            pending_++;
+                            poco_debug(Logger(), fmt::format("Upgrade Pending: {} : {}", Device.serialNumber, Status));
+                        } else {
+                            upgraded_++;
+                            poco_debug(Logger(), fmt::format("Upgrade Success: {} : {}", Device.serialNumber, Status));
+                        }
 					} else {
 						poco_information(Logger(), fmt::format("{}: Not Upgraded to {}.",
 															   Device.serialNumber, revision_));
@@ -53,10 +58,9 @@ namespace OpenWifi {
 				}
 			}
 			done_ = true;
-			// std::cout << "Done push for " << Device.serialNumber << std::endl;
 		}
 
-		std::uint64_t upgraded_ = 0, not_connected_ = 0, skipped_ = 0, no_firmware_ = 0;
+		std::uint64_t upgraded_ = 0, not_connected_ = 0, skipped_ = 0, no_firmware_ = 0, pending_ = 0;
 		bool started_ = false, done_ = false;
 		std::string SerialNumber;
 
@@ -85,7 +89,7 @@ namespace OpenWifi {
 			ProvWebSocketNotifications::VenueFWUpgradeList_t N;
 
 			ProvObjects::Venue Venue;
-			uint64_t upgraded_ = 0, not_connected_ = 0, skipped_ = 0, no_firmware_ = 0;
+			uint64_t upgraded_ = 0, not_connected_ = 0, skipped_ = 0, no_firmware_ = 0, pending_=0;
 			if (StorageService()->VenueDB().GetRecord("id", VenueUUID_, Venue)) {
 
 				N.content.title = fmt::format("Upgrading {} devices.", Venue.info.name);
@@ -96,8 +100,10 @@ namespace OpenWifi {
 				ProvObjects::DeviceRules Rules;
 
 				StorageService()->VenueDB().EvaluateDeviceRules(Venue.info.id, Rules);
+                std::vector<std::string> DeviceList;
+                StorageService()->InventoryDB().GetDevicesUUIDForVenue(Venue.info.id, DeviceList);
 
-				for (const auto &uuid : Venue.devices) {
+				for (const auto &uuid : DeviceList) {
 					auto NewTask =
 						new VenueDeviceUpgrade(uuid, Venue.info.name, Revision_, Rules, Logger());
 					bool TaskAdded = false;
@@ -121,10 +127,13 @@ namespace OpenWifi {
 								N.content.not_connected.push_back(current_job->SerialNumber);
 							else if (current_job->no_firmware_)
 								N.content.no_firmware.push_back(current_job->SerialNumber);
+                            else if (current_job->pending_)
+                                N.content.pending.push_back(current_job->SerialNumber);
 							upgraded_ += current_job->upgraded_;
 							skipped_ += current_job->skipped_;
 							no_firmware_ += current_job->no_firmware_;
 							not_connected_ += current_job->not_connected_;
+                            pending_ += current_job->pending_;
 							job_it = JobList.erase(job_it);
 							delete current_job;
 						} else {
@@ -146,10 +155,13 @@ namespace OpenWifi {
 							N.content.not_connected.push_back(current_job->SerialNumber);
 						else if (current_job->no_firmware_)
 							N.content.no_firmware.push_back(current_job->SerialNumber);
+                        else if (current_job->pending_)
+                            N.content.pending.push_back(current_job->SerialNumber);
 						upgraded_ += current_job->upgraded_;
 						skipped_ += current_job->skipped_;
 						no_firmware_ += current_job->no_firmware_;
 						not_connected_ += current_job->not_connected_;
+                        pending_ += current_job->pending_;
 						job_it = JobList.erase(job_it);
 						delete current_job;
 					} else {
@@ -158,8 +170,8 @@ namespace OpenWifi {
 				}
 
 				N.content.details = fmt::format(
-					"Job {} Completed: {} upgraded, {} not connected, {} skipped, {} no firmware.",
-					JobId(), upgraded_, not_connected_, skipped_, no_firmware_);
+					"Job {} Completed: {} upgraded, {} not connected, {} skipped, {} no firmware, {} pending.",
+					JobId(), upgraded_, not_connected_, skipped_, no_firmware_, pending_);
 			} else {
 				N.content.details = fmt::format("Venue {} no longer exists.", VenueUUID_);
 				Logger().warning(N.content.details);

src/UI_Prov_WebSocketNotifications.cpp

@@ -60,6 +60,7 @@ namespace OpenWifi::ProvWebSocketNotifications {
 		RESTAPI_utils::field_to_json(Obj, "success", success);
 		RESTAPI_utils::field_to_json(Obj, "notConnected", not_connected);
 		RESTAPI_utils::field_to_json(Obj, "noFirmware", no_firmware);
+        RESTAPI_utils::field_to_json(Obj, "pending", pending);
 		RESTAPI_utils::field_to_json(Obj, "skipped", skipped);
 		RESTAPI_utils::field_to_json(Obj, "timeStamp", timeStamp);
 		RESTAPI_utils::field_to_json(Obj, "details", details);
@@ -71,6 +72,7 @@ namespace OpenWifi::ProvWebSocketNotifications {
 			RESTAPI_utils::field_from_json(Obj, "jobId", jobId);
 			RESTAPI_utils::field_from_json(Obj, "success", success);
 			RESTAPI_utils::field_from_json(Obj, "notConnected", not_connected);
+            RESTAPI_utils::field_from_json(Obj, "pending", pending);
 			RESTAPI_utils::field_from_json(Obj, "noFirmware", no_firmware);
 			RESTAPI_utils::field_from_json(Obj, "skipped", skipped);
 			RESTAPI_utils::field_from_json(Obj, "timeStamp", timeStamp);

src/UI_Prov_WebSocketNotifications.h

@@ -32,7 +32,7 @@ namespace OpenWifi::ProvWebSocketNotifications {
 
 	struct FWUpgradeList {
 		std::string title, details, jobId;
-		std::vector<std::string> success, skipped, no_firmware, not_connected;
+		std::vector<std::string> success, skipped, no_firmware, not_connected, pending;
 		uint64_t timeStamp = OpenWifi::Utils::Now();
 
 		void to_json(Poco::JSON::Object &Obj) const;

src/framework/AppServiceRegistry.h

@@ -67,9 +67,9 @@ namespace OpenWifi {
 
         bool Get(const char *key, std::vector<std::string> &Value) {
             if(Registry_->has(key) && !Registry_->isNull(key) && Registry_->isArray(key)) {
-                auto Arr = Registry_->get(key);
-                for(const auto &v:Arr) {
-                    Value.emplace_back(v);
+                auto pArr = Registry_->getArray(key);
+                for(const auto &v : *pArr) {
+                    Value.emplace_back(v.toString());
                 }
                 return true;
             }
@@ -91,4 +91,4 @@ namespace OpenWifi {
 
 	inline auto AppServiceRegistry() { return AppServiceRegistry::instance(); }
 
-} // namespace OpenWifi
+} // namespace OpenWifi

src/framework/ConfigurationValidator.h

@@ -17,33 +17,42 @@
 namespace OpenWifi {
 	class ConfigurationValidator : public SubSystemServer {
 	  public:
+
+		enum class ConfigurationType { AP = 0 , SWITCH = 1};
+
 		static auto instance() {
 			static auto instance_ = new ConfigurationValidator;
 			return instance_;
 		}
 
-		bool Validate(const std::string &C, std::vector<std::string> &Errors, bool Strict);
+		bool Validate(ConfigurationType Type, const std::string &C, std::string &Errors, bool Strict);
 		int Start() override;
 		void Stop() override;
 		void reinitialize(Poco::Util::Application &self) override;
 
+		inline static ConfigurationType GetType(const std::string &type) {
+			std::string Type = Poco::toUpper(type);
+			if (Type == "AP")
+				return ConfigurationType::AP;
+			if (Type == "SWITCH")
+				return ConfigurationType::SWITCH;
+			return ConfigurationType::AP;
+		}
+
 	  private:
 		bool Initialized_ = false;
 		bool Working_ = false;
 		void Init();
-		std::unique_ptr<valijson::Schema> RootSchema_;
-		std::unique_ptr<valijson::SchemaParser> SchemaParser_;
-		std::unique_ptr<valijson::adapters::PocoJsonAdapter> PocoJsonAdapter_;
-		Poco::JSON::Object::Ptr SchemaDocPtr_;
-		bool SetSchema(const std::string &SchemaStr);
+		std::array<valijson::Schema,2> 			RootSchema_;
+		bool SetSchema(ConfigurationType Type, const std::string &SchemaStr);
 
 		ConfigurationValidator()
 			: SubSystemServer("ConfigValidator", "CFG-VALIDATOR", "config.validator") {}
 	};
 
 	inline auto ConfigurationValidator() { return ConfigurationValidator::instance(); }
-	inline bool ValidateUCentralConfiguration(const std::string &C, std::vector<std::string> &Error,
+	inline bool ValidateUCentralConfiguration(ConfigurationValidator::ConfigurationType Type, const std::string &C, std::string &Errors,
 											  bool strict) {
-		return ConfigurationValidator::instance()->Validate(C, Error, strict);
+		return ConfigurationValidator::instance()->Validate(Type, C, Errors, strict);
 	}
 } // namespace OpenWifi

src/framework/EventBusManager.cpp

@@ -9,8 +9,6 @@
 
 namespace OpenWifi {
 
-	EventBusManager::EventBusManager(Poco::Logger &L) : Logger_(L) {}
-
 	void EventBusManager::run() {
 		Running_ = true;
 		Utils::SetThreadName("fmwk:EventMgr");
@@ -18,9 +16,9 @@ namespace OpenWifi {
 		KafkaManager()->PostMessage(KafkaTopics::SERVICE_EVENTS, MicroServicePrivateEndPoint(), Msg,
 									false);
 		while (Running_) {
-			Poco::Thread::trySleep((unsigned long)MicroServiceDaemonBusTimer());
-			if (!Running_)
-				break;
+			if(!Poco::Thread::trySleep((unsigned long)MicroServiceDaemonBusTimer())) {
+                break;
+            }
 			Msg = (MicroServiceMakeSystemEventMessage(KafkaTopics::ServiceEvents::EVENT_KEEP_ALIVE));
 			KafkaManager()->PostMessage(KafkaTopics::SERVICE_EVENTS, MicroServicePrivateEndPoint(),
 										Msg, false);
@@ -31,7 +29,7 @@ namespace OpenWifi {
 	};
 
 	void EventBusManager::Start() {
-		poco_information(Logger(), "Starting...");
+		poco_information(Logger_, "Starting...");
 		if (KafkaManager()->Enabled()) {
 			Thread_.start(*this);
 		}
@@ -39,11 +37,11 @@ namespace OpenWifi {
 
 	void EventBusManager::Stop() {
 		if (KafkaManager()->Enabled()) {
-			poco_information(Logger(), "Stopping...");
+			poco_information(Logger_, "Stopping...");
 			Running_ = false;
 			Thread_.wakeUp();
 			Thread_.join();
-			poco_information(Logger(), "Stopped...");
+			poco_information(Logger_, "Stopped...");
 		}
 	}
 

src/framework/EventBusManager.h

@@ -12,7 +12,16 @@ namespace OpenWifi {
 
 	class EventBusManager : public Poco::Runnable {
 	  public:
-		explicit EventBusManager(Poco::Logger &L);
+		EventBusManager() :
+			Logger_(Poco::Logger::create(
+				"EventBusManager", Poco::Logger::root().getChannel(), Poco::Logger::root().getLevel())) {
+		}
+
+		static auto instance() {
+			static auto instance_ = new EventBusManager;
+			return instance_;
+		}
+
 		void run() final;
 		void Start();
 		void Stop();
@@ -24,4 +33,6 @@ namespace OpenWifi {
 		Poco::Logger &Logger_;
 	};
 
+	inline auto EventBusManager() { return EventBusManager::instance(); }
+
 } // namespace OpenWifi

src/framework/KafkaManager.cpp

@@ -79,8 +79,10 @@ namespace OpenWifi {
 		Utils::SetThreadName("Kafka:Prod");
 		cppkafka::Configuration Config(
 			{{"client.id", MicroServiceConfigGetString("openwifi.kafka.client.id", "")},
-			 {"metadata.broker.list",
-			  MicroServiceConfigGetString("openwifi.kafka.brokerlist", "")}});
+			 {"metadata.broker.list",MicroServiceConfigGetString("openwifi.kafka.brokerlist", "")} // ,
+			 // {"send.buffer.bytes", KafkaManager()->KafkaManagerMaximumPayloadSize() }
+			}
+ 		);
 
 		AddKafkaSecurity(Config);
 
@@ -105,6 +107,19 @@ namespace OpenWifi {
 					NewMessage.partition(0);
 					NewMessage.payload(Msg->Payload());
 					Producer.produce(NewMessage);
+					if (Queue_.size() < 100) {
+						// use flush when internal queue is lightly loaded, i.e. flush after each
+						// message
+						Producer.flush();
+					}
+					else {
+						// use poll when internal queue is loaded to allow messages to be sent in
+						// batches
+						Producer.poll((std::chrono::milliseconds) 0);
+					}
+				}
+		                if (Queue_.size() == 0) {
+					// message queue is empty, flush all previously sent messages
 					Producer.flush();
 				}
 			} catch (const cppkafka::HandleException &E) {
@@ -117,6 +132,7 @@ namespace OpenWifi {
 			}
 			Note = Queue_.waitDequeueNotification();
 		}
+		Producer.flush();
 		poco_information(Logger_, "Stopped...");
 	}
 
@@ -275,6 +291,7 @@ namespace OpenWifi {
 	int KafkaManager::Start() {
 		if (!KafkaEnabled_)
 			return 0;
+		MaxPayloadSize_ = MicroServiceConfigGetInt("openwifi.kafka.max.payload", 250000);
 		ConsumerThr_.Start();
 		ProducerThr_.Start();
 		return 0;
@@ -321,4 +338,4 @@ namespace OpenWifi {
 											   partitions.front().get_partition()));
 	}
 
-} // namespace OpenWifi
+} // namespace OpenWifi

src/framework/KafkaManager.h

@@ -94,11 +94,14 @@ namespace OpenWifi {
 			return ConsumerThr_.UnregisterTopicWatcher(Topic,Id);
 		}
 
+		std::uint64_t KafkaManagerMaximumPayloadSize() const { return MaxPayloadSize_; }
+
 	  private:
 		bool KafkaEnabled_ = false;
 		std::string SystemInfoWrapper_;
 		KafkaProducer ProducerThr_;
 		KafkaConsumer ConsumerThr_;
+		std::uint64_t MaxPayloadSize_ = 250000;
 
 		void PartitionAssignment(const cppkafka::TopicPartitionList &partitions);
 		void PartitionRevocation(const cppkafka::TopicPartitionList &partitions);

src/framework/MicroService.cpp

@@ -1,4 +1,5 @@
 //
+//
 // Created by stephane bourque on 2022-10-26.
 //
 
@@ -29,13 +30,29 @@
 #include "framework/WebSocketLogger.h"
 #include "framework/utils.h"
 
+#ifdef  USE_MEDUSA_CLIENT
+#include <medusa/MedusaClient.h>
+#endif
+
 namespace OpenWifi {
 
-	void MicroService::Exit(int Reason) { std::exit(Reason); }
+	static std::string MakeServiceListString(const Types::MicroServiceMetaMap &Services) {
+        std::string SvcList;
+        for (const auto &Svc : Services) {
+            if (SvcList.empty())
+                SvcList = Svc.second.Type;
+            else
+                SvcList += ", " + Svc.second.Type;
+        }
+        return SvcList;
+    }
 
 	void MicroService::BusMessageReceived([[maybe_unused]] const std::string &Key,
 										  const std::string &Payload) {
 		std::lock_guard G(InfraMutex_);
+
+		Poco::Logger &BusLogger = EventBusManager()->Logger();
+
 		try {
 			Poco::JSON::Parser P;
 			auto Object = P.parse(Payload).extract<Poco::JSON::Object::Ptr>();
@@ -55,13 +72,10 @@ namespace OpenWifi {
 							Object->has(KafkaTopics::ServiceEvents::Fields::KEY)) {
 							auto PrivateEndPoint =
 								Object->get(KafkaTopics::ServiceEvents::Fields::PRIVATE).toString();
-							if (Event == KafkaTopics::ServiceEvents::EVENT_KEEP_ALIVE &&
-								Services_.find(PrivateEndPoint) != Services_.end()) {
-								Services_[PrivateEndPoint].LastUpdate = Utils::Now();
-							} else if (Event == KafkaTopics::ServiceEvents::EVENT_LEAVE) {
+							if (Event == KafkaTopics::ServiceEvents::EVENT_LEAVE) {
 								Services_.erase(PrivateEndPoint);
-								poco_debug(
-									logger(),
+								poco_information(
+									BusLogger,
 									fmt::format(
 										"Service {} ID={} leaving system.",
 										Object->get(KafkaTopics::ServiceEvents::Fields::PRIVATE)
@@ -69,14 +83,7 @@ namespace OpenWifi {
 										ID));
 							} else if (Event == KafkaTopics::ServiceEvents::EVENT_JOIN ||
 									   Event == KafkaTopics::ServiceEvents::EVENT_KEEP_ALIVE) {
-								poco_debug(
-									logger(),
-									fmt::format(
-										"Service {} ID={} joining system.",
-										Object->get(KafkaTopics::ServiceEvents::Fields::PRIVATE)
-											.toString(),
-										ID));
-								Services_[PrivateEndPoint] = Types::MicroServiceMeta{
+								auto ServiceInfo = Types::MicroServiceMeta{
 									.Id = ID,
 									.Type = Poco::toLower(
 										Object->get(KafkaTopics::ServiceEvents::Fields::TYPE)
@@ -94,20 +101,46 @@ namespace OpenWifi {
 												   .toString(),
 									.LastUpdate = Utils::Now()};
 
-								std::string SvcList;
-								for (const auto &Svc : Services_) {
-									if (SvcList.empty())
-										SvcList = Svc.second.Type;
-									else
-										SvcList += ", " + Svc.second.Type;
+                                auto s1 = MakeServiceListString(Services_);
+								auto PreviousSize = Services_.size();
+								Services_[PrivateEndPoint] = ServiceInfo;
+								auto CurrentSize = Services_.size();
+								if(Event == KafkaTopics::ServiceEvents::EVENT_JOIN) {
+									if(!s1.empty()) {
+										poco_information(
+											BusLogger,
+											fmt::format(
+												"Service {} ID={} is joining the system.",
+												Object
+													->get(
+														KafkaTopics::ServiceEvents::Fields::PRIVATE)
+													.toString(),
+												ID));
+									}
+									std::string SvcList;
+									for (const auto &Svc : Services_) {
+										if (SvcList.empty())
+											SvcList = Svc.second.Type;
+										else
+											SvcList += ", " + Svc.second.Type;
+									}
+									poco_information(
+										BusLogger,
+										fmt::format("Current list of microservices: {}", SvcList));
+								} else if(CurrentSize!=PreviousSize) {
+									poco_information(
+										BusLogger,
+										fmt::format(
+											"Service {} ID={} is being added back in.",
+											Object
+												->get(KafkaTopics::ServiceEvents::Fields::PRIVATE)
+												.toString(),
+											ID));
 								}
-								poco_information(
-									logger(),
-									fmt::format("Current list of microservices: {}", SvcList));
 							}
 						} else {
-							poco_error(
-								logger(),
+							poco_information(
+								BusLogger,
 								fmt::format("KAFKA-MSG: invalid event '{}', missing a field.",
 											Event));
 						}
@@ -118,32 +151,39 @@ namespace OpenWifi {
 								Object->get(KafkaTopics::ServiceEvents::Fields::TOKEN).toString());
 #endif
 						} else {
-							poco_error(
-								logger(),
+							poco_information(
+								BusLogger,
 								fmt::format("KAFKA-MSG: invalid event '{}', missing token", Event));
 						}
 					} else {
-						poco_error(logger(),
+						poco_information(BusLogger,
 								   fmt::format("Unknown Event: {} Source: {}", Event, ID));
 					}
 				}
 			} else {
-				poco_error(logger(), "Bad bus message.");
-                std::ostringstream os;
-                Object->stringify(std::cout);
+				std::ostringstream os;
+				Object->stringify(std::cout);
+				poco_error(BusLogger, fmt::format("Bad bus message: {}", os.str()));
 			}
 
-			auto i = Services_.begin();
+			auto ServiceHint = Services_.begin();
 			auto now = Utils::Now();
-			for (; i != Services_.end();) {
-				if ((now - i->second.LastUpdate) > 60) {
-					i = Services_.erase(i);
+            auto si1 = Services_.size();
+            auto ss1 = MakeServiceListString(Services_);
+			while(ServiceHint!=Services_.end()) {
+				if ((now - ServiceHint->second.LastUpdate) > 120) {
+					poco_information(BusLogger, fmt::format("ZombieService: Removing service {}, ", ServiceHint->second.PublicEndPoint));
+					ServiceHint = Services_.erase(ServiceHint);
 				} else
-					++i;
+					++ServiceHint;
 			}
+            if(Services_.size() != si1) {
+                auto ss2 = MakeServiceListString(Services_);
+                poco_information(BusLogger, fmt::format("Current list of microservices: {} -> {}", ss1, ss2));
+            }
 
 		} catch (const Poco::Exception &E) {
-			logger().log(E);
+			BusLogger.log(E);
 		}
 	}
 
@@ -167,25 +207,29 @@ namespace OpenWifi {
 			Res.push_back(ServiceRec);
 		}
 		return Res;
+
 	}
 
 	void MicroService::LoadConfigurationFile() {
-		std::string Location = Poco::Environment::get(DAEMON_CONFIG_ENV_VAR, ".");
-		ConfigFileName_ =
-			ConfigFileName_.empty() ? Location + "/" + DAEMON_PROPERTIES_FILENAME : ConfigFileName_;
-		Poco::Path ConfigFile(ConfigFileName_);
+        if(ConfigContent_.empty()) {
+            std::string Location = Poco::Environment::get(DAEMON_CONFIG_ENV_VAR, ".");
+            ConfigFileName_ =
+                ConfigFileName_.empty() ? Location + "/" + DAEMON_PROPERTIES_FILENAME : ConfigFileName_;
+            Poco::Path ConfigFile(ConfigFileName_);
 
-		if (!ConfigFile.isFile()) {
-			std::cerr << DAEMON_APP_NAME << ": Configuration " << ConfigFile.toString()
-					  << " does not seem to exist. Please set " + DAEMON_CONFIG_ENV_VAR +
-							 " env variable the path of the " + DAEMON_PROPERTIES_FILENAME +
-							 " file."
-					  << std::endl;
-			std::exit(Poco::Util::Application::EXIT_CONFIG);
-		}
-
-		// 	    loadConfiguration(ConfigFile.toString());
-		PropConfigurationFile_ = new Poco::Util::PropertyFileConfiguration(ConfigFile.toString());
+            if (!ConfigFile.isFile()) {
+                std::cerr << DAEMON_APP_NAME << ": Configuration " << ConfigFile.toString()
+                          << " does not seem to exist. Please set " + DAEMON_CONFIG_ENV_VAR +
+                                 " env variable the path of the " + DAEMON_PROPERTIES_FILENAME +
+                                 " file."
+                          << std::endl;
+                std::exit(Poco::Util::Application::EXIT_CONFIG);
+            }
+            PropConfigurationFile_ = new Poco::Util::PropertyFileConfiguration(ConfigFile.toString());
+        } else {
+            std::istringstream is(ConfigContent_);
+            PropConfigurationFile_ = new Poco::Util::PropertyFileConfiguration(is);
+        }
 		configPtr()->addWriteable(PropConfigurationFile_, PRIO_DEFAULT);
 	}
 
@@ -388,49 +432,69 @@ namespace OpenWifi {
 
 	void DaemonPostInitialization(Poco::Util::Application &self);
 
-	void MicroService::initialize(Poco::Util::Application &self) {
-		// add the default services
-		LoadConfigurationFile();
-		InitializeLoggingSystem();
+    void MicroService::StartEverything(Poco::Util::Application &self) {
+        LoadConfigurationFile();
+        InitializeLoggingSystem();
 
-		SubSystems_.push_back(KafkaManager());
-		SubSystems_.push_back(ALBHealthCheckServer());
-		SubSystems_.push_back(RESTAPI_ExtServer());
-		SubSystems_.push_back(RESTAPI_IntServer());
+        static bool InitializedBaseService=false;
+        if(!InitializedBaseService) {
+            InitializedBaseService = true;
+            SubSystems_.push_back(KafkaManager());
+            SubSystems_.push_back(ALBHealthCheckServer());
+            SubSystems_.push_back(RESTAPI_ExtServer());
+            SubSystems_.push_back(RESTAPI_IntServer());
 #ifndef TIP_SECURITY_SERVICE
-		SubSystems_.push_back(AuthClient());
+            SubSystems_.push_back(AuthClient());
 #endif
-		Poco::Net::initializeSSL();
-		Poco::Net::HTTPStreamFactory::registerFactory();
-		Poco::Net::HTTPSStreamFactory::registerFactory();
-		Poco::Net::FTPStreamFactory::registerFactory();
-		Poco::Net::FTPSStreamFactory::registerFactory();
 
-		Poco::File DataDir(ConfigPath("openwifi.system.data"));
-		DataDir_ = DataDir.path();
-		if (!DataDir.exists()) {
-			try {
-				DataDir.createDirectory();
-			} catch (const Poco::Exception &E) {
-				logger().log(E);
-			}
-		}
-		WWWAssetsDir_ = ConfigPath("openwifi.restapi.wwwassets", "");
-		if (WWWAssetsDir_.empty())
-			WWWAssetsDir_ = DataDir_;
+            Poco::Net::initializeSSL();
+            Poco::Net::HTTPStreamFactory::registerFactory();
+            Poco::Net::HTTPSStreamFactory::registerFactory();
+            Poco::Net::FTPStreamFactory::registerFactory();
+            Poco::Net::FTPSStreamFactory::registerFactory();
+        }
 
-		LoadMyConfig();
+        Poco::File DataDir(ConfigPath("openwifi.system.data"));
+        DataDir_ = DataDir.path();
+        if (!DataDir.exists()) {
+            try {
+                DataDir.createDirectory();
+            } catch (const Poco::Exception &E) {
+                Logger_.log(E);
+            }
+        }
+        WWWAssetsDir_ = ConfigPath("openwifi.restapi.wwwassets", "");
+        if (WWWAssetsDir_.empty())
+            WWWAssetsDir_ = DataDir_;
 
-		AllowExternalMicroServices_ = ConfigGetBool("allowexternalmicroservices", true);
+        LoadMyConfig();
 
-		InitializeSubSystemServers();
-		ServerApplication::initialize(self);
-		DaemonPostInitialization(self);
+        AllowExternalMicroServices_ = ConfigGetBool("allowexternalmicroservices", true);
 
-		Types::TopicNotifyFunction F = [this](const std::string &Key, const std::string &Payload) {
-			this->BusMessageReceived(Key, Payload);
-		};
-		KafkaManager()->RegisterTopicWatcher(KafkaTopics::SERVICE_EVENTS, F);
+        InitializeSubSystemServers();
+        ServerApplication::initialize(self);
+        DaemonPostInitialization(self);
+
+        Types::TopicNotifyFunction F = [this](const std::string &Key, const std::string &Payload) {
+            this->BusMessageReceived(Key, Payload);
+        };
+        KafkaManager()->RegisterTopicWatcher(KafkaTopics::SERVICE_EVENTS, F);
+    }
+
+    void MicroService::StopEverything([[maybe_unused]] Poco::Util::Application &self) {
+        LoadConfigurationFile();
+        InitializeLoggingSystem();
+
+        Types::TopicNotifyFunction F = [this](const std::string &Key, const std::string &Payload) {
+            this->BusMessageReceived(Key, Payload);
+        };
+        KafkaManager()->RegisterTopicWatcher(KafkaTopics::SERVICE_EVENTS, F);
+    }
+
+    void MicroService::initialize([[maybe_unused]] Poco::Util::Application &self) {
+#ifndef USE_MEDUSA_CLIENT
+        StartEverything(self);
+#endif
 	}
 
 	void MicroService::uninitialize() {
@@ -530,14 +594,12 @@ namespace OpenWifi {
 		for (auto i : SubSystems_) {
 			i->Start();
 		}
-		EventBusManager_ = std::make_unique<EventBusManager>(Poco::Logger::create(
-			"EventBusManager", Poco::Logger::root().getChannel(), Poco::Logger::root().getLevel()));
-		EventBusManager_->Start();
+		EventBusManager()->Start();
 	}
 
 	void MicroService::StopSubSystemServers() {
 		AddActivity("Stopping");
-		EventBusManager_->Stop();
+		EventBusManager()->Stop();
 		for (auto i = SubSystems_.rbegin(); i != SubSystems_.rend(); ++i) {
 			(*i)->Stop();
 		}
@@ -697,7 +759,7 @@ namespace OpenWifi {
 			auto APIKEY = Request.get("X-API-KEY");
 			return APIKEY == MyHash_;
 		} catch (const Poco::Exception &E) {
-			logger().log(E);
+			Logger_.log(E);
 		}
 		return false;
 	}
@@ -718,6 +780,8 @@ namespace OpenWifi {
 		MicroServiceErrorHandler ErrorHandler(*this);
 		Poco::ErrorHandler::set(&ErrorHandler);
 
+        Args_ = args;
+
 		if (!HelpRequested_) {
 			SavePID();
 
@@ -733,11 +797,18 @@ namespace OpenWifi {
 				poco_information(logger, "Starting as a daemon.");
 			}
 
+#ifdef USE_MEDUSA_CLIENT
+            MedusaClient::instance()->SetSubSystems(SubSystems_);
+            MedusaClient::instance()->Start();
+			waitForTerminationRequest();
+            MedusaClient::instance()->Stop();
+#else
 			poco_information(logger, fmt::format("System ID set to {}", ID_));
 			StartSubSystemServers();
 			waitForTerminationRequest();
 			StopSubSystemServers();
 			logger.notice(fmt::format("Stopped {}...", DAEMON_APP_NAME));
+#endif
 		}
 
 		return Application::EXIT_OK;

src/framework/MicroService.h

@@ -55,9 +55,6 @@ namespace OpenWifi {
 #include "nlohmann/json.hpp"
 #include "ow_version.h"
 
-#define _OWDEBUG_ std::cout << __FILE__ << ":" << __LINE__ << std::endl;
-// #define _OWDEBUG_ Logger().debug(Poco::format("%s: %lu",__FILE__,__LINE__));
-
 namespace OpenWifi {
 
 	class MicroService : public Poco::Util::ServerApplication {
@@ -70,7 +67,6 @@ namespace OpenWifi {
 			  SubSystems_(std::move(Subsystems)), Logger_(Poco::Logger::get("FRAMEWORK")) {
 			instance_ = this;
 			RandomEngine_.seed(std::chrono::steady_clock::now().time_since_epoch().count());
-			// Logger_ = Poco::Logger::root().get("BASE-SVC");
 		}
 
 		inline static const char *ExtraConfigurationFilename = "/configuration_override.json";
@@ -92,7 +88,7 @@ namespace OpenWifi {
 		inline uint64_t DaemonBusTimer() const { return DAEMON_BUS_TIMER; };
 		[[nodiscard]] const std::string &AppName() { return DAEMON_APP_NAME; }
 		static inline uint64_t GetPID() { return Poco::Process::id(); };
-		[[nodiscard]] inline const std::string GetPublicAPIEndPoint() {
+		[[nodiscard]] inline std::string GetPublicAPIEndPoint() const {
 			return MyPublicEndPoint_ + "/api/v1";
 		};
 		[[nodiscard]] inline const std::string &GetUIURI() const { return UIURI_; };
@@ -107,7 +103,8 @@ namespace OpenWifi {
 		}
 		static MicroService &instance() { return *instance_; }
 
-		inline void Exit(int Reason);
+		inline void Exit(int Reason) { std::exit(Reason); }
+
 		void BusMessageReceived(const std::string &Key, const std::string &Payload);
 		Types::MicroServiceMetaVec GetServices(const std::string &Type);
 		Types::MicroServiceMetaVec GetServices();
@@ -115,7 +112,9 @@ namespace OpenWifi {
 		void Reload();
 		void LoadMyConfig();
 		void initialize(Poco::Util::Application &self) override;
-		void uninitialize() override;
+        void StartEverything(Poco::Util::Application &self);
+        void StopEverything(Poco::Util::Application &self);
+        void uninitialize() override;
 		void reinitialize(Poco::Util::Application &self) override;
 		void defineOptions(Poco::Util::OptionSet &options) override;
 		void handleHelp(const std::string &name, const std::string &value);
@@ -132,7 +131,7 @@ namespace OpenWifi {
 		void Reload(const std::string &Sub);
 		Types::StringVec GetSubSystems() const;
 		Types::StringPairVec GetLogLevels();
-		const Types::StringVec &GetLogLevelNames();
+		static const Types::StringVec &GetLogLevelNames();
 		uint64_t ConfigGetInt(const std::string &Key, uint64_t Default);
 		uint64_t ConfigGetInt(const std::string &Key);
 		uint64_t ConfigGetBool(const std::string &Key, bool Default);
@@ -166,12 +165,25 @@ namespace OpenWifi {
 								const std::string &FormatterPattern,
 								const std::string &root_env_var);
 		inline bool AllowExternalMicroServices() const { return AllowExternalMicroServices_; }
+        const ArgVec &Args() const { return Args_; }
+
+        inline void SetConfigContent(const std::string &Content) { ConfigContent_ = Content; }
+
+        inline std::optional<OpenWifi::Types::MicroServiceMeta> GetPrivateEndPointServiceKey( const std::string & ServicePrivateEndPoint ) {
+            std::lock_guard   G(InfraMutex_);
+            auto K = Services_.find(ServicePrivateEndPoint);
+            if(K==end(Services_)) {
+                return std::nullopt;
+            }
+            return K->second;
+        }
 
 	  private:
 		static MicroService *instance_;
 		bool HelpRequested_ = false;
 		std::string LogDir_;
 		std::string ConfigFileName_;
+        std::string ConfigContent_;
 		uint64_t ID_ = 1;
 		Poco::SharedPtr<Poco::Crypto::RSAKey> AppKey_;
 		bool DebugMode_ = false;
@@ -201,7 +213,7 @@ namespace OpenWifi {
 		Poco::JWT::Signer Signer_;
 		Poco::Logger &Logger_;
 		Poco::ThreadPool TimerPool_{"timer:pool", 2, 32};
-		std::unique_ptr<EventBusManager> EventBusManager_;
+        ArgVec Args_;
 	};
 
 	inline MicroService *MicroService::instance_ = nullptr;

src/framework/MicroServiceFuncs.cpp

@@ -133,4 +133,8 @@ namespace OpenWifi {
 		return MicroService::instance().Hash();
 	}
 
+    std::optional<OpenWifi::Types::MicroServiceMeta> MicroServicePrivateAccessKey(const std::string &servicePrivateEndPoint) {
+        return MicroService::instance().GetPrivateEndPointServiceKey(servicePrivateEndPoint);
+    }
+
 } // namespace OpenWifi

src/framework/MicroServiceFuncs.h

@@ -23,7 +23,9 @@ namespace OpenWifi {
 	std::string MicroServiceConfigGetString(const std::string &Key,
 											const std::string &DefaultValue);
 	std::string MicroServiceAccessKey();
-	bool MicroServiceConfigGetBool(const std::string &Key, bool DefaultValue);
+    std::optional<OpenWifi::Types::MicroServiceMeta> MicroServicePrivateAccessKey(const std::string &servicePrivateEndPoint);
+
+    bool MicroServiceConfigGetBool(const std::string &Key, bool DefaultValue);
 	std::uint64_t MicroServiceConfigGetInt(const std::string &Key, std::uint64_t DefaultValue);
 	std::string MicroServicePrivateEndPoint();
 	std::uint64_t MicroServiceID();

src/framework/StorageClass.h

@@ -47,6 +47,8 @@ namespace OpenWifi {
 
         }
 
+		Poco::Data::SessionPool &Pool() { return *Pool_; }
+
 	  private:
 		inline int Setup_SQLite();
 		inline int Setup_MySQL();

src/framework/UI_WebSocketClientServer.cpp

@@ -58,11 +58,9 @@ namespace OpenWifi {
 	void UI_WebSocketClientServer::run() {
 		Running_ = true;
 		while (Running_) {
-			Poco::Thread::trySleep(2000);
-
-			if (!Running_)
-				break;
-
+			if(!Poco::Thread::trySleep(2000)) {
+                break;
+            }
 			std::lock_guard G(LocalMutex_);
 			for (const auto i : ToBeRemoved_) {
 				// std::cout << "Erasing old WS UI connection..." << std::endl;

src/framework/orm.h

@@ -576,8 +576,8 @@ namespace ORM {
 		bool UpdateRecord(field_name_t FieldName, const T &Value, const RecordType &R) {
 			try {
 				assert(ValidFieldName(FieldName));
-
 				Poco::Data::Session Session = Pool_.get();
+                Session.begin();
 				Poco::Data::Statement Update(Session);
 
 				RecordTuple RT;
@@ -593,6 +593,7 @@ namespace ORM {
 				Update.execute();
 				if (Cache_)
 					Cache_->UpdateCache(R);
+                Session.commit();
 				return true;
 			} catch (const Poco::Exception &E) {
 				Logger_.log(E);
@@ -662,6 +663,7 @@ namespace ORM {
 				assert(ValidFieldName(FieldName));
 
 				Poco::Data::Session Session = Pool_.get();
+                Session.begin();
 				Poco::Data::Statement Delete(Session);
 
 				std::string St = "delete from " + TableName_ + " where " + FieldName + "=?";
@@ -671,6 +673,7 @@ namespace ORM {
 				Delete.execute();
 				if (Cache_)
 					Cache_->Delete(FieldName, Value);
+                Session.commit();
 				return true;
 			} catch (const Poco::Exception &E) {
 				Logger_.log(E);
@@ -682,11 +685,13 @@ namespace ORM {
 			try {
 				assert(!WhereClause.empty());
 				Poco::Data::Session Session = Pool_.get();
+                Session.begin();
 				Poco::Data::Statement Delete(Session);
 
 				std::string St = "delete from " + TableName_ + " where " + WhereClause;
 				Delete << St;
 				Delete.execute();
+                Session.commit();
 				return true;
 			} catch (const Poco::Exception &E) {
 				Logger_.log(E);

src/framework/ow_constants.h

@@ -565,6 +565,7 @@ namespace OpenWifi::RESTAPI::Protocol {
 
 	static const char *TRANSFER = "transfer";
 	static const char *CERTUPDATE = "certupdate";
+	static const char *POWERCYCLE = "powercycle";
 	static const char *RRM = "rrm";
 
 	static const char *REQUIREMENTS = "requirements";
@@ -687,6 +688,7 @@ namespace OpenWifi::uCentralProtocol {
 
 	static const char *TRANSFER = "transfer";
 	static const char *CERTUPDATE = "certupdate";
+	static const char *POWERCYCLE = "powercycle";
 	static const char *RRM = "rrm";
 	static const char *ACTIONS = "actions";
 
@@ -785,6 +787,7 @@ namespace OpenWifi::APCommands {
 		rrm,
 		certupdate,
 		transfer,
+		powercycle,
 		unknown
 	};
 
@@ -799,7 +802,7 @@ namespace OpenWifi::APCommands {
 		RESTAPI::Protocol::EVENTQUEUE,	 RESTAPI::Protocol::TELEMETRY,
 		RESTAPI::Protocol::PING,		 RESTAPI::Protocol::SCRIPT,
 		RESTAPI::Protocol::RRM,		 	 RESTAPI::Protocol::CERTUPDATE,
-		RESTAPI::Protocol::TRANSFER
+		RESTAPI::Protocol::TRANSFER,	 RESTAPI::Protocol::POWERCYCLE
 	};
 
 	inline const char *to_string(Commands Cmd) { return uCentralAPCommands[(uint8_t)Cmd]; }

src/framework/utils.h

@@ -316,5 +316,90 @@ namespace OpenWifi::Utils {
         uint32_t        Port;
     };
 
+	class CompressedString {
+	  public:
+		CompressedString() {
+			DecompressedSize_ = 0;
+		};
+
+		explicit CompressedString(const std::string &Data) : DecompressedSize_(Data.size()) {
+			CompressIt(Data);
+		}
+
+		CompressedString(const CompressedString &Data) {
+			this->DecompressedSize_ = Data.DecompressedSize_;
+			this->CompressedData_ = Data.CompressedData_;
+		}
+
+		CompressedString& operator=(const CompressedString& rhs) {
+			if (this != &rhs) {
+				this->DecompressedSize_ = rhs.DecompressedSize_;
+				this->CompressedData_ = rhs.CompressedData_;
+			}
+			return *this;
+		}
+
+		CompressedString& operator=(CompressedString&& rhs) {
+			if (this != &rhs) {
+				this->DecompressedSize_ = rhs.DecompressedSize_;
+				this->CompressedData_ = rhs.CompressedData_;
+			}
+			return *this;
+		}
+
+		~CompressedString() = default;
+
+		operator std::string() const {
+			return DecompressIt();
+		}
+
+		CompressedString &operator=(const std::string &Data) {
+			DecompressedSize_ = Data.size();
+			CompressIt(Data);
+			return *this;
+		}
+
+		auto CompressedSize() const { return CompressedData_.size(); }
+		auto DecompressedSize() const { return DecompressedSize_; }
+
+	  private:
+		std::string     CompressedData_;
+		std::size_t     DecompressedSize_;
+
+		inline void CompressIt(const std::string &Data) {
+			z_stream strm; // = {0};
+			CompressedData_.resize(Data.size());
+			strm.next_in = (Bytef *)Data.data();
+			strm.avail_in = Data.size();
+			strm.next_out = (Bytef *)CompressedData_.data();
+			strm.avail_out = Data.size();
+			strm.zalloc = Z_NULL;
+			strm.zfree = Z_NULL;
+			strm.opaque = Z_NULL;
+			deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY);
+			deflate(&strm, Z_FINISH);
+			deflateEnd(&strm);
+			CompressedData_.resize(strm.total_out);
+		}
+
+		[[nodiscard]] std::string DecompressIt() const {
+			std::string Result;
+			if(DecompressedSize_!=0) {
+				Result.resize(DecompressedSize_);
+				z_stream strm ; //= {0};
+				strm.next_in = (Bytef *)CompressedData_.data();
+				strm.avail_in = CompressedData_.size();
+				strm.next_out = (Bytef *)Result.data();
+				strm.avail_out = Result.size();
+				strm.zalloc = Z_NULL;
+				strm.zfree = Z_NULL;
+				strm.opaque = Z_NULL;
+				inflateInit2(&strm, 15 + 32);
+				inflate(&strm, Z_FINISH);
+				inflateEnd(&strm);
+			}
+			return Result;
+		}
+	};
 
 } // namespace OpenWifi::Utils

src/sdks/SDK_fms.cpp

@@ -36,15 +36,18 @@ namespace OpenWifi::SDK::FMS {
 			static const std::string EndPoint{"/api/v1/firmwares"};
 
 			OpenWifi::OpenAPIRequestGet API(uSERVICE_FIRMWARE, EndPoint,
-											{{"deviceType", device_type}}, 50000);
+											{{"deviceType", device_type},
+                                             {"offset","0"},
+                                             {"limit","1000"}}, 50000);
 
 			auto CallResponse = Poco::makeShared<Poco::JSON::Object>();
 			auto StatusCode = API.Do(CallResponse);
 			if (StatusCode == Poco::Net::HTTPResponse::HTTP_OK) {
 				Poco::JSON::Array::Ptr FirmwareArr = CallResponse->getArray("firmwares");
-				for (uint64_t i = 0; i < FirmwareArr->size(); i++) {
+                for(const auto &firmware:*FirmwareArr) {
+                    auto Object = firmware.extract<Poco::JSON::Object::Ptr>();
 					FMSObjects::Firmware F;
-					F.from_json(FirmwareArr->getObject(i));
+					F.from_json(Object);
 					FirmWares.emplace_back(F);
 				}
 				return true;

src/sdks/SDK_gw.cpp

@@ -79,7 +79,7 @@ namespace OpenWifi::SDK::GW {
 		}
 
 		bool Upgrade(RESTAPIHandler *client, const std::string &SerialNumber, uint64_t When,
-					 const std::string &ImageName) {
+					 const std::string &ImageName, std::string &status) {
 			Poco::JSON::Object Body;
 
 			Body.set(RESTAPI::Protocol::SERIALNUMBER, SerialNumber);
@@ -92,6 +92,7 @@ namespace OpenWifi::SDK::GW {
 			auto ResponseStatus =
 				API.Do(CallResponse, client ? client->UserInfo_.webtoken.access_token_ : "");
 			if (ResponseStatus == Poco::Net::HTTPResponse::HTTP_OK) {
+                status = CallResponse->get("status").toString();
 				return true;
 			}
 			return false;

src/sdks/SDK_gw.h

@@ -23,7 +23,7 @@ namespace OpenWifi::SDK::GW {
 		bool Configure(RESTAPIHandler *client, const std::string &Mac,
 					   Poco::JSON::Object::Ptr &Configuration, Poco::JSON::Object::Ptr &Response);
 		bool Upgrade(RESTAPIHandler *client, const std::string &Mac, uint64_t When,
-					 const std::string &ImageName);
+					 const std::string &ImageName, std::string &status);
 
 		bool SetVenue(RESTAPIHandler *client, const std::string &SerialNumber,
 					  const std::string &uuid);

src/storage/storage_inventory.cpp

@@ -44,7 +44,10 @@ namespace OpenWifi {
 		ORM::Field{"devClass", ORM::FieldType::FT_TEXT},
 		ORM::Field{"locale", ORM::FieldType::FT_TEXT},
 		ORM::Field{"realMacAddress", ORM::FieldType::FT_TEXT},
-		ORM::Field{"doNotAllowOverrides", ORM::FieldType::FT_BOOLEAN}};
+		ORM::Field{"doNotAllowOverrides", ORM::FieldType::FT_BOOLEAN},
+        ORM::Field{"imported", ORM::FieldType::FT_BIGINT},
+        ORM::Field{"connected", ORM::FieldType::FT_BIGINT},
+        ORM::Field{"platform", ORM::FieldType::FT_TEXT}};
 
 	static ORM::IndexVec InventoryDB_Indexes{
 		{std::string("inventory_name_index"),
@@ -60,6 +63,9 @@ namespace OpenWifi {
 			"alter table " + TableName_ + " add column realMacAddress text",
 			"alter table " + TableName_ + " add column devClass text",
 			"alter table " + TableName_ + " add column deviceRules text",
+            "alter table " + TableName_ + " add column platform text default 'AP'",
+            "alter table " + TableName_ + " add column imported bigint",
+            "alter table " + TableName_ + " add column connected bigint",
 			"alter table " + TableName_ + " add column doNotAllowOverrides boolean"};
 
 		for (const auto &i : Script) {
@@ -80,7 +86,8 @@ namespace OpenWifi {
 	bool InventoryDB::CreateFromConnection(const std::string &SerialNumberRaw,
 										   const std::string &ConnectionInfo,
 										   const std::string &DeviceType,
-										   const std::string &Locale) {
+										   const std::string &Locale,
+										   const bool isConnection) {
 
 		ProvObjects::InventoryTag ExistingDevice;
 		auto SerialNumber = Poco::toLower(SerialNumberRaw);
@@ -106,6 +113,8 @@ namespace OpenWifi {
 			StateDoc["date"] = Utils::Now();
 			NewDevice.state = to_string(StateDoc);
 			NewDevice.devClass = "any";
+            NewDevice.connected = Now;
+            NewDevice.imported = 0;
 			if (!IP.empty()) {
 				StorageService()->VenueDB().GetByIP(IP, NewDevice.venue);
 				if (NewDevice.venue.empty()) {
@@ -176,9 +185,32 @@ namespace OpenWifi {
 
 			if (modified) {
 				ExistingDevice.info.modified = Utils::Now();
+                ExistingDevice.connected = Utils::Now();
 				StorageService()->InventoryDB().UpdateRecord("id", ExistingDevice.info.id,
 															 ExistingDevice);
 			}
+
+			// Push entity and venue down to GW but only on connect (not ping)
+			if (isConnection && !ExistingDevice.venue.empty()) {
+				if (SDK::GW::Device::SetVenue(nullptr, ExistingDevice.serialNumber, ExistingDevice.venue)) {
+						Logger().information(Poco::format("%s: GW set venue property.",
+														  ExistingDevice.serialNumber));
+				} else {
+					Logger().information(Poco::format(
+						"%s: could not set GW venue property.", ExistingDevice.serialNumber));
+				}
+			}
+
+			if (isConnection && !ExistingDevice.entity.empty()) {
+				if (SDK::GW::Device::SetEntity(nullptr, ExistingDevice.serialNumber, ExistingDevice.entity)) {
+						Logger().information(Poco::format("%s: GW set entity property.",
+														  ExistingDevice.serialNumber));
+				} else {
+					Logger().information(Poco::format(
+						"%s: could not set GW entity property.", ExistingDevice.serialNumber));
+				}
+			}
+
 		}
 		return false;
 	}
@@ -232,6 +264,69 @@ namespace OpenWifi {
 		}
 		return true;
 	}
+
+    bool InventoryDB::GetDevicesForVenue(const std::string &venue_uuid, std::vector<std::string> &devices) {
+        try {
+            std::vector<ProvObjects::InventoryTag> device_list;
+            if(GetRecords(0, 1000, device_list, fmt::format(" venue='{}' ", venue_uuid))) {
+                for(auto &i:device_list) {
+                    devices.push_back(i.serialNumber);
+                }
+                return true;
+            }
+        } catch(const Poco::Exception &E) {
+            Logger().log(E);
+            return false;
+        } catch(const std::exception &E) {
+            Logger().error(fmt::format("std::exception: {}",E.what()));
+            return false;
+        } catch(...) {
+            Logger().error("Unknown exception");
+            return false;
+        }
+        return false;
+    }
+
+    bool InventoryDB::GetDevicesUUIDForVenue(const std::string &venue_uuid, std::vector<std::string> &devices) {
+        try {
+            std::vector<ProvObjects::InventoryTag> device_list;
+            if(GetRecords(0, 1000, device_list, fmt::format(" venue='{}' ", venue_uuid))) {
+                for(auto &i:device_list) {
+                    devices.push_back(i.info.id);
+                }
+                return true;
+            }
+        } catch(const Poco::Exception &E) {
+            Logger().log(E);
+            return false;
+        } catch(const std::exception &E) {
+            Logger().error(fmt::format("std::exception: {}",E.what()));
+            return false;
+        } catch(...) {
+            Logger().error("Unknown exception");
+            return false;
+        }
+        return false;
+    }
+
+    bool InventoryDB::GetDevicesForVenue(const std::string &venue_uuid, std::vector<ProvObjects::InventoryTag> &devices) {
+        try {
+            return GetRecords(0, 1000, devices, fmt::format(" venue='{}' ", venue_uuid));
+        } catch(const Poco::Exception &E) {
+            Logger().log(E);
+            return false;
+        } catch(const std::exception &E) {
+            Logger().error(fmt::format("std::exception: {}",E.what()));
+            return false;
+        } catch(...) {
+            Logger().error("Unknown exception");
+            return false;
+        }
+
+        return false;
+    }
+
+
 } // namespace OpenWifi
 
 template <>
@@ -263,6 +358,9 @@ void ORM::DB<OpenWifi::InventoryDBRecordType, OpenWifi::ProvObjects::InventoryTa
 	Out.locale = In.get<21>();
 	Out.realMacAddress = In.get<22>();
 	Out.doNotAllowOverrides = In.get<23>();
+    Out.imported = In.get<24>();
+    Out.connected = In.get<25>();
+    Out.platform = In.get<26>();
 }
 
 template <>
@@ -292,4 +390,7 @@ void ORM::DB<OpenWifi::InventoryDBRecordType, OpenWifi::ProvObjects::InventoryTa
 	Out.set<21>(In.locale);
 	Out.set<22>(In.realMacAddress);
 	Out.set<23>(In.doNotAllowOverrides);
+    Out.set<24>(In.imported);
+    Out.set<25>(In.connected);
+    Out.set<26>(In.platform);
 }

src/storage/storage_inventory.h

@@ -16,7 +16,7 @@ namespace OpenWifi {
 						std::string, std::string, std::string, std::string, std::string,
 						std::string, std::string, std::string, std::string, std::string,
 						std::string, std::string, std::string, std::string, std::string,
-						std::string, std::string, bool>
+						std::string, std::string, bool, uint64_t, uint64_t, std::string>
 		InventoryDBRecordType;
 
 	class InventoryDB : public ORM::DB<InventoryDBRecordType, ProvObjects::InventoryTag> {
@@ -25,7 +25,8 @@ namespace OpenWifi {
 		virtual ~InventoryDB(){};
 		bool CreateFromConnection(const std::string &SerialNumber,
 								  const std::string &ConnectionInfo, const std::string &DeviceType,
-								  const std::string &Locale);
+								  const std::string &Locale,
+								  const bool isConnection);
 
 		void InitializeSerialCache();
 		bool GetRRMDeviceList(Types::UUIDvec_t &DeviceList);
@@ -38,6 +39,10 @@ namespace OpenWifi {
 
 		bool Upgrade(uint32_t from, uint32_t &to) override;
 
+        bool GetDevicesForVenue(const std::string &uuid, std::vector<std::string> &devices);
+        bool GetDevicesUUIDForVenue(const std::string &uuid, std::vector<std::string> &devices);
+        bool GetDevicesForVenue(const std::string &uuid, std::vector<ProvObjects::InventoryTag> &devices);
+
 	  private:
 		bool EvaluateDeviceRules(const ProvObjects::InventoryTag &T,
 								 ProvObjects::DeviceRules &Rules);