From cdbe71ccce78daba9c1785625e593ff8c5bd4682 Mon Sep 17 00:00:00 2001 From: stephb9959 Date: Wed, 5 Jan 2022 22:50:58 -0800 Subject: [PATCH] Updating the framework. --- CMakeLists.txt | 2 +- build | 2 +- cmake/CppKafkaConfig.cmake | 57 ++++ cmake/FindRdKafka.cmake | 73 +++++ ...ESTAPI_Routers.cpp => RESTAPI_routers.cpp} | 0 src/RESTObjects/RESTAPI_SecurityObjects.cpp | 5 + src/RESTObjects/RESTAPI_SecurityObjects.h | 2 +- src/Simulation.cpp | 8 +- src/StorageService.cpp | 8 +- src/framework/ConfigurationValidator.cpp | 8 +- src/framework/MicroService.h | 295 +++++++++++++----- src/framework/StorageClass.h | 10 +- src/framework/orm.h | 14 +- 13 files changed, 383 insertions(+), 101 deletions(-) create mode 100644 cmake/CppKafkaConfig.cmake create mode 100644 cmake/FindRdKafka.cmake rename src/RESTAPI/{RESTAPI_Routers.cpp => RESTAPI_routers.cpp} (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 45efe1c..df4df81 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,7 +71,7 @@ add_executable( owls src/framework/uCentral_Protocol.h src/framework/ConfigurationValidator.cpp src/framework/ConfigurationValidator.h - src/RESTAPI/RESTAPI_Routers.cpp + src/RESTAPI/RESTAPI_routers.cpp src/Daemon.cpp src/Daemon.h src/Dashboard.cpp src/Dashboard.h src/uCentralClient.cpp diff --git a/build b/build index 19c7bdb..b393560 100644 --- a/build +++ b/build @@ -1 +1 @@ -16 \ No newline at end of file +23 \ No newline at end of file diff --git a/cmake/CppKafkaConfig.cmake b/cmake/CppKafkaConfig.cmake new file mode 100644 index 0000000..656d2d6 --- /dev/null +++ b/cmake/CppKafkaConfig.cmake @@ -0,0 +1,57 @@ + +####### Expanded from @PACKAGE_INIT@ by configure_package_config_file() ####### +####### Any changes to this file will be overwritten by the next CMake run #### +####### The input file was config.cmake.in ######## + +get_filename_component(PACKAGE_PREFIX_DIR "${CMAKE_CURRENT_LIST_DIR}/../../../" ABSOLUTE) + +macro(set_and_check _var _file) + set(${_var} "${_file}") + if(NOT EXISTS "${_file}") + message(FATAL_ERROR "File or directory ${_file} referenced by variable ${_var} does not exist !") + endif() +endmacro() + +macro(check_required_components _NAME) + foreach(comp ${${_NAME}_FIND_COMPONENTS}) + if(NOT ${_NAME}_${comp}_FOUND) + if(${_NAME}_FIND_REQUIRED_${comp}) + set(${_NAME}_FOUND FALSE) + endif() + endif() + endforeach() +endmacro() + +#################################################################################### + +include(CMakeFindDependencyMacro) + +# Add FindRdKafka.cmake +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_LIST_DIR}") + +set(RDKAFKA_MIN_VERSION_HEX "0x00090400") + +# Find boost optional +find_dependency(Boost REQUIRED) + +# Try to find the RdKafka configuration file if present. +# This will search default system locations as well as RdKafka_ROOT and RdKafka_DIR paths if specified. +find_package(RdKafka QUIET CONFIG) +set(RDKAFKA_TARGET_IMPORTS ${RdKafka_FOUND}) +if (NOT RdKafka_FOUND) + find_dependency(RdKafka REQUIRED MODULE) +endif() + +include("${CMAKE_CURRENT_LIST_DIR}/CppKafkaTargets.cmake") + +# Export 'CppKafka_ROOT' +set_and_check(CppKafka_ROOT "${PACKAGE_PREFIX_DIR}") + +# Export 'CppKafka_INSTALL_INCLUDE_DIR' +set_and_check(CppKafka_INSTALL_INCLUDE_DIR "${PACKAGE_PREFIX_DIR}/include") + +# Export 'CppKafka_INSTALL_LIB_DIR' +set_and_check(CppKafka_INSTALL_LIB_DIR "${PACKAGE_PREFIX_DIR}/lib") + +# Validate installed components +check_required_components("CppKafka") diff --git a/cmake/FindRdKafka.cmake b/cmake/FindRdKafka.cmake new file mode 100644 index 0000000..d402fe1 --- /dev/null +++ b/cmake/FindRdKafka.cmake @@ -0,0 +1,73 @@ +# This find module helps find the RdKafka module. It exports the following variables: +# - RdKafka_INCLUDE_DIR : The directory where rdkafka.h is located. +# - RdKafka_LIBNAME : The name of the library, i.e. librdkafka.a, librdkafka.so, etc. +# - RdKafka_LIBRARY_PATH : The full library path i.e. /${RdKafka_LIBNAME} +# - RdKafka::rdkafka : Imported library containing all above properties set. + +if (CPPKAFKA_RDKAFKA_STATIC_LIB) + set(RDKAFKA_PREFIX ${CMAKE_STATIC_LIBRARY_PREFIX}) + set(RDKAFKA_SUFFIX ${CMAKE_STATIC_LIBRARY_SUFFIX}) + set(RDKAFKA_LIBRARY_TYPE STATIC) +else() + set(RDKAFKA_PREFIX ${CMAKE_SHARED_LIBRARY_PREFIX}) + set(RDKAFKA_SUFFIX ${CMAKE_SHARED_LIBRARY_SUFFIX}) + set(RDKAFKA_LIBRARY_TYPE SHARED) +endif() + +set(RdKafka_LIBNAME ${RDKAFKA_PREFIX}rdkafka${RDKAFKA_SUFFIX}) + +find_path(RdKafka_INCLUDE_DIR + NAMES librdkafka/rdkafka.h + HINTS ${RdKafka_ROOT}/include +) + +find_library(RdKafka_LIBRARY_PATH + NAMES ${RdKafka_LIBNAME} rdkafka + HINTS ${RdKafka_ROOT}/lib ${RdKafka_ROOT}/lib64 +) + +# Check lib paths +if (CPPKAFKA_CMAKE_VERBOSE) + get_property(FIND_LIBRARY_32 GLOBAL PROPERTY FIND_LIBRARY_USE_LIB32_PATHS) + get_property(FIND_LIBRARY_64 GLOBAL PROPERTY FIND_LIBRARY_USE_LIB64_PATHS) + message(STATUS "RDKAFKA search 32-bit library paths: ${FIND_LIBRARY_32}") + message(STATUS "RDKAFKA search 64-bit library paths: ${FIND_LIBRARY_64}") + message(STATUS "RdKafka_ROOT = ${RdKafka_ROOT}") + message(STATUS "RdKafka_INCLUDE_DIR = ${RdKafka_INCLUDE_DIR}") + message(STATUS "RdKafka_LIBNAME = ${RdKafka_LIBNAME}") + message(STATUS "RdKafka_LIBRARY_PATH = ${RdKafka_LIBRARY_PATH}") +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(RdKafka DEFAULT_MSG + RdKafka_LIBNAME + RdKafka_LIBRARY_PATH + RdKafka_INCLUDE_DIR +) + +set(CONTENTS "#include \n #if RD_KAFKA_VERSION >= ${RDKAFKA_MIN_VERSION_HEX}\n int main() { }\n #endif") +set(FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/rdkafka_version_test.cpp) +file(WRITE ${FILE_NAME} ${CONTENTS}) + +try_compile(RdKafka_FOUND ${CMAKE_CURRENT_BINARY_DIR} + SOURCES ${FILE_NAME} + CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${RdKafka_INCLUDE_DIR}") + +if (RdKafka_FOUND) + add_library(RdKafka::rdkafka ${RDKAFKA_LIBRARY_TYPE} IMPORTED GLOBAL) + set(RDKAFKA_DEPENDENCIES pthread) +# set(RDKAFKA_DEPENDENCIES pthread rt crypto dl z) + set_target_properties(RdKafka::rdkafka PROPERTIES + IMPORTED_NAME RdKafka + IMPORTED_LOCATION "${RdKafka_LIBRARY_PATH}" + INTERFACE_INCLUDE_DIRECTORIES "${RdKafka_INCLUDE_DIR}" + INTERFACE_LINK_LIBRARIES "${RDKAFKA_DEPENDENCIES}") + message(STATUS "Found valid rdkafka version") + mark_as_advanced( + RDKAFKA_LIBRARY + RdKafka_INCLUDE_DIR + RdKafka_LIBRARY_PATH + ) +else() + message(FATAL_ERROR "Failed to find valid rdkafka version") +endif() diff --git a/src/RESTAPI/RESTAPI_Routers.cpp b/src/RESTAPI/RESTAPI_routers.cpp similarity index 100% rename from src/RESTAPI/RESTAPI_Routers.cpp rename to src/RESTAPI/RESTAPI_routers.cpp diff --git a/src/RESTObjects/RESTAPI_SecurityObjects.cpp b/src/RESTObjects/RESTAPI_SecurityObjects.cpp index 613c22d..58403b2 100644 --- a/src/RESTObjects/RESTAPI_SecurityObjects.cpp +++ b/src/RESTObjects/RESTAPI_SecurityObjects.cpp @@ -54,6 +54,8 @@ namespace OpenWifi::SecurityObjects { return ADMIN; else if (!Poco::icompare(U,"subscriber")) return SUBSCRIBER; + else if (!Poco::icompare(U,"partner")) + return PARTNER; else if (!Poco::icompare(U,"csr")) return CSR; else if (!Poco::icompare(U, "system")) @@ -72,6 +74,7 @@ namespace OpenWifi::SecurityObjects { case ROOT: return "root"; case ADMIN: return "admin"; case SUBSCRIBER: return "subscriber"; + case PARTNER: return "partner"; case CSR: return "csr"; case SYSTEM: return "system"; case INSTALLER: return "installer"; @@ -265,6 +268,8 @@ namespace OpenWifi::SecurityObjects { field_from_json(Obj,"currentLoginURI",currentLoginURI); field_from_json(Obj,"locale",locale); field_from_json(Obj,"notes",notes); + field_from_json(Obj,"location", location); + field_from_json(Obj,"owner", owner); field_from_json(Obj,"userRole",userRole, UserTypeFromString); field_from_json(Obj,"securityPolicy",securityPolicy); field_from_json(Obj,"userTypeProprietaryInfo",userTypeProprietaryInfo); diff --git a/src/RESTObjects/RESTAPI_SecurityObjects.h b/src/RESTObjects/RESTAPI_SecurityObjects.h index 48d0628..fd01baa 100644 --- a/src/RESTObjects/RESTAPI_SecurityObjects.h +++ b/src/RESTObjects/RESTAPI_SecurityObjects.h @@ -43,7 +43,7 @@ namespace OpenWifi { }; enum USER_ROLE { - UNKNOWN, ROOT, ADMIN, SUBSCRIBER, CSR, SYSTEM, INSTALLER, NOC, ACCOUNTING + UNKNOWN, ROOT, ADMIN, SUBSCRIBER, CSR, SYSTEM, INSTALLER, NOC, ACCOUNTING, PARTNER }; USER_ROLE UserTypeFromString(const std::string &U); diff --git a/src/Simulation.cpp b/src/Simulation.cpp index 78c1831..e2f6023 100644 --- a/src/Simulation.cpp +++ b/src/Simulation.cpp @@ -55,14 +55,14 @@ namespace OpenWifi { } void SimulationCoordinator::StartSimulators() { - Logger_.notice("Starting simulation threads..."); + Logger().notice("Starting simulation threads..."); for(const auto &i:SimThreads_) { i->Thread.start(i->Sim); } } void SimulationCoordinator::CancelSimulators() { - Logger_.notice("Cancel simulation threads..."); + Logger().notice("Cancel simulation threads..."); SimStats()->EndSim(); for(const auto &i:SimThreads_) { i->Sim.stop(); @@ -72,7 +72,7 @@ namespace OpenWifi { } void SimulationCoordinator::StopSimulators() { - Logger_.notice("Stopping simulation threads..."); + Logger().notice("Stopping simulation threads..."); SimStats()->EndSim(); for(const auto &i:SimThreads_) { i->Sim.stop(); @@ -122,7 +122,7 @@ namespace OpenWifi { for(auto i=0;ClientCount;i++) { auto Clients = std::min(ClientCount,NumClientsPerThread); - auto NewSimThread = std::make_unique(i,CurrentSim_.macPrefix,Clients, Logger_); + auto NewSimThread = std::make_unique(i,CurrentSim_.macPrefix,Clients, Logger()); NewSimThread->Sim.Initialize(); SimThreads_.push_back(std::move(NewSimThread)); ClientCount -= Clients; diff --git a/src/StorageService.cpp b/src/StorageService.cpp index d240478..9f5c46d 100644 --- a/src/StorageService.cpp +++ b/src/StorageService.cpp @@ -12,13 +12,13 @@ namespace OpenWifi { int Storage::Start() { std::lock_guard Guard(Mutex_); - Logger_.notice("Starting."); + Logger().notice("Starting."); StorageClass::Start(); - SimulationDB_ = std::make_unique(dbType_,*Pool_, Logger_); + SimulationDB_ = std::make_unique(dbType_,*Pool_, Logger()); SimulationDB_->Create(); - SimulationResultsDB_ = std::make_unique(dbType_,*Pool_, Logger_); + SimulationResultsDB_ = std::make_unique(dbType_,*Pool_, Logger()); SimulationResultsDB_->Create(); return 0; @@ -26,7 +26,7 @@ namespace OpenWifi { void Storage::Stop() { std::lock_guard Guard(Mutex_); - Logger_.notice("Stopping."); + Logger().notice("Stopping."); StorageClass::Stop(); } diff --git a/src/framework/ConfigurationValidator.cpp b/src/framework/ConfigurationValidator.cpp index b395909..a7f1d7e 100644 --- a/src/framework/ConfigurationValidator.cpp +++ b/src/framework/ConfigurationValidator.cpp @@ -2327,7 +2327,7 @@ namespace OpenWifi { if(Utils::wgets(GitUCentralJSONSchemaFile, GitSchema)) { auto schema = json::parse(GitSchema); Validator_->set_root_schema(schema); - Logger_.information("Using uCentral validation schema from GIT."); + Logger().information("Using uCentral validation schema from GIT."); } else { std::string FileName{ MicroService::instance().DataDir() + "/ucentral.schema.json" }; std::ifstream input(FileName); @@ -2336,11 +2336,11 @@ namespace OpenWifi { input.close(); auto schema = json::parse(schema_file.str()); Validator_->set_root_schema(schema); - Logger_.information("Using uCentral validation schema from local file."); + Logger().information("Using uCentral validation schema from local file."); } } catch (const Poco::Exception &E) { Validator_->set_root_schema(DefaultUCentralSchema); - Logger_.information("Using uCentral validation from built-in default."); + Logger().information("Using uCentral validation from built-in default."); } Initialized_ = Working_ = true; } @@ -2471,7 +2471,7 @@ namespace OpenWifi { } void ConfigurationValidator::reinitialize(Poco::Util::Application &self) { - Logger_.information("Reinitializing."); + Logger().information("Reinitializing."); Working_ = Initialized_ = false; Init(); } diff --git a/src/framework/MicroService.h b/src/framework/MicroService.h index f4f7ca7..c0aaff2 100644 --- a/src/framework/MicroService.h +++ b/src/framework/MicroService.h @@ -61,6 +61,13 @@ using namespace std::chrono_literals; #include "Poco/JSON/Object.h" #include "Poco/JSON/Parser.h" #include "Poco/StringTokenizer.h" +#include "Poco/AsyncChannel.h" +#include "Poco/ConsoleChannel.h" +#include "Poco/AutoPtr.h" +#include "Poco/FormattingChannel.h" +#include "Poco/PatternFormatter.h" +#include "Poco/FileChannel.h" +#include "Poco/SimpleFileChannel.h" #include "cppkafka/cppkafka.h" @@ -74,6 +81,8 @@ using namespace std::chrono_literals; #include "ow_version.h" +#define _OWDEBUG_ std::cout<< __FILE__ <<":" << __LINE__ << std::endl; + namespace OpenWifi { enum UNAUTHORIZED_REASON { @@ -87,7 +96,9 @@ namespace OpenWifi { ACCESS_DENIED, INVALID_TOKEN, EXPIRED_TOKEN, - RATE_LIMIT_EXCEEDED + RATE_LIMIT_EXCEEDED, + BAD_MFA_TRANSACTION, + MFA_FAILURE }; class AppServiceRegistry { @@ -95,7 +106,7 @@ namespace OpenWifi { inline AppServiceRegistry(); static AppServiceRegistry & instance() { - static AppServiceRegistry *instance_= new AppServiceRegistry; + static auto instance_= new AppServiceRegistry; return *instance_; } @@ -1311,17 +1322,13 @@ namespace OpenWifi { class SubSystemServer : public Poco::Util::Application::Subsystem { public: SubSystemServer(std::string Name, const std::string &LoggingPrefix, - std::string SubSystemConfigPrefix) - : Name_(std::move(Name)), Logger_(Poco::Logger::get(LoggingPrefix)), - SubSystemConfigPrefix_(std::move(SubSystemConfigPrefix)) { - Logger_.setLevel(Poco::Message::PRIO_NOTICE); - } + std::string SubSystemConfigPrefix); inline void initialize(Poco::Util::Application &self) override; inline void uninitialize() override { } inline void reinitialize(Poco::Util::Application &self) override { - Logger_.information("Reloading of this subsystem is not supported."); + Logger().information("Reloading of this subsystem is not supported."); } inline void defineOptions(Poco::Util::OptionSet &options) override { } @@ -1330,18 +1337,29 @@ namespace OpenWifi { inline const PropertiesFileServerEntry & Host(uint64_t index) { return ConfigServersList_[index]; }; inline uint64_t HostSize() const { return ConfigServersList_.size(); } - inline Poco::Logger &Logger() { return Logger_; }; - inline void SetLoggingLevel(Poco::Message::Priority NewPriority) { Logger_.setLevel(NewPriority); } - inline int GetLoggingLevel() { return Logger_.getLevel(); } + inline Poco::Logger &Logger() { if(Log_) + return Log_->L; + return Poco::Logger::get("tmp"); + }; + inline void SetLoggingLevel(Poco::Message::Priority NewPriority) { Logger().setLevel(NewPriority); } + inline int GetLoggingLevel() { return Logger().getLevel(); } virtual int Start() = 0; virtual void Stop() = 0; + struct LoggerWrapper { + Poco::Logger &L; + inline LoggerWrapper(Poco::Logger &Logger) : L(Logger) {} + }; + protected: std::recursive_mutex Mutex_; - Poco::Logger &Logger_; + std::vector ConfigServersList_; + private: + std::unique_ptr Log_; + // Poco::Logger &Logger_; std::string Name_; - std::vector ConfigServersList_; + std::string LoggerPrefix_; std::string SubSystemConfigPrefix_; }; @@ -1456,7 +1474,7 @@ namespace OpenWifi { }; static RESTAPI_RateLimiter *instance() { - static RESTAPI_RateLimiter * instance_ = new RESTAPI_RateLimiter; + static auto instance_ = new RESTAPI_RateLimiter; return instance_; } @@ -1476,7 +1494,7 @@ namespace OpenWifi { E->Count++; Cache_.update(H,E); if(E->Count > MaxCalls) { - Logger_.warning(Poco::format("RATE-LIMIT-EXCEEDED: from '%s'", R.clientAddress().toString())); + Logger().warning(Poco::format("RATE-LIMIT-EXCEEDED: from '%s'", R.clientAddress().toString())); return true; } return false; @@ -1508,7 +1526,8 @@ namespace OpenWifi { public: struct QueryBlock { uint64_t StartDate = 0 , EndDate = 0 , Offset = 0 , Limit = 0, LogType = 0 ; - std::string SerialNumber, Filter, Select; + std::string SerialNumber, Filter; + std::vector Select; bool Lifetime=false, LastOnly=false, Newest=false, CountOnly=false, AdditionalInfo=false; }; typedef std::map BindingMap; @@ -1562,7 +1581,7 @@ namespace OpenWifi { if (AlwaysAuthorize_ && !IsAuthorized(Expired, SubOnlyService_)) { if(Expired) return UnAuthorized(RESTAPI::Errors::ExpiredToken, EXPIRED_TOKEN); - return UnAuthorized(RESTAPI::Errors::InvalidCredentials, ACCESS_DENIED); + return UnAuthorized(RESTAPI::Errors::InvalidCredentials, INVALID_TOKEN); } std::string Reason; @@ -1572,24 +1591,24 @@ namespace OpenWifi { ParseParameters(); if (Request->getMethod() == Poco::Net::HTTPRequest::HTTP_GET) - DoGet(); + return DoGet(); else if (Request->getMethod() == Poco::Net::HTTPRequest::HTTP_POST) - DoPost(); + return DoPost(); else if (Request->getMethod() == Poco::Net::HTTPRequest::HTTP_DELETE) - DoDelete(); + return DoDelete(); else if (Request->getMethod() == Poco::Net::HTTPRequest::HTTP_PUT) - DoPut(); + return DoPut(); else - BadRequest(RESTAPI::Errors::UnsupportedHTTPMethod); + return BadRequest(RESTAPI::Errors::UnsupportedHTTPMethod); return; } catch (const Poco::Exception &E) { Logger_.log(E); - BadRequest(RESTAPI::Errors::InternalError); + return BadRequest(RESTAPI::Errors::InternalError); } } inline bool NeedAdditionalInfo() const { return QB_.AdditionalInfo; } - inline const std::string & SelectedRecords() const { return QB_.Select; } + inline const std::vector & SelectedRecords() const { return QB_.Select; } inline const Poco::JSON::Object::Ptr & ParseStream() { return IncomingParser_.parse(Request->stream()).extract(); @@ -1827,6 +1846,22 @@ namespace OpenWifi { } } + inline void SendCompressedTarFile(const std::string & FileName, const std::string & Content) { + Response->set("Content-Type","application/gzip"); + Response->set("Content-Disposition", "attachment; filename=" + FileName ); + Response->set("Content-Transfer-Encoding","binary"); + Response->set("Accept-Ranges", "bytes"); + Response->set("Cache-Control", "private"); + Response->set("Pragma", "private"); + Response->set("Expires", "Mon, 26 Jul 2027 05:00:00 GMT"); + Response->setStatus(Poco::Net::HTTPResponse::HTTP_OK); + AddCORS(); + Response->setContentLength(Content.size()); + Response->setChunkedTransferEncoding(true); + std::ostream& OutputStream = Response->send(); + OutputStream << Content; + } + inline void SendFile(Poco::File & File, const std::string & UUID) { Response->set("Content-Type","application/octet-stream"); Response->set("Content-Disposition", "attachment; filename=" + UUID ); @@ -1865,11 +1900,30 @@ namespace OpenWifi { Response->set("Cache-Control", "private"); Response->set("Pragma", "private"); Response->set("Expires", "Mon, 26 Jul 2027 05:00:00 GMT"); + Response->setContentLength(TempAvatar.getSize()); AddCORS(); Response->sendFile(TempAvatar.path(),MT.ContentType); } - inline void SendHTMLFileBack(Poco::File & File, + inline void SendFileContent(const std::string &Content, const std::string &Type, const std::string & Name) { + auto MT = Utils::FindMediaType(Name); + if(MT.Encoding==Utils::BINARY) { + Response->set("Content-Transfer-Encoding","binary"); + Response->set("Accept-Ranges", "bytes"); + } + Response->set("Content-Disposition", "attachment; filename=" + Name ); + Response->set("Accept-Ranges", "bytes"); + Response->set("Cache-Control", "private"); + Response->set("Pragma", "private"); + Response->set("Expires", "Mon, 26 Jul 2027 05:00:00 GMT"); + Response->setContentLength(Content.size()); + Response->setContentType(Type ); + AddCORS(); + auto & OutputStream = Response->send(); + OutputStream << Content ; + } + + inline void SendHTMLFileBack(Poco::File & File, const Types::StringPairVec & FormVars) { Response->set("Pragma", "private"); Response->set("Expires", "Mon, 26 Jul 2027 05:00:00 GMT"); @@ -1929,7 +1983,6 @@ namespace OpenWifi { QB_.Offset = GetParameter(RESTAPI::Protocol::OFFSET, 0); QB_.Limit = GetParameter(RESTAPI::Protocol::LIMIT, 100); QB_.Filter = GetParameter(RESTAPI::Protocol::FILTER, ""); - QB_.Select = GetParameter(RESTAPI::Protocol::SELECT, ""); QB_.Lifetime = GetBoolParameter(RESTAPI::Protocol::LIFETIME,false); QB_.LogType = GetParameter(RESTAPI::Protocol::LOGTYPE,0); QB_.LastOnly = GetBoolParameter(RESTAPI::Protocol::LASTONLY,false); @@ -1937,6 +1990,11 @@ namespace OpenWifi { QB_.CountOnly = GetBoolParameter(RESTAPI::Protocol::COUNTONLY,false); QB_.AdditionalInfo = GetBoolParameter(RESTAPI::Protocol::WITHEXTENDEDINFO,false); + auto RawSelect = GetParameter(RESTAPI::Protocol::SELECT, ""); + + auto Entries = Poco::StringTokenizer(RawSelect,","); + for(const auto &i:Entries) + QB_.Select.emplace_back(i); if(QB_.Offset<1) QB_.Offset=0; return true; @@ -2057,7 +2115,7 @@ namespace OpenWifi { EndPoint_(EndPoint), QueryData_(QueryData), msTimeout_(msTimeout) {}; - inline Poco::Net::HTTPServerResponse::HTTPStatus Do(Poco::JSON::Object::Ptr &ResponseObject); + inline Poco::Net::HTTPServerResponse::HTTPStatus Do(Poco::JSON::Object::Ptr &ResponseObject, const std::string & BearerToken = ""); private: std::string Type_; std::string EndPoint_; @@ -2078,7 +2136,7 @@ namespace OpenWifi { msTimeout_(msTimeout), Body_(Body){}; - inline Poco::Net::HTTPServerResponse::HTTPStatus Do(Poco::JSON::Object::Ptr &ResponseObject); + inline Poco::Net::HTTPServerResponse::HTTPStatus Do(Poco::JSON::Object::Ptr &ResponseObject, const std::string & BearerToken = ""); private: std::string Type_; @@ -2100,7 +2158,7 @@ namespace OpenWifi { QueryData_(QueryData), msTimeout_(msTimeout), Body_(Body){}; - inline Poco::Net::HTTPServerResponse::HTTPStatus Do(Poco::JSON::Object::Ptr &ResponseObject); + inline Poco::Net::HTTPServerResponse::HTTPStatus Do(Poco::JSON::Object::Ptr &ResponseObject, const std::string & BearerToken = ""); private: std::string Type_; std::string EndPoint_; @@ -2167,7 +2225,7 @@ namespace OpenWifi { inline void initialize(Poco::Util::Application & self) override; static KafkaManager *instance() { - static KafkaManager * instance_ = new KafkaManager; + static auto instance_ = new KafkaManager; return instance_; } @@ -2248,10 +2306,10 @@ namespace OpenWifi { KafkaConsumer ConsumerThr_; inline void PartitionAssignment(const cppkafka::TopicPartitionList& partitions) { - Logger_.information(Poco::format("Partition assigned: %Lu...",(uint64_t )partitions.front().get_partition())); + Logger().information(Poco::format("Partition assigned: %Lu...",(uint64_t )partitions.front().get_partition())); } inline void PartitionRevocation(const cppkafka::TopicPartitionList& partitions) { - Logger_.information(Poco::format("Partition revocation: %Lu...",(uint64_t )partitions.front().get_partition())); + Logger().information(Poco::format("Partition revocation: %Lu...",(uint64_t )partitions.front().get_partition())); } KafkaManager() noexcept: @@ -2270,7 +2328,7 @@ namespace OpenWifi { } static AuthClient *instance() { - static AuthClient * instance_ = new AuthClient; + static auto instance_ = new AuthClient; return instance_; } @@ -2424,12 +2482,12 @@ namespace OpenWifi { class RESTAPI_ExtServer : public SubSystemServer { public: static RESTAPI_ExtServer *instance() { - static RESTAPI_ExtServer *instance_ = new RESTAPI_ExtServer; + static auto instance_ = new RESTAPI_ExtServer; return instance_; } int Start() override; inline void Stop() override { - Logger_.information("Stopping "); + Logger().information("Stopping "); for( const auto & svr : RESTServers_ ) svr->stop(); Pool_.joinAll(); @@ -2440,7 +2498,7 @@ namespace OpenWifi { inline Poco::Net::HTTPRequestHandler *CallServer(const char *Path, uint64_t Id) { RESTAPIHandler::BindingMap Bindings; - return RESTAPI_ExtRouter(Path, Bindings, Logger_, Server_, Id); + return RESTAPI_ExtRouter(Path, Bindings, Logger(), Server_, Id); } private: @@ -2478,18 +2536,17 @@ namespace OpenWifi { }; inline int RESTAPI_ExtServer::Start() { - Logger_.information("Starting."); Server_.InitLogging(); for(const auto & Svr: ConfigServersList_) { - Logger_.information(Poco::format("Starting: %s:%s Keyfile:%s CertFile: %s", Svr.Address(), std::to_string(Svr.Port()), + Logger().information(Poco::format("Starting: %s:%s Keyfile:%s CertFile: %s", Svr.Address(), std::to_string(Svr.Port()), Svr.KeyFile(),Svr.CertFile())); - auto Sock{Svr.CreateSecureSocket(Logger_)}; + auto Sock{Svr.CreateSecureSocket(Logger())}; - Svr.LogCert(Logger_); + Svr.LogCert(Logger()); if(!Svr.RootCA().empty()) - Svr.LogCas(Logger_); + Svr.LogCas(Logger()); Poco::Net::HTTPServerParams::Ptr Params = new Poco::Net::HTTPServerParams; Params->setMaxThreads(50); @@ -2508,13 +2565,13 @@ namespace OpenWifi { public: static RESTAPI_IntServer *instance() { - static RESTAPI_IntServer *instance_ = new RESTAPI_IntServer; + static auto instance_ = new RESTAPI_IntServer; return instance_; } inline int Start() override; inline void Stop() override { - Logger_.information("Stopping "); + Logger().information("Stopping "); for( const auto & svr : RESTServers_ ) svr->stop(); Pool_.stopAll(); @@ -2524,7 +2581,7 @@ namespace OpenWifi { inline Poco::Net::HTTPRequestHandler *CallServer(const char *Path, uint64_t Id) { RESTAPIHandler::BindingMap Bindings; - return RESTAPI_IntRouter(Path, Bindings, Logger_, Server_, Id); + return RESTAPI_IntRouter(Path, Bindings, Logger(), Server_, Id); } private: std::vector> RESTServers_; @@ -2556,18 +2613,18 @@ namespace OpenWifi { }; inline int RESTAPI_IntServer::Start() { - Logger_.information("Starting."); + Logger().information("Starting."); Server_.InitLogging(); for(const auto & Svr: ConfigServersList_) { - Logger_.information(Poco::format("Starting: %s:%s Keyfile:%s CertFile: %s", Svr.Address(), std::to_string(Svr.Port()), + Logger().information(Poco::format("Starting: %s:%s Keyfile:%s CertFile: %s", Svr.Address(), std::to_string(Svr.Port()), Svr.KeyFile(),Svr.CertFile())); - auto Sock{Svr.CreateSecureSocket(Logger_)}; + auto Sock{Svr.CreateSecureSocket(Logger())}; - Svr.LogCert(Logger_); + Svr.LogCert(Logger()); if(!Svr.RootCA().empty()) - Svr.LogCas(Logger_); + Svr.LogCas(Logger()); auto Params = new Poco::Net::HTTPServerParams; Params->setMaxThreads(50); Params->setMaxQueued(200); @@ -2640,6 +2697,16 @@ namespace OpenWifi { return ((RandomEngine_() % (max-min)) + min); } + inline Poco::Logger & GetLogger(const std::string &Name) { + static auto initilized = false; + + if(!initilized) { + initilized = true; + InitializeLoggingSystem(); + } + return Poco::Logger::get(Name); + } + inline void Exit(int Reason); inline void BusMessageReceived(const std::string &Key, const std::string & Message); inline MicroServiceMetaVec GetServices(const std::string & Type); @@ -2682,6 +2749,7 @@ namespace OpenWifi { inline static void SavePID(); inline int main(const ArgVec &args) override; static MicroService & instance() { return *instance_; } + inline void InitializeLoggingSystem(); private: static MicroService * instance_; @@ -2850,8 +2918,58 @@ namespace OpenWifi { void MicroServicePostInitialization(); + inline void MicroService::InitializeLoggingSystem() { + static auto initialized = false; + + if(!initialized) { + initialized = true; + LoadConfigurationFile(); + + auto LoggingDestination = MicroService::instance().ConfigGetString("logging.type", "file"); + auto LoggingFormat = MicroService::instance().ConfigGetString("logging.format", + "%Y-%m-%d %H:%M:%S %s: [%p] %t"); + if (LoggingDestination == "console") { + Poco::AutoPtr Console(new Poco::ConsoleChannel); + Poco::AutoPtr Formatter(new Poco::PatternFormatter); + Formatter->setProperty("pattern", LoggingFormat); + Poco::AutoPtr FormattingChannel( + new Poco::FormattingChannel(Formatter, Console)); + Poco::Logger::root().setChannel(FormattingChannel); + } else if (LoggingDestination == "colorconsole") { + Poco::AutoPtr Console(new Poco::ColorConsoleChannel); + Poco::AutoPtr Formatter(new Poco::PatternFormatter); + Formatter->setProperty("pattern", LoggingFormat); + Poco::AutoPtr FormattingChannel( + new Poco::FormattingChannel(Formatter, Console)); + Poco::Logger::root().setChannel(FormattingChannel); + } else if (LoggingDestination == "sql") { + + } else if (LoggingDestination == "syslog") { + //"CREATE TABLE T_POCO_LOG (Source VARCHAR, Name VARCHAR, ProcessId INTEGER, Thread VARCHAR, ThreadId INTEGER, Priority INTEGER, Text VARCHAR, DateTime DATE)" + } else { + auto LoggingLocation = + MicroService::instance().ConfigPath("logging.path", "$OWCERT_ROOT/logs") + "/log"; + + Poco::AutoPtr FileChannel(new Poco::FileChannel); + FileChannel->setProperty("rotation", "10 M"); + FileChannel->setProperty("archive", "timestamp"); + FileChannel->setProperty("path", LoggingLocation); + Poco::AutoPtr Formatter(new Poco::PatternFormatter); + Formatter->setProperty("pattern", LoggingFormat); + Poco::AutoPtr FormattingChannel( + new Poco::FormattingChannel(Formatter, FileChannel)); + Poco::Logger::root().setChannel(FormattingChannel); + } + auto Level = Poco::Logger::parseLevel(MicroService::instance().ConfigGetString("logging.level", "debug")); + Poco::Logger::root().setLevel(Level); + } + } + inline void MicroService::initialize(Poco::Util::Application &self) { // add the default services + LoadConfigurationFile(); + InitializeLoggingSystem(); + SubSystems_.push_back(KafkaManager()); SubSystems_.push_back(ALBHealthCheckServer()); SubSystems_.push_back(RESTAPI_ExtServer()); @@ -2863,17 +2981,6 @@ namespace OpenWifi { Poco::Net::FTPStreamFactory::registerFactory(); Poco::Net::FTPSStreamFactory::registerFactory(); - LoadConfigurationFile(); - - static const char * LogFilePathKey = "logging.channels.c2.path"; - - if(LogDir_.empty()) { - std::string OriginalLogFileValue = ConfigPath(LogFilePathKey); - config().setString(LogFilePathKey, OriginalLogFileValue); - } else { - config().setString(LogFilePathKey, LogDir_); - } - Poco::File DataDir(ConfigPath("openwifi.system.data")); DataDir_ = DataDir.path(); if(!DataDir.exists()) { @@ -3167,6 +3274,13 @@ namespace OpenWifi { } } + inline SubSystemServer::SubSystemServer(std::string Name, const std::string &LoggingPrefix, + std::string SubSystemConfigPrefix): + Name_(std::move(Name)), + LoggerPrefix_(LoggingPrefix), + SubSystemConfigPrefix_(std::move(SubSystemConfigPrefix)) { + } + inline int MicroService::main(const ArgVec &args) { MyErrorHandler ErrorHandler(*this); @@ -3174,6 +3288,7 @@ namespace OpenWifi { if (!HelpRequested_) { SavePID(); + Poco::Logger &logger = Poco::Logger::get(DAEMON_APP_NAME); logger.notice(Poco::format("Starting %s version %s.",DAEMON_APP_NAME, Version())); @@ -3185,6 +3300,7 @@ namespace OpenWifi { if (config().getBool("application.runAsDaemon", false)) { logger.information("Starting as a daemon."); } + logger.information(Poco::format("System ID set to %Lu",ID_)); StartSubSystemServers(); waitForTerminationRequest(); @@ -3212,10 +3328,11 @@ namespace OpenWifi { } inline void SubSystemServer::initialize(Poco::Util::Application &self) { - Logger_.notice("Initializing..."); auto i = 0; bool good = true; + Log_ = std::make_unique(Poco::Logger::get(LoggerPrefix_)); + ConfigServersList_.clear(); while (good) { std::string root{SubSystemConfigPrefix_ + ".host." + std::to_string(i) + "."}; @@ -3272,7 +3389,7 @@ namespace OpenWifi { Port_ = (int)MicroService::instance().ConfigGetInt("alb.port",15015); Socket_ = std::make_unique(Port_); auto Params = new Poco::Net::HTTPServerParams; - Server_ = std::make_unique(new ALBRequestHandlerFactory(Logger_), *Socket_, Params); + Server_ = std::make_unique(new ALBRequestHandlerFactory(Logger()), *Socket_, Params); Server_->start(); } @@ -3340,9 +3457,9 @@ namespace OpenWifi { if(Num) Producer.flush(); } catch (const cppkafka::HandleException &E ) { - KafkaManager()->Logger_.warning(Poco::format("Caught a Kafka exception (producer): %s",std::string{E.what()})); + KafkaManager()->Logger().warning(Poco::format("Caught a Kafka exception (producer): %s",std::string{E.what()})); } catch (const Poco::Exception &E) { - KafkaManager()->Logger_.log(E); + KafkaManager()->Logger().log(E); } } Producer.flush(); @@ -3368,13 +3485,13 @@ namespace OpenWifi { cppkafka::Consumer Consumer(Config); Consumer.set_assignment_callback([this](cppkafka::TopicPartitionList& partitions) { if(!partitions.empty()) { - KafkaManager()->Logger_.information(Poco::format("Partition assigned: %Lu...", + KafkaManager()->Logger().information(Poco::format("Partition assigned: %Lu...", (uint64_t)partitions.front().get_partition())); } }); Consumer.set_revocation_callback([this](const cppkafka::TopicPartitionList& partitions) { if(!partitions.empty()) { - KafkaManager()->Logger_.information(Poco::format("Partition revocation: %Lu...", + KafkaManager()->Logger().information(Poco::format("Partition revocation: %Lu...", (uint64_t)partitions.front().get_partition())); } }); @@ -3397,7 +3514,7 @@ namespace OpenWifi { continue; if (Msg.get_error()) { if (!Msg.is_eof()) { - KafkaManager()->Logger_.error(Poco::format("Error: %s", Msg.get_error().to_string())); + KafkaManager()->Logger().error(Poco::format("Error: %s", Msg.get_error().to_string())); }if(!AutoCommit) Consumer.async_commit(Msg); continue; @@ -3417,9 +3534,9 @@ namespace OpenWifi { Consumer.async_commit(Msg); } } catch (const cppkafka::HandleException &E) { - KafkaManager()->Logger_.warning(Poco::format("Caught a Kafka exception (consumer): %s",std::string{E.what()})); + KafkaManager()->Logger().warning(Poco::format("Caught a Kafka exception (consumer): %s",std::string{E.what()})); } catch (const Poco::Exception &E) { - KafkaManager()->Logger_.log(E); + KafkaManager()->Logger().log(E); } } Consumer.unsubscribe(); @@ -3427,14 +3544,14 @@ namespace OpenWifi { inline void RESTAPI_ExtServer::reinitialize(Poco::Util::Application &self) { MicroService::instance().LoadConfigurationFile(); - Logger_.information("Reinitializing."); + Logger().information("Reinitializing."); Stop(); Start(); } void RESTAPI_IntServer::reinitialize(Poco::Util::Application &self) { MicroService::instance().LoadConfigurationFile(); - Logger_.information("Reinitializing."); + Logger().information("Reinitializing."); Stop(); Start(); } @@ -3573,7 +3690,7 @@ namespace OpenWifi { void DoDelete() final {}; }; - inline Poco::Net::HTTPServerResponse::HTTPStatus OpenAPIRequestGet::Do(Poco::JSON::Object::Ptr &ResponseObject) { + inline Poco::Net::HTTPServerResponse::HTTPStatus OpenAPIRequestGet::Do(Poco::JSON::Object::Ptr &ResponseObject, const std::string & BearerToken) { try { auto Services = MicroService::instance().GetServices(Type_); for(auto const &Svc:Services) { @@ -3590,8 +3707,15 @@ namespace OpenWifi { Poco::Net::HTTPRequest Request(Poco::Net::HTTPRequest::HTTP_GET, Path, Poco::Net::HTTPMessage::HTTP_1_1); - Request.add("X-API-KEY", Svc.AccessKey); - Request.add("X-INTERNAL-NAME", MicroService::instance().PublicEndPoint()); + + if(BearerToken.empty()) { + Request.add("X-API-KEY", Svc.AccessKey); + Request.add("X-INTERNAL-NAME", MicroService::instance().PublicEndPoint()); + } else { + // Authorization: Bearer ${token} + Request.add("Authorization", "Bearer " + BearerToken); + } + Session.sendRequest(Request); Poco::Net::HTTPResponse Response; @@ -3610,7 +3734,7 @@ namespace OpenWifi { return Poco::Net::HTTPServerResponse::HTTP_GATEWAY_TIMEOUT; } - inline Poco::Net::HTTPServerResponse::HTTPStatus OpenAPIRequestPut::Do(Poco::JSON::Object::Ptr &ResponseObject) { + inline Poco::Net::HTTPServerResponse::HTTPStatus OpenAPIRequestPut::Do(Poco::JSON::Object::Ptr &ResponseObject, const std::string & BearerToken) { try { auto Services = MicroService::instance().GetServices(Type_); for(auto const &Svc:Services) { @@ -3633,8 +3757,13 @@ namespace OpenWifi { Request.setContentType("application/json"); Request.setContentLength(obody.str().size()); - Request.add("X-API-KEY", Svc.AccessKey); - Request.add("X-INTERNAL-NAME", MicroService::instance().PublicEndPoint()); + if(BearerToken.empty()) { + Request.add("X-API-KEY", Svc.AccessKey); + Request.add("X-INTERNAL-NAME", MicroService::instance().PublicEndPoint()); + } else { + // Authorization: Bearer ${token} + Request.add("Authorization", "Bearer " + BearerToken); + } std::ostream & os = Session.sendRequest(Request); os << obody.str(); @@ -3658,9 +3787,10 @@ namespace OpenWifi { return Poco::Net::HTTPServerResponse::HTTP_GATEWAY_TIMEOUT; } - inline Poco::Net::HTTPServerResponse::HTTPStatus OpenAPIRequestPost::Do(Poco::JSON::Object::Ptr &ResponseObject) { + inline Poco::Net::HTTPServerResponse::HTTPStatus OpenAPIRequestPost::Do(Poco::JSON::Object::Ptr &ResponseObject, const std::string & BearerToken) { try { auto Services = MicroService::instance().GetServices(Type_); + for(auto const &Svc:Services) { Poco::URI URI(Svc.PrivateEndPoint); Poco::Net::HTTPSClientSession Session(URI.getHost(), URI.getPort()); @@ -3681,8 +3811,13 @@ namespace OpenWifi { Request.setContentType("application/json"); Request.setContentLength(obody.str().size()); - Request.add("X-API-KEY", Svc.AccessKey); - Request.add("X-INTERNAL-NAME", MicroService::instance().PublicEndPoint()); + if(BearerToken.empty()) { + Request.add("X-API-KEY", Svc.AccessKey); + Request.add("X-INTERNAL-NAME", MicroService::instance().PublicEndPoint()); + } else { + // Authorization: Bearer ${token} + Request.add("Authorization", "Bearer " + BearerToken); + } std::ostream & os = Session.sendRequest(Request); os << obody.str(); diff --git a/src/framework/StorageClass.h b/src/framework/StorageClass.h index 95dba62..9a4ca56 100644 --- a/src/framework/StorageClass.h +++ b/src/framework/StorageClass.h @@ -33,8 +33,8 @@ namespace OpenWifi { int Start() override { std::lock_guard Guard(Mutex_); - Logger_.setLevel(Poco::Message::PRIO_NOTICE); - Logger_.notice("Starting."); + Logger().setLevel(Poco::Message::PRIO_NOTICE); + Logger().notice("Starting."); std::string DBType = MicroService::instance().ConfigGetString("storage.type"); if (DBType == "sqlite") { @@ -101,7 +101,7 @@ namespace OpenWifi { #else inline int StorageClass::Setup_SQLite() { - Logger_.notice("SQLite StorageClass enabled."); + Logger().notice("SQLite StorageClass enabled."); dbType_ = sqlite; auto DBName = MicroService::instance().DataDir() + "/" + MicroService::instance().ConfigGetString("storage.type.sqlite.db"); auto NumSessions = MicroService::instance().ConfigGetInt("storage.type.sqlite.maxsessions", 64); @@ -112,7 +112,7 @@ namespace OpenWifi { } inline int StorageClass::Setup_MySQL() { - Logger_.notice("MySQL StorageClass enabled."); + Logger().notice("MySQL StorageClass enabled."); dbType_ = mysql; auto NumSessions = MicroService::instance().ConfigGetInt("storage.type.mysql.maxsessions", 64); auto IdleTime = MicroService::instance().ConfigGetInt("storage.type.mysql.idletime", 60); @@ -137,7 +137,7 @@ namespace OpenWifi { } inline int StorageClass::Setup_PostgreSQL() { - Logger_.notice("PostgreSQL StorageClass enabled."); + Logger().notice("PostgreSQL StorageClass enabled."); dbType_ = pgsql; auto NumSessions = MicroService::instance().ConfigGetInt("storage.type.postgresql.maxsessions", 64); auto IdleTime = MicroService::instance().ConfigGetInt("storage.type.postgresql.idletime", 60); diff --git a/src/framework/orm.h b/src/framework/orm.h index f3f17fe..e283b92 100644 --- a/src/framework/orm.h +++ b/src/framework/orm.h @@ -488,6 +488,18 @@ namespace ORM { return false; } + template bool ReplaceRecord( const char *FieldName, T & Value, RecordType & R) { + try { + if(Exists(FieldName, Value)) { + return UpdateRecord(FieldName,Value,R); + } + return CreateRecord(R); + } catch (const Poco::Exception &E) { + Logger_.log(E); + } + return false; + } + template bool GetNameAndDescription(const char *FieldName, T & Value, std::string & Name, std::string & Description ) { try { assert( FieldNames_.find(FieldName) != FieldNames_.end() ); @@ -776,9 +788,9 @@ namespace ORM { protected: Poco::Data::SessionPool &Pool_; Poco::Logger &Logger_; + std::string DBName_; private: OpenWifi::DBType Type_; - std::string DBName_; std::string CreateFields_; std::string SelectFields_; std::string SelectList_;