scottk/wlan-cloud-analytics
1
0
Fork 0
mirror of https://github.com/Telecominfraproject/wlan-cloud-analytics.git synced 2025-11-01 19:08:09 +00:00
Code Issues Packages Projects Releases 10 Wiki Activity

Added averages and so on to DeviceTimePoint.

Browse Source
This commit is contained in:
stephb9959
2022-03-26 21:31:21 -07:00
parent 4ef98b3cba
commit d617e3247e
11 changed files with 222 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
build
View File

@@ -1 +1 @@
42
43

60
openapi/owanalytics.yaml
View File

@@ -355,26 +355,19 @@ components:
items:
$ref: '#/components/schemas/UETimePoint'
tx_bytes_bw:
type:
$ref: '#/components/schemas/AveragePoint'
$ref: '#/components/schemas/AveragePoint'
rx_bytes_bw:
type:
$ref: '#/components/schemas/AveragePoint'
$ref: '#/components/schemas/AveragePoint'
tx_packets_bw:
type:
$ref: '#/components/schemas/AveragePoint'
$ref: '#/components/schemas/AveragePoint'
rx_packets_bw:
type:
$ref: '#/components/schemas/AveragePoint'
$ref: '#/components/schemas/AveragePoint'
tx_failed_pct:
type:
$ref: '#/components/schemas/AveragePoint'
$ref: '#/components/schemas/AveragePoint'
tx_retries_pct:
type:
$ref: '#/components/schemas/AveragePoint'
$ref: '#/components/schemas/AveragePoint'
tx_duration_pct:
type:
$ref: '#/components/schemas/AveragePoint'
$ref: '#/components/schemas/AveragePoint'
APTimePoint:
type: object
@@ -487,6 +480,41 @@ components:
device_info:
$ref: '#/components/schemas/DeviceInfo'
DeviceTimePointAnalysis:
type: object
properties:
timestamp:
type: integer
format: int64
noise:
$ref: '#/components/schemas/AveragePoint'
active_pct:
$ref: '#/components/schemas/AveragePoint'
busy_pct:
$ref: '#/components/schemas/AveragePoint'
receive_pct:
$ref: '#/components/schemas/AveragePoint'
transmit_pct:
$ref: '#/components/schemas/AveragePoint'
tx_power:
$ref: '#/components/schemas/AveragePoint'
tx_bytes_bw:
$ref: '#/components/schemas/AveragePoint'
rx_bytes_bw:
$ref: '#/components/schemas/AveragePoint'
rx_dropped_pct:
$ref: '#/components/schemas/AveragePoint'
tx_dropped_pct:
$ref: '#/components/schemas/AveragePoint'
rx_packets_bw:
$ref: '#/components/schemas/AveragePoint'
tx_packets_bw:
$ref: '#/components/schemas/AveragePoint'
rx_errors_pct =:
$ref: '#/components/schemas/AveragePoint'
tx_errors_pct =:
$ref: '#/components/schemas/AveragePoint'
DeviceTimePointList:
type: object
properties:
@@ -494,6 +522,10 @@ components:
type: array
items:
$ref: '#/components/schemas/DeviceTimePoint'
stats:
type: array
items:
$ref: '#/components/schemas/DeviceTimePointAnalysis'
DeviceTimePointStats:
type: object

2
owanalytics.properties
View File

@@ -58,6 +58,8 @@ openwifi.kafka.ssl.certificate.location =
openwifi.kafka.ssl.key.location =
openwifi.kafka.ssl.key.password =
storage.cleanup.interval = 21600
#
# This section select which form of persistence you need
# Only one selected at a time. If you select multiple, this service will die if a horrible

4
src/RESTAPI/RESTAPI_board_handler.cpp
View File

@@ -33,10 +33,12 @@ namespace OpenWifi {
return NotFound();
}
VenueCoordinator()->StopBoard(id);
if(!StorageService()->BoardsDB().DeleteRecord("id",id)) {
return NotFound();
}
VenueCoordinator()->StopBoard(id);
StorageService()->TimePointsDB().DeleteBoard(id);
return OK();
}

72
src/RESTAPI/RESTAPI_board_timepoint_handler.cpp
View File

@@ -54,6 +54,32 @@ namespace OpenWifi {
}
}
template <typename X, typename M> void AverageAPData( X T, const std::vector<M> &Values, AnalyticsObjects::AveragePoint &P) {
if(Values.empty())
return;
double sum = 0.0;
for(const auto &v:Values) {
sum += (v.ap_data.*T);
P.min = std::min(P.min,(v.ap_data.*T));
P.max = std::min(P.max,(v.ap_data.*T));
}
P.avg = sum / (double) Values.size();
}
template <typename X, typename M> void AverageRadioData( X T, const std::vector<M> &Values, AnalyticsObjects::AveragePoint &P) {
if(Values.empty())
return;
double sum = 0.0;
for(const auto &value:Values) {
for(const auto &radio:value.radio_data) {
sum += (radio.*T);
P.min = std::min((double)P.min, (double)(radio.*T));
P.max = std::max((double)P.max, (double)(radio.*T));
}
}
P.avg = sum / (double) Values.size();
}
void RESTAPI_board_timepoint_handler::DoGet() {
auto id = GetBinding("id","");
@@ -125,17 +151,47 @@ namespace OpenWifi {
}
Poco::JSON::Object Answer;
Poco::JSON::Array Outer;
for(const auto &i:sp) {
Poco::JSON::Array InnerArray;
for(const auto &j:i) {
Poco::JSON::Array Points_OuterArray;
for(const auto &point_list:sp) {
Poco::JSON::Array Points_InnerArray;
for(const auto &point:point_list) {
Poco::JSON::Object O;
j.to_json(O);
InnerArray.add(O);
point.to_json(O);
Points_InnerArray.add(O);
}
Outer.add(InnerArray);
Points_OuterArray.add(Points_InnerArray);
}
Answer.set("points",Outer);
// calculate the stats for each time slot
Poco::JSON::Array Stats_Array;
for(const auto &point_list:sp) {
AnalyticsObjects::DeviceTimePointAnalysis DTPA;
DTPA.timestamp = point_list[0].timestamp;
AverageAPData(&AnalyticsObjects::APTimePoint::tx_bytes_bw,point_list,DTPA.tx_bytes_bw);
AverageAPData(&AnalyticsObjects::APTimePoint::rx_bytes_bw,point_list,DTPA.rx_bytes_bw);
AverageAPData(&AnalyticsObjects::APTimePoint::rx_dropped_pct,point_list,DTPA.rx_dropped_pct);
AverageAPData(&AnalyticsObjects::APTimePoint::tx_dropped_pct,point_list,DTPA.tx_dropped_pct);
AverageAPData(&AnalyticsObjects::APTimePoint::rx_packets_bw,point_list,DTPA.rx_packets_bw);
AverageAPData(&AnalyticsObjects::APTimePoint::tx_packets_bw,point_list,DTPA.tx_packets_bw);
AverageAPData(&AnalyticsObjects::APTimePoint::rx_errors_pct,point_list,DTPA.rx_errors_pct);
AverageAPData(&AnalyticsObjects::APTimePoint::tx_errors_pct,point_list,DTPA.tx_errors_pct);
AverageRadioData(&AnalyticsObjects::RadioTimePoint::noise, point_list, DTPA.noise);
AverageRadioData(&AnalyticsObjects::RadioTimePoint::tx_power, point_list, DTPA.tx_power);
AverageRadioData(&AnalyticsObjects::RadioTimePoint::active_pct, point_list, DTPA.active_pct);
AverageRadioData(&AnalyticsObjects::RadioTimePoint::busy_pct, point_list, DTPA.busy_pct);
AverageRadioData(&AnalyticsObjects::RadioTimePoint::receive_pct, point_list, DTPA.receive_pct);
AverageRadioData(&AnalyticsObjects::RadioTimePoint::transmit_pct, point_list, DTPA.transmit_pct);
Poco::JSON::Object Stats_point;
DTPA.to_json(Stats_point);
Stats_Array.add(Stats_point);
}
Answer.set("points",Points_OuterArray);
Answer.set("stats",Stats_Array);
/* static int f=0;

42
src/RESTObjects/RESTAPI_AnalyticsObjects.cpp
View File

@@ -397,13 +397,55 @@ namespace OpenWifi::AnalyticsObjects {
return false;
}
void DeviceTimePointAnalysis::to_json(Poco::JSON::Object &Obj) const {
field_to_json(Obj,"noise",noise);
field_to_json(Obj,"active_pct",active_pct);
field_to_json(Obj,"busy_pct",busy_pct);
field_to_json(Obj,"receive_pct",receive_pct);
field_to_json(Obj,"transmit_pct",transmit_pct);
field_to_json(Obj,"tx_power",tx_power);
field_to_json(Obj,"tx_bytes_bw",tx_bytes_bw);
field_to_json(Obj,"rx_bytes_bw",rx_bytes_bw);
field_to_json(Obj,"rx_dropped_pct",rx_dropped_pct);
field_to_json(Obj,"tx_dropped_pct",tx_dropped_pct);
field_to_json(Obj,"rx_packets_bw",rx_packets_bw);
field_to_json(Obj,"tx_packets_bw",tx_packets_bw);
field_to_json(Obj,"rx_errors_pct",rx_errors_pct);
field_to_json(Obj,"tx_errors_pct",tx_errors_pct);
}
bool DeviceTimePointAnalysis::from_json(const Poco::JSON::Object::Ptr &Obj) {
try {
field_from_json(Obj,"noise",noise);
field_from_json(Obj,"active_pct",active_pct);
field_from_json(Obj,"busy_pct",busy_pct);
field_from_json(Obj,"receive_pct",receive_pct);
field_from_json(Obj,"transmit_pct",transmit_pct);
field_from_json(Obj,"tx_power",tx_power);
field_from_json(Obj,"tx_bytes_bw",tx_bytes_bw);
field_from_json(Obj,"rx_bytes_bw",rx_bytes_bw);
field_from_json(Obj,"rx_dropped_pct",rx_dropped_pct);
field_from_json(Obj,"tx_dropped_pct",tx_dropped_pct);
field_from_json(Obj,"rx_packets_bw",rx_packets_bw);
field_from_json(Obj,"tx_packets_bw",tx_packets_bw);
field_from_json(Obj,"rx_errors_pct",rx_errors_pct);
field_from_json(Obj,"tx_errors_pct",tx_errors_pct);
return true;
} catch(...) {
}
return false;
}
void DeviceTimePointList::to_json(Poco::JSON::Object &Obj) const {
field_to_json(Obj,"points",points);
field_to_json(Obj,"stats",stats);
}
bool DeviceTimePointList::from_json(const Poco::JSON::Object::Ptr &Obj) {
try {
field_from_json(Obj,"points",points);
field_from_json(Obj,"stats",stats);
return true;
} catch(...) {

42
src/RESTObjects/RESTAPI_AnalyticsObjects.h
View File

@@ -240,13 +240,13 @@ namespace OpenWifi {
struct DeviceTimePoint {
std::string id;
std::string boardId;
uint64_t timestamp = 0;
APTimePoint ap_data;
std::vector<SSIDTimePoint> ssid_data;
std::vector<RadioTimePoint> radio_data;
AnalyticsObjects::DeviceInfo device_info;
std::string id;
std::string boardId;
uint64_t timestamp = 0;
APTimePoint ap_data;
std::vector<SSIDTimePoint> ssid_data;
std::vector<RadioTimePoint> radio_data;
AnalyticsObjects::DeviceInfo device_info;
void to_json(Poco::JSON::Object &Obj) const;
bool from_json(const Poco::JSON::Object::Ptr &Obj);
@@ -277,9 +277,33 @@ namespace OpenWifi {
};
struct DeviceTimePointList {
std::vector<DeviceTimePoint> points;
struct DeviceTimePointAnalysis {
uint64_t timestamp;
AveragePoint noise;
AveragePoint active_pct;
AveragePoint busy_pct;
AveragePoint receive_pct;
AveragePoint transmit_pct;
AveragePoint tx_power;
AveragePoint tx_bytes_bw;
AveragePoint rx_bytes_bw;
AveragePoint rx_dropped_pct;
AveragePoint tx_dropped_pct;
AveragePoint rx_packets_bw;
AveragePoint tx_packets_bw;
AveragePoint rx_errors_pct;
AveragePoint tx_errors_pct;
void to_json(Poco::JSON::Object &Obj) const;
bool from_json(const Poco::JSON::Object::Ptr &Obj);
};
struct DeviceTimePointList {
std::vector<DeviceTimePoint> points;
std::vector<DeviceTimePointAnalysis> stats;
void to_json(Poco::JSON::Object &Obj) const;
bool from_json(const Poco::JSON::Object::Ptr &Obj);
};

27
src/StorageService.cpp
View File

@@ -20,18 +20,40 @@ namespace OpenWifi {
BoardsDB_->Create();
TimePointsDB_ = std::make_unique<OpenWifi::TimePointDB>(dbType_,*Pool_, Logger());
TimePointsDB_->Create();
PeriodicCleanup_ = MicroService::instance().ConfigGetInt("storage.cleanup.interval", 6*60*60);
if(PeriodicCleanup_<1*60*60)
PeriodicCleanup_ = 1*60*60;
Updater_.start(*this);
TimerCallback_ = std::make_unique<Poco::TimerCallback<Storage>>(*this,&Storage::onTimer);
Timer_.setStartInterval( 20 * 1000); // first run in 20 seconds
Timer_.setPeriodicInterval(1 * 60 * 60 * 1000); // 1 hours
Timer_.setStartInterval( 60 * 1000); // first run in 20 seconds
Timer_.setPeriodicInterval((long)PeriodicCleanup_ * 1000); // 1 hours
Timer_.start(*TimerCallback_);
return 0;
}
void Storage::onTimer([[maybe_unused]] Poco::Timer &timer) {
BoardsDB::RecordVec BoardList;
uint64_t start = 0 ;
bool done = false;
const uint64_t batch=100;
Logger().information("Starting cleanup of TimePoint Database");
while(!done) {
if(!BoardsDB().GetRecords(start,batch,BoardList)) {
for(const auto &board:BoardList) {
for(const auto &venue:board.venueList) {
auto now = OpenWifi::Now();
auto lower_bound = now - venue.retention;
Logger().information(fmt::format("Removing old records for board '{}'",board.info.name));
BoardsDB().DeleteRecords(fmt::format(" boardId='{}' and timestamp<{}", board.info.id, lower_bound));
}
}
}
done = (BoardList.size() < batch);
}
Logger().information(fmt::format("Done cleanup of TimePoint Database. Next run in {} seconds.", PeriodicCleanup_));
}
void Storage::run() {
@@ -50,6 +72,7 @@ namespace OpenWifi {
void Storage::Stop() {
Running_=false;
Timer_.stop();
Updater_.wakeUp();
Updater_.join();
Logger().notice("Stopping.");

1
src/StorageService.h
View File

@@ -36,6 +36,7 @@ namespace OpenWifi {
std::atomic_bool Running_=false;
Poco::Timer Timer_;
std::unique_ptr<Poco::TimerCallback<Storage>> TimerCallback_;
uint64_t PeriodicCleanup_=6*60*60;
};
inline auto StorageService() { return Storage::instance(); }
} // namespace

4
src/storage/storage_timepoints.cpp
View File

@@ -64,6 +64,10 @@ namespace OpenWifi {
GetRecords(0,MaxRecords,Recs,WhereClause," order by timestamp ASC ");
return true;
}
bool TimePointDB::DeleteBoard(const std::string &boardId) {
return DeleteRecords(fmt::format(" boardId='{}' ", boardId));
}
}
template<> void ORM::DB<OpenWifi::TimePointDBRecordType, OpenWifi::AnalyticsObjects::DeviceTimePoint>::Convert(const OpenWifi::TimePointDBRecordType &In, OpenWifi::AnalyticsObjects::DeviceTimePoint &Out) {

1
src/storage/storage_timepoints.h
View File

@@ -23,6 +23,7 @@ namespace OpenWifi {
TimePointDB( OpenWifi::DBType T, Poco::Data::SessionPool & P, Poco::Logger &L);
bool GetStats(const std::string &id, AnalyticsObjects::DeviceTimePointStats &S);
bool SelectRecords(const std::string &boardId, uint64_t FromDate, uint64_t LastDate, uint64_t MaxRecords, DB::RecordVec & Recs);
bool DeleteBoard(const std::string &boardId);
virtual ~TimePointDB() {};
private:
bool Upgrade(uint32_t from, uint32_t &to) override;