From a1593e759acdb250677228fd3ecdfa969f17348d Mon Sep 17 00:00:00 2001 From: Kris Moore Date: Fri, 12 Feb 2016 12:54:45 -0500 Subject: [PATCH] Add initial Event watcher for system health, needs fixing since QTimer isn't working at the moment, kenmoore will investigate --- src/server/EventWatcher.cpp | 85 ++++++++++++++++++++++++++++++++++++- src/server/EventWatcher.h | 7 ++- src/server/LogManager.h | 3 +- 3 files changed, 91 insertions(+), 4 deletions(-) diff --git a/src/server/EventWatcher.cpp b/src/server/EventWatcher.cpp index 4e5aee4..a96f838 100644 --- a/src/server/EventWatcher.cpp +++ b/src/server/EventWatcher.cpp @@ -6,6 +6,7 @@ #include "EventWatcher.h" #include "globals.h" +#include "sysadm-general.h" // === PUBLIC === EventWatcher::EventWatcher(){ @@ -14,11 +15,17 @@ EventWatcher::EventWatcher(){ starting = true; watcher = new QFileSystemWatcher(this); filechecktimer = new QTimer(this); - filechecktimer->setSingleShot(false); - filechecktimer->setInterval(3600000); //1-hour checks (also run on new event notices) + filechecktimer->setSingleShot(false); + filechecktimer->setInterval(3600000); //1-hour checks (also run on new event notices) connect(watcher, SIGNAL(fileChanged(const QString&)), this, SLOT(WatcherUpdate(const QString&)) ); connect(watcher, SIGNAL(directoryChanged(const QString&)), this, SLOT(WatcherUpdate(const QString&)) ); connect(filechecktimer, SIGNAL(timeout()), this, SLOT( CheckLogFiles()) ); + + oldhostname = sysadm::General::RunCommand("hostname").simplified(); + syschecktimer = new QTimer(this); + syschecktimer->setSingleShot(false); + syschecktimer->setInterval(900000); //15 minute checks + connect(syschecktimer, SIGNAL(timeout()), this, SLOT( CheckSystemState()) ); } EventWatcher::~EventWatcher(){ @@ -303,3 +310,77 @@ void EventWatcher::ReadLPRepFile(){ if(repTotK!="??"){CONFIG->setValue("internal/"+QString(WS_MODE ? "ws" : "tcp")+"/lp-rep-totk",repTotK); } CONFIG->setValue("internal/"+QString(WS_MODE ? "ws" : "tcp")+"/lp-rep-lastsize",lastSize); } + +// Periodic check to monitor the health of the running system +void EventWatcher::CheckSystemState(){ + QJsonObject obj; + QString tmp, line; + bool ok; + int priority = 0; + + qDebug() << "Starting health check"; + // Query the system, check how things are running + + // First up, get the hostname + QString newhostname = sysadm::General::RunCommand("hostname").simplified(); + if ( newhostname != oldhostname ) + { + // Interesting, hostname changed, lets notify + obj.insert("hostnamechanged","true"); + oldhostname = newhostname; + if ( priority < 3 ) + priority = 3; + } + obj.insert("hostname",oldhostname); + + // Next check zpools + QStringList output = sysadm::General::RunCommand("zpool list -H").split("\n"); + for ( int i = 0; i < output.size(); i++) + { + line = output.at(i).simplified(); + + // Get the individual elements + QString pool = line.section(" ", 0, 0); + QString total = line.section(" ", 1, 1); + QString used = line.section(" ", 2, 2); + QString free = line.section(" ", 3, 3); + QString frag = line.section(" ", 5, 5).replace("%", ""); + QString cap = line.section(" ", 6, 6).replace("%", ""); + QString health = line.section(" ", 8, 8); + + // If the health is bad, we need to notify + if ( health != "ONLINE" ) + if ( priority < 9 ) + priority = 9; + + // Check the capacity, if over 90% we should warn + cap.toInt(&ok); + if ( ok ) { + if ( cap.toInt() > 90 ) { + if ( priority < 9 ) + priority = 6; + } + } + + QJsonObject zpool; + QJsonObject zstats; + zstats.insert("size", total); + zstats.insert("used", used); + zstats.insert("free", free); + zstats.insert("frag", frag); + zstats.insert("capacity", cap); + zstats.insert("health", health); + zpool.insert(pool, zstats); + obj.insert("zpools", zpool ); + } + + // Priority 0-10 + obj.insert("priority", DisplayPriority(priority) ); + + qDebug() << "Done health check"; + + // Log and send out event + LogManager::log(LogManager::EV_STATE, obj); + emit NewEvent(STATE, obj); +} + diff --git a/src/server/EventWatcher.h b/src/server/EventWatcher.h index 9149e4f..4d4ebad 100644 --- a/src/server/EventWatcher.h +++ b/src/server/EventWatcher.h @@ -17,7 +17,7 @@ class EventWatcher : public QObject{ Q_OBJECT public: //Add more event types here as needed - enum EVENT_TYPE{ BADEVENT, DISPATCHER, LIFEPRESERVER}; + enum EVENT_TYPE{ BADEVENT, DISPATCHER, LIFEPRESERVER, STATE}; EventWatcher(); ~EventWatcher(); @@ -33,6 +33,7 @@ private: QFileSystemWatcher *watcher; QHash HASH; QTimer *filechecktimer; + QTimer *syschecktimer; bool starting; //HASH Note: Fields 1-99 reserved for EVENT_TYPE enum (last message of that type) // Fields 100-199 reserved for Life Preserver logs (all types) @@ -46,6 +47,9 @@ private: QString readFile(QString path); double displayToDoubleK(QString); + // For health monitoring + QString oldhostname; + public slots: void start(); @@ -57,6 +61,7 @@ private slots: //File watcher signals void WatcherUpdate(const QString&); void CheckLogFiles(); //catch/load any new log files into the watcher + void CheckSystemState(); // Periodic check to monitor health of system //LP File changed signals/slots void ReadLPLogFile(); diff --git a/src/server/LogManager.h b/src/server/LogManager.h index b7855cc..b035595 100644 --- a/src/server/LogManager.h +++ b/src/server/LogManager.h @@ -24,7 +24,7 @@ class LogManager{ public: //Enumeration of common log files (will automatically use proper file) // === ADD NEW FILE SUPPORT HERE === - enum LOG_FILE {HOST, DISPATCH, EV_DISPATCH, EV_LP}; + enum LOG_FILE {HOST, DISPATCH, EV_DISPATCH, EV_LP, EV_STATE}; //Conversion function for flag->path static QString flagToPath(LogManager::LOG_FILE flag){ QString filepath; @@ -32,6 +32,7 @@ public: else if(flag==DISPATCH){ filepath.append("dispatcher"); } else if(flag==EV_DISPATCH){ filepath.append("events-dispatcher"); } else if(flag==EV_LP){ filepath.append("events-lifepreserver"); } + else if(flag==EV_STATE){ filepath.append("events-state"); } else{ return ""; } //invalid file given //Now add the datestamp to the filename - so logs are saved/pruned daily filepath.append("-%1.log");