Merge pull request #349 from bill1600/siglog

fix shutdown close msg to agree with splunk log

CHANGELOG.md

@@ -16,6 +16,20 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - update to use nopoll v 1.0.2
 - Add pause/resume heartBeatTimer
 - parodus event handler to listen to interface_down and interface_up event
+- Pause connection retry during interface_down event
+- Add callback handler for ping status change event
+- Fixed nopoll_conn_unref crash
+- Update retry timestamp in connection-health-file
+- fix so that parodus can be killed, even if in a connection fail loop
+- provide signal handlers so we shut down properly when INCLUDE_BREAKPAD active
+- send status code and reason in websocket close message
+- dont try to install handler for signal 9
+- on connect retry, requery jwt only if it failed before 
+- put two timestamps in connection health file; start conn and current
+- change health file update interval to 240sec
+- use jitter in backoff delay
+- sendMessage to check cloud status == ONLINE before sending
+- when killed with SIGTERM, close will use msg in close reason file.
 
 ## [1.0.2] - 2019-02-08
 - Refactored connection.c and updated corresponding unit tests

README.md

@@ -1,7 +1,7 @@
 # parodus
 
-[![Build Status](https://travis-ci.org/Comcast/parodus.svg?branch=master)](https://travis-ci.org/Comcast/parodus)
-[![codecov.io](http://codecov.io/github/Comcast/parodus/coverage.svg?branch=master)](http://codecov.io/github/Comcast/parodus?branch=master)
+[![Build Status](https://travis-ci.org/xmidt-org/parodus.svg?branch=master)](https://travis-ci.org/xmidt-org/parodus)
+[![codecov.io](http://codecov.io/github/xmidt-org/parodus/coverage.svg?branch=master)](http://codecov.io/github/Comcast/parodus?branch=master)
 [![Coverity](https://img.shields.io/coverity/scan/11192.svg)](https://scan.coverity.com/projects/comcast-parodus)
 [![Apache V2 License](http://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/Comcast/parodus/blob/master/LICENSE)
 [![GitHub release](https://img.shields.io/github/release/Comcast/parodus.svg)](CHANGELOG.md)

src/CMakeLists.txt

@@ -14,7 +14,7 @@
 set(SOURCES main.c mutex.c networking.c nopoll_helpers.c heartBeat.c nopoll_handlers.c
     ParodusInternal.c string_helpers.c time.c config.c conn_interface.c
     connection.c spin_thread.c client_list.c service_alive.c
-    upstream.c downstream.c thread_tasks.c partners_check.c token.c 
+    upstream.c downstream.c thread_tasks.c partners_check.c token.c event_handler.c
 	crud_interface.c crud_tasks.c crud_internal.c close_retry.c auth_token.c)
 
 if (ENABLE_SESHAT)

src/ParodusInternal.c

@@ -28,6 +28,7 @@
 bool interface_down_event = false;
 
 pthread_mutex_t interface_down_mut=PTHREAD_MUTEX_INITIALIZER;
+pthread_cond_t interface_down_con=PTHREAD_COND_INITIALIZER;
 
 /*----------------------------------------------------------------------------*/
 /*                             External Functions                             */
@@ -130,6 +131,27 @@ char* getWebpaConveyHeader()
     return NULL;
 }
 
+int readFromFile(const char *file_name, char **data)
+{
+	FILE *fp;
+	int ch_count = 0;
+	fp = fopen(file_name, "r+");
+	if (fp == NULL)
+	{
+		ParodusError("Failed to open file %s (errno %d)\n", file_name, errno);
+		return 0;
+	}
+	fseek(fp, 0, SEEK_END);
+	ch_count = ftell(fp);
+	fseek(fp, 0, SEEK_SET);
+	*data = (char *) malloc(sizeof(char) * (ch_count + 1));
+	fread(*data, 1, ch_count,fp);
+	(*data)[ch_count] ='\0';
+	fclose(fp);
+	return 1;
+}
+
+
 void timespec_diff(struct timespec *start, struct timespec *stop,
                    struct timespec *diff)
 {
@@ -164,6 +186,7 @@ void reset_interface_down_event()
 {
 	pthread_mutex_lock (&interface_down_mut);
 	interface_down_event = false;
+	pthread_cond_signal(&interface_down_con);
 	pthread_mutex_unlock (&interface_down_mut);
 }
 
@@ -175,6 +198,14 @@ void set_interface_down_event()
     	pthread_mutex_unlock (&interface_down_mut);
 }
 
-
+pthread_cond_t *get_interface_down_con(void)
+{
+    return &interface_down_con;
+}
+
+pthread_mutex_t *get_interface_down_mut(void)
+{
+    return &interface_down_mut;
+}
 
 

src/ParodusInternal.h

@@ -49,6 +49,7 @@
 /*----------------------------------------------------------------------------*/
 /*                                   Macros                                   */
 /*----------------------------------------------------------------------------*/
+
 #define UNUSED(x) (void )(x)
 #define NANO_SOCKET_SEND_TIMEOUT_MS                     2000
 #define NANO_SOCKET_RCV_TIMEOUT_MS			500
@@ -107,6 +108,8 @@ typedef struct {
 
 //--- Used in connection.c for backoff delay timer
 typedef struct {
+  unsigned long start_time;
+  struct timespec ts;
   int count;
   int max_count;
   int delay;
@@ -124,7 +127,7 @@ typedef struct {
 // wait_connection_ready, and nopoll_connect 
 typedef struct {
   noPollCtx *nopoll_ctx;
-  server_list_t server_list;
+  server_list_t *server_list;
   server_t *current_server;
   header_info_t header_info;
   char *extra_headers;		// need to be freed
@@ -154,6 +157,8 @@ char* getWebpaConveyHeader();
 void *CRUDHandlerTask();
 void addCRUDmsgToQueue(wrp_msg_t *crudMsg);
 
+int readFromFile(const char *file_name, char **data);
+
 void timespec_diff(struct timespec *start, struct timespec *stop,
                    struct timespec *result);
 
@@ -170,6 +175,10 @@ void reset_interface_down_event();
 
 // Set value of interface_down_event to true
 void set_interface_down_event();
+
+pthread_cond_t *get_interface_down_con();
+
+pthread_mutex_t *get_interface_down_mut();
   
 
 #ifdef __cplusplus

src/auth_token.c

@@ -38,9 +38,14 @@
 /*                            File Scoped Variables                           */
 /*----------------------------------------------------------------------------*/
 void createCurlheader(char *mac_header, char *serial_header, char *uuid_header, char *transaction_uuid, struct curl_slist *list, struct curl_slist **header_list);
+long g_response_code;
 /*----------------------------------------------------------------------------*/
 /*                             External Functions                             */
 /*----------------------------------------------------------------------------*/
+int getGlobalResponseCode()
+{
+	return g_response_code;
+}
 /*
 * @brief Initialize curl object with required options. create newToken using libcurl.
 * @param[out] newToken auth token string obtained from JWT curl response
@@ -62,7 +67,6 @@ int requestNewAuthToken(char *newToken, size_t len, int r_count)
 	char *uuid_header = NULL;
 	char *transaction_uuid = NULL;
 	double total;
-	long response_code;
 
 	struct token_data data;
 	data.size = 0;
@@ -118,8 +122,8 @@ int requestNewAuthToken(char *newToken, size_t len, int r_count)
 		/* Perform the request, res will get the return code */
 		res = curl_easy_perform(curl);
 
-		curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
-		ParodusInfo("themis curl response %d http_code %d\n", res, response_code);
+		curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &g_response_code);
+		ParodusInfo("themis curl response %d http_code %d\n", res, g_response_code);
 
 		time_res = curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &total);
 		if(time_res == 0)
@@ -137,10 +141,18 @@ int requestNewAuthToken(char *newToken, size_t len, int r_count)
 		}
 		else
 		{
-			if(response_code == 200)
+			if(getGlobalResponseCode() == 200)
 			{
 				ParodusInfo("cURL success\n");
 			}
+			else
+			{
+				ParodusError("Failed response from auth token server %s\n", data.data);
+				curl_easy_cleanup(curl);
+				data.size = 0;
+				memset (data.data, 0, len);
+				return -1;
+			}
 		}
 		curl_easy_cleanup(curl);
 	}

src/config.c

@@ -366,6 +366,7 @@ int parseCommandLine(int argc,char **argv,ParodusCfg * cfg)
 	{"token-server-url",        required_argument, 0, 'U'},
 	{"crud-config-file",        required_argument, 0, 'C'},
 	{"connection-health-file",  required_argument, 0, 'S'},
+	{"close-reason-file",  		required_argument, 0, 'R'},
 	{"mtls-client-key-path",    required_argument, 0, 'K'},
 	{"mtls-client-cert-path",    required_argument, 0,'M'},
         {0, 0, 0, 0}
@@ -384,6 +385,7 @@ int parseCommandLine(int argc,char **argv,ParodusCfg * cfg)
 	parStrncpy (cfg->jwt_key, "", sizeof(cfg->jwt_key));
 	cfg->crud_config_file = NULL;
 	cfg->connection_health_file = NULL;
+	cfg->close_reason_file = NULL;
 	cfg->client_cert_path = NULL;
 	cfg->token_server_url = NULL;
 	cfg->cloud_status = NULL;
@@ -394,8 +396,9 @@ int parseCommandLine(int argc,char **argv,ParodusCfg * cfg)
 
       /* getopt_long stores the option index here. */
       int option_index = 0;
-      c = getopt_long (argc, argv, "m:s:f:d:r:n:b:u:t:o:i:l:p:e:D:j:a:k:c:T:w:J:46:C:S:K:M",
-				long_options, &option_index);
+      c = getopt_long (argc, argv, 
+			"m:s:f:d:r:n:b:u:t:o:i:l:p:e:D:j:a:k:c:T:w:J:46:C:S:R:K:M",
+			long_options, &option_index);
 
       /* Detect the end of the options. */
       if (c == -1)
@@ -538,17 +541,22 @@ int parseCommandLine(int argc,char **argv,ParodusCfg * cfg)
 	  ParodusInfo("connection_health_file is %s\n", cfg->connection_health_file);
 	  break;
 
+	case 'R':
+	  cfg->close_reason_file = strdup(optarg);
+	  ParodusInfo("sigterm_close_reason_file is %s\n", cfg->close_reason_file);
+	  break;
+
 	case 'C':
 	  cfg->crud_config_file = strdup(optarg);
 	  ParodusInfo("crud_config_file is %s\n", cfg->crud_config_file);
 	  break;
 
-		case 'P':
+	case 'P':
 		cfg->client_cert_path = strdup(optarg);
 		ParodusInfo("client_cert_path is %s\n", cfg->client_cert_path);
 		break;
 
-		case 'U':
+	case 'U':
 		cfg->token_server_url = strdup(optarg);
 		ParodusInfo("token_server_url is %s\n", cfg->token_server_url);
 		break;
@@ -637,6 +645,7 @@ void setDefaultValuesToCfg(ParodusCfg *cfg)
     ParodusPrint("cfg->webpa_uuid is :%s\n", cfg->webpa_uuid);
     cfg->crud_config_file = NULL;
     cfg->connection_health_file = NULL;
+    cfg->close_reason_file = NULL;
     cfg->client_cert_path = NULL;
     cfg->token_server_url = NULL;
 	
@@ -797,6 +806,15 @@ void loadParodusCfg(ParodusCfg * config,ParodusCfg *cfg)
         ParodusPrint("connection_health_file is NULL. set to empty\n");
     }
 
+    if(config->close_reason_file != NULL)
+    {
+        cfg->close_reason_file = strdup(config->close_reason_file);
+    }
+    else
+    {
+        ParodusPrint("close_reason_file is NULL. set to empty\n");
+    }
+
     if(config->crud_config_file != NULL)
     {
         cfg->crud_config_file = strdup(config->crud_config_file);

src/config.h

@@ -102,6 +102,7 @@ typedef struct
     char *client_cert_path;
     char *token_server_url;
     char *connection_health_file;
+    char *close_reason_file;
     char *mtls_client_key_path;
     char *mtls_client_cert_path;
 	char *crud_config_file;

src/conn_interface.c

@@ -33,6 +33,7 @@
 #include "spin_thread.h"
 #include "service_alive.h"
 #include "seshat_interface.h"
+#include "event_handler.h"
 #include "crud_interface.h"
 #include "heartBeat.h"
 #include "close_retry.h"
@@ -73,6 +74,7 @@ void createSocketConnection(void (* initKeypress)())
 {
     //ParodusCfg *tmpCfg = (ParodusCfg*)config_in;
     noPollCtx *ctx;
+    server_list_t server_list;  
     bool seshat_registered = false;
     int create_conn_rtn = 0;
     unsigned int webpa_ping_timeout_ms = 1000 * get_parodus_cfg()->webpa_ping_timeout;
@@ -95,9 +97,10 @@ void createSocketConnection(void (* initKeypress)())
     nopoll_log_set_handler (ctx, __report_log, NULL);
     #endif
 
-    start_conn_in_progress ();
-    create_conn_rtn = createNopollConnection(ctx);
-    stop_conn_in_progress ();
+    EventHandler();
+    
+    set_server_list_null (&server_list);
+    create_conn_rtn = createNopollConnection(ctx, &server_list);
     if(!create_conn_rtn)
     {
 		ParodusError("Unrecovered error, terminating the process\n");
@@ -148,6 +151,11 @@ void createSocketConnection(void (* initKeypress)())
                 OnboardLog("Reconnect detected, setting Ping_Miss reason for Reconnect\n");
                 set_global_reconnect_reason("Ping_Miss");
                 set_global_reconnect_status(true);
+		// Invoke the ping status change event callback as "missed" ping
+		if(NULL != on_ping_status_change)
+		{ 
+			on_ping_status_change("missed");
+		}
                 set_close_retry();
             }
             else
@@ -163,15 +171,17 @@ void createSocketConnection(void (* initKeypress)())
         if( false == seshat_registered ) {
             seshat_registered = __registerWithSeshat();
         }
+        
+        if (get_interface_down_event ())
+          if (0 != wait_while_interface_down ())
+            break;
 
         if(get_close_retry())
         {
             ParodusInfo("close_retry is %d, hence closing the connection and retrying\n", get_close_retry());
-            close_and_unref_connection(get_global_conn());
+            close_and_unref_connection(get_global_conn(), false);
             set_global_conn(NULL);
 
-            get_parodus_cfg()->cloud_status = CLOUD_STATUS_OFFLINE;
-            ParodusInfo("cloud_status set as %s after connection close\n", get_parodus_cfg()->cloud_status);
             if(get_parodus_cfg()->cloud_disconnect !=NULL)
             {
                 ParodusPrint("get_parodus_cfg()->cloud_disconnect is %s\n", get_parodus_cfg()->cloud_disconnect);
@@ -183,9 +193,7 @@ void createSocketConnection(void (* initKeypress)())
 		free(get_parodus_cfg()->cloud_disconnect);
 		reset_cloud_disconnect_reason(get_parodus_cfg());
             }
-            start_conn_in_progress ();
-            createNopollConnection(ctx);
-            stop_conn_in_progress ();
+            createNopollConnection(ctx, &server_list);
         }
        } while(!get_close_retry() && !g_shutdown);
 
@@ -211,13 +219,17 @@ void createSocketConnection(void (* initKeypress)())
 
     deleteAllClients ();
 
-    close_and_unref_connection(get_global_conn());
+    ParodusInfo ("reconnect reason at close %s\n", get_global_reconnect_reason());
+    close_and_unref_connection(get_global_conn(), true);
     nopoll_ctx_unref(ctx);
     nopoll_cleanup_library();
     curl_global_cleanup();
 }
 
-void shutdownSocketConnection(void) {
+void shutdownSocketConnection(char *reason) {
+   set_global_shutdown_reason (reason);
    g_shutdown = true;
+   reset_interface_down_event ();
+   terminate_backoff_delay ();
 }
 

src/conn_interface.h

@@ -45,7 +45,7 @@ extern UpStreamMsg *UpStreamMsgQ;
  *  and creates the intial connection and manages the connection wait, close mechanisms.
  */
 void createSocketConnection(void (* initKeypress)());
-void shutdownSocketConnection(void);
+void shutdownSocketConnection(char *reason);
    
 #ifdef __cplusplus
 }

src/connection.c

@@ -38,11 +38,34 @@
 
 #define HTTP_CUSTOM_HEADER_COUNT                    	5
 #define INITIAL_CJWT_RETRY                    	-2
+#define UPDATE_HEALTH_FILE_INTERVAL_SECS		240
+
+/* Close codes defined in RFC 6455, section 11.7. */
+enum {
+	CloseNormalClosure           = 1000,
+	CloseGoingAway               = 1001,
+	CloseProtocolError           = 1002,
+	CloseUnsupportedData         = 1003,
+	CloseNoStatus		     = 1005,
+	CloseAbnormalClosure         = 1006,
+	CloseInvalidFramePayloadData = 1007,
+	ClosePolicyViolation         = 1008,
+	CloseMessageTooBig           = 1009,
+	CloseMandatoryExtension      = 1010,
+	CloseInternalServerErr       = 1011,
+	CloseServiceRestart          = 1012,
+	CloseTryAgainLater           = 1013,
+	CloseTLSHandshake            = 1015
+};
 
 /*----------------------------------------------------------------------------*/
 /*                            File Scoped Variables                           */
 /*----------------------------------------------------------------------------*/
 
+pthread_mutex_t backoff_delay_mut=PTHREAD_MUTEX_INITIALIZER;
+pthread_cond_t backoff_delay_con=PTHREAD_COND_INITIALIZER;
+
+static char *shutdown_reason = SHUTDOWN_REASON_PARODUS_STOP;  /* goes in the close message */
 static char *reconnect_reason = "webpa_process_starts";
 static int cloud_disconnect_max_time = 5;
 static noPollConn *g_conn = NULL;
@@ -65,6 +88,16 @@ void set_global_conn(noPollConn *conn)
     g_conn = conn;
 }
 
+char *get_global_shutdown_reason()
+{
+    return shutdown_reason;
+}
+
+void set_global_shutdown_reason(char *reason)
+{
+    shutdown_reason = reason;
+}
+
 char *get_global_reconnect_reason()
 {
     return reconnect_reason;
@@ -199,23 +232,114 @@ void init_backoff_timer (backoff_timer_t *timer, int max_count)
   timer->count = 1;
   timer->max_count = max_count;
   timer->delay = 1;
+  clock_gettime (CLOCK_REALTIME, &timer->ts);
+  timer->start_time = time(NULL);
 }
 
-int update_backoff_delay (backoff_timer_t *timer)
+unsigned update_backoff_delay (backoff_timer_t *timer)
 {
   if (timer->count < timer->max_count) {
     timer->count += 1;
     timer->delay = timer->delay + timer->delay + 1;
     // 3,7,15,31 ..
   }
-  return timer->delay;
+  return (unsigned) timer->delay;
 }  
 
-static void backoff_delay (backoff_timer_t *timer)
+void add_timespec (struct timespec *t1, struct timespec *t2)
 {
-  update_backoff_delay (timer);
-  ParodusInfo("Waiting with backoffRetryTime %d seconds\n", timer->delay);
-  sleep (timer->delay);
+	t2->tv_sec += t1->tv_sec;
+	t2->tv_nsec += t1->tv_nsec;
+	if (t2->tv_nsec >= 1000000000) {
+	  t2->tv_sec += 1;
+	  t2->tv_nsec -= 1000000000;
+	}
+}
+
+unsigned calc_random_secs (int random_num, unsigned max_secs)
+{
+  unsigned delay_secs = (unsigned) random_num & max_secs;
+  if (delay_secs < 3)
+    return delay_secs + 3;
+  else
+    return delay_secs;
+}
+
+unsigned calc_random_nsecs (int random_num)
+{
+	/* random _num is in range 0..2147483648 */
+	unsigned n = (unsigned) random_num >> 1;
+	/* n is in range 0..1073741824 */
+	if (n < 1000000000)
+	  return n;
+	return n - 1000000000;
+}
+
+void calc_random_expiration (int random_num1, int random_num2, backoff_timer_t *timer, struct timespec *ts)
+{
+	unsigned max_secs = update_backoff_delay (timer); // 3,7,15,31
+	struct timespec ts_delay = {3, 0};
+	
+	if (max_secs > 3) {
+	  ts_delay.tv_sec = calc_random_secs (random_num1, max_secs);
+	  ts_delay.tv_nsec = calc_random_nsecs (random_num2);
+	}
+    ParodusInfo("Waiting max delay %u backoffRetryTime %ld secs %ld usecs\n", 
+      max_secs, ts_delay.tv_sec, ts_delay.tv_nsec/1000);
+      
+	/* Add delay to expire time */
+    add_timespec (&ts_delay, ts);
+}
+
+void terminate_backoff_delay (void)
+{
+	pthread_mutex_lock (&backoff_delay_mut);
+	pthread_cond_signal(&backoff_delay_con);
+	pthread_mutex_unlock (&backoff_delay_mut);
+}
+
+#define BACKOFF_ERR -1
+#define BACKOFF_SHUTDOWN   1
+#define BACKOFF_DELAY_TAKEN 0
+
+void start_conn_in_progress (unsigned long start_time);
+
+/* backoff_delay
+ * 
+ * delays for the number of seconds specified in parameter timer
+ * g_shutdown can break out of the delay.
+ *
+ * returns -1 pthread_cond_timedwait error
+ *  1   shutdown
+ *  0    delay taken
+*/  
+static int backoff_delay (backoff_timer_t *timer)
+{
+  struct timespec ts;
+  int rtn;
+
+  // periodically update the health file.
+  clock_gettime (CLOCK_REALTIME, &ts);
+  if ((ts.tv_sec - timer->ts.tv_sec) >= UPDATE_HEALTH_FILE_INTERVAL_SECS) {
+    start_conn_in_progress (timer->start_time);
+    timer->ts.tv_sec += UPDATE_HEALTH_FILE_INTERVAL_SECS;
+  }	  
+
+  calc_random_expiration (random(), random(), timer, &ts);
+
+  pthread_mutex_lock (&backoff_delay_mut);
+  // The condition variable will only be set if we shut down.
+  rtn = pthread_cond_timedwait (&backoff_delay_con, &backoff_delay_mut, &ts);
+  pthread_mutex_unlock (&backoff_delay_mut);
+
+  if (g_shutdown)
+    return BACKOFF_SHUTDOWN;
+  if ((rtn != 0) && (rtn != ETIMEDOUT)) {
+    ParodusError ("pthread_cond_timedwait error (%d) in backoff_delay.\n", rtn);
+    return BACKOFF_ERR;
+  }
+
+  return BACKOFF_DELAY_TAKEN;
 }  
 
 //--------------------------------------------------------------------
@@ -293,7 +417,7 @@ char *build_extra_hdrs (header_info_t *header_info)
 //--------------------------------------------------------------------
 void set_current_server (create_connection_ctx_t *ctx)
 {
-  ctx->current_server = get_current_server (&ctx->server_list);
+  ctx->current_server = get_current_server (ctx->server_list);
 }
 
 void free_extra_headers (create_connection_ctx_t *ctx)
@@ -319,7 +443,8 @@ void free_connection_ctx (create_connection_ctx_t *ctx)
 {
   free_extra_headers (ctx);
   free_header_info (&ctx->header_info);
-  free_server_list (&ctx->server_list);
+  if (NULL != ctx->server_list)
+    free_server_list (ctx->server_list);
 }
 
 
@@ -336,8 +461,10 @@ int find_servers (server_list_t *server_list)
 
   free_server_list (server_list);
   // parse default server URL
-  if (parse_server_url (get_parodus_cfg()->webpa_url, default_server) < 0)
-     return FIND_INVALID_DEFAULT;	// must have valid default url
+  if (parse_server_url (get_parodus_cfg()->webpa_url, default_server) < 0) {
+	ParodusError ("Invalid Default URL\n");
+    return FIND_INVALID_DEFAULT;
+  }
   ParodusInfo("default server_Address %s\n", default_server->server_addr);
   ParodusInfo("default port %u\n", default_server->port);
 #ifdef FEATURE_DNS_QUERY
@@ -355,7 +482,6 @@ int find_servers (server_list_t *server_list)
   return FIND_SUCCESS;
 }
 
-
 //--------------------------------------------------------------------
 // connect to current server
 int nopoll_connect (create_connection_ctx_t *ctx, int is_ipv6)
@@ -391,7 +517,7 @@ int nopoll_connect (create_connection_ctx_t *ctx, int is_ipv6)
    }
    if ((NULL == connection) && (!is_ipv6)) {
      if((checkHostIp(server->server_addr) == -2)) {
-       if (check_timer_expired (&ctx->connect_timer, 15*60*1000)) {
+       if (check_timer_expired (&ctx->connect_timer, 15*60*1000) && !get_interface_down_event()) {
   	 ParodusError("WebPA unable to connect due to DNS resolving to 10.0.0.1 for over 15 minutes; crashing service.\n");
 	 OnboardLog("WebPA unable to connect due to DNS resolving to 10.0.0.1 for over 15 minutes; crashing service.\n");
 	 OnboardLog("Reconnect detected, setting Dns_Res_webpa_reconnect reason for Reconnect\n");
@@ -417,7 +543,7 @@ int nopoll_connect (create_connection_ctx_t *ctx, int is_ipv6)
 
 int wait_connection_ready (create_connection_ctx_t *ctx)
 {
-  int wait_status;
+  int wait_status = 0;
   char *redirectURL = NULL;
 
   if(nopoll_conn_wait_for_status_until_connection_ready(get_global_conn(), 10, 
@@ -429,12 +555,12 @@ int wait_connection_ready (create_connection_ctx_t *ctx)
   if(wait_status == 307 || wait_status == 302 || wait_status == 303)    // only when there is a http redirect
   {
 	char *redirect_ptr = redirectURL;
-	ParodusError("Received temporary redirection response message %s\n", redirectURL);
+	ParodusInfo("Received temporary redirection response message %s\n", redirectURL);
 	// Extract server Address and port from the redirectURL
 	if (strncmp (redirect_ptr, "Redirect:", 9) == 0)
 	    redirect_ptr += 9;
-	free_server (&ctx->server_list.redirect);
-	if (parse_server_url (redirect_ptr, &ctx->server_list.redirect) < 0) {
+	free_server (&ctx->server_list->redirect);
+	if (parse_server_url (redirect_ptr, &ctx->server_list->redirect) < 0) {
 	  ParodusError ("Redirect url error %s\n", redirectURL);
   	  free (redirectURL);
 	  return WAIT_FAIL;
@@ -463,7 +589,7 @@ int wait_connection_ready (create_connection_ctx_t *ctx)
 // Return codes for connect_and_wait
 #define CONN_WAIT_SUCCESS	 0
 #define CONN_WAIT_ACTION_RETRY	 1	// if wait_status is 307, 302, 303, or 403
-#define CONN_WAIT_RETRY_DNS 	 2
+#define CONN_WAIT_FAIL 	 2
 
 int connect_and_wait (create_connection_ctx_t *ctx)
 {
@@ -494,7 +620,7 @@ int connect_and_wait (create_connection_ctx_t *ctx)
     } // nopoll_connected
     
     if (nopoll_connected) {
-	close_and_unref_connection(get_global_conn());
+      close_and_unref_connection(get_global_conn(), false);
 	set_global_conn(NULL);
     }
 
@@ -507,7 +633,7 @@ int connect_and_wait (create_connection_ctx_t *ctx)
       continue;
     }
     
-    return CONN_WAIT_RETRY_DNS;
+    return CONN_WAIT_FAIL;
   }
 }
 
@@ -520,7 +646,7 @@ int keep_trying_to_connect (create_connection_ctx_t *ctx,
 {
     int rtn;
     
-    while (true)
+    while (!g_shutdown)
     {
       set_extra_headers (ctx);
 
@@ -530,13 +656,51 @@ int keep_trying_to_connect (create_connection_ctx_t *ctx,
 
       if (rtn == CONN_WAIT_ACTION_RETRY) // if redirected or build_headers
         continue;
-      backoff_delay (backoff_timer); // 3,7,15,31 ..
-      if (rtn == CONN_WAIT_RETRY_DNS)
-        return false;  //find_server again
+      // If interface down event is set, stop retry
+      // and wait till interface is up again.
+      if(get_interface_down_event()) {
+	    if (0 != wait_while_interface_down())
+	      return false;
+	    start_conn_in_progress(backoff_timer->start_time);
+	    ParodusInfo("Interface is back up, re-initializing the convey header\n");
+	    // Reset the reconnect reason by initializing the convey header again
+	    ((header_info_t *)(&ctx->header_info))->conveyHeader = getWebpaConveyHeader();
+	    ParodusInfo("Received reconnect_reason as:%s\n", reconnect_reason);  
+      } else { 
+        if (backoff_delay (backoff_timer) // 3,7,15,31 ..
+              != BACKOFF_DELAY_TAKEN) // shutdown or cond wait error
+          return false;
+      }
+      if (rtn == CONN_WAIT_FAIL) {
+        return false;
+      }
       // else retry
     }
+    return false;
 }
 
+int wait_while_interface_down()
+
+{
+	int rtn;
+	
+	ParodusError("Interface is down, hence waiting until its up\n");
+	close_and_unref_connection (get_global_conn(), false);
+	set_global_conn(NULL);
+
+	while (get_interface_down_event ()) {
+	  pthread_mutex_lock(get_interface_down_mut());
+  	  rtn = pthread_cond_wait(get_interface_down_con(), get_interface_down_mut());
+      pthread_mutex_unlock (get_interface_down_mut());
+  	  if (rtn != 0)
+  	    ParodusError 
+  	      ("Error on pthread_cond_wait (%d) in wait_while_interface_down\n", rtn);
+  	  if ((rtn != 0) || g_shutdown) {
+	    return -1;
+	  }
+	}
+    return 0;
+}
 
 //--------------------------------------------------------------------
 
@@ -544,11 +708,10 @@ int keep_trying_to_connect (create_connection_ctx_t *ctx,
  * @brief createNopollConnection interface to create WebSocket client connections.
  *Loads the WebPA config file and creates the intial connection and manages the connection wait, close mechanisms.
  */
-int createNopollConnection(noPollCtx *ctx)
+int createNopollConnection(noPollCtx *ctx, server_list_t *server_list)
 {
   create_connection_ctx_t conn_ctx;
   int max_retry_count;
-  int query_dns_status;
   struct timespec connect_time,*connectTimePtr;
   connectTimePtr = &connect_time;
   backoff_timer_t backoff_timer;
@@ -568,18 +731,33 @@ int createNopollConnection(noPollCtx *ctx)
 	conn_ctx.nopoll_ctx = ctx;
 	init_expire_timer (&conn_ctx.connect_timer);
 	init_header_info (&conn_ctx.header_info);
-        set_server_list_null (&conn_ctx.server_list);
-        init_backoff_timer (&backoff_timer, max_retry_count);
+	/* look up server information if we don't already have it */
+	if (server_is_null (&server_list->defaults))
+	  if (find_servers (server_list) == FIND_INVALID_DEFAULT) {
+        return nopoll_false;		  
+      }
+	conn_ctx.server_list = server_list;
+    init_backoff_timer (&backoff_timer, max_retry_count);
+    start_conn_in_progress (backoff_timer.start_time); 
   
 	while (!g_shutdown)
 	{
-	  query_dns_status = find_servers (&conn_ctx.server_list);
-	  if (query_dns_status == FIND_INVALID_DEFAULT)
-		return nopoll_false;
 	  set_current_server (&conn_ctx);
-	  if (keep_trying_to_connect (&conn_ctx, &backoff_timer))
+	  if (keep_trying_to_connect (&conn_ctx, &backoff_timer)) {
+		// Don't reuse the redirect server during reconnect
+		free_server (&conn_ctx.server_list->redirect);
 		break;
-	  // retry dns query
+	  }
+	  /* if we failed to connect, don't reuse the redirect server */	
+      free_server (&conn_ctx.server_list->redirect);
+#ifdef FEATURE_DNS_QUERY
+      /* if we don't already have a valid jwt, look up server information */
+      if (server_is_null (&conn_ctx.server_list->jwt))
+        if (find_servers (conn_ctx.server_list) == FIND_INVALID_DEFAULT) {
+		  /* since we already found a default server, we don't expect FIND_INVALID_DEFAULT */
+		  g_shutdown = true;
+	    }
+#endif		
 	}
       
 	if(conn_ctx.current_server->allow_insecure <= 0)
@@ -596,6 +774,12 @@ int createNopollConnection(noPollCtx *ctx)
 	get_parodus_cfg()->cloud_status = CLOUD_STATUS_ONLINE;
 	ParodusInfo("cloud_status set as %s after successful connection\n", get_parodus_cfg()->cloud_status);
 
+	// Invoke the ping status change event callback as "received" ping
+	if(NULL != on_ping_status_change)
+	{
+		on_ping_status_change("received");
+	}
+
 	if((get_parodus_cfg()->boot_time != 0) && init) {
 		getCurrentTime(connectTimePtr);
 		ParodusInfo("connect_time-diff-boot_time=%d\n", connectTimePtr->tv_sec - get_parodus_cfg()->boot_time);
@@ -603,8 +787,7 @@ int createNopollConnection(noPollCtx *ctx)
 	}
 
 	free_extra_headers (&conn_ctx);
-        free_header_info (&conn_ctx.header_info);
-        free_server_list (&conn_ctx.server_list);
+	free_header_info (&conn_ctx.header_info);
         
 	// Reset close_retry flag and heartbeatTimer once the connection retry is successful
 	ParodusPrint("createNopollConnection(): reset_close_retry\n");
@@ -614,7 +797,7 @@ int createNopollConnection(noPollCtx *ctx)
 	set_global_reconnect_status(false);
 	ParodusPrint("LastReasonStatus reset after successful connection\n");
 	setMessageHandlers();
-
+    stop_conn_in_progress ();
 	return nopoll_true;
 }          
 
@@ -664,18 +847,47 @@ static noPollConnOpts * createConnOpts (char * extra_headers, bool secure)
 	return opts;   
 }
 
-
-void close_and_unref_connection(noPollConn *conn)
+static void close_conn ( noPollConn *conn, bool is_shutting_down)
+{
+	const char *effective_reason = get_global_shutdown_reason();
+	
+	if (NULL == effective_reason) {
+          effective_reason = SHUTDOWN_REASON_SYSTEM_RESTART;
+    }
+    else if (strcmp (effective_reason, SHUTDOWN_REASON_SIGTERM) == 0) {
+		char *sigterm_reason;
+		char *reason_file = get_parodus_cfg()->close_reason_file;
+        if ((NULL != reason_file) && readFromFile (reason_file, &sigterm_reason) &&
+            (strlen(sigterm_reason) != 0)) 
+        {
+          nopoll_conn_close_ext(conn, CloseNormalClosure, sigterm_reason, 
+			strlen (sigterm_reason));
+		  ParodusInfo ("Closed by SIGTERM, reason: %s\n", sigterm_reason);
+		  if (is_shutting_down)
+            ParodusInfo ("shutdown reason at close %s\n", sigterm_reason); 
+		  free (sigterm_reason);
+		  return;
+		}
+		/* file could not be read. use canned message. */
+        effective_reason = SHUTDOWN_REASON_SYSTEM_RESTART;
+        ParodusError ("Closed by SIGTERM, but no reason file\n");
+    }
+    nopoll_conn_close_ext(conn, CloseNormalClosure, effective_reason, 
+      strlen (effective_reason)); 
+    if (is_shutting_down)
+      ParodusInfo ("shutdown reason at close %s\n", effective_reason); 
+}    		
+		
+void close_and_unref_connection(noPollConn *conn, bool is_shutting_down)
 {
     if (conn) {
-        nopoll_conn_close(conn);
-        if (0 < nopoll_conn_ref_count (conn)) {
-            nopoll_conn_unref(conn);
-        }
+      close_conn (conn, is_shutting_down);
+      get_parodus_cfg()->cloud_status = CLOUD_STATUS_OFFLINE;
+      ParodusInfo("cloud_status set as %s after connection close\n", get_parodus_cfg()->cloud_status);
     }
 }
 
-void write_conn_in_prog_file (const char *msg)
+void write_conn_in_prog_file (bool is_starting, unsigned long start_time)
 {
   int fd;
   FILE *fp;
@@ -696,17 +908,27 @@ void write_conn_in_prog_file (const char *msg)
     return;
   }
   timestamp = (unsigned long) time(NULL);
-  fprintf (fp, "{%s=%lu}\n", msg, timestamp);
+  if (is_starting)
+    fprintf (fp, "{START=%lu,%lu}\n", start_time, timestamp);
+  else
+    fprintf (fp, "{STOP=%lu}\n", timestamp);
+  
   fclose (fp);
 }
 
-void start_conn_in_progress (void)
+void start_conn_in_progress (unsigned long start_time)
 {
-  write_conn_in_prog_file ("START");
+  write_conn_in_prog_file (true, start_time);
 }   
 
 void stop_conn_in_progress (void)
 {
-  write_conn_in_prog_file ("STOP");
+  write_conn_in_prog_file (false, 0);
 }   
 
+
+void registerParodusOnPingStatusChangeHandler(parodusOnPingStatusChangeHandler callback_func)
+{
+	on_ping_status_change = callback_func;
+}
+

src/connection.h

@@ -33,22 +33,37 @@ extern "C" {
 /*----------------------------------------------------------------------------*/
 /*                            File Scoped Variables                           */
 /*----------------------------------------------------------------------------*/
+#define SHUTDOWN_REASON_PARODUS_STOP    "parodus_stopping"
+#define SHUTDOWN_REASON_SYSTEM_RESTART  "system_restarting"
+#define SHUTDOWN_REASON_SIGTERM 		"SIGTERM"
 
+/**
+* parodusOnPingStatusChangeHandler - Function pointer
+* Used to define callback function to do additional processing 
+* when websocket Ping status change event 
+* i.e. ping_miss or ping receive after miss
+*/
+typedef void (*parodusOnPingStatusChangeHandler) (char * status);
+parodusOnPingStatusChangeHandler on_ping_status_change;
 
 /*----------------------------------------------------------------------------*/
 /*                             Function Prototypes                            */
 /*----------------------------------------------------------------------------*/
 
-int createNopollConnection(noPollCtx *);
+void set_server_list_null (server_list_t *server_list);
+int createNopollConnection(noPollCtx *, server_list_t *);
 
 /**
  * @brief Interface to terminate WebSocket client connections and clean up resources.
  */
-void close_and_unref_connection(noPollConn *);
+void close_and_unref_connection(noPollConn *conn, bool is_shutting_down);
 
 noPollConn *get_global_conn(void);
 void set_global_conn(noPollConn *);
 
+char *get_global_shutdown_reason();
+void set_global_shutdown_reason(char *reason);
+
 char *get_global_reconnect_reason();
 void set_global_reconnect_reason(char *reason);
 
@@ -61,9 +76,17 @@ void set_cloud_disconnect_time(int disconnTime);
 /**
  * @brief Interface to self heal connection in progress getting stuck
  */
-void start_conn_in_progress (void);
+void start_conn_in_progress (unsigned long start_time);
 void stop_conn_in_progress (void);
 
+// To Register parodusOnPingStatusChangeHandler Callback function
+void registerParodusOnPingStatusChangeHandler(parodusOnPingStatusChangeHandler on_ping_status_change);
+
+// To stop connection and wait duing interface down state
+int wait_while_interface_down (void);
+
+void terminate_backoff_delay (void);
+
 #ifdef __cplusplus
 }
 #endif

src/crud_internal.c

@@ -58,22 +58,7 @@ int writeToJSON(char *data)
 
 int readFromJSON(char **data)
 {
-	FILE *fp;
-	int ch_count = 0;
-	fp = fopen(get_parodus_cfg()->crud_config_file, "r+");
-	if (fp == NULL)
-	{
-		ParodusError("Failed to open file %s\n", get_parodus_cfg()->crud_config_file);
-		return 0;
-	}
-	fseek(fp, 0, SEEK_END);
-	ch_count = ftell(fp);
-	fseek(fp, 0, SEEK_SET);
-	*data = (char *) malloc(sizeof(char) * (ch_count + 1));
-	fread(*data, 1, ch_count,fp);
-	(*data)[ch_count] ='\0';
-	fclose(fp);
-	return 1;
+	return readFromFile (get_parodus_cfg()->crud_config_file, data);
 }
 /*
 *	@res_obj 	json object to add it in crud config json file

src/event_handler.c

@@ -0,0 +1,26 @@
+/*
+ * If not stated otherwise in this file or this component's Licenses.txt file the
+ * following copyright and licenses apply:
+ *
+ * Copyright 2016 RDK Management
+ * Copyright [2014] [Cisco Systems, Inc.]
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+/* This is a stub file that will be overridden in a patch */
+
+#include "event_handler.h"
+
+void EventHandler()
+{
+}

src/event_handler.h

@@ -0,0 +1,21 @@
+/*
+ * If not stated otherwise in this file or this component's Licenses.txt file the
+ * following copyright and licenses apply:
+ *
+ * Copyright 2016 RDK Management
+ * Copyright [2014] [Cisco Systems, Inc.]
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+void EventHandler();
+#define LOGGING_INTERVAL_SECS    ( 60 * 60 )

src/main.c

@@ -19,14 +19,14 @@
 #include "stdlib.h"
 #include "config.h"
 #include "auth_token.h"
+#include "connection.h"
 #include "conn_interface.h"
 #include "parodus_log.h"
 #include <curl/curl.h>
 #ifdef INCLUDE_BREAKPAD
 #include "breakpad_wrapper.h"
-#else
-#include "signal.h"
 #endif
+#include "signal.h"
 
 /*----------------------------------------------------------------------------*/
 /*                                   Macros                                   */
@@ -36,7 +36,7 @@
 /*----------------------------------------------------------------------------*/
 /*                               Data Structures                              */
 /*----------------------------------------------------------------------------*/
-/* none */
+typedef    void    Sigfunc(int);
 
 /*----------------------------------------------------------------------------*/
 /*                            File Scoped Variables                           */
@@ -46,33 +46,59 @@
 /*----------------------------------------------------------------------------*/
 /*                             Function Prototypes                            */
 /*----------------------------------------------------------------------------*/
-#ifndef INCLUDE_BREAKPAD
 static void sig_handler(int sig);
+
+Sigfunc *
+signal (int signo, Sigfunc *func)
+{
+    struct sigaction act, oact;
+
+    act.sa_handler = func;
+    sigemptyset (&act.sa_mask);
+    act.sa_flags = 0;
+    if (signo == SIGALRM) {
+#ifdef  SA_INTERRUPT
+        act.sa_flags |= SA_INTERRUPT;     /* SunOS 4.x */
 #endif
+    } else {
+#ifdef  SA_RESTART
+        act.sa_flags |= SA_RESTART; /* SVR4, 4.4BSD */
+#endif
+    }
+    if (sigaction (signo, &act, &oact) < 0) {
+	ParodusError ("Signal Handler for signal %d not installed!\n", signo);
+        return (SIG_ERR);
+    }
+    return (oact.sa_handler);
+}
 
 /*----------------------------------------------------------------------------*/
 /*                             External Functions                             */
 /*----------------------------------------------------------------------------*/
 int main( int argc, char **argv)
 {
-#ifdef INCLUDE_BREAKPAD
-    breakpad_ExceptionHandler();
-#else
+    set_global_shutdown_reason (SHUTDOWN_REASON_PARODUS_STOP);
     signal(SIGTERM, sig_handler);
-	signal(SIGINT, sig_handler);
+    signal(SIGINT, sig_handler);
 	signal(SIGUSR1, sig_handler);
 	signal(SIGUSR2, sig_handler);
+	signal(SIGQUIT, sig_handler);
+	signal(SIGHUP, sig_handler);   
+	signal(SIGALRM, sig_handler);
+#ifdef INCLUDE_BREAKPAD
+    /* breakpad handles the signals SIGSEGV, SIGBUS, SIGFPE, and SIGILL */
+    breakpad_ExceptionHandler();
+#else
 	signal(SIGSEGV, sig_handler);
 	signal(SIGBUS, sig_handler);
-	signal(SIGKILL, sig_handler);
 	signal(SIGFPE, sig_handler);
 	signal(SIGILL, sig_handler);
-	signal(SIGQUIT, sig_handler);
-	signal(SIGHUP, sig_handler);
-	signal(SIGALRM, sig_handler);
 #endif	
     ParodusCfg *cfg;
 
+	ParodusInfo ("RAND_MAX is %ld (0x%lx)\n", RAND_MAX, RAND_MAX);
+	srandom (getpid());
+
     /* TODO not ideal, but it fixes a more major problem for now. */
     cfg = get_parodus_cfg();
     memset(cfg,0,sizeof(ParodusCfg));
@@ -97,7 +123,6 @@ const char *rdk_logger_module_fetch(void)
 /*----------------------------------------------------------------------------*/
 /*                             Internal functions                             */
 /*----------------------------------------------------------------------------*/
-#ifndef INCLUDE_BREAKPAD
 static void sig_handler(int sig)
 {
 
@@ -105,17 +130,25 @@ static void sig_handler(int sig)
 	{
 		signal(SIGINT, sig_handler); /* reset it to this function */
 		ParodusInfo("SIGINT received!\n");
-		shutdownSocketConnection();
+		shutdownSocketConnection(SHUTDOWN_REASON_PARODUS_STOP);
 	}
 	else if ( sig == SIGUSR1 ) 
 	{
 		signal(SIGUSR1, sig_handler); /* reset it to this function */
 		ParodusInfo("SIGUSR1 received!\n");
+		shutdownSocketConnection(SHUTDOWN_REASON_SYSTEM_RESTART);
 	}
 	else if ( sig == SIGUSR2 ) 
 	{
+		signal(SIGUSR2, sig_handler); /* reset it to this function */
 		ParodusInfo("SIGUSR2 received!\n");
 	}
+	else if ( sig == SIGTERM ) 
+	{
+		signal(SIGTERM, sig_handler); /* reset it to this function */
+		ParodusInfo("SIGTERM received!\n");
+		shutdownSocketConnection(SHUTDOWN_REASON_SIGTERM);
+	}
 	else if ( sig == SIGCHLD ) 
 	{
 		signal(SIGCHLD, sig_handler); /* reset it to this function */
@@ -134,8 +167,7 @@ static void sig_handler(int sig)
 	else 
 	{
 		ParodusInfo("Signal %d received!\n", sig);
-		shutdownSocketConnection();
+		shutdownSocketConnection(SHUTDOWN_REASON_PARODUS_STOP);
 	}
 	
 }
-#endif

src/nopoll_helpers.c

@@ -26,6 +26,7 @@
 #include "nopoll_helpers.h"
 #include "nopoll_handlers.h"
 #include "time.h"
+#include "config.h"
 
 /*----------------------------------------------------------------------------*/
 /*                                   Macros                                   */
@@ -49,59 +50,33 @@ void setMessageHandlers()
     nopoll_conn_set_on_close(get_global_conn(), (noPollOnCloseHandler)listenerOnCloseMessage, NULL);
 }
 
+static int cloud_status_is_online (void)
+{
+	const char *status = get_parodus_cfg()->cloud_status;
+	if (NULL == status)
+	  return false;
+	return (strcmp (status, CLOUD_STATUS_ONLINE) == 0);
+}
+
 /** To send upstream msgs to server ***/
 
 void sendMessage(noPollConn *conn, void *msg, size_t len)
 {
     int bytesWritten = 0;
-    static int connErr=0;
-    long timeDiff = 0;
+
+    if (!cloud_status_is_online ()) {
+        ParodusError("Failed to send msg upstream as connection is not OK\n");
+        OnboardLog("Failed to send msg upstream as connection is not OK\n");
+		return;
+	}
 
     ParodusInfo("sendMessage length %zu\n", len);
 
-    if(nopoll_conn_is_ok(conn) && nopoll_conn_is_ready(conn))
+    bytesWritten = sendResponse(conn, msg, len);
+    ParodusPrint("Number of bytes written: %d\n", bytesWritten);
+    if (bytesWritten != (int) len) 
     {
-        //bytesWritten = nopoll_conn_send_binary(conn, msg, len);
-        bytesWritten = sendResponse(conn, msg, len);
-        ParodusPrint("Number of bytes written: %d\n", bytesWritten);
-        if (bytesWritten != (int) len) 
-        {
-            ParodusError("Failed to send bytes %zu, bytes written were=%d (errno=%d, %s)..\n", len, bytesWritten, errno, strerror(errno));
-        }
-	connErr = 0;
-    }
-    else
-    {
-                ParodusError("Failed to send msg upstream as connection is not OK\n");
-                OnboardLog("Failed to send msg upstream as connection is not OK\n");
-
-                if(get_interface_down_event())
-                {     
-                     ParodusError("Unable to connect to server since interface is down\n");
-                }
-                else
-                {
-			if (connErr == 0)
-			{
-				getCurrentTime(connStuck_startPtr);
-				ParodusInfo("Conn got stuck, initialized the first timer\n");
-				connErr = 1;
-			}
-			else
-			{
-				getCurrentTime(connStuck_endPtr);
-				timeDiff = timeValDiff(connStuck_startPtr, connStuck_endPtr);
-				ParodusPrint("checking timeout difference:%ld\n", timeDiff);
-
-				if( timeDiff >= (10*60*1000))
-				{
-					ParodusError("conn got stuck for over 10 minutes; crashing service.\n");
-					OnboardLog("conn got stuck for over 10 minutes; crashing service.\n");
-					kill(getpid(),SIGTERM);
-				}
-
-			}
-                 }
+        ParodusError("Failed to send bytes %zu, bytes written were=%d (errno=%d, %s)..\n", len, bytesWritten, errno, strerror(errno));
     }
 }
 

tests/CMakeLists.txt

@@ -16,7 +16,9 @@ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -W -g -fprofile-arcs -ftest-coverage
 set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DTEST ")
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -W  -g -fprofile-arcs -ftest-coverage -O0")
 set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage -O0")
-set (PARODUS_COMMON_SRC ../src/string_helpers.c ../src/mutex.c ../src/time.c ../src/config.c ../src/spin_thread.c ../src/token.c)
+set (PARODUS_COMMON_SRC ../src/string_helpers.c ../src/mutex.c 
+  ../src/time.c ../src/config.c ../src/spin_thread.c 
+  ../src/event_handler.c ../src/token.c)
 set (PARODUS_COMMON_LIBS gcov -lcunit -lcimplog -lwrp-c
  -luuid -lmsgpackc -lnopoll -lnanomsg -lpthread
  -Wl,--no-as-needed -lcjson -lcjwt -ltrower-base64
@@ -222,7 +224,8 @@ target_link_libraries (test_crud_tasks -lcmocka ${PARODUS_COMMON_LIBS} )
 #   test_crud_internal
 #-------------------------------------------------------------------------------
 add_test(NAME test_crud_internal COMMAND ${MEMORY_CHECK} ./test_crud_internal)
-add_executable(test_crud_internal test_crud_internal.c ../src/config.c ../src/close_retry.c ../src/string_helpers.c ../src/crud_internal.c )
+add_executable(test_crud_internal test_crud_internal.c ../src/config.c ../src/close_retry.c 
+  ../src/ParodusInternal.c ../src/string_helpers.c ../src/crud_internal.c )
 target_link_libraries (test_crud_internal -lcmocka ${PARODUS_COMMON_LIBS} -lcurl -luuid)
 
 #-------------------------------------------------------------------------------
@@ -283,6 +286,7 @@ set(CONIFC_SRC test_conn_interface.c
   ../src/mutex.c
   ../src/heartBeat.c
   ../src/close_retry.c
+  ../src/event_handler.c
 )
 if (ENABLE_SESHAT)
 set(CONIFC_SRC ${CONIFC_SRC} ../src/seshat_interface.c)
@@ -320,7 +324,7 @@ set(TOKEN_SRC ../src/conn_interface.c ../src/config.c
  ../src/thread_tasks.c ../src/time.c
  ../src/string_helpers.c ../src/mutex.c 
  ../src/token.c ../src/heartBeat.c
- ../src/close_retry.c
+ ../src/close_retry.c ../src/event_handler.c
  )
 
 if (ENABLE_SESHAT)

tests/mock_event_handler.c

@@ -0,0 +1,104 @@
+/*
+ * If not stated otherwise in this file or this component's Licenses.txt file the
+ * following copyright and licenses apply:
+ *
+ * Copyright 2016 RDK Management
+ * Copyright [2014] [Cisco Systems, Inc.]
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+/* This is a test version of event_handler.c that can be used
+ * to simulate interface down, interface up event
+ * You overwrite event_handler.c in the src diectory with this
+ * version.  It will generate interface down / interface up events
+ * at random intervals between 60 secs and 124 secs
+ */
+
+#include "parodus_log.h"
+#include "event_handler.h"
+#include "connection.h"
+#include "config.h"
+#include "heartBeat.h"
+#include <stdlib.h>
+#include <string.h>
+#include <pthread.h>
+#include <unistd.h>
+#include "time.h"
+#include "close_retry.h"
+
+extern bool g_shutdown;
+
+static pthread_t sysevent_tid;
+
+static void start_interface_down (void)
+{
+	  	set_interface_down_event();
+		ParodusInfo("Interface_down_event is set\n");				
+		pause_heartBeatTimer();
+}
+
+static void end_interface_down (void)
+{
+		reset_interface_down_event();
+		ParodusInfo("Interface_down_event is reset\n");
+		resume_heartBeatTimer();
+		set_close_retry();
+}
+
+// waits from 60 to 124 secs
+int wait_random (const char *msg)
+{
+  #define HALF_SEC 500000l
+  long delay = (random() >> 5) + 60000000l;
+  long secs, usecs;
+  struct timeval timeout;
+
+  secs = delay / 1000000;
+  usecs = delay % 1000000;
+  ParodusInfo ("Waiting %ld secs %ld usecs for %s\n", secs, usecs, msg);
+  
+  while (!g_shutdown) {
+	timeout.tv_sec = 0;
+	if (delay <= HALF_SEC) {
+	  timeout.tv_usec = delay;
+	  select (0, NULL, NULL, NULL, &timeout);
+	  return 0;
+	}
+	timeout.tv_usec = HALF_SEC;
+	delay -= HALF_SEC;
+	select (0, NULL, NULL, NULL, &timeout);
+  }
+  return -1;    
+}
+
+
+static void *parodus_sysevent_handler (void *data)
+{
+	while (!g_shutdown) {
+	  if (wait_random ("interface down") != 0)
+	    break;
+	  start_interface_down ();
+	  wait_random ("interface up");
+	  end_interface_down ();	
+	}
+	ParodusInfo ("Exiting event handler\n");
+	return data;
+}
+
+void EventHandler()
+{
+	ParodusInfo ("RAND_MAX is %ld (0x%lx)\n", RAND_MAX, RAND_MAX);
+	srandom (getpid());
+
+	pthread_create(&sysevent_tid, NULL, parodus_sysevent_handler, NULL);
+}

tests/test_auth_token.c

@@ -54,12 +54,21 @@ int curl_easy_perform(CURL *curl)
 	  write_callback_fn (msg, 1, strlen(msg), &test_data);
 	return rtn;
 }
+int g_response_code=0;
+void setGlobalResponseCode (int response_code)
+{
+	g_response_code = response_code;
+}
 
 int curl_easy_getinfo(CURL *curl, CURLINFO CURLINFO_RESPONSE_CODE, long response_code)
 {
 	UNUSED(curl);
 	UNUSED(CURLINFO_RESPONSE_CODE);
-	UNUSED(response_code);
+	if (0 != g_response_code)
+	{
+		response_code= g_response_code;
+		ParodusInfo("response_code is %ld\n", response_code);
+	}
 	function_called();
 	return (int) mock();
 }
@@ -166,6 +175,7 @@ void test_requestNewAuthToken ()
 
   test_data.size = 0;
   test_data.data = token;
+  setGlobalResponseCode(200);
   will_return (curl_easy_perform, 0);
   expect_function_calls (curl_easy_perform, 1);
   will_return (curl_easy_getinfo, 0);
@@ -216,6 +226,7 @@ void test_getAuthToken ()
 	/* To test curl success case */
 	test_data.size = 0;
 	test_data.data = cfg.webpa_auth_token;
+	setGlobalResponseCode(200);
 	will_return (curl_easy_perform, 0);
 	expect_function_calls (curl_easy_perform, 1);
 
@@ -313,6 +324,86 @@ void test_write_callback_fn ()
 	free(data.data);
 }
 
+void test_requestNewAuthToken_non200 ()
+{
+  char token[1024];
+  ParodusCfg cfg;
+  int output;
+  memset(&cfg,0,sizeof(cfg));
+
+  cfg.token_server_url = strdup("https://dev.comcast.net/token");
+  parStrncpy(cfg.cert_path , "/etc/ssl/certs/ca-certificates.crt", sizeof(cfg.cert_path));
+  parStrncpy(cfg.webpa_interface_used , "eth0", sizeof(cfg.webpa_interface_used));
+  parStrncpy(cfg.hw_serial_number, "Fer23u948590", sizeof(cfg.hw_serial_number));
+  parStrncpy(cfg.hw_mac , "123567892366", sizeof(cfg.hw_mac));
+  set_parodus_cfg(&cfg);
+
+  test_data.size = 0;
+  test_data.data = token;
+  setGlobalResponseCode(404);
+  will_return (curl_easy_perform, 0);
+  expect_function_calls (curl_easy_perform, 1);
+  will_return (curl_easy_getinfo, 0);
+  expect_function_calls (curl_easy_getinfo, 1);
+
+  will_return (curl_easy_getinfo, 0);
+  expect_function_calls (curl_easy_getinfo, 1);
+
+  output = requestNewAuthToken (token, sizeof(token), 1);
+  assert_int_equal (output, -1);
+  assert_string_equal (token, "");
+  ParodusInfo("requestNewAuthToken output: %d token empty len: %lu\n", output, strlen(token));
+  free(cfg.token_server_url);
+}
+
+void test_getAuthTokenFailure_non200 ()
+{
+	ParodusCfg cfg;
+	memset(&cfg,0,sizeof(cfg));
+
+	cfg.token_server_url = strdup("https://dev.comcast.net/token");
+	cfg.client_cert_path = strdup("testcert");
+	parStrncpy(cfg.cert_path , "/etc/ssl/certs/ca-certificates.crt", sizeof(cfg.cert_path));
+	parStrncpy(cfg.webpa_interface_used , "eth0", sizeof(cfg.webpa_interface_used));
+	parStrncpy(cfg.hw_serial_number, "Fer23u948590", sizeof(cfg.hw_serial_number));
+	parStrncpy(cfg.hw_mac , "123567892366", sizeof(cfg.hw_mac));
+	set_parodus_cfg(&cfg);
+
+	will_return (curl_easy_perform, 0);
+	expect_function_calls (curl_easy_perform, 1);
+
+	will_return (curl_easy_getinfo, 0);
+	expect_function_calls (curl_easy_getinfo, 1);
+
+	will_return (curl_easy_getinfo, 0);
+	expect_function_calls (curl_easy_getinfo, 1);
+
+	will_return (curl_easy_perform, 0);
+	expect_function_calls (curl_easy_perform, 1);
+
+	will_return (curl_easy_getinfo, 0);
+	expect_function_calls (curl_easy_getinfo, 1);
+
+	will_return (curl_easy_getinfo, 0);
+	expect_function_calls (curl_easy_getinfo, 1);
+
+	will_return (curl_easy_perform, 0);
+	expect_function_calls (curl_easy_perform, 1);
+
+	will_return (curl_easy_getinfo, 0);
+	expect_function_calls (curl_easy_getinfo, 1);
+
+	will_return (curl_easy_getinfo, 0);
+	expect_function_calls (curl_easy_getinfo, 1);
+	setGlobalResponseCode(504);
+	getAuthToken(&cfg);
+
+	assert_string_equal( cfg.webpa_auth_token, "");
+
+	free(cfg.client_cert_path);
+	free(cfg.token_server_url);
+}
+
 /*----------------------------------------------------------------------------*/
 /*                             External Functions                             */
 /*----------------------------------------------------------------------------*/
@@ -328,6 +419,8 @@ int main(void)
 	cmocka_unit_test(getAuthToken_MacNull),
 	cmocka_unit_test(test_getAuthToken),
 	cmocka_unit_test(test_getAuthTokenFailure),
+	cmocka_unit_test(test_requestNewAuthToken_non200),
+	cmocka_unit_test(test_getAuthTokenFailure_non200)
     };
 
     return cmocka_run_group_tests(tests, NULL, NULL);

tests/test_conn_interface.c

@@ -33,6 +33,7 @@
 /*----------------------------------------------------------------------------*/
 /*                            File Scoped Variables                           */
 /*----------------------------------------------------------------------------*/
+static char *reconnect_reason = "webpa_process_starts";
 UpStreamMsg *UpStreamMsgQ;
 ParodusMsg *ParodusMsgQ;
 pthread_mutex_t g_mutex=PTHREAD_MUTEX_INITIALIZER;
@@ -46,9 +47,20 @@ pthread_cond_t svc_con=PTHREAD_COND_INITIALIZER;
 /*----------------------------------------------------------------------------*/
 /*                                   Mocks                                    */
 /*----------------------------------------------------------------------------*/
-int createNopollConnection(noPollCtx *ctx)
+void set_server_list_null (server_list_t *server_list)
 {
-    UNUSED(ctx);
+	UNUSED(server_list);
+}
+
+int find_servers (server_list_t *server_list)
+{
+	UNUSED(server_list);
+	return FIND_SUCCESS;
+}
+
+int createNopollConnection(noPollCtx *ctx, server_list_t *server_list)
+{
+    UNUSED(ctx); UNUSED(server_list);
     function_called();
     return (int) mock();
 }
@@ -75,14 +87,52 @@ noPollMutexUnlock 	mutex_unlock
     function_called();
 }
 
-void start_conn_in_progress (void)
+char *get_global_reconnect_reason()
 {
+    return reconnect_reason;
+}
+
+char *get_global_shutdown_reason()
+{
+    return SHUTDOWN_REASON_PARODUS_STOP;
+}
+
+void set_global_shutdown_reason(char *reason)
+{
+    UNUSED(reason);
+}
+
+void start_conn_in_progress (unsigned long start_time)
+{
+	UNUSED(start_time);
 }   
 
 void stop_conn_in_progress (void)
 {
 }   
 
+void reset_interface_down_event (void)
+{
+}
+
+bool get_interface_down_event (void)
+{
+	return false;
+}
+
+void set_interface_down_event (void)
+{
+}
+
+int wait_while_interface_down (void)
+{
+	return 0;
+}
+
+void terminate_backoff_delay (void)
+{
+}
+
 void packMetaData()
 {
     function_called();
@@ -140,12 +190,19 @@ void set_global_reconnect_status(bool status)
     function_called();
 }
 
-void close_and_unref_connection(noPollConn *conn)
+void close_and_unref_connection(noPollConn *conn, bool is_shutting_down)
 {
-    UNUSED(conn);
+    UNUSED(conn); UNUSED(is_shutting_down);
     function_called();
 }
 
+int readFromFile(const char *file_name, char **data)
+{
+	UNUSED(file_name);
+	*data = strdup ("parodus_stopping");
+	return 1;
+}
+
 void nopoll_cleanup_library	()	
 {
     function_called();
@@ -309,7 +366,6 @@ void test_createSocketConnection1()
     expect_function_call(nopoll_ctx_unref);
     expect_function_call(nopoll_cleanup_library);
     createSocketConnection(NULL);
-    assert_string_equal(get_parodus_cfg()->cloud_status, CLOUD_STATUS_OFFLINE);
     
 }
 

tests/test_connection.c

@@ -15,15 +15,14 @@
  */
 #include <stdarg.h>
 
-#include <CUnit/Basic.h>
 #include <stdbool.h>
 #include <stdarg.h>
 #include <stddef.h>
 #include <setjmp.h>
 #include <cmocka.h>
 #include <assert.h>
-#include <CUnit/Basic.h>
 #include <nopoll.h>
+#include <pthread.h>
 
 #include "../src/ParodusInternal.h"
 #include "../src/connection.h"
@@ -38,6 +37,11 @@ extern void init_expire_timer (expire_timer_t *timer);
 extern int check_timer_expired (expire_timer_t *timer, long timeout_ms);
 extern void init_backoff_timer (backoff_timer_t *timer, int max_count);
 extern int update_backoff_delay (backoff_timer_t *timer);
+extern void add_timespec (struct timespec *t1, struct timespec *t2);
+extern unsigned calc_random_secs (int random_num, unsigned max_secs);
+extern unsigned calc_random_nsecs (int random_num);
+void calc_random_expiration (int random_num1, int random_num2, 
+  backoff_timer_t *timer, struct timespec *ts);
 extern int init_header_info (header_info_t *header_info);
 extern void free_header_info (header_info_t *header_info);
 extern char *build_extra_hdrs (header_info_t *header_info);
@@ -60,7 +64,10 @@ extern int keep_trying_to_connect (create_connection_ctx_t *ctx,
 
 bool close_retry;
 bool LastReasonStatus;
-pthread_mutex_t close_mut;
+bool interface_down_event = false;
+pthread_mutex_t close_mut; 
+pthread_mutex_t interface_down_mut=PTHREAD_MUTEX_INITIALIZER;
+pthread_cond_t interface_down_con=PTHREAD_COND_INITIALIZER;
 
 // Mock values
 bool g_shutdown = false;
@@ -80,6 +87,42 @@ char* getWebpaConveyHeader()
     return (char*) "WebPA-1.6 (TG1682)";
 }
 
+
+void set_interface_down_event()
+{
+        interface_down_event = true;
+}
+
+void reset_interface_down_event() 
+{
+	pthread_mutex_lock (&interface_down_mut);
+	interface_down_event = false;
+	pthread_cond_signal(&interface_down_con);
+	pthread_mutex_unlock (&interface_down_mut);
+}
+
+bool get_interface_down_event() 
+{
+	return interface_down_event;
+}
+
+pthread_cond_t *get_interface_down_con(void)
+{
+    return &interface_down_con;
+}
+
+pthread_mutex_t *get_interface_down_mut(void)
+{
+    return &interface_down_mut;
+}
+
+int readFromFile(const char *file_name, char **data)
+{
+	UNUSED(file_name);
+	*data = strdup ("parodus_stopping");
+	return 1;
+}
+
 noPollConn * nopoll_conn_new_opts (noPollCtx  * ctx, noPollConnOpts  * opts, const char  * host_ip, const char  * host_port, const char  * host_name,const char  * get_url,const char  * protocols, const char * origin)
 {
     UNUSED(host_port); UNUSED(host_name); UNUSED(get_url); UNUSED(protocols); 
@@ -148,11 +191,9 @@ void nopoll_conn_close (noPollConn *conn)
     UNUSED(conn);
 }
 
-int nopoll_conn_ref_count (noPollConn *conn)
+void nopoll_conn_close_ext (noPollConn *conn, int status, const char *reason, int reason_size)
 {
-    UNUSED(conn);
-    function_called ();
-    return (nopoll_bool) mock();
+    UNUSED(conn); UNUSED(status); UNUSED(reason); UNUSED(reason_size);
 }
 
 int checkHostIp(char * serverIP)
@@ -200,6 +241,7 @@ int allow_insecure_conn (char **server_addr, unsigned int *port)
 /*                                   Tests                                    */
 /*----------------------------------------------------------------------------*/
 
+
 void test_get_global_conn()
 {
     assert_null(get_global_conn());
@@ -226,7 +268,7 @@ void test_set_global_reconnect_reason()
 
 void test_closeConnection()
 {
-    close_and_unref_connection(get_global_conn());
+    close_and_unref_connection(get_global_conn(), false);
 }
 
 void test_server_is_null()
@@ -303,13 +345,67 @@ void test_expire_timer()
 
 void test_backoff_delay_timer()
 {
+  struct timespec t1;
+  struct timespec t2;
   backoff_timer_t btimer;
+
   init_backoff_timer (&btimer, 5);
   assert_int_equal (3, update_backoff_delay (&btimer));
   assert_int_equal (7, update_backoff_delay (&btimer));
   assert_int_equal (15, update_backoff_delay (&btimer));
   assert_int_equal (31, update_backoff_delay (&btimer));
-  assert_int_equal (31, update_backoff_delay (&btimer));
+
+  t1.tv_sec = 3; t1.tv_nsec = 0;
+  t2.tv_sec = 3; t2.tv_nsec = 0;
+  add_timespec (&t1, &t2);
+  assert_int_equal (6, t2.tv_sec);
+  assert_int_equal (0, t2.tv_nsec);
+
+  t1.tv_sec = 3; t1.tv_nsec = 500*1000000;
+  t2.tv_sec = 3; t2.tv_nsec = 499*1000000;
+  add_timespec (&t1, &t2);
+  assert_int_equal (6, t2.tv_sec);
+  assert_int_equal (999*1000000, t2.tv_nsec);
+  
+  t1.tv_sec = 3; t1.tv_nsec = 500*1000000;
+  t2.tv_sec = 3; t2.tv_nsec = 501*1000000;
+  add_timespec (&t1, &t2);
+  assert_int_equal (7, t2.tv_sec);
+  assert_int_equal (1000000, t2.tv_nsec);
+  
+  assert_int_equal (3, calc_random_secs (0, 7));
+  assert_int_equal (4, calc_random_secs (1, 7));
+  assert_int_equal (7, calc_random_secs (15, 7));
+  assert_int_equal (3, calc_random_secs (16, 15));
+  assert_int_equal (14, calc_random_secs (30,15));
+  
+  assert_int_equal (250000, calc_random_nsecs (500000));
+  assert_int_equal (1, calc_random_nsecs (2000000002));
+
+  init_backoff_timer (&btimer, 5);
+  t1.tv_sec = 0; t1.tv_nsec = 0;
+  /* max delay is 3 */
+  calc_random_expiration (0, 0, &btimer, &t1);
+  assert_int_equal (3, t1.tv_sec); 
+  assert_int_equal (0, t1.tv_nsec);  
+  
+  t1.tv_sec = 0; t1.tv_nsec = 0;
+  /* max delay is 7*/
+  calc_random_expiration (15, 1073741824, &btimer, &t1);
+  assert_int_equal (7, t1.tv_sec);
+  assert_int_equal (536870912, t1.tv_nsec);  
+
+  t1.tv_sec = 0; t1.tv_nsec = 0;
+  /* max delay is 15 */
+  calc_random_expiration (30, 2000000002, &btimer, &t1);
+  assert_int_equal (14, t1.tv_sec);
+  assert_int_equal (1, t1.tv_nsec);  
+
+  t1.tv_sec = 0; t1.tv_nsec = 0;
+  /* max delay is 31 */
+  calc_random_expiration (32, 1, &btimer, &t1);
+  assert_int_equal (3, t1.tv_sec);
+  assert_int_equal (0, t1.tv_nsec);  
 }
 
 
@@ -365,16 +461,19 @@ void test_extra_headers ()
 
 void test_set_current_server()
 {
+  server_list_t server_list;
   create_connection_ctx_t ctx;
-  memset (&ctx, 0xFF, sizeof(ctx));
+  memset (&server_list, 0xFF, sizeof(server_list));
+  memset (&ctx, 0, sizeof(ctx));
+  ctx.server_list = &server_list;
   set_current_server (&ctx);
-  assert_ptr_equal (&ctx.server_list.redirect, ctx.current_server);
-  set_server_null (&ctx.server_list.redirect);
+  assert_ptr_equal (&ctx.server_list->redirect, ctx.current_server);
+  set_server_null (&ctx.server_list->redirect);
   set_current_server (&ctx);
-  assert_ptr_equal (&ctx.server_list.jwt, ctx.current_server);
-  set_server_null (&ctx.server_list.jwt);
+  assert_ptr_equal (&ctx.server_list->jwt, ctx.current_server);
+  set_server_null (&ctx.server_list->jwt);
   set_current_server (&ctx);
-  assert_ptr_equal (&ctx.server_list.defaults, ctx.current_server);
+  assert_ptr_equal (&ctx.server_list->defaults, ctx.current_server);
 }
 
 void init_cfg_header_info (ParodusCfg *cfg)
@@ -590,10 +689,12 @@ void test_nopoll_connect ()
 
 void test_wait_connection_ready ()
 {
+  server_list_t server_list;
   create_connection_ctx_t ctx;
 
+  set_server_list_null (&server_list);
   memset(&ctx,0,sizeof(ctx));
-  set_server_list_null (&ctx.server_list);
+  ctx.server_list = &server_list;
 
   mock_wait_status = 0;
   mock_redirect = NULL;
@@ -622,22 +723,22 @@ void test_wait_connection_ready ()
   will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_false);
   expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
   assert_int_equal (wait_connection_ready (&ctx), WAIT_ACTION_RETRY);
-  assert_string_equal (ctx.server_list.redirect.server_addr, "mydns.mycom.net");
-  assert_int_equal (ctx.server_list.redirect.port, 8080);
-  assert_int_equal (0, ctx.server_list.redirect.allow_insecure);
-  assert_ptr_equal (ctx.current_server, &ctx.server_list.redirect);
-  free_server (&ctx.server_list.redirect);
+  assert_string_equal (ctx.server_list->redirect.server_addr, "mydns.mycom.net");
+  assert_int_equal (ctx.server_list->redirect.port, 8080);
+  assert_int_equal (0, ctx.server_list->redirect.allow_insecure);
+  assert_ptr_equal (ctx.current_server, &ctx.server_list->redirect);
+  free_server (&ctx.server_list->redirect);
   
   mock_wait_status = 303;
   mock_redirect = "http://mydns.mycom.net";
   will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_false);
   expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
   assert_int_equal (wait_connection_ready (&ctx), WAIT_ACTION_RETRY);
-  assert_string_equal (ctx.server_list.redirect.server_addr, "mydns.mycom.net");
-  assert_int_equal (ctx.server_list.redirect.port, 80);
-  assert_int_equal (1, ctx.server_list.redirect.allow_insecure);
-  assert_ptr_equal (ctx.current_server, &ctx.server_list.redirect);
-  free_server (&ctx.server_list.redirect);
+  assert_string_equal (ctx.server_list->redirect.server_addr, "mydns.mycom.net");
+  assert_int_equal (ctx.server_list->redirect.port, 80);
+  assert_int_equal (1, ctx.server_list->redirect.allow_insecure);
+  assert_ptr_equal (ctx.current_server, &ctx.server_list->redirect);
+  free_server (&ctx.server_list->redirect);
   
     mock_wait_status = 403;
     will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_false);
@@ -655,6 +756,7 @@ void test_wait_connection_ready ()
 
 void test_connect_and_wait ()
 {
+  server_list_t server_list;
   create_connection_ctx_t ctx;
   noPollCtx test_nopoll_ctx;
   server_t test_server;
@@ -671,7 +773,9 @@ void test_connect_and_wait ()
 
   mock_wait_status = 0;
   
+  set_server_list_null (&server_list);
   memset(&ctx,0,sizeof(ctx));
+  ctx.server_list = &server_list;
   ctx.nopoll_ctx = &test_nopoll_ctx;
   ctx.current_server = &test_server;
   ctx.extra_headers = test_extra_headers;
@@ -695,8 +799,6 @@ void test_connect_and_wait ()
   expect_function_call (nopoll_conn_is_ok);
   will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_false);
   expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
-  will_return (nopoll_conn_ref_count, 0);
-  expect_function_call (nopoll_conn_ref_count);
   assert_int_equal (connect_and_wait (&ctx), CONN_WAIT_RETRY_DNS);
 
   Cfg.flags = 0;
@@ -706,8 +808,6 @@ void test_connect_and_wait ()
   expect_function_call (nopoll_conn_new_opts);
   will_return (nopoll_conn_is_ok, nopoll_false);
   expect_function_call (nopoll_conn_is_ok);
-  will_return (nopoll_conn_ref_count, 0);
-  expect_function_call (nopoll_conn_ref_count);
   assert_int_equal (connect_and_wait (&ctx), CONN_WAIT_RETRY_DNS);
 
   will_return (nopoll_conn_new_opts, &connection1);
@@ -745,8 +845,6 @@ void test_connect_and_wait ()
   expect_function_call (nopoll_conn_tls_new6);
   will_return (nopoll_conn_is_ok, nopoll_false);
   expect_function_call (nopoll_conn_is_ok);
-  will_return (nopoll_conn_ref_count, 0);
-  expect_function_call (nopoll_conn_ref_count);
   will_return (nopoll_conn_tls_new, NULL);
   expect_function_call (nopoll_conn_tls_new);
   will_return (checkHostIp, 0);
@@ -757,8 +855,6 @@ void test_connect_and_wait ()
   expect_function_call (nopoll_conn_tls_new6);
   will_return (nopoll_conn_is_ok, nopoll_false);
   expect_function_call (nopoll_conn_is_ok);
-  will_return (nopoll_conn_ref_count, 0);
-  expect_function_call (nopoll_conn_ref_count);
   will_return (nopoll_conn_tls_new, &connection1);
   expect_function_call (nopoll_conn_tls_new);
   will_return (nopoll_conn_is_ok, nopoll_true);
@@ -778,8 +874,6 @@ void test_connect_and_wait ()
   mock_redirect = "mydns.mycom.net";
   will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_false);
   expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
-  will_return (nopoll_conn_ref_count, 0);
-  expect_function_call (nopoll_conn_ref_count);
   assert_int_equal (connect_and_wait (&ctx), CONN_WAIT_RETRY_DNS);
 
   will_return (nopoll_conn_tls_new, &connection1);
@@ -790,17 +884,15 @@ void test_connect_and_wait ()
   mock_redirect = "https://mydns.mycom.net";
   will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_false);
   expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
-  will_return (nopoll_conn_ref_count, 0);
-  expect_function_call (nopoll_conn_ref_count);
   assert_int_equal (connect_and_wait (&ctx), CONN_WAIT_ACTION_RETRY);
 }
 
 void test_keep_trying ()
 {
   int rtn;	
+  server_list_t server_list;
   create_connection_ctx_t ctx;
   noPollCtx test_nopoll_ctx;
-  server_t test_server;
   backoff_timer_t backoff_timer;
   ParodusCfg Cfg;
 
@@ -810,14 +902,16 @@ void test_keep_trying ()
 
   mock_wait_status = 0;
 
+  set_server_list_null (&server_list);
   memset(&ctx,0,sizeof(ctx));
+  ctx.server_list = &server_list;
   ctx.nopoll_ctx = &test_nopoll_ctx;
-  ctx.current_server = &test_server;
-
-  test_server.allow_insecure = 1;
-  test_server.server_addr = "mydns.mycom.net";
-  test_server.port = 8080;
 
+  server_list.defaults.allow_insecure = 1;
+  server_list.defaults.server_addr = "mydns.mycom.net";
+  server_list.defaults.port = 8080;
+  set_current_server (&ctx);
+  
   Cfg.flags = 0;
   set_parodus_cfg(&Cfg);
 
@@ -831,7 +925,7 @@ void test_keep_trying ()
   rtn = keep_trying_to_connect (&ctx, &backoff_timer);
   assert_int_equal (rtn, true);
 
-  test_server.allow_insecure = 0;
+  server_list.defaults.allow_insecure = 0;
   Cfg.flags = FLAGS_IPV4_ONLY;
   set_parodus_cfg(&Cfg);
 
@@ -843,8 +937,6 @@ void test_keep_trying ()
   mock_redirect = "https://mydns.mycom.net";
   will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_false);
   expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
-  will_return (nopoll_conn_ref_count, 0);
-  expect_function_call (nopoll_conn_ref_count);
   will_return (nopoll_conn_tls_new, &connection1);
   expect_function_call (nopoll_conn_tls_new);
   will_return (nopoll_conn_is_ok, nopoll_true);
@@ -863,16 +955,12 @@ void test_keep_trying ()
   mock_redirect = "https://mydns.mycom.net";
   will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_false);
   expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
-  will_return (nopoll_conn_ref_count, 0);
-  expect_function_call (nopoll_conn_ref_count);
   will_return (nopoll_conn_tls_new, &connection1);
   expect_function_call (nopoll_conn_tls_new);
   will_return (nopoll_conn_is_ok, nopoll_true);
   expect_function_call (nopoll_conn_is_ok);
   will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_false);
   expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
-  will_return (nopoll_conn_ref_count, 0);
-  expect_function_call (nopoll_conn_ref_count);
   init_backoff_timer (&backoff_timer, 5);
   rtn = keep_trying_to_connect (&ctx, &backoff_timer);
   assert_int_equal (rtn, false);
@@ -895,8 +983,6 @@ void test_keep_trying ()
   expect_function_call (nopoll_conn_is_ok);
   will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_false);
   expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
-  will_return (nopoll_conn_ref_count, 0);
-  expect_function_call (nopoll_conn_ref_count);
   init_backoff_timer (&backoff_timer, 5);
   rtn = keep_trying_to_connect (&ctx, &backoff_timer);
   assert_int_equal (rtn, false);
@@ -906,8 +992,182 @@ void test_create_nopoll_connection()
 {
   int rtn;
   ParodusCfg cfg;
+  server_list_t server_list;
   noPollCtx test_nopoll_ctx;
 
+  set_server_list_null (&server_list);
+  memset(&cfg,0,sizeof(cfg));
+
+  cfg.flags = 0;
+  parStrncpy (cfg.webpa_url, "mydns.mycom.net:8080", sizeof(cfg.webpa_url));
+  cfg.boot_time = 25;
+  parStrncpy (cfg.hw_last_reboot_reason, "Test reason", sizeof(cfg.hw_last_reboot_reason));
+  cfg.webpa_backoff_max = 30;
+  parStrncpy (cfg.webpa_auth_token, "Auth---", sizeof (cfg.webpa_auth_token));
+  parStrncpy(cfg.hw_model, "TG1682", sizeof(cfg.hw_model));
+  parStrncpy(cfg.hw_manufacturer , "ARRISGroup,Inc.", sizeof(cfg.hw_manufacturer));
+  parStrncpy(cfg.hw_mac , "123567892366", sizeof(cfg.hw_mac));
+  parStrncpy(cfg.fw_name , "2.364s2", sizeof(cfg.fw_name));
+  parStrncpy(cfg.webpa_protocol , "WebPA-1.6", sizeof(cfg.webpa_protocol));
+  set_parodus_cfg(&cfg);
+  rtn = createNopollConnection (&test_nopoll_ctx, &server_list);
+  assert_int_equal (rtn, nopoll_false);
+
+  parStrncpy (cfg.webpa_url, "http://mydns.mycom.net:8080", sizeof(cfg.webpa_url));
+  set_parodus_cfg(&cfg);
+  set_server_list_null (&server_list);
+
+  mock_wait_status = 0;
+  
+  will_return (nopoll_conn_new_opts, &connection1);
+  expect_function_call (nopoll_conn_new_opts);
+  will_return (nopoll_conn_is_ok, nopoll_true);
+  expect_function_call (nopoll_conn_is_ok);
+  will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_true);
+  expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
+  rtn = createNopollConnection (&test_nopoll_ctx, &server_list);
+  assert_int_equal (rtn, nopoll_true);
+
+  parStrncpy (cfg.webpa_url, "https://mydns.mycom.net:8080", sizeof(cfg.webpa_url));
+  cfg.flags = 0;
+  set_parodus_cfg(&cfg);
+  set_server_list_null (&server_list);
+
+  will_return (nopoll_conn_tls_new6, &connection1);
+  expect_function_call (nopoll_conn_tls_new6);
+  will_return (nopoll_conn_is_ok, nopoll_false);
+  expect_function_call (nopoll_conn_is_ok);
+  will_return (nopoll_conn_tls_new, &connection1);
+  expect_function_call (nopoll_conn_tls_new);
+  will_return (nopoll_conn_is_ok, nopoll_true);
+  expect_function_call (nopoll_conn_is_ok);
+  will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_true);
+  expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
+  rtn = createNopollConnection (&test_nopoll_ctx, &server_list);
+  assert_int_equal (rtn, nopoll_true);
+
+  set_server_list_null (&server_list);
+
+  will_return (nopoll_conn_tls_new6, &connection1);
+  expect_function_call (nopoll_conn_tls_new6);
+  will_return (nopoll_conn_is_ok, nopoll_false);
+  expect_function_call (nopoll_conn_is_ok);
+  will_return (nopoll_conn_tls_new, &connection1);
+  expect_function_call (nopoll_conn_tls_new);
+  will_return (nopoll_conn_is_ok, nopoll_true);
+  expect_function_call (nopoll_conn_is_ok);
+  mock_wait_status = 302;
+  mock_redirect = "https://mydns.mycom.net";
+  will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_false);
+  expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
+
+  will_return (nopoll_conn_tls_new6, &connection1);
+  expect_function_call (nopoll_conn_tls_new6);
+  will_return (nopoll_conn_is_ok, nopoll_false);
+  expect_function_call (nopoll_conn_is_ok);
+  will_return (nopoll_conn_tls_new, &connection1);
+  expect_function_call (nopoll_conn_tls_new);
+  will_return (nopoll_conn_is_ok, nopoll_true);
+  expect_function_call (nopoll_conn_is_ok);
+  mock_wait_status = 0;
+  will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_true);
+  expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
+  rtn = createNopollConnection (&test_nopoll_ctx, &server_list);
+  assert_int_equal (rtn, nopoll_true);
+
+#ifdef FEATURE_DNS_QUERY
+  cfg.acquire_jwt = 1;
+  cfg.flags = FLAGS_IPV4_ONLY;
+  set_parodus_cfg(&cfg);
+  set_server_list_null (&server_list);
+  
+  will_return (allow_insecure_conn, -1);
+  expect_function_call (allow_insecure_conn);
+  will_return (nopoll_conn_tls_new, NULL);
+  expect_function_call (nopoll_conn_tls_new);
+  will_return (checkHostIp, 0);
+  expect_function_call (checkHostIp);
+  mock_server_addr = "mydns.myjwtcom.net";
+  mock_port = 80;
+  will_return (allow_insecure_conn, 0);
+  expect_function_call (allow_insecure_conn);
+  will_return (nopoll_conn_tls_new, &connection1);
+  expect_function_call (nopoll_conn_tls_new);
+  will_return (nopoll_conn_is_ok, nopoll_true);
+  expect_function_call (nopoll_conn_is_ok);
+  will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_true);
+  expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
+  rtn = createNopollConnection (&test_nopoll_ctx, &server_list);
+  assert_int_equal (rtn, nopoll_true);
+    
+  set_server_list_null (&server_list);
+  mock_server_addr = "mydns.myjwtcom.net";
+  mock_port = 80;
+  will_return (allow_insecure_conn, 0);
+  expect_function_call (allow_insecure_conn);
+  will_return (nopoll_conn_tls_new, &connection1);
+  expect_function_call (nopoll_conn_tls_new);
+  will_return (nopoll_conn_is_ok, nopoll_true);
+  expect_function_call (nopoll_conn_is_ok);
+  will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_false);
+  expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
+  will_return (nopoll_conn_tls_new, &connection1);
+  expect_function_call (nopoll_conn_tls_new);
+  will_return (nopoll_conn_is_ok, nopoll_true);
+  expect_function_call (nopoll_conn_is_ok);
+  will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_true);
+  expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
+  rtn = createNopollConnection (&test_nopoll_ctx, &server_list);
+  assert_int_equal (rtn, nopoll_true);
+
+  cfg.flags = 0;
+  set_parodus_cfg(&cfg);
+  set_server_list_null (&server_list);
+
+  will_return (allow_insecure_conn, -1);
+  expect_function_call (allow_insecure_conn);
+  will_return (nopoll_conn_tls_new6, &connection1);
+  expect_function_call (nopoll_conn_tls_new6);
+  will_return (nopoll_conn_is_ok, nopoll_false);
+  expect_function_call (nopoll_conn_is_ok);
+  mock_wait_status = 0;
+  will_return (nopoll_conn_tls_new, &connection1);
+  expect_function_call (nopoll_conn_tls_new);
+  will_return (nopoll_conn_is_ok, nopoll_true);
+  expect_function_call (nopoll_conn_is_ok);
+  will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_true);
+  expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
+  rtn = createNopollConnection (&test_nopoll_ctx, &server_list);
+  assert_int_equal (rtn, nopoll_true);
+#endif
+
+}
+
+void test_get_interface_down_event()
+{
+    assert_false(get_interface_down_event());
+    set_interface_down_event();
+}
+
+void *a()
+{   
+    sleep(15);
+    reset_interface_down_event();
+    pthread_exit(0);
+    return NULL;
+}
+
+
+void test_interface_down_retry()
+{ 
+  int rtn;
+  ParodusCfg cfg;
+  noPollCtx test_nopoll_ctx;
+  server_list_t server_list;
+  pthread_t thread_a;
+
+  pthread_create(&thread_a, NULL, a, NULL);
+
   memset(&cfg,0,sizeof(cfg));
   cfg.flags = 0;
   parStrncpy (cfg.webpa_url, "mydns.mycom.net:8080", sizeof(cfg.webpa_url));
@@ -921,7 +1181,8 @@ void test_create_nopoll_connection()
   parStrncpy(cfg.fw_name , "2.364s2", sizeof(cfg.fw_name));
   parStrncpy(cfg.webpa_protocol , "WebPA-1.6", sizeof(cfg.webpa_protocol));
   set_parodus_cfg(&cfg);
-  rtn = createNopollConnection (&test_nopoll_ctx);
+  set_server_list_null (&server_list);
+  rtn = createNopollConnection (&test_nopoll_ctx, &server_list);
   assert_int_equal (rtn, nopoll_false);
 
   parStrncpy (cfg.webpa_url, "http://mydns.mycom.net:8080", sizeof(cfg.webpa_url));
@@ -935,7 +1196,8 @@ void test_create_nopoll_connection()
   expect_function_call (nopoll_conn_is_ok);
   will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_true);
   expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
-  rtn = createNopollConnection (&test_nopoll_ctx);
+  set_server_list_null (&server_list);
+  rtn = createNopollConnection (&test_nopoll_ctx, &server_list);
   assert_int_equal (rtn, nopoll_true);
 
   parStrncpy (cfg.webpa_url, "https://mydns.mycom.net:8080", sizeof(cfg.webpa_url));
@@ -946,23 +1208,20 @@ void test_create_nopoll_connection()
   expect_function_call (nopoll_conn_tls_new6);
   will_return (nopoll_conn_is_ok, nopoll_false);
   expect_function_call (nopoll_conn_is_ok);
-  will_return (nopoll_conn_ref_count, 0);
-  expect_function_call (nopoll_conn_ref_count);
   will_return (nopoll_conn_tls_new, &connection1);
   expect_function_call (nopoll_conn_tls_new);
   will_return (nopoll_conn_is_ok, nopoll_true);
   expect_function_call (nopoll_conn_is_ok);
   will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_true);
   expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
-  rtn = createNopollConnection (&test_nopoll_ctx);
+  set_server_list_null (&server_list);
+  rtn = createNopollConnection (&test_nopoll_ctx, &server_list);
   assert_int_equal (rtn, nopoll_true);
 
   will_return (nopoll_conn_tls_new6, &connection1);
   expect_function_call (nopoll_conn_tls_new6);
   will_return (nopoll_conn_is_ok, nopoll_false);
   expect_function_call (nopoll_conn_is_ok);
-  will_return (nopoll_conn_ref_count, 0);
-  expect_function_call (nopoll_conn_ref_count);
   will_return (nopoll_conn_tls_new, &connection1);
   expect_function_call (nopoll_conn_tls_new);
   will_return (nopoll_conn_is_ok, nopoll_true);
@@ -971,15 +1230,11 @@ void test_create_nopoll_connection()
   mock_redirect = "https://mydns.mycom.net";
   will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_false);
   expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
-  will_return (nopoll_conn_ref_count, 0);
-  expect_function_call (nopoll_conn_ref_count);
 
   will_return (nopoll_conn_tls_new6, &connection1);
   expect_function_call (nopoll_conn_tls_new6);
   will_return (nopoll_conn_is_ok, nopoll_false);
   expect_function_call (nopoll_conn_is_ok);
-  will_return (nopoll_conn_ref_count, 0);
-  expect_function_call (nopoll_conn_ref_count);
   will_return (nopoll_conn_tls_new, &connection1);
   expect_function_call (nopoll_conn_tls_new);
   will_return (nopoll_conn_is_ok, nopoll_true);
@@ -987,7 +1242,8 @@ void test_create_nopoll_connection()
   mock_wait_status = 0;
   will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_true);
   expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
-  rtn = createNopollConnection (&test_nopoll_ctx);
+  set_server_list_null (&server_list);
+  rtn = createNopollConnection (&test_nopoll_ctx, &server_list);
   assert_int_equal (rtn, nopoll_true);
 
 #ifdef FEATURE_DNS_QUERY
@@ -1011,7 +1267,8 @@ void test_create_nopoll_connection()
   expect_function_call (nopoll_conn_is_ok);
   will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_true);
   expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
-  rtn = createNopollConnection (&test_nopoll_ctx);
+  set_server_list_null (&server_list);
+  rtn = createNopollConnection (&test_nopoll_ctx, &server_list);
   assert_int_equal (rtn, nopoll_true);
     
   cfg.flags = 0;
@@ -1023,8 +1280,6 @@ void test_create_nopoll_connection()
   expect_function_call (nopoll_conn_tls_new6);
   will_return (nopoll_conn_is_ok, nopoll_false);
   expect_function_call (nopoll_conn_is_ok);
-  will_return (nopoll_conn_ref_count, 0);
-  expect_function_call (nopoll_conn_ref_count);
   mock_wait_status = 0;
   will_return (nopoll_conn_tls_new, &connection1);
   expect_function_call (nopoll_conn_tls_new);
@@ -1032,12 +1287,12 @@ void test_create_nopoll_connection()
   expect_function_call (nopoll_conn_is_ok);
   will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_true);
   expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
-  rtn = createNopollConnection (&test_nopoll_ctx);
+  set_server_list_null (&server_list);
+  rtn = createNopollConnection (&test_nopoll_ctx, &server_list);
   assert_int_equal (rtn, nopoll_true);
 #endif
-
-}
-
+   pthread_join(thread_a, NULL);
+} 
 
 /*----------------------------------------------------------------------------*/
 /*                             External Functions                             */
@@ -1065,7 +1320,9 @@ int main(void)
         cmocka_unit_test(test_wait_connection_ready),
         cmocka_unit_test(test_connect_and_wait),
         cmocka_unit_test(test_keep_trying),
-	cmocka_unit_test(test_create_nopoll_connection)
+	    cmocka_unit_test(test_create_nopoll_connection),
+        cmocka_unit_test(test_get_interface_down_event),
+        cmocka_unit_test(test_interface_down_retry)
     };
 
     return cmocka_run_group_tests(tests, NULL, NULL);

tests/test_crud_internal.c

@@ -35,6 +35,11 @@
 bool LastReasonStatus;
 pthread_mutex_t close_mut;
 
+char *get_global_reconnect_reason()
+{
+    return "parodus_stopping";
+}
+
 void set_global_reconnect_reason(char *reason)
 {
 	UNUSED(reason);

tests/test_nopoll_handlers.c

@@ -30,8 +30,11 @@
 /*----------------------------------------------------------------------------*/
 volatile unsigned int heartBeatTimer;
 bool LastReasonStatus;
+bool interface_down_event = false;
 int closeReason = 0;
-pthread_mutex_t close_mut;
+pthread_mutex_t close_mut; 
+pthread_mutex_t interface_down_mut=PTHREAD_MUTEX_INITIALIZER;
+pthread_cond_t interface_down_con=PTHREAD_COND_INITIALIZER;
 bool close_retry;
 /*----------------------------------------------------------------------------*/
 /*                                   Mocks                                    */
@@ -47,9 +50,22 @@ bool get_global_reconnect_status()
     return LastReasonStatus;
 }
 
-bool get_interface_down_event()
+void set_interface_down_event()
 {
-     return false;
+        interface_down_event = true;
+}
+
+void reset_interface_down_event() 
+{
+	pthread_mutex_lock (&interface_down_mut);
+	interface_down_event = false;
+	pthread_cond_signal(&interface_down_con);
+	pthread_mutex_unlock (&interface_down_mut);
+}
+
+bool get_interface_down_event() 
+{
+	return interface_down_event;
 }
 
 void set_global_reconnect_status(bool status)
@@ -185,6 +201,30 @@ void test_listenerOnPingMessage()
     listenerOnPingMessage(NULL, NULL, NULL, NULL);
 }
 
+void test_getInterfaceDownEvent()
+{   
+    set_interface_down_event();
+    CU_ASSERT_TRUE(get_interface_down_event());
+}
+
+void test_interfaceDownEvent()
+{
+    char str[] = "SSL_Socket_Close";
+    set_global_reconnect_status(true);
+    set_interface_down_event();
+    listenerOnCloseMessage(NULL, NULL, (noPollPtr) str);
+    
+}
+
+void test_noInterfaceDownEvent()
+{
+    char str[] = "SSL_Socket_Close";
+    set_global_reconnect_status(true);
+    reset_interface_down_event();
+    listenerOnCloseMessage(NULL, NULL, (noPollPtr) str);
+}
+
+
 void add_suites( CU_pSuite *suite )
 {
     ParodusInfo("--------Start of Test Cases Execution ---------\n");
@@ -192,6 +232,9 @@ void add_suites( CU_pSuite *suite )
     CU_add_test( *suite, "Test 1", test_listenerOnMessage_queue );
     CU_add_test( *suite, "Test 2", test_listenerOnCloseMessage );
     CU_add_test( *suite, "Test 3", test_listenerOnPingMessage );
+    CU_add_test( *suite, "Test 4", test_getInterfaceDownEvent );
+    CU_add_test( *suite, "Test 5", test_interfaceDownEvent );
+    CU_add_test( *suite, "Test 6", test_noInterfaceDownEvent );
 }
 
 /*----------------------------------------------------------------------------*/

tests/test_nopoll_helpers.c

@@ -23,6 +23,7 @@
 
 #include "../src/parodus_log.h"
 #include "../src/nopoll_helpers.h"
+#include "../src/config.h"
 
 /*----------------------------------------------------------------------------*/
 /*                                   Macros                                   */
@@ -32,7 +33,9 @@
 /*----------------------------------------------------------------------------*/
 /*                            File Scoped Variables                           */
 /*----------------------------------------------------------------------------*/
- static noPollConn *conn;
+ static noPollConn *conn = NULL;
+ static ParodusCfg cfg;
+ 
 /*----------------------------------------------------------------------------*/
 /*                                   Mocks                                    */
 /*----------------------------------------------------------------------------*/
@@ -54,6 +57,12 @@ nopoll_bool nopoll_conn_is_ready( noPollConn *conn )
     return (nopoll_bool)mock();
 }
 
+ParodusCfg *get_parodus_cfg(void)
+{
+	function_called();
+    return &cfg;
+}
+
 int  __nopoll_conn_send_common (noPollConn * conn, const char * content, long length, nopoll_bool  has_fin, long       sleep_in_header, noPollOpCode frame_type)
 {
     UNUSED(has_fin); UNUSED(sleep_in_header); UNUSED(frame_type); UNUSED(content);
@@ -212,13 +221,8 @@ void test_sendMessage()
 {
     int len = strlen("Hello Parodus!");
     
-    expect_value(nopoll_conn_is_ok, (intptr_t)conn, (intptr_t)conn);
-    will_return(nopoll_conn_is_ok, nopoll_true);
-    expect_function_call(nopoll_conn_is_ok);
-    
-    expect_value(nopoll_conn_is_ready, (intptr_t)conn, (intptr_t)conn);
-    will_return(nopoll_conn_is_ready, nopoll_true);
-    expect_function_call(nopoll_conn_is_ready);
+    cfg.cloud_status = CLOUD_STATUS_ONLINE;
+    expect_function_calls (get_parodus_cfg, 1);
     
     expect_value(__nopoll_conn_send_common, (intptr_t)conn, (intptr_t)conn);
     expect_value(__nopoll_conn_send_common, length, len);
@@ -228,81 +232,23 @@ void test_sendMessage()
     sendMessage(conn, "Hello Parodus!", len);
 }
 
-void connStuck_sendMessage()
+void test_sendMessageOffline()
 {
-   int len = strlen("Hello Parodus!");
-
-    /* Initialize the timer when connection gets stuck */
-    expect_value(nopoll_conn_is_ok, (intptr_t)conn, (intptr_t)NULL);
-    will_return(nopoll_conn_is_ok, nopoll_false);
-    expect_function_call(nopoll_conn_is_ok);
-
-    expect_function_call(getCurrentTime);
-    sendMessage(NULL, "Hello Parodus!", len);
-
-    /* When connection recovers within 10 mins, it should be able to re-connect */
-    expect_value(nopoll_conn_is_ok, (intptr_t)conn, (intptr_t)NULL);
-    will_return(nopoll_conn_is_ok, nopoll_false);
-    expect_function_call(nopoll_conn_is_ok);
-
-    expect_function_call(getCurrentTime);
-
-    will_return(timeValDiff, 5*60*1000);
-    expect_function_call(timeValDiff);
-
-    expect_value(nopoll_conn_is_ok, (intptr_t)conn, (intptr_t)conn);
-    will_return(nopoll_conn_is_ok, nopoll_true);
-    expect_function_call(nopoll_conn_is_ok);
-
-    expect_value(nopoll_conn_is_ready, (intptr_t)conn, (intptr_t)conn);
-    will_return(nopoll_conn_is_ready, nopoll_true);
-    expect_function_call(nopoll_conn_is_ready);
-
-    expect_value(__nopoll_conn_send_common, (intptr_t)conn, (intptr_t)conn);
-    expect_value(__nopoll_conn_send_common, length, len);
-    will_return(__nopoll_conn_send_common, len);
-    expect_function_calls(__nopoll_conn_send_common, 1);
-
+    int len = strlen("Hello Parodus!");
+    
+    cfg.cloud_status = CLOUD_STATUS_OFFLINE;
+    expect_function_calls (get_parodus_cfg, 1);
     sendMessage(conn, "Hello Parodus!", len);
-
-    /* When timer exceeds more than 10 mins kill the process */
-    expect_value(nopoll_conn_is_ok, (intptr_t)conn, (intptr_t)NULL);
-    will_return(nopoll_conn_is_ok, nopoll_false);
-    expect_function_call(nopoll_conn_is_ok);
-
-    sendMessage(NULL, "Hello Parodus!", len);
-
-    expect_function_call(getCurrentTime);
-
-    sendMessage(NULL, "Hello Parodus!", len);
-
-    expect_value(nopoll_conn_is_ok, (intptr_t)conn, (intptr_t)NULL);
-    will_return(nopoll_conn_is_ok, nopoll_false);
-    expect_function_call(nopoll_conn_is_ok);
-
-    expect_function_call(getCurrentTime);
-
-    will_return(timeValDiff, 10*60*1000);
-    expect_function_call(timeValDiff);
-
-    will_return(kill, 1);
-    expect_function_call(kill);
-
-    sendMessage(NULL, "Hello Parodus!", len);
+    
 }
 
 void err_sendMessage()
 {
     int len = strlen("Hello Parodus!");
     
-    expect_value(nopoll_conn_is_ok, (intptr_t)conn, (intptr_t)conn);
-    will_return(nopoll_conn_is_ok, nopoll_true);
-    expect_function_call(nopoll_conn_is_ok);
-    
-    expect_value(nopoll_conn_is_ready, (intptr_t)conn, (intptr_t)conn);
-    will_return(nopoll_conn_is_ready, nopoll_true);
-    expect_function_call(nopoll_conn_is_ready);
-    
+    cfg.cloud_status = CLOUD_STATUS_ONLINE;
+    expect_function_calls (get_parodus_cfg, 1);
+
     expect_value(__nopoll_conn_send_common, (intptr_t)conn,(intptr_t) conn);
     expect_value(__nopoll_conn_send_common, length, len);
     will_return(__nopoll_conn_send_common, len-2);
@@ -320,11 +266,13 @@ void err_sendMessageConnNull()
 {
     int len = strlen("Hello Parodus!");
     
-    expect_value(nopoll_conn_is_ok, (intptr_t)conn, (intptr_t)NULL);
-    will_return(nopoll_conn_is_ok, nopoll_false);
-    expect_function_call(nopoll_conn_is_ok);
+    cfg.cloud_status = CLOUD_STATUS_ONLINE;
+    expect_function_calls (get_parodus_cfg, 1);
 
-    expect_function_call(getCurrentTime);
+    expect_value(__nopoll_conn_send_common, (intptr_t)conn, NULL);
+    expect_value(__nopoll_conn_send_common, length, len);
+    will_return(__nopoll_conn_send_common, len);
+    expect_function_calls(__nopoll_conn_send_common, 1);
 
     sendMessage(NULL, "Hello Parodus!", len);
 }
@@ -349,7 +297,7 @@ int main(void)
         cmocka_unit_test(err_sendResponseFlushWrites),
         cmocka_unit_test(err_sendResponseConnNull),
         cmocka_unit_test(test_sendMessage),
-        cmocka_unit_test(connStuck_sendMessage),
+        cmocka_unit_test(test_sendMessageOffline),
         cmocka_unit_test(err_sendMessage),
         cmocka_unit_test(err_sendMessageConnNull),
         cmocka_unit_test(test_reportLog),