Merge pull request #314 from guruchandru/iface_event

Pause retry during interface down event

CHANGELOG.md

@@ -6,6 +6,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 - Security: Added support to use auth token during initial connect to cloud
+- Fix re-registration fails that lose a socket
+- Fix mutex error in service alive
+- Security: Mutual Authentication (mTLS or two way TLS)
+- Rename command line options for MTLS cert and Key
+- Update to use nanomsg v. 1.1.4
+- requestNewAuthToken will clear the token if it fails.
+- request auth token on every retry, not just after 403
+- update to use nopoll v 1.0.2
+- Add pause/resume heartBeatTimer
+- parodus event handler to listen to interface_down and interface_up event
 
 ## [1.0.2] - 2019-02-08
 - Refactored connection.c and updated corresponding unit tests

CMakeLists.txt

@@ -85,7 +85,7 @@ add_dependencies(libtrower-base64 trower-base64)
 ExternalProject_Add(nopoll
     PREFIX ${PREFIX_DIR}/nopoll
     GIT_REPOSITORY https://github.com/Comcast/nopoll.git
-    GIT_TAG "1.0.1"
+    GIT_TAG "1.0.2"
     CONFIGURE_COMMAND COMMAND <SOURCE_DIR>/autogen.sh --prefix=${PREFIX}
                                                      --includedir=${INCLUDE_DIR}
                                                      --libdir=${LIBRARY_DIR}
@@ -101,7 +101,7 @@ add_dependencies(libnopoll nopoll)
 ExternalProject_Add(nanomsg
     PREFIX ${CMAKE_CURRENT_BINARY_DIR}/_prefix/nanomsg
     GIT_REPOSITORY https://github.com/nanomsg/nanomsg.git
-    GIT_TAG "1.1.2"
+    GIT_TAG "1.1.4"
     CMAKE_ARGS += -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}
 )
 add_library(libnanomsg STATIC SHARED IMPORTED)
@@ -152,7 +152,7 @@ add_dependencies(libcurl curl)
 ExternalProject_Add(cimplog
     PREFIX ${CMAKE_CURRENT_BINARY_DIR}/_prefix/cimplog
     GIT_REPOSITORY https://github.com/Comcast/cimplog.git
-    GIT_TAG "1.0.1"
+    GIT_TAG "1.0.2"
     CMAKE_ARGS += -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -DBUILD_TESTING=OFF
 )
 add_library(libcimplog STATIC SHARED IMPORTED)

README.md

@@ -75,6 +75,12 @@ make test
 
 - /jwt-public-key-file -JWT token validation key
 
+# if ENABLE_MUTUAL_AUTH is enabled
+
+- /mtls-client-cert-path - Provide the client cert for establishing a mutual auth secure websocket connection
+
+- /mtls-client-key-path - Provide the client cert key for establishing a mutual auth secure websocket connection
+
 ```
 
 # Sample parodus start commands:

src/ParodusInternal.c

@@ -25,6 +25,10 @@
 #include "config.h"
 #include "connection.h"
 
+bool interface_down_event = false;
+
+pthread_mutex_t interface_down_mut=PTHREAD_MUTEX_INITIALIZER;
+
 /*----------------------------------------------------------------------------*/
 /*                             External Functions                             */
 /*----------------------------------------------------------------------------*/
@@ -139,3 +143,38 @@ void timespec_diff(struct timespec *start, struct timespec *stop,
 
     return;
 }
+
+
+/*------------------------------------------------------------------------------*/
+/*                        For interface_down_event Flag                         */
+/*------------------------------------------------------------------------------*/
+
+// Get value of interface_down_event
+bool get_interface_down_event() 
+{
+	bool tmp = false;
+	pthread_mutex_lock (&interface_down_mut);
+	tmp = interface_down_event;
+	pthread_mutex_unlock (&interface_down_mut);
+	return tmp;
+}
+
+// Reset value of interface_down_event to false
+void reset_interface_down_event() 
+{
+	pthread_mutex_lock (&interface_down_mut);
+	interface_down_event = false;
+	pthread_mutex_unlock (&interface_down_mut);
+}
+
+// set value of interface_down_event to true
+void set_interface_down_event() 
+{
+	pthread_mutex_lock (&interface_down_mut);
+    	interface_down_event = true;
+    	pthread_mutex_unlock (&interface_down_mut);
+}
+
+
+
+

src/ParodusInternal.h

@@ -157,6 +157,21 @@ void addCRUDmsgToQueue(wrp_msg_t *crudMsg);
 void timespec_diff(struct timespec *start, struct timespec *stop,
                    struct timespec *result);
 
+
+/*------------------------------------------------------------------------------*/
+/*                        For interface_down_event Flag                         */
+/*------------------------------------------------------------------------------*/
+
+// Get value of interface_down_event
+bool get_interface_down_event();
+
+// Reset value of interface_down_event to false
+void reset_interface_down_event();
+
+// Set value of interface_down_event to true
+void set_interface_down_event();
+  
+
 #ifdef __cplusplus
 }
 #endif

src/auth_token.c

@@ -66,17 +66,11 @@ int requestNewAuthToken(char *newToken, size_t len, int r_count)
 
 	struct token_data data;
 	data.size = 0;
+	data.data = newToken;
 
 	curl = curl_easy_init();
 	if(curl)
 	{
-		//this memory will be dynamically grown by write call back fn as required
-		data.data = (char *) malloc(sizeof(char) * 1);
-		if(NULL == data.data)
-		{
-			ParodusError("Failed to allocate memory.\n");
-			return -1;
-		}
 		data.data[0] = '\0';
 
 		createCurlheader(mac_header, serial_header, uuid_header, transaction_uuid, list, &headers_list);
@@ -137,11 +131,8 @@ int requestNewAuthToken(char *newToken, size_t len, int r_count)
 		{
 			ParodusError("curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
 			curl_easy_cleanup(curl);
-			if(data.data)
-			{
-				free(data.data);
-				data.data = NULL;
-			}
+			data.size = 0;
+			memset (data.data, 0, len);
 			return -1;
 		}
 		else
@@ -149,19 +140,15 @@ int requestNewAuthToken(char *newToken, size_t len, int r_count)
 			if(response_code == 200)
 			{
 				ParodusInfo("cURL success\n");
-				strncpy(newToken, data.data, len);
 			}
 		}
-		if(data.data)
-		{
-			free(data.data);
-			data.data = NULL;
-		}
 		curl_easy_cleanup(curl);
 	}
 	else
 	{
 		ParodusError("curl init failure\n");
+		data.size = 0;
+		memset (data.data, 0, len);
 		return -1;
 	}
 
@@ -179,6 +166,7 @@ void getAuthToken(ParodusCfg *cfg)
 	int status = -1;
 	int retry_count = 0;
 
+	memset (cfg->webpa_auth_token, 0, sizeof(cfg->webpa_auth_token));
 	if( cfg->hw_mac != NULL && strlen(cfg->hw_mac) !=0 )
 	{
 		if( cfg->client_cert_path !=NULL && strlen(cfg->client_cert_path) !=0 )
@@ -225,28 +213,23 @@ void getAuthToken(ParodusCfg *cfg)
 */
 size_t write_callback_fn(void *buffer, size_t size, size_t nmemb, struct token_data *data)
 {
+    ParodusCfg *cfg;
+    size_t max_data_size = sizeof (cfg->webpa_auth_token);
     size_t index = data->size;
     size_t n = (size * nmemb);
-    char* tmp;
 
-    data->size += (size * nmemb);
+    data->size += n;
 
-    tmp = realloc(data->data, data->size + 1); /* +1 for '\0' */
-
-    if(tmp) {
-        data->data = tmp;
-    } else {
-        if(data->data) {
-            free(data->data);
-        }
-        ParodusError("Failed to allocate memory for data\n");
+    if (data->size >= max_data_size) {
+        ParodusError("Auth token data overruns buffer\n");
+	data->size = 0;
         return 0;
     }
 
     memcpy((data->data + index), buffer, n);
     data->data[data->size] = '\0';
 
-    return size * nmemb;
+    return n;
 }
 
 /* @brief function to generate random uuid.

src/client_list.c

@@ -76,7 +76,11 @@ int addToList( wrp_msg_t **msg)
             if(rc < 0)
             {
                 ParodusError ("Unable to connect socket (errno=%d, %s)\n",errno, strerror(errno));
-                nn_close (sock);
+		if (nn_close (sock) < 0)
+                {
+                   ParodusError ("nn_close socket=%d (err=%d, %s)\n", 
+			sock, errno, strerror(errno));
+		}
                 
             }
             else
@@ -87,6 +91,7 @@ int addToList( wrp_msg_t **msg)
 		{
     			memset( new_node, 0, sizeof( reg_list_item_t ) );
     			new_node->sock = sock;
+                        new_node->endpoint = rc;
     			ParodusPrint("new_node->sock is %d\n", new_node->sock);
     			
     			
@@ -219,6 +224,16 @@ int deleteFromList(char* service_name)
 			}
 			
 			ParodusPrint("Deleting the node\n");
+                        if(nn_shutdown(curr_node->sock, curr_node->endpoint) < 0)
+                        {
+                           ParodusError ("nn_shutdown socket=%d endpt=%d, err=%d\n", 
+				curr_node->sock, curr_node->endpoint, errno);
+                        }
+			if (nn_close (curr_node->sock) < 0)
+                        {
+                           ParodusError ("nn_close socket=%d err=%d\n", 
+				curr_node->sock, errno);
+                        }
 			free( curr_node );
 			curr_node = NULL;
 			ParodusInfo("Deleted successfully and returning..\n");

src/client_list.h

@@ -30,6 +30,7 @@
 typedef struct reg_list_item
 {
 	int sock;
+	int endpoint;
 	char service_name[32];
 	char url[100];
 	struct reg_list_item *next;

src/close_retry.c

@@ -26,6 +26,17 @@
 bool close_retry = false;
 
 pthread_mutex_t close_mut=PTHREAD_MUTEX_INITIALIZER;
+pthread_cond_t close_con=PTHREAD_COND_INITIALIZER;
+
+pthread_cond_t *get_global_close_retry_con(void)
+{
+    return &close_con;
+}
+
+pthread_mutex_t *get_global_close_retry_mut(void)
+{
+    return &close_mut;
+}
 
 // Get value of close_retry
 bool get_close_retry() 
@@ -48,8 +59,9 @@ void reset_close_retry()
 // set value of close_retry to true
 void set_close_retry() 
 {
-	pthread_mutex_lock (&close_mut);
+    pthread_mutex_lock (&close_mut);
     close_retry = true;
+    pthread_cond_signal(&close_con);
     pthread_mutex_unlock (&close_mut);
 }
 

src/close_retry.h

@@ -31,6 +31,12 @@
 extern "C" {
 #endif
 
+// Get the condition variable
+pthread_cond_t *get_global_close_retry_con();
+
+// Get the mutex variable
+pthread_mutex_t *get_global_close_retry_mut();
+
 // Get value of close_retry
 bool get_close_retry();
 

src/config.c

@@ -365,6 +365,9 @@ int parseCommandLine(int argc,char **argv,ParodusCfg * cfg)
 	{"client-cert-path",        required_argument, 0, 'P'},
 	{"token-server-url",        required_argument, 0, 'U'},
 	{"crud-config-file",        required_argument, 0, 'C'},
+	{"connection-health-file",  required_argument, 0, 'S'},
+	{"mtls-client-key-path",    required_argument, 0, 'K'},
+	{"mtls-client-cert-path",    required_argument, 0,'M'},
         {0, 0, 0, 0}
     };
     int c;
@@ -380,6 +383,7 @@ int parseCommandLine(int argc,char **argv,ParodusCfg * cfg)
 	cfg->jwt_algo = 0;
 	parStrncpy (cfg->jwt_key, "", sizeof(cfg->jwt_key));
 	cfg->crud_config_file = NULL;
+	cfg->connection_health_file = NULL;
 	cfg->client_cert_path = NULL;
 	cfg->token_server_url = NULL;
 	cfg->cloud_status = NULL;
@@ -390,7 +394,7 @@ int parseCommandLine(int argc,char **argv,ParodusCfg * cfg)
 
       /* getopt_long stores the option index here. */
       int option_index = 0;
-      c = getopt_long (argc, argv, "m:s:f:d:r:n:b:u:t:o:i:l:p:e:D:j:a:k:c:T:w:J:46:C",
+      c = getopt_long (argc, argv, "m:s:f:d:r:n:b:u:t:o:i:l:p:e:D:j:a:k:c:T:w:J:46:C:S:K:M",
 				long_options, &option_index);
 
       /* Detect the end of the options. */
@@ -431,6 +435,7 @@ int parseCommandLine(int argc,char **argv,ParodusCfg * cfg)
         case 'r':
           parStrncpy(cfg->hw_last_reboot_reason, optarg,sizeof(cfg->hw_last_reboot_reason));
           ParodusInfo("hw_last_reboot_reason is %s\n",cfg->hw_last_reboot_reason);
+          OnboardLog("Last reboot reason is %s\n",cfg->hw_last_reboot_reason);
           break;
 
         case 'n':
@@ -528,10 +533,15 @@ int parseCommandLine(int argc,char **argv,ParodusCfg * cfg)
           ParodusInfo("boot_retry_wait is %d\n",cfg->boot_retry_wait);
           break;
 
-		case 'C':
-		  cfg->crud_config_file = strdup(optarg);
-		  ParodusInfo("crud_config_file is %s\n", cfg->crud_config_file);
-		  break;
+	case 'S':
+	  cfg->connection_health_file = strdup(optarg);
+	  ParodusInfo("connection_health_file is %s\n", cfg->connection_health_file);
+	  break;
+
+	case 'C':
+	  cfg->crud_config_file = strdup(optarg);
+	  ParodusInfo("crud_config_file is %s\n", cfg->crud_config_file);
+	  break;
 
 		case 'P':
 		cfg->client_cert_path = strdup(optarg);
@@ -543,6 +553,16 @@ int parseCommandLine(int argc,char **argv,ParodusCfg * cfg)
 		ParodusInfo("token_server_url is %s\n", cfg->token_server_url);
 		break;
 
+        case 'K':
+          cfg->mtls_client_key_path = strdup(optarg);
+          ParodusInfo("mtls_client_key_path is %s\n", cfg->mtls_client_key_path);
+          break;
+
+	case 'M':
+          cfg->mtls_client_cert_path = strdup(optarg);
+          ParodusInfo("mtls_client_cert_path is %s\n", cfg->mtls_client_cert_path);
+          break;
+
         case '?':
           /* getopt_long already printed an error message. */
           break;
@@ -616,6 +636,7 @@ void setDefaultValuesToCfg(ParodusCfg *cfg)
     parStrncpy(cfg->webpa_uuid, "1234567-345456546",sizeof(cfg->webpa_uuid));
     ParodusPrint("cfg->webpa_uuid is :%s\n", cfg->webpa_uuid);
     cfg->crud_config_file = NULL;
+    cfg->connection_health_file = NULL;
     cfg->client_cert_path = NULL;
     cfg->token_server_url = NULL;
 	
@@ -767,6 +788,15 @@ void loadParodusCfg(ParodusCfg * config,ParodusCfg *cfg)
     parStrncpy(cfg->webpa_uuid, "1234567-345456546",sizeof(cfg->webpa_uuid));
     ParodusPrint("cfg->webpa_uuid is :%s\n", cfg->webpa_uuid);
     
+    if(config->connection_health_file != NULL)
+    {
+        cfg->connection_health_file = strdup(config->connection_health_file);
+    }
+    else
+    {
+        ParodusPrint("connection_health_file is NULL. set to empty\n");
+    }
+
     if(config->crud_config_file != NULL)
     {
         cfg->crud_config_file = strdup(config->crud_config_file);

src/config.h

@@ -101,6 +101,9 @@ typedef struct
     char token_read_script[64];
     char *client_cert_path;
     char *token_server_url;
+    char *connection_health_file;
+    char *mtls_client_key_path;
+    char *mtls_client_cert_path;
 	char *crud_config_file;
 	char *cloud_status;
 	char *cloud_disconnect;

src/conn_interface.c

@@ -74,6 +74,7 @@ void createSocketConnection(void (* initKeypress)())
     //ParodusCfg *tmpCfg = (ParodusCfg*)config_in;
     noPollCtx *ctx;
     bool seshat_registered = false;
+    int create_conn_rtn = 0;
     unsigned int webpa_ping_timeout_ms = 1000 * get_parodus_cfg()->webpa_ping_timeout;
     unsigned int heartBeatTimer = 0;
     struct timespec start_svc_alive_timer;
@@ -94,9 +95,13 @@ void createSocketConnection(void (* initKeypress)())
     nopoll_log_set_handler (ctx, __report_log, NULL);
     #endif
 
-    if(!createNopollConnection(ctx))
+    start_conn_in_progress ();
+    create_conn_rtn = createNopollConnection(ctx);
+    stop_conn_in_progress ();
+    if(!create_conn_rtn)
     {
 		ParodusError("Unrecovered error, terminating the process\n");
+		OnboardLog("Unrecovered error, terminating the process\n");
 		abort();
     }
     packMetaData();
@@ -140,6 +145,7 @@ void createSocketConnection(void (* initKeypress)())
             {
                 ParodusError("ping wait time > %d . Terminating the connection with WebPA server and retrying\n", webpa_ping_timeout_ms / 1000);
                 ParodusInfo("Reconnect detected, setting Ping_Miss reason for Reconnect\n");
+                OnboardLog("Reconnect detected, setting Ping_Miss reason for Reconnect\n");
                 set_global_reconnect_reason("Ping_Miss");
                 set_global_reconnect_status(true);
                 set_close_retry();
@@ -177,9 +183,11 @@ void createSocketConnection(void (* initKeypress)())
 		free(get_parodus_cfg()->cloud_disconnect);
 		reset_cloud_disconnect_reason(get_parodus_cfg());
             }
+            start_conn_in_progress ();
             createNopollConnection(ctx);
+            stop_conn_in_progress ();
         }
-       } while(!get_close_retry() && !g_shutdown);
+       } while(!g_shutdown);
 
     pthread_mutex_lock (get_global_svc_mut());
     pthread_cond_signal (get_global_svc_con());

src/connection.c

@@ -296,21 +296,25 @@ void set_current_server (create_connection_ctx_t *ctx)
   ctx->current_server = get_current_server (&ctx->server_list);
 }
 
-void set_extra_headers (create_connection_ctx_t *ctx, int reauthorize)
-{
-  if (reauthorize && (get_parodus_cfg()->client_cert_path !=NULL && strlen(get_parodus_cfg()->client_cert_path) >0))
-  {
-    getAuthToken(get_parodus_cfg());
-  }
-  
-  ctx->extra_headers = build_extra_hdrs (&ctx->header_info);
-}
-
 void free_extra_headers (create_connection_ctx_t *ctx)
 {
   FREE_PTR_VAR (ctx->extra_headers)
 }
 
+void set_extra_headers (create_connection_ctx_t *ctx)
+{
+  ParodusCfg * cfg = get_parodus_cfg();
+
+  free_extra_headers (ctx);
+  if ((strlen(cfg->webpa_auth_token) == 0) &&
+      (cfg->client_cert_path != NULL) && (strlen(cfg->client_cert_path) > 0))
+  {
+    getAuthToken(cfg);
+  }
+  
+  ctx->extra_headers = build_extra_hdrs (&ctx->header_info);
+}
+
 void free_connection_ctx (create_connection_ctx_t *ctx)
 {
   free_extra_headers (ctx);
@@ -387,8 +391,10 @@ int nopoll_connect (create_connection_ctx_t *ctx, int is_ipv6)
    }
    if ((NULL == connection) && (!is_ipv6)) {
      if((checkHostIp(server->server_addr) == -2)) {
-       if (check_timer_expired (&ctx->connect_timer, 15*60*1000)) {
+       if (check_timer_expired (&ctx->connect_timer, 15*60*1000) && !get_interface_down_event()) {
   	 ParodusError("WebPA unable to connect due to DNS resolving to 10.0.0.1 for over 15 minutes; crashing service.\n");
+	 OnboardLog("WebPA unable to connect due to DNS resolving to 10.0.0.1 for over 15 minutes; crashing service.\n");
+	 OnboardLog("Reconnect detected, setting Dns_Res_webpa_reconnect reason for Reconnect\n");
 	 set_global_reconnect_reason("Dns_Res_webpa_reconnect");
 	 set_global_reconnect_status(true);
 						
@@ -404,7 +410,7 @@ int nopoll_connect (create_connection_ctx_t *ctx, int is_ipv6)
 //--------------------------------------------------------------------
 // Return codes for wait_connection_ready
 #define WAIT_SUCCESS	0
-#define WAIT_ACTION_RETRY	1	// if wait_status is 307, 302, 303 or 403
+#define WAIT_ACTION_RETRY	1	// if wait_status is 307, 302, 303, or 403
 #define WAIT_FAIL 	2
 
 #define FREE_NON_NULL_PTR(ptr) if (NULL != ptr) free(ptr)
@@ -440,10 +446,12 @@ int wait_connection_ready (create_connection_ctx_t *ctx)
   FREE_NON_NULL_PTR (redirectURL);
   if(wait_status == 403) 
   {
-	ParodusError("Received Unauthorized response with status: %d\n", wait_status);
-	free_extra_headers (ctx);
-	set_extra_headers (ctx, true);
-	return WAIT_ACTION_RETRY;
+    ParodusCfg *cfg = get_parodus_cfg();
+    /* clear auth token in cfg so that we will refetch auth token */
+    memset (cfg->webpa_auth_token, 0, sizeof(cfg->webpa_auth_token));
+    ParodusError("Received Unauthorized response with status: %d\n", wait_status);
+	OnboardLog("Received Unauthorized response with status: %d\n", wait_status);
+    return WAIT_ACTION_RETRY;
   }
   ParodusError("Client connection timeout\n");	
   ParodusError("RDK-10037 - WebPA Connection Lost\n");
@@ -453,9 +461,9 @@ int wait_connection_ready (create_connection_ctx_t *ctx)
  
 //--------------------------------------------------------------------
 // Return codes for connect_and_wait
-#define CONN_WAIT_SUCCESS	0
-#define CONN_WAIT_ACTION_RETRY	1	// if wait_status is 307, 302, 303 or 403
-#define CONN_WAIT_RETRY_DNS 	2
+#define CONN_WAIT_SUCCESS	 0
+#define CONN_WAIT_ACTION_RETRY	 1	// if wait_status is 307, 302, 303, or 403
+#define CONN_WAIT_RETRY_DNS 	 2
 
 int connect_and_wait (create_connection_ctx_t *ctx)
 {
@@ -514,9 +522,12 @@ int keep_trying_to_connect (create_connection_ctx_t *ctx,
     
     while (true)
     {
+      set_extra_headers (ctx);
+
       rtn = connect_and_wait (ctx);
       if (rtn == CONN_WAIT_SUCCESS)
         return true;
+
       if (rtn == CONN_WAIT_ACTION_RETRY) // if redirected or build_headers
         continue;
       backoff_delay (backoff_timer); // 3,7,15,31 ..
@@ -552,11 +563,11 @@ int createNopollConnection(noPollCtx *ctx)
 	
 	max_retry_count = (int) get_parodus_cfg()->webpa_backoff_max;
 	ParodusPrint("max_retry_count is %d\n", max_retry_count );
-  
+
+	memset (&conn_ctx, 0, sizeof(create_connection_ctx_t));
 	conn_ctx.nopoll_ctx = ctx;
 	init_expire_timer (&conn_ctx.connect_timer);
 	init_header_info (&conn_ctx.header_info);
-	set_extra_headers (&conn_ctx, false);
         set_server_list_null (&conn_ctx.server_list);
         init_backoff_timer (&backoff_timer, max_retry_count);
   
@@ -569,15 +580,26 @@ int createNopollConnection(noPollCtx *ctx)
 	  if (keep_trying_to_connect (&conn_ctx, &backoff_timer))
 		break;
 	  // retry dns query
+
+	  // If close_retry is reset due to interface down event, stop retry
+	  // and wait till close_retry is set again.
+	  if(!get_close_retry() && get_interface_down_event()) {
+		pthread_mutex_lock(get_global_close_retry_mut());
+		pthread_cond_wait(get_global_close_retry_con(), get_global_close_retry_mut());
+		pthread_mutex_unlock (get_global_close_retry_mut());
+		//&conn_ctx.header_info->conveyHeader = getWebpaConveyHeader(); 
+	  }
 	}
       
 	if(conn_ctx.current_server->allow_insecure <= 0)
 	{
 		ParodusInfo("Connected to server over SSL\n");
+		OnboardLog("Connected to server over SSL\n");
 	}
 	else 
 	{
 		ParodusInfo("Connected to server\n");
+		OnboardLog("Connected to server\n");
 	}
 	
 	get_parodus_cfg()->cloud_status = CLOUD_STATUS_ONLINE;
@@ -628,13 +650,20 @@ static char* build_extra_headers( const char *auth, const char *device_id,
 static noPollConnOpts * createConnOpts (char * extra_headers, bool secure)
 {
     noPollConnOpts * opts;
+    char * mtls_client_cert_path = NULL;
+    char * mtls_client_key_path = NULL;
     
     opts = nopoll_conn_opts_new ();
     if(secure) 
 	{
 	    if(strlen(get_parodus_cfg()->cert_path) > 0)
             {
-                nopoll_conn_opts_set_ssl_certs(opts, NULL, NULL, NULL, get_parodus_cfg()->cert_path);
+                if( ( get_parodus_cfg()->mtls_client_cert_path !=NULL && strlen(get_parodus_cfg()->mtls_client_cert_path) > 0) && (get_parodus_cfg()->mtls_client_key_path !=NULL && strlen(get_parodus_cfg()->mtls_client_key_path) > 0) )
+                {
+                       mtls_client_cert_path = get_parodus_cfg()->mtls_client_cert_path;
+                       mtls_client_key_path = get_parodus_cfg()->mtls_client_key_path;
+                }
+                nopoll_conn_opts_set_ssl_certs(opts, mtls_client_cert_path, mtls_client_key_path, NULL, get_parodus_cfg()->cert_path);
             }
 	    nopoll_conn_opts_ssl_peer_verify (opts, nopoll_true);
 	    nopoll_conn_opts_set_ssl_protocol (opts, NOPOLL_METHOD_TLSV1_2);
@@ -655,3 +684,38 @@ void close_and_unref_connection(noPollConn *conn)
     }
 }
 
+void write_conn_in_prog_file (const char *msg)
+{
+  int fd;
+  FILE *fp;
+  unsigned long timestamp;
+  ParodusCfg *cfg = get_parodus_cfg();
+
+  if (NULL == cfg->connection_health_file)
+    return;
+  fd = open (cfg->connection_health_file, O_CREAT | O_WRONLY | O_SYNC, 0666);
+  if (fd < 0) {
+    ParodusError ("Error(1) %d opening file %s\n", errno, cfg->connection_health_file);
+    return;
+  }
+  ftruncate (fd, 0);
+  fp = fdopen (fd, "w");
+  if (fp == NULL) {
+    ParodusError ("Error(2) %d opening file %s\n", errno, cfg->connection_health_file);
+    return;
+  }
+  timestamp = (unsigned long) time(NULL);
+  fprintf (fp, "{%s=%lu}\n", msg, timestamp);
+  fclose (fp);
+}
+
+void start_conn_in_progress (void)
+{
+  write_conn_in_prog_file ("START");
+}   
+
+void stop_conn_in_progress (void)
+{
+  write_conn_in_prog_file ("STOP");
+}   
+

src/connection.h

@@ -57,6 +57,13 @@ void set_global_reconnect_status(bool status);
 
 int get_cloud_disconnect_time();
 void set_cloud_disconnect_time(int disconnTime);
+
+/**
+ * @brief Interface to self heal connection in progress getting stuck
+ */
+void start_conn_in_progress (void);
+void stop_conn_in_progress (void);
+
 #ifdef __cplusplus
 }
 #endif

src/crud_internal.c

@@ -1346,6 +1346,7 @@ static int ConnDisconnectFromCloud(char *disconn_reason)
 	if(!close_retry)
 	{
 		ParodusInfo("Reconnect detected, setting reason %s for Reconnect\n", disconn_reason);
+		OnboardLog("Reconnect detected, setting reason %s for Reconnect\n", disconn_reason);
 		set_global_reconnect_reason(disconn_reason);
 		set_global_reconnect_status(true);
 		set_close_retry();

src/downstream.c

@@ -121,6 +121,9 @@ void listenerOnMessage(void * msg, size_t msgSize)
                         ParodusInfo("Received downstream dest as :%s and transaction_uuid :%s\n", dest, 
                             ((WRP_MSG_TYPE__REQ   == msgType) ? message->u.req.transaction_uuid : 
                             ((WRP_MSG_TYPE__EVENT == msgType) ? "NA" : message->u.crud.transaction_uuid)));
+			OnboardLog("%s\n",
+                            ((WRP_MSG_TYPE__REQ   == msgType) ? message->u.req.transaction_uuid :
+                            ((WRP_MSG_TYPE__EVENT == msgType) ? "NA" : message->u.crud.transaction_uuid)));
                         
                         free(destVal);
 

src/heartBeat.c

@@ -22,8 +22,10 @@
  */
 
 #include "heartBeat.h"
+#include <stdbool.h>
 
 volatile unsigned int heartBeatTimer = 0;
+volatile bool paused = false;
 
 pthread_mutex_t heartBeat_mut=PTHREAD_MUTEX_INITIALIZER;
 
@@ -49,8 +51,27 @@ void reset_heartBeatTimer()
 void increment_heartBeatTimer(unsigned int inc_time_ms) 
 {
 	pthread_mutex_lock (&heartBeat_mut);
-	heartBeatTimer += inc_time_ms;
+	if (!paused)
+	  heartBeatTimer += inc_time_ms;
+	pthread_mutex_unlock (&heartBeat_mut);
+}
+
+// Pause heartBeatTimer, i.e. stop incrementing
+void pause_heartBeatTimer()
+{
+	pthread_mutex_lock (&heartBeat_mut);
+	heartBeatTimer = 0;
+	paused = true;
+	pthread_mutex_unlock (&heartBeat_mut);
+}
+
+// Resume heartBeatTimer, i.e. resume incrementing
+void resume_heartBeatTimer()
+{
+	pthread_mutex_lock (&heartBeat_mut);
+	paused = false;
 	pthread_mutex_unlock (&heartBeat_mut);
 }
 
 
+

src/heartBeat.h

@@ -39,6 +39,12 @@ void reset_heartBeatTimer();
 // Increment value of heartBeatTimer to desired value
 void increment_heartBeatTimer(unsigned int inc_time_ms);
 
+// Pause heartBeatTimer, i.e. stop incrementing
+void pause_heartBeatTimer();
+
+// Resume heartBeatTimer, i.e. resume incrementing
+void resume_heartBeatTimer();
+
 #ifdef __cplusplus
 }
 #endif

src/main.c

@@ -83,7 +83,6 @@ int main( int argc, char **argv)
 		abort();
 	}
     curl_global_init(CURL_GLOBAL_DEFAULT);
-    getAuthToken(cfg);
      
     createSocketConnection( NULL);
     

src/networking.c

@@ -70,6 +70,7 @@ int checkHostIp(char * serverIP)
 	if (retVal != 0) 
 	{
 		ParodusError("getaddrinfo: %s\n", gai_strerror(retVal));
+		OnboardLog("getaddrinfo: %s\n", gai_strerror(retVal));
 	}
 	else
 	{

src/nopoll_handlers.c

@@ -167,16 +167,25 @@ void listenerOnCloseMessage (noPollCtx * ctx, noPollConn * conn, noPollPtr user_
     if( closeStatus == 1006 && !get_global_reconnect_status())
     {
 	ParodusInfo("Reconnect detected, setting default Reconnect reason %s\n",defaultReason);
+	OnboardLog("Reconnect detected, setting default Reconnect reason %s\n",defaultReason);
         set_global_reconnect_reason(defaultReason);
         set_global_reconnect_status(true);
     }
     else if(!get_global_reconnect_status())
     {
     	ParodusInfo("Reconnect detected, setting Reconnect reason as Unknown\n");
+	OnboardLog("Reconnect detected, setting Reconnect reason as Unknown\n");
         set_global_reconnect_reason("Unknown");
     }
+    
+    if(!get_interface_down_event())
+    {
+	ParodusInfo("Setting the close and retry connection\n");
+    	set_close_retry();
+    }
+    else
+    	ParodusInfo("Not Setting the close and retry connection as interface is down\n");
 
-    set_close_retry();
     ParodusPrint("listenerOnCloseMessage(): mutex unlock in producer thread\n");
 }
 

src/nopoll_helpers.c

@@ -73,25 +73,35 @@ void sendMessage(noPollConn *conn, void *msg, size_t len)
     else
     {
                 ParodusError("Failed to send msg upstream as connection is not OK\n");
-		if (connErr == 0)
-		{
-			getCurrentTime(connStuck_startPtr);
-			ParodusInfo("Conn got stuck, initialized the first timer\n");
-			connErr = 1;
-		}
-		else
-		{
-			getCurrentTime(connStuck_endPtr);
-			timeDiff = timeValDiff(connStuck_startPtr, connStuck_endPtr);
-			ParodusPrint("checking timeout difference:%ld\n", timeDiff);
+                OnboardLog("Failed to send msg upstream as connection is not OK\n");
 
-			if( timeDiff >= (10*60*1000))
+                if(get_interface_down_event())
+                {     
+                     ParodusError("Unable to connect to server since interface is down\n");
+                }
+                else
+                {
+			if (connErr == 0)
 			{
-				ParodusError("conn got stuck for over 10 minutes; crashing service.\n");
-				kill(getpid(),SIGTERM);
+				getCurrentTime(connStuck_startPtr);
+				ParodusInfo("Conn got stuck, initialized the first timer\n");
+				connErr = 1;
 			}
+			else
+			{
+				getCurrentTime(connStuck_endPtr);
+				timeDiff = timeValDiff(connStuck_startPtr, connStuck_endPtr);
+				ParodusPrint("checking timeout difference:%ld\n", timeDiff);
 
-		}
+				if( timeDiff >= (10*60*1000))
+				{
+					ParodusError("conn got stuck for over 10 minutes; crashing service.\n");
+					OnboardLog("conn got stuck for over 10 minutes; crashing service.\n");
+					kill(getpid(),SIGTERM);
+				}
+
+			}
+                 }
     }
 }
 
@@ -149,6 +159,7 @@ void __report_log (noPollCtx * ctx, noPollDebugLevel level, const char * log_msg
     if (level == NOPOLL_LEVEL_CRITICAL) 
     {
         ParodusError("%s\n", log_msg );
+	OnboardLog("%s\n", log_msg );
     }
     return;
 }

src/parodus_log.h

@@ -33,3 +33,9 @@
 #define ParodusError(...)                   cimplog_error(LOGGING_MODULE, __VA_ARGS__)
 #define ParodusInfo(...)                    cimplog_info(LOGGING_MODULE, __VA_ARGS__)
 #define ParodusPrint(...)                   cimplog_debug(LOGGING_MODULE, __VA_ARGS__)
+#ifdef FEATURE_SUPPORT_ONBOARD_LOGGING
+#define OnboardLog(...)                     onboarding_log(LOGGING_MODULE, __VA_ARGS__)
+#else
+#define OnboardLog(...)
+#endif
+

src/partners_check.c

@@ -178,6 +178,7 @@ int validate_partner_id(wrp_msg_t *msg, partners_t **partnerIds)
                 if(matchFlag != 1)
                 {
                     ParodusError("Invalid partner_id %s\n",temp);
+                    OnboardLog("Invalid partner_id %s\n",temp);
                     if(partnersList != NULL)
                     {
                         for(j=0; j<partnersList->count; j++)

src/token.c

@@ -139,6 +139,7 @@ int analyze_jwt (const cjwt_t *jwt, char **url_buf, unsigned int *port)
 		show_times (exp_time, cur_time);
 		if (exp_time < cur_time) {
 			ParodusError ("JWT has expired\n");
+			OnboardLog ("JWT has expired\n");
 			return TOKEN_ERR_JWT_EXPIRED;
 		}
 	}
@@ -146,6 +147,7 @@ int analyze_jwt (const cjwt_t *jwt, char **url_buf, unsigned int *port)
 		url_buf, port);
 	if (http_match < 0) {
 		ParodusError ("Invalid endpoint claim in JWT\n");
+		OnboardLog("Invalid endpoint claim in JWT\n");
 		return TOKEN_ERR_BAD_ENDPOINT;
 	}
 	ParodusInfo ("JWT is_http strncmp: %d\n", http_match);

src/upstream.c

@@ -190,6 +190,16 @@ void *handle_upstream()
                     ParodusError("failure in allocation for message\n");
                 }
             }
+	    if(nn_shutdown(sock, bind) < 0)
+	    {
+	        ParodusError ("nn_shutdown bind socket=%d endpt=%d, err=%d\n", 
+		    sock, bind, errno);
+	    }
+	    if (nn_close (sock) < 0)
+	    {
+	        ParodusError ("nn_close bind socket=%d err=%d\n", 
+		    sock, errno);
+	    }
         }
     }
     else
@@ -234,7 +244,7 @@ void *processUpstreamMessage()
             if(rv > 0)
             {
                 msgType = msg->msg_type;				   
-                if(msgType == 9)
+                if(msgType == WRP_MSG_TYPE__SVC_REGISTRATION)
                 {
                     ParodusInfo("\n Nanomsg client Registration for Upstream\n");
                     //Extract serviceName and url & store it in a linked list for reg_clients
@@ -249,12 +259,16 @@ void *processUpstreamMessage()
                             {
                                 ParodusInfo("match found, client is already registered\n");
                                 parStrncpy(temp->url,msg->u.reg.url, sizeof(temp->url));
-                                if(nn_shutdown(temp->sock, 0) < 0)
+                                if(nn_shutdown(temp->sock, temp->endpoint) < 0)
                                 {
-                                    ParodusError ("nn_shutdown socket=%d err=%d\n", 
+                                    ParodusError ("nn_shutdown socket=%d endpt=%d, err=%d\n", 
+					temp->sock, temp->endpoint, errno);
+                                }
+				if (nn_close (temp->sock) < 0)
+                                {
+                                    ParodusError ("nn_close socket=%d err=%d\n", 
 					temp->sock, errno);
                                 }
-				nn_close (temp->sock);
 
                                 temp->sock = nn_socket(AF_SP,NN_PUSH );
                                 if(temp->sock >= 0)
@@ -272,6 +286,7 @@ void *processUpstreamMessage()
                                     }
                                     else
                                     {
+                                        temp->endpoint = rc;
                                         ParodusInfo("Client registered before. Sending ack on socket %d\n", temp->sock); 
                                         status =sendAuthStatus(temp);
 
@@ -402,9 +417,8 @@ void *processUpstreamMessage()
 								{
 									ParodusError("Failed to get device_id\n");
 								}
-						}
-						else
-						{
+						} else if (WRP_MSG_TYPE__SVC_ALIVE != msgType) {
+						  /* Don't reply to service alive message */
 							sendUpstreamMsgToServer(&message->msg, message->len);
 						}
 					}
@@ -575,6 +589,7 @@ void sendUpstreamMsgToServer(void **resp_bytes, size_t resp_size)
 		else
 		{
 			ParodusInfo("close_retry is %d, unable to send response as connection retry is in progress\n", close_retry);
+			OnboardLog("close_retry is %d, unable to send response as connection retry is in progress\n", close_retry);
 		}
 		free(appendData);
 		appendData =NULL;

tests/CMakeLists.txt

@@ -117,13 +117,13 @@ target_link_libraries (test_nopoll_handlers_fragment -lnopoll -lcunit -lcimplog
 #-------------------------------------------------------------------------------
 #   test_connection
 #-------------------------------------------------------------------------------
-add_test(NAME test_connection COMMAND ${MEMORY_CHECK} ./test_connection)
-set (CONN_SRC ../src/connection.c 
-  ../src/string_helpers.c ../src/mutex.c ../src/time.c 
-  ../src/config.c ../src/auth_token.c ../src/spin_thread.c  ../src/heartBeat.c ../src/close_retry.c)
+#add_test(NAME test_connection COMMAND ${MEMORY_CHECK} ./test_connection)
+#set (CONN_SRC ../src/connection.c 
+#  ../src/string_helpers.c ../src/mutex.c ../src/time.c 
+#  ../src/config.c ../src/auth_token.c ../src/spin_thread.c  ../src/heartBeat.c ../src/close_retry.c)
 #set(CONN_SRC ../src/connection.c ${PARODUS_COMMON_SRC})
-add_executable(test_connection test_connection.c ${CONN_SRC})
-target_link_libraries (test_connection ${PARODUS_COMMON_LIBS} -lcmocka -lcurl -luuid)
+#add_executable(test_connection test_connection.c ${CONN_SRC})
+#target_link_libraries (test_connection ${PARODUS_COMMON_LIBS} -lcmocka -lcurl -luuid)
 
 #-------------------------------------------------------------------------------
 #   test_connection - function createNopollConnection
@@ -193,6 +193,17 @@ target_link_libraries (test_auth_token -lcmocka
  -Wl,--no-as-needed -lcimplog
  -lcjson -lcjwt -ltrower-base64 -lssl -lcrypto -lrt -lm -lcurl -luuid
 )
+
+#-------------------------------------------------------------------------------
+#   test_auth_token_more
+#-------------------------------------------------------------------------------
+add_test(NAME test_auth_token_more COMMAND ${MEMORY_CHECK} ./test_auth_token_more)
+add_executable(test_auth_token_more test_auth_token_more.c ../src/config.c ../src/auth_token.c ../src/string_helpers.c)
+target_link_libraries (test_auth_token_more -lcmocka
+ -Wl,--no-as-needed -lcimplog
+ -lcjson -lcjwt -ltrower-base64 -lssl -lcrypto -lrt -lm -lcurl -luuid
+)
+
 #-------------------------------------------------------------------------------
 #   test_crud_interface
 #-------------------------------------------------------------------------------
@@ -224,6 +235,17 @@ target_link_libraries (test_upstream -lcmocka gcov -lcunit -lcimplog
  -Wl,--no-as-needed -lcjson -lcjwt -ltrower-base64
  -lssl -lcrypto -lrt -lm)
 
+#-------------------------------------------------------------------------------
+#   test_upstream_sock
+#-------------------------------------------------------------------------------
+add_test(NAME test_upstream_sock COMMAND ${MEMORY_CHECK} ./test_upstream_sock)
+add_executable(test_upstream_sock test_upstream_sock.c ../src/upstream.c 
+  ../src/client_list.c ../src/close_retry.c ../src/string_helpers.c)
+target_link_libraries (test_upstream_sock -lcmocka gcov -lcunit -lcimplog
+ -lwrp-c -luuid -lpthread -lmsgpackc -lnopoll -lnanomsg
+ -Wl,--no-as-needed -lcjson -lcjwt -ltrower-base64
+ -lssl -lcrypto -lrt -lm)
+
 #-------------------------------------------------------------------------------
 #   test_downstream
 #-------------------------------------------------------------------------------

tests/test_auth_token.c

@@ -40,11 +40,19 @@ typedef enum {
   CURLINFO_TOTAL_TIME
 } CURLINFO;
 
+struct token_data test_data;
+
 int curl_easy_perform(CURL *curl)
 {
 	UNUSED(curl);
+	char *msg = "response";
+	int rtn;
+
 	function_called();
-	return (int) mock();
+	rtn = (int) mock();
+	if (0 == rtn)
+	  write_callback_fn (msg, 1, strlen(msg), &test_data);
+	return rtn;
 }
 
 int curl_easy_getinfo(CURL *curl, CURLINFO CURLINFO_RESPONSE_CODE, long response_code)
@@ -67,6 +75,7 @@ void getAuthToken_Null()
 	getAuthToken(&cfg);
 	set_parodus_cfg(&cfg);
 	assert( cfg.client_cert_path == NULL);
+	assert_int_equal (0, (int) cfg.webpa_auth_token[0]);
 }
 
 void getAuthToken_MacNull()
@@ -77,8 +86,10 @@ void getAuthToken_MacNull()
     getAuthToken(&cfg);
     set_parodus_cfg(&cfg);
     assert( cfg.client_cert_path == NULL);
+    assert_int_equal (0, (int) cfg.webpa_auth_token[0]);
 }
 
+#if 0
 void test_requestNewAuthToken_init_fail ()
 {
 	char token[32];
@@ -106,7 +117,7 @@ void test_requestNewAuthToken_init_fail ()
 	assert_int_equal (output, -1);
 	free(cfg.token_server_url);
 }
-
+#endif
 
 void test_requestNewAuthToken_failure ()
 {
@@ -121,6 +132,8 @@ void test_requestNewAuthToken_failure ()
 	parStrncpy(cfg.hw_mac , "123567892366", sizeof(cfg.hw_mac));
 	set_parodus_cfg(&cfg);
 
+	test_data.size = 0;
+	test_data.data = token;
 	will_return (curl_easy_perform, -1);
 	expect_function_calls (curl_easy_perform, 1);
 
@@ -132,6 +145,7 @@ void test_requestNewAuthToken_failure ()
 
 	requestNewAuthToken (token, sizeof(token), 2);
 	assert_int_equal (output, -1);
+	assert_int_equal (0, (int) token[0]);
 	free(cfg.token_server_url);
 }
 
@@ -150,6 +164,8 @@ void test_requestNewAuthToken ()
   parStrncpy(cfg.hw_mac , "123567892366", sizeof(cfg.hw_mac));
   set_parodus_cfg(&cfg);
 
+  test_data.size = 0;
+  test_data.data = token;
   will_return (curl_easy_perform, 0);
   expect_function_calls (curl_easy_perform, 1);
   will_return (curl_easy_getinfo, 0);
@@ -160,6 +176,7 @@ void test_requestNewAuthToken ()
 
   output = requestNewAuthToken (token, sizeof(token), 1);
   assert_int_equal (output, 0);
+  assert_string_equal (token, "response");
   free(cfg.token_server_url);
 }
 
@@ -197,6 +214,8 @@ void test_getAuthToken ()
 	expect_function_calls (curl_easy_getinfo, 1);
 
 	/* To test curl success case */
+	test_data.size = 0;
+	test_data.data = cfg.webpa_auth_token;
 	will_return (curl_easy_perform, 0);
 	expect_function_calls (curl_easy_perform, 1);
 
@@ -207,7 +226,7 @@ void test_getAuthToken ()
 	expect_function_calls (curl_easy_getinfo, 1);
 
 	getAuthToken(&cfg);
-
+	assert_string_equal (cfg.webpa_auth_token, "response");
 	free(cfg.client_cert_path);
 	free(cfg.token_server_url);
 }
@@ -224,6 +243,7 @@ void test_getAuthTokenFailure ()
 	parStrncpy(cfg.hw_serial_number, "Fer23u948590", sizeof(cfg.hw_serial_number));
 	parStrncpy(cfg.hw_mac , "123567892366", sizeof(cfg.hw_mac));
 	set_parodus_cfg(&cfg);
+
 	will_return (curl_easy_perform, -1);
 	expect_function_calls (curl_easy_perform, 1);
 
@@ -261,20 +281,35 @@ void test_getAuthTokenFailure ()
 
 void test_write_callback_fn ()
 {
-	void *buffer;
-	size_t size = 1;
-	size_t nmemb =8;
+	ParodusCfg *cfg;
+	size_t max_data_size = sizeof (cfg->webpa_auth_token);
+	char *buffer1 = "response1";
+	size_t buf1len = strlen(buffer1);
+        char *buffer2 = "R2";
+	size_t buf2len = strlen(buffer2);
+	char buffer3[max_data_size];
 	int out_len=0;
 	struct token_data data;
 	data.size = 0;
-	buffer = strdup("response");
 
-	data.data = (char *) malloc(sizeof(char) * 5);
+	data.data = (char *) malloc(max_data_size);
 	data.data[0] = '\0';
 
-	out_len = write_callback_fn(buffer, size, nmemb, &data);
-	assert_string_equal(data.data, buffer);
-	assert_int_equal( out_len, strlen(buffer));
+	out_len = write_callback_fn(buffer1, 1, buf1len, &data);
+	assert_string_equal(data.data, buffer1);
+	assert_int_equal (out_len, buf1len);
+	assert_int_equal (data.size, buf1len);
+
+	out_len = write_callback_fn(buffer2, 1, buf2len, &data);
+	assert_string_equal(data.data, "response1R2");
+	assert_int_equal (out_len, buf2len);
+	assert_int_equal (data.size, buf1len+buf2len);
+
+	memset (buffer3, 'x', max_data_size);
+	out_len = write_callback_fn(buffer3, 1, max_data_size, &data);
+	assert_int_equal (out_len, 0);
+	assert_int_equal (data.size, 0);
+
 	free(data.data);
 }
 
@@ -285,14 +320,14 @@ void test_write_callback_fn ()
 int main(void)
 {
     const struct CMUnitTest tests[] = {
+	cmocka_unit_test(test_write_callback_fn),
         cmocka_unit_test(test_requestNewAuthToken),
-	cmocka_unit_test(test_requestNewAuthToken_init_fail),
+	// cmocka_unit_test(test_requestNewAuthToken_init_fail),
 	cmocka_unit_test(test_requestNewAuthToken_failure),
 	cmocka_unit_test(getAuthToken_Null),
 	cmocka_unit_test(getAuthToken_MacNull),
 	cmocka_unit_test(test_getAuthToken),
 	cmocka_unit_test(test_getAuthTokenFailure),
-	cmocka_unit_test(test_write_callback_fn),
     };
 
     return cmocka_run_group_tests(tests, NULL, NULL);

tests/test_auth_token_more.c

@@ -0,0 +1,109 @@
+/**
+ *  Copyright 2010-2016 Comcast Cable Communications Management, LLC
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+#include <assert.h>
+#include <errno.h>
+#include <pthread.h>
+#include <malloc.h>
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+
+#include <CUnit/Basic.h>
+
+#include "../src/config.h"
+#include "../src/auth_token.h"
+#include "../src/ParodusInternal.h"
+
+extern int requestNewAuthToken(char *newToken, size_t len, int r_count);
+
+/*----------------------------------------------------------------------------*/
+/*                                   Mocks                                    */
+/*----------------------------------------------------------------------------*/
+typedef void CURL;
+
+typedef enum {
+  CURLINFO_RESPONSE_CODE    = 2,
+  CURLINFO_TOTAL_TIME
+} CURLINFO;
+
+struct token_data test_data;
+
+CURL *curl_easy_init  ()
+{
+	function_called();
+	return (CURL *) mock();
+}
+
+int curl_easy_perform(CURL *curl)
+{
+	UNUSED(curl);
+	char *msg = "response";
+	int rtn;
+
+	function_called();
+	rtn = (int) mock();
+	if (0 == rtn)
+	  write_callback_fn (msg, 1, strlen(msg), &test_data);
+	return rtn;
+}
+
+int curl_easy_getinfo(CURL *curl, CURLINFO CURLINFO_RESPONSE_CODE, long response_code)
+{
+	UNUSED(curl);
+	UNUSED(CURLINFO_RESPONSE_CODE);
+	UNUSED(response_code);
+	function_called();
+	return (int) mock();
+}
+/*----------------------------------------------------------------------------*/
+/*                                   Tests                                    */
+/*----------------------------------------------------------------------------*/
+void test_requestNewAuthToken_init_fail ()
+{
+	char token[32];
+	ParodusCfg cfg;
+	int output = -1;
+	memset(&cfg,0,sizeof(cfg));
+
+	cfg.token_server_url = strdup("https://dev.comcast.net/token");
+	parStrncpy(cfg.cert_path , "/etc/ssl/certs/ca-certificates.crt", sizeof(cfg.cert_path));
+	parStrncpy(cfg.hw_serial_number, "Fer23u948590", sizeof(cfg.hw_serial_number));
+	parStrncpy(cfg.hw_mac , "123567892366", sizeof(cfg.hw_mac));
+
+	set_parodus_cfg(&cfg);
+
+	will_return (curl_easy_init, NULL);
+	expect_function_calls (curl_easy_init, 1);
+
+	requestNewAuthToken (token, sizeof(token), 2);
+	assert_int_equal (output, -1);
+	assert_int_equal (0, (int) token[0]);
+	free(cfg.token_server_url);
+}
+
+/*----------------------------------------------------------------------------*/
+/*                             External Functions                             */
+/*----------------------------------------------------------------------------*/
+
+int main(void)
+{
+    const struct CMUnitTest tests[] = {
+	cmocka_unit_test(test_requestNewAuthToken_init_fail)
+    };
+
+    return cmocka_run_group_tests(tests, NULL, NULL);
+}

tests/test_conn_interface.c

@@ -74,6 +74,15 @@ noPollMutexUnlock 	mutex_unlock
     UNUSED(mutex_create); UNUSED(mutex_destroy); UNUSED(mutex_lock); UNUSED(mutex_unlock);
     function_called();
 }
+
+void start_conn_in_progress (void)
+{
+}   
+
+void stop_conn_in_progress (void)
+{
+}   
+
 void packMetaData()
 {
     function_called();

tests/test_connection.c

@@ -117,6 +117,8 @@ nopoll_bool  nopoll_conn_wait_for_status_until_connection_ready (noPollConn * co
 	int timeout, int *status, char ** message)
 {
     UNUSED(conn); UNUSED(timeout);
+
+    *message = NULL;
     if (mock_wait_status >= 1000) {
       *status = mock_wait_status / 1000;
       mock_wait_status = mock_wait_status % 1000;
@@ -375,6 +377,15 @@ void test_set_current_server()
   assert_ptr_equal (&ctx.server_list.defaults, ctx.current_server);
 }
 
+void init_cfg_header_info (ParodusCfg *cfg)
+{
+  parStrncpy(cfg->hw_mac , "123567892366", sizeof(cfg->hw_mac));
+  parStrncpy(cfg->hw_model, "TG1682", sizeof(cfg->hw_model));
+  parStrncpy(cfg->hw_manufacturer , "ARRISGroup,Inc.", sizeof(cfg->hw_manufacturer));
+  parStrncpy(cfg->fw_name , "2.364s2", sizeof(cfg->fw_name));
+  parStrncpy(cfg->webpa_protocol , "WebPA-1.6", sizeof(cfg->webpa_protocol));
+}
+
 void test_set_extra_headers ()
 {
   int rtn;
@@ -391,12 +402,8 @@ void test_set_extra_headers ()
 
   cfg.client_cert_path = strdup("testcert");
   parStrncpy(cfg.hw_serial_number, "Fer23u948590", sizeof(cfg.hw_serial_number));
-  parStrncpy(cfg.hw_mac , "123567892366", sizeof(cfg.hw_mac));
-  parStrncpy(cfg.hw_model, "TG1682", sizeof(cfg.hw_model));
-  parStrncpy(cfg.hw_manufacturer , "ARRISGroup,Inc.", sizeof(cfg.hw_manufacturer));
-  parStrncpy(cfg.fw_name , "2.364s2", sizeof(cfg.fw_name));
+  init_cfg_header_info (&cfg);
   parStrncpy(cfg.cert_path , "/etc/ssl/certs/ca-certificates.crt", sizeof(cfg.cert_path));
-  parStrncpy(cfg.webpa_protocol , "WebPA-1.6", sizeof(cfg.webpa_protocol));
 	
   set_parodus_cfg(&cfg);
   rtn = init_header_info (&ctx.header_info);
@@ -578,19 +585,12 @@ void test_nopoll_connect ()
 
 // Return codes for wait_connection_ready
 #define WAIT_SUCCESS	0
-#define WAIT_ACTION_RETRY	1	// if wait_status is 307, 302, 303 or 403
+#define WAIT_ACTION_RETRY	1	// if wait_status is 307, 302, 303, or 403
 #define WAIT_FAIL 	2
 
 void test_wait_connection_ready ()
 {
   create_connection_ctx_t ctx;
-  ParodusCfg Cfg;
-  const char *expected_extra_headers =
-      "\r\nAuthorization: Bearer Auth---"
-      "\r\nX-WebPA-Device-Name: mac:123567892366"
-      "\r\nX-WebPA-Device-Protocols: wrp-0.11,getset-0.1"
-      "\r\nUser-Agent: WebPA-1.6 (2.364s2; TG1682/ARRISGroup,Inc.;)"
-      "\r\nX-WebPA-Convey: WebPA-1.6 (TG1682)";
 
   memset(&ctx,0,sizeof(ctx));
   set_server_list_null (&ctx.server_list);
@@ -605,7 +605,7 @@ void test_wait_connection_ready ()
   will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_false);
   expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
   assert_int_equal (wait_connection_ready (&ctx), WAIT_FAIL);
-  
+
   mock_wait_status = 503;
   will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_false);
   expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
@@ -640,30 +640,18 @@ void test_wait_connection_ready ()
   free_server (&ctx.server_list.redirect);
   
     mock_wait_status = 403;
-    memset(&Cfg, 0, sizeof(ParodusCfg));
-
-    parStrncpy (Cfg.webpa_auth_token, "Auth---", sizeof (Cfg.webpa_auth_token));
-    parStrncpy(Cfg.hw_model, "TG1682", sizeof(Cfg.hw_model));
-    parStrncpy(Cfg.hw_manufacturer , "ARRISGroup,Inc.", sizeof(Cfg.hw_manufacturer));
-    parStrncpy(Cfg.hw_mac , "123567892366", sizeof(Cfg.hw_mac));
-    parStrncpy(Cfg.fw_name , "2.364s2", sizeof(Cfg.fw_name));
-    parStrncpy(Cfg.webpa_protocol , "WebPA-1.6", sizeof(Cfg.webpa_protocol));
-    set_parodus_cfg(&Cfg);
-  
-    init_header_info (&ctx.header_info);
     will_return (nopoll_conn_wait_for_status_until_connection_ready, nopoll_false);
     expect_function_call (nopoll_conn_wait_for_status_until_connection_ready);
     assert_int_equal (wait_connection_ready (&ctx), WAIT_ACTION_RETRY);
-    
-    assert_string_equal (ctx.extra_headers, expected_extra_headers);
+
     free_extra_headers (&ctx);
     free_header_info (&ctx.header_info);
 }
 
 // Return codes for connect_and_wait
 #define CONN_WAIT_SUCCESS	0
-#define CONN_WAIT_ACTION_RETRY	1	// if wait_status is 307, 302, 303 or 403
-#define CONN_WAIT_RETRY_DNS 	2
+#define CONN_WAIT_ACTION_RETRY	 1	// if wait_status is 307, 302, 303, or 403
+#define CONN_WAIT_RETRY_DNS 	 2
 
 void test_connect_and_wait ()
 {
@@ -815,22 +803,16 @@ void test_keep_trying ()
   server_t test_server;
   backoff_timer_t backoff_timer;
   ParodusCfg Cfg;
-  char *test_extra_headers =
-      "\r\nAuthorization: Bearer SER_MAC Fer23u948590 123567892366"
-      "\r\nX-WebPA-Device-Name: mac:123567892366"
-      "\r\nX-WebPA-Device-Protocols: wrp-0.11,getset-0.1"
-      "\r\nUser-Agent: WebPA-1.6 (2.364s2; TG1682/ARRISGroup,Inc.;)"
-      "\r\nX-WebPA-Convey: WebPA-1.6 (TG1682)";
 
   memset(&Cfg, 0, sizeof(ParodusCfg));
   parStrncpy (Cfg.webpa_url, "http://mydns.mycom.net:8080", sizeof(Cfg.webpa_url));
+  init_cfg_header_info (&Cfg);
 
   mock_wait_status = 0;
 
   memset(&ctx,0,sizeof(ctx));
   ctx.nopoll_ctx = &test_nopoll_ctx;
   ctx.current_server = &test_server;
-  ctx.extra_headers = test_extra_headers;
 
   test_server.allow_insecure = 1;
   test_server.server_addr = "mydns.mycom.net";

tests/test_nopoll_handlers.c

@@ -21,6 +21,7 @@
 #include <nopoll_private.h>
 #include <pthread.h>
 
+#include "../src/ParodusInternal.h"
 #include "../src/nopoll_handlers.h"
 #include "../src/parodus_log.h"
 
@@ -46,6 +47,11 @@ bool get_global_reconnect_status()
     return LastReasonStatus;
 }
 
+bool get_interface_down_event()
+{
+     return false;
+}
+
 void set_global_reconnect_status(bool status)
 {
     (void) status ;

tests/test_nopoll_handlers_fragment.c

@@ -48,6 +48,11 @@ bool get_global_reconnect_status()
     return LastReasonStatus;
 }
 
+bool get_interface_down_event()
+{
+     return false;
+}
+
 void set_global_reconnect_status(bool status)
 {
     (void) status ;

tests/test_nopoll_helpers.c

@@ -16,6 +16,7 @@
  
 #include <stdarg.h>
 #include <stddef.h>
+#include <stdbool.h>
 #include <setjmp.h>
 #include <cmocka.h>
 #include <nopoll.h>
@@ -116,6 +117,11 @@ int kill(pid_t pid, int sig)
     return (int) mock();
 }
 
+bool get_interface_down_event()
+{
+     return false;
+}
+
 /*----------------------------------------------------------------------------*/
 /*                                   Tests                                    */
 /*----------------------------------------------------------------------------*/

tests/test_upstream.c

@@ -273,6 +273,10 @@ void test_handleUpstreamNull()
     expect_function_call(nn_bind);
     will_return(nn_recv, 12);
     expect_function_call(nn_recv);
+    will_return(nn_shutdown, 0);
+    expect_function_call(nn_shutdown);
+    will_return(nn_close, 0);
+    expect_function_call(nn_close);
     handle_upstream();
 }
 
@@ -294,6 +298,10 @@ void test_handle_upstream()
     expect_function_call(nn_bind);
     will_return(nn_recv, 12);
     expect_function_call(nn_recv);
+    will_return(nn_shutdown, 0);
+    expect_function_call(nn_shutdown);
+    will_return(nn_close, 0);
+    expect_function_call(nn_close);
     handle_upstream();
     free(UpStreamMsgQ->next);
     free(UpStreamMsgQ);

tests/test_upstream_sock.c

@@ -0,0 +1,249 @@
+/**
+ *  Copyright 2010-2016 Comcast Cable Communications Management, LLC
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+#include <assert.h>
+#include <errno.h>
+#include <pthread.h>
+#include <malloc.h>
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <cmocka.h>
+
+#include <nopoll.h>
+#include <wrp-c.h>
+#include <nanomsg/nn.h>
+
+#include "../src/upstream.h"
+#include "../src/config.h"
+#include "../src/client_list.h"
+#include "../src/ParodusInternal.h"
+#include "../src/partners_check.h"
+#include "../src/close_retry.h"
+
+/*----------------------------------------------------------------------------*/
+/*                            File Scoped Variables                           */
+/*----------------------------------------------------------------------------*/
+#define GOOD_CLIENT_URL "tcp://127.0.0.1:6667"
+
+static noPollConn *conn;
+static char *reconnect_reason = "webpa_process_starts";
+bool g_shutdown  = false;
+static ParodusCfg parodusCfg;
+extern size_t metaPackSize;
+extern UpStreamMsg *UpStreamMsgQ;
+int numLoops = 1;
+int deviceIDNull =0;
+wrp_msg_t *reg_msg = NULL;
+extern pthread_mutex_t nano_mut;
+extern pthread_cond_t nano_con;
+
+/*----------------------------------------------------------------------------*/
+/*                                   Mocks                                    */
+/*----------------------------------------------------------------------------*/
+reg_list_item_t *get_reg_list()
+{
+  reg_list_item_t *item = get_global_node();
+  release_global_node();
+  return item;
+}
+
+noPollConn *get_global_conn()
+{
+    return conn;   
+}
+
+char *get_global_reconnect_reason()
+{
+    return reconnect_reason;
+}
+
+void addCRUDmsgToQueue(wrp_msg_t *crudMsg)
+{
+	(void)crudMsg;
+	function_called();
+	return;
+}
+
+void sendMessage(noPollConn *conn, void *msg, size_t len)
+{
+    (void) conn; (void) msg; (void) len;
+    function_called();
+}
+
+void set_parodus_cfg(ParodusCfg *cfg)
+{
+    memcpy(&parodusCfg, cfg, sizeof(ParodusCfg));
+}
+
+ParodusCfg *get_parodus_cfg(void) 
+{
+	ParodusCfg cfg;
+	memset(&cfg,0,sizeof(cfg));
+	parStrncpy(cfg.hw_mac , "14cfe2142xxx", sizeof(cfg.hw_mac));
+	if(deviceIDNull)
+	{
+		parStrncpy(cfg.hw_mac , "", sizeof(cfg.hw_mac));
+	}
+	set_parodus_cfg(&cfg);
+    return &parodusCfg;
+}
+
+/*-------------------------------------------
+int nn_connect (int s, const char *addr)
+{
+    (void) s; (void) addr;
+    printf ("nn_connect, socket %d\n", s);
+    return 1;
+}
+---------------------------------------------*/
+
+ssize_t wrp_pack_metadata( const data_t *packData, void **data )
+{
+    (void) packData; (void) data;
+    function_called();
+    
+    return (ssize_t)mock();
+}
+
+size_t appendEncodedData( void **appendData, void *encodedBuffer, size_t encodedSize, void *metadataPack, size_t metadataSize )
+{
+    (void) encodedBuffer; (void) encodedSize; (void) metadataPack; (void) metadataSize;
+    function_called();
+    char *data = (char *) malloc (sizeof(char) * 100);
+    parStrncpy(data, "AAAAAAAAYYYYIGkYTUYFJH", 100);
+    *appendData = data;
+    return (size_t)mock();
+}
+
+int nn_send (int s, const void *buf, size_t len, int flags)
+{
+    UNUSED(s); UNUSED(buf); UNUSED(len); UNUSED(flags);
+    function_called();
+    return (int)mock();
+}
+
+int pthread_cond_wait(pthread_cond_t *restrict cond, pthread_mutex_t *restrict mutex)
+{
+    UNUSED(cond); UNUSED(mutex);
+    function_called();
+    return (int)mock();
+}
+
+ssize_t wrp_to_struct( const void *bytes, const size_t length, const enum wrp_format fmt, wrp_msg_t **msg )
+{
+    UNUSED(bytes); UNUSED(length); UNUSED(fmt);
+    function_called();
+    *msg = reg_msg;
+    return (ssize_t)mock();
+}
+
+void wrp_free_struct( wrp_msg_t *msg )
+{
+    UNUSED(msg);
+    function_called();
+}
+
+int nn_freemsg (void *msg)
+{
+    UNUSED(msg);
+    function_called();
+    return (int)mock();
+}
+
+int validate_partner_id(wrp_msg_t *msg, partners_t **partnerIds)
+{
+    UNUSED(msg); UNUSED(partnerIds);
+    function_called();
+    return (int) mock();
+}
+
+/*----------------------------------------------------------------------------*/
+/*                                   Tests                                    */
+/*----------------------------------------------------------------------------*/
+
+
+void test_message()
+{
+    metaPackSize = 20;
+    UpStreamMsgQ = (UpStreamMsg *) malloc(sizeof(UpStreamMsg));
+    UpStreamMsgQ->msg = "First Message";
+    UpStreamMsgQ->len = 13;
+    UpStreamMsgQ->next = NULL;
+
+    numLoops = 1;
+    reg_msg = (wrp_msg_t *) malloc(sizeof(wrp_msg_t));
+    memset(reg_msg,0,sizeof(wrp_msg_t));
+    reg_msg->msg_type = WRP_MSG_TYPE__SVC_REGISTRATION;
+    reg_msg->u.reg.service_name = "config";
+    reg_msg->u.reg.url = GOOD_CLIENT_URL;
+
+    will_return(wrp_to_struct, 12);
+    expect_function_call(wrp_to_struct);
+
+    will_return(nn_send, 1);
+    expect_function_call(nn_send);
+
+    will_return(nn_freemsg, 0);
+    expect_function_call(nn_freemsg);
+    expect_function_call(wrp_free_struct);
+
+    processUpstreamMessage();
+    free(reg_msg);
+    free(UpStreamMsgQ);
+}
+
+void test_processUpstreamMessage()
+{
+  int last_sock = -1;
+  reg_list_item_t * reg_item = get_reg_list ();
+
+  assert_null (reg_item);
+  test_message();
+  reg_item = get_reg_list ();
+  assert_non_null (reg_item);
+  if (NULL == reg_item)
+    return;
+
+  last_sock = reg_item->sock;
+  test_message ();
+  assert_int_equal (get_numOfClients(), 1);
+  if (get_numOfClients() != 1)
+    return;
+  reg_item = get_reg_list ();
+  assert_int_equal (last_sock, reg_item->sock);
+  if (last_sock != reg_item->sock)
+    return;
+
+  test_message ();
+  assert_int_equal (get_numOfClients(), 1);
+  if (get_numOfClients() != 1)
+    return;
+  reg_item = get_reg_list ();
+  assert_int_equal (last_sock, reg_item->sock);
+
+}
+
+/*----------------------------------------------------------------------------*/
+/*                             External Functions                             */
+/*----------------------------------------------------------------------------*/
+
+int main(void)
+{
+    const struct CMUnitTest tests[] = {
+        cmocka_unit_test(test_processUpstreamMessage)
+    };
+    return cmocka_run_group_tests(tests, NULL, NULL);
+}