WIFI-540: Introduce methods to cleanup stale GW and Routing record entries when we bounce the GW pod and it gets a new IP Address

opensync-ext-cloud/pom.xml

@@ -28,6 +28,11 @@
         <groupId>com.telecominfraproject.wlan</groupId>
         <version>${tip-wlan-cloud.release.version}</version>
       </dependency>
+        <dependency>
+            <artifactId>base-client</artifactId>
+            <groupId>com.telecominfraproject.wlan</groupId>
+            <version>${tip-wlan-cloud.release.version}</version>
+        </dependency>
 
       <dependency>
         <artifactId>equipment-gateway-models</artifactId>

opensync-ext-cloud/src/main/java/com/telecominfraproject/wlan/opensync/external/integration/controller/OpensyncCloudGatewayController.java

@@ -14,6 +14,9 @@ import java.util.function.Consumer;
 
 import javax.servlet.http.HttpServletRequest;
 
+import com.telecominfraproject.wlan.core.model.service.GatewayType;
+import com.telecominfraproject.wlan.core.client.PingClient;
+import com.telecominfraproject.wlan.datastore.exceptions.DsEntityNotFoundException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -78,6 +81,9 @@ public class OpensyncCloudGatewayController {
     @Autowired
     private OvsdbClientInterface tipwlanOvsdbClient;
 
+    @Autowired
+    private PingClient pingClient;
+
     /**
      * Flag indicates if this gateway has registered with routing service
      */
@@ -215,7 +221,6 @@ public class OpensyncCloudGatewayController {
      *
      * @param session
      * @param command
-     * @param protocolVersion
      * @return NoRouteToCE if route Id does not match or Success
      */
     private EquipmentCommandResponse checkEquipmentRouting(OvsdbSession session, CEGWRouteCheck command) {
@@ -267,7 +272,6 @@ public class OpensyncCloudGatewayController {
      * @param session
      * @param inventoryId
      * @param command
-     * @param request
      * @return
      */
     private EquipmentCommandResponse sendMessage(OvsdbSession session, String inventoryId, EquipmentCommand command) {
@@ -373,6 +377,8 @@ public class OpensyncCloudGatewayController {
                 throw new ConfigurationException(
                         "Unable to register gateway with routing service: routing service interface not initialized");
             }
+
+            cleanupStaleGwRecord();
             EquipmentGatewayRecord gwRecord = new EquipmentGatewayRecord();
 
             // external facing service, protected by the client certificate auth
@@ -396,6 +402,42 @@ public class OpensyncCloudGatewayController {
         }
     }
 
+    /**
+     * This method does the following:
+     * See WIFI-540
+     * 1. Retrieves the existing list of Gateway entries from the Routing Service
+     * 2. Check each one of them for reachability (using PING method)
+     * 3. If the Gw does not respond (stale IP), they will be unregistered/cleaned
+     */
+    protected void cleanupStaleGwRecord() {
+        LOG.debug("In CleanUp stale registered Gateways records ");
+        // Get Equipment gateway list
+        List<EquipmentGatewayRecord> eqGwRecList = eqRoutingSvc.getGateway(GatewayType.CEGW);
+        if (eqGwRecList != null) {
+            for (EquipmentGatewayRecord eqpRec : eqGwRecList) {
+                if (!isGwReachable(eqpRec.getIpAddr(), eqpRec.getPort())) {
+                    // GW isn't reachable --> invoke deleteGw
+                    LOG.debug("Gateway {} is not-reachable... deleting from Routing Svc", eqpRec.getHostname());
+                    try {
+                        eqRoutingSvc.deleteGateway(eqpRec.getId());
+                    } catch (RuntimeException e) {
+                        // failed
+                        LOG.error("Failed to delete Equipment Gateway (name={}) from Routing Service: {}",
+                                eqpRec.getHostname(), e.getLocalizedMessage());
+                    }
+                } else {
+                    LOG.debug("Gateway {} is reachable.", eqpRec.getHostname());
+                }
+            }
+        } else {
+            LOG.debug("No gateways registered with Routing Service");
+        }
+    }
+
+    private boolean isGwReachable(String ipAddr, int port) {
+        return pingClient.isReachable(ipAddr, port);
+    }
+
     /**
      * Return the current deployment identifier
      *
@@ -445,6 +487,8 @@ public class OpensyncCloudGatewayController {
                     equipmentId);
             return null;
         }
+        // Clean up stale records
+        cleanupStaleEqptRoutingRecord(equipmentId);
         EquipmentRoutingRecord routingRecord = new EquipmentRoutingRecord();
         routingRecord.setCustomerId(customerId);
         routingRecord.setEquipmentId(equipmentId);
@@ -463,6 +507,61 @@ public class OpensyncCloudGatewayController {
         return null;
     }
 
+    /**
+     * Deletes the Equipment to Gateway relationship for gateway's that don't respond
+     * See WIFI-540
+     * 1. Get List of EquipmentRoutingRecords for an Equipment
+     * 2. Get the GW from GW-Id associated with 'this' EquipmentRoutingRecord
+     * 3. Try to ping the gateway
+     * 4. If ping fails or Gateway does not exist, delete the equipmentRouting entry.
+     * @param equipmentId: Equipment's ID
+     */
+    protected void cleanupStaleEqptRoutingRecord(Long equipmentId) {
+        LOG.debug("In Clean Up stale Equipment Routing record for Equipment ID {}", equipmentId);
+        List<EquipmentRoutingRecord> eqptRoutingRecsList = eqRoutingSvc.getRegisteredRouteList(equipmentId);
+        if (eqptRoutingRecsList != null) {
+            for(EquipmentRoutingRecord eqRouting : eqptRoutingRecsList) {
+                try {
+                    EquipmentGatewayRecord gwRec = eqRoutingSvc.getGateway(eqRouting.getGatewayId());
+                    if (gwRec != null) {
+                        if (!isGwReachable(gwRec.getIpAddr(), gwRec.getPort())) {
+                            // GW isn't reachable --> invoke unregister
+                            LOG.debug("Gateway {} is not-reachable... Deleting the equipment routing entry", gwRec.getHostname());
+                            deleteUnresponiveGwRoutingRecord(eqRouting.getId(), equipmentId);
+                        } else {
+                            LOG.debug("Gateway {} is reachable.", gwRec.getHostname());
+                        }
+                    } else {
+                        LOG.debug("Gateway with ID {} not found. Deleting the equipment routing entry ", eqRouting.getGatewayId());
+                        deleteUnresponiveGwRoutingRecord(eqRouting.getId(), equipmentId);
+                    }
+                } catch ( DsEntityNotFoundException entityNotFoundException) {
+                    LOG.debug("Gateway ID: {} not found... Deleting the equipment routing entry", eqRouting.getGatewayId());
+                    deleteUnresponiveGwRoutingRecord(eqRouting.getId(), equipmentId);
+                } catch ( Exception genericException) {
+                    LOG.debug("Generic Exception encountered when trying to query/delete " +
+                            "the Gateway with ID: {}. Error: {} ", eqRouting.getGatewayId(), genericException.getMessage());
+                }
+            }
+
+        } else {
+            LOG.debug("No gateways registered with Routing Service for Equipment ID {}", equipmentId);
+        }
+
+    }
+
+    private boolean deleteUnresponiveGwRoutingRecord(Long routingId, Long eqptId) {
+        try {
+            eqRoutingSvc.delete(routingId);
+        } catch (RuntimeException e) {
+            // failed
+            LOG.error("Failed to delete Equipment routing record (ID={}) from Routing Service: {}",
+                    eqptId, e.getLocalizedMessage());
+            return false;
+        }
+        return true;
+    }
+
     public void deregisterCustomerEquipment(Long routingId, String equipmentName, Long equipmentId) {
         if (!registeredWithRoutingService) {
             LOG.error("Unable to deregister customer equipement (name={},id={}): gateway not registered", equipmentName,

opensync-ext-cloud/src/test/java/com/telecominfraproject/wlan/opensync/external/integration/controller/OpensyncCloudGatewayControllerTest.java

@@ -0,0 +1,119 @@
+package com.telecominfraproject.wlan.opensync.external.integration.controller;
+
+import com.telecominfraproject.wlan.core.client.PingClient;
+import com.telecominfraproject.wlan.core.model.service.GatewayType;
+import com.telecominfraproject.wlan.datastore.exceptions.DsEntityNotFoundException;
+import com.telecominfraproject.wlan.routing.RoutingServiceInterface;
+import com.telecominfraproject.wlan.routing.models.EquipmentGatewayRecord;
+import com.telecominfraproject.wlan.routing.models.EquipmentRoutingRecord;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.atLeast;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.verify;
+
+
+class OpensyncCloudGatewayControllerTest {
+
+    OpensyncCloudGatewayController opensyncCloudGatewayController = new OpensyncCloudGatewayController();
+
+    RoutingServiceInterface routingSvc = Mockito.mock(RoutingServiceInterface.class);
+    PingClient pingClient = Mockito.mock(PingClient.class);
+
+    @BeforeEach
+    void setUp() {
+        ReflectionTestUtils.setField(opensyncCloudGatewayController, "eqRoutingSvc", routingSvc);
+        ReflectionTestUtils.setField(opensyncCloudGatewayController, "pingClient", pingClient);
+    }
+
+    @Test
+    void cleanupStaleGwRecord() throws IOException {
+        EquipmentGatewayRecord gatewayRecord = readGatewayRecord();
+        Mockito.when(pingClient.isReachable(anyString(), anyInt())).thenReturn(true);
+        opensyncCloudGatewayController.cleanupStaleGwRecord();
+        verify(pingClient, atLeastOnce()).isReachable("1.1.1.1",123);
+    }
+
+    @Test
+    void cleanupStaleGwRecord_pingFail() throws IOException {
+        EquipmentGatewayRecord gatewayRecord = readGatewayRecord();
+        Mockito.when(pingClient.isReachable(anyString(), anyInt())).thenReturn(false);
+        Mockito.when(routingSvc.deleteGateway(anyLong())).thenReturn(gatewayRecord);
+        opensyncCloudGatewayController.cleanupStaleGwRecord();
+        verify(routingSvc, atLeastOnce()).deleteGateway(gatewayRecord.getId());
+    }
+
+    @Test
+    void cleanupStaleEqptRoutingRecord_NoEntityFound() throws IOException {
+        EquipmentRoutingRecord routingRecord = readRoutingRecord();
+        Mockito.when(routingSvc.getGateway(anyLong())).thenThrow(new DsEntityNotFoundException());
+        Mockito.when(routingSvc.delete(anyLong())).thenReturn(new EquipmentRoutingRecord());
+
+        opensyncCloudGatewayController.cleanupStaleEqptRoutingRecord(123L);
+        verify(routingSvc, atLeastOnce()).delete(routingRecord.getId());
+
+    }
+
+    @Test
+    void cleanupStaleEqptRoutingRecord_GenericException() throws IOException {
+        EquipmentRoutingRecord routingRecord = readRoutingRecord();
+        Mockito.when(routingSvc.getGateway(anyLong())).thenThrow(new RuntimeException("Throwing exception to test"));
+        Mockito.when(routingSvc.delete(anyLong())).thenReturn(new EquipmentRoutingRecord());
+
+        opensyncCloudGatewayController.cleanupStaleEqptRoutingRecord(123L);
+        verify(routingSvc, atLeast(0)).delete(routingRecord.getId());
+    }
+
+    @Test
+    void cleanupStaleEqptRoutingRecord_GatewayResponds() throws IOException {
+        EquipmentRoutingRecord routingRecord = readRoutingRecord();
+        EquipmentGatewayRecord gatewayRecord = readGatewayRecord();
+        Mockito.when(routingSvc.getGateway(anyLong())).thenReturn(gatewayRecord);
+        Mockito.when(routingSvc.delete(anyLong())).thenReturn(new EquipmentRoutingRecord());
+        Mockito.when(pingClient.isReachable(anyString(), anyInt())).thenReturn(true);
+
+        opensyncCloudGatewayController.cleanupStaleEqptRoutingRecord(123L);
+        verify(routingSvc, atLeast(0)).delete(routingRecord.getId());
+        verify(pingClient, atLeastOnce()).isReachable(anyString(), anyInt());
+    }
+
+    @Test
+    void cleanupStaleEqptRoutingRecord_GatewayFails() throws IOException {
+        EquipmentRoutingRecord routingRecord = readRoutingRecord();
+        EquipmentGatewayRecord gatewayRecord = readGatewayRecord();
+        Mockito.when(routingSvc.getGateway(anyLong())).thenReturn(gatewayRecord);
+        Mockito.when(routingSvc.delete(anyLong())).thenReturn(new EquipmentRoutingRecord());
+        Mockito.when(pingClient.isReachable(anyString(), anyInt())).thenReturn(false);
+
+        opensyncCloudGatewayController.cleanupStaleEqptRoutingRecord(123L);
+        verify(routingSvc, atLeastOnce()).delete(routingRecord.getId());
+        verify(pingClient, atLeastOnce()).isReachable(anyString(), anyInt());
+    }
+
+    private EquipmentGatewayRecord readGatewayRecord() throws IOException {
+        EquipmentGatewayRecord gatewayRecord = EquipmentGatewayRecord.fromFile(
+                OpensyncCloudGatewayControllerTest.class.getResource("EquipmentGatewayRecords.json").getFile(),
+                EquipmentGatewayRecord.class);
+
+        Mockito.when(routingSvc.getGateway(GatewayType.CEGW)).thenReturn(Arrays.asList(gatewayRecord));
+        return gatewayRecord;
+    }
+
+    private EquipmentRoutingRecord readRoutingRecord() throws IOException {
+        EquipmentRoutingRecord routingRecord = EquipmentRoutingRecord.fromFile(
+                OpensyncCloudGatewayControllerTest.class.getResource("EquipmentRoutingRecord.json").getFile(),
+                EquipmentRoutingRecord.class);
+
+        Mockito.when(routingSvc.getRegisteredRouteList(anyLong())).thenReturn(Arrays.asList(routingRecord));
+        return routingRecord;
+    }
+}

opensync-ext-cloud/src/test/resources/com/telecominfraproject/wlan/opensync/external/integration/controller/EquipmentGatewayRecords.json

@@ -0,0 +1,9 @@
+{
+  "id": "1",
+  "hostname": "Host-1",
+  "ipAddr": "1.1.1.1",
+  "port": 123,
+  "gatewayType": "CEGW",
+  "createdTimestamp": 0,
+  "lastModifiedTimestamp": 0
+}

opensync-ext-cloud/src/test/resources/com/telecominfraproject/wlan/opensync/external/integration/controller/EquipmentRoutingRecord.json

@@ -0,0 +1,8 @@
+{
+  "id": 12,
+  "equipmentId": 123,
+  "customerId": 2,
+  "gatewayId": 110,
+  "createdTimestamp": 0,
+  "lastModifiedTimestamp": 0
+}

opensync-gateway-cloud-process/pom.xml

@@ -29,7 +29,11 @@
 			<groupId>com.telecominfraproject.wlan</groupId>
 			<version>${tip-wlan-cloud.release.version}</version>
 		</dependency>
-
+		<dependency>
+			<artifactId>base-client</artifactId>
+			<groupId>com.telecominfraproject.wlan</groupId>
+			<version>${tip-wlan-cloud.release.version}</version>
+		</dependency>
         <dependency>
             <artifactId>filestore-service</artifactId>
             <groupId>com.telecominfraproject.wlan</groupId>