[WIFI-7888] Add concurrent burst config handling in opensync gateway

[WIFI-7708] Add BSSID handling for client session

opensync-ext-cloud/src/main/java/com/telecominfraproject/wlan/opensync/external/integration/OpensyncExternalIntegrationCloud.java

@@ -25,6 +25,8 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -98,7 +100,6 @@ import com.telecominfraproject.wlan.profile.models.ProfileType;
 import com.telecominfraproject.wlan.profile.network.models.ApNetworkConfiguration;
 import com.telecominfraproject.wlan.profile.network.models.RadioProfileConfiguration;
 import com.telecominfraproject.wlan.profile.rf.models.RfConfiguration;
-import com.telecominfraproject.wlan.profile.rf.models.RfElementConfiguration;
 import com.telecominfraproject.wlan.profile.ssid.models.SsidConfiguration;
 import com.telecominfraproject.wlan.profile.ssid.models.SsidConfiguration.SecureMode;
 import com.telecominfraproject.wlan.routing.RoutingServiceInterface;
@@ -143,6 +144,8 @@ public class OpensyncExternalIntegrationCloud implements OpensyncExternalIntegra
     private static final String NATIVE_VLAN_ID = "pvid";
     private static final String SPACE_SEPERATOR = " ";
 
+    public final static int MAX_ALLOWED_2GHz_CHANNEL_NUMBER = 14;
+    
     private static final Logger LOG = LoggerFactory.getLogger(OpensyncExternalIntegrationCloud.class);
 
     @Autowired
@@ -767,7 +770,7 @@ public class OpensyncExternalIntegrationCloud implements OpensyncExternalIntegra
             String firmwareVersion) {
 
         LOG.debug("reconcileFwVersionToTrack for AP {} with active firmware version {} model {}", ce.getInventoryId(), activeFirmwareImageAp, model);
-        Status statusRecord = statusServiceInterface.getOrNull(autoProvisionedCustomerId, autoProvisionedCustomerId, StatusDataType.FIRMWARE);
+        Status statusRecord = statusServiceInterface.getOrNull(ce.getCustomerId(), ce.getId(), StatusDataType.FIRMWARE);
         if (statusRecord == null) {
             statusRecord = new Status();
             statusRecord.setCreatedTimestamp(System.currentTimeMillis());
@@ -777,7 +780,7 @@ public class OpensyncExternalIntegrationCloud implements OpensyncExternalIntegra
             statusRecord.setDetails(new EquipmentUpgradeStatusData());
             ((EquipmentUpgradeStatusData) statusRecord.getDetails()).setActiveSwVersion(activeFirmwareImageAp);
             if (inactiveFirmwareImageAp != null) {
-                ((EquipmentUpgradeStatusData) statusRecord.getDetails()).setActiveSwVersion(inactiveFirmwareImageAp);
+                ((EquipmentUpgradeStatusData) statusRecord.getDetails()).setAlternateSwVersion(inactiveFirmwareImageAp);
             }
         }
         EquipmentUpgradeStatusData fwUpgradeStatusData = (EquipmentUpgradeStatusData) statusRecord.getDetails();
@@ -795,8 +798,8 @@ public class OpensyncExternalIntegrationCloud implements OpensyncExternalIntegra
         // determine if AP requires FW upgrade before cloud
         // connection/provision
         if (trackSettings.getAutoUpgradeDeprecatedOnBind().equals(TrackFlag.ALWAYS) || trackSettings.getAutoUpgradeUnknownOnBind().equals(TrackFlag.ALWAYS)) {
-            LOG.debug("reconcileFwVersionToTrack for AP {} track flag for auto-upgrade {}", ce.getInventoryId(),
+            LOG.debug("reconcileFwVersionToTrack for AP {} track flag for auto-upgrade; deprecated flag {} ; unknown flag {} ", ce.getInventoryId(),
-                    trackSettings.getAutoUpgradeDeprecatedOnBind());
+                    trackSettings.getAutoUpgradeDeprecatedOnBind(), trackSettings.getAutoUpgradeUnknownOnBind());
             // check the reported fw version for the AP, if it is < than
             // the default version for the cloud, then download and
             // flash the firmware before proceeding.
@@ -885,23 +888,17 @@ public class OpensyncExternalIntegrationCloud implements OpensyncExternalIntegra
     }
 
     @Override
-    public void apDisconnected(String apId) {
+    public void apDisconnected(String apId, Long ctxRoutingId) {
-        LOG.info("AP {} got disconnected from the gateway", apId);
+        LOG.info("AP {} got disconnected from the gateway, remove ctxRoutingId {} ", apId, ctxRoutingId);
         try {
-
+            if (ctxRoutingId != null && ctxRoutingId > 0L) {
-            OvsdbSession ovsdbSession = ovsdbSessionMapInterface.getSession(apId);
+                try {
-
+                    routingServiceInterface.delete(ctxRoutingId);
-            if (ovsdbSession != null) {
+                } catch (Exception e) {
-                if (ovsdbSession.getRoutingId() > 0L) {
+                    LOG.warn("Unable to delete routing service Id {} for ap {}. {}", ctxRoutingId, apId, e);
-                    try {
-                        routingServiceInterface.delete(ovsdbSession.getRoutingId());
-                    } catch (Exception e) {
-                        LOG.warn("Unable to delete routing service Id {} for ap {}. {}", ovsdbSession.getRoutingId(), apId, e);
-                    }
                 }
-            } else {
-                LOG.warn("Cannot find ap {} in inventory", apId);
             }
+
             Equipment ce = equipmentServiceInterface.getByInventoryIdOrNull(apId);
             if (ce != null) {
                 List<Status> deletedStatuses = statusServiceInterface.deleteOnEquipmentDisconnect(ce.getCustomerId(), ce.getId());
@@ -917,6 +914,47 @@ public class OpensyncExternalIntegrationCloud implements OpensyncExternalIntegra
         }
 
     }
+    
+    @Override
+    public Long getLatestRoutingId(String apId) {
+        try {
+            Equipment ce = equipmentServiceInterface.getByInventoryIdOrNull(apId);
+            if (ce != null) {
+                List<EquipmentRoutingRecord> recordList = routingServiceInterface.getRegisteredRouteList(ce.getId());
+                LOG.info("{} routing records found for AP {}, sort and return latest routingRecordId", recordList.size(), apId);
+                if (!recordList.isEmpty()) {
+                    // Sort by latest record first
+                    Collections.sort(recordList, new Comparator<EquipmentRoutingRecord>() {
+                        @Override
+                        public int compare(EquipmentRoutingRecord o1, EquipmentRoutingRecord o2) {
+                            return Long.compare(o2.getLastModifiedTimestamp(), o1.getLastModifiedTimestamp());
+                        }
+                    });
+                    return recordList.get(0).getId();
+                } else {
+                    LOG.debug("No records found for AP {} equipmentId {}", apId, ce.getId());
+                    return null;
+                }
+            } else {
+                // Equipment doesn't exist, nothing to clean up
+                LOG.debug("No equipment found for AP {} ", apId);
+                return null;
+            }
+        } catch (Exception e) {
+            LOG.error("Exception when registering ap routing {}", apId, e);
+            // Equipment doesn't exist, nothing to clean up
+            return null;
+        }
+    }
+
+    @Override
+    public void removeRoutingRecord(Long routingId) {
+        try {
+            routingServiceInterface.delete(routingId);
+        } catch (Exception e) {
+            LOG.warn("Unable to delete routing service Id {}. {}", routingId, e);
+        }
+    }
 
     private void updateApDisconnectedStatus(String apId, Equipment ce) {
         LOG.info("updateApDisconnectedStatus disconnected AP {}", apId);
@@ -1123,11 +1161,6 @@ public class OpensyncExternalIntegrationCloud implements OpensyncExternalIntegra
             return;
         }
 
-        ApElementConfiguration apElementConfig = (ApElementConfiguration) apNode.getDetails();
-
-        ProfileContainer profileContainer = new ProfileContainer(profileServiceInterface.getProfileWithChildren(apNode.getProfileId()));
-        RfConfiguration rfConfig = (RfConfiguration) profileContainer.getChildOfTypeOrNull(apNode.getProfileId(), ProfileType.rf).getDetails();
-
         for (OpensyncAPVIFState vifState : vifStateTables) {
 
             LOG.debug("Processing vifState for interface {} on AP {}", vifState.getIfName(), apId);
@@ -1158,19 +1191,10 @@ public class OpensyncExternalIntegrationCloud implements OpensyncExternalIntegra
                     vifState.getAssociatedClients(), channel);
 
             RadioType radioType = null;
-            Map<RadioType, RfElementConfiguration> rfElementMap = rfConfig.getRfConfigMap();
+            if (channel <= MAX_ALLOWED_2GHz_CHANNEL_NUMBER) {
-            Map<RadioType, ElementRadioConfiguration> elementRadioMap = apElementConfig.getRadioMap();
+            	radioType = RadioType.is2dot4GHz;
-
+            } else {
-            if (apElementConfig.getAdvancedRadioMap().isEmpty()) {
+            	radioType = RadioType.is5GHz; // 8300 isn't supported
-                LOG.warn("No AdvancedRadioMap for ap {} {}", apId, apElementConfig);
-                continue;
-            }
-            for (RadioType rType : elementRadioMap.keySet()) {
-                boolean autoChannelSelection = rfElementMap.get(rType).getAutoChannelSelection();
-                if (elementRadioMap.get(rType).getActiveChannel(autoChannelSelection) == channel) {
-                    radioType = rType;
-                    break;
-                }
             }
 
             updateActiveBssids(customerId, equipmentId, apId, ssid, radioType, bssid, numClients);
@@ -1461,8 +1485,10 @@ public class OpensyncExternalIntegrationCloud implements OpensyncExternalIntegra
         if (currentActiveBSSIDs == null) {
             currentActiveBSSIDs = new ArrayList<>();
         } else {
-            currentActiveBSSIDs = currentActiveBSSIDs.stream().filter(p -> (p.getRadioType() != null && p.getSsid() != null))
+            currentActiveBSSIDs = currentActiveBSSIDs.stream()
-                    .filter(p -> !p.getRadioType().equals(freqBand) || !p.getSsid().equals(ssid)).collect(Collectors.toList());
+            		.filter(p -> (p.getRadioType() != null && p.getSsid() != null))
+                    .filter(p -> !(p.getRadioType().equals(freqBand) && p.getSsid().equals(ssid)) )
+                    .collect(Collectors.toList());
             LOG.debug("Processing Wifi_VIF_State table update for AP {}, activeBSSIDs bssidList without current radio freq {} and ssid {}", apId,
                     currentActiveBSSIDs, ssid);
         }
@@ -2248,10 +2274,13 @@ public class OpensyncExternalIntegrationCloud implements OpensyncExternalIntegra
                 Client client = clientServiceInterface.getOrNull(customerId, clientMacAddress);
                 if (client == null) {
                     LOG.info("Cannot find client instance for {}", clientMacAddress);
-                    ClientSession session = clientServiceInterface.getSessionOrNull(customerId, equipmentId, clientMacAddress);
+                    ClientSession clientSession = clientServiceInterface.getSessionOrNull(customerId, equipmentId, clientMacAddress);
-                    if (session != null) {
+                    if (clientSession != null && clientSession.getDetails() != null && clientSession.getDetails().getAssociationState() != null) {
-                        LOG.info("Delete clientSession for client that was removed from the Dhcp_Leased_IP table {}",
+                        if (!clientSession.getDetails().getAssociationState().equals(AssociationState.Disconnected)) {
-                                clientServiceInterface.deleteSession(customerId, equipmentId, clientMacAddress));
+                            clientSession.getDetails().setAssociationState(AssociationState.Disconnected);
+                            clientSession = clientServiceInterface.updateSession(clientSession);
+                            LOG.info("Session {} for client {} is now disconnected.", clientSession, clientSession.getMacAddress());
+                        }
                     }
                 } else if (clientMacAddress.equals(equipmentServiceInterface.get(equipmentId).getBaseMacAddress())) {
                     LOG.info("Not a client device {} ", dhcpLeasedIps);

opensync-ext-cloud/src/main/java/com/telecominfraproject/wlan/opensync/external/integration/utils/AsyncPublishService.java

@@ -152,6 +152,9 @@ public class AsyncPublishService {
         clientSession.getDetails().setSessionId(Long.toUnsignedString(apEventClientSession.getSessionId()));
         clientSession.getDetails().setRadioType(OvsdbToWlanCloudTypeMappingUtility.getRadioTypeFromOpensyncStatsRadioBandType(apClientEvent.getBand()));
         clientSession.getDetails().setSsid(apClientEvent.getSsid());
+        if (apClientEvent.hasBssid()) {
+            clientSession.getDetails().setBssid(MacAddress.valueOf(apClientEvent.getBssid()));
+        }
         if (apClientEvent.hasAuthStatus()) {
             clientSession.getDetails().setAssociationStatus(apClientEvent.getAuthStatus());
         }
@@ -202,6 +205,9 @@ public class AsyncPublishService {
         clientSession.getDetails().setSessionId(Long.toUnsignedString(apEventClientSession.getSessionId()));
         clientSession.getDetails().setRadioType(OvsdbToWlanCloudTypeMappingUtility.getRadioTypeFromOpensyncStatsRadioBandType(apClientEvent.getBand()));
         clientSession.getDetails().setSsid(apClientEvent.getSsid());
+        if (apClientEvent.hasBssid()) {
+            clientSession.getDetails().setBssid(MacAddress.valueOf(apClientEvent.getBssid()));
+        }
         if (apClientEvent.hasStatus()) {
             clientSession.getDetails().setAssociationStatus(apClientEvent.getStatus());
         }
@@ -320,6 +326,9 @@ public class AsyncPublishService {
         clientSession.getDetails().setSessionId(Long.toUnsignedString(apEventClientSession.getSessionId()));
         clientSession.getDetails().setRadioType(OvsdbToWlanCloudTypeMappingUtility.getRadioTypeFromOpensyncStatsRadioBandType(apClientEvent.getBand()));
         clientSession.getDetails().setSsid(apClientEvent.getSsid());
+        if (apClientEvent.hasBssid()) {
+            clientSession.getDetails().setBssid(MacAddress.valueOf(apClientEvent.getBssid()));
+        }
         if (apClientEvent.hasDevType()) {
             if (apClientEvent.getDevType().equals(DeviceType.DEV_AP)) {
                 clientSession.getDetails().setDisconnectByApTimestamp(apClientEvent.getTimestampMs());

opensync-ext-cloud/src/main/java/com/telecominfraproject/wlan/opensync/external/integration/utils/MqttStatsPublisher.java

@@ -1190,12 +1190,12 @@ public class MqttStatsPublisher implements StatsPublisherInterface {
         }
 
         String[] parts = topic.split("/");
-        if (parts.length < 3) {
+        if (parts.length < 4) {
             return null;
         }
 
         // apId is the third element in the topic
-        return parts[2];
+        return parts[3];
     }
 
     /**

opensync-ext-cloud/src/test/java/com/telecominfraproject/wlan/opensync/external/integration/OpensyncExternalIntegrationCloudTest.java

@@ -394,7 +394,7 @@ public class OpensyncExternalIntegrationCloudTest {
 
     @Test
     public void testApDisconnected() {
-        opensyncExternalIntegrationCloud.apDisconnected("Test_Client_21P10C68818122");
+        opensyncExternalIntegrationCloud.apDisconnected("Test_Client_21P10C68818122", 0L);
     }
 
     @Test

opensync-ext-interface/src/main/java/com/telecominfraproject/wlan/opensync/external/integration/OpensyncExternalIntegrationInterface.java

@@ -22,7 +22,11 @@ public interface OpensyncExternalIntegrationInterface {
         INIT, INSERT, DELETE, MODIFY
     }
 
-    void apDisconnected(String apId);
+    void apDisconnected(String apId, Long ctxRoutingId);
+    
+    Long getLatestRoutingId(String apId);
+    
+    void removeRoutingRecord(Long routingId);
 
     OpensyncAPConfig getApConfig(String apId);
 

opensync-ext-interface/src/main/java/com/telecominfraproject/wlan/opensync/external/integration/OvsdbSession.java

@@ -1,5 +1,7 @@
 package com.telecominfraproject.wlan.opensync.external.integration;
 
+import java.util.concurrent.atomic.AtomicInteger;
+
 import com.vmware.ovsdb.service.OvsdbClient;
 
 public class OvsdbSession {
@@ -7,6 +9,7 @@ public class OvsdbSession {
     private String apId;
     private long routingId;
     private long equipmentId;
+    private AtomicInteger currentConfigNumInFlight = new AtomicInteger();
     
     public OvsdbClient getOvsdbClient() {
         return ovsdbClient;
@@ -32,5 +35,9 @@ public class OvsdbSession {
     public void setEquipmentId(long equipmentId) {
         this.equipmentId = equipmentId;
     }
+    
+    public AtomicInteger getCurrentConfigNumInFlight() {
+        return currentConfigNumInFlight;
+    }
  
 }

opensync-ext-static/src/main/java/com/telecominfraproject/wlan/opensync/external/integration/OpensyncExternalIntegrationSimple.java

@@ -64,7 +64,7 @@ public class OpensyncExternalIntegrationSimple implements OpensyncExternalIntegr
         LOG.info("ConnectNodeInfo {}", connectNodeInfo);
     }
 
-    public void apDisconnected(String apId) {
+    public void apDisconnected(String apId, Long ctxRoutingId) {
         LOG.info("AP {} got disconnected from the gateway", apId);
     }
 
@@ -222,4 +222,15 @@ public class OpensyncExternalIntegrationSimple implements OpensyncExternalIntegr
     public void updateConfigVersionInStatus(String apId, long configVersionFromProfiles) {
         // do nothing here
     }
+
+    @Override
+    public Long getLatestRoutingId(String apId) {
+        LOG.info("getLatestRoutingId for AP {}", apId);
+        return null;
+    }
+
+    @Override
+    public void removeRoutingRecord(Long routingId) {
+        LOG.info("removeRoutingRecord for routingId {}", routingId);        
+    }
 }

opensync-gateway/src/main/java/com/telecominfraproject/wlan/opensync/mqtt/OpensyncMqttClient.java

@@ -60,6 +60,9 @@ public class OpensyncMqttClient implements ApplicationListener<ContextClosedEven
     @Autowired
     private OpensyncExternalIntegrationInterface opensyncExternalIntegrationInterface;
 
+    @Value("${tip.wlan.internalHostName:localhost}") 
+    private String internalHostName;
+    
     // dtop: use anonymous constructor to ensure that the following code always
     // get executed,
     // even when somebody adds another constructor in here
@@ -138,7 +141,7 @@ public class OpensyncMqttClient implements ApplicationListener<ContextClosedEven
                         MQTT mqtt = new MQTT();
                         mqtt.setHost("tls://" + mqttBrokerAddress + ":" + mqttBrokerListenPort);
                         LOG.info("Connecting to MQTT broker at {}", mqtt.getHost());
-                        mqtt.setClientId("opensync_mqtt");
+                        mqtt.setClientId("opensync_mqtt_" + internalHostName);
                         mqtt.setUserName(username);
                         mqtt.setPassword(password);
                         blockingConnection = mqtt.blockingConnection();
@@ -147,7 +150,7 @@ public class OpensyncMqttClient implements ApplicationListener<ContextClosedEven
                         LOG.debug("Connected to MQTT broker at {}", mqtt.getHost());
 
                         // NB. setting to AT_MOST_ONCE to match the APs message level
-                        Topic[] topics = {new Topic("/ap/#", QoS.AT_MOST_ONCE),};
+                        Topic[] topics = {new Topic("/ap/opensync_mqtt_" + internalHostName + "/#", QoS.AT_MOST_ONCE),};
 
                         blockingConnection.subscribe(topics);
                         LOG.info("Subscribed to mqtt topics {}", Arrays.asList(topics));

opensync-gateway/src/main/java/com/telecominfraproject/wlan/opensync/ovsdb/TipWlanOvsdbClient.java

@@ -20,6 +20,7 @@ import com.telecominfraproject.wlan.opensync.external.integration.models.*;
 import com.telecominfraproject.wlan.opensync.ovsdb.dao.OvsdbDao;
 import com.telecominfraproject.wlan.opensync.ovsdb.metrics.OvsdbClientWithMetrics;
 import com.telecominfraproject.wlan.opensync.ovsdb.metrics.OvsdbMetrics;
+import com.telecominfraproject.wlan.opensync.util.OvsdbClientUtil;
 import com.telecominfraproject.wlan.opensync.util.OvsdbStringConstants;
 import com.telecominfraproject.wlan.opensync.util.SslUtil;
 import com.vmware.ovsdb.callback.ConnectionCallback;
@@ -141,26 +142,32 @@ public class TipWlanOvsdbClient implements OvsdbClientInterface {
 					subjectDn = ((X509Certificate) remoteCertificate).getSubjectDN().getName();
 
                     String clientCn = SslUtil.extractCN(subjectDn);
-                    LOG.info("ovsdbClient connecting from {} on port {} clientCn {}", remoteHost, localPort, clientCn);
+                    if (clientCn != null && !clientCn.isEmpty()) {
-
+                        LOG.info("ovsdbClient connecting from {} on port {} clientCn {}", remoteHost, localPort, clientCn);
-                    ConnectNodeInfo connectNodeInfo = ovsdbDao.getConnectNodeInfo(ovsdbClient);
+    
-
+                        ConnectNodeInfo connectNodeInfo = ovsdbDao.getConnectNodeInfo(ovsdbClient);
-                    // successfully connected - register it in our
+    
-                    // connectedClients table
+                        // successfully connected - register it in our
-
+                        // connectedClients table
-                    String key = alterClientCnIfRequired(clientCn, connectNodeInfo);
+    
-                    ovsdbSessionMapInterface.newSession(key, ovsdbClient);
+                        String key = alterClientCnIfRequired(clientCn, connectNodeInfo);
-
+                        OvsdbSession ovsdbSession = ovsdbSessionMapInterface.newSession(key, ovsdbClient);
-                    extIntegrationInterface.apConnected(key, connectNodeInfo);
+                        extIntegrationInterface.apConnected(key, connectNodeInfo);
-
+                        // DT: at this point the routing Id is associated with the session, let's store it into the
-                    processConnectRequest(ovsdbClient, clientCn, connectNodeInfo);
+                        // connectionInfo object so that the disconnect handler has access to it
-
+                        OvsdbClientUtil.setRoutingId(ovsdbClient, ovsdbSession.getRoutingId());
-                    monitorOvsdbStateTables(ovsdbClient, key);
+    
-
+                        processConnectRequest(ovsdbClient, clientCn, connectNodeInfo);
-                    connectionsCreated.increment();
+    
-                    LOG.info("ovsdbClient connected from {} on port {} AP {} ", remoteHost, localPort, key);
+                        monitorOvsdbStateTables(ovsdbClient, key);
-                    LOG.info("ovsdbClient connectedClients = {}", ovsdbSessionMapInterface.getNumSessions());
+    
-
+                        connectionsCreated.increment();
+                        LOG.info("ovsdbClient connected from {} on port {} AP {} ", remoteHost, localPort, key);
+                        LOG.info("ovsdbClient connectedClients = {}", ovsdbSessionMapInterface.getNumSessions());
+                    } else {
+                        LOG.debug("ovsdbClient: clientCn is empty - not connecting.");
+                    }
+                    
                 } catch (IllegalStateException e) {
                     connectionsFailed.increment();
                     LOG.error("autoprovisioning error {}", e.getMessage(), e);
@@ -181,6 +188,7 @@ public class TipWlanOvsdbClient implements OvsdbClientInterface {
                 connectionsDropped.increment();
 
                 String remoteHost;
+                int remotePort;
                 int localPort;
                 String clientCn;
 
@@ -196,6 +204,7 @@ public class TipWlanOvsdbClient implements OvsdbClientInterface {
 
                 try {
                     remoteHost = ovsdbClient.getConnectionInfo().getRemoteAddress().getHostAddress();
+                    remotePort = ovsdbClient.getConnectionInfo().getRemotePort();
                     localPort = ovsdbClient.getConnectionInfo().getLocalPort();
                     String subjectDn = null;
                     try {
@@ -206,17 +215,63 @@ public class TipWlanOvsdbClient implements OvsdbClientInterface {
                         // do nothing
                     }
                     clientCn = SslUtil.extractCN(subjectDn);
-                    key = ovsdbSessionMapInterface.lookupClientId(ovsdbClient);
+                    if (clientCn != null && !clientCn.isEmpty()) {
-                    if (key != null) {
+                        Long ctxRoutingId =  OvsdbClientUtil.getRoutingId(ovsdbClient);
-                        try {
+                        Long latestDbRoutingId = extIntegrationInterface.getLatestRoutingId(clientCn);
-                            extIntegrationInterface.apDisconnected(key);
+                        OvsdbSession ovsdbSession = ovsdbSessionMapInterface.getSession(clientCn);
-                            ovsdbSessionMapInterface.removeSession(key);
+
-                        } catch (Exception e) {
+                        LOG.info("Determine true disconnect with AP {} ctxRoutingId {} latestDbRoutingId {} ", clientCn, ctxRoutingId, latestDbRoutingId);
-                            LOG.debug("Unable to process ap disconnect. {}", e);
+                        // Determine whether this is a true disconnect based on latest routing record
+                        if (ctxRoutingId != null && latestDbRoutingId != null) {
+                            // if context matches latest DB routingId, this is a true disconnect
+                            if (ctxRoutingId.equals(latestDbRoutingId)) {
+                                try {
+                                    extIntegrationInterface.apDisconnected(clientCn, ctxRoutingId);
+                                    ovsdbSessionMapInterface.removeSession(clientCn);
+                                } catch (Exception e) {
+                                    LOG.debug("Unable to process ap disconnect. {}", e);
+                                }
+                            } else {
+                                if (ovsdbSession != null) {
+                                    if (!latestDbRoutingId.equals(ovsdbSession.getRoutingId())) {
+                                        ovsdbSessionMapInterface.removeSession(clientCn);
+                                    }
+                                }
+                                
+                                extIntegrationInterface.removeRoutingRecord(ctxRoutingId);
+                            }
+                            
+                        // else, clearly handle all other cases
+                        } else if (ctxRoutingId == null && latestDbRoutingId != null) {
+                            // session was not initialized properly in connect path, we can remove this session
+                            LOG.debug("ctxRoutingId null latestDbRoutingId {}, remove non-initialized session for AP {} ", latestDbRoutingId, clientCn);
+                            if (ovsdbSession != null) {
+                                if (ovsdbSession.getRoutingId() == 0L) {
+                                    ovsdbSessionMapInterface.removeSession(clientCn);
+                                }
+                            }
+                            
+                        } else if (ctxRoutingId != null && latestDbRoutingId == null) {
+                            // no routing exist at all, so we know this session is not using any routing record, we can remove this session
+                            LOG.debug("ctxRoutingId {} latestDbRoutingId null, no routing exists, remove session for AP {} ", latestDbRoutingId, clientCn);
+                            if (ovsdbSession != null) {
+                                if (ctxRoutingId.equals(ovsdbSession.getRoutingId())) {
+                                    ovsdbSessionMapInterface.removeSession(clientCn);
+                                }
+                            }
+                            
+                        } else {
+                            // ctxRoutingId == null && latestDbRoutingId == null
+                            // This session is not initialized properly and no routings exist for any session
+                            // We can remove this session without any checks
+                            LOG.debug("ctxRoutingId null latestDbRoutingId null, remove session for AP {} ", latestDbRoutingId, clientCn);
+                            ovsdbSessionMapInterface.removeSession(clientCn);
                         }
+                        
+                        LOG.info("ovsdbClient disconnected from {} on local port {} remote port {} clientCn {} ", remoteHost, localPort,
+                                remotePort, clientCn);
+                        LOG.info("ovsdbClient connectedClients = {}", ovsdbSessionMapInterface.getNumSessions());
                     }
-                    LOG.info("ovsdbClient disconnected from {} on port {} clientCn {} AP {} ", remoteHost, localPort, clientCn, key);
-                    LOG.info("ovsdbClient connectedClients = {}", ovsdbSessionMapInterface.getNumSessions());
                 } finally {
                     try {
                         ovsdbClient.shutdown();
@@ -385,29 +440,56 @@ public class TipWlanOvsdbClient implements OvsdbClientInterface {
 
         OvsdbClient ovsdbClient = ovsdbSession.getOvsdbClient();
 
-        OpensyncAPConfig opensyncAPConfig = extIntegrationInterface.getApConfig(apId);
+        int currentConfigCount = ovsdbSession.getCurrentConfigNumInFlight().get();
-
+        if (currentConfigCount == 0L) {
-        if (opensyncAPConfig == null) {
+            // Current count is 0, start this config push
-            LOG.warn("AP with id " + apId + " does not have a config to apply.");
+            // Increment other incoming configs into the count until this config push is done
-            return;
+            currentConfigCount = ovsdbSession.getCurrentConfigNumInFlight().incrementAndGet();
+            do {
+                try {
+                    OpensyncAPConfig opensyncAPConfig = extIntegrationInterface.getApConfig(apId);
+                    
+                    if (opensyncAPConfig == null) {
+                        LOG.warn("AP with id " + apId + " does not have a config to apply.");
+                        return;
+                    }
+            
+                    //get last known configVersion from the AP
+                    long configVersionFromAp = checkBeforePushConfigVersionFromAp ? ovsdbDao.getConfigVersionFromNode(ovsdbClient) : 0;
+                    
+                    //get last known configVersion from the EquipmentProtocolStatusData        
+                    long configVersionFromStatus = checkBeforePushConfigVersionFromStatus ? extIntegrationInterface.getConfigVersionFromStatus(apId) : 0;
+                    
+                    //get current configVersion from the profiles and equipment
+                    long configVersionFromProfiles = opensyncAPConfig.getConfigVersion();
+                    
+                    boolean needToPushConfigToAP = needToPushToAp(configVersionFromAp, configVersionFromStatus, configVersionFromProfiles);
+            
+                    if(needToPushConfigToAP) {
+                        pushConfigToAp(ovsdbClient, opensyncAPConfig, apId, configVersionFromProfiles);
+                    }
+                    LOG.debug("Finished processConfigChanged for {}", apId);
+                } catch (Exception ex) {
+                    // If anything fails in the ovsdb config push, clean up count and exit
+                    ovsdbSession.getCurrentConfigNumInFlight().set(0);
+                    return;
+                }
+                int checkCount = ovsdbSession.getCurrentConfigNumInFlight().get();
+                if (checkCount == currentConfigCount) {
+                    // Count didn't change from pre-config push, we can clean up and exit
+                    if (ovsdbSession.getCurrentConfigNumInFlight().compareAndSet(currentConfigCount, 0)) {
+                        return;
+                    }
+                } else {
+                    // Count has changed, update the currentConfigCount and rerun config push
+                    ovsdbSession.getCurrentConfigNumInFlight().incrementAndGet();
+                }
+            } while (ovsdbSession.getCurrentConfigNumInFlight().get() != 0);
+        } else {
+            // Count is not 0, another request is being processed for this OvsdbSession
+            // Remember this request, and the other thread will check the count when it's done
+            ovsdbSession.getCurrentConfigNumInFlight().incrementAndGet();
         }
-
-        //get last known configVersion from the AP
-        long configVersionFromAp = checkBeforePushConfigVersionFromAp ? ovsdbDao.getConfigVersionFromNode(ovsdbClient) : 0;
-        
-        //get last known configVersion from the EquipmentProtocolStatusData        
-        long configVersionFromStatus = checkBeforePushConfigVersionFromStatus ? extIntegrationInterface.getConfigVersionFromStatus(apId) : 0;
-        
-        //get current configVersion from the profiles and equipment
-        long configVersionFromProfiles = opensyncAPConfig.getConfigVersion();
-        
-        boolean needToPushConfigToAP = needToPushToAp(configVersionFromAp, configVersionFromStatus, configVersionFromProfiles);
-
-        if(needToPushConfigToAP) {
-            pushConfigToAp(ovsdbClient, opensyncAPConfig, apId, configVersionFromProfiles);
-        }
-        
-        LOG.debug("Finished processConfigChanged for {}", apId);
     }
 
     @Override

opensync-gateway/src/main/java/com/telecominfraproject/wlan/opensync/ovsdb/dao/OvsdbNode.java

@@ -32,6 +32,10 @@ import com.vmware.ovsdb.service.OvsdbClient;
 
 @Component
 public class OvsdbNode extends OvsdbDaoBase {
+	
+	@org.springframework.beans.factory.annotation.Value("${tip.wlan.internalHostName:localhost}") 
+    private String internalHostName;
+	
     String changeRedirectorAddress(OvsdbClient ovsdbClient, String apId, String newRedirectorAddress) {
         try {
             List<Operation> operations = new ArrayList<>();
@@ -380,7 +384,7 @@ public class OvsdbNode extends OvsdbDaoBase {
             Map<String, String> newMqttSettings = new HashMap<>();
             newMqttSettings.put("broker", mqttBrokerAddress);
             String mqttClientName = OvsdbToWlanCloudTypeMappingUtility.getAlteredClientCnIfRequired(clientCn, incomingConnectNodeInfo, preventCnAlteration);
-            newMqttSettings.put("topics", "/ap/" + mqttClientName + "/opensync");
+            newMqttSettings.put("topics", "/ap/opensync_mqtt_" + internalHostName + "/" + mqttClientName + "/opensync");
             newMqttSettings.put("port", "" + mqttBrokerExternalPort);
             newMqttSettings.put("compress", "zlib");
             newMqttSettings.put("qos", "0");

opensync-gateway/src/main/java/com/telecominfraproject/wlan/opensync/ovsdb/metrics/OvsdbClientWithMetrics.java

@@ -25,6 +25,10 @@ public class OvsdbClientWithMetrics implements OvsdbClient {
         this.delegate = delegate;
         this.metrics = metrics;
     }
+    
+    public OvsdbClient getDelegate() {
+        return delegate;
+    }
 
     public CompletableFuture<String[]> listDatabases() throws OvsdbClientException {
         metrics.listDatabases.increment();

opensync-gateway/src/main/java/com/telecominfraproject/wlan/opensync/util/OvsdbClientUtil.java

@@ -0,0 +1,65 @@
+package com.telecominfraproject.wlan.opensync.util;
+
+import java.lang.reflect.Field;
+
+import com.telecominfraproject.wlan.opensync.ovsdb.metrics.OvsdbClientWithMetrics;
+import com.vmware.ovsdb.service.OvsdbClient;
+import com.vmware.ovsdb.service.impl.OvsdbClientImpl;
+
+import io.netty.channel.Channel;
+import io.netty.util.AttributeKey;
+
+/**
+ * Uses reflection to associate routingId with low-level netty channel.
+ * 
+ * @author dtop
+ * 
+ */
+public class OvsdbClientUtil {
+    
+    public static final AttributeKey<String> routingRecordIdAttrKey = AttributeKey.valueOf("routingRecordId");
+
+
+    public static void setRoutingId(OvsdbClient ovsdbClient, long routingId) {
+        Channel channel = getChannel(ovsdbClient);
+        channel.attr(routingRecordIdAttrKey).set(Long.toString(routingId));
+    }
+
+    public static Long getRoutingId(OvsdbClient ovsdbClient) {
+        Channel channel = getChannel(ovsdbClient);
+        String strVal = channel.attr(routingRecordIdAttrKey).get();
+        return strVal==null?null:Long.parseLong(strVal);        
+    }
+
+    
+    private static Channel getChannel(OvsdbClient ovsdbClient) {
+        if(ovsdbClient instanceof OvsdbClientWithMetrics) {
+            //unwrap the object, if needed
+            ovsdbClient = ((OvsdbClientWithMetrics) ovsdbClient).getDelegate();
+        }
+        
+        if(! (ovsdbClient instanceof OvsdbClientImpl)) {
+            throw new RuntimeException("Do not know how to handle "+ ovsdbClient.getClass().getName()+" - expected OvsdbClientImpl");
+        }
+        
+        try {
+            Field jsonRpcClientField = ovsdbClient.getClass().getDeclaredField("jsonRpcClient");
+            jsonRpcClientField.setAccessible(true);
+            Object jsonRpcClientObj = jsonRpcClientField.get(ovsdbClient);
+            
+            Field transporterField = jsonRpcClientObj.getClass().getDeclaredField("transporter");
+            transporterField.setAccessible(true);
+            Object transporterObj = transporterField.get(jsonRpcClientObj);
+            
+            Field channelField = transporterObj.getClass().getDeclaredField("val$channel");
+            channelField.setAccessible(true);
+            
+            Channel channel = (Channel) channelField.get(transporterObj);
+            
+            return channel;
+        }catch(Exception e) {
+            throw new RuntimeException("Cannot get the channel for the ovsdbClient "+ ovsdbClient, e);            
+        }
+    }
+    
+}

opensync-gateway/src/test/java/com/telecominfraproject/wlan/opensync/ovsdb/dao/OvsdbNodeTest.java

@@ -14,6 +14,7 @@ import org.mockito.Mockito;
 import org.mockito.MockitoSession;
 import org.mockito.quality.Strictness;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
 import org.springframework.boot.test.mock.mockito.MockBean;
@@ -85,7 +86,10 @@ public class OvsdbNodeTest {
     OvsdbRadiusProxyConfig ovsdbRadiusProxyConfig;
     @MockBean(answer = Answers.RETURNS_MOCKS)
     OvsdbGet ovsdbGet;
-
+    
+    @Value("${tip.wlan.internalHostName:localhost}") 
+    private String internalHostName;
+    
     MockitoSession mockito;
 
     @Configuration
@@ -122,7 +126,7 @@ public class OvsdbNodeTest {
         newMqttSettings.put("broker", ovsdbNode.mqttBrokerAddress);
         String mqttClientName = OvsdbToWlanCloudTypeMappingUtility.getAlteredClientCnIfRequired("AP-1", connectNodeInfo,
                 false);
-        newMqttSettings.put("topics", "/ap/" + mqttClientName + "/opensync");
+        newMqttSettings.put("topics", "/ap/opensync_mqtt_" + internalHostName + "/" + mqttClientName + "/opensync");
         newMqttSettings.put("port", "" + ovsdbNode.mqttBrokerExternalPort);
         newMqttSettings.put("compress", "zlib");
         newMqttSettings.put("qos", "0");