Telecominfraproject/wlan-cloud-services
1
0
Fork 0
mirror of https://github.com/Telecominfraproject/wlan-cloud-services.git synced 2026-01-27 10:23:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

added a component to periodically purge old ServiceMetrics

Browse Source
This commit is contained in:
Dmitry Toptygin
2020-06-02 11:49:21 -04:00
parent 46f2dbe24d
commit 7fa02ca1c5
8 changed files with 103 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
all-cloud-in-one-process/src/main/resources/application.properties
View File

@@ -13,7 +13,7 @@ app.name=AllCloudInOneServer
#spring.profiles.include=use_ssl,client_certificate_auth,use_single_ds,RestTemplateConfiguration_X509_client_cert_auth
#spring.profiles.include=use_ssl_with_client_cert_and_basic_auth,client_certificate_and_basic_auth,use_single_ds,RestTemplateConfiguration_X509_client_cert_auth
#spring.profiles.include=use_ssl,webtoken_auth,use_single_ds,RestTemplateConfiguration_X509_client_cert_auth
spring.profiles.include=use_ssl_with_client_cert_and_webtoken_auth,client_certificate_and_webtoken_auth,use_single_ds,RestTemplateConfiguration_X509_client_cert_auth
spring.profiles.include=use_ssl_with_client_cert_and_webtoken_auth,client_certificate_and_webtoken_auth,use_single_ds,RestTemplateConfiguration_X509_client_cert_auth,purgeOldServiceMetrics
spring.main.show-banner=false
# primary port uses client-certificate auth

1
service-metric-datastore-common-test/src/main/java/com/telecominfraproject/wlan/servicemetric/datastore/BaseServiceMetricDatastoreTest.java
View File

@@ -232,6 +232,7 @@ public abstract class BaseServiceMetricDatastoreTest {
assertEquals(expectedPage1SingleSortDescStrings, actualPage1SingleSortDescStrings);
testInterface.delete(System.currentTimeMillis());
}

14
service-metric-datastore-inmemory/src/main/java/com/telecominfraproject/wlan/servicemetric/datastore/inmemory/ServiceMetricDatastoreInMemory.java
View File

@@ -111,6 +111,20 @@ public class ServiceMetricDatastoreInMemory extends BaseInMemoryDatastore implem
LOG.debug("Deleted ServiceMetric s for customer {} equipment {} createdBefore {}", customerId, equipmentId, createdBeforeTimestamp);
}
@Override
public void delete(long createdBeforeTimestamp) {
List<ServiceMetricKey> keysToRemove = new ArrayList<>();
idToServiceMetricMap.keySet().forEach( k -> {
if(k.createdTimestamp < createdBeforeTimestamp) {
keysToRemove.add(k);
}
});
keysToRemove.forEach(k -> idToServiceMetricMap.remove(k) );
LOG.debug("Deleted {} ServiceMetrics createdBefore {}", keysToRemove.size(), createdBeforeTimestamp);
}
@Override
public PaginationResponse<ServiceMetric> getForCustomer(long fromTime, long toTime, int customerId,

4
service-metric-datastore-interface/src/main/java/com/telecominfraproject/wlan/servicemetric/datastore/ServiceMetricDatastore.java
View File

@@ -19,7 +19,9 @@ public interface ServiceMetricDatastore {
void create(ServiceMetric serviceMetric);
void create(List<ServiceMetric> serviceMetrics);
void delete(int customerId, long equipmentId, long createdBeforeTimestamp);
void delete(long createdBeforeTimestamp);
/**
* <br>Retrieves all of the ServiceMetric records that are mapped to the provided customerId.
* Results are returned in pages.

10
service-metric-datastore-rdbms/src/main/java/com/telecominfraproject/wlan/servicemetric/datastore/rdbms/ServiceMetricDAO.java
View File

@@ -131,6 +131,8 @@ public class ServiceMetricDAO extends BaseJdbcDao {
private static final String SQL_DELETE =
"delete from "+TABLE_NAME+" where customerId = ? and equipmentId = ? and createdTimestamp < ?";
private static final String SQL_PURGE_OLD_RECORDS =
"delete from "+TABLE_NAME+" where createdTimestamp < ?";
private static final String SQL_PAGING_SUFFIX = " LIMIT ? OFFSET ? ";
private static final String SORT_SUFFIX = "";
@@ -380,4 +382,12 @@ public class ServiceMetricDAO extends BaseJdbcDao {
LOG.debug("Stored {} ServiceMetrics", serviceMetrics.size());
}
public void delete(long createdBeforeTimestamp) {
this.jdbcTemplate.update(SQL_PURGE_OLD_RECORDS, createdBeforeTimestamp);
LOG.debug("Deleted ServiceMetrics created before {}", createdBeforeTimestamp);
}
}

5
service-metric-datastore-rdbms/src/main/java/com/telecominfraproject/wlan/servicemetric/datastore/rdbms/ServiceMetricDatastoreRdbms.java
View File

@@ -39,6 +39,11 @@ public class ServiceMetricDatastoreRdbms implements ServiceMetricDatastore {
serviceMetricDAO.delete(customerId, equipmentId, createdBeforeTimestamp);
}
@Override
public void delete(long createdBeforeTimestamp) {
serviceMetricDAO.delete(createdBeforeTimestamp);
}
@Override
public PaginationResponse<ServiceMetric> getForCustomer(long fromTime, long toTime, int customerId,
Set<Long> equipmentIds, Set<MacAddress> clientMacAdresses, Set<ServiceMetricDataType> dataTypes,

8
service-metric-datastore-rdbms/src/test/java/com/telecominfraproject/wlan/servicemetric/datastore/rdbms/ServiceMetricDatastoreRdbmsPlumbingTests.java
View File

@@ -36,11 +36,11 @@ public class ServiceMetricDatastoreRdbmsPlumbingTests extends BaseJdbcTest {
if(db!=null){
//this is a simple test to see if embedded db is working in test environment
JdbcTemplate jdbcTemplate = new JdbcTemplate(db);
Long ret = jdbcTemplate.queryForObject(
"select customerId from service_metric where customerid = ? and equipmentId = ? and clientMac = ? and dataType = ?",
jdbcTemplate.queryForObject(
"select count(1) from service_metric where customerid = ? and equipmentId = ? and clientMac = ? and dataType = ?",
Long.class, 1,1,0,1);
assertEquals((Long)1L, ret);
//this is not a stable test, its outcome depends on the order of execution
//we'll leave it here though - to guard the table name and the key columns
}
}

65
service-metric-service/src/main/java/com/telecominfraproject/wlan/servicemetric/purge/ServiceMetricOldDataPurgeJob.java Normal file
View File

@@ -0,0 +1,65 @@
package com.telecominfraproject.wlan.servicemetric.purge;
import java.util.concurrent.atomic.AtomicBoolean;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Profile;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import com.telecominfraproject.wlan.servicemetric.datastore.ServiceMetricDatastore;
/**
* Purges service metric records older than configured time.
*
* @author dtop
*
*/
@Component
@EnableScheduling
@Profile("purgeOldServiceMetrics")
public class ServiceMetricOldDataPurgeJob {
private static final Logger LOG = LoggerFactory.getLogger(ServiceMetricOldDataPurgeJob.class);
@Autowired private ServiceMetricDatastore serviceMetricDatastore;
private AtomicBoolean jobInProgress = new AtomicBoolean();
@Value("${tip.wlan.purgeServiceMetricsOlderThanSec:14400}") //default is 4 hours
private long purgeServiceMetricsOlderThanSec;
@Scheduled(initialDelay=1200000, fixedDelay=3600000)
public void scheduleScanRuleAgentQueueAssignements(){
LOG.info("ServiceMetricOldDataPurgeJob periodic task started");
if(!jobInProgress.compareAndSet(false, true)){
LOG.info("another ServiceMetricOldDataPurgeJob was started before this one, will not start a new one ontil the old one is done");
return;
}
try{
performPurge();
} catch(Exception e) {
LOG.warn("exception when performing ServiceMetricOldDataPurgeJob", e);
} finally {
jobInProgress.set(false);
LOG.info("ServiceMetricOldDataPurgeJob completed");
}
}
private void performPurge() {
if(purgeServiceMetricsOlderThanSec>60) {
LOG.info("Deleting ServiceMetrics older than {} sec", purgeServiceMetricsOlderThanSec);
serviceMetricDatastore.delete(System.currentTimeMillis() - (purgeServiceMetricsOlderThanSec * 1000) );
}
}
}