scottk/siembol
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/siembol.git synced 2025-11-01 19:07:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Sending correlated alerts fields after matchign the rule

Browse Source
This commit is contained in:
Marian Novotny
2022-08-25 12:04:41 +01:00
parent 5f0c216e8d
commit e129a1263f
32 changed files with 327 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
alerting/alerting-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -35,7 +35,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>

3
alerting/alerting-core/src/main/java/uk/co/gresearch/siembol/alerts/common/AlertingFields.java
View File

@@ -9,7 +9,8 @@ public enum AlertingFields {
MAX_PER_HOUR_FIELD("max_per_hour"),
MAX_PER_DAY_FIELD("max_per_day"),
EXCEPTION("exception"),
PROCESSING_TIME("processing_time");
PROCESSING_TIME("processing_time"),
CORRELATED_ALERTS("correlated_alerts");
private final String name;
private static final String ALERTS_PREFIX = "siembol_alerts";

4
alerting/alerting-core/src/main/java/uk/co/gresearch/siembol/alerts/compiler/AlertingCorrelationRulesCompiler.java
View File

@@ -94,6 +94,10 @@ public class AlertingCorrelationRulesCompiler implements AlertingCompiler {
.protections(protections)
.version(ruleDto.getRuleVersion());
if (attr.getFieldsToSend() != null) {
builder.fieldNamesToSend(attr.getFieldsToSend());
}
for (CorrelationAlertDto alert : attr.getAlerts()) {
EnumSet<AlertCounterMetadata.Flags> flags = alert.getMandatory()
? EnumSet.of(AlertCounterMetadata.Flags.MANDATORY)

36
alerting/alerting-core/src/main/java/uk/co/gresearch/siembol/alerts/correlationengine/AlertContext.java Normal file
View File

@@ -0,0 +1,36 @@
package uk.co.gresearch.siembol.alerts.correlationengine;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
public class AlertContext implements Comparable<AlertContext> {
private final long timestamp;
private final Object[] fieldsToSend;
public AlertContext(long timestamp, Object[] fieldsToSend) {
this.timestamp = timestamp;
this.fieldsToSend = fieldsToSend;
}
public long getTimestamp() {
return timestamp;
}
public Map<String, Object> getFields(List<String> fieldNames) {
Map<String, Object> ret = new LinkedHashMap<>();
for (int i = 0; i < fieldsToSend.length; i++) {
if (fieldsToSend[i] == null) {
continue;
}
ret.put(fieldNames.get(i), fieldsToSend[i]);
}
return ret;
}
@Override
public int compareTo(AlertContext o) {
return Long.compare(this.timestamp, o.timestamp);
}
}

34
alerting/alerting-core/src/main/java/uk/co/gresearch/siembol/alerts/correlationengine/AlertCounter.java
View File

@@ -1,50 +1,60 @@
package uk.co.gresearch.siembol.alerts.correlationengine;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.PriorityQueue;
public class AlertCounter {
private final AlertCounterMetadata counterMetadata;
private final PriorityQueue<Long> timestampsHeap = new PriorityQueue<>();
private final PriorityQueue<AlertContext> alertContextHeap = new PriorityQueue<>();
public AlertCounter(AlertCounterMetadata counterMetadata) {
this.counterMetadata = counterMetadata;
}
public void update(long eventTime) {
public void update(AlertContext alertContext) {
if (getSize() == counterMetadata.getThreshold()) {
timestampsHeap.poll();
alertContextHeap.poll();
}
timestampsHeap.add(eventTime);
alertContextHeap.add(alertContext);
}
public void clean(long waterMark) {
if (!timestampsHeap.isEmpty() && timestampsHeap.peek() < waterMark - counterMetadata.getExtendedWindowSize()) {
timestampsHeap.clear();
if (!alertContextHeap.isEmpty()
&& alertContextHeap.peek().getTimestamp() < waterMark - counterMetadata.getExtendedWindowSize()) {
alertContextHeap.clear();
return;
}
while (!timestampsHeap.isEmpty() && timestampsHeap.peek() < waterMark) {
timestampsHeap.poll();
while (!alertContextHeap.isEmpty() && alertContextHeap.peek().getTimestamp() < waterMark) {
alertContextHeap.poll();
}
}
public boolean isEmpty() {
return timestampsHeap.isEmpty();
return alertContextHeap.isEmpty();
}
public int getSize() {
return timestampsHeap.size();
return alertContextHeap.size();
}
public Long getOldest() {
return timestampsHeap.peek();
return alertContextHeap.isEmpty() ? null : alertContextHeap.peek().getTimestamp();
}
public boolean matchThreshold() {
return timestampsHeap.size() >= counterMetadata.getThreshold();
return alertContextHeap.size() >= counterMetadata.getThreshold();
}
public boolean isMandatory() {
return counterMetadata.isMandatory();
}
public List<Map<String, Object>> getCorrelatedAlerts(List<String> fieldNames) {
List<Map<String, Object>> ret = new ArrayList<>();
alertContextHeap.forEach(x -> ret.add(x.getFields(fieldNames)));
return ret;
}
}

43
alerting/alerting-core/src/main/java/uk/co/gresearch/siembol/alerts/correlationengine/CorrelationRule.java
View File

@@ -5,8 +5,10 @@ import uk.co.gresearch.siembol.alerts.common.AlertingAttributes;
import uk.co.gresearch.siembol.alerts.common.AlertingFields;
import uk.co.gresearch.siembol.alerts.common.AlertingResult;
import uk.co.gresearch.siembol.alerts.engine.AbstractRule;
import uk.co.gresearch.siembol.common.constants.SiembolConstants;
import java.util.*;
import java.util.stream.Collectors;
import static uk.co.gresearch.siembol.alerts.common.AlertingTags.CORRELATION_KEY_TAG_NAME;
@@ -14,7 +16,7 @@ public class CorrelationRule extends AbstractRule {
public enum Flags {
USE_EVENT_TIME,
}
private static final String EVENT_TIMESTAMP_FIELD = "timestamp";
private static final String EVENT_TIMESTAMP_FIELD = SiembolConstants.TIMESTAMP;
private final EnumSet<Flags> flags;
private final int alertsThresholds;
private final long timeWindowInMs;
@@ -24,6 +26,8 @@ public class CorrelationRule extends AbstractRule {
private final Map<String, Integer> alertToCounterIndex;
private final Map<String, ArrayList<AlertCounter>> alertCounters = new HashMap<>();
private final List<String> fieldNamesToSend;
protected CorrelationRule(Builder<?> builder) {
super(builder);
this.alertsThresholds = builder.alertsThresholds;
@@ -32,6 +36,7 @@ public class CorrelationRule extends AbstractRule {
this.flags = builder.flags;
this.alertCountersMetadata = builder.alertCountersMetadata;
this.alertToCounterIndex = builder.alertToCounterIndex;
this.fieldNamesToSend = builder.fieldNamesToSend;
}
@Override
public AlertingResult match(Map<String, Object> alert) {
@@ -42,17 +47,23 @@ public class CorrelationRule extends AbstractRule {
&& (alert.get(EVENT_TIMESTAMP_FIELD) instanceof Number)
? ((Number)alert.get(EVENT_TIMESTAMP_FIELD)).longValue()
: processingTime;
Object[] fieldsToSend = new Object[fieldNamesToSend.size()];
for (int i = 0; i < fieldNamesToSend.size(); i++) {
fieldsToSend[i] = alert.get(fieldNamesToSend.get(i));
}
AlertContext alertContext = new AlertContext(eventTime, fieldsToSend);
try {
if (EvaluationResult.NO_MATCH == evaluate(key, alertName, eventTime, processingTime)) {
if (EvaluationResult.NO_MATCH == evaluate(key, alertName, alertContext, processingTime)) {
return AlertingResult.fromEvaluationResult(EvaluationResult.NO_MATCH, alert);
}
Map<String, Object> outAlert = createOutputAlert(alert);
Map<String, Object> outAlert = createOutputAlert(alert, key);
alertCounters.remove(key);
return AlertingResult.fromEvaluationResult(EvaluationResult.MATCH, outAlert);
} catch (Exception e) {
AlertingAttributes attr = new AlertingAttributes();
Map<String, Object> outAlert = createOutputAlert(alert);
Map<String, Object> outAlert = createOutputAlert(alert, key);
outAlert.put(AlertingFields.EXCEPTION.getCorrelationAlertingName(), ExceptionUtils.getStackTrace(e));
attr.setEvent(outAlert);
return new AlertingResult(AlertingResult.StatusCode.ERROR, attr);
@@ -69,7 +80,7 @@ public class CorrelationRule extends AbstractRule {
}
private EvaluationResult evaluate(String key, String ruleName, long eventTime, long processingTime) {
private EvaluationResult evaluate(String key, String ruleName, AlertContext alertContext, long processingTime) {
ArrayList<AlertCounter> currentCounterList = alertCounters.get(key);
if (currentCounterList == null) {
currentCounterList = createAlertCounters();
@@ -80,7 +91,8 @@ public class CorrelationRule extends AbstractRule {
int index = alertToCounterIndex.get(ruleName);
AlertCounter currentCounter = currentCounterList.get(index);
currentCounter.update(eventTime);
currentCounter.update(alertContext);
if (currentCounter.matchThreshold()) {
return evaluateRule(currentCounterList);
} else {
@@ -88,10 +100,19 @@ public class CorrelationRule extends AbstractRule {
}
}
private Map<String, Object> createOutputAlert(Map<String, Object> alert) {
private Map<String, Object> createOutputAlert(Map<String, Object> alert, String key) {
Map<String, Object> ret = new HashMap<>(alert);
ret.put(AlertingFields.RULE_NAME.getCorrelationAlertingName(), getRuleName());
ret.put(AlertingFields.FULL_RULE_NAME.getCorrelationAlertingName(), getFullRuleName());
List<Map<String, Object>> correlatedAlerts = alertCounters.get(key).stream()
.flatMap(x -> x.getCorrelatedAlerts(fieldNamesToSend).stream())
.filter(x -> !x.isEmpty())
.collect(Collectors.toList());
if (!correlatedAlerts.isEmpty()) {
ret.put(AlertingFields.CORRELATED_ALERTS.getCorrelationAlertingName(), correlatedAlerts);
}
return ret;
}
@@ -143,6 +164,7 @@ public class CorrelationRule extends AbstractRule {
protected ArrayList<AlertCounterMetadata> alertCountersMetadata = new ArrayList<>();
protected Map<String, Integer> alertToCounterIndex = new HashMap<>();
protected EnumSet<Flags> flags = EnumSet.noneOf(Flags.class);
protected List<String> fieldNamesToSend = new ArrayList<>();
public Builder<T> alertsThresholds(Integer alertThresholds) {
this.alertsThresholds = alertThresholds;
@@ -173,6 +195,11 @@ public class CorrelationRule extends AbstractRule {
alertCountersMetadataTemp.add(metadata);
return this;
}
public Builder<T> fieldNamesToSend(List<String> fieldNames) {
this.fieldNamesToSend = fieldNames;
return this;
}
}
public static CorrelationRule.Builder<CorrelationRule> builder() {
@@ -186,7 +213,7 @@ public class CorrelationRule extends AbstractRule {
if (alertCountersMetadataTemp.isEmpty()) {
throw new IllegalArgumentException(EMPTY_ALERT_COUNTERS_MSG);
}
if (timeWindowInMs == null || maxLagTimeInSec == null) {
if (timeWindowInMs == null || maxLagTimeInSec == null || fieldNamesToSend == null) {
throw new IllegalArgumentException(MISSING_REQUIRED_ATTRIBUTES);
}
maxLagTimeInMs = maxLagTimeInSec * MILLI_MULTIPLIER;

14
alerting/alerting-core/src/main/java/uk/co/gresearch/siembol/alerts/model/CorrelationAttributesDto.java
View File

@@ -35,6 +35,12 @@ public class CorrelationAttributesDto {
@Attributes(required = true, description = "The alerts used for the correlation", minItems = 1)
private List<CorrelationAlertDto> alerts;
@JsonProperty("fields_to_send")
@Attributes(required = true, description = "The list of fields of correlated alerts " +
"that will be included in the triggered alert", minItems = 1)
private List<String> fieldsToSend;
public Integer getTimeWindow() {
return timeWindow;
}
@@ -82,4 +88,12 @@ public class CorrelationAttributesDto {
public void setMaxTimeLagInSec(Integer maxTimeLagInSec) {
this.maxTimeLagInSec = maxTimeLagInSec;
}
public List<String> getFieldsToSend() {
return fieldsToSend;
}
public void setFieldsToSend(List<String> fieldsToSend) {
this.fieldsToSend = fieldsToSend;
}
}

60
alerting/alerting-core/src/test/java/uk/co/gresearch/siembol/alerts/correlationengine/AlertCounterTest.java
View File

@@ -4,15 +4,21 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
public class AlertCounterTest {
private final int threshold = 1000;
private AlertCounterMetadata counterMetadata;
private AlertCounter alertCounter;
private List<String> fieldNames;
@Before
public void setUp() {
fieldNames = Arrays.asList("constant", "variable");
counterMetadata = new AlertCounterMetadata("dummy",
threshold, 1000, EnumSet.noneOf(AlertCounterMetadata.Flags.class));
alertCounter = new AlertCounter(counterMetadata);
@@ -21,19 +27,33 @@ public class AlertCounterTest {
@Test
public void reachingMaxSize() {
for (long i = 1; i <= counterMetadata.getThreshold(); i++) {
alertCounter.update(i);
alertCounter.update(new AlertContext(i, new Object[]{"secret", i}));
Assert.assertEquals(1, alertCounter.getOldest().longValue());
Assert.assertEquals(i, alertCounter.getSize());
}
alertCounter.update(8);
alertCounter.update(new AlertContext(8, new Object[]{"secret", 1001}));
Assert.assertEquals(2, alertCounter.getOldest().longValue());
Assert.assertEquals(counterMetadata.getThreshold(), alertCounter.getSize());
var fieldsToSend = alertCounter.getCorrelatedAlerts(fieldNames);
Assert.assertFalse(fieldsToSend.isEmpty());
Assert.assertEquals(1000, fieldsToSend.size());
var variableSet = new HashSet<Long>();
for (var stringObjectMap : fieldsToSend) {
Assert.assertEquals(2, stringObjectMap.size());
Assert.assertTrue(stringObjectMap.get("constant") instanceof String);
Assert.assertTrue(stringObjectMap.get("variable") instanceof Number);
Assert.assertEquals("secret", stringObjectMap.get("constant"));
variableSet.add(((Number) stringObjectMap.get("variable")).longValue());
}
Assert.assertEquals(1000, variableSet.size());
}
@Test
public void cleaningCounterPartially() {
for (long i = 1; i <= counterMetadata.getThreshold(); i++) {
alertCounter.update(i);
alertCounter.update(new AlertContext(i, new Object[]{"secret", i}));
}
Assert.assertEquals(1, alertCounter.getOldest().longValue());
@@ -47,7 +67,7 @@ public class AlertCounterTest {
@Test
public void cleaningCounterComplete() {
alertCounter.update(1);
alertCounter.update(new AlertContext(1, new Object[]{"secret", 1}));
Assert.assertEquals(1, alertCounter.getOldest().longValue());
Assert.assertEquals(1, alertCounter.getSize());
Assert.assertFalse(alertCounter.isEmpty());
@@ -60,10 +80,40 @@ public class AlertCounterTest {
@Test
public void cleaningCounterVeryOld() {
alertCounter.update(1);
alertCounter.update(new AlertContext(1, new Object[]{"secret", 1}));
alertCounter.clean(1002);
Assert.assertNull(alertCounter.getOldest());
Assert.assertEquals(0, alertCounter.getSize());
Assert.assertTrue(alertCounter.isEmpty());
}
@Test
public void getFieldsToSendWithNullFields() {
alertCounter.update(new AlertContext(1, new Object[]{"secret", 1}));
alertCounter.update(new AlertContext(2, new Object[]{null, 2}));
alertCounter.update(new AlertContext(3, new Object[]{null, null}));
Assert.assertEquals(3, alertCounter.getSize());
var fieldsToSend = alertCounter.getCorrelatedAlerts(fieldNames);
Assert.assertEquals(3, fieldsToSend.size());
for (var stringObjectMap : fieldsToSend) {
if (stringObjectMap.get("variable") == null) {
Assert.assertTrue(stringObjectMap.isEmpty());
continue;
}
Assert.assertTrue(stringObjectMap.get("variable") instanceof Number);
if (((Number) stringObjectMap.get("variable")).longValue() == 2) {
Assert.assertEquals(1, stringObjectMap.size());
} else {
Assert.assertEquals(2, stringObjectMap.size());
Assert.assertTrue(stringObjectMap.get("constant") instanceof String);
Assert.assertTrue(stringObjectMap.get("variable") instanceof Number);
Assert.assertEquals("secret", stringObjectMap.get("constant"));
Assert.assertEquals(1, stringObjectMap.get("variable"));
}
}
}
}

100
alerting/alerting-core/src/test/java/uk/co/gresearch/siembol/alerts/correlationengine/CorrelationRuleTest.java
View File

@@ -25,9 +25,11 @@ public class CorrelationRuleTest {
private List<Map<String, Object>> alerts;
private final String correlationKey = "1.2.3.4";
private List<String> fieldNames;
@Before
public void setUp() {
fieldNames = Arrays.asList("constant", "variable");
builder = CorrelationRule.builder();
builder
.timeWindowInMs(timeWindowInMs)
@@ -35,6 +37,7 @@ public class CorrelationRuleTest {
.addAlertCounter("alert1", 1, counterFlags)
.addAlertCounter("alert2", 2, counterFlags)
.addAlertCounter("alert3", 3, counterFlags)
.fieldNamesToSend(fieldNames)
.name(ruleName)
.version(1);
}
@@ -132,6 +135,14 @@ public class CorrelationRuleTest {
AlertingResult ret = rule.match(alerts.get(0));
Assert.assertEquals(OK, ret.getStatusCode());
Assert.assertEquals(MATCH, ret.getAttributes().getEvaluationResult());
Assert.assertNotNull(ret.getAttributes().getEvent()
.get(AlertingFields.CORRELATED_ALERTS.getCorrelationAlertingName()));
Assert.assertTrue(ret.getAttributes().getEvent()
.get(AlertingFields.CORRELATED_ALERTS.getCorrelationAlertingName()) instanceof List<?>);
Assert.assertEquals(6, ((List<?>) ret.getAttributes().getEvent()
.get(AlertingFields.CORRELATED_ALERTS.getCorrelationAlertingName())).size());
alerts = createAlert(1, correlationKey, "alert3", 30003);
ret = rule.match(alerts.get(0));
@@ -139,6 +150,83 @@ public class CorrelationRuleTest {
Assert.assertEquals(NO_MATCH, ret.getAttributes().getEvaluationResult());
}
@Test
public void matchEventTimeNoFieldsToSend() {
ruleFlags = EnumSet.of(CorrelationRule.Flags.USE_EVENT_TIME);
rule = builder
.flags(ruleFlags)
.fieldNamesToSend(new ArrayList<>())
.build();
alerts = createAlert(1, correlationKey, "alert1", 30000);
for (Map<String, Object> alert : alerts) {
AlertingResult ret = rule.match(alert);
Assert.assertEquals(OK, ret.getStatusCode());
Assert.assertEquals(NO_MATCH, ret.getAttributes().getEvaluationResult());
}
alerts = createAlert(2, correlationKey, "alert2", 30001);
for (Map<String, Object> alert : alerts) {
AlertingResult ret = rule.match(alert);
Assert.assertEquals(OK, ret.getStatusCode());
Assert.assertEquals(NO_MATCH, ret.getAttributes().getEvaluationResult());
}
alerts = createAlert(2, correlationKey, "alert3", 30002);
for (Map<String, Object> alert : alerts) {
AlertingResult ret = rule.match(alert);
Assert.assertEquals(OK, ret.getStatusCode());
Assert.assertEquals(NO_MATCH, ret.getAttributes().getEvaluationResult());
}
alerts = createAlert(1, correlationKey, "alert3", 30003);
AlertingResult ret = rule.match(alerts.get(0));
Assert.assertEquals(OK, ret.getStatusCode());
Assert.assertEquals(MATCH, ret.getAttributes().getEvaluationResult());
Assert.assertNull(ret.getAttributes().getEvent()
.get(AlertingFields.CORRELATED_ALERTS.getCorrelationAlertingName()));
}
@Test
public void matchEventTimeTimeNoFieldsToSendInEvent() {
ruleFlags = EnumSet.of(CorrelationRule.Flags.USE_EVENT_TIME);
rule = builder.flags(ruleFlags).build();
alerts = createAlert(1, correlationKey, "alert1", 30000);
alerts.forEach(x -> x.remove("constant"));
alerts.forEach(x -> x.remove("variable"));
for (Map<String, Object> alert : alerts) {
AlertingResult ret = rule.match(alert);
Assert.assertEquals(OK, ret.getStatusCode());
Assert.assertEquals(NO_MATCH, ret.getAttributes().getEvaluationResult());
}
alerts = createAlert(2, correlationKey, "alert2", 30001);
alerts.forEach(x -> x.remove("constant"));
alerts.forEach(x -> x.remove("variable"));
for (Map<String, Object> alert : alerts) {
AlertingResult ret = rule.match(alert);
Assert.assertEquals(OK, ret.getStatusCode());
Assert.assertEquals(NO_MATCH, ret.getAttributes().getEvaluationResult());
}
alerts = createAlert(2, correlationKey, "alert3", 30002);
alerts.forEach(x -> x.remove("constant"));
alerts.forEach(x -> x.remove("variable"));
for (Map<String, Object> alert : alerts) {
AlertingResult ret = rule.match(alert);
Assert.assertEquals(OK, ret.getStatusCode());
Assert.assertEquals(NO_MATCH, ret.getAttributes().getEvaluationResult());
}
alerts = createAlert(1, correlationKey, "alert3", 30003);
alerts.forEach(x -> x.remove("constant"));
alerts.forEach(x -> x.remove("variable"));
AlertingResult ret = rule.match(alerts.get(0));
Assert.assertEquals(OK, ret.getStatusCode());
Assert.assertEquals(MATCH, ret.getAttributes().getEvaluationResult());
Assert.assertNull(ret.getAttributes().getEvent()
.get(AlertingFields.CORRELATED_ALERTS.getCorrelationAlertingName()));
}
@Test
public void clean() {
ruleFlags = EnumSet.of(CorrelationRule.Flags.USE_EVENT_TIME);
@@ -223,16 +311,18 @@ public class CorrelationRuleTest {
Assert.assertEquals(MATCH, ret.getAttributes().getEvaluationResult());
}
private List<Map<String, Object>> createAlert(int numbers, String key, String alertName, long processingTime) {
Map<String, Object> alert = new HashMap<>();
alert.put("constant", "secret");
alert.put(AlertingFields.RULE_NAME.getAlertingName(), alertName);
alert.put(CORRELATION_KEY_TAG_NAME.toString(), key);
alert.put(AlertingFields.PROCESSING_TIME.getCorrelationAlertingName(), processingTime);
alert.put("timestamp", processingTime);
return new ArrayList<>(Collections.nCopies(numbers, alert));
List<Map<String, Object>> ret = Collections.nCopies(numbers, alert);
for (int i = 0; i < ret.size(); i++) {
ret.get(i).put("variable", i);
}
return ret;
}
}

4
alerting/alerting-spark/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -23,7 +23,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
<exclusions>
<exclusion>
<artifactId>jackson-databind</artifactId>

4
alerting/alerting-storm/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -51,7 +51,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

2
alerting/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</parent>
<modules>
<module>alerting-core</module>

4
config-editor/config-editor-core/pom.xml
View File

@@ -9,13 +9,13 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>

18
config-editor/config-editor-rest/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</parent>
<dependencyManagement>
<dependencies>
@@ -56,7 +56,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -67,22 +67,22 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-core</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-services</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-sync</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -93,7 +93,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -104,7 +104,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-core</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -115,7 +115,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding-core</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

14
config-editor/config-editor-services/pom.xml
View File

@@ -10,7 +10,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -41,32 +41,32 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-core</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-core</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding-core</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>

8
config-editor/config-editor-sync/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -20,17 +20,17 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-core</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>

2
config-editor/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</parent>
<modules>
<module>config-editor-core</module>

4
deployment/siembol-monitoring/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<dependencyManagement>
@@ -37,7 +37,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

4
deployment/storm-topology-manager/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<dependencyManagement>
@@ -43,7 +43,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

13
docs/services/siembol_alerting_services.md
View File

@@ -69,12 +69,13 @@ The correlation alert allows you to group several detections together before rai
- `time_computation_type` - You can configure how the time window is calculated
- `event_time` - The time window is calculated using the `timestamp` field in the events, the `timestamp` field is usually computed during parsing from the log
- `processing_time` - The time window is calculated using the current time (when an alert is evaluated), the events need to be processed by the correlation alert component within the time window
- `max_time_lag_in_sec` - The event with timestamp older than the current time minus the lag (in seconds) will be discarded
- `alerts_threshold` - The alert's threshold allows you to configure how many detections (you can specify which detections later) need to trigger in the time window for the alert to trigger. This field accepts an integer value, if it is left empty then all detections need to trigger before an alert is created
- `alerts` - The list of alerts for correlation
- `alert` - The alert name used for correlation
- `threshold` - The number of times the alert has to trigger in the time window
- `mandatory` - The alert must pass the threshold for the rule to match
- `max_time_lag_in_sec` - The event with timestamp older than the current time minus the lag (in seconds) will be discarded
- `alerts_threshold` - The alert's threshold allows you to configure how many detections (you can specify which detections later) need to trigger in the time window for the alert to trigger. This field accepts an integer value, if it is left empty then all detections need to trigger before an alert is created
- `alerts` - The list of alerts for correlation
- `alert` - The alert name used for correlation
- `threshold` - The number of times the alert has to trigger in the time window
- `mandatory` - The alert must pass the threshold for the rule to match
- `fields_to_send` - The list of fields of correlated alerts that will be included in the triggered alert after matching
## Admin config
### Common admin config fields
- `alerts.topology.name` - The name of storm topology

6
enriching/enriching-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -35,12 +35,12 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>

4
enriching/enriching-storm/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -75,7 +75,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-core</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

2
enriching/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</parent>
<modules>
<module>enriching-core</module>

6
parsing/parsing-app/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -39,12 +39,12 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-core</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>

4
parsing/parsing-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -50,7 +50,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>

4
parsing/parsing-storm/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -75,7 +75,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

2
parsing/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</parent>
<modules>
<module>parsing-core</module>

2
pom.xml
View File

@@ -6,7 +6,7 @@
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<name>siembol</name>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
<description>A scalable, advanced security analytics framework based on open-source big data technologies.</description>
<inceptionYear>2019</inceptionYear>
<url>https://siembol.io/</url>

2
responding/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</parent>
<modules>
<module>responding-core</module>

6
responding/responding-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -35,12 +35,12 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.jayway.jsonpath</groupId>

6
responding/responding-stream/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</parent>
<dependencyManagement>
<dependencies>
@@ -51,7 +51,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -62,7 +62,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding-core</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>

2
siembol-common/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.6.5-SNAPSHOT</version>
<version>2.6.6-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>