scottk/siembol
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/siembol.git synced 2025-12-09 05:25:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Upgrading java to java 11 (java 13 in tests) (#405)

Browse Source 
Upgrading Storm to 2.3.0
Supporting downloading http enrichment tables
Remove multiline string dependency
This commit is contained in:
Marian Novotny
2021-11-04 16:12:54 +00:00
committed by GitHub
parent 2628339ddb
commit 3415ce3909
131 changed files with 4013 additions and 4538 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.github/workflows/ci.yml vendored
View File

@@ -15,10 +15,11 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Set up JDK 1.8
uses: actions/setup-java@v1
- name: Set up JDK 13
uses: actions/setup-java@v2
with:
java-version: 1.8
distribution: 'adopt'
java-version: '13'
- name: Cache Maven packages
uses: actions/cache@v2
with:

7
.github/workflows/codeql-analysis.yml vendored
View File

@@ -65,11 +65,12 @@ jobs:
# uses a compiled language
# java build
- name: Set up JDK 1.8
- name: Set up JDK 13
if: ${{ matrix.language == 'java' }}
uses: actions/setup-java@v1
uses: actions/setup-java@v2
with:
java-version: 1.8
distribution: 'adopt'
java-version: '13'
- name: Cache Maven packages
if: ${{ matrix.language == 'java' }}
uses: actions/cache@v2

10
alerting/alerting-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -35,13 +35,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>1.4.2-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>

244
alerting/alerting-core/src/test/java/uk/co/gresearch/siembol/alerts/compiler/AlertingRulesCompilerTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.alerts.compiler;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -10,137 +9,128 @@ import uk.co.gresearch.siembol.alerts.common.AlertingResult;
import uk.co.gresearch.siembol.alerts.engine.AlertingEngineImpl;
import java.util.Arrays;
import java.util.List;
public class AlertingRulesCompilerTest {
/**
*{
* "rules_version" :1,
* "tags" : [ { "tag_name" : "detection_source", "tag_value" : "alerts" } ],
* "rules" : [ {
* "rule_name" : "siembol_alert_generic",
* "rule_version" : 1,
* "rule_author" : "dummy",
* "rule_description": "Test rule - is_alert is equal to true",
* "source_type" : "*",
* "matchers" : [ {
* "matcher_type" : "REGEX_MATCH",
* "is_negated" : false,
* "field" : "is_alert",
* "data" : "(?i)true" },
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "source_type",
* "data": "(?<sensor>.*)"
* }
* ]
* }]
*}
**/
@Multiline
public static String alertRules;
private final String alertRules = """
{
"rules_version" :1,
"tags" : [ { "tag_name" : "detection_source", "tag_value" : "alerts" } ],
"rules" : [ {
"rule_name" : "siembol_alert_generic",
"rule_version" : 1,
"rule_author" : "dummy",
"rule_description": "Test rule - is_alert is equal to true",
"source_type" : "*",
"matchers" : [ {
"matcher_type" : "REGEX_MATCH",
"is_negated" : false,
"field" : "is_alert",
"data" : "(?i)true" },
{
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "source_type",
"data": "(?<sensor>.*)"
}
]
}]
}
""";
/**
*{
* "rule_name" : "siembol_alert_generic",
* "rule_version" : 1,
* "rule_author" : "dummy",
* "rule_description": "Test rule - is_alert is equal to true",
* "source_type" : "*",
* "matchers" : [ {
* "matcher_type" : "REGEX_MATCH",
* "is_negated" : false,
* "field" : "is_alert",
* "data" : "(?i)true" },
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "source_type",
* "data": "(?<sensor>.*)"
* }]
*}
**/
@Multiline
public static String alertRule;
private final String alertRule = """
{
"rule_name" : "siembol_alert_generic",
"rule_version" : 1,
"rule_author" : "dummy",
"rule_description": "Test rule - is_alert is equal to true",
"source_type" : "*",
"matchers" : [ {
"matcher_type" : "REGEX_MATCH",
"is_negated" : false,
"field" : "is_alert",
"data" : "(?i)true" },
{
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "source_type",
"data": "(?<sensor>.*)"
}]
}
""";
/**
*{
* "source_type" : "secret",
* "is_alert" : "TruE",
* "dummy_field_int" : 1,
* "dummy_field_boolean" : false
*}
**/
@Multiline
public static String goodAlert;
private final String goodAlert = """
{
"source_type" : "secret",
"is_alert" : "TruE",
"dummy_field_int" : 1,
"dummy_field_boolean" : false
}
""";
/**
*{
* "source_type" : "secret",
* "is_alert" : "TruE",
* "dummy_field_int" : 1,
* "dummy_field_boolean" : false,
* "is_secret" : "true"
*}
**/
@Multiline
public static String goodAlertWithSecret;
private final String goodAlertWithSecret = """
{
"source_type" : "secret",
"is_alert" : "TruE",
"dummy_field_int" : 1,
"dummy_field_boolean" : false,
"is_secret" : "true"
}
""";
/**
* {
* "rule_name": "siembol_alert_generic_with_composite_matchers",
* "rule_version": 1,
* "rule_author": "dummy",
* "rule_description": "Test rule with composite matchers",
* "source_type": "*",
* "matchers": [
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "is_alert",
* "data": "(?i)true"
* },
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "source_type",
* "data": "(?<sensor>.*)"
* },
* {
* "matcher_type": "COMPOSITE_OR",
* "is_negated": false,
* "matchers": [
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "is_secret",
* "data": "(?i)true"
* },
* {
* "matcher_type": "COMPOSITE_AND",
* "is_negated": false,
* "matchers": [
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "is_public",
* "data": "(?i)true"
* },
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "is_detected",
* "data": "(?i)yes"
* }
* ]
* }
* ]
* }
* ]
* }
**/
@Multiline
public static String ruleWithCompositeMatchers;
private final String ruleWithCompositeMatchers = """
{
"rule_name": "siembol_alert_generic_with_composite_matchers",
"rule_version": 1,
"rule_author": "dummy",
"rule_description": "Test rule with composite matchers",
"source_type": "*",
"matchers": [
{
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_alert",
"data": "(?i)true"
},
{
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "source_type",
"data": "(?<sensor>.*)"
},
{
"matcher_type": "COMPOSITE_OR",
"is_negated": false,
"matchers": [
{
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_secret",
"data": "(?i)true"
},
{
"matcher_type": "COMPOSITE_AND",
"is_negated": false,
"matchers": [
{
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_public",
"data": "(?i)true"
},
{
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_detected",
"data": "(?i)yes"
}
]
}
]
}
]
}
""";
private AlertingCompiler compiler;
@@ -305,7 +295,7 @@ public class AlertingRulesCompilerTest {
@Test
public void compileRulesListSizeOne() {
AlertingResult compileResult = compiler.compile(Arrays.asList(alertRules));
AlertingResult compileResult = compiler.compile(List.of(alertRules));
Assert.assertEquals(AlertingResult.StatusCode.OK, compileResult.getStatusCode());
Assert.assertNotNull(compileResult.getAttributes().getEngine());
Assert.assertTrue(compileResult.getAttributes().getEngine() instanceof AlertingEngineImpl);

115
alerting/alerting-core/src/test/java/uk/co/gresearch/siembol/alerts/compiler/CorrelationRulesCompilerTest.java
View File

@@ -1,72 +1,67 @@
package uk.co.gresearch.siembol.alerts.compiler;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import uk.co.gresearch.siembol.alerts.common.AlertingResult;
public class CorrelationRulesCompilerTest {
/**
*{
* "rules_version" :1,
* "tags" : [ { "tag_name" : "detection_source", "tag_value" : "siembol_correlation_alerts" } ],
* "rules" : [ {
* "tags" : [ { "tag_name" : "test", "tag_value" : "true" } ],
* "rule_protection": {
* "max_per_hour": 500,
* "max_per_day": 1000
* },
* "rule_name" : "test_rule",
* "rule_version" : 1,
* "rule_author" : "dummy",
* "rule_description": "Testing rule",
* "correlation_attributes" : {
* "time_unit" : "seconds",
* "time_window" : 500,
* "time_computation_type" : "processing_time",
* "alerts" : [
* {
* "alert" : "alert1",
* "threshold" : 5
* },
* {
* "alert" : "alert2",
* "threshold" : 5
* }]
* }
* }]
*}
**/
@Multiline
public static String rulesWithSimpleCorrelationRule;
/**{
* "rule_name" : "test_rule_event_time",
* "rule_version" : 1,
* "rule_author" : "dummy",
* "rule_description": "Testing rule",
* "correlation_attributes" : {
* "time_unit" : "seconds",
* "time_window" : 500,
* "time_computation_type" : "event_time",
* "max_time_lag_in_sec": 30,
* "alerts" : [
* {
* "alert" : "alert1",
* "threshold" : 5
* },
* {
* "mandatory": true,
* "alert" : "alert2",
* "threshold" : 5
* }]
* }
* }
**/
@Multiline
public static String simpleCorrelationRule;
private final String rulesWithSimpleCorrelationRule = """
{
"rules_version" :1,
"tags" : [ { "tag_name" : "detection_source", "tag_value" : "siembol_correlation_alerts" } ],
"rules" : [ {
"tags" : [ { "tag_name" : "test", "tag_value" : "true" } ],
"rule_protection": {
"max_per_hour": 500,
"max_per_day": 1000
},
"rule_name" : "test_rule",
"rule_version" : 1,
"rule_author" : "dummy",
"rule_description": "Testing rule",
"correlation_attributes" : {
"time_unit" : "seconds",
"time_window" : 500,
"time_computation_type" : "processing_time",
"alerts" : [
{
"alert" : "alert1",
"threshold" : 5
},
{
"alert" : "alert2",
"threshold" : 5
}]
}
}]
}
""";
private final String simpleCorrelationRule = """
{
"rule_name" : "test_rule_event_time",
"rule_version" : 1,
"rule_author" : "dummy",
"rule_description": "Testing rule",
"correlation_attributes" : {
"time_unit" : "seconds",
"time_window" : 500,
"time_computation_type" : "event_time",
"max_time_lag_in_sec": 30,
"alerts" : [
{
"alert" : "alert1",
"threshold" : 5
},
{
"mandatory": true,
"alert" : "alert2",
"threshold" : 5
}]
}
}
""";
private AlertingCompiler compiler;

3
alerting/alerting-core/src/test/java/uk/co/gresearch/siembol/alerts/correlationengine/AlertCounterTest.java
View File

@@ -7,7 +7,7 @@ import org.junit.Test;
import java.util.EnumSet;
public class AlertCounterTest {
private int threshold = 1000;
private final int threshold = 1000;
private AlertCounterMetadata counterMetadata;
private AlertCounter alertCounter;
@@ -66,5 +66,4 @@ public class AlertCounterTest {
Assert.assertEquals(0, alertCounter.getSize());
Assert.assertTrue(alertCounter.isEmpty());
}
}

2
alerting/alerting-core/src/test/java/uk/co/gresearch/siembol/alerts/correlationengine/CorrelationEngineTest.java
View File

@@ -93,7 +93,7 @@ public class CorrelationEngineTest {
public void testMatchOneOverwriteRuleProtection() {
doAnswer(invocation -> {
Object[] args = invocation.getArguments();
((Map<String, Object>)args[0]).put(AlertingFields.MAX_PER_HOUR_FIELD.getCorrelationAlertingName(), 2);
((Map<String, Object>) args[0]).put(AlertingFields.MAX_PER_HOUR_FIELD.getCorrelationAlertingName(), 2);
return null;
}).when(rule1).addOutputFieldsToEvent(anyMap());

5
alerting/alerting-core/src/test/java/uk/co/gresearch/siembol/alerts/correlationengine/CorrelationRuleTest.java
View File

@@ -6,7 +6,6 @@ import org.junit.Test;
import uk.co.gresearch.siembol.alerts.common.AlertingFields;
import uk.co.gresearch.siembol.alerts.common.AlertingResult;
import java.util.*;
import static uk.co.gresearch.siembol.alerts.common.EvaluationResult.MATCH;
@@ -24,7 +23,7 @@ public class CorrelationRuleTest {
private final int maxTimeLagInSec = 5;
private final String ruleName = "test_rule";
private List<Map<String, Object>> alerts;
private String correlationKey = "1.2.3.4";
private final String correlationKey = "1.2.3.4";
@Before
@@ -176,7 +175,7 @@ public class CorrelationRuleTest {
rule = builder.flags(ruleFlags).alertsThresholds(1).build();
for (int i = 1; i < 100; i++) {
alerts = createAlert(2, correlationKey + String.valueOf(i),
alerts = createAlert(2, correlationKey + i,
"alert3",
30000 + i);
for (Map<String, Object> alert : alerts) {

22
alerting/alerting-core/src/test/java/uk/co/gresearch/siembol/alerts/engine/AlertingEngineImplTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.alerts.engine;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang3.tuple.Pair;
import org.junit.Assert;
import org.junit.Before;
@@ -17,14 +16,13 @@ import java.util.*;
import static org.mockito.Mockito.when;
public class AlertingEngineImplTest {
/**
*{"source_type" : "test_source", "dummy_field" : "true"}
*/
@Multiline
public static String knownSourceType;
private final String knownSourceType = """
{ "source_type" : "test_source",
"dummy_field" : "true"
}
""";
private String sourceType = "test_source";
private final String sourceType = "test_source";
private List<Pair<String, String>> constants;
private List<Pair<String, Object>> protections;
private List<Pair<String, Rule>> rules;
@@ -36,8 +34,8 @@ public class AlertingEngineImplTest {
@Before
public void setUp() {
constants = Arrays.asList(Pair.of("detection_source", "siembol_alerts"));
protections = Arrays.asList(Pair.of(AlertingFields.MAX_PER_HOUR_FIELD.getAlertingName(), Integer.valueOf(1)));
constants = List.of(Pair.of("detection_source", "siembol_alerts"));
protections = List.of(Pair.of(AlertingFields.MAX_PER_HOUR_FIELD.getAlertingName(), 1));
rule1 = Mockito.mock(Rule.class);
rule2 = Mockito.mock(Rule.class);
resultRule1 = AlertingResult.fromEvaluationResult(EvaluationResult.MATCH, new HashMap<>());
@@ -46,10 +44,10 @@ public class AlertingEngineImplTest {
when(rule1.getRuleName()).thenReturn("rule1");
when(rule1.getFullRuleName()).thenReturn("rule1_v1");
when(rule1.match(ArgumentMatchers.<Map<String, Object>>any())).thenReturn(resultRule1);
when(rule1.match(ArgumentMatchers.any())).thenReturn(resultRule1);
when(rule2.getRuleName()).thenReturn("rule2");
when(rule2.getFullRuleName()).thenReturn("rule2_v1");
when(rule2.match(ArgumentMatchers.<Map<String, Object>>any())).thenReturn(resultRule2);
when(rule2.match(ArgumentMatchers.any())).thenReturn(resultRule2);
rules = Arrays.asList(Pair.of(sourceType, rule1),
Pair.of("*", rule2));

47
alerting/alerting-core/src/test/java/uk/co/gresearch/siembol/alerts/engine/IsInSetTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.alerts.engine;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -14,32 +13,24 @@ public class IsInSetTest {
private Map<String, Object> event;
private IsInSetMatcher matcher;
/**
*metron
*alerts
*response
*stoRm
*123
**/
@Multiline
private String strings;
private final String strings = """
metron
alerts
response
stoRm
123""";
/**
*${variable_field1}
*aa ${variable.field2} ${variable:field1}
*http://${host}/${path}
**/
@Multiline
private String variableStrings;
private final String variableStrings = """
${variable_field1}
aa ${variable.field2} ${variable:field1}
http://${host}/${path}""";
/**
*alerts
*${variable:field1}
*aa ${variable.field2} b
*Metron
**/
@Multiline
private String mixedVariablesConstants;
private final String mixedVariablesConstants = """
alerts
${variable:field1}
aa ${variable.field2} b
Metron
""";
@Before
public void setUp() {
@@ -163,8 +154,8 @@ public class IsInSetTest {
event.put(field, "Metron");
EvaluationResult rest = matcher.match(event);
Assert.assertEquals(rest, EvaluationResult.MATCH);
Assert.assertEquals(matcher.canModifyEvent(), false);
Assert.assertEquals(EvaluationResult.MATCH, rest);
Assert.assertFalse(matcher.canModifyEvent());
}
@Test
@@ -275,6 +266,4 @@ public class IsInSetTest {
ret = matcher.match(event);
Assert.assertEquals(EvaluationResult.MATCH, ret);
}
}

22
alerting/alerting-core/src/test/java/uk/co/gresearch/siembol/alerts/engine/RegexMatcherTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.alerts.engine;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -10,22 +9,16 @@ import java.util.Map;
import java.util.HashMap;
public class RegexMatcherTest {
private String field = "test_field";
private final String field = "test_field";
Map<String, Object> event;
RegexMatcher matcher;
/**
* Threat Level=(?<vof_threat_level>\d) Category=(?<vof_threat_cat>\S+) Type=(?<vof_threat_type>.*?)
**/
@Multiline
public static String goodVofDetail;
private final String goodVofDetail = """
Threat Level=(?<vof_threat_level>\\d) Category=(?<vof_threat_cat>\\S+) Type=(?<vof_threat_type>.*?)""";
/**
* Threat Level=1 Category=UNKNOWN Type=a
*bc
**/
@Multiline
public static String vofDetailInstance;
private final String vofDetailInstance = """
Threat Level=1 Category=UNKNOWN Type=a
bc""";
@Before
public void setUp() {
@@ -73,7 +66,7 @@ public class RegexMatcherTest {
EvaluationResult rest = matcher.match(event);
Assert.assertEquals(EvaluationResult.NO_MATCH, rest);
Assert.assertEquals(matcher.canModifyEvent(), true);
Assert.assertTrue(matcher.canModifyEvent());
}
@Test
@@ -189,5 +182,4 @@ public class RegexMatcherTest {
.pattern("valid")
.build();
}
}

36
alerting/alerting-core/src/test/java/uk/co/gresearch/siembol/alerts/engine/RuleTest.java
View File

@@ -15,9 +15,9 @@ import java.util.*;
import static org.mockito.Mockito.when;
public class RuleTest {
private String name = "test_rule";
private Integer version = 1;
private Map<String, Object> event = new HashMap<>();
private final String name = "test_rule";
private final Integer version = 1;
private final Map<String, Object> event = new HashMap<>();
private List<Pair<String, String>> constants;
private List<Pair<String, Object>> protections;
private BasicMatcher matcher;
@@ -25,16 +25,16 @@ public class RuleTest {
@Before
public void setUp() {
constants = Arrays.asList(Pair.of("detection_source", "alerts"));
protections = Arrays.asList(Pair.of(AlertingFields.MAX_PER_HOUR_FIELD.toString(), Integer.valueOf(1)));
constants = List.of(Pair.of("detection_source", "alerts"));
protections = List.of(Pair.of(AlertingFields.MAX_PER_HOUR_FIELD.toString(), 1));
matcher = Mockito.mock(BasicMatcher.class);
when(matcher.match(ArgumentMatchers.<Map<String, Object>>any())).thenReturn(EvaluationResult.MATCH);
when(matcher.match(ArgumentMatchers.any())).thenReturn(EvaluationResult.MATCH);
}
@Test
public void testGoodMetadata() {
rule = Rule.builder()
.matchers(Arrays.asList(matcher))
.matchers(List.of(matcher))
.name(name)
.version(version)
.tags(constants)
@@ -52,7 +52,7 @@ public class RuleTest {
Assert.assertEquals("alerts", event.get("detection_source"));
Assert.assertEquals(Integer.valueOf(1), event.get(AlertingFields.MAX_PER_HOUR_FIELD.toString()));
Assert.assertEquals(1, event.get(AlertingFields.MAX_PER_HOUR_FIELD.toString()));
Assert.assertFalse(rule.canModifyEvent());
}
@@ -61,7 +61,7 @@ public class RuleTest {
constants = new ArrayList<>(constants);
constants.add(Pair.of("malicious_url", "http://${dummy_host}/${dummy_path}"));
rule = Rule.builder()
.matchers(Arrays.asList(matcher))
.matchers(List.of(matcher))
.name(name)
.version(version)
.tags(constants)
@@ -82,7 +82,7 @@ public class RuleTest {
Assert.assertEquals("http://alerts.com/about", event.get("malicious_url"));
Assert.assertEquals(Integer.valueOf(1), event.get(AlertingFields.MAX_PER_HOUR_FIELD.toString()));
Assert.assertEquals(1, event.get(AlertingFields.MAX_PER_HOUR_FIELD.toString()));
Assert.assertFalse(rule.canModifyEvent());
}
@@ -91,7 +91,7 @@ public class RuleTest {
when(matcher.canModifyEvent()).thenReturn(true);
rule = Rule.builder()
.matchers(Arrays.asList(matcher))
.matchers(List.of(matcher))
.name(name)
.version(version)
.tags(constants)
@@ -104,7 +104,7 @@ public class RuleTest {
@Test
public void testGoodMatch() {
rule = Rule.builder()
.matchers(Arrays.asList(matcher))
.matchers(List.of(matcher))
.name(name)
.version(version)
.tags(constants)
@@ -118,9 +118,9 @@ public class RuleTest {
@Test
public void testGoodNoMatch() {
when(matcher.match(ArgumentMatchers.<Map<String, Object>>any())).thenReturn(EvaluationResult.NO_MATCH);
when(matcher.match(ArgumentMatchers.any())).thenReturn(EvaluationResult.NO_MATCH);
rule = Rule.builder()
.matchers(Arrays.asList(matcher))
.matchers(List.of(matcher))
.name(name)
.version(version)
.tags(constants)
@@ -134,9 +134,9 @@ public class RuleTest {
@Test(expected = RuntimeException.class)
public void testThrowsException() throws RuntimeException {
when(matcher.match(ArgumentMatchers.<Map<String, Object>>any())).thenThrow(new RuntimeException());
when(matcher.match(ArgumentMatchers.any())).thenThrow(new RuntimeException());
rule = Rule.builder()
.matchers(Arrays.asList(matcher))
.matchers(List.of(matcher))
.name(name)
.version(version)
.tags(constants)
@@ -149,7 +149,7 @@ public class RuleTest {
@Test(expected = IllegalArgumentException.class)
public void missingName() {
rule = Rule.builder()
.matchers(Arrays.asList(matcher))
.matchers(List.of(matcher))
.version(version)
.tags(constants)
.protections(protections)
@@ -159,7 +159,7 @@ public class RuleTest {
@Test(expected = IllegalArgumentException.class)
public void missingVersion() {
Rule.builder()
.matchers(Arrays.asList(matcher))
.matchers(List.of(matcher))
.name(name)
.tags(constants)
.protections(protections)

2
alerting/alerting-core/src/test/java/uk/co/gresearch/siembol/alerts/protection/RuleProtectionSystemTest.java
View File

@@ -6,7 +6,7 @@ import uk.co.gresearch.siembol.alerts.common.AlertingResult;
public class RuleProtectionSystemTest {
private RuleProtectionSystem protection;
private String ruleName = "test";
private final String ruleName = "test";
@Before
public void setUp() {

10
alerting/alerting-spark/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -23,7 +23,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<artifactId>jackson-databind</artifactId>
@@ -45,12 +45,6 @@
<version>${junit_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>

72
alerting/alerting-spark/src/test/java/uk/co/gresearch/siembol/spark/AlertingSparkEngineTest.java
View File

@@ -1,48 +1,42 @@
package uk.co.gresearch.siembol.spark;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang3.SerializationUtils;
import org.junit.Assert;
import org.junit.Test;
public class AlertingSparkEngineTest {
/**
*{
* "rules_version" :1,
* "tags" : [ { "tag_name" : "detection:source", "tag_value" : "alerts" } ],
* "rules" : [ {
* "rule_name" : "test_rule",
* "rule_version" : 1,
* "rule_author" : "dummy",
* "rule_protection" : {
* "max_per_hour" : 100,
* "max_per_day" : 10000
* },
* "rule_description": "test rule - is_alert is equal to true",
* "source_type" : "*",
* "matchers" : [ {
* "matcher_type" : "REGEX_MATCH",
* "is_negated" : false,
* "field" : "is_alert",
* "data" : "(?i)true" }
* ]
* }]
*}
**/
@Multiline
public static String isAlertRules;
/**
*{
* "source_type" : "secret",
* "is_alert" : "TruE",
* "dummy_field_int" : 1,
* "dummy_field_boolean" : false
*}
**/
@Multiline
public static String goodAlert;
private final String isAlertRules = """
{
"rules_version" :1,
"tags" : [ { "tag_name" : "detection:source", "tag_value" : "alerts" } ],
"rules" : [ {
"rule_name" : "test_rule",
"rule_version" : 1,
"rule_author" : "dummy",
"rule_protection" : {
"max_per_hour" : 100,
"max_per_day" : 10000
},
"rule_description": "test rule - is_alert is equal to true",
"source_type" : " ",
"matchers" : [ {
"matcher_type" : "REGEX_MATCH",
"is_negated" : false,
"field" : "is_alert",
"data" : "(?i)true" }
]
}]
}
""";
private final String goodAlert = """
{
"source_type" : "secret",
"is_alert" : "TruE",
"dummy_field_int" : 1,
"dummy_field_boolean" : false
}
""";
@Test
public void serializableTest() throws Exception {
@@ -57,7 +51,7 @@ public class AlertingSparkEngineTest {
Assert.assertEquals(ret.getMatchesTotal(), retClone.getMatchesTotal());
Assert.assertEquals(ret.getExceptionsTotal(), retClone.getExceptionsTotal());
Assert.assertTrue(ret.getExceptions().equals(retClone.getExceptions()));
Assert.assertTrue(ret.getMatches().equals(retClone.getMatches()));
Assert.assertEquals(ret.getExceptions(), retClone.getExceptions());
Assert.assertEquals(ret.getMatches(), retClone.getMatches());
}
}

87
alerting/alerting-spark/src/test/java/uk/co/gresearch/siembol/spark/AlertingSparkJobTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.spark;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
@@ -8,58 +7,52 @@ import org.junit.*;
import java.util.ArrayList;
public class AlertingSparkJobTest {
/**
*{
* "rules_version" :1,
* "tags" : [ { "tag_name" : "detection_source", "tag_value" : "siembol_alerts" } ],
* "rules" : [ {
* "rule_name" : "test_rule",
* "rule_version" : 1,
* "rule_author" : "dummy",
* "rule_protection" : {
* "max_per_hour" : 100,
* "max_per_day" : 10000
* },
* "rule_description": "Testing rule",
* "source_type" : "*",
* "matchers" : [ {
* "matcher_type" : "REGEX_MATCH",
* "is_negated" : false,
* "field" : "is_alert",
* "data" : "(?i)true" }
* ]
* }]
*}
**/
@Multiline
public static String isAlertRules;
private final String isAlertRules = """
{
"rules_version" :1,
"tags" : [ { "tag_name" : "detection_source", "tag_value" : "siembol_alerts" } ],
"rules" : [ {
"rule_name" : "test_rule",
"rule_version" : 1,
"rule_author" : "dummy",
"rule_protection" : {
"max_per_hour" : 100,
"max_per_day" : 10000
},
"rule_description": "Testing rule",
"source_type" : "*",
"matchers" : [ {
"matcher_type" : "REGEX_MATCH",
"is_negated" : false,
"field" : "is_alert",
"data" : "(?i)true" }
]
}]
}
""";
/**
*{
* "source_type" : "secret",
* "is_alert" : "TruE",
* "dummy_field_int" : 1,
* "dummy_field_boolean" : false
*}
**/
@Multiline
public static String goodAlert;
private final String goodAlert = """
{
"source_type" : "secret",
"is_alert" : "TruE",
"dummy_field_int" : 1,
"dummy_field_boolean" : false
}
""";
/**
*{
* "source_type" : "secret",
* "dummy_field_int" : 1,
* "dummy_field_boolean" : false
*}
**/
@Multiline
public static String eventWithoutAlert;
private final String eventWithoutAlert = """
{
"source_type" : "secret",
"dummy_field_int" : 1,
"dummy_field_boolean" : false
}
""";
private JavaSparkContext sc;
private AlertingSparkJob job;
private int maxResult = 100;
private final int maxResult = 100;
@Before
public void setup() {

59
alerting/alerting-spark/src/test/java/uk/co/gresearch/siembol/spark/AlertingSparkTest.java
View File

@@ -1,42 +1,39 @@
package uk.co.gresearch.siembol.spark;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Ignore;
import org.junit.Test;
import java.util.Base64;
public class AlertingSparkTest {
/**
*{
* "source_type" : "secret",
* "from_date" : "2019-06-18",
* "to_date" : "2019-06-19",
* "rules" :{
* "rules_version" :1,
* "tags" : [ { "tag_name" : "detection_source", "tag_value" : "siembol_alerts" } ],
* "rules" : [ {
* "rule_name" : "test_rule",
* "rule_version" : 1,
* "rule_author" : "dummy",
* "rule_protection" : {
* "max_per_hour" : 100,
* "max_per_day" : 10000
* },
* "rule_description": "Testing rule",
* "source_type" : "*",
* "matchers" : [ {
* "matcher_type" : "REGEX_MATCH",
* "is_negated" : false,
* "field" : "is_alert",
* "data" : "(?i)true" }
* ]
* }]
*}
*}
**/
@Multiline
public static String testAttributes;
private final String testAttributes = """
{
"source_type" : "secret",
"from_date" : "2019-06-18",
"to_date" : "2019-06-19",
"rules" :{
"rules_version" :1,
"tags" : [ { "tag_name" : "detection_source", "tag_value" : "siembol_alerts" } ],
"rules" : [ {
"rule_name" : "test_rule",
"rule_version" : 1,
"rule_author" : "dummy",
"rule_protection" : {
"max_per_hour" : 100,
"max_per_day" : 10000
},
"rule_description": "Testing rule",
"source_type" : "*",
"matchers" : [ {
"matcher_type" : "REGEX_MATCH",
"is_negated" : false,
"field" : "is_alert",
"data" : "(?i)true" }
]
}]
}
}
""";
@Test
@Ignore

38
alerting/alerting-spark/src/test/java/uk/co/gresearch/siembol/spark/SparkResultTest.java
View File

@@ -7,16 +7,13 @@ import org.junit.Test;
import uk.co.gresearch.siembol.alerts.common.AlertingAttributes;
import uk.co.gresearch.siembol.alerts.common.AlertingResult;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.util.HashMap;
import java.util.*;
public class SparkResultTest {
private AlertingAttributes attributes;
private AlertingResult alertingResult;
private AlertingSparkResult alertingSparkResult;
private int maxResult = 100;
private final int maxResult = 100;
private Map<String, Object> event;
@Before
@@ -27,7 +24,7 @@ public class SparkResultTest {
}
@Test
public void wrongStatusCodeTest() throws Exception {
public void wrongStatusCodeTest() {
alertingResult = new AlertingResult(AlertingResult.StatusCode.ERROR, attributes);
alertingSparkResult = new AlertingSparkResult(alertingResult, maxResult);
Assert.assertEquals(1, alertingSparkResult.getExceptionsTotal());
@@ -38,9 +35,9 @@ public class SparkResultTest {
}
@Test
public void singleEventTest() throws Exception {
public void singleEventTest() {
event.put("test", "true");
attributes.setOutputEvents(Arrays.asList(event));
attributes.setOutputEvents(List.of(event));
alertingSparkResult = new AlertingSparkResult(alertingResult, maxResult);
Assert.assertEquals(0, alertingSparkResult.getExceptionsTotal());
Assert.assertEquals(1, alertingSparkResult.getMatchesTotal());
@@ -50,10 +47,10 @@ public class SparkResultTest {
}
@Test
public void maxResultEventTest() throws Exception {
public void maxResultEventTest() {
event.put("test", "true");
ArrayList<Map<String, Object>> events = new ArrayList<>();
for(int i = 0; i <= maxResult; i++) {
for (int i = 0; i <= maxResult; i++) {
events.add(event);
}
@@ -67,9 +64,9 @@ public class SparkResultTest {
}
@Test
public void singleExceptionTest() throws Exception {
public void singleExceptionTest() {
event.put("test", "true");
attributes.setExceptionEvents(Arrays.asList(event));
attributes.setExceptionEvents(List.of(event));
alertingSparkResult = new AlertingSparkResult(alertingResult, maxResult);
Assert.assertEquals(1, alertingSparkResult.getExceptionsTotal());
Assert.assertEquals(0, alertingSparkResult.getMatchesTotal());
@@ -79,10 +76,10 @@ public class SparkResultTest {
}
@Test
public void maxResultExceptionTest() throws Exception {
public void maxResultExceptionTest() {
event.put("test", "true");
ArrayList<Map<String, Object>> events = new ArrayList<>();
for(int i = 0; i <= maxResult; i++) {
for (int i = 0; i <= maxResult; i++) {
events.add(event);
}
@@ -96,10 +93,10 @@ public class SparkResultTest {
}
@Test
public void mergeTest() throws Exception {
public void mergeTest() {
event.put("test", "true");
ArrayList<Map<String, Object>> events = new ArrayList<>();
for(int i = 0; i < 10; i++) {
for (int i = 0; i < 10; i++) {
events.add(event);
}
@@ -114,7 +111,7 @@ public class SparkResultTest {
event.put("test", "false");
ArrayList<Map<String, Object>> eventsOther = new ArrayList<>();
for(int i = 0; i < maxResult + 1; i++) {
for (int i = 0; i < maxResult + 1; i++) {
eventsOther.add(event);
}
@@ -122,7 +119,7 @@ public class SparkResultTest {
attributesOther.setOutputEvents(eventsOther);
attributesOther.setExceptionEvents(eventsOther);
AlertingSparkResult alertingSparkResultOther = new AlertingSparkResult(
new AlertingResult(AlertingResult.StatusCode.OK, attributesOther),maxResult);
new AlertingResult(AlertingResult.StatusCode.OK, attributesOther), maxResult);
Assert.assertEquals(maxResult + 1, alertingSparkResultOther.getMatchesTotal());
Assert.assertEquals(maxResult + 1, alertingSparkResultOther.getExceptionsTotal());
@@ -137,16 +134,15 @@ public class SparkResultTest {
}
@Test
public void serializableTest() throws Exception {
public void serializableTest() {
event.put("test", "true");
attributes.setOutputEvents(Arrays.asList(event));
attributes.setOutputEvents(List.of(event));
alertingSparkResult = new AlertingSparkResult(alertingResult, maxResult);
byte[] blob = SerializationUtils.serialize(alertingSparkResult);
Assert.assertTrue(blob.length > 0);
AlertingSparkResult clone = SerializationUtils.clone(alertingSparkResult);
Assert.assertEquals(0, clone.getExceptionsTotal());
Assert.assertEquals(1, clone.getMatchesTotal());
Assert.assertTrue(clone.getExceptions().isEmpty());

15
alerting/alerting-storm/pom.xml
View File

@@ -9,9 +9,14 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>23.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
@@ -46,7 +51,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -64,12 +69,6 @@
<version>${junit_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.11</artifactId>

169
alerting/alerting-storm/src/test/java/uk/co/gresearch/siembol/alerts/storm/AlertingEngineBoltTest.java
View File

@@ -3,7 +3,6 @@ package uk.co.gresearch.siembol.alerts.storm;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
@@ -33,90 +32,84 @@ import static org.mockito.internal.verification.VerificationModeFactory.times;
public class AlertingEngineBoltTest {
private static final ObjectReader JSON_READER = new ObjectMapper()
.readerFor(new TypeReference<Map<String, Object>>() {});
.readerFor(new TypeReference<Map<String, Object>>() {
});
/**
*{
* "source_type" : "secret",
* "is_alert" : "TruE",
* "dummy_field_int" : 1,
* "dummy_field_boolean" : false
*}
**/
@Multiline
public static String event;
private final String event = """
{
"source_type" : "secret",
"is_alert" : "TruE",
"dummy_field_int" : 1,
"dummy_field_boolean" : false
}
""";
private final String simpleTestRules = """
{
"rules_version" :1,
"tags" : [ { "tag_name" : "detection_source", "tag_value" : "siembol_alerts" } ],
"rules" : [ {
"rule_name" : "siembol_alert_generic",
"rule_version" : 1,
"rule_author" : "dummy",
"rule_description": "Test rule - is_alert is equal to true",
"source_type" : "*",
"matchers" : [ {
"matcher_type" : "REGEX_MATCH",
"is_negated" : false,
"field" : "is_alert",
"data" : "(?i)true" },
{
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "source_type",
"data": "(?<sensor>.*)"
}
]
}]
}
""";
/**
*{
* "rules_version" :1,
* "tags" : [ { "tag_name" : "detection_source", "tag_value" : "siembol_alerts" } ],
* "rules" : [ {
* "rule_name" : "siembol_alert_generic",
* "rule_version" : 1,
* "rule_author" : "dummy",
* "rule_description": "Test rule - is_alert is equal to true",
* "source_type" : "*",
* "matchers" : [ {
* "matcher_type" : "REGEX_MATCH",
* "is_negated" : false,
* "field" : "is_alert",
* "data" : "(?i)true" },
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "source_type",
* "data": "(?<sensor>.*)"
* }
* ]
* }]
*}
**/
@Multiline
public static String simpleTestRules;
/**
* {
* "rules_version": 1,
* "tags": [
* {
* "tag_name": "detection_source",
* "tag_value": "siembol_alerts"
* }
* ],
* "rules": [
* {
* "rule_name": "siembol_alert_generic",
* "rule_version": 1,
* "rule_author": "dummy",
* "rule_description": "Test rule - is_alert is equal to true",
* "source_type": "*",
* "matchers": [
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "is_alert",
* "data": "(?i)true"
* },
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "source_type",
* "data": "(?<sensor>.*)"
* }
* ],
* "tags": [
* {
* "tag_name": "correlation_key",
* "tag_value": "${dummy_field_int}"
* }
* ]
* }
* ]
* }
**/
@Multiline
public static String rulesForCorrelation;
private final String rulesForCorrelation = """
{
"rules_version": 1,
"tags": [
{
"tag_name": "detection_source",
"tag_value": "siembol_alerts"
}
],
"rules": [
{
"rule_name": "siembol_alert_generic",
"rule_version": 1,
"rule_author": "dummy",
"rule_description": "Test rule - is_alert is equal to true",
"source_type": "*",
"matchers": [
{
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_alert",
"data": "(?i)true"
},
{
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "source_type",
"data": "(?<sensor>.*)"
}
],
"tags": [
{
"tag_name": "correlation_key",
"tag_value": "${dummy_field_int}"
}
]
}
]
}
""";
private Tuple tuple;
private OutputCollector collector;
@@ -159,7 +152,7 @@ public class AlertingEngineBoltTest {
Assert.assertTrue(values.get(0) instanceof AlertMessages);
Assert.assertTrue(values.get(1) instanceof ExceptionMessages);
AlertMessages alerts = (AlertMessages)values.get(0);
AlertMessages alerts = (AlertMessages) values.get(0);
Assert.assertEquals(1, alerts.size());
Assert.assertTrue(alerts.get(0).isVisibleAlert());
Assert.assertEquals("siembol_alert_generic_v1", alerts.get(0).getFullRuleName());
@@ -201,7 +194,7 @@ public class AlertingEngineBoltTest {
Assert.assertTrue(values.get(0) instanceof AlertMessages);
Assert.assertTrue(values.get(1) instanceof ExceptionMessages);
AlertMessages alerts = (AlertMessages)values.get(0);
AlertMessages alerts = (AlertMessages) values.get(0);
Assert.assertEquals(1, alerts.size());
Assert.assertTrue(alerts.get(0).isCorrelationAlert());
Assert.assertFalse(alerts.get(0).isVisibleAlert());
@@ -225,7 +218,7 @@ public class AlertingEngineBoltTest {
}
@Test
public void testException(){
public void testException() {
when(tuple.getStringByField(eq(TupleFieldNames.EVENT.toString())))
.thenReturn("INVALID");
@@ -235,8 +228,8 @@ public class AlertingEngineBoltTest {
Assert.assertEquals(2, values.size());
Assert.assertTrue(values.get(0) instanceof AlertMessages);
Assert.assertTrue(values.get(1) instanceof ExceptionMessages);
Assert.assertTrue(((AlertMessages)values.get(0)).isEmpty());
Assert.assertEquals(1, ((ExceptionMessages)values.get(1)).size());
Assert.assertTrue(((ExceptionMessages)values.get(1)).get(0).contains("JsonParseException"));
Assert.assertTrue(((AlertMessages) values.get(0)).isEmpty());
Assert.assertEquals(1, ((ExceptionMessages) values.get(1)).size());
Assert.assertTrue(((ExceptionMessages) values.get(1)).get(0).contains("JsonParseException"));
}
}

133
alerting/alerting-storm/src/test/java/uk/co/gresearch/siembol/alerts/storm/AlertingStormApplicationTest.java
View File

@@ -5,7 +5,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import com.github.charithe.kafka.EphemeralKafkaBroker;
import com.github.charithe.kafka.KafkaJunitRule;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.generated.StormTopology;
@@ -29,77 +28,71 @@ public class AlertingStormApplicationTest {
private static final ObjectReader JSON_PARSERS_CONFIG_READER = new ObjectMapper()
.readerFor(AlertingStormAttributesDto.class);
private static final ObjectReader JSON_READER = new ObjectMapper()
.readerFor(new TypeReference<Map<String, Object>>() {});
.readerFor(new TypeReference<Map<String, Object>>() {
});
/**
*{
* "rules_version" :1,
* "tags" : [ { "tag_name" : "detection_source", "tag_value" : "siembol_alerts" } ],
* "rules" : [ {
* "rule_name" : "test_rule",
* "rule_version" : 1,
* "rule_author" : "dummy",
* "rule_protection" : {
* "max_per_hour" : 100,
* "max_per_day" : 10000
* },
* "rule_description": "test rule - is_alert is equal to true",
* "source_type" : "*",
* "matchers" : [ {
* "matcher_type" : "REGEX_MATCH",
* "is_negated" : false,
* "field" : "is_alert",
* "data" : "(?i)true" }
* ]
* }]
*}
**/
@Multiline
private static String testRules;
private final String testRules = """
{
"rules_version" :1,
"tags" : [ { "tag_name" : "detection_source", "tag_value" : "siembol_alerts" } ],
"rules" : [ {
"rule_name" : "test_rule",
"rule_version" : 1,
"rule_author" : "dummy",
"rule_protection" : {
"max_per_hour" : 100,
"max_per_day" : 10000
},
"rule_description": "test rule - is_alert is equal to true",
"source_type" : "*",
"matchers" : [ {
"matcher_type" : "REGEX_MATCH",
"is_negated" : false,
"field" : "is_alert",
"data" : "(?i)true" }
]
}]
}
""";
/**
*{
* "source_type" : "secret",
* "is_alert" : "TruE",
* "dummy_field_int" : 1,
* "dummy_field_boolean" : false
*}
**/
@Multiline
private static String goodAlert;
private final String goodAlert = """
{
"source_type" : "secret",
"is_alert" : "TruE",
"dummy_field_int" : 1,
"dummy_field_boolean" : false
}
""";
/**
* {
* "alerts.engine": "siembol_alerts",
* "alerts.input.topics": [ "input" ],
* "alerts.correlation.output.topic": "correlation.alerts",
* "kafka.error.topic": "errors",
* "alerts.output.topic": "alerts",
* "storm.attributes": {
* "first.pool.offset.strategy": "EARLIEST",
* "kafka.spout.properties": {
* "group.id": "alerts.reader",
* "security.protocol": "PLAINTEXT"
* }
* },
* "kafka.spout.num.executors": 1,
* "alerts.engine.bolt.num.executors": 1,
* "kafka.writer.bolt.num.executors": 1,
* "kafka.producer.properties": {
* "compression.type": "snappy",
* "security.protocol": "PLAINTEXT",
* "client.id": "test_producer"
* },
* "zookeeper.attributes": {
* "zk.path": "rules",
* "zk.base.sleep.ms": 1000,
* "zk.max.retries": 10
* }
* }
**/
@Multiline
public static String testConfig;
private final String testConfig = """
{
"alerts.engine": "siembol_alerts",
"alerts.input.topics": [ "input" ],
"alerts.correlation.output.topic": "correlation.alerts",
"kafka.error.topic": "errors",
"alerts.output.topic": "alerts",
"storm.attributes": {
"first.pool.offset.strategy": "EARLIEST",
"kafka.spout.properties": {
"group.id": "alerts.reader",
"security.protocol": "PLAINTEXT"
}
},
"kafka.spout.num.executors": 1,
"alerts.engine.bolt.num.executors": 1,
"kafka.writer.bolt.num.executors": 1,
"kafka.producer.properties": {
"compression.type": "snappy",
"security.protocol": "PLAINTEXT",
"client.id": "test_producer"
},
"zookeeper.attributes": {
"zk.path": "rules",
"zk.base.sleep.ms": 1000,
"zk.max.retries": 10
}
}
""";
@ClassRule
public static KafkaJunitRule kafkaRule = new KafkaJunitRule(EphemeralKafkaBroker.create());
@@ -134,7 +127,7 @@ public class AlertingStormApplicationTest {
cluster.submitTopology("test", config, topology);
}
@Test(timeout=200000)
@Test(timeout = 200000)
public void integrationTest() throws Exception {
kafkaRule.helper().produceStrings("input", goodAlert.trim());
List<String> outputEvent = kafkaRule.helper().consumeStrings("alerts", 1)

213
alerting/alerting-storm/src/test/java/uk/co/gresearch/siembol/alerts/storm/CorrelationAlertingTest.java
View File

@@ -5,7 +5,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import com.github.charithe.kafka.EphemeralKafkaBroker;
import com.github.charithe.kafka.KafkaJunitRule;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.generated.StormTopology;
@@ -27,120 +26,112 @@ import static org.mockito.Mockito.withSettings;
public class CorrelationAlertingTest {
private static final ObjectReader JSON_PARSERS_CONFIG_READER = new ObjectMapper()
.readerFor(AlertingStormAttributesDto.class);
private static ObjectReader JSON_READER = new ObjectMapper()
.readerFor(new TypeReference<Map<String, Object>>() {});
private static final ObjectReader JSON_READER = new ObjectMapper()
.readerFor(new TypeReference<Map<String, Object>>() {
});
/**
* {
* "siembol_alerts_full_rule_name": "alert1_v3",
* "siembol_alerts_rule_name": "alert1",
* "correlation_key": "evil",
* "siembol_alerts_max_per_hour": 200,
* "siembol_alerts_test": "true",
* "source_type": "a",
* "siembol_alerts_max_per_day": 10000
* }
**/
@Multiline
public static String alert1;
private final String alert1 = """
{
"siembol_alerts_full_rule_name": "alert1_v3",
"siembol_alerts_rule_name": "alert1",
"correlation_key": "evil",
"siembol_alerts_max_per_hour": 200,
"siembol_alerts_test": "true",
"source_type": "a",
"siembol_alerts_max_per_day": 10000
}
""";
/**
* {
* "siembol_alerts_full_rule_name": "alert1_v3",
* "siembol_alerts_rule_name": "alert2",
* "correlation_key": "evil",
* "sensor": "a",
* "siembol_alerts_max_per_hour": 200,
* "siembol_alerts_test": "true",
* "source_type": "a",
* "siembol_alerts_max_per_day": 10000
* }
**/
@Multiline
public static String alert2;
private final String alert2 = """
{
"siembol_alerts_full_rule_name": "alert1_v3",
"siembol_alerts_rule_name": "alert2",
"correlation_key": "evil",
"sensor": "a",
"siembol_alerts_max_per_hour": 200,
"siembol_alerts_test": "true",
"source_type": "a",
"siembol_alerts_max_per_day": 10000
}
""";
private final String simpleCorrelationRules = """
{
"rules_version": 1,
"tags": [
{
"tag_name": "detection_source",
"tag_value": "siembol_correlation_alerts_instance"
}
],
"rules": [
{
"tags": [
{
"tag_name": "test",
"tag_value": "true"
}
],
"rule_protection": {
"max_per_hour": 500,
"max_per_day": 1000
},
"rule_name": "test_rule",
"rule_version": 1,
"rule_author": "dummy",
"rule_description": "Testing rule",
"correlation_attributes": {
"time_unit": "seconds",
"time_window": 500,
"time_computation_type": "processing_time",
"alerts": [
{
"alert": "alert1",
"threshold": 2
},
{
"alert": "alert2",
"threshold": 1
}
]
}
}
]
}
}
""";
/**
* {
* "rules_version": 1,
* "tags": [
* {
* "tag_name": "detection_source",
* "tag_value": "siembol_correlation_alerts_instance"
* }
* ],
* "rules": [
* {
* "tags": [
* {
* "tag_name": "test",
* "tag_value": "true"
* }
* ],
* "rule_protection": {
* "max_per_hour": 500,
* "max_per_day": 1000
* },
* "rule_name": "test_rule",
* "rule_version": 1,
* "rule_author": "dummy",
* "rule_description": "Testing rule",
* "correlation_attributes": {
* "time_unit": "seconds",
* "time_window": 500,
* "time_computation_type": "processing_time",
* "alerts": [
* {
* "alert": "alert1",
* "threshold": 2
* },
* {
* "alert": "alert2",
* "threshold": 1
* }
* ]
* }
* }
* ]
* }
*}
**/
@Multiline
public static String simpleCorrelationRules;
/**
* {
* "alerts.engine": "siembol_correlation_alerts",
* "alerts.input.topics": [ "input" ],
* "alerts.correlation.output.topic": "correlation.alerts",
* "kafka.error.topic": "errors",
* "alerts.output.topic": "alerts",
* "alerts.engine.clean.interval.sec" : 2,
* "storm.attributes": {
* "first.pool.offset.strategy": "EARLIEST",
* "kafka.spout.properties": {
* "group.id": "alerts.reader",
* "security.protocol": "PLAINTEXT"
* }
* },
* "kafka.spout.num.executors": 1,
* "alerts.engine.bolt.num.executors": 1,
* "kafka.writer.bolt.num.executors": 1,
* "kafka.producer.properties": {
* "compression.type": "snappy",
* "security.protocol": "PLAINTEXT",
* "client.id": "test_producer"
* },
* "zookeeper.attributes": {
* "zk.path": "rules",
* "zk.base.sleep.ms": 1000,
* "zk.max.retries": 10
* }
* }
**/
@Multiline
public static String testConfig;
private final String testConfig = """
{
"alerts.engine": "siembol_correlation_alerts",
"alerts.input.topics": [ "input" ],
"alerts.correlation.output.topic": "correlation.alerts",
"kafka.error.topic": "errors",
"alerts.output.topic": "alerts",
"alerts.engine.clean.interval.sec" : 2,
"storm.attributes": {
"first.pool.offset.strategy": "EARLIEST",
"kafka.spout.properties": {
"group.id": "alerts.reader",
"security.protocol": "PLAINTEXT"
}
},
"kafka.spout.num.executors": 1,
"alerts.engine.bolt.num.executors": 1,
"kafka.writer.bolt.num.executors": 1,
"kafka.producer.properties": {
"compression.type": "snappy",
"security.protocol": "PLAINTEXT",
"client.id": "test_producer"
},
"zookeeper.attributes": {
"zk.path": "rules",
"zk.base.sleep.ms": 1000,
"zk.max.retries": 10
}
}
""";
@ClassRule
public static KafkaJunitRule kafkaRule = new KafkaJunitRule(EphemeralKafkaBroker.create());

160
alerting/alerting-storm/src/test/java/uk/co/gresearch/siembol/alerts/storm/CorrelationEngineBoltTest.java
View File

@@ -3,7 +3,6 @@ package uk.co.gresearch.siembol.alerts.storm;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
@@ -33,86 +32,81 @@ import static org.mockito.Mockito.when;
import static org.mockito.internal.verification.VerificationModeFactory.times;
public class CorrelationEngineBoltTest {
private static ObjectReader JSON_READER = new ObjectMapper()
.readerFor(new TypeReference<Map<String, Object>>() {});
private static final ObjectReader JSON_READER = new ObjectMapper()
.readerFor(new TypeReference<Map<String, Object>>() {
});
/**
* {
* "siembol_alerts_full_rule_name": "alert1_v3",
* "siembol_alerts_rule_name": "alert1",
* "correlation_key": "evil",
* "siembol_alerts_max_per_hour": 200,
* "siembol_alerts_test": "true",
* "source_type": "a",
* "siembol_alerts_max_per_day": 10000
* }
**/
@Multiline
public static String alert1;
private final String alert1 = """
{
"siembol_alerts_full_rule_name": "alert1_v3",
"siembol_alerts_rule_name": "alert1",
"correlation_key": "evil",
"siembol_alerts_max_per_hour": 200,
"siembol_alerts_test": "true",
"source_type": "a",
"siembol_alerts_max_per_day": 10000
}
""";
/**
* {
* "siembol_alerts_full_rule_name": "alert1_v3",
* "siembol_alerts_rule_name": "alert2",
* "correlation_key": "evil",
* "sensor": "a",
* "siembol_alerts_max_per_hour": 200,
* "siembol_alerts_test": "true",
* "source_type": "a",
* "siembol_alerts_max_per_day": 10000
* }
**/
@Multiline
public static String alert2;
private final String alert2 = """
{
"siembol_alerts_full_rule_name": "alert1_v3",
"siembol_alerts_rule_name": "alert2",
"correlation_key": "evil",
"sensor": "a",
"siembol_alerts_max_per_hour": 200,
"siembol_alerts_test": "true",
"source_type": "a",
"siembol_alerts_max_per_day": 10000
}
""";
private final String simpleCorrelationRules = """
{
"rules_version": 1,
"tags": [
{
"tag_name": "detection_source",
"tag_value": "siembol_correlation_alerts_instance"
}
],
"rules": [
{
"tags": [
{
"tag_name": "test",
"tag_value": "true"
}
],
"rule_protection": {
"max_per_hour": 500,
"max_per_day": 1000
},
"rule_name": "test_rule",
"rule_version": 1,
"rule_author": "dummy",
"rule_description": "Testing rule",
"correlation_attributes": {
"time_unit": "seconds",
"time_window": 500,
"time_computation_type": "processing_time",
"alerts": [
{
"alert": "alert1",
"threshold": 2
},
{
"alert": "alert2",
"threshold": 1
}
]
}
}
]
}
}
""";
/**
* {
* "rules_version": 1,
* "tags": [
* {
* "tag_name": "detection_source",
* "tag_value": "siembol_correlation_alerts_instance"
* }
* ],
* "rules": [
* {
* "tags": [
* {
* "tag_name": "test",
* "tag_value": "true"
* }
* ],
* "rule_protection": {
* "max_per_hour": 500,
* "max_per_day": 1000
* },
* "rule_name": "test_rule",
* "rule_version": 1,
* "rule_author": "dummy",
* "rule_description": "Testing rule",
* "correlation_attributes": {
* "time_unit": "seconds",
* "time_window": 500,
* "time_computation_type": "processing_time",
* "alerts": [
* {
* "alert": "alert1",
* "threshold": 2
* },
* {
* "alert": "alert2",
* "threshold": 1
* }
* ]
* }
* }
* ]
* }
*}
**/
@Multiline
public static String simpleCorrelationRules;
private Tuple tuple;
private OutputCollector collector;
@@ -160,7 +154,7 @@ public class CorrelationEngineBoltTest {
Assert.assertTrue(values.get(0) instanceof AlertMessages);
Assert.assertTrue(values.get(1) instanceof ExceptionMessages);
AlertMessages alerts = (AlertMessages)values.get(0);
AlertMessages alerts = (AlertMessages) values.get(0);
Assert.assertEquals(1, alerts.size());
Assert.assertTrue(alerts.get(0).isVisibleAlert());
Assert.assertFalse(alerts.get(0).isCorrelationAlert());
@@ -180,7 +174,7 @@ public class CorrelationEngineBoltTest {
}
@Test
public void testException(){
public void testException() {
when(tuple.getStringByField(eq(TupleFieldNames.EVENT.toString())))
.thenReturn("INVALID");
@@ -190,16 +184,16 @@ public class CorrelationEngineBoltTest {
Assert.assertEquals(2, values.size());
Assert.assertTrue(values.get(0) instanceof AlertMessages);
Assert.assertTrue(values.get(1) instanceof ExceptionMessages);
Assert.assertTrue(((AlertMessages)values.get(0)).isEmpty());
Assert.assertEquals(1, ((ExceptionMessages)values.get(1)).size());
Assert.assertTrue(((ExceptionMessages)values.get(1)).get(0).contains("JsonParseException"));
Assert.assertTrue(((AlertMessages) values.get(0)).isEmpty());
Assert.assertEquals(1, ((ExceptionMessages) values.get(1)).size());
Assert.assertTrue(((ExceptionMessages) values.get(1)).get(0).contains("JsonParseException"));
}
@Test
public void testNoMatchRule() {
when(tuple.getStringByField(eq(TupleFieldNames.EVENT.toString()))).thenReturn(
alert1, alert1, alert1, alert1, alert1,
alert1, alert1, alert1, alert1, alert1 );
alert1, alert1, alert1, alert1, alert1);
for (int i = 0; i < 10; i++) {
correlationAlertingEngineBolt.execute(tuple);

120
alerting/alerting-storm/src/test/java/uk/co/gresearch/siembol/alerts/storm/KafkaWriterBoltTest.java
View File

@@ -5,13 +5,9 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import com.github.charithe.kafka.EphemeralKafkaBroker;
import com.github.charithe.kafka.KafkaJunitRule;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
import org.junit.Assert;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.*;
import org.mockito.Mockito;
import uk.co.gresearch.siembol.alerts.common.AlertingEngineType;
@@ -32,68 +28,62 @@ public class KafkaWriterBoltTest {
private static final ObjectReader JSON_PARSERS_CONFIG_READER = new ObjectMapper()
.readerFor(AlertingStormAttributesDto.class);
private static final ObjectReader JSON_MAP_READER = new ObjectMapper()
.readerFor(new TypeReference<Map<String, Object>>() {});
.readerFor(new TypeReference<Map<String, Object>>() {
});
/**
*{
* "alerts.engine": "siembol_alerts",
* "alerts.input.topics": [ "enrichmnents" ],
* "alerts.correlation.output.topic": "correlation.alerts",
* "kafka.error.topic": "errors",
* "alerts.output.topic": "alerts",
* "storm.attributes": {
* "first.pool.offset.strategy": "EARLIEST",
* "kafka.spout.properties": {
* "group.id": "alerts.reader",
* "security.protocol": "PLAINTEXT"
* }
* },
* "kafka.spout.num.executors": 1,
* "alerts.engine.bolt.num.executors": 1,
* "kafka.writer.bolt.num.executors": 1,
* "kafka.producer.properties": {
* "compression.type": "snappy",
* "security.protocol": "PLAINTEXT",
* "client.id": "test_producer"
* }
* }
**/
@Multiline
public static String alertingStormConfig;
private final String alertingStormConfig = """
{
"alerts.engine": "siembol_alerts",
"alerts.input.topics": [ "enrichmnents" ],
"alerts.correlation.output.topic": "correlation.alerts",
"kafka.error.topic": "errors",
"alerts.output.topic": "alerts",
"storm.attributes": {
"first.pool.offset.strategy": "EARLIEST",
"kafka.spout.properties": {
"group.id": "alerts.reader",
"security.protocol": "PLAINTEXT"
}
},
"kafka.spout.num.executors": 1,
"alerts.engine.bolt.num.executors": 1,
"kafka.writer.bolt.num.executors": 1,
"kafka.producer.properties": {
"compression.type": "snappy",
"security.protocol": "PLAINTEXT",
"client.id": "test_producer"
}
}
""";
/**
* {
* "ip_src_addr": "1.2.3.4",
* "b": 1,
* "is_alert": "true",
* "source_type": "test",
* "detection_source": "alerts",
* "siembol_alerts_full_rule_name": "alert1_v1",
* "siembol_alerts_rule_name": "alert1",
* "siembol_alerts_max_per_day": 1,
* "siembol_alerts_max_per_hour": 1
* }
**/
@Multiline
public static String AlertMessageStr;
/**
* {
* "ip_src_addr": "1.2.3.4",
* "b": 1,
* "is_alert": "true",
* "source_type": "test",
* "detection_source": "alerts",
* "siembol_alerts_full_rule_name": "alert1_v1",
* "siembol_alerts_rule_name": "alert1",
* "siembol_alerts_max_per_day": 1,
* "siembol_alerts_max_per_hour": 1,
* "correlation_key" : "evil"
* }
**/
@Multiline
public static String AlertMessageCorrelationStr;
private final String AlertMessageStr = """
{
"ip_src_addr": "1.2.3.4",
"b": 1,
"is_alert": "true",
"source_type": "test",
"detection_source": "alerts",
"siembol_alerts_full_rule_name": "alert1_v1",
"siembol_alerts_rule_name": "alert1",
"siembol_alerts_max_per_day": 1,
"siembol_alerts_max_per_hour": 1
}
""";
private final String AlertMessageCorrelationStr = """
{
"ip_src_addr": "1.2.3.4",
"b": 1,
"is_alert": "true",
"source_type": "test",
"detection_source": "alerts",
"siembol_alerts_full_rule_name": "alert1_v1",
"siembol_alerts_rule_name": "alert1",
"siembol_alerts_max_per_day": 1,
"siembol_alerts_max_per_hour": 1,
"correlation_key" : "evil"
}
""";
@ClassRule
public static KafkaJunitRule kafkaRule = new KafkaJunitRule(EphemeralKafkaBroker.create());
@@ -132,7 +122,7 @@ public class KafkaWriterBoltTest {
AlertMessage alert = new AlertMessage(AlertingEngineType.SIEMBOL_ALERTS, alertMap, AlertMessageStr);
AlertMessages.add(alert);
writerBolt.execute(tuple);
List<String> outputAlert= kafkaRule.helper().consumeStrings("alerts", 1)
List<String> outputAlert = kafkaRule.helper().consumeStrings("alerts", 1)
.get(10, TimeUnit.SECONDS);
Assert.assertNotNull(outputAlert);
Assert.assertEquals(1, outputAlert.size());

2
alerting/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</parent>
<modules>
<module>alerting-core</module>

10
config-editor/config-editor-core/pom.xml
View File

@@ -9,13 +9,13 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
@@ -79,12 +79,6 @@
<version>${junit_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>

440
config-editor/config-editor-core/src/test/java/uk/co/gresearch/siembol/configeditor/common/ConfigEditorUtilsTest.java
View File

@@ -4,7 +4,6 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Test;
@@ -13,240 +12,229 @@ import java.util.Map;
import java.util.Optional;
public class ConfigEditorUtilsTest {
/**
* {
* "rules_schema": {
* "type": "object",
* "description": "Incident Response Rules",
* "title": "rules",
* "properties": {
* "rules_version": {
* "type": "integer",
* "description": "Incident response rules version",
* "default": 0
* },
* "rules": {
* "type": "array",
* "items": {
* "type": "object",
* "description": "Response rule that should handle response to a siembol alert",
* "title": "rule",
* "properties": {
* "rule_name": {
* "type": "string",
* "description": "ResponseRule name that uniquely identifies the rule"
* },
* "rule_author": {
* "type": "string",
* "description": "The owner of the rule"
* },
* "rule_version": {
* "type": "integer",
* "description": "The version of the rule",
* "default": 0
* },
* "rule_description": {
* "type": "string",
* "description": "The description of the rule"
* },
* "evaluators": {
* "type": "array",
* "items": {
* "type": "object",
* "description": "Response evaluator used in response rules",
* "title": "response evaluator",
* "oneOf": [
* {
* "type": "object",
* "title": "matching_evaluator",
* "properties": {
* "evaluator_type": {
* "enum": [
* "matching_evaluator"
* ],
* "default": "matching_evaluator"
* },
* "evaluator_attributes": {
* "type": "object",
* "description": "Attributes for matching evaluator",
* "title": "matching evaluator attributes",
* "properties": {
* "evaluation_result": {
* "enum": [
* "match",
* "filtered"
* ],
* "type": "string",
* "description": "Evaluation result returned by the evaluator after matching",
* "default": "match"
* },
* "matchers": {
* "type": "array",
* "items": {
* "type": "object",
* "description": "Matcher for matching fields in response rules",
* "title": "matcher",
* "properties": {
* "matcher_type": {
* "enum": [
* "REGEX_MATCH",
* "IS_IN_SET"
* ],
* "type": "string",
* "description": "Type of matcher, either Regex match or list of strings (newline delimited)"
* },
* "is_negated": {
* "type": "boolean",
* "description": "The matcher is negated",
* "default": false
* },
* "field": {
* "type": "string",
* "description": "Field on which the matcher will be evaluated"
* },
* "case_insensitive": {
* "type": "boolean",
* "description": "Use case insensitive string compare",
* "default": false
* },
* "data": {
* "type": "string",
* "description": "Matcher expression as defined by matcher type"
* }
* },
* "required": [
* "data",
* "field",
* "matcher_type"
* ]
* },
* "description": "Matchers of the evaluator",
* "minItems": 1
* }
* },
* "required": [
* "evaluation_result",
* "matchers"
* ]
* }
* },
* "required": [
* "evaluator_type",
* "evaluator_attributes"
* ]
* }
* ]
* },
* "description": "Evaluators of the rule",
* "minItems": 1
* }
* },
* "required": [
* "evaluators",
* "rule_author",
* "rule_name",
* "rule_version"
* ]
* },
* "description": "Response rules",
* "minItems": 1
* }
* },
* "required": [
* "rules",
* "rules_version"
* ]
* }
* }
*/
@Multiline
public static String rulesSchema;
private final String rulesSchema = """
{
"rules_schema": {
"type": "object",
"description": "Incident Response Rules",
"title": "rules",
"properties": {
"rules_version": {
"type": "integer",
"description": "Incident response rules version",
"default": 0
},
"rules": {
"type": "array",
"items": {
"type": "object",
"description": "Response rule that should handle response to a siembol alert",
"title": "rule",
"properties": {
"rule_name": {
"type": "string",
"description": "ResponseRule name that uniquely identifies the rule"
},
"rule_author": {
"type": "string",
"description": "The owner of the rule"
},
"rule_version": {
"type": "integer",
"description": "The version of the rule",
"default": 0
},
"rule_description": {
"type": "string",
"description": "The description of the rule"
},
"evaluators": {
"type": "array",
"items": {
"type": "object",
"description": "Response evaluator used in response rules",
"title": "response evaluator",
"oneOf": [
{
"type": "object",
"title": "matching_evaluator",
"properties": {
"evaluator_type": {
"enum": [
"matching_evaluator"
],
"default": "matching_evaluator"
},
"evaluator_attributes": {
"type": "object",
"description": "Attributes for matching evaluator",
"title": "matching evaluator attributes",
"properties": {
"evaluation_result": {
"enum": [
"match",
"filtered"
],
"type": "string",
"description": "Evaluation result returned by the evaluator after matching",
"default": "match"
},
"matchers": {
"type": "array",
"items": {
"type": "object",
"description": "Matcher for matching fields in response rules",
"title": "matcher",
"properties": {
"matcher_type": {
"enum": [
"REGEX_MATCH",
"IS_IN_SET"
],
"type": "string",
"description": "Type of matcher, either Regex match or list of strings (newline delimited)"
},
"is_negated": {
"type": "boolean",
"description": "The matcher is negated",
"default": false
},
"field": {
"type": "string",
"description": "Field on which the matcher will be evaluated"
},
"case_insensitive": {
"type": "boolean",
"description": "Use case insensitive string compare",
"default": false
},
"data": {
"type": "string",
"description": "Matcher expression as defined by matcher type"
}
},
"required": [
"data",
"field",
"matcher_type"
]
},
"description": "Matchers of the evaluator",
"minItems": 1
}
},
"required": [
"evaluation_result",
"matchers"
]
}
},
"required": [
"evaluator_type",
"evaluator_attributes"
]
}
]
},
"description": "Evaluators of the rule",
"minItems": 1
}
},
"required": [
"evaluators",
"rule_author",
"rule_name",
"rule_version"
]
},
"description": "Response rules",
"minItems": 1
}
},
"required": [
"rules",
"rules_version"
]
}
}
""";
/**
* {
* "$..evaluators": {
* "widget": {
* "formlyConfig": {
* "type": "tab-array"
* }
* }
* },
* "$..rule_description": {
* "widget": {
* "formlyConfig": {
* "type": "textarea",
* "wrappers": []
* }
* }
* },
* "$..matchers.items": {
* "widget": {
* "formlyConfig": {
* "wrappers": [
* "expansion-panel"
* ]
* }
* }
* },
* "$..matchers.items.properties.data": {
* "title" : "changed"
* }
* }
*/
@Multiline
public static String layoutConfig;
private final String layoutConfig = """
{
"$..evaluators": {
"widget": {
"formlyConfig": {
"type": "tab-array"
}
}
},
"$..rule_description": {
"widget": {
"formlyConfig": {
"type": "textarea",
"wrappers": []
}
}
},
"$..matchers.items": {
"widget": {
"formlyConfig": {
"wrappers": [
"expansion-panel"
]
}
}
},
"$..matchers.items.properties.data": {
"title" : "changed"
}
}
""";
/**
* {
* "robots": {
* "type": "tab-array"
* }
* }
*/
@Multiline
public static String unknownKeyConfig;
/**
* {
* "$..items": {
* "type": "tab-array"
* }
* }
*/
@Multiline
public static String multipleKeyConfig;
private final String unknownKeyConfig = """
{
"robots": {
"type": "tab-array"
}
}
""";
/**
* {
* "rules_schema.description": {
* "type": "tab-array"
* }
* }
*/
@Multiline
public static String valueWithString;
private final String multipleKeyConfig = """
{
"$..items": {
"type": "tab-array"
}
}
""";
/**
* {
* "$..evaluators": {
* }
* }
*/
@Multiline
public static String valueEmptyObject;
private final String valueWithString = """
{
"rules_schema.description": {
"type": "tab-array"
}
}
""";
/**
* {
* "$..evaluators": "dummy"
* }
*/
@Multiline
public static String valueString;
private final String valueEmptyObject = """
{
"$..evaluators": {
}
}
""";
private final String valueString = """
{
"$..evaluators": "dummy"
}
""";
private static final ObjectReader JSON_OBJECT_READER = new ObjectMapper()
.readerFor(new TypeReference<Map<String, Object>>() {});
.readerFor(new TypeReference<Map<String, Object>>() {
});
private static final ObjectReader FORM_ATTRIBUTES_READER = new ObjectMapper()
.readerFor(new TypeReference<Map<String, JsonNode>>() {});
.readerFor(new TypeReference<Map<String, JsonNode>>() {
});
@Test

20
config-editor/config-editor-core/src/test/java/uk/co/gresearch/siembol/configeditor/configinfo/AdminConfigInfoProviderTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.configeditor.configinfo;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -9,16 +8,14 @@ import uk.co.gresearch.siembol.configeditor.common.UserInfo;
import uk.co.gresearch.siembol.configeditor.common.ConfigInfo;
public class AdminConfigInfoProviderTest {
/**
* {
* "config_version": 1,
* "secret": "john",
* "object": { },
* "actions": [ "test" , "siembol"]
* }
**/
@Multiline
public static String testConfig;
private final String testConfig = """
{
"config_version": 1,
"secret": "john",
"object": { },
"actions": [ "test" , "siembol"]
}
""";
private final ConfigInfoProvider infoProvider = new AdminConfigInfoProvider();
private UserInfo steve;
@@ -54,5 +51,4 @@ public class AdminConfigInfoProviderTest {
Assert.assertEquals(infoProvider.isReleaseFile("admin_config.json"), true);
Assert.assertEquals(infoProvider.isReleaseFile("rules.json"), false);
}
}

176
config-editor/config-editor-core/src/test/java/uk/co/gresearch/siembol/configeditor/configinfo/JsonRuleConfigInfoProviderTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.configeditor.configinfo;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -14,102 +13,91 @@ import java.util.ArrayList;
import java.util.List;
public class JsonRuleConfigInfoProviderTest {
/**
* {
* "rule_name": "info_provider-test",
* "rule_author": "john",
* "rule_version": 12345,
* "rule_description": "Test rule",
* "enrichments": { },
* "actions": { }
* }
**/
@Multiline
public static String testRule;
/**
* {
* "rule_name": "info_provider_test",
* "rule_author": "john",
* "rule_version": 0,
* "rule_description": "Test rule",
* "enrichments": { },
* "actions": { }
* }
**/
@Multiline
public static String testNewRule;
private final String testRule = """
{
"rule_name": "info_provider-test",
"rule_author": "john",
"rule_version": 12345,
"rule_description": "Test rule",
"enrichments": { },
"actions": { }
}
""";
/**
* {
* "rules_version" : 1,
* "rules": [{
* "rule_name": "info_provider_test",
* "rule_author": "mark",
* "rule_version": 12,
* "rule_description": "Test rule",
* "enrichments": { },
* "actions": { }
* }]
* }
**/
@Multiline
public static String release;
private final String testNewRule = """
{
"rule_name": "info_provider_test",
"rule_author": "john",
"rule_version": 0,
"rule_description": "Test rule",
"enrichments": { },
"actions": { }
}
""";
/**
* {
* "rules_version" : 1,
* "rules": []
* }
**/
@Multiline
public static String releaseNoRules;
private final String release = """
{
"rules_version" : 1,
"rules": [{
"rule_name": "info_provider_test",
"rule_author": "mark",
"rule_version": 12,
"rule_description": "Test rule",
"enrichments": { },
"actions": { }
}]
}
""";
/**
* {
* "rules_version": 1,
* "rules": [
* {
* "rule_name": "info_provider_test",
* "rule_author": "mark",
* "rule_version": 1,
* "rule_description": "Test rule",
* "enrichments": {},
* "actions": {}
* },
* {
* "rule_name": "info_provider_extra_test",
* "rule_author": "mark",
* "rule_version": 1,
* "rule_description": "Test rule",
* "enrichments": {},
* "actions": {}
* },
* {
* "rule_name": "secret_test",
* "rule_author": "mark",
* "rule_version": 1,
* "rule_description": "Test rule",
* "enrichments": {},
* "actions": {}
* }
* ]
* }
**/
@Multiline
public static String releaseThreeRules;
private final String releaseNoRules = """
{
"rules_version" : 1,
"rules": []
}
""";
/**
* {
* "rule_name": "../../../test",
* "rule_author": "steve",
* "rule_version": 12345,
* "rule_description": "Test rule",
* "enrichments": { },
* "actions": { }
* }
**/
@Multiline
public static String maliciousRule;
private final String releaseThreeRules = """
{
"rules_version": 1,
"rules": [
{
"rule_name": "info_provider_test",
"rule_author": "mark",
"rule_version": 1,
"rule_description": "Test rule",
"enrichments": {},
"actions": {}
},
{
"rule_name": "info_provider_extra_test",
"rule_author": "mark",
"rule_version": 1,
"rule_description": "Test rule",
"enrichments": {},
"actions": {}
},
{
"rule_name": "secret_test",
"rule_author": "mark",
"rule_version": 1,
"rule_description": "Test rule",
"enrichments": {},
"actions": {}
}
]
}
""";
private final String maliciousRule = """
{
"rule_name": "../../../test",
"rule_author": "steve",
"rule_version": 12345,
"rule_description": "Test rule",
"enrichments": { },
"actions": { }
}
""";
private final ConfigInfoProvider infoProvider = JsonRuleConfigInfoProvider.create();
@@ -181,7 +169,7 @@ public class JsonRuleConfigInfoProviderTest {
@Test(expected = java.lang.IllegalArgumentException.class)
public void ruleInfoWrongJson() {
infoProvider.getConfigInfo(steve,"WRONG JSON");
infoProvider.getConfigInfo(steve, "WRONG JSON");
}
@Test(expected = java.lang.IllegalArgumentException.class)

188
config-editor/config-editor-core/src/test/java/uk/co/gresearch/siembol/configeditor/configinfo/TestCaseConfigInfoProviderTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.configeditor.configinfo;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -11,105 +10,98 @@ import uk.co.gresearch.siembol.configeditor.common.ConfigInfoType;
import java.util.ArrayList;
public class TestCaseConfigInfoProviderTest {
/**
* {
* "test_case_name": "test_case",
* "version": 12345,
* "author": "john",
* "config_name": "syslog",
* "description": "unitest test case",
* "test_specification": {
* "secret": true
* },
* "assertions": [
* {
* "assertion_type": "path_and_value_matches",
* "json_path": "$.a",
* "expected_pattern": "^.*mp$",
* "negated_pattern": false,
* "description": "match string",
* "active": true
* },
* {
* "assertion_type": "only_if_path_exists",
* "json_path": "s",
* "expected_pattern": "secret",
* "negated_pattern": true,
* "description": "skipped assertion",
* "active": false
* }
* ]
* }
*/
@Multiline
public static String testCase;
private final String testCase = """
{
"test_case_name": "test_case",
"version": 12345,
"author": "john",
"config_name": "syslog",
"description": "unitest test case",
"test_specification": {
"secret": true
},
"assertions": [
{
"assertion_type": "path_and_value_matches",
"json_path": "$.a",
"expected_pattern": "^.*mp$",
"negated_pattern": false,
"description": "match string",
"active": true
},
{
"assertion_type": "only_if_path_exists",
"json_path": "s",
"expected_pattern": "secret",
"negated_pattern": true,
"description": "skipped assertion",
"active": false
}
]
}
""";
/**
* {
* "test_case_name": "test_case",
* "version": 0,
* "author": "john",
* "config_name": "syslog",
* "description": "unitest test case",
* "test_specification": {
* "secret": true
* },
* "assertions": [
* {
* "assertion_type": "path_and_value_matches",
* "json_path": "$.a",
* "expected_pattern": "^.*mp$",
* "negated_pattern": false,
* "description": "match string",
* "active": true
* },
* {
* "assertion_type": "only_if_path_exists",
* "json_path": "s",
* "expected_pattern": "secret",
* "negated_pattern": true,
* "description": "skipped assertion",
* "active": false
* }
* ]
* }
*/
@Multiline
public static String testCaseNew;
/**
* {
* "test_case_name": "./../../test",
* "version": 1,
* "author": "john",
* "config_name": "syslog",
* "description": "unitest test case",
* "test_specification": {
* "secret": true
* },
* "assertions": [
* {
* "assertion_type": "path_and_value_matches",
* "json_path": "$.a",
* "expected_pattern": "^.*mp$",
* "negated_pattern": false,
* "description": "match string",
* "active": true
* },
* {
* "assertion_type": "only_if_path_exists",
* "json_path": "s",
* "expected_pattern": "secret",
* "negated_pattern": true,
* "description": "skipped assertion",
* "active": false
* }
* ]
* }
*/
@Multiline
public static String maliciousTestCase;
private final String testCaseNew = """
{
"test_case_name": "test_case",
"version": 0,
"author": "john",
"config_name": "syslog",
"description": "unitest test case",
"test_specification": {
"secret": true
},
"assertions": [
{
"assertion_type": "path_and_value_matches",
"json_path": "$.a",
"expected_pattern": "^.*mp$",
"negated_pattern": false,
"description": "match string",
"active": true
},
{
"assertion_type": "only_if_path_exists",
"json_path": "s",
"expected_pattern": "secret",
"negated_pattern": true,
"description": "skipped assertion",
"active": false
}
]
}
""";
private final String maliciousTestCase = """
{
"test_case_name": "./../../test",
"version": 1,
"author": "john",
"config_name": "syslog",
"description": "unitest test case",
"test_specification": {
"secret": true
},
"assertions": [
{
"assertion_type": "path_and_value_matches",
"json_path": "$.a",
"expected_pattern": "^.*mp$",
"negated_pattern": false,
"description": "match string",
"active": true
},
{
"assertion_type": "only_if_path_exists",
"json_path": "s",
"expected_pattern": "secret",
"negated_pattern": true,
"description": "skipped assertion",
"active": false
}
]
}
""";
private final TestCaseInfoProvider infoProvider = new TestCaseInfoProvider();
private UserInfo steve;

317
config-editor/config-editor-core/src/test/java/uk/co/gresearch/siembol/configeditor/testcase/TestCaseEvaluatorImplTest.java
View File

@@ -1,7 +1,7 @@
package uk.co.gresearch.siembol.configeditor.testcase;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -13,173 +13,162 @@ import static uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult.Stat
import static uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult.StatusCode.OK;
public class TestCaseEvaluatorImplTest {
/**
* {
* "a" : "tmp",
* "b" : true,
* "c" : "test",
* "d" : { "e" : "test"},
* "f" : [ "1", "2"]
* }
**/
@Multiline
public static String testResult;
/**
* {
* "test_case_name": "test",
* "version": 1,
* "author": "john",
* "config_name": "syslog",
* "description": "unitest test case",
* "test_specification": {
* "secret": true
* },
* "assertions": [
* {
* "assertion_type": "path_and_value_matches",
* "json_path": "$.a",
* "expected_pattern": "^.*mp$",
* "negated_pattern": false,
* "description": "match string",
* "active": true
* },
* {
* "assertion_type": "only_if_path_exists",
* "json_path": "s",
* "expected_pattern": "secret",
* "negated_pattern": true,
* "description": "skipped assertion",
* "active": false
* }
* ]
* }
*/
@Multiline
public static String simpleTestCaseString;
private final String testResult = """
{
"a" : "tmp",
"b" : true,
"c" : "test",
"d" : { "e" : "test"},
"f" : [ "1", "2"]
}
""";
/**
* {
* "test_case_name": "test",
* "version": 1,
* "author": "john",
* "config_name": "syslog",
* "description": "unitest test case",
* "test_specification": {
* "secret": true
* },
* "assertions": [
* {
* "assertion_type": "only_if_path_exists",
* "json_path": "$.b",
* "expected_pattern": "secret",
* "negated_pattern": true,
* "description": "negated match of boolean",
* "active": true
* },
* {
* "assertion_type": "only_if_path_exists",
* "json_path": "s",
* "expected_pattern": "secret",
* "negated_pattern": true,
* "description": "skipped assertion",
* "active": false
* }
* ]
* }
*/
@Multiline
public static String simpleTestCaseBoolean;
private final String simpleTestCaseString = """
{
"test_case_name": "test",
"version": 1,
"author": "john",
"config_name": "syslog",
"description": "unitest test case",
"test_specification": {
"secret": true
},
"assertions": [
{
"assertion_type": "path_and_value_matches",
"json_path": "$.a",
"expected_pattern": "^.*mp$",
"negated_pattern": false,
"description": "match string",
"active": true
},
{
"assertion_type": "only_if_path_exists",
"json_path": "s",
"expected_pattern": "secret",
"negated_pattern": true,
"description": "skipped assertion",
"active": false
}
]
}
""";
/**
* {
* "test_case_name": "test",
* "version": 1,
* "author": "john",
* "config_name": "syslog",
* "description": "unitest test case",
* "test_specification": {
* "secret": true
* },
* "assertions": [
* {
* "assertion_type": "only_if_path_exists",
* "json_path": "$.d",
* "expected_pattern": ".*ask",
* "negated_pattern": false,
* "description": "fail to match object",
* "active": true
* },
* {
* "assertion_type": "only_if_path_exists",
* "json_path": "s",
* "expected_pattern": "secret",
* "negated_pattern": true,
* "description": "skipped assertion",
* "active": false
* }
* ]
* }
*/
@Multiline
public static String simpleTestCaseObject;
private final String simpleTestCaseBoolean = """
{
"test_case_name": "test",
"version": 1,
"author": "john",
"config_name": "syslog",
"description": "unitest test case",
"test_specification": {
"secret": true
},
"assertions": [
{
"assertion_type": "only_if_path_exists",
"json_path": "$.b",
"expected_pattern": "secret",
"negated_pattern": true,
"description": "negated match of boolean",
"active": true
},
{
"assertion_type": "only_if_path_exists",
"json_path": "s",
"expected_pattern": "secret",
"negated_pattern": true,
"description": "skipped assertion",
"active": false
}
]
}
""";
/**
* {
* "test_case_name": "test",
* "version": 1,
* "author": "john",
* "config_name": "syslog",
* "description": "unitest test case",
* "test_specification": {
* "secret": true
* },
* "assertions": [
* {
* "assertion_type": "only_if_path_exists",
* "json_path": "$.f",
* "expected_pattern": ".*1.*",
* "negated_pattern": false,
* "description": "match in array",
* "active": true
* }
* ]
* }
*/
@Multiline
public static String simpleTestCaseArray;
private final String simpleTestCaseObject = """
{
"test_case_name": "test",
"version": 1,
"author": "john",
"config_name": "syslog",
"description": "unitest test case",
"test_specification": {
"secret": true
},
"assertions": [
{
"assertion_type": "only_if_path_exists",
"json_path": "$.d",
"expected_pattern": ".*ask",
"negated_pattern": false,
"description": "fail to match object",
"active": true
},
{
"assertion_type": "only_if_path_exists",
"json_path": "s",
"expected_pattern": "secret",
"negated_pattern": true,
"description": "skipped assertion",
"active": false
}
]
}
""";
/**
* {
* "test_case_name": "test",
* "version": 1,
* "author": "john",
* "config_name": "syslog",
* "description": "unitest test case",
* "test_specification": {
* "secret": true
* },
* "assertions": [
* {
* "assertion_type": "only_if_path_exists",
* "json_path": "$.g",
* "expected_pattern": "secret",
* "negated_pattern": true,
* "description": "only if path exists test",
* "active": true
* },
* {
* "assertion_type": "only_if_path_exists",
* "json_path": "s",
* "expected_pattern": "secret",
* "negated_pattern": true,
* "description": "skipped assertion",
* "active": false
* }
* ]
* }
*/
@Multiline
public static String simpleTestCaseMissing;
private final String simpleTestCaseArray = """
{
"test_case_name": "test",
"version": 1,
"author": "john",
"config_name": "syslog",
"description": "unitest test case",
"test_specification": {
"secret": true
},
"assertions": [
{
"assertion_type": "only_if_path_exists",
"json_path": "$.f",
"expected_pattern": ".*1.*",
"negated_pattern": false,
"description": "match in array",
"active": true
}
]
}
""";
private final String simpleTestCaseMissing = """
{
"test_case_name": "test",
"version": 1,
"author": "john",
"config_name": "syslog",
"description": "unitest test case",
"test_specification": {
"secret": true
},
"assertions": [
{
"assertion_type": "only_if_path_exists",
"json_path": "$.g",
"expected_pattern": "secret",
"negated_pattern": true,
"description": "only if path exists test",
"active": true
},
{
"assertion_type": "only_if_path_exists",
"json_path": "s",
"expected_pattern": "secret",
"negated_pattern": true,
"description": "skipped assertion",
"active": false
}
]
}
""";
private TestCaseEvaluator testCaseEvaluator;

18
config-editor/config-editor-rest/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</parent>
<dependencyManagement>
<dependencies>
@@ -56,7 +56,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -67,22 +67,22 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-core</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-services</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-sync</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -93,7 +93,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -104,7 +104,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-core</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -115,7 +115,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding-core</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

45
config-editor/config-editor-rest/src/main/java/uk/co/gresearch/siembol/configeditor/rest/application/ConfigEditorConfiguration.java
View File

@@ -7,6 +7,7 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.DependsOn;
import org.springframework.util.ResourceUtils;
import uk.co.gresearch.siembol.common.model.ZooKeeperAttributesDto;
import uk.co.gresearch.siembol.common.testing.TestingZooKeeperConnectorFactory;
@@ -50,7 +51,7 @@ public class ConfigEditorConfiguration implements DisposableBean {
private ServiceAggregator serviceAggregator;
@Bean
@Bean("serviceAggregator")
ServiceAggregator serviceAggregator() throws Exception {
Map<String, ConfigStoreProperties> configStorePropertiesMap = ConfigEditorHelper
.getConfigStoreProperties(this.properties);
@@ -76,33 +77,28 @@ public class ConfigEditorConfiguration implements DisposableBean {
return serviceAggregator;
}
@Bean
@Bean("testCaseEvaluator")
TestCaseEvaluator testCaseEvaluator() throws Exception {
ConfigEditorUiLayout uiLayout = ConfigEditorUtils.readUiLayoutFile(properties.getTestCasesUiConfigFileName());
return new TestCaseEvaluatorImpl(uiLayout);
}
@Bean
@Bean("stormApplicationProvider")
@ConditionalOnProperty(prefix = "config-editor", value = "synchronisation")
StormApplicationProvider stormApplicationProvider() throws Exception {
return StormApplicationProviderImpl.create(zooKeeperConnectorFactory(), properties.getStormTopologiesZooKeeper());
@DependsOn("zooKeeperConnectorFactory")
StormApplicationProvider stormApplicationProvider(
@Autowired ZooKeeperConnectorFactory zooKeeperConnectorFactory) throws Exception {
return StormApplicationProviderImpl.create(zooKeeperConnectorFactory, properties.getStormTopologiesZooKeeper());
}
@Bean
@Bean("synchronisationService")
@ConditionalOnProperty(prefix = "config-editor", value = "synchronisation")
SynchronisationService synchronisationService() throws Exception {
@DependsOn({"zooKeeperConnectorFactory", "stormApplicationProvider"})
SynchronisationService synchronisationService(
@Autowired ZooKeeperConnectorFactory zooKeeperConnectorFactory,
@Autowired StormApplicationProvider stormApplicationProvider) throws Exception {
serviceAggregator = serviceAggregator();
ZooKeeperConnectorFactory zooKeeperConnectorFactory = zooKeeperConnectorFactory();
Map<String, ZooKeeperConnector> zooKeeperConnectorMap = new HashMap<>();
if (properties.getEnrichmentTablesZooKeeper() != null) {
for (Map.Entry<String, ZooKeeperAttributesDto> entry : properties.getEnrichmentTablesZooKeeper().entrySet()) {
zooKeeperConnectorMap.put(entry.getKey(),
zooKeeperConnectorFactory.createZookeeperConnector(entry.getValue()));
}
}
enrichmentTablesProvider(zooKeeperConnectorMap);
StormApplicationProvider stormApplicationProvider = stormApplicationProvider();
List<ConfigServiceHelper> aggregatorServices = serviceAggregator
.getAggregatorServices()
.stream()
@@ -117,7 +113,7 @@ public class ConfigEditorConfiguration implements DisposableBean {
return ret;
}
@Bean
@Bean("zooKeeperConnectorFactory")
@ConditionalOnProperty(prefix = "config-editor", value = "synchronisation")
ZooKeeperConnectorFactory zooKeeperConnectorFactory() throws Exception {
if (properties.getTestingZookeeperFiles() == null) {
@@ -134,9 +130,18 @@ public class ConfigEditorConfiguration implements DisposableBean {
return ret;
}
@Bean
@Bean("enrichmentTablesProvider")
@ConditionalOnProperty(prefix = "config-editor", value = "synchronisation")
EnrichmentTablesProvider enrichmentTablesProvider(Map<String, ZooKeeperConnector> zooKeeperConnectorMap) {
@DependsOn("zooKeeperConnectorFactory")
EnrichmentTablesProvider enrichmentTablesProvider(
@Autowired ZooKeeperConnectorFactory zooKeeperConnectorFactory) throws Exception {
Map<String, ZooKeeperConnector> zooKeeperConnectorMap = new HashMap<>();
if (properties.getEnrichmentTablesZooKeeper() != null) {
for (Map.Entry<String, ZooKeeperAttributesDto> entry : properties.getEnrichmentTablesZooKeeper().entrySet()) {
zooKeeperConnectorMap.put(entry.getKey(),
zooKeeperConnectorFactory.createZookeeperConnector(entry.getValue()));
}
}
return new EnrichmentTablesProviderImpl(zooKeeperConnectorMap);
}

26
config-editor/config-editor-services/pom.xml
View File

@@ -10,53 +10,53 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-core</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-storm</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-storm</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-storm</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-core</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding-core</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>
@@ -70,12 +70,6 @@
<version>${mockito_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-yaml</artifactId>

34
config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/alerts/AlertingRuleSchemaServiceTest.java
View File

@@ -1,6 +1,6 @@
package uk.co.gresearch.siembol.configeditor.service.alerts;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -29,27 +29,23 @@ import static uk.co.gresearch.siembol.common.result.SiembolResult.StatusCode.OK;
public class AlertingRuleSchemaServiceTest {
/**
* {"test_event":"true"}
**/
@Multiline
public static String testEvent;
/**
* {
* "event" : {"test_event":"true"}
* }
**/
@Multiline
public static String testSpecification;
private final String testEvent = """
{"test_event":"true"}""";
private final String testSpecification = """
{
"event" : {"test_event":"true"}
}
""";
private AlertingRuleSchemaService alertingRuleSchemaService;
private final String ruleSchema = "dummmy schema";
private final String testSchema = "dummmy test schema";
private final String adminSchema = "dummmy admin config schema";
private final String testRule = "dummmy rule";
private final String testRules = "dummmy rules";
private final String ruleSchema = "dummy schema";
private final String testSchema = "dummy test schema";
private final String adminSchema = "dummy admin config schema";
private final String testRule = "dummy rule";
private final String testRules = "dummy rules";
private final String testResultOutput = "test output";
private final String testConfig = "dummmy config";
private final String testConfig = "dummy config";
private AlertingCompiler alertingCompiler;
private AlertingResult alertingResult;
private AlertingAttributes alertingAttributes;

30
config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/alerts/sigma/SigmaConditionTokenTest.java
View File

@@ -1,6 +1,6 @@
package uk.co.gresearch.siembol.configeditor.service.alerts.sigma;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang3.tuple.Pair;
import org.junit.Assert;
import org.junit.Before;
@@ -14,29 +14,17 @@ import java.util.*;
import static org.mockito.Mockito.when;
public class SigmaConditionTokenTest {
/**
* not 1 of filter*
**/
@Multiline
private static String condition1;
private final String condition1 = """
not 1 of filter*""";
/**
* ( 1 of filter* and not secret) or long
**/
@Multiline
private static String condition2;
private final String condition2 = """
( 1 of filter* and not secret) or long""";
/**
* 1 of A | 1 of B
**/
@Multiline
private static String conditionUnsupported;
private final String conditionUnsupported = """
1 of A | 1 of B""";
/**
* 1 of A $ 1 of B
**/
@Multiline
private static String conditionUnknownToken;
private final String conditionUnknownToken = """
1 of A $ 1 of B""";
private SigmaConditionTokenNode node;
Map<String, SigmaSearch> searches;

123
config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/alerts/sigma/SigmaRuleImporterTest.java
View File

@@ -3,7 +3,7 @@ package uk.co.gresearch.siembol.configeditor.service.alerts.sigma;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -18,69 +18,64 @@ import static uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult.Stat
public class SigmaRuleImporterTest {
private static final ObjectReader ALERTING_RULE_READER = new ObjectMapper().readerFor(RuleDto.class);
/**
* {
* "field_mapping": [
* {
* "sigma_field": "sigma_user",
* "siembol_field": "siembol_user"
* }
* ],
* "rule_metadata_mapping": {
* "rule_name": "based_on_${title}",
* "rule_description": "generated from ${description} and id: ${id}",
* "source_type": "secret_data",
* "tags": [
* {
* "tag_name": "sigma_tags",
* "tag_value": "${tags}"
* }
* ]
* }
* }
**/
@Multiline
private static String importerAttributes;
private final String importerAttributes = """
{
"field_mapping": [
{
"sigma_field": "sigma_user",
"siembol_field": "siembol_user"
}
],
"rule_metadata_mapping": {
"rule_name": "based_on_${title}",
"rule_description": "generated from ${description} and id: ${id}",
"source_type": "secret_data",
"tags": [
{
"tag_name": "sigma_tags",
"tag_value": "${tags}"
}
]
}
}
""";
/**
* title: Sigma Title( Experimental???
* id: d06be400-8045-4200-0067-740a2009db25
* status: experimental
* description: Detects secret
* references:
* - https://github.com/siembol
* author: Joe
* date: 2021/10/09
* logsource:
* category: process_creation
* product: windows
* detection:
* image_path:
* Image|endswith: 'secret.exe'
* cmd_s:
* CommandLine|contains: '/S'
* cmd_c:
* CommandLine|contains: '/C'
* net_utility:
* Image|endswith:
* - '\net.exe'
* - '\net1.exe'
* CommandLine|contains:
* - ' user '
* - ' use '
* - ' group '
* condition: image_path and cmd_c and (cmd_s or not net_utility)
* fields:
* - CommandLine
* falsepositives:
* - Unknown
* level: medium
* tags:
* - attack.defense_evasion
* - attack.example
*/
@Multiline
private static String sigmaRuleExample;
private final String sigmaRuleExample = """
title: Sigma Title( Experimental???
id: d06be400-8045-4200-0067-740a2009db25
status: experimental
description: Detects secret
references:
- https://github.com/siembol
author: Joe
date: 2021/10/09
logsource:
category: process_creation
product: windows
detection:
image_path:
Image|endswith: 'secret.exe'
cmd_s:
CommandLine|contains: '/S'
cmd_c:
CommandLine|contains: '/C'
net_utility:
Image|endswith:
- '\\net.exe'
- '\\net1.exe'
CommandLine|contains:
- ' user '
- ' use '
- ' group '
condition: image_path and cmd_c and (cmd_s or not net_utility)
fields:
- CommandLine
falsepositives:
- Unknown
level: medium
tags:
- attack.defense_evasion
- attack.example""";
private SigmaRuleImporter importer;
@@ -117,7 +112,7 @@ public class SigmaRuleImporterTest {
@Test
public void validateAttributesMissingRequired() {
ConfigEditorResult result = importer.validateImporterAttributes(
importerAttributes.replace("source_type", "uknown"));
importerAttributes.replace("source_type", "unknown"));
Assert.assertEquals(BAD_REQUEST, result.getStatusCode());
Assert.assertNotNull(result.getAttributes().getMessage());
}

61
config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/alerts/sigma/SigmaSearchTest.java
View File

@@ -3,7 +3,7 @@ package uk.co.gresearch.siembol.configeditor.service.alerts.sigma;
import com.fasterxml.jackson.core.*;
import com.fasterxml.jackson.databind.*;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -16,40 +16,34 @@ import uk.co.gresearch.siembol.configeditor.service.alerts.sigma.model.SigmaDete
import java.util.*;
public class SigmaSearchTest {
/**
* iptables:
* Image|endswith: '/service'
* CommandLine|contains|all:
* - 'iptables'
* - 'stop'
* - 1
* keywords:
* - entered promiscuous mode
* - 1
* - test secret word
* keyword:
* - single
*/
@Multiline
private static String sigmaDetectionExample;
private final String sigmaDetectionExample = """
iptables:
Image|endswith: '/service'
CommandLine|contains|all:
- 'iptables'
- 'stop'
- 1
keywords:
- entered promiscuous mode
- 1
- test secret word
keyword:
- single
""";
/**
* iptables:
* Image: null
* test: ''
*/
@Multiline
private static String sigmaDetectionExampleEmptyValues;
private final String sigmaDetectionExampleEmptyValues = """
iptables:
Image: null
test: ''
""";
/**
* iptables:
* Image: 'abc'
* CommandLine|contains|all:
* - true
* - 1
*/
@Multiline
private static String sigmaDetectionExampleBooleanValue;
private final String sigmaDetectionExampleBooleanValue = """
iptables:
Image: 'abc'
CommandLine|contains|all:
- true
- 1
""";
private static final ObjectReader SIGMA_DETECTION_READER = new ObjectMapper(new YAMLFactory())
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
@@ -209,5 +203,4 @@ public class SigmaSearchTest {
.fieldNames()
.forEachRemaining(x -> builder.addMapEntry(x, searchesMap.get("iptables").get(x)));
}
}

154
config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/parserconfig/ParserConfigConfigInfoProviderTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.configeditor.service.parserconfig;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -13,84 +12,77 @@ import java.util.ArrayList;
import java.util.List;
public class ParserConfigConfigInfoProviderTest {
/**
* {
* "parser_name": "test_parser",
* "parser_author": "john",
* "parser_version": 12345,
* "parser_config": {
* "parser_attributes": {
* "parser_type": "syslog",
* "syslog_config": {
* "syslog_version": "RFC_3164",
* "timezone": "UTC"
* }
* }
* }
* }
**/
@Multiline
public static String testParser;
/**
* {
* "parser_name": "test_parser",
* "parser_author": "john",
* "parser_version": 0,
* "parser_config": {
* "parser_attributes": {
* "parser_type": "syslog",
* "syslog_config": {
* "syslog_version": "RFC_3164",
* "timezone": "UTC"
* }
* }
* }
* }
**/
@Multiline
public static String testNewParser;
private final String testParser = """
{
"parser_name": "test_parser",
"parser_author": "john",
"parser_version": 12345,
"parser_config": {
"parser_attributes": {
"parser_type": "syslog",
"syslog_config": {
"syslog_version": "RFC_3164",
"timezone": "UTC"
}
}
}
}
""";
/**
* {
* "parsers_version" : 1,
* "parser_configurations": [
* {
* "parser_name": "test_parser",
* "parser_author": "john",
* "parser_version": 1,
* "parser_config": {
* "parser_attributes": {
* "parser_type": "syslog",
* "syslog_config": {
* "syslog_version": "RFC_3164",
* "timezone": "UTC"
* }
* }
* }
* }]
* }
**/
@Multiline
public static String release;
private final String testNewParser = """
{
"parser_name": "test_parser",
"parser_author": "john",
"parser_version": 0,
"parser_config": {
"parser_attributes": {
"parser_type": "syslog",
"syslog_config": {
"syslog_version": "RFC_3164",
"timezone": "UTC"
}
}
}
}
""";
/**
* {
* "parser_name": "../../../test_parser",
* "parser_author": "john",
* "parser_version": 12345,
* "parser_config": {
* "parser_attributes": {
* "parser_type": "syslog",
* "syslog_config": {
* "syslog_version": "RFC_3164",
* "timezone": "UTC"
* }
* }
* }
* }
**/
@Multiline
public static String maliciousConfig;
private final String release = """
{
"parsers_version" : 1,
"parser_configurations": [
{
"parser_name": "test_parser",
"parser_author": "john",
"parser_version": 1,
"parser_config": {
"parser_attributes": {
"parser_type": "syslog",
"syslog_config": {
"syslog_version": "RFC_3164",
"timezone": "UTC"
}
}
}
}]
}
""";
private final String maliciousConfig = """
{
"parser_name": "../../../test_parser",
"parser_author": "john",
"parser_version": 12345,
"parser_config": {
"parser_attributes": {
"parser_type": "syslog",
"syslog_config": {
"syslog_version": "RFC_3164",
"timezone": "UTC"
}
}
}
}
""";
private final ConfigInfoProvider infoProvider = ParserConfigConfigInfoProvider.create();
@@ -121,6 +113,7 @@ public class ParserConfigConfigInfoProviderTest {
Assert.assertEquals(1, info.getFilesContent().size());
Assert.assertTrue(info.getFilesContent().containsKey("test_parser.json"));
Assert.assertTrue(info.getFilesContent().get("test_parser.json").isPresent());
Assert.assertTrue(info.getFilesContent()
.get("test_parser.json").get().indexOf("\"parser_version\": 12346,") > 0);
Assert.assertTrue(info.getFilesContent()
@@ -135,8 +128,9 @@ public class ParserConfigConfigInfoProviderTest {
Assert.assertEquals("john", info.getCommitter());
Assert.assertEquals("Updating configuration: test_parser to version: 12346", info.getCommitMessage());
Assert.assertEquals("john@secret.net", info.getCommitterEmail());
Assert.assertEquals( 1, info.getFilesContent().size());
Assert.assertEquals(1, info.getFilesContent().size());
Assert.assertTrue(info.getFilesContent().containsKey("test_parser.json"));
Assert.assertTrue(info.getFilesContent().get("test_parser.json").isPresent());
Assert.assertTrue(info.getFilesContent()
.get("test_parser.json").get().indexOf("\"parser_version\": 12346,") > 0);
Assert.assertTrue(info.getFilesContent()
@@ -153,6 +147,7 @@ public class ParserConfigConfigInfoProviderTest {
Assert.assertEquals(info.getCommitterEmail(), steve.getEmail());
Assert.assertEquals(1, info.getFilesContent().size());
Assert.assertTrue(info.getFilesContent().containsKey("test_parser.json"));
Assert.assertTrue(info.getFilesContent().get("test_parser.json").isPresent());
Assert.assertTrue(info.getFilesContent()
.get("test_parser.json").get().indexOf("\"parser_version\": 1,") > 0);
Assert.assertTrue(info.isNewConfig());
@@ -160,7 +155,7 @@ public class ParserConfigConfigInfoProviderTest {
@Test(expected = java.lang.IllegalArgumentException.class)
public void ConfigInfoWrongJson() {
infoProvider.getConfigInfo(steve,"WRONG JSON");
infoProvider.getConfigInfo(steve, "WRONG JSON");
}
@Test(expected = java.lang.IllegalArgumentException.class)
@@ -192,6 +187,7 @@ public class ParserConfigConfigInfoProviderTest {
Assert.assertEquals(1, info.getFilesContent().size());
Assert.assertTrue(info.getFilesContent().containsKey("parsers.json"));
Assert.assertTrue(info.getFilesContent().get("parsers.json").isPresent());
Assert.assertTrue(info.getFilesContent()
.get("parsers.json").get().indexOf("\"parsers_version\": 2,") > 0);

29
config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/parserconfig/ParserConfigSchemaServiceTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.configeditor.service.parserconfig;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -22,23 +21,19 @@ import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.times;
public class ParserConfigSchemaServiceTest {
/**
* {
* "encoding" : "utf8_string",
* "log" : "dummy log"
* }
**/
@Multiline
public static String logUtf8;
private final String logUtf8 = """
{
"encoding" : "utf8_string",
"log" : "dummy log"
}
""";
/**
* {
* "encoding" : "hex_string",
* "log" : "64756D6D79206C6F67"
* }
**/
@Multiline
public static String logHex;
private final String logHex = """
{
"encoding" : "hex_string",
"log" : "64756D6D79206C6F67"
}
""";
private ParserConfigSchemaService parserConfigSchemaService;
private final String schema = "dummmy schema";

167
config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/parsingapp/ParsingAppConfigInfoProviderTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.configeditor.service.parsingapp;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -10,93 +9,85 @@ import uk.co.gresearch.siembol.configeditor.common.ConfigInfoProvider;
public class ParsingAppConfigInfoProviderTest {
/**
*{
* "parsing_app_name": "test",
* "parsing_app_version": 12345,
* "parsing_app_author": "dummy",
* "parsing_app_description": "Description of parser application",
* "parsing_app_settings": {
* "input_topics": [
* "secret"
* ],
* "error_topic": "error",
* "input_parallelism": 1,
* "parsing_parallelism": 2,
* "output_parallelism": 3,
* "parsing_app_type": "single_parser"
* },
* "parsing_settings": {
* "single_parser": {
* "parser_name": "single",
* "output_topic": "output"
* }
* }
* }
**/
@Multiline
static String simpleSingleApplicationParser;
private final String simpleSingleApplicationParser = """
{
"parsing_app_name": "test",
"parsing_app_version": 12345,
"parsing_app_author": "dummy",
"parsing_app_description": "Description of parser application",
"parsing_app_settings": {
"input_topics": [
"secret"
],
"error_topic": "error",
"input_parallelism": 1,
"parsing_parallelism": 2,
"output_parallelism": 3,
"parsing_app_type": "single_parser"
},
"parsing_settings": {
"single_parser": {
"parser_name": "single",
"output_topic": "output"
}
}
}
""";
/**
*{
* "parsing_app_name": "test",
* "parsing_app_version": 0,
* "parsing_app_author": "dummy",
* "parsing_app_description": "Description of parser application",
* "parsing_app_settings": {
* "input_topics": [
* "secret"
* ],
* "error_topic": "error",
* "input_parallelism": 1,
* "parsing_parallelism": 2,
* "output_parallelism": 3,
* "parsing_app_type": "single_parser"
* },
* "parsing_settings": {
* "single_parser": {
* "parser_name": "single",
* "output_topic": "output"
* }
* }
* }
**/
@Multiline
static String simpleSingleApplicationParserNew;
/**
*{
* "parsing_applications_version" : 1,
* "parsing_applications" : [
* {
* "parsing_app_name": "test",
* "parsing_app_version": 12345,
* "parsing_app_author": "dummy",
* "parsing_app_description": "Description of parser application",
* "parsing_app_settings": {
* "input_topics": [
* "secret"
* ],
* "error_topic": "error",
* "input_parallelism": 1,
* "parsing_parallelism": 2,
* "output_parallelism": 3,
* "parsing_app_type": "single_parser"
* },
* "parsing_settings": {
* "single_parser": {
* "parser_name": "single",
* "output_topic": "output"
* }
* }
* }
* ]
* }
*
**/
@Multiline
static String release;
private final String simpleSingleApplicationParserNew = """
{
"parsing_app_name": "test",
"parsing_app_version": 0,
"parsing_app_author": "dummy",
"parsing_app_description": "Description of parser application",
"parsing_app_settings": {
"input_topics": [
"secret"
],
"error_topic": "error",
"input_parallelism": 1,
"parsing_parallelism": 2,
"output_parallelism": 3,
"parsing_app_type": "single_parser"
},
"parsing_settings": {
"single_parser": {
"parser_name": "single",
"output_topic": "output"
}
}
}
""";
private final String release = """
{
"parsing_applications_version" : 1,
"parsing_applications" : [
{
"parsing_app_name": "test",
"parsing_app_version": 12345,
"parsing_app_author": "dummy",
"parsing_app_description": "Description of parser application",
"parsing_app_settings": {
"input_topics": [
"secret"
],
"error_topic": "error",
"input_parallelism": 1,
"parsing_parallelism": 2,
"output_parallelism": 3,
"parsing_app_type": "single_parser"
},
"parsing_settings": {
"single_parser": {
"parser_name": "single",
"output_topic": "output"
}
}
}
]
}
""";
static String user = "unknown@secret.net";
private final ConfigInfoProvider infoProvider = ParsingAppConfigInfoProvider.create();
@@ -127,6 +118,7 @@ public class ParsingAppConfigInfoProviderTest {
Assert.assertEquals(1, info.getFilesContent().size());
Assert.assertTrue(info.getFilesContent().containsKey("test.json"));
Assert.assertTrue(info.getFilesContent().get("test.json").isPresent());
Assert.assertTrue(info.getFilesContent()
.get("test.json").get().indexOf("\"parsing_app_version\": 12346,") > 0);
Assert.assertTrue(info.getFilesContent()
@@ -143,6 +135,7 @@ public class ParsingAppConfigInfoProviderTest {
Assert.assertEquals("dummy@secret.net", info.getCommitterEmail());
Assert.assertEquals(1, info.getFilesContent().size());
Assert.assertTrue(info.getFilesContent().containsKey("test.json"));
Assert.assertTrue(info.getFilesContent().get("test.json").isPresent());
Assert.assertTrue(info.getFilesContent()
.get("test.json").get().indexOf("\"parsing_app_version\": 12346,") > 0);
Assert.assertTrue(info.getFilesContent()
@@ -159,6 +152,7 @@ public class ParsingAppConfigInfoProviderTest {
Assert.assertEquals(user, info.getCommitterEmail());
Assert.assertEquals(1, info.getFilesContent().size());
Assert.assertTrue(info.getFilesContent().containsKey("test.json"));
Assert.assertTrue(info.getFilesContent().get("test.json").isPresent());
Assert.assertTrue(info.getFilesContent()
.get("test.json").get().indexOf("\"parsing_app_version\": 1,") > 0);
Assert.assertTrue(info.getFilesContent()
@@ -195,6 +189,7 @@ public class ParsingAppConfigInfoProviderTest {
Assert.assertEquals(1, info.getFilesContent().size());
Assert.assertTrue(info.getFilesContent().containsKey("parsing_applications.json"));
Assert.assertTrue(info.getFilesContent().get("parsing_applications.json").isPresent());
Assert.assertTrue(info.getFilesContent()
.get("parsing_applications.json").get().indexOf("\"parsing_applications_version\": 2,") > 0);
}

2
config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/parsingapp/ParsingAppConfigSchemaServiceTest.java
View File

@@ -129,7 +129,7 @@ public class ParsingAppConfigSchemaServiceTest {
}
@Test
public void getImportersEmpty() throws Exception {
public void getImportersEmpty() {
ConfigEditorResult ret = parsingAppConfigSchemaService.getImporters();
Assert.assertEquals(ConfigEditorResult.StatusCode.OK, ret.getStatusCode());
Assert.assertNotNull(ret.getAttributes().getConfigImporters());

341
config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/response/ResponseSchemaServiceTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.configeditor.service.response;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -18,187 +17,179 @@ import static uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult.Stat
import static uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult.StatusCode.OK;
public class ResponseSchemaServiceTest {
/**
* {
* "statusCode": "OK",
* "attributes": {
* "rules_schema": {
* "type": "object",
* "description": "Incident Response Rules",
* "title": "rules",
* "properties": {
* "rules_version": {
* "type": "integer",
* "description": "Incident response rules version",
* "default": 0
* },
* "rules": {
* "type": "array",
* "items": {
* "type": "object",
* "description": "Response rule that should handle response to a siembol alert",
* "title": "rule",
* "properties": {
* "rule_name": {
* "type": "string",
* "description": "ResponseRule name that uniquely identifies the rule"
* },
* "rule_author": {
* "type": "string",
* "description": "The owner of the rule"
* },
* "rule_version": {
* "type": "integer",
* "description": "The version of the rule",
* "default": 0
* },
* "rule_description": {
* "type": "string",
* "description": "The description of the rule"
* },
* "evaluators": {
* "type": "array",
* "items": {
* "type": "object",
* "description": "Response evaluator used in response rules",
* "title": "response evaluator",
* "oneOf": [
* {
* "type": "object",
* "title": "matching_evaluator",
* "properties": {
* "evaluator_type": {
* "enum": [
* "matching_evaluator"
* ],
* "default": "matching_evaluator"
* },
* "evaluator_attributes": {
* "type": "object",
* "description": "Attributes for matching evaluator",
* "title": "matching evaluator attributes",
* "properties": {
* "evaluation_result": {
* "enum": [
* "match",
* "filtered"
* ],
* "type": "string",
* "description": "Evaluation result returned by the evaluator after matching",
* "default": "match"
* },
* "matchers": {
* "type": "array",
* "items": {
* "type": "object",
* "description": "Matcher for matching fields in response rules",
* "title": "matcher",
* "properties": {
* "matcher_type": {
* "enum": [
* "REGEX_MATCH",
* "IS_IN_SET"
* ],
* "type": "string",
* "description": "Type of matcher, either Regex match or list of strings (newline delimited)"
* },
* "is_negated": {
* "type": "boolean",
* "description": "The matcher is negated",
* "default": false
* },
* "field": {
* "type": "string",
* "description": "Field on which the matcher will be evaluated"
* },
* "case_insensitive": {
* "type": "boolean",
* "description": "Use case insensitive string compare",
* "default": false
* },
* "data": {
* "type": "string",
* "description": "Matcher expression as defined by matcher type"
* }
* },
* "required": [
* "data",
* "field",
* "matcher_type"
* ]
* },
* "description": "Matchers of the evaluator",
* "minItems": 1
* }
* },
* "required": [
* "evaluation_result",
* "matchers"
* ]
* }
* },
* "required": [
* "evaluator_type",
* "evaluator_attributes"
* ]
* }
* ]
* },
* "description": "Evaluators of the rule",
* "minItems": 1
* }
* },
* "required": [
* "evaluators",
* "rule_author",
* "rule_name",
* "rule_version"
* ]
* },
* "description": "Response rules",
* "minItems": 1
* }
* },
* "required": [
* "rules",
* "rules_version"
* ]
* }
* }
* }
*/
@Multiline
public static String rulesSchema;
private final String rulesSchema = """
{
"statusCode": "OK",
"attributes": {
"rules_schema": {
"type": "object",
"description": "Incident Response Rules",
"title": "rules",
"properties": {
"rules_version": {
"type": "integer",
"description": "Incident response rules version",
"default": 0
},
"rules": {
"type": "array",
"items": {
"type": "object",
"description": "Response rule that should handle response to a siembol alert",
"title": "rule",
"properties": {
"rule_name": {
"type": "string",
"description": "ResponseRule name that uniquely identifies the rule"
},
"rule_author": {
"type": "string",
"description": "The owner of the rule"
},
"rule_version": {
"type": "integer",
"description": "The version of the rule",
"default": 0
},
"rule_description": {
"type": "string",
"description": "The description of the rule"
},
"evaluators": {
"type": "array",
"items": {
"type": "object",
"description": "Response evaluator used in response rules",
"title": "response evaluator",
"oneOf": [
{
"type": "object",
"title": "matching_evaluator",
"properties": {
"evaluator_type": {
"enum": [
"matching_evaluator"
],
"default": "matching_evaluator"
},
"evaluator_attributes": {
"type": "object",
"description": "Attributes for matching evaluator",
"title": "matching evaluator attributes",
"properties": {
"evaluation_result": {
"enum": [
"match",
"filtered"
],
"type": "string",
"description": "Evaluation result returned by the evaluator after matching",
"default": "match"
},
"matchers": {
"type": "array",
"items": {
"type": "object",
"description": "Matcher for matching fields in response rules",
"title": "matcher",
"properties": {
"matcher_type": {
"enum": [
"REGEX_MATCH",
"IS_IN_SET"
],
"type": "string",
"description": "Type of matcher, either Regex match or list of strings (newline delimited)"
},
"is_negated": {
"type": "boolean",
"description": "The matcher is negated",
"default": false
},
"field": {
"type": "string",
"description": "Field on which the matcher will be evaluated"
},
"case_insensitive": {
"type": "boolean",
"description": "Use case insensitive string compare",
"default": false
},
"data": {
"type": "string",
"description": "Matcher expression as defined by matcher type"
}
},
"required": [
"data",
"field",
"matcher_type"
]
},
"description": "Matchers of the evaluator",
"minItems": 1
}
},
"required": [
"evaluation_result",
"matchers"
]
}
},
"required": [
"evaluator_type",
"evaluator_attributes"
]
}
]
},
"description": "Evaluators of the rule",
"minItems": 1
}
},
"required": [
"evaluators",
"rule_author",
"rule_name",
"rule_version"
]
},
"description": "Response rules",
"minItems": 1
}
},
"required": [
"rules",
"rules_version"
]
}
}
}
""";
/**
* {"statusCode":"OK","attributes":{"test_schema":{ "type" : "object", "description" : "Specification for testing responding rules", "title" : "response test specification", "properties" : { "event" : { "type" : "object", "description" : "Alert for response alerts evaluation", "title" : "json raw string" } }, "required" : [ "event" ]}}}
*/
@Multiline
public static String testSchema;
private final String testSchema = """
{"statusCode":"OK","attributes":{"test_schema":{ "type" : "object", "description" : "Specification for testing responding rules", "title" : "response test specification", "properties" : { "event" : { "type" : "object", "description" : "Alert for response alerts evaluation", "title" : "json raw string" } }, "required" : [ "event" ]}}}
""";
/**
* {
* "statusCode":"ERROR",
* "attributes":{ "message" : "dummy"}}
* }
*/
@Multiline
public static String errorMessage;
private final String errorMessage = """
{
"statusCode":"ERROR",
"attributes":{ "message" : "dummy"}}
}
""";
/**
* {
* "statusCode":"OK",
* "attributes":{ "message" : "dummy"}}
* }
*/
@Multiline
public static String okMessage;
private final String okMessage = """
{
"statusCode":"OK",
"attributes":{ "message" : "dummy"}}
}
""";
private ResponseSchemaService.Builder builder;
private ResponseSchemaService responseSchemaService;
private HttpProvider httpProvider;
private String dummyJsonObject = "{ \"dummy\" : true }";
private String dummyJsonObject2 = "{ \"dummy2\" : true }";
private final String dummyJsonObject = "{ \"dummy\" : true }";
private final String dummyJsonObject2 = "{ \"dummy2\" : true }";
@Before
public void setup() throws Exception {

14
config-editor/config-editor-sync/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -20,17 +20,17 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-core</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
@@ -126,12 +126,6 @@
<version>${junit_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>

4
config-editor/config-editor-sync/src/test/java/uk/co/gresearch/siembol/configeditor/sync/actions/GetAdminConfigActionTest.java
View File

@@ -17,10 +17,10 @@ import static uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult.Stat
public class GetAdminConfigActionTest {
private ConfigServiceHelper serviceHelper;
private String adminConfig = "ADMIN_CONFIG";
private final String adminConfig = "ADMIN_CONFIG";
private GetAdminConfigAction getAdminConfigAction;
private ConfigEditorServiceContext context;
private int version = 1;
private final int version = 1;
@Before
public void setUp() {

292
config-editor/config-editor-sync/src/test/java/uk/co/gresearch/siembol/configeditor/sync/actions/GetParsingAppStormTopologyActionTest.java
View File

@@ -2,7 +2,7 @@ package uk.co.gresearch.siembol.configeditor.sync.actions;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -25,156 +25,150 @@ public class GetParsingAppStormTopologyActionTest {
private static final ObjectReader ADMIN_CONFIG_READER = new ObjectMapper()
.readerFor(StormParsingApplicationAttributesDto.class);
/**
*{
* "config_version": 1,
* "client.id.prefix": "siembol.writer",
* "group.id.prefix": "siembol.reader",
* "zookeeper.attributes": {
* "zk.url": "global_url",
* "zk.path": "global_path",
* "zk.base.sleep.ms": 1000,
* "zk.max.retries": 3
* },
* "kafka.batch.writer.attributes": {
* "batch.size": 50,
* "producer.properties": {
* "bootstrap.servers": "global_servers",
* "security.protocol": "SASL_PLAINTEXT"
* }
* },
* "storm.attributes": {
* "bootstrap.servers": "dummy",
* "first.pool.offset.strategy": "UNCOMMITTED_LATEST",
* "kafka.spout.properties": {
* "session.timeout.ms": 300000,
* "security.protocol": "SASL_PLAINTEXT"
* },
* "storm.config": {
* "num.workers": 1
* }
* },
* "overridden.applications": [
* {
* "application.name": "secret",
* "kafka.batch.writer.attributes": {
* "batch.size": 1,
* "producer.properties": {
* "bootstrap.servers": "dummy",
* "security.protocol": "SASL_PLAINTEXT"
* }
* },
* "storm.attributes": {
* "bootstrap.servers": "dummy",
* "first.pool.offset.strategy": "UNCOMMITTED_LATEST",
* "kafka.spout.properties": {
* "session.timeout.ms": 300000,
* "security.protocol": "SASL_PLAINTEXT"
* },
* "storm.config": {
* "num.workers": 2
* }
* }
* }
* ]
* }
**/
@Multiline
public static String adminConfig;
private final String adminConfig = """
{
"config_version": 1,
"client.id.prefix": "siembol.writer",
"group.id.prefix": "siembol.reader",
"zookeeper.attributes": {
"zk.url": "global_url",
"zk.path": "global_path",
"zk.base.sleep.ms": 1000,
"zk.max.retries": 3
},
"kafka.batch.writer.attributes": {
"batch.size": 50,
"producer.properties": {
"bootstrap.servers": "global_servers",
"security.protocol": "SASL_PLAINTEXT"
}
},
"storm.attributes": {
"bootstrap.servers": "dummy",
"first.pool.offset.strategy": "UNCOMMITTED_LATEST",
"kafka.spout.properties": {
"session.timeout.ms": 300000,
"security.protocol": "SASL_PLAINTEXT"
},
"storm.config": {
"num.workers": 1
}
},
"overridden.applications": [
{
"application.name": "secret",
"kafka.batch.writer.attributes": {
"batch.size": 1,
"producer.properties": {
"bootstrap.servers": "dummy",
"security.protocol": "SASL_PLAINTEXT"
}
},
"storm.attributes": {
"bootstrap.servers": "dummy",
"first.pool.offset.strategy": "UNCOMMITTED_LATEST",
"kafka.spout.properties": {
"session.timeout.ms": 300000,
"security.protocol": "SASL_PLAINTEXT"
},
"storm.config": {
"num.workers": 2
}
}
}
]
}
""";
/**
*{
* "config_version": 1,
* "client.id.prefix": "siembol.writer",
* "group.id.prefix": "siembol.reader",
* "zookeeper.attributes": {
* "zk.url": "global_url",
* "zk.path": "global_path",
* "zk.base.sleep.ms": 1000,
* "zk.max.retries": 3
* },
* "kafka.batch.writer.attributes": {
* "batch.size": 50,
* "producer.properties": {
* "bootstrap.servers": "global_servers",
* "security.protocol": "SASL_PLAINTEXT"
* }
* },
* "storm.attributes": {
* "bootstrap.servers": "dummy",
* "first.pool.offset.strategy": "UNCOMMITTED_LATEST",
* "kafka.spout.properties": {
* "session.timeout.ms": 300000,
* "security.protocol": "SASL_PLAINTEXT"
* },
* "storm.config": {
* "num.workers": 1
* }
* }
* }
**/
@Multiline
public static String adminConfigNoOverriddenApplications;
private final String adminConfigNoOverriddenApplications = """
{
"config_version": 1,
"client.id.prefix": "siembol.writer",
"group.id.prefix": "siembol.reader",
"zookeeper.attributes": {
"zk.url": "global_url",
"zk.path": "global_path",
"zk.base.sleep.ms": 1000,
"zk.max.retries": 3
},
"kafka.batch.writer.attributes": {
"batch.size": 50,
"producer.properties": {
"bootstrap.servers": "global_servers",
"security.protocol": "SASL_PLAINTEXT"
}
},
"storm.attributes": {
"bootstrap.servers": "dummy",
"first.pool.offset.strategy": "UNCOMMITTED_LATEST",
"kafka.spout.properties": {
"session.timeout.ms": 300000,
"security.protocol": "SASL_PLAINTEXT"
},
"storm.config": {
"num.workers": 1
}
}
}
""";
/**
*{
* "parsing_applications_version": 0,
* "parsing_applications": [
* {
* "parsing_app_name": "secret",
* "parsing_app_version": 1,
* "parsing_app_author": "siembol",
* "parsing_app_settings": {
* "parsing_app_type": "single_parser",
* "input_topics": [
* "test"
* ],
* "error_topic": "test",
* "input_parallelism": 4,
* "parsing_parallelism": 4,
* "output_parallelism": 4,
* "parse_metadata": false
* },
* "parsing_settings": {
* "single_parser": {
* "output_topic": "test",
* "parser_name": "test"
* }
* }
* },
* {
* "parsing_app_name": "public",
* "parsing_app_version": 1,
* "parsing_app_author": "siembol",
* "parsing_app_settings": {
* "parsing_app_type": "single_parser",
* "input_topics": [
* "test"
* ],
* "error_topic": "test",
* "input_parallelism": 4,
* "parsing_parallelism": 4,
* "output_parallelism": 4,
* "parse_metadata": false
* },
* "parsing_settings": {
* "single_parser": {
* "output_topic": "test",
* "parser_name": "test"
* }
* }
* }
* ]
* }
**/
@Multiline
public static String release;
private final String release = """
{
"parsing_applications_version": 0,
"parsing_applications": [
{
"parsing_app_name": "secret",
"parsing_app_version": 1,
"parsing_app_author": "siembol",
"parsing_app_settings": {
"parsing_app_type": "single_parser",
"input_topics": [
"test"
],
"error_topic": "test",
"input_parallelism": 4,
"parsing_parallelism": 4,
"output_parallelism": 4,
"parse_metadata": false
},
"parsing_settings": {
"single_parser": {
"output_topic": "test",
"parser_name": "test"
}
}
},
{
"parsing_app_name": "public",
"parsing_app_version": 1,
"parsing_app_author": "siembol",
"parsing_app_settings": {
"parsing_app_type": "single_parser",
"input_topics": [
"test"
],
"error_topic": "test",
"input_parallelism": 4,
"parsing_parallelism": 4,
"output_parallelism": 4,
"parse_metadata": false
},
"parsing_settings": {
"single_parser": {
"output_topic": "test",
"parser_name": "test"
}
}
}
]
}
""";
private ConfigServiceHelper serviceHelper;
private GetParsingAppStormTopologyAction getStormTopologyAction;
private ConfigEditorServiceContext context;
private String topologyImage = "dummyImage";
private String serviceName = "dummyService";
private final String topologyImage = "dummyImage";
private final String serviceName = "dummyService";
@Before
public void setUp() {
@@ -229,9 +223,9 @@ public class GetParsingAppStormTopologyActionTest {
Assert.assertEquals(1, adminConfigSecret.getKafkaBatchWriterAttributes().getBatchSize().intValue());
Assert.assertEquals(50, adminConfigPublic.getKafkaBatchWriterAttributes().getBatchSize().intValue());
Assert.assertEquals(Integer.valueOf(2),
Assert.assertEquals(2,
adminConfigSecret.getStormAttributes().getStormConfig().getRawMap().get("num.workers"));
Assert.assertEquals(Integer.valueOf(1),
Assert.assertEquals(1,
adminConfigPublic.getStormAttributes().getStormConfig().getRawMap().get("num.workers"));
}
@@ -279,9 +273,9 @@ public class GetParsingAppStormTopologyActionTest {
Assert.assertEquals(50, adminConfigSecret.getKafkaBatchWriterAttributes().getBatchSize().intValue());
Assert.assertEquals(50, adminConfigPublic.getKafkaBatchWriterAttributes().getBatchSize().intValue());
Assert.assertEquals(Integer.valueOf(1),
Assert.assertEquals(1,
adminConfigSecret.getStormAttributes().getStormConfig().getRawMap().get("num.workers"));
Assert.assertEquals(Integer.valueOf(1),
Assert.assertEquals(1,
adminConfigPublic.getStormAttributes().getStormConfig().getRawMap().get("num.workers"));
}

6
config-editor/config-editor-sync/src/test/java/uk/co/gresearch/siembol/configeditor/sync/actions/GetReleaseActionTest.java
View File

@@ -18,16 +18,18 @@ import static uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult.Stat
public class GetReleaseActionTest {
private ConfigServiceHelper serviceHelper;
private String release = "RELEASE";
private final String release = "RELEASE";
private GetReleaseAction getReleaseAction;
private ConfigEditorServiceContext context;
private int version = 1;
private final int version = 1;
@Before
public void setUp() {
context = new ConfigEditorServiceContext();
serviceHelper = Mockito.mock(ConfigServiceHelper.class);
when(serviceHelper.getConfigsRelease()).thenReturn(Optional.of(release));
when(serviceHelper.getReleaseVersion(eq(release))).thenReturn(version);
when(serviceHelper.validateConfigurations(eq(release))).thenReturn(true);
getReleaseAction = new GetReleaseAction(serviceHelper);

8
config-editor/config-editor-sync/src/test/java/uk/co/gresearch/siembol/configeditor/sync/actions/GetStormTopologyActionTest.java
View File

@@ -19,12 +19,12 @@ import static uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult.Stat
public class GetStormTopologyActionTest {
private ConfigServiceHelper serviceHelper;
private String adminConfig = "ADMIN_CONFIG";
private final String adminConfig = "ADMIN_CONFIG";
private GetStormTopologyAction getStormTopologyAction;
private ConfigEditorServiceContext context;
private String topologyName = "dummyTopologyName";
private String topologyImage = "dummyImage";
private String serviceName = "dummyService";
private final String topologyName = "dummyTopologyName";
private final String topologyImage = "dummyImage";
private final String serviceName = "dummyService";
@Before
public void setUp() {

4
config-editor/config-editor-sync/src/test/java/uk/co/gresearch/siembol/configeditor/sync/actions/UpdateRulesInZookeeperActionTest.java
View File

@@ -19,11 +19,11 @@ import static uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult.Stat
public class UpdateRulesInZookeeperActionTest {
private ConfigServiceHelper serviceHelper;
private String release = "RELEASE";
private final String release = "RELEASE";
private UpdateReleaseInZookeeperAction updateReleaseInZookeeperAction;
private ZooKeeperConnector zooKeeperConnector;
private ConfigEditorServiceContext context;
private String currentRelease = "ZK_RELEASE";
private final String currentRelease = "ZK_RELEASE";
@Before
public void setUp() throws Exception {

75
config-editor/config-editor-sync/src/test/java/uk/co/gresearch/siembol/configeditor/sync/service/EnrichmentTablesProviderTest.java
View File

@@ -1,67 +1,57 @@
package uk.co.gresearch.siembol.configeditor.sync.service;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import uk.co.gresearch.siembol.common.model.EnrichmentTableDto;
import uk.co.gresearch.siembol.common.model.StormTopologyDto;
import uk.co.gresearch.siembol.common.zookeeper.ZooKeeperConnector;
import uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.mockito.Mockito.when;
public class EnrichmentTablesProviderTest {
/**
*{
* "enrichment_tables": [
* {
* "name": "test_table",
* "path": "/siembol/tables/enrichment/test.json"
* },
* {
* "name": "dns_table",
* "path": "/siembol/tables/enrichment/dns.json"
* }
* ]
* }
**/
@Multiline
public static String tablesUpdateServiceA;
private final String tablesUpdateServiceA = """
{
"enrichment_tables": [
{
"name": "test_table",
"path": "/siembol/tables/enrichment/test.json"
},
{
"name": "dns_table",
"path": "/siembol/tables/enrichment/dns.json"
}
]
}
""";
/**
*{
* "enrichment_tables": [
* {
* "name": "users_table",
* "path": "/siembol/tables/enrichment/users.json"
* },
* {
* "name": "url_table",
* "path": "/siembol/tables/enrichment/url.json"
* },
* {
* "name": "ioc_table",
* "path": "/siembol/tables/enrichment/ioc.json"
* }
* ]
* }
**/
@Multiline
public static String tablesUpdateServiceB;
private final String tablesUpdateServiceB = """
{
"enrichment_tables": [
{
"name": "users_table",
"path": "/siembol/tables/enrichment/users.json"
},
{
"name": "url_table",
"path": "/siembol/tables/enrichment/url.json"
},
{
"name": "ioc_table",
"path": "/siembol/tables/enrichment/ioc.json"
}
]
}
""";
private Map<String, ZooKeeperConnector> zooKeeperConnectorMap;
private EnrichmentTablesProviderImpl enrichmentTablesProvider;
private Set<String> services;
private List<StormTopologyDto> topologiesToUpdate;
private final String serviceA = "a";
private final String serviceB = "b";
@@ -184,5 +174,4 @@ public class EnrichmentTablesProviderTest {
Assert.assertEquals(ConfigEditorResult.StatusCode.BAD_REQUEST, result.getStatusCode());
Assert.assertNotNull(result.getAttributes().getMessage());
}
}

183
config-editor/config-editor-sync/src/test/java/uk/co/gresearch/siembol/configeditor/sync/service/StormApplicationProviderTest.java
View File

@@ -2,7 +2,7 @@ package uk.co.gresearch.siembol.configeditor.sync.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -23,93 +23,88 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.*;
public class StormApplicationProviderTest {
private static ObjectReader TOPOLOGIES_READER = new ObjectMapper()
private static final ObjectReader TOPOLOGIES_READER = new ObjectMapper()
.readerFor(StormTopologiesDto.class);
/**
* {
* "timestamp": 1,
* "topologies": [
* {
* "topology_name": "a",
* "topology_id": "1",
* "image": "secret",
* "service_name": "alert",
* "attributes": [
* "a1",
* "a2"
* ]
* },
* {
* "topology_name": "b",
* "topology_id": "2",
* "image": "secret",
* "service_name": "parsing",
* "attributes": [
* "a1",
* "a2",
* "a3"
* ]
* },
* {
* "topology_name": "c",
* "topology_id": "3",
* "image": "secret",
* "service_name": "parsing",
* "attributes": [
* "a1",
* "a2",
* "a3"
* ]
* }
* ]
* }
**/
@Multiline
public static String initTopologies;
/**
* {
* "timestamp": 1,
* "topologies": [
* {
* "topology_name": "a",
* "topology_id": "1",
* "image": "secret",
* "service_name": "alert",
* "attributes": [
* "a1",
* "a3"
* ]
* },
* {
* "topology_name": "b",
* "topology_id": "2",
* "image": "secret",
* "service_name": "parsing",
* "attributes": [
* "a1",
* "a2",
* "a3"
* ]
* },
* {
* "topology_name": "c",
* "topology_id": "3",
* "image": "secret",
* "service_name": "parsing",
* "attributes": [
* "a1",
* "a2",
* "a4"
* ]
* }
* ]
* }
**/
@Multiline
public static String updatedTopologies;
private final String initTopologies = """
{
"timestamp": 1,
"topologies": [
{
"topology_name": "a",
"topology_id": "1",
"image": "secret",
"service_name": "alert",
"attributes": [
"a1",
"a2"
]
},
{
"topology_name": "b",
"topology_id": "2",
"image": "secret",
"service_name": "parsing",
"attributes": [
"a1",
"a2",
"a3"
]
},
{
"topology_name": "c",
"topology_id": "3",
"image": "secret",
"service_name": "parsing",
"attributes": [
"a1",
"a2",
"a3"
]
}
]
}
""";
private final String updatedTopologies = """
{
"timestamp": 1,
"topologies": [
{
"topology_name": "a",
"topology_id": "1",
"image": "secret",
"service_name": "alert",
"attributes": [
"a1",
"a3"
]
},
{
"topology_name": "b",
"topology_id": "2",
"image": "secret",
"service_name": "parsing",
"attributes": [
"a1",
"a2",
"a3"
]
},
{
"topology_name": "c",
"topology_id": "3",
"image": "secret",
"service_name": "parsing",
"attributes": [
"a1",
"a2",
"a4"
]
}
]
}
""";
private ZooKeeperConnector zooKeeperConnector;
private StormApplicationProviderImpl stormApplicationProvider;
@@ -164,13 +159,13 @@ public class StormApplicationProviderTest {
Assert.assertEquals(ConfigEditorResult.StatusCode.OK, result.getStatusCode());
Assert.assertNotNull(result.getAttributes().getTopologies());
Assert.assertEquals(3, result.getAttributes().getTopologies().size());
StormTopologyDto restarted = result.getAttributes().getTopologies().stream()
Optional<StormTopologyDto> restarted = result.getAttributes().getTopologies().stream()
.filter(x -> x.getTopologyName().equals("a"))
.findFirst().orElseGet(null);
.findFirst();
Assert.assertNotNull(restarted);
Assert.assertNotNull(restarted.getTopologyId());
Assert.assertNotEquals("1", restarted.getTopologyId());
Assert.assertTrue(restarted.isPresent());
Assert.assertNotNull(restarted.get().getTopologyId());
Assert.assertNotEquals("1", restarted.get().getTopologyId());
}
@Test
@@ -203,7 +198,7 @@ public class StormApplicationProviderTest {
@Test
public void removeTopologiesFromOneService() throws Exception {
services.addAll(Arrays.asList("parsing"));
services.addAll(List.of("parsing"));
ConfigEditorResult result = stormApplicationProvider.updateStormTopologies(new ArrayList<>(), services);
Assert.assertEquals(ConfigEditorResult.StatusCode.OK, result.getStatusCode());
Assert.assertNotNull(result.getAttributes().getTopologies());
@@ -236,20 +231,20 @@ public class StormApplicationProviderTest {
}
@Test(expected = IllegalStateException.class)
public void wrongInitTopologies() throws Exception {
public void wrongInitTopologies() {
when(zooKeeperConnector.getData()).thenReturn("INVALID");
doNothing().when(zooKeeperConnector).addCacheListener(any());
stormApplicationProvider = new StormApplicationProviderImpl(zooKeeperConnector);
}
@Test
public void testHealth() throws Exception {
public void testHealth() {
Health health = stormApplicationProvider.checkHealth();
Assert.assertEquals(Status.UP, health.getStatus());
}
@Test
public void updateDuplicatesError() throws Exception {
public void updateDuplicatesError() {
topologiesToUpdate.get(0).setTopologyName("b");
services.addAll(Arrays.asList("alert", "parsing"));
ConfigEditorResult result = stormApplicationProvider.updateStormTopologies(topologiesToUpdate, services);

2
config-editor/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</parent>
<modules>
<module>config-editor-core</module>

27
config/config-editor-rest/application.properties
View File

@@ -30,6 +30,10 @@ config-editor.test-cases-ui-config-file-name=ui-config/test-cases-layout-config.
config-editor.synchronisation=ALL
config-editor.storm-topologies-zookeeper.zk-path=/siembol/synchronise
config-editor.storm-topologies-zookeeper.zk-url=siembol-zookeeper:2181
config-editor.storm-topologies-zookeeper.init-value-if-not-exists={}
config-editor.enrichment-tables-zookeeper.enrichment.zk-path=/siembol/enrichment_tables
config-editor.enrichment-tables-zookeeper.enrichment.zk-url=siembol-zookeeper:2181
config-editor.enrichment-tables-zookeeper.enrichment.init-value-if-not-exists={}
config-editor.services.alert.type=alert
config-editor.services.alert.config-store.git-user-name=${GITHUB_USER}
@@ -47,6 +51,7 @@ config-editor.services.alert.ui-config-file-name=ui-config/alert-layout-config.j
config-editor.services.alert.synchronisation=ALL
config-editor.services.alert.release-zookeeper.zk-path=/siembol/alerts
config-editor.services.alert.release-zookeeper.zk-url=siembol-zookeeper:2181
config-editor.services.alert.release-zookeeper.init-value-if-not-exists={}
config-editor.services.alert.topology-image=gresearchdev/siembol-alerting-storm:latest
config-editor.services.correlation-alert.type=correlationalert
@@ -65,6 +70,7 @@ config-editor.services.correlation-alert.ui-config-file-name=ui-config/correlati
config-editor.services.correlation-alert.synchronisation=ALL
config-editor.services.correlation-alert.release-zookeeper.zk-path=/siembol/correlation_alerts
config-editor.services.correlation-alert.release-zookeeper.zk-url=siembol-zookeeper:2181
config-editor.services.correlation-alert.release-zookeeper.init-value-if-not-exists={}
config-editor.services.correlation-alert.topology-image=gresearchdev/siembol-alerting-storm:latest
config-editor.services.parser-config.type=parserconfig
@@ -82,6 +88,7 @@ config-editor.services.parser-config.ui-config-file-name=ui-config/parser-config
config-editor.services.parser-config.synchronisation=RELEASE
config-editor.services.parser-config.release-zookeeper.zk-path=/siembol/parser_configs
config-editor.services.parser-config.release-zookeeper.zk-url=siembol-zookeeper:2181
config-editor.services.parser-config.release-zookeeper.init-value-if-not-exists={}
config-editor.services.parsing-application.type=parsingapp
config-editor.services.parsing-application.config-store.git-user-name=${GITHUB_USER}
@@ -100,3 +107,23 @@ config-editor.services.parsing-application.ui-config-file-name=ui-config/parsing
config-editor.services.parsing-application.synchronisation=ALL
config-editor.services.parsing-application.topology-image=gresearchdev/siembol-parsing-storm:latest
config-editor.services.enrichment.type=enrichment
config-editor.services.enrichment.config-store.git-user-name=${GITHUB_USER}
config-editor.services.enrichment.config-store.git-password=${GITHUB_TOKEN}
config-editor.services.enrichment.config-store.github-url=${GITHUB_URL}
config-editor.services.enrichment.config-store.store-repository-name=${GITHUB_REPO_NAME}
config-editor.services.enrichment.config-store.release-repository-name=${GITHUB_REPO_NAME}
config-editor.services.enrichment.config-store.admin-config-repository-name=${GITHUB_REPO_NAME}
config-editor.services.enrichment.config-store.store-repository-path=/tmp/siembol-config
config-editor.services.enrichment.config-store.store-directory=enrichment/rules
config-editor.services.enrichment.config-store.release-directory=enrichment/release
config-editor.services.enrichment.config-store.test-case-directory=enrichment/testcases
config-editor.services.enrichment.config-store.admin-config-directory=enrichment/adminconfig
config-editor.services.enrichment.ui-config-file-name=ui-config/enrichment-layout-config.json
config-editor.services.enrichment.synchronisation=ALL
config-editor.services.enrichment.release-zookeeper.zk-path=/siembol/enrichment_rules
config-editor.services.enrichment.release-zookeeper.zk-url=siembol-zookeeper:2181
config-editor.services.enrichment.release-zookeeper.init-value-if-not-exists={}
config-editor.services.enrichment.topology-image=gresearchdev/siembol-enriching-storm:latest

1
config/storm-topology-manager/application.properties
View File

@@ -31,6 +31,7 @@ topology-manager.desired-state.zk-url=siembol-zookeeper:2181
topology-manager.saved-state.zk-path=/siembol/cache
topology-manager.saved-state.zk-url=siembol-zookeeper:2181
topology-manager.saved-state.init-value-if-not-exists={}
topology-manager.k8s.namespace=siembol
topology-manager.k8s.storm-submit-job-template-file=storm-submit.yaml

6
config/storm-topology-manager/storm-submit.yaml
View File

@@ -4,6 +4,7 @@ metadata:
name: --name--
namespace: --namespace--
spec:
ttlSecondsAfterFinished: 5
template:
spec:
restartPolicy: Never
@@ -14,5 +15,10 @@ spec:
args: --args--
image: --image--
name: --name--
resources:
requests:
memory: "64Mi"
limits:
memory: "128Mi"
securityContext:
runAsUser: 101

2
deployment/docker/Dockerfile.java
View File

@@ -1,4 +1,4 @@
FROM openjdk:8-jre-slim
FROM openjdk:11-jre-slim
ARG APP
ENV APP=$APP

2
deployment/docker/Dockerfile.storm
View File

@@ -1,4 +1,4 @@
FROM storm:1.2.3
FROM storm:2.2.0
ARG JAR
ARG CLASS

2
deployment/docker/storm-entrypoint.sh
View File

@@ -2,7 +2,7 @@
# if command starts with something that is not executable, prepend our deploy command
if ! which "${1}" >/dev/null; then
set -- storm -c nimbus.seeds="${NIMBUS_SEEDS:-"[\"nimbus\"]"}" -c nimbus.thrift.port=${NIMBUS_PORT:-6627} jar $TOPOLOGY_JAR $TOPOLOGY_CLASS "$@"
set -- storm jar $TOPOLOGY_JAR $TOPOLOGY_CLASS "$@" -c nimbus.seeds="${NIMBUS_SEEDS:-"[\"nimbus\"]"}" -c nimbus.thrift.port=${NIMBUS_PORT:-6627}
fi
exec "$@"

10
deployment/storm-topology-manager/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<dependencyManagement>
@@ -43,7 +43,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -159,12 +159,6 @@
<version>${junit_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-launcher</artifactId>

69
deployment/storm-topology-manager/src/test/java/uk/co/gresearch/siembol/deployment/storm/providers/KubernetesProviderImplTest.java
View File

@@ -3,7 +3,6 @@ package uk.co.gresearch.siembol.deployment.storm.providers;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import io.fabric8.kubernetes.client.server.mock.KubernetesServer;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Rule;
@@ -18,43 +17,39 @@ public class KubernetesProviderImplTest {
private static final ObjectReader READER = new ObjectMapper()
.readerFor(StormTopologyDto.class);
/**
* {
* "topology_name": "parsing-heartbeat",
* "topology_id": "id1234",
* "image": "gr/siembol-parsing-storm:1.72-SNAPSHOT",
* "service_name": "parsing",
* "attributes": [
* "testattributes1",
* "testattributes2"
* ]
* }
**/
@Multiline
private static String topologyConfig;
private final String topologyConfig = """
{
"topology_name": "parsing-heartbeat",
"topology_id": "id1234",
"image": "gr/siembol-parsing-storm:1.72-SNAPSHOT",
"service_name": "parsing",
"attributes": [
"testattributes1",
"testattributes2"
]
}
""";
/**
*apiVersion: batch/v1
*kind: Job
*metadata:
* name: parsing-heartbeat
* namespace: siembol
*spec:
* template:
* spec:
* restartPolicy: Never
* containers:
* - env:
* - name: NIMBUS_SEEDS
* value: '["nimbus"]'
* args: ["testattributes1", "testattributes2"]
* image: gr/siembol-parsing-storm:1.72-SNAPSHOT
* name: parsing-heartbeat
* securityContext:
* runAsUser: 1000
*/
@Multiline
private static String expectedYaml;
private final String expectedYaml = """
apiVersion: batch/v1
kind: Job
metadata:
name: parsing-heartbeat
namespace: siembol
spec:
template:
spec:
restartPolicy: Never
containers:
- env:
- name: NIMBUS_SEEDS
value: '["nimbus"]'
args: ["testattributes1", "testattributes2"]
image: gr/siembol-parsing-storm:1.72-SNAPSHOT
name: parsing-heartbeat
securityContext:
runAsUser: 1000
""";
KubernetesProvider provider;

9
deployment/storm-topology-manager/src/test/java/uk/co/gresearch/siembol/deployment/storm/providers/StormProviderImplTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.deployment.storm.providers;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -17,11 +16,9 @@ import static org.mockito.Mockito.when;
public class StormProviderImplTest {
/**
* {"topologies":[{"id":"id1","name":"topology1"}]}
*/
@Multiline
private static String runningTopologies;
private final String runningTopologies = """
{"topologies":[{"id":"id1","name":"topology1"}]}
""";
HttpProvider httpProvider = mock(HttpProvider.class);
StormProvider stormProvider;

140
deployment/storm-topology-manager/src/test/java/uk/co/gresearch/siembol/deployment/storm/service/TopologyManagerServiceImplTests.java
View File

@@ -5,7 +5,6 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import com.fasterxml.jackson.databind.ObjectWriter;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.junit.Assert;
import org.junit.Before;
@@ -34,100 +33,73 @@ public class TopologyManagerServiceImplTests {
.setSerializationInclusion(JsonInclude.Include.NON_NULL)
.writerFor(StormTopologiesDto.class);
/**
* {"topologies":[{"id":"id1","name":"topology1"}]}
*/
@Multiline
private static String runningTopologies;
private final String runningTopologies = """
{"topologies":[{"id":"id1","name":"topology1"}]}
""";
/**
* {"topologies":[{"id":"id1","name":"topology1"},{"id":"id2","name":"topology2"},{"id":"id3","name":"topology3"},{"id":"id4","name":"topology4"}]}
*/
@Multiline
private static String runningTopologies2;
/**
* {"topologies": []}
*
**/
@Multiline
private static String empty;
private final String runningTopologies2 = """
{"topologies":[{"id":"id1","name":"topology1"},{"id":"id2","name":"topology2"},{"id":"id3","name":"topology3"},{"id":"id4","name":"topology4"}]}
""";
/**
* {"topologies": [
* { "topology_id": "t1", "topology_name": "topology1", "image": "", "attributes": ["", ""] }
* ]}
*
**/
@Multiline
private static String topologies1;
private final String empty = """
{"topologies": []}
""";
/**
* {"topologies": [
* { "topology_id": "t2", "topology_name": "topology1", "image": "", "attributes": ["", ""] }
* ]}
*
**/
@Multiline
private static String topologies1changed;
private final String topologies1 = """
{"topologies": [
{ "topology_id": "t1", "topology_name": "topology1", "image": "", "attributes": ["", ""] }
]}
""";
/**
* {"topologies": [
* { "topology_id": "t1", "topology_name": "topology1", "image": "", "attributes": ["", ""] },
* { "topology_id": "t2", "topology_name": "topology2", "image": "", "attributes": ["", ""] }
* ]}
*
**/
@Multiline
private static String topologies2;
/**
* {"topologies": [
* { "topology_id": "t3", "topology_name": "topology1", "image": "", "attributes": ["", ""] },
* { "topology_id": "t4", "topology_name": "topology2", "image": "", "attributes": ["", ""] }
* ]}
*
**/
@Multiline
private static String topologies2changed;
private final String topologies1changed = """
{"topologies": [
{ "topology_id": "t2", "topology_name": "topology1", "image": "", "attributes": ["", ""] }
]}
""";
/**
* {"topologies": [
* { "topology_id": "t1", "topology_name": "topology1", "image": "", "attributes": ["", ""] },
* { "topology_id": "t2", "topology_name": "topology2", "image": "", "attributes": ["", ""] },
* { "topology_id": "t3", "topology_name": "topology3", "image": "", "attributes": ["", ""] }
* ]}
*
**/
@Multiline
private static String topologies3;
private final String topologies2 = """
{"topologies": [
{ "topology_id": "t1", "topology_name": "topology1", "image": "", "attributes": ["", ""] },
{ "topology_id": "t2", "topology_name": "topology2", "image": "", "attributes": ["", ""] }
]}
""";
/**
* {"topologies": [
* { "topology_id": "t1", "topology_name": "topology1", "image": "", "attributes": ["", ""] },
* { "topology_id": "t2", "topology_name": "topology2", "image": "", "attributes": ["", ""] },
* { "topology_id": "t3", "topology_name": "topology3", "image": "", "attributes": ["", ""] },
* { "topology_id": "t4", "topology_name": "topology4", "image": "", "attributes": ["", ""] }
* ]}
*
**/
@Multiline
private static String topologies4;
private final String topologies2changed = """
{"topologies": [
{ "topology_id": "t3", "topology_name": "topology1", "image": "", "attributes": ["", ""] },
{ "topology_id": "t4", "topology_name": "topology2", "image": "", "attributes": ["", ""] }
]}
""";
/**
* {"topologies": [
* { "topology_id": "t8", "topology_name": "topology1", "image": "", "attributes": ["", ""] },
* { "topology_id": "t2", "topology_name": "topology2", "image": "", "attributes": ["", ""] },
* { "topology_id": "t3", "topology_name": "topology3", "image": "", "attributes": ["", ""] },
* { "topology_id": "t4", "topology_name": "topology4", "image": "", "attributes": ["", ""] }
* ]}
*
**/
@Multiline
private static String topologies4changed;
private final String topologies3 = """
{"topologies": [
{ "topology_id": "t1", "topology_name": "topology1", "image": "", "attributes": ["", ""] },
{ "topology_id": "t2", "topology_name": "topology2", "image": "", "attributes": ["", ""] },
{ "topology_id": "t3", "topology_name": "topology3", "image": "", "attributes": ["", ""] }
]}
""";
private final String topologies4 = """
{"topologies": [
{ "topology_id": "t1", "topology_name": "topology1", "image": "", "attributes": ["", ""] },
{ "topology_id": "t2", "topology_name": "topology2", "image": "", "attributes": ["", ""] },
{ "topology_id": "t3", "topology_name": "topology3", "image": "", "attributes": ["", ""] },
{ "topology_id": "t4", "topology_name": "topology4", "image": "", "attributes": ["", ""] }
]}
""";
private final String topologies4changed = """
{"topologies": [
{ "topology_id": "t8", "topology_name": "topology1", "image": "", "attributes": ["", ""] },
{ "topology_id": "t2", "topology_name": "topology2", "image": "", "attributes": ["", ""] },
{ "topology_id": "t3", "topology_name": "topology3", "image": "", "attributes": ["", ""] },
{ "topology_id": "t4", "topology_name": "topology4", "image": "", "attributes": ["", ""] }
]}
""";
StormResponseDto stormTopologies = READER_STORM.readValue(runningTopologies);
TopologyManagerService service;
KubernetesProvider kubernetesProvider = mock(KubernetesProvider.class);
StormProvider stormProvider = mock(StormProvider.class);

2
docs/services/how-tos/how_to_set_up_enrichment_table.md
View File

@@ -11,7 +11,7 @@ An enrichment table is defined in a JSON file, it only supports mappings of `str
After updating an enrichment table it is necessary to inform the enrichment topology of the changes, this is done using Zookeeper. The Zookeeper node is configured in the admin config of enrichment ([see here](../siembol_enrichment_service.md)). This message is stored in a Zookeeper node in JSON format. In it the tables are in a list in the "hdfs_tables" key and each table has a name and the path to the latest table. Here is an example with two tables:
{
"hdfs_tables":[
"enrichment_tables":[
{
"name":"employees",
"path":"/siembol-enrichment/employees/1.json"

12
enriching/enriching-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -35,18 +35,12 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>1.4.2-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>

414
enriching/enriching-core/src/test/java/uk/co/gresearch/siembol/enrichments/compiler/EnrichmentCompilerTest.java
View File

@@ -1,10 +1,9 @@
package uk.co.gresearch.siembol.enrichments.compiler;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -20,226 +19,209 @@ import static uk.co.gresearch.siembol.enrichments.compiler.EnrichmentCompilerImp
public class EnrichmentCompilerTest {
private static final ObjectReader JSON_MAP_READER = new ObjectMapper()
.readerFor(new TypeReference<Map<String, Object>>() { });
.readerFor(new TypeReference<Map<String, Object>>() {
});
/**
*{
* "rules_version": 1,
* "rules": [
* {
* "rule_name": "siembol_enrichments_test",
* "rule_version": 1,
* "rule_author": "dummy",
* "rule_description": "Test rule",
* "source_type": "*",
* "matchers": [
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "is_alert",
* "data": "(?i)true"
* }
* ],
* "table_mapping": {
* "table_name": "test_table",
* "joining_key": "${ip_src_addr}",
* "tags": [
* {
* "tag_name": "is_test_tag",
* "tag_value": "true"
* }
* ],
* "enriching_fields": [
* {
* "table_field_name": "dns_name",
* "event_field_name": "siembol:enrichments:dns"
* }
* ]
* }
* }
* ]
* }
*
**/
@Multiline
public static String testRules;
private final String testRules = """
{
"rules_version": 1,
"rules": [
{
"rule_name": "siembol_enrichments_test",
"rule_version": 1,
"rule_author": "dummy",
"rule_description": "Test rule",
"source_type": "*",
"matchers": [
{
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_alert",
"data": "(?i)true"
}
],
"table_mapping": {
"table_name": "test_table",
"joining_key": "${ip_src_addr}",
"tags": [
{
"tag_name": "is_test_tag",
"tag_value": "true"
}
],
"enriching_fields": [
{
"table_field_name": "dns_name",
"event_field_name": "siembol:enrichments:dns"
}
]
}
}
]
}
""";
/**
*{
* "rules_version": 1,
* "rules": [
* {
* "rule_name": "siembol_enrichments_test",
* "rule_version": 1,
* "rule_author": "dummy",
* "rule_description": "Test rule",
* "source_type": "*",
* "matchers": [
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "is_alert",
* "data": "(?i)true"
* }
* ],
* "table_mapping": {
* "table_name": "test_table",
* "joining_key": "${ip_src_addr}",
* "tags": [
* {
* "tag_name": "is_test_tag",
* "tag_value": "true"
* }
* ]
* }
* }
* ]
* }
*
**/
@Multiline
public static String testRulesTagsOnly;
private final String testRulesTagsOnly = """
{
"rules_version": 1,
"rules": [
{
"rule_name": "siembol_enrichments_test",
"rule_version": 1,
"rule_author": "dummy",
"rule_description": "Test rule",
"source_type": "*",
"matchers": [
{
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_alert",
"data": "(?i)true"
}
],
"table_mapping": {
"table_name": "test_table",
"joining_key": "${ip_src_addr}",
"tags": [
{
"tag_name": "is_test_tag",
"tag_value": "true"
}
]
}
}
]
}
""";
/**
*{
* "rules_version": 1,
* "rules": [
* {
* "rule_name": "siembol_enrichments_test",
* "rule_version": 1,
* "rule_author": "dummy",
* "rule_description": "Test rule",
* "source_type": "*",
* "matchers": [
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "is_alert",
* "data": "(?i)true"
* }
* ],
* "table_mapping": {
* "table_name": "test_table",
* "joining_key": "${ip_src_addr}",
* "enriching_fields": [
* {
* "table_field_name": "dns_name",
* "event_field_name": "siembol:enrichments:dns"
* }
* ]
* }
* }
* ]
* }
*
**/
@Multiline
public static String testRulesEnrichingFieldsOnly;
/**
*{
* "rules_version": 1,
* "rules": [
* {
* "rule_name": "siembol_enrichments_test",
* "rule_version": 1,
* "rule_author": "dummy",
* "rule_description": "Test rule",
* "source_type": "*",
* "matchers": [
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "is_alert",
* "data": "(?i)true"
* }
* ],
* "table_mapping": {
* "table_name": "test_table",
* "joining_key": "${ip_src_addr}"
* }
* }
* ]
* }
*
**/
@Multiline
public static String testRulesMissingTagsAndEnrichingFields;
private final String testRulesEnrichingFieldsOnly = """
{
"rules_version": 1,
"rules": [
{
"rule_name": "siembol_enrichments_test",
"rule_version": 1,
"rule_author": "dummy",
"rule_description": "Test rule",
"source_type": "*",
"matchers": [
{
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_alert",
"data": "(?i)true"
}
],
"table_mapping": {
"table_name": "test_table",
"joining_key": "${ip_src_addr}",
"enriching_fields": [
{
"table_field_name": "dns_name",
"event_field_name": "siembol:enrichments:dns"
}
]
}
}
]
}
""";
/**
*{
* "rule_name": "siembol_enrichments_test",
* "rule_version": 1,
* "rule_author": "dummy",
* "rule_description": "Test rule",
* "source_type": "*",
* "matchers": [
* {
* "matcher_type": "IS_IN_SET",
* "is_negated": false,
* "field": "is_alert",
* "data": "true"
* }
* ],
* "table_mapping": {
* "table_name": "test_table",
* "joining_key": "${ip_src_addr}",
* "tags": [
* {
* "tag_name": "is_test_tag",
* "tag_value": "true"
* }
* ],
* "enriching_fields": [
* {
* "table_field_name": "dns_name",
* "event_field_name": "siembol:enrichments:dns"
* }
* ]
* }
* }
*
*
**/
@Multiline
public static String testRule;
/**
* {
* "event": {
* "source_type": "secret",
* "is_alert": "true",
* "ip_src_addr": "1.2.3.4"
* },
* "testing_table_name": "test_table",
* "testing_table_mapping": {
* "1.2.3.4": {
* "dns_name": "secret.abc"
* }
* }
*
* }
**/
@Multiline
public static String testSpecification;
private final String testRulesMissingTagsAndEnrichingFields = """
{
"rules_version": 1,
"rules": [
{
"rule_name": "siembol_enrichments_test",
"rule_version": 1,
"rule_author": "dummy",
"rule_description": "Test rule",
"source_type": "*",
"matchers": [
{
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_alert",
"data": "(?i)true"
}
],
"table_mapping": {
"table_name": "test_table",
"joining_key": "${ip_src_addr}"
}
}
]
}
/**
* {
* "event": {
* "source_type": "secret",
* "is_alert": "false",
* "ip_src_addr": "1.2.3.4"
* },
* "testing_table_name": "test_table",
* "testing_table_mapping": {
* "1.2.3.4": {
* "dns_name": "secret.abc"
* }
* }
* }
**/
@Multiline
public static String testSpecificationNoMatch;
""";
private final String testRule = """
{
"rule_name": "siembol_enrichments_test",
"rule_version": 1,
"rule_author": "dummy",
"rule_description": "Test rule",
"source_type": "*",
"matchers": [
{
"matcher_type": "IS_IN_SET",
"is_negated": false,
"field": "is_alert",
"data": "true"
}
],
"table_mapping": {
"table_name": "test_table",
"joining_key": "${ip_src_addr}",
"tags": [
{
"tag_name": "is_test_tag",
"tag_value": "true"
}
],
"enriching_fields": [
{
"table_field_name": "dns_name",
"event_field_name": "siembol:enrichments:dns"
}
]
}
}
""";
private final String testSpecification = """
{
"event": {
"source_type": "secret",
"is_alert": "true",
"ip_src_addr": "1.2.3.4"
},
"testing_table_name": "test_table",
"testing_table_mapping": {
"1.2.3.4": {
"dns_name": "secret.abc"
}
}
}
""";
private final String testSpecificationNoMatch = """
{
"event": {
"source_type": "secret",
"is_alert": "false",
"ip_src_addr": "1.2.3.4"
},
"testing_table_name": "test_table",
"testing_table_mapping": {
"1.2.3.4": {
"dns_name": "secret.abc"
}
}
}
""";
private EnrichmentCompiler enrichmentCompiler;
@@ -380,7 +362,7 @@ public class EnrichmentCompilerTest {
}
@Test
public void testRulesMissingTagsAndEnrichingFields() throws IOException {
public void testRulesMissingTagsAndEnrichingFields() {
EnrichmentResult result = enrichmentCompiler.testConfigurations(testRulesMissingTagsAndEnrichingFields,
testSpecification);
Assert.assertEquals(ERROR, result.getStatusCode());

20
enriching/enriching-core/src/test/java/uk/co/gresearch/siembol/enrichments/evaluation/AlertingEnrichmentEvaluatorTest.java
View File

@@ -1,7 +1,5 @@
package uk.co.gresearch.siembol.enrichments.evaluation;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang3.tuple.Pair;
import org.junit.Assert;
import org.junit.Before;
@@ -23,16 +21,14 @@ import static uk.co.gresearch.siembol.enrichments.common.EnrichmentResult.Status
import static uk.co.gresearch.siembol.enrichments.common.EnrichmentResult.StatusCode.OK;
public class AlertingEnrichmentEvaluatorTest {
/**
* {
* "timestamp" : 1,
* "dummy_bool" : true,
* "dummy_str" : "test",
* "a" : "conflict"
* }
**/
@Multiline
public static String simpleEvent;
private final String simpleEvent = """
{
"timestamp" : 1,
"dummy_bool" : true,
"dummy_str" : "test",
"a" : "conflict"
}
""";
private AlertingAttributes alertingAttributes;
private AlertingResult alertingResult;

8
enriching/enriching-core/src/test/java/uk/co/gresearch/siembol/enrichments/evaluation/EnrichingRuleTest.java
View File

@@ -21,9 +21,9 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
public class EnrichingRuleTest {
private String name = "test_rule";
private Integer version = 1;
private String tableName = "test_table";
private final String name = "test_rule";
private final Integer version = 1;
private final String tableName = "test_table";
private String key = "test_key";
private Map<String, Object> event;
@@ -40,7 +40,7 @@ public class EnrichingRuleTest {
enrichmentTags.add(Pair.of("is_test", "true"));
matcher = Mockito.mock(BasicMatcher.class);
event = new HashMap<>();
when(matcher.match(ArgumentMatchers.<Map<String, Object>>any())).thenReturn(EvaluationResult.MATCH);
when(matcher.match(ArgumentMatchers.any())).thenReturn(EvaluationResult.MATCH);
}
@Test

20
enriching/enriching-core/src/test/java/uk/co/gresearch/siembol/enrichments/evaluation/EnrichmentEvaluatorLibraryTest.java
View File

@@ -3,7 +3,7 @@ package uk.co.gresearch.siembol.enrichments.evaluation;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang3.tuple.Pair;
import org.junit.Assert;
import org.junit.Before;
@@ -20,16 +20,14 @@ import static org.mockito.Mockito.when;
public class EnrichmentEvaluatorLibraryTest {
private static final ObjectReader JSON_MAP_READER =
new ObjectMapper().readerFor(new TypeReference<Map<String, Object>>() {});
/**
* {
* "timestamp" : 1,
* "dummy_bool" : true,
* "dummy_str" : "test",
* "a" : "conflict"
* }
**/
@Multiline
public static String simpleEvent;
private final String simpleEvent = """
{
"timestamp" : 1,
"dummy_bool" : true,
"dummy_str" : "test",
"a" : "conflict"
}
""";
private List<Pair<String, String>> enrichments;
private List<EnrichmentCommand> commands;

79
enriching/enriching-core/src/test/java/uk/co/gresearch/siembol/enrichments/table/EnrichmentsMemoryTableTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.enrichments.table;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang3.tuple.Pair;
import org.junit.Assert;
import org.junit.Test;
@@ -13,53 +12,42 @@ import java.util.*;
public class EnrichmentsMemoryTableTest {
/**
*
* {
* "1.2.3.1" : {},
* "1.2.3.2" : {},
* "1.2.3.3" : {},
* "1.2.3.4" : {},
* "1.2.3.5" : {}
* }
**/
@Multiline
public static String simpleEmptyFields;
private final String simpleEmptyFields = """
{
"1.2.3.1" : {},
"1.2.3.2" : {},
"1.2.3.3" : {},
"1.2.3.4" : {},
"1.2.3.5" : {}
}
""";
/**
*
* {
* "1.2.3.1" : {"is_ioc" : 1}
* }
**/
@Multiline
public static String unsupportedFieldType;
private final String unsupportedFieldType = """
{
"1.2.3.1" : {"is_ioc" : 1}
}
""";
/**
*
* {
* "1.2.3.1" : { "is_malicious" : "true" },
* "1.2.3.2" : { "is_malicious" : "true"},
* "1.2.3.3" : {"is_malicious" : "false"},
* "1.2.3.4" : {"is_malicious" : "true"},
* "1.2.3.5" : {"is_malicious" : "true"}
* }
**/
@Multiline
public static String simpleOneField;
private final String simpleOneField = """
{
"1.2.3.1" : { "is_malicious" : "true" },
"1.2.3.2" : { "is_malicious" : "true"},
"1.2.3.3" : {"is_malicious" : "false"},
"1.2.3.4" : {"is_malicious" : "true"},
"1.2.3.5" : {"is_malicious" : "true"}
}
""";
private final String simpleMixedFields = """
{
"1.2.3.1" : { "is_malicious" : "true", "is_ioc" : "false" },
"1.2.3.2" : {},
"1.2.3.3" : {"is_malicious" : "true", "is_ioc" : "false", "is_alert" : "true"},
"1.2.3.4" : {},
"1.2.3.5" : {"is_malicious" : "true"}
}
""";
/**
*
* {
* "1.2.3.1" : { "is_malicious" : "true", "is_ioc" : "false" },
* "1.2.3.2" : {},
* "1.2.3.3" : {"is_malicious" : "true", "is_ioc" : "false", "is_alert" : "true"},
* "1.2.3.4" : {},
* "1.2.3.5" : {"is_malicious" : "true"}
* }
**/
@Multiline
public static String simpleMixedFields;
private EnrichmentMemoryTable table;
@Test
@@ -186,5 +174,4 @@ public class EnrichmentsMemoryTableTest {
EnrichmentMemoryTable.fromJsonStream(is);
}
}
}

15
enriching/enriching-storm/pom.xml
View File

@@ -9,9 +9,14 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>23.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
@@ -62,7 +67,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-core</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -80,12 +85,6 @@
<version>${junit_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>

8
enriching/enriching-storm/src/main/java/uk/co/gresearch/siembol/enrichments/storm/EnrichmentEvaluatorBolt.java
View File

@@ -49,11 +49,11 @@ public class EnrichmentEvaluatorBolt extends BaseRichBolt {
private OutputCollector collector;
private ZooKeeperConnector zooKeeperConnector;
private final ZooKeeperAttributesDto zookeperAttributes;
private final ZooKeeperAttributesDto zooKeeperAttributes;
private final ZooKeeperConnectorFactory zooKeeperConnectorFactory;
EnrichmentEvaluatorBolt(StormEnrichmentAttributesDto attributes, ZooKeeperConnectorFactory zooKeeperConnectorFactory) {
this.zookeperAttributes = attributes.getEnrichingRulesZookeperAttributes();
this.zooKeeperAttributes = attributes.getEnrichingRulesZookeperAttributes();
this.zooKeeperConnectorFactory = zooKeeperConnectorFactory;
}
@@ -61,13 +61,12 @@ public class EnrichmentEvaluatorBolt extends BaseRichBolt {
this(attributes, new ZooKeeperConnectorFactoryImpl());
}
@SuppressWarnings("rawtypes")
@Override
public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
this.collector = outputCollector;
try {
LOG.info(ENGINE_INIT_START);
zooKeeperConnector = zooKeeperConnectorFactory.createZookeeperConnector(zookeperAttributes);
zooKeeperConnector = zooKeeperConnectorFactory.createZookeeperConnector(zooKeeperAttributes);
updateRules();
if (enrichmentEvaluator.get() == null) {
@@ -96,7 +95,6 @@ public class EnrichmentEvaluatorBolt extends BaseRichBolt {
LOG.info(ENGINE_UPDATE_COMPLETED);
} catch (Exception e) {
LOG.error(UPDATE_EXCEPTION_LOG, ExceptionUtils.getStackTrace(e));
return;
}
}

17
enriching/enriching-storm/src/main/java/uk/co/gresearch/siembol/enrichments/storm/MemoryTableEnrichmentBolt.java
View File

@@ -15,9 +15,9 @@ import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import uk.co.gresearch.siembol.common.filesystem.HdfsFileSystemFactory;
import uk.co.gresearch.siembol.common.filesystem.SiembolFileSystem;
import uk.co.gresearch.siembol.common.filesystem.SiembolFileSystemFactory;
import uk.co.gresearch.siembol.common.filesystem.SupportedFileSystem;
import uk.co.gresearch.siembol.common.model.EnrichmentTableDto;
import uk.co.gresearch.siembol.common.model.EnrichmentTablesUpdateDto;
import uk.co.gresearch.siembol.common.model.StormEnrichmentAttributesDto;
@@ -58,7 +58,7 @@ public class MemoryTableEnrichmentBolt extends BaseRichBolt {
private static final String INVALID_TYPE_IN_TUPLES = "Invalid type in tuple provided";
private final AtomicReference<Map<String, EnrichmentTable>> enrichmentTables = new AtomicReference<>();
private final ZooKeeperAttributesDto zookeperAttributes;
private final ZooKeeperAttributesDto zooKeeeperAttributes;
private final ZooKeeperConnectorFactory zooKeeperConnectorFactory;
private final SiembolFileSystemFactory fileSystemFactory;
@@ -68,7 +68,7 @@ public class MemoryTableEnrichmentBolt extends BaseRichBolt {
MemoryTableEnrichmentBolt(StormEnrichmentAttributesDto attributes,
ZooKeeperConnectorFactory zooKeeperConnectorFactory,
SiembolFileSystemFactory fileSystemFactory) {
this.zookeperAttributes = attributes.getEnrichingTablesAttributes();
this.zooKeeeperAttributes = attributes.getEnrichingTablesAttributes();
this.zooKeeperConnectorFactory = zooKeeperConnectorFactory;
this.fileSystemFactory = fileSystemFactory;
}
@@ -76,17 +76,16 @@ public class MemoryTableEnrichmentBolt extends BaseRichBolt {
public MemoryTableEnrichmentBolt(StormEnrichmentAttributesDto attributes) {
this(attributes,
new ZooKeeperConnectorFactoryImpl(),
new HdfsFileSystemFactory(attributes.getEnrichingTablesHdfsUri()));
SupportedFileSystem.fromUri(attributes.getEnrichingTablesUri()));
}
@SuppressWarnings("rawtypes")
@Override
public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
this.collector = outputCollector;
try {
LOG.info(TABLES_INIT_START);
zooKeeperConnector = zooKeeperConnectorFactory.createZookeeperConnector(zookeperAttributes);
zooKeeperConnector = zooKeeperConnectorFactory.createZookeeperConnector(zooKeeeperAttributes);
updateTables();
if (enrichmentTables.get() == null) {
@@ -124,7 +123,6 @@ public class MemoryTableEnrichmentBolt extends BaseRichBolt {
LOG.info(TABLES_UPDATES_COMPLETED);
} catch (Exception e) {
LOG.error(TABLES_UPDATE_EXCEPTION_FORMAT, ExceptionUtils.getStackTrace(e));
return;
}
}
@@ -155,10 +153,7 @@ public class MemoryTableEnrichmentBolt extends BaseRichBolt {
}
Optional<List<Pair<String, String>>> result = table.getValues(command);
if (result.isPresent()) {
enrichments.addAll(result.get());
}
result.ifPresent(enrichments::addAll);
}
collector.emit(tuple, new Values(event, enrichments, exceptions));
collector.ack(tuple);

6
enriching/enriching-storm/src/main/java/uk/co/gresearch/siembol/enrichments/storm/StormEnrichingApplication.java
View File

@@ -12,8 +12,8 @@ import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import uk.co.gresearch.siembol.common.filesystem.HdfsFileSystemFactory;
import uk.co.gresearch.siembol.common.filesystem.SiembolFileSystemFactory;
import uk.co.gresearch.siembol.common.filesystem.SupportedFileSystem;
import uk.co.gresearch.siembol.common.storm.KafkaBatchWriterBolt;
import uk.co.gresearch.siembol.common.model.StormAttributesDto;
import uk.co.gresearch.siembol.common.storm.StormHelper;
@@ -101,9 +101,9 @@ public class StormEnrichingApplication {
config.putAll(attributes.getStormAttributes().getStormConfig().getRawMap());
StormTopology topology = createTopology(attributes,
new ZooKeeperConnectorFactoryImpl(),
new HdfsFileSystemFactory(attributes.getEnrichingTablesHdfsUri()));
SupportedFileSystem.fromUri(attributes.getEnrichingTablesUri()));
LOG.info(SUBMIT_INFO_MSG, attributesStr);
LOG.info(SUBMIT_INFO_MSG, attributes.getTopologyName(), attributesStr);
StormSubmitter.submitTopology(attributes.getTopologyName(), config, topology);
}
}

112
enriching/enriching-storm/src/test/java/uk/co/gresearch/siembol/enrichments/storm/EnrichmentEvaluatorBoltTest.java
View File

@@ -1,6 +1,6 @@
package uk.co.gresearch.siembol.enrichments.storm;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
@@ -20,57 +20,53 @@ import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
public class EnrichmentEvaluatorBoltTest {
/**
* {"a" : "tmp_string", "b" : 1, "is_alert" : "true", "source_type" : "test"}
**/
@Multiline
public static String event;
private final String event = """
{"a" : "tmp_string", "b" : 1, "is_alert" : "true", "source_type" : "test"}
""";
/**
* {
* "rules_version": 1,
* "rules": [
* {
* "rule_name": "test_rule",
* "rule_version": 1,
* "rule_author": "john",
* "rule_description": "Test rule",
* "source_type": "*",
* "matchers": [
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "is_alert",
* "data": "(?i)true"
* }
* ],
* "table_mapping": {
* "table_name": "test_table",
* "joining_key": "${a}",
* "tags": [
* {
* "tag_name": "is_test_tag",
* "tag_value": "true"
* }
* ],
* "enriching_fields": [
* {
* "table_field_name": "dns_name",
* "event_field_name": "siembol:enrichments:dns"
* }
* ]
* }
* }
* ]
* }
**/
@Multiline
public static String testRules;
private final String testRules = """
{
"rules_version": 1,
"rules": [
{
"rule_name": "test_rule",
"rule_version": 1,
"rule_author": "john",
"rule_description": "Test rule",
"source_type": "*",
"matchers": [
{
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_alert",
"data": "(?i)true"
}
],
"table_mapping": {
"table_name": "test_table",
"joining_key": "${a}",
"tags": [
{
"tag_name": "is_test_tag",
"tag_value": "true"
}
],
"enriching_fields": [
{
"table_field_name": "dns_name",
"event_field_name": "siembol:enrichments:dns"
}
]
}
}
]
}
""";
private Tuple tuple;
private OutputCollector collector;
EnrichmentEvaluatorBolt enrichmentEvaluatorBolt;
ZooKeeperAttributesDto zookeperAttributes;
ZooKeeperAttributesDto zooKeeperAttributes;
StormEnrichmentAttributesDto attributes;
ZooKeeperConnector zooKeeperConnector;
ZooKeeperConnectorFactory zooKeeperConnectorFactory;
@@ -78,9 +74,9 @@ public class EnrichmentEvaluatorBoltTest {
@Before
public void setUp() throws Exception {
zookeperAttributes = new ZooKeeperAttributesDto();
zooKeeperAttributes = new ZooKeeperAttributesDto();
attributes = new StormEnrichmentAttributesDto();
attributes.setEnrichingRulesZookeperAttributes(zookeperAttributes);
attributes.setEnrichingRulesZookeperAttributes(zooKeeperAttributes);
tuple = Mockito.mock(Tuple.class);
collector = Mockito.mock(OutputCollector.class);
@@ -89,7 +85,7 @@ public class EnrichmentEvaluatorBoltTest {
zooKeeperConnector = Mockito.mock(ZooKeeperConnector.class);
when(zooKeeperConnectorFactory.createZookeeperConnector(zookeperAttributes)).thenReturn(zooKeeperConnector);
when(zooKeeperConnectorFactory.createZookeeperConnector(zooKeeperAttributes)).thenReturn(zooKeeperConnector);
when(zooKeeperConnector.getData()).thenReturn(testRules);
when(tuple.getStringByField(eq(EnrichmentTuples.EVENT.toString()))).thenReturn(event);
@@ -109,13 +105,13 @@ public class EnrichmentEvaluatorBoltTest {
Assert.assertTrue(values.get(1) instanceof EnrichmentCommands);
Assert.assertTrue(values.get(2) instanceof EnrichmentExceptions);
Assert.assertEquals(event, values.get(0));
EnrichmentCommands commands = (EnrichmentCommands)values.get(1);
EnrichmentCommands commands = (EnrichmentCommands) values.get(1);
Assert.assertEquals(1, commands.size());
Assert.assertEquals("tmp_string", commands.get(0).getKey());
Assert.assertEquals("tmp_string", commands.get(0).getKey());
Assert.assertEquals(1, commands.get(0).getTags().size());
Assert.assertEquals(1, commands.get(0).getEnrichmentFields().size());
Assert.assertTrue(((EnrichmentExceptions)values.get(2)).isEmpty());
Assert.assertTrue(((EnrichmentExceptions) values.get(2)).isEmpty());
}
@Test
@@ -128,8 +124,8 @@ public class EnrichmentEvaluatorBoltTest {
Assert.assertTrue(values.get(0) instanceof String);
Assert.assertTrue(values.get(1) instanceof EnrichmentCommands);
Assert.assertTrue(values.get(2) instanceof EnrichmentExceptions);
Assert.assertTrue(((EnrichmentCommands)values.get(1)).isEmpty());
Assert.assertTrue(((EnrichmentExceptions)values.get(2)).isEmpty());
Assert.assertTrue(((EnrichmentCommands) values.get(1)).isEmpty());
Assert.assertTrue(((EnrichmentExceptions) values.get(2)).isEmpty());
}
@Test
@@ -142,9 +138,9 @@ public class EnrichmentEvaluatorBoltTest {
Assert.assertTrue(values.get(0) instanceof String);
Assert.assertTrue(values.get(1) instanceof EnrichmentCommands);
Assert.assertTrue(values.get(2) instanceof EnrichmentExceptions);
Assert.assertTrue(((EnrichmentCommands)values.get(1)).isEmpty());
Assert.assertFalse(((EnrichmentExceptions)values.get(2)).isEmpty());
Assert.assertEquals(1, ((EnrichmentExceptions)values.get(2)).size());
Assert.assertTrue(((EnrichmentExceptions)values.get(2)).get(0).contains("JsonParseException"));
Assert.assertTrue(((EnrichmentCommands) values.get(1)).isEmpty());
Assert.assertFalse(((EnrichmentExceptions) values.get(2)).isEmpty());
Assert.assertEquals(1, ((EnrichmentExceptions) values.get(2)).size());
Assert.assertTrue(((EnrichmentExceptions) values.get(2)).get(0).contains("JsonParseException"));
}
}

24
enriching/enriching-storm/src/test/java/uk/co/gresearch/siembol/enrichments/storm/EnrichmentMergerBoltTest.java
View File

@@ -1,6 +1,6 @@
package uk.co.gresearch.siembol.enrichments.storm;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
@@ -24,20 +24,16 @@ import static org.mockito.Mockito.when;
public class EnrichmentMergerBoltTest {
/**
* {"a": "string","b": 1,"c": true}
**/
@Multiline
public static String event;
private final String event = """
{"a": "string","b": 1,"c": true}
""";
/**
* {"a":"string","b":1,"c":true,"siembol_enriching_ts":
**/
@Multiline
public static String enrichedEventPrefix;
private final String enrichedEventPrefix = """
{"a":"string","b":1,"c":true,"siembol_enriching_ts":
""";
private String errorTopic = "error";
private String outputTopic = "output";
private final String errorTopic = "error";
private final String outputTopic = "output";
private Tuple tuple;
private OutputCollector collector;
@@ -91,7 +87,7 @@ public class EnrichmentMergerBoltTest {
KafkaBatchWriterMessages messages = (KafkaBatchWriterMessages)values.get(0);
Assert.assertEquals(outputTopic, messages.get(0).getTopic());
Assert.assertTrue(messages.get(0).getMessage().contains(""));
Assert.assertFalse(messages.get(0).getMessage().isEmpty());
Assert.assertEquals(errorTopic, messages.get(1).getTopic());
Assert.assertEquals("dummy1", messages.get(1).getMessage());

71
enriching/enriching-storm/src/test/java/uk/co/gresearch/siembol/enrichments/storm/MemoryTableEnrichmentBoltTest.java
View File

@@ -1,6 +1,6 @@
package uk.co.gresearch.siembol.enrichments.storm;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
@@ -28,36 +28,29 @@ import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
public class MemoryTableEnrichmentBoltTest {
/**
* {"a": "string", "b": 1, "c": true}
**/
@Multiline
public static String event;
private final String event = """
{"a": "string", "b": 1, "c": true}
""";
/**
* {
* "enrichment_tables" : [
* {
* "name" : "test_table",
* "path": "/siembol/tables/enrichment/test.json"
* }]
* }
**/
@Multiline
public static String tablesUpdate;
private final String tablesUpdate = """
{
"enrichment_tables" : [
{
"name" : "test_table",
"path": "/siembol/tables/enrichment/test.json"
}]
}
""";
/**
*
* {
* "1.2.3.1" : { "is_malicious" : "true" },
* "1.2.3.2" : { "is_malicious" : "true"},
* "1.2.3.3" : {"is_malicious" : "false"},
* "1.2.3.4" : {"is_malicious" : "true"},
* "1.2.3.5" : {"is_malicious" : "true"}
* }
**/
@Multiline
public static String simpleOneField;
private final String simpleOneField = """
{
"1.2.3.1" : { "is_malicious" : "true" },
"1.2.3.2" : { "is_malicious" : "true"},
"1.2.3.3" : { "is_malicious" : "false"},
"1.2.3.4" : { "is_malicious" : "true"},
"1.2.3.5" : { "is_malicious" : "true"}
}
""";
private Tuple tuple;
private OutputCollector collector;
@@ -112,8 +105,8 @@ public class MemoryTableEnrichmentBoltTest {
Assert.assertTrue(values.get(1) instanceof EnrichmentPairs);
Assert.assertTrue(values.get(2) instanceof EnrichmentExceptions);
Assert.assertEquals(event, values.get(0));
Assert.assertTrue(((EnrichmentPairs)values.get(1)).isEmpty());
Assert.assertTrue(((EnrichmentExceptions)values.get(2)).isEmpty());
Assert.assertTrue(((EnrichmentPairs) values.get(1)).isEmpty());
Assert.assertTrue(((EnrichmentExceptions) values.get(2)).isEmpty());
}
@Test
@@ -128,8 +121,8 @@ public class MemoryTableEnrichmentBoltTest {
Assert.assertTrue(values.get(1) instanceof EnrichmentPairs);
Assert.assertTrue(values.get(2) instanceof EnrichmentExceptions);
Assert.assertEquals(event, values.get(0));
Assert.assertTrue(((EnrichmentPairs)values.get(1)).isEmpty());
EnrichmentExceptions exceptions = (EnrichmentExceptions)values.get(2);
Assert.assertTrue(((EnrichmentPairs) values.get(1)).isEmpty());
EnrichmentExceptions exceptions = (EnrichmentExceptions) values.get(2);
Assert.assertEquals(2, exceptions.size());
Assert.assertEquals("dummy1", exceptions.get(0));
Assert.assertEquals("dummy2", exceptions.get(1));
@@ -148,8 +141,8 @@ public class MemoryTableEnrichmentBoltTest {
Assert.assertTrue(values.get(1) instanceof EnrichmentPairs);
Assert.assertTrue(values.get(2) instanceof EnrichmentExceptions);
Assert.assertEquals(event, values.get(0));
Assert.assertTrue(((EnrichmentPairs)values.get(1)).isEmpty());
Assert.assertTrue(((EnrichmentExceptions)values.get(2)).isEmpty());
Assert.assertTrue(((EnrichmentPairs) values.get(1)).isEmpty());
Assert.assertTrue(((EnrichmentExceptions) values.get(2)).isEmpty());
}
@Test
@@ -167,9 +160,9 @@ public class MemoryTableEnrichmentBoltTest {
Assert.assertTrue(values.get(1) instanceof EnrichmentPairs);
Assert.assertTrue(values.get(2) instanceof EnrichmentExceptions);
Assert.assertEquals(event, values.get(0));
EnrichmentPairs enrichments = (EnrichmentPairs)values.get(1);
EnrichmentPairs enrichments = (EnrichmentPairs) values.get(1);
Assert.assertEquals(1, enrichments.size());
Assert.assertTrue(((EnrichmentExceptions)values.get(2)).isEmpty());
Assert.assertTrue(((EnrichmentExceptions) values.get(2)).isEmpty());
}
@Test
@@ -186,7 +179,7 @@ public class MemoryTableEnrichmentBoltTest {
Assert.assertTrue(values.get(1) instanceof EnrichmentPairs);
Assert.assertTrue(values.get(2) instanceof EnrichmentExceptions);
Assert.assertEquals(event, values.get(0));
Assert.assertTrue(((EnrichmentPairs)values.get(1)).isEmpty());
Assert.assertTrue(((EnrichmentExceptions)values.get(2)).isEmpty());
Assert.assertTrue(((EnrichmentPairs) values.get(1)).isEmpty());
Assert.assertTrue(((EnrichmentExceptions) values.get(2)).isEmpty());
}
}

279
enriching/enriching-storm/src/test/java/uk/co/gresearch/siembol/enrichments/storm/StormEnrichingApplicationTest.java
View File

@@ -4,7 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import com.github.charithe.kafka.EphemeralKafkaBroker;
import com.github.charithe.kafka.KafkaJunitRule;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
@@ -29,156 +29,141 @@ public class StormEnrichingApplicationTest {
private static final ObjectReader JSON_PARSERS_CONFIG_READER = new ObjectMapper()
.readerFor(StormEnrichmentAttributesDto.class);
/**
* {
* "topology.name": "testing",
* "kafka.spout.num.executors": 1,
* "enriching.engine.bolt.num.executors": 1,
* "memory.enriching.bolt.num.executors": 1,
* "merging.bolt.num.executors": 1,
* "kafka.writer.bolt.num.executors": 1,
* "enriching.input.topics" : [ "enrichments" ],
* "enriching.output.topic": "output",
* "enriching.error.topic": "error",
* "enriching.tables.hdfs.uri": "hdfs://secret",
* "enriching.rules.zookeeper.attributes": {
* "zk.path": "/enrichment/rules",
* "zk.base.sleep.ms": 1000,
* "zk.max.retries": 10
* },
* "enriching.tables.zookeeper.attributes": {
* "zk.path": "/enrichment/tables",
* "zk.base.sleep.ms": 1000,
* "zk.max.retries": 10
* },
* "kafka.batch.writer.attributes": {
* "batch.size": 1,
* "producer.properties": {
* "client.id": "writer",
* "compression.type": "snappy",
* "security.protocol": "PLAINTEXT"
* }
* },
* "storm.attributes": {
* "first.pool.offset.strategy": "EARLIEST",
* "kafka.spout.properties": {
* "security.protocol": "PLAINTEXT"
* },
* "storm.config": {
* "session.timeout.ms": 100000
* }
* }
* }
**/
@Multiline
public static String testEnrichmentStormConfig;
private final String testEnrichmentStormConfig = """
{
"topology.name": "testing",
"kafka.spout.num.executors": 1,
"enriching.engine.bolt.num.executors": 1,
"memory.enriching.bolt.num.executors": 1,
"merging.bolt.num.executors": 1,
"kafka.writer.bolt.num.executors": 1,
"enriching.input.topics" : [ "enrichments" ],
"enriching.output.topic": "output",
"enriching.error.topic": "error",
"enriching.tables.uri": "hdfs://secret",
"enriching.rules.zookeeper.attributes": {
"zk.path": "/enrichment/rules",
"zk.base.sleep.ms": 1000,
"zk.max.retries": 10
},
"enriching.tables.zookeeper.attributes": {
"zk.path": "/enrichment/tables",
"zk.base.sleep.ms": 1000,
"zk.max.retries": 10
},
"kafka.batch.writer.attributes": {
"batch.size": 1,
"producer.properties": {
"client.id": "writer",
"compression.type": "snappy",
"security.protocol": "PLAINTEXT"
}
},
"storm.attributes": {
"first.pool.offset.strategy": "EARLIEST",
"kafka.spout.properties": {
"security.protocol": "PLAINTEXT"
},
"storm.config": {
"session.timeout.ms": 100000
}
}
}
""";
/**
* {
* "enrichment_tables" : [
* {
* "name" : "test_table",
* "path": "/siembol/tables/enrichment/test.json"
* }]
* }
**/
@Multiline
public static String tablesUpdate;
private final String tablesUpdate = """
{
"enrichment_tables" : [
{
"name" : "test_table",
"path": "/siembol/tables/enrichment/test.json"
}]
}
""";
/**
*
* {
* "1.2.3.4" : { "dns_name" : "secret.unknown" },
* "1.2.3.5" : { "dns_name" : "secret.known" }
* }
**/
@Multiline
public static String simpleOneField;
private final String simpleOneField = """
{
"1.2.3.4" : { "dns_name" : "secret.unknown" },
"1.2.3.5" : { "dns_name" : "secret.known" }
}
""";
/**
*{"ip_src_addr":"1.2.3.4","ip_dst_addr":"1.2.3.5","b":1,"is_alert":"true","source_type":"test","is_test_tag_first":"true","src_dns_name":"secret.unknown","is_test_tag_second":"true","dst_dns_name":"secret.known","siembol_enriching_ts":
**/
@Multiline
public static String expectedEvent;
private final String expectedEvent = """
{"ip_src_addr":"1.2.3.4","ip_dst_addr":"1.2.3.5","b":1,"is_alert":"true","source_type":"test","is_test_tag_first":"true","src_dns_name":"secret.unknown","is_test_tag_second":"true","dst_dns_name":"secret.known","siembol_enriching_ts":""";
/**
* {"ip_src_addr" : "1.2.3.4", "ip_dst_addr" : "1.2.3.5", "b" : 1, "is_alert" : "true", "source_type" : "test"}
**/
@Multiline
public static String event;
/**
* {
* "rules_version": 1,
* "rules": [
* {
* "rule_name": "test_rule_first",
* "rule_version": 1,
* "rule_author": "john",
* "rule_description": "Test rule",
* "source_type": "*",
* "matchers": [
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "is_alert",
* "data": "(?i)true"
* }
* ],
* "table_mapping": {
* "table_name": "test_table",
* "joining_key": "${ip_src_addr}",
* "tags": [
* {
* "tag_name": "is_test_tag_first",
* "tag_value": "true"
* }
* ],
* "enriching_fields": [
* {
* "table_field_name": "dns_name",
* "event_field_name": "src_dns_name"
* }
* ]
* }
* },
* {
* "rule_name": "test_rule_second",
* "rule_version": 1,
* "rule_author": "john",
* "rule_description": "Test rule",
* "source_type": "*",
* "matchers": [
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "is_alert",
* "data": "(?i)true"
* }
* ],
* "table_mapping": {
* "table_name": "test_table",
* "joining_key": "${ip_dst_addr}",
* "tags": [
* {
* "tag_name": "is_test_tag_second",
* "tag_value": "true"
* }
* ],
* "enriching_fields": [
* {
* "table_field_name": "dns_name",
* "event_field_name": "dst_dns_name"
* }
* ]
* }
* }
* ]
* }
**/
@Multiline
public static String testRules;
private final String event = """
{"ip_src_addr" : "1.2.3.4", "ip_dst_addr" : "1.2.3.5", "b" : 1, "is_alert" : "true", "source_type" : "test"}
""";
private final String testRules = """
{
"rules_version": 1,
"rules": [
{
"rule_name": "test_rule_first",
"rule_version": 1,
"rule_author": "john",
"rule_description": "Test rule",
"source_type": "*",
"matchers": [
{
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_alert",
"data": "(?i)true"
}
],
"table_mapping": {
"table_name": "test_table",
"joining_key": "${ip_src_addr}",
"tags": [
{
"tag_name": "is_test_tag_first",
"tag_value": "true"
}
],
"enriching_fields": [
{
"table_field_name": "dns_name",
"event_field_name": "src_dns_name"
}
]
}
},
{
"rule_name": "test_rule_second",
"rule_version": 1,
"rule_author": "john",
"rule_description": "Test rule",
"source_type": "*",
"matchers": [
{
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_alert",
"data": "(?i)true"
}
],
"table_mapping": {
"table_name": "test_table",
"joining_key": "${ip_dst_addr}",
"tags": [
{
"tag_name": "is_test_tag_second",
"tag_value": "true"
}
],
"enriching_fields": [
{
"table_field_name": "dns_name",
"event_field_name": "dst_dns_name"
}
]
}
}
]
}
""";
@ClassRule
public static KafkaJunitRule kafkaRule = new KafkaJunitRule(EphemeralKafkaBroker.create());
@@ -232,7 +217,7 @@ public class StormEnrichingApplicationTest {
cluster.submitTopology("test", config, topology);
}
@Test(timeout=200000)
@Test(timeout = 200000)
public void testEnrichMessageOK() throws Exception {
kafkaRule.helper().produceStrings("enrichments", event.trim());
List<String> outputEvent = kafkaRule.helper().consumeStrings("output", 1)

2
enriching/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</parent>
<modules>
<module>enriching-core</module>

12
parsing/parsing-app/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -39,18 +39,12 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-core</artifactId>
<version>1.4.2-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>

243
parsing/parsing-app/src/test/java/uk/co/gresearch/siembol/parsers/application/factory/ParsingApplicationFactoryImplTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.parsers.application.factory;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Test;
@@ -10,136 +9,128 @@ public class ParsingApplicationFactoryImplTest {
factory = new ParsingApplicationFactoryImpl();
}
/**
*{
* "parsing_app_name": "test",
* "parsing_app_version": 1,
* "parsing_app_author": "dummy",
* "parsing_app_description": "Description of parser application",
* "parsing_app_settings": {
* "input_topics": [
* "secret"
* ],
* "error_topic": "error",
* "input_parallelism": 1,
* "parsing_parallelism": 2,
* "output_parallelism": 3,
* "parsing_app_type": "single_parser"
* },
* "parsing_settings": {
* "single_parser": {
* "parser_name": "single",
* "output_topic": "output"
* }
* }
* }
**/
@Multiline
public static String simpleSingleApplicationParser;
private final String simpleSingleApplicationParser = """
{
"parsing_app_name": "test",
"parsing_app_version": 1,
"parsing_app_author": "dummy",
"parsing_app_description": "Description of parser application",
"parsing_app_settings": {
"input_topics": [
"secret"
],
"error_topic": "error",
"input_parallelism": 1,
"parsing_parallelism": 2,
"output_parallelism": 3,
"parsing_app_type": "single_parser"
},
"parsing_settings": {
"single_parser": {
"parser_name": "single",
"output_topic": "output"
}
}
}
""";
private final String simpleRoutingApplicationParser = """
{
"parsing_app_name": "test",
"parsing_app_version": 1,
"parsing_app_author": "dummy",
"parsing_app_description": "Description of parser application",
"parsing_app_settings": {
"input_topics": [
"secret"
],
"error_topic": "error",
"input_parallelism": 1,
"parsing_parallelism": 2,
"output_parallelism": 3,
"parsing_app_type": "router_parsing"
},
"parsing_settings": {
"routing_parser": {
"router_parser_name": "router",
"routing_field": "host",
"routing_message": "msg",
"merged_fields": [
"timestamp",
"syslog_host"
],
"default_parser": {
"parser_name": "default",
"output_topic": "output_default"
},
"parsers": [
{
"routing_field_pattern": "secret",
"parser_properties": {
"parser_name": "single",
"output_topic": "out_secret"
}
}
]
}
}
}
""";
/**
* {
* "parsing_app_name": "test",
* "parsing_app_version": 1,
* "parsing_app_author": "dummy",
* "parsing_app_description": "Description of parser application",
* "parsing_app_settings": {
* "input_topics": [
* "secret"
* ],
* "error_topic": "error",
* "input_parallelism": 1,
* "parsing_parallelism": 2,
* "output_parallelism": 3,
* "parsing_app_type": "router_parsing"
* },
* "parsing_settings": {
* "routing_parser": {
* "router_parser_name": "router",
* "routing_field": "host",
* "routing_message": "msg",
* "merged_fields": [
* "timestamp",
* "syslog_host"
* ],
* "default_parser": {
* "parser_name": "default",
* "output_topic": "output_default"
* },
* "parsers": [
* {
* "routing_field_pattern": "secret",
* "parser_properties": {
* "parser_name": "single",
* "output_topic": "out_secret"
* }
* }
* ]
* }
* }
* }
**/
@Multiline
public static String simpleRoutingApplicationParser;
/**
* {
* "parsers_version": 1,
* "parsers_configurations": [
* {
* "parser_description": "for testing single app parser",
* "parser_version": 2,
* "parser_name": "single",
* "parser_author": "dummy",
* "parser_attributes": {
* "parser_type": "generic"
* }
* },
* {
* "parser_description": "for testing routing app paerser",
* "parser_version": 2,
* "parser_name": "router",
* "parser_author": "dummy",
* "parser_attributes": {
* "parser_type": "generic"
* }
* },
* {
* "parser_description": "for testing routing app parser",
* "parser_version": 2,
* "parser_name": "default",
* "parser_author": "dummy",
* "parser_attributes": {
* "parser_type": "generic"
* }
* }
* ]
* }
**/
@Multiline
public static String testParsersConfigs;
private final String testParsersConfigs = """
{
"parsers_version": 1,
"parsers_configurations": [
{
"parser_description": "for testing single app parser",
"parser_version": 2,
"parser_name": "single",
"parser_author": "dummy",
"parser_attributes": {
"parser_type": "generic"
}
},
{
"parser_description": "for testing routing app parser",
"parser_version": 2,
"parser_name": "router",
"parser_author": "dummy",
"parser_attributes": {
"parser_type": "generic"
}
},
{
"parser_description": "for testing routing app parser",
"parser_version": 2,
"parser_name": "default",
"parser_author": "dummy",
"parser_attributes": {
"parser_type": "generic"
}
}
]
}
""";
@Test
public void testGetSchema() {
ParsingApplicationFactoryResult schemaResult = factory.getSchema();
Assert.assertTrue(schemaResult.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.OK);
Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.OK, schemaResult.getStatusCode());
Assert.assertFalse(schemaResult.getAttributes().getJsonSchema().isEmpty());
}
@Test
public void testValidationSingleGood() {
ParsingApplicationFactoryResult result = factory.validateConfiguration(simpleSingleApplicationParser);
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.OK);
Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.OK, result.getStatusCode());
}
@Test
public void testValidationSingleFail() {
ParsingApplicationFactoryResult result = factory.validateConfiguration(simpleSingleApplicationParser
.replace("error_topic", "dummy"));
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR);
Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("error_topic"));
}
@@ -147,7 +138,7 @@ public class ParsingApplicationFactoryImplTest {
public void testValidationSingleFail2() {
ParsingApplicationFactoryResult result = factory.validateConfiguration(simpleSingleApplicationParser
.replace("\"parsing_parallelism\": 2,", ""));
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR);
Assert.assertSame( ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage()
.contains("missing required properties ([\"parsing_parallelism\"])"));
}
@@ -155,7 +146,7 @@ public class ParsingApplicationFactoryImplTest {
@Test
public void testCreationSingleGood() {
ParsingApplicationFactoryResult result = factory.create(simpleSingleApplicationParser, testParsersConfigs);
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.OK);
Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.OK, result.getStatusCode());
Assert.assertEquals("test", result.getAttributes().getName());
Assert.assertEquals(1, result.getAttributes().getInputParallelism().intValue());
Assert.assertEquals(2, result.getAttributes().getParsingParallelism().intValue());
@@ -169,14 +160,14 @@ public class ParsingApplicationFactoryImplTest {
ParsingApplicationFactoryResult result = factory.create(
simpleSingleApplicationParser.replace("error_topic", "dummy"),
testParsersConfigs);
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR);
Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("error_topic"));
}
@Test
public void testCreationSingleWrongParserConfigs() {
ParsingApplicationFactoryResult result = factory.create(simpleSingleApplicationParser, "INVALID");
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR);
Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("INVALID"));
}
@@ -184,28 +175,28 @@ public class ParsingApplicationFactoryImplTest {
public void testCreationSingleMissingParserConfigs() {
ParsingApplicationFactoryResult result = factory.create(simpleSingleApplicationParser,
testParsersConfigs.replace("single", "unwanted"));
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR);
Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("Missing parser: single"));
}
@Test
public void testValidationRoutingGood() {
ParsingApplicationFactoryResult result = factory.validateConfiguration(simpleRoutingApplicationParser);
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.OK);
Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.OK, result.getStatusCode());
}
@Test
public void testValidationRoutingFail() {
ParsingApplicationFactoryResult result = factory.validateConfiguration(simpleRoutingApplicationParser
.replace("error_topic", "dummy"));
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR);
Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("error_topic"));
}
@Test
public void testCreationRoutingGood() {
ParsingApplicationFactoryResult result = factory.create(simpleRoutingApplicationParser, testParsersConfigs);
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.OK);
Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.OK, result.getStatusCode());
Assert.assertEquals("test", result.getAttributes().getName());
Assert.assertEquals(1, result.getAttributes().getInputParallelism().intValue());
Assert.assertEquals(2, result.getAttributes().getParsingParallelism().intValue());
@@ -219,14 +210,14 @@ public class ParsingApplicationFactoryImplTest {
ParsingApplicationFactoryResult result = factory.create(
simpleRoutingApplicationParser.replace("error_topic", "dummy"),
testParsersConfigs);
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR);
Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("error_topic"));
}
@Test
public void testCreationRoutingWrongParserConfigs() {
ParsingApplicationFactoryResult result = factory.create(simpleRoutingApplicationParser, "INVALID");
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR);
Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("INVALID"));
}
@@ -234,7 +225,7 @@ public class ParsingApplicationFactoryImplTest {
public void testCreationRoutingMissingRouterParserConfigs() {
ParsingApplicationFactoryResult result = factory.create(simpleRoutingApplicationParser,
testParsersConfigs.replace("router", "unwanted"));
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR);
Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("Missing parser: router"));
}
@@ -242,7 +233,7 @@ public class ParsingApplicationFactoryImplTest {
public void testCreationRoutingMissingDefaultParserConfigs() {
ParsingApplicationFactoryResult result = factory.create(simpleRoutingApplicationParser,
testParsersConfigs.replace("default", "unwanted"));
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR);
Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("Missing parser: default"));
}
@@ -250,7 +241,7 @@ public class ParsingApplicationFactoryImplTest {
public void testCreationRoutingMissingParserConfigs() {
ParsingApplicationFactoryResult result = factory.create(simpleRoutingApplicationParser,
testParsersConfigs.replace("single", "unwanted"));
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR);
Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("Missing parser: single"));
}
}

69
parsing/parsing-app/src/test/java/uk/co/gresearch/siembol/parsers/application/parsing/RoutingParsingApplicationParserTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.parsers.application.parsing;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -17,38 +16,34 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class RoutingParsingApplicationParserTest {
/**
* {
*
* "a": "string",
* "b": 1,
* "c": true
* }
**/
@Multiline
public static String metadata;
private final String metadata = """
{
"a": "string",
"b": 1,
"c": true
}
""";
private SerializableSiembolParser routerParser;
private SerializableSiembolParser defaultParser;
private SerializableSiembolParser routedParser1;
private SerializableSiembolParser routedParser2;
private String routingConditionField = "test_field";
private String routingMessageField = "original_string";
private final String routingConditionField = "test_field";
private final String routingMessageField = "original_string";
private RoutingParsingApplicationParser appParser;
private Map<String, Object> message1;
private Map<String, Object> message2;
private List<Map<String, Object>> parsed;
private String errorTopic = "error";
private String outputTopic = "output";
private byte[] input = "test".getBytes();
private final String errorTopic = "error";
private final String outputTopic = "output";
private final byte[] input = "test".getBytes();
private ParserResult routerParserResult;
private ParserResult routedParserResult1;
private ParserResult routedParserResult2;
TimeProvider timeProvider;
long currentTime = 1L;
@Before
public void setUp() {
timeProvider = Mockito.mock(TimeProvider.class);
@@ -95,7 +90,7 @@ public class RoutingParsingApplicationParserTest {
}
@Test(expected = IllegalArgumentException.class)
public void testMissingARguments() {
public void testMissingArguments() {
appParser = RoutingParsingApplicationParser.builder()
.errorTopic(errorTopic)
.build();
@@ -138,13 +133,13 @@ public class RoutingParsingApplicationParserTest {
Assert.assertEquals(outputTopic, result.get(0).getTopic());
Assert.assertEquals(1, result.size());
Assert.assertEquals(1, result.get(0).getMessages().size());
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("output_field" + "\":\"routed"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("original_string" + "\":\"test"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":3"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"default-parser\""));
SiembolMessageFields.SENSOR_TYPE + "\":\"default-parser\""));
}
@Test
@@ -194,14 +189,14 @@ public class RoutingParsingApplicationParserTest {
Assert.assertEquals(outputTopic, result.get(0).getTopic());
Assert.assertEquals(1, result.size());
Assert.assertEquals(1, result.get(0).getMessages().size());
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("output_field" + "\":\"routed"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("original_string" + "\":\"test"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":3"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("guid" + "\":"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"default-parser\""));
SiembolMessageFields.SENSOR_TYPE + "\":\"default-parser\""));
}
@Test
@@ -285,13 +280,13 @@ public class RoutingParsingApplicationParserTest {
Assert.assertEquals(outputTopic, result.get(0).getTopic());
Assert.assertEquals(1, result.size());
Assert.assertEquals(1, result.get(0).getMessages().size());
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("output_field" + "\":\"routed"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("original_string" + "\":\"test"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":3"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"default-parser\""));
SiembolMessageFields.SENSOR_TYPE + "\":\"default-parser\""));
}
@Test
@@ -320,10 +315,10 @@ public class RoutingParsingApplicationParserTest {
Assert.assertEquals(2, result.size());
Assert.assertEquals("dummy1", result.get(0).getTopic());
Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"routed-parser1\""));
SiembolMessageFields.SENSOR_TYPE + "\":\"routed-parser1\""));
Assert.assertEquals("dummy2", result.get(1).getTopic());
Assert.assertTrue(result.get(1).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"routed-parser2\""));
SiembolMessageFields.SENSOR_TYPE + "\":\"routed-parser2\""));
}
@Test
@@ -335,7 +330,7 @@ public class RoutingParsingApplicationParserTest {
.routingMessageField(routingMessageField)
.addParser("dummy1", routedParser1, "a")
.addParser("dummy2", routedParser2, "b")
.mergedFields(Arrays.asList("timestamp"))
.mergedFields(List.of("timestamp"))
.name("test")
.errorTopic(errorTopic)
.timeProvider(timeProvider)
@@ -356,9 +351,9 @@ public class RoutingParsingApplicationParserTest {
Assert.assertEquals("dummy2", result.get(1).getTopic());
Assert.assertTrue(result.get(1).getMessages().get(0).contains("timestamp" + "\":2"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"routed-parser1\""));
SiembolMessageFields.SENSOR_TYPE + "\":\"routed-parser1\""));
Assert.assertTrue(result.get(1).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"routed-parser2\""));
SiembolMessageFields.SENSOR_TYPE + "\":\"routed-parser2\""));
}
@Test
@@ -370,7 +365,7 @@ public class RoutingParsingApplicationParserTest {
.routingMessageField(routingMessageField)
.addParser("dummy1", routedParser1, "a")
.addParser("dummy2", routedParser2, "b")
.mergedFields(Arrays.asList("timestamp"))
.mergedFields(List.of("timestamp"))
.name("test")
.errorTopic(errorTopic)
.timeProvider(timeProvider)
@@ -392,9 +387,9 @@ public class RoutingParsingApplicationParserTest {
Assert.assertEquals("dummy2", result.get(1).getTopic());
Assert.assertTrue(result.get(1).getMessages().get(0).contains("timestamp" + "\":2"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"routed-parser1\""));
SiembolMessageFields.SENSOR_TYPE + "\":\"routed-parser1\""));
Assert.assertTrue(result.get(1).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"routed-parser2\""));
SiembolMessageFields.SENSOR_TYPE + "\":\"routed-parser2\""));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("guid" + "\":"));
Assert.assertTrue(result.get(1).getMessages().get(0).contains("guid" + "\":"));
}
@@ -408,7 +403,7 @@ public class RoutingParsingApplicationParserTest {
.routingMessageField(routingMessageField)
.addParser("dummy1", routedParser1, "a")
.addParser("dummy2", routedParser2, "b")
.mergedFields(Arrays.asList("timestamp"))
.mergedFields(List.of("timestamp"))
.name("test")
.errorTopic(errorTopic)
.timeProvider(timeProvider)
@@ -431,6 +426,6 @@ public class RoutingParsingApplicationParserTest {
Assert.assertEquals("dummy2", result.get(1).getTopic());
Assert.assertTrue(result.get(1).getMessages().get(0).contains("timestamp" + "\":2"));
Assert.assertTrue(result.get(1).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"routed-parser2\""));
SiembolMessageFields.SENSOR_TYPE + "\":\"routed-parser2\""));
}
}

71
parsing/parsing-app/src/test/java/uk/co/gresearch/siembol/parsers/application/parsing/SingleApplicationParserTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.parsers.application.parsing;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -17,26 +16,24 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class SingleApplicationParserTest {
/**
* {
*
* "a": "string",
* "b": 1,
* "c": true
* }
**/
@Multiline
public static String metadata;
private final String metadata = """
{
"a": "string",
"b": 1,
"c": true
}
""";
private SerializableSiembolParser siembolParser;
private String sourceType = "test_type";
private final String sourceType = "test_type";
private SingleApplicationParser appParser;
private Map<String, Object> message1;
private Map<String, Object> message2;
private List<Map<String, Object>> parsed;
private String errorTopic = "error";
private String outputTopic = "output";
private byte[] input = "test".getBytes();
private final String errorTopic = "error";
private final String outputTopic = "output";
private final byte[] input = "test".getBytes();
private ParserResult parserResult;
TimeProvider timeProvider;
long currentTime = 1L;
@@ -65,14 +62,14 @@ public class SingleApplicationParserTest {
}
@Test(expected = IllegalArgumentException.class)
public void testMissingARguments() {
public void testMissingArguments() {
appParser = SingleApplicationParser.builder()
.errorTopic(errorTopic)
.build();
}
@Test(expected = IllegalArgumentException.class)
public void testMissingARguments2() throws Exception {
public void testMissingArguments2() throws Exception {
appParser = SingleApplicationParser.builder()
.parser(outputTopic, siembolParser)
.name("test")
@@ -97,16 +94,16 @@ public class SingleApplicationParserTest {
Assert.assertEquals(1, result.size());
Assert.assertEquals(2, result.get(0).getMessages().size());
Assert.assertEquals(outputTopic, result.get(0).getTopic());
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_field" + "\":\"a"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"test_type\""));
Assert.assertTrue(result.get(0).getMessages().get(1).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1"));
SiembolMessageFields.SENSOR_TYPE + "\":\"test_type\""));
Assert.assertTrue(result.get(0).getMessages().get(1).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_field" + "\":\"b"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("timestamp" + "\":2"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"test_type\""));
SiembolMessageFields.SENSOR_TYPE + "\":\"test_type\""));
}
@Test
@@ -126,12 +123,12 @@ public class SingleApplicationParserTest {
Assert.assertEquals(1, result.size());
Assert.assertEquals(1, result.get(0).getMessages().size());
Assert.assertEquals(outputTopic, result.get(0).getTopic());
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_field" + "\":\"a"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":1"));
Assert.assertFalse(result.get(0).getMessages().get(0).contains("guid" + "\":"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"test_type\""));
SiembolMessageFields.SENSOR_TYPE + "\":\"test_type\""));
}
@Test
@@ -152,12 +149,12 @@ public class SingleApplicationParserTest {
Assert.assertEquals(1, result.size());
Assert.assertEquals(1, result.get(0).getMessages().size());
Assert.assertEquals(outputTopic, result.get(0).getTopic());
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_field" + "\":\"a"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("guid" + "\":"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"test_type\""));
SiembolMessageFields.SENSOR_TYPE + "\":\"test_type\""));
}
@Test
@@ -256,22 +253,22 @@ public class SingleApplicationParserTest {
Assert.assertEquals(1, result.size());
Assert.assertEquals(2, result.get(0).getMessages().size());
Assert.assertEquals(outputTopic, result.get(0).getTopic());
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_field" + "\":\"a"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_metadata:a" + "\":\"string\""));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_metadata:b" + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_metadata:c" + "\":true"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"test_type\""));
Assert.assertTrue(result.get(0).getMessages().get(1).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1"));
SiembolMessageFields.SENSOR_TYPE + "\":\"test_type\""));
Assert.assertTrue(result.get(0).getMessages().get(1).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_field" + "\":\"b"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("timestamp" + "\":2"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_metadata:a" + "\":\"string\""));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_metadata:b" + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_metadata:c" + "\":true"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"test_type\""));
SiembolMessageFields.SENSOR_TYPE + "\":\"test_type\""));
}
@Test
@@ -293,7 +290,7 @@ public class SingleApplicationParserTest {
Assert.assertEquals(1, result.size());
Assert.assertEquals(2, result.get(0).getMessages().size());
Assert.assertEquals(outputTopic, result.get(0).getTopic());
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_field" + "\":\"a"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_metadata:a" + "\":\"string\""));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_metadata:b" + "\":1"));
@@ -301,15 +298,15 @@ public class SingleApplicationParserTest {
Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("guid" + "\":"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"test_type\""));
Assert.assertTrue(result.get(0).getMessages().get(1).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1"));
SiembolMessageFields.SENSOR_TYPE + "\":\"test_type\""));
Assert.assertTrue(result.get(0).getMessages().get(1).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_field" + "\":\"b"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("timestamp" + "\":2"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_metadata:a" + "\":\"string\""));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_metadata:b" + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_metadata:c" + "\":true"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"test_type\""));
SiembolMessageFields.SENSOR_TYPE + "\":\"test_type\""));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("guid" + "\":"));
}
@@ -330,21 +327,21 @@ public class SingleApplicationParserTest {
Assert.assertEquals(1, result.size());
Assert.assertEquals(2, result.get(0).getMessages().size());
Assert.assertEquals(outputTopic, result.get(0).getTopic());
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_field" + "\":\"a"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("a" + "\":\"string\""));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("b" + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("c" + "\":true"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"test_type\""));
Assert.assertTrue(result.get(0).getMessages().get(1).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1"));
SiembolMessageFields.SENSOR_TYPE + "\":\"test_type\""));
Assert.assertTrue(result.get(0).getMessages().get(1).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_field" + "\":\"b"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("timestamp" + "\":2"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("a" + "\":\"string\""));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("b" + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("c" + "\":true"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"test_type\""));
SiembolMessageFields.SENSOR_TYPE + "\":\"test_type\""));
}
}

10
parsing/parsing-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -45,19 +45,13 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.10.13</version>
</dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>

71
parsing/parsing-core/src/test/java/uk/co/gresearch/siembol/parsers/common/SerializableSiembolParserTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.parsers.common;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang3.SerializationUtils;
import org.junit.Assert;
import org.junit.Test;
@@ -9,43 +8,39 @@ import java.util.List;
import java.util.Map;
public class SerializableSiembolParserTest {
/**
* {
* "parser_name" : "test",
* "parser_version" : 1,
* "parser_author" : "dummy",
* "parser_attributes": {
* "parser_type": "generic"
* },
* "parser_extractors": [
* {
* "extractor_type": "json_extractor",
* "name": "test",
* "field": "original_string",
* "post_processing_functions": [
* "format_timestamp"
* ],
* "attributes": {
* "should_overwrite_fields": true,
* "should_remove_field": false,
* "time_formats": [
* {
* "time_format": "yyyy-MM-dd HH:mm:ss.SSS 'Z'"
* }
* ]
* }
* }
* ]
* }
**/
@Multiline
public static String simpleGenericParser;
private final String simpleGenericParser = """
{
"parser_name" : "test",
"parser_version" : 1,
"parser_author" : "dummy",
"parser_attributes": {
"parser_type": "generic"
},
"parser_extractors": [
{
"extractor_type": "json_extractor",
"name": "test",
"field": "original_string",
"post_processing_functions": [
"format_timestamp"
],
"attributes": {
"should_overwrite_fields": true,
"should_remove_field": false,
"time_formats": [
{
"time_format": "yyyy-MM-dd HH:mm:ss.SSS 'Z'"
}
]
}
}
]
}
""";
/**
* {"timestamp":"2019-03-27 18:52:02.732 Z"}
**/
@Multiline
public static String message;
private final String message = """
{"timestamp":"2019-03-27 18:52:02.732 Z"}
""";
@Test
public void serializableTest() throws Exception {
@@ -57,7 +52,7 @@ public class SerializableSiembolParserTest {
SerializableSiembolParser clone = SerializationUtils.clone(original);
List<Map<String, Object>> parsedClone = clone.parse(message.getBytes());
Assert.assertTrue(parsedOriginal.equals(parsedClone));
Assert.assertEquals(parsedOriginal, parsedClone);
Assert.assertEquals(1553712722732L, parsedClone.get(0).get("timestamp"));
}

53
parsing/parsing-core/src/test/java/uk/co/gresearch/siembol/parsers/extractors/CSVExtractorTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.parsers.extractors;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -8,46 +7,34 @@ import org.junit.Test;
import java.util.*;
public class CSVExtractorTest {
private String name = "test_name";
private String field = "test_field";
private final String name = "test_name";
private final String field = "test_field";
private List<ColumnNames> columnNamesList;
private EnumSet<ParserExtractor.ParserExtractorFlags> extractorFlags;
/**
* a,bb,ccc,,ee
**/
@Multiline
public static String simpleNoQuotes;
private final String simpleNoQuotes = """
a,bb,ccc,,ee
""";
/**
* a,bb,ccc,,ee,
**/
@Multiline
public static String simpleEmptyLastColumn;
private final String simpleEmptyLastColumn = """
a,bb,ccc,,ee,
""";
/**
* a||bb||ccc||||ee||
**/
@Multiline
public static String stringDelimiterEmptyLastColumn;
private final String stringDelimiterEmptyLastColumn = """
a||bb||ccc||||ee||
""";
/**
* a;bb;ccc;;ee
**/
@Multiline
public static String simpleChangedDelimiter;
private final String simpleChangedDelimiter = """
a;bb;ccc;;ee
""";
/**
* a,"b,,,b",cc""c,"","ee
**/
@Multiline
public static String simpleQuotes;
private final String simpleQuotes = """
a,"b,,,b",cc""c,"","ee
""";
/**
* a||bb||ccc||||ee
**/
@Multiline
public static String noQuotesStringDelimiter;
private final String noQuotesStringDelimiter = """
a||bb||ccc||||ee
""";
@Before
public void setUp() {

24
parsing/parsing-core/src/test/java/uk/co/gresearch/siembol/parsers/extractors/JsonExtractorTest.java
View File

@@ -1,30 +1,24 @@
package uk.co.gresearch.siembol.parsers.extractors;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Test;
import java.util.EnumSet;
import java.util.Map;
public class JsonExtractorTest {
private String name = "test_name";
private String field = "test_field";
private EnumSet<ParserExtractor.ParserExtractorFlags> extractorFlags =
private final String name = "test_name";
private final String field = "test_field";
private final EnumSet<ParserExtractor.ParserExtractorFlags> extractorFlags =
EnumSet.of(ParserExtractor.ParserExtractorFlags.SHOULD_REMOVE_FIELD);
/**
* {"key1":"bbb", "key2":2, "key3": true, "key4": {"nested1": { "neste21" : 1, "nested22" : true, "nested23" : {}, "nested24": []} }} {"ignored": "hopefully"}
**/
@Multiline
public static String simpleJson;
private final String simpleJson = """
{"key1":"bbb", "key2":2, "key3": true, "key4": {"nested1": { "neste21" : 1, "nested22" : true, "nested23" : {}, "nested24": []} }} {"ignored": "hopefully"}
""";
/**
* {"key1":"bbb", "key2": {"nested1": [{"order" : 1}, {"order" : 2}]}} {"ignored": "hopefully"}
**/
@Multiline
public static String simpleArrayJson;
private final String simpleArrayJson = """
{"key1":"bbb", "key2": {"nested1": [{"order" : 1}, {"order" : 2}]}} {"ignored": "hopefully"}
""";
@Test
public void testGoodNested() {

49
parsing/parsing-core/src/test/java/uk/co/gresearch/siembol/parsers/extractors/KeyValueExtractorTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.parsers.extractors;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -10,33 +9,27 @@ import java.util.Map;
public class KeyValueExtractorTest {
private String name = "test_name";
private String field = "test_field";
private final String name = "test_name";
private final String field = "test_field";
private EnumSet<ParserExtractor.ParserExtractorFlags> extractorFlags;
private EnumSet<KeyValueExtractor.KeyValueExtractorFlags> keyValueFlags;
/**
* Level=1 Category=UNKNOWN Type=abc
**/
@Multiline
public static String simpleNoQuotas;
/**
* Threat=Evil Level='A' Category="UN =KNOWN"
**/
@Multiline
public static String simpleQuotes;
private final String simpleNoQuotas = """
Level=1 Category=UNKNOWN Type=abc
""";
/**
* Threat|Evil,Level|'A',Category|"UN,|KNOWN"
**/
@Multiline
public static String nonStandardDelimiters;
private final String simpleQuotes = """
Threat=Evil Level='A' Category="UN =KNOWN"
""";
/**
* Threat|Evil,Level|'\'A',Category|"UN,|KN\"OWN"
**/
@Multiline
public static String nonStandartDelimitersEscaping;
private final String nonStandardDelimiters = """
Threat|Evil,Level|'A',Category|"UN,|KNOWN"
""";
private final String nonStandardDelimitersEscaping = """
Threat|Evil,Level|'\\'A',Category|"UN,|KN\\"OWN"
""";
@Before
public void setUp() {
@@ -120,7 +113,7 @@ public class KeyValueExtractorTest {
Assert.assertEquals("'A'", out.get("Level"));
}
@Test
public void testGoodNonStandartsDelimiter() {
public void testGoodNonStandardsDelimiter() {
extractorFlags.add(
ParserExtractor.ParserExtractorFlags.SHOULD_OVERWRITE_FIELDS);
keyValueFlags.add(KeyValueExtractor.KeyValueExtractorFlags.QUOTA_VALUE_HANDLING);
@@ -147,7 +140,7 @@ public class KeyValueExtractorTest {
}
@Test
public void testGoodNonStandartsDelimiterEscaping() {
public void testGoodNonStandardsDelimiterEscaping() {
extractorFlags.add(
ParserExtractor.ParserExtractorFlags.SHOULD_OVERWRITE_FIELDS);
keyValueFlags.add(KeyValueExtractor.KeyValueExtractorFlags.QUOTA_VALUE_HANDLING);
@@ -168,7 +161,7 @@ public class KeyValueExtractorTest {
Assert.assertFalse(extractor.shouldRemoveField());
Assert.assertTrue(extractor.shouldOverwiteFields());
Map<String, Object> out = extractor.extract(nonStandartDelimitersEscaping.trim());
Map<String, Object> out = extractor.extract(nonStandardDelimitersEscaping.trim());
Assert.assertEquals(3, out.size());
Assert.assertEquals("Evil", out.get("Threat"));
Assert.assertEquals("\"UN,|KN\\\"OWN\"", out.get("Category"));
@@ -176,7 +169,7 @@ public class KeyValueExtractorTest {
}
@Test
public void testGoodNonStandartsDelimiterEscapingNextKey() {
public void testGoodNonStandardsDelimiterEscapingNextKey() {
extractorFlags.add(
ParserExtractor.ParserExtractorFlags.SHOULD_OVERWRITE_FIELDS);
keyValueFlags.add(KeyValueExtractor.KeyValueExtractorFlags.QUOTA_VALUE_HANDLING);
@@ -198,7 +191,7 @@ public class KeyValueExtractorTest {
Assert.assertFalse(extractor.shouldRemoveField());
Assert.assertTrue(extractor.shouldOverwiteFields());
Map<String, Object> out = extractor.extract(nonStandartDelimitersEscaping.trim());
Map<String, Object> out = extractor.extract(nonStandardDelimitersEscaping.trim());
Assert.assertEquals(3, out.size());
Assert.assertEquals("Evil", out.get("Threat"));
Assert.assertEquals("\"UN,|KN\\\"OWN\"", out.get("Category"));

47
parsing/parsing-core/src/test/java/uk/co/gresearch/siembol/parsers/extractors/PatternExtractorTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.parsers.extractors;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -12,41 +11,25 @@ import java.util.Map;
public class PatternExtractorTest {
private String name = "test_name";
private String field = "test_field";
private final String name = "test_name";
private final String field = "test_field";
private List<String> stringPatterns;
/**
* MID (?<my_mid>\d+)
**/
@Multiline
public static String goodMid;
private final String goodMid = """
MID (?<my_mid>\\d+)""";
private final String goodVofDetail = """
Threat Level=(?<vof_threat_level>\\d) Category=(?<vof_threat_cat>\\S+) Type=(?<vof_threat_type>.*?)""";
/**
* Threat Level=(?<vof_threat_level>\d) Category=(?<vof_threat_cat>\S+) Type=(?<vof_threat_type>.*?)
**/
@Multiline
public static String goodVofDetail;
private final String vofDetailInstance = """
Threat Level=1 Category=UNKNOWN Type=a
bc""";
/**
* Threat Level=1 Category=UNKNOWN Type=a
*bc
**/
@Multiline
public static String vofDetailInstance;
private final String vofDetailInstanceWrong1 = """
Threat Level=A Category=UNKNOWN Type=abc""";
/**
* Threat Level=A Category=UNKNOWN Type=abc
**/
@Multiline
public static String vofDetailInstanceWrong1;
/**
* Threat Level=1 Category= Type=abc
**/
@Multiline
public static String vofDetailInstanceWrong2;
private final String vofDetailInstanceWrong2 = """
Threat Level=1 Category= Type=abc""";
@Before
public void setUp() {
@@ -55,7 +38,6 @@ public class PatternExtractorTest {
@Test
public void testGoodMid() {
stringPatterns.add(goodMid.trim());
PatternExtractor extractor = PatternExtractor.builder()
.patterns(stringPatterns)
@@ -82,7 +64,6 @@ public class PatternExtractorTest {
@Test
public void testGoodMid2() {
stringPatterns.add(goodMid.trim());
stringPatterns.add(".*" + goodMid.trim());
@@ -110,7 +91,6 @@ public class PatternExtractorTest {
@Test
public void testGoodVofDetail() {
stringPatterns.add(goodVofDetail.trim());
PatternExtractor extractor = PatternExtractor.builder()
.patterns(stringPatterns)
@@ -137,7 +117,6 @@ public class PatternExtractorTest {
@Test
public void testGoodVofDetail2() {
stringPatterns.add(goodVofDetail.trim());
stringPatterns.add(".*" + goodVofDetail.trim());
PatternExtractor extractor = PatternExtractor.builder()

51
parsing/parsing-core/src/test/java/uk/co/gresearch/siembol/parsers/extractors/RegexSelectExtractorTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.parsers.extractors;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang3.tuple.Pair;
import org.junit.Assert;
import org.junit.Before;
@@ -9,45 +8,29 @@ import org.junit.Test;
import java.util.*;
public class RegexSelectExtractorTest {
private String name = "test_name";
private String field = "test_field";
private final String name = "test_name";
private final String field = "test_field";
private EnumSet<ParserExtractor.ParserExtractorFlags> extractorFlags;
private List<Pair<String, String>> patterns = new ArrayList<>();
private final List<Pair<String, String>> patterns = new ArrayList<>();
/**
* node=nqptick1 type=EOE msg=audit(1526397806.509:3436363485):
**/
@Multiline
private String auditdMessage;
private String auditdRegex = "^node=";
private final String auditdMessage = """
node=nqptick1 type=EOE msg=audit(1526397806.509:3436363485):""";
private final String auditdRegex = "^node=";
/**
* <13>Jan 14 13:26:58 prod-1.k8s.abc fluentd: stream:stdout docker:{"container_id"=>"88751a072197197da7fa50987c485c04fdd7325a98831a533291ac113b558278"} kubernetes:{"container_name"=>"dummy", "namespace_name"=>"dev", "pod_name"=>"dummy", "container_image"=>"unknown.net/service:cwh10r-gb4ys-km514euza-3azyc-niutqq", "container_image_id"=>"docker-pullable://docker.artifactory.net", "pod_id"=>"416af93b-15c3-11e9-add7-48df3701a2c4", "labels"=>{"app"=>"abc", "master"=>"false", "pod-template-hash"=>"1710645932"}, "host"=>"abc", "master_url"=>"https://1.2.3.4:443/api", "namespace_id"=>"3993c1be-b01f-11e8-bee0-30e1716064fc", "namespace_labels"=>{"istio-injection"=>"disabled", "opa-validating-webhook"=>"enabled", "spooning"=>"dev"}}
**/
@Multiline
private String k8sMessage;
private String k8sRegex = "^<\\d+>\\w+\\s+\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}\\s+[\\w-\\.]+\\.k8s";
private final String k8sMessage = """
<13>Jan 14 13:26:58 prod-1.k8s.abc fluentd: stream:stdout docker:{"container_id"=>"88751a072197197da7fa50987c485c04fdd7325a98831a533291ac113b558278"} kubernetes:{"container_name"=>"dummy", "namespace_name"=>"dev", "pod_name"=>"dummy", "container_image"=>"unknown.net/service:cwh10r-gb4ys-km514euza-3azyc-niutqq", "container_image_id"=>"docker-pullable://docker.artifactory.net", "pod_id"=>"416af93b-15c3-11e9-add7-48df3701a2c4", "labels"=>{"app"=>"abc", "master"=>"false", "pod-template-hash"=>"1710645932"}, "host"=>"abc", "master_url"=>"https://1.2.3.4:443/api", "namespace_id"=>"3993c1be-b01f-11e8-bee0-30e1716064fc", "namespace_labels"=>{"istio-injection"=>"disabled", "opa-validating-webhook"=>"enabled", "spooning"=>"dev"}}""";
private final String k8sRegex = "^<\\d+>\\w+\\s+\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}\\s+[\\w-\\.]+\\.k8s";
/**
* <85>1 2019-02-18T17:37:47 10.18.9.141 CP-GW - Log [Fields@1.3.6.1.4.1.2620 Action="accept" UUid="{0x5c6aed6a,0x67,0x8d09120a,0xc0000002}" rule="313" rule_uid="{DB44DDE8-CD96-4B37-8A14-3E978D5BC322}" rule_name="RZ Splunk Fwd and Dep" src="192.168.55.11" dst="192.168.41.43" proto="6" product="VPN-1 & FireWall-1" service="9997" s_port="50172" product_family="Network"]
**/
@Multiline
private String checkpointMessage;
private String checkpointRegex = "^<\\d+>\\d+\\s+\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(.\\d{6})?([+-]\\d{2}:\\d{2})?\\s+[^\\s]+\\sCP-GW";
private final String checkpointMessage = """
<85>1 2019-02-18T17:37:47 10.18.9.141 CP-GW - Log [Fields@1.3.6.1.4.1.2620 Action="accept" UUid="{0x5c6aed6a,0x67,0x8d09120a,0xc0000002}" rule="313" rule_uid="{DB44DDE8-CD96-4B37-8A14-3E978D5BC322}" rule_name="RZ Splunk Fwd and Dep" src="192.168.55.11" dst="192.168.41.43" proto="6" product="VPN-1 & FireWall-1" service="9997" s_port="50172" product_family="Network"]""";
private final String checkpointRegex = "^<\\d+>\\d+\\s+\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(.\\d{6})?([+-]\\d{2}:\\d{2})?\\s+[^\\s]+\\sCP-GW";
/**
* CEF:0|2019-02-18T17:37:47Z|GSPARK3|Cyber-Ark|Vault|9.99.0000|295|Retrieve password|5|act="Retrieve password" duser="dummmy" fname="Root\abc.net" src="172.22.1.1" cs1Label="Affected User Name" cs1="" cs2Label="Safe Name" cs2="AD" cs3Label="Location" cs3="" cs4Label="Property Name" cs4="" cs5Label="Target User Name" cs5="" cs6Label="Gateway Address" cs6="" cn1Label="Request Id" cn1="" msg="[AppID: Switches] Getting account password for Switches", , Retrieve password
**/
@Multiline
private String cyberarkMessage;
private String cyberarkRegex = "(?i)(\\w+\\s+\\d{1,2}\\s+\\d{2}:\\d{2}:\\d{2}|.+?\\|.+?\\|)\\s*...ark\\d";
/**
* this is a simple message
**/
@Multiline
String simpleMessage;
private final String cyberarkMessage = """
CEF:0|2019-02-18T17:37:47Z|GSPARK3|Cyber-Ark|Vault|9.99.0000|295|Retrieve password|5|act="Retrieve password" duser="dummmy" fname="Root\\abc.net" src="172.22.1.1" cs1Label="Affected User Name" cs1="" cs2Label="Safe Name" cs2="AD" cs3Label="Location" cs3="" cs4Label="Property Name" cs4="" cs5Label="Target User Name" cs5="" cs6Label="Gateway Address" cs6="" cn1Label="Request Id" cn1="" msg="[AppID: Switches] Getting account password for Switches", , Retrieve password""";
private final String cyberarkRegex = "(?i)(\\w+\\s+\\d{1,2}\\s+\\d{2}:\\d{2}:\\d{2}|.+?\\|.+?\\|)\\s*...ark\\d";
private final String simpleMessage = """
this is a simple message""";
@Before
public void setUp() {

100
parsing/parsing-core/src/test/java/uk/co/gresearch/siembol/parsers/factory/ParserFactoryImplTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.parsers.factory;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Test;
import uk.co.gresearch.siembol.common.constants.SiembolMessageFields;
@@ -14,65 +13,60 @@ public class ParserFactoryImplTest {
factory = ParserFactoryImpl.createParserFactory();
}
/**
* {
* "parser_name" : "test",
* "parser_version" : 1,
* "parser_author" : "dummy",
* "parser_attributes": {
* "parser_type": "generic"
* },
* "parser_extractors": [
* {
* "extractor_type": "json_extractor",
* "name": "test",
* "field": "original_string",
* "post_processing_functions": [
* "format_timestamp"
* ],
* "attributes": {
* "should_overwrite_fields": true,
* "should_remove_field": false,
* "time_formats": [
* {
* "time_format": "yyyy-MM-dd HH:mm:ss.SSS 'Z'"
* }
* ]
* }
* }
* ],
* "transformations": [
* {
* "transformation_type": "field_name_string_replace",
* "attributes": {
* "string_replace_target": " ",
* "string_replace_replacement": "_"
* }
* }
* ]
* }
**/
@Multiline
public static String simpleGenericParser;
private final String simpleGenericParser = """
{
"parser_name" : "test",
"parser_version" : 1,
"parser_author" : "dummy",
"parser_attributes": {
"parser_type": "generic"
},
"parser_extractors": [
{
"extractor_type": "json_extractor",
"name": "test",
"field": "original_string",
"post_processing_functions": [
"format_timestamp"
],
"attributes": {
"should_overwrite_fields": true,
"should_remove_field": false,
"time_formats": [
{
"time_format": "yyyy-MM-dd HH:mm:ss.SSS 'Z'"
}
]
}
}
],
"transformations": [
{
"transformation_type": "field_name_string_replace",
"attributes": {
"string_replace_target": " ",
"string_replace_replacement": "_"
}
}
]
}
""";
/**
* {"timestamp":"2019-03-27 18:52:02.732 Z", "test field" : true, "test_field1" : " message ", "test_field2" : " message "}
**/
@Multiline
public static String message;
private final String message = """
{"timestamp":"2019-03-27 18:52:02.732 Z", "test field" : true, "test_field1" : " message ", "test_field2" : " message "}""";
@Test
public void testGetSchema() {
ParserFactoryResult schemaResult = factory.getSchema();
Assert.assertTrue(schemaResult.getStatusCode() == ParserFactoryResult.StatusCode.OK);
Assert.assertSame(ParserFactoryResult.StatusCode.OK, schemaResult.getStatusCode());
Assert.assertFalse(schemaResult.getAttributes().getJsonSchema().isEmpty());
}
@Test
public void testGoodCreate() {
ParserFactoryResult result = factory.create(simpleGenericParser);
Assert.assertTrue(result.getStatusCode() == ParserFactoryResult.StatusCode.OK);
Assert.assertTrue(result.getAttributes().getSiembolParser() != null);
Assert.assertSame(ParserFactoryResult.StatusCode.OK, result.getStatusCode());
Assert.assertNotNull(result.getAttributes().getSiembolParser());
List<Map<String, Object>> parsed = result.getAttributes().getSiembolParser().parse(message.getBytes());
Assert.assertEquals(1553712722732L, parsed.get(0).get("timestamp"));
@@ -83,20 +77,20 @@ public class ParserFactoryImplTest {
@Test
public void testInvalidCreate() {
ParserFactoryResult result = factory.create("INVALID");
Assert.assertTrue(result.getStatusCode() == ParserFactoryResult.StatusCode.ERROR);
Assert.assertTrue(result.getAttributes().getMessage() != null);
Assert.assertSame(ParserFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertNotNull(result.getAttributes().getMessage());
}
@Test
public void testValidationGood() {
ParserFactoryResult result = factory.validateConfiguration(simpleGenericParser);
Assert.assertTrue(result.getStatusCode() == ParserFactoryResult.StatusCode.OK);
Assert.assertSame(ParserFactoryResult.StatusCode.OK, result.getStatusCode());
}
@Test
public void testTestingGood() {
ParserFactoryResult result = factory.test(simpleGenericParser, null, message.getBytes());
Assert.assertTrue(result.getStatusCode() == ParserFactoryResult.StatusCode.OK);
Assert.assertSame(result.getStatusCode(), ParserFactoryResult.StatusCode.OK);
List<Map<String, Object>> parsed = result.getAttributes().getParserResult().getParsedMessages();
Assert.assertEquals(1553712722732L, parsed.get(0).get("timestamp"));
Assert.assertEquals(true, parsed.get(0).get("test_field"));

131
parsing/parsing-core/src/test/java/uk/co/gresearch/siembol/parsers/generic/SiembolGenericParserTest.java
View File

@@ -1,5 +1,4 @@
package uk.co.gresearch.siembol.parsers.generic;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -14,77 +13,69 @@ public class SiembolGenericParserTest {
private SiembolParser genericParser;
private ParserFactory factory;
/**
* {
* "parser_attributes": {
* "parser_type": "generic"
* },
* "parser_extractors" : [
* {
* "extractor_type": "pattern_extractor",
* "name": "simple_message",
* "field": "original_string",
* "attributes": {
* "regular_expressions": [
* "^msg:\\s(?<secret_msg>.*)$",
* "^msg2:\\s(?<timestamp>.*)$"
* ],
* "should_remove_field" : false
* }
* }],
* "transformations" : [
* {
* "transformation_type": "field_name_string_replace_all",
* "attributes": {
* "string_replace_target": "secret_msg",
* "string_replace_replacement": "dummy"
* }
* }]
* }
**/
@Multiline
public static String simpleGenericParserConfig;
private final String simpleGenericParserConfig = """
{
"parser_attributes": {
"parser_type": "generic"
},
"parser_extractors" : [
{
"extractor_type": "pattern_extractor",
"name": "simple_message",
"field": "original_string",
"attributes": {
"regular_expressions": [
"^msg:\\\\s(?<secret_msg>.*)$",
"^msg2:\\\\s(?<timestamp>.*)$"
],
"should_remove_field" : false
}
}],
"transformations" : [
{
"transformation_type": "field_name_string_replace_all",
"attributes": {
"string_replace_target": "secret_msg",
"string_replace_replacement": "dummy"
}
}]
}
""";
/**
* {
* "parser_attributes": {
* "parser_type": "generic"
* },
* "parser_extractors" : [
* {
* "extractor_type": "pattern_extractor",
* "name": "simple_message",
* "field": "original_string",
* "attributes": {
* "regular_expressions": [
* "^msg:\\s(?<secret_msg>.*)$"
* ],
* "should_remove_field" : false
* }
* }],
* "transformations" : [
* {
* "transformation_type": "filter_message",
* "attributes": {
* "message_filter" : {
* "matchers" : [
* {
* "field_name" : "secret_msg",
* "pattern" : "secret",
* "negated" : false
* }]
* }}}]
* }
**/
@Multiline
public static String simpleGenericParserFiltered;
/**
* msg: secret
**/
@Multiline
public static String simpleMessage;
private final String simpleGenericParserFiltered = """
{
"parser_attributes": {
"parser_type": "generic"
},
"parser_extractors" : [
{
"extractor_type": "pattern_extractor",
"name": "simple_message",
"field": "original_string",
"attributes": {
"regular_expressions": [
"^msg:\\\\s(?<secret_msg>.*)$"
],
"should_remove_field" : false
}
}],
"transformations" : [
{
"transformation_type": "filter_message",
"attributes": {
"message_filter" : {
"matchers" : [
{
"field_name" : "secret_msg",
"pattern" : "secret",
"negated" : false
}]
}}}]
}
""";
private final String simpleMessage = """
msg: secret""";
@Before
public void setUp() throws Exception {

283
parsing/parsing-core/src/test/java/uk/co/gresearch/siembol/parsers/syslog/BasicSyslogSiembolParserTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.parsers.syslog;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -26,163 +25,124 @@ public class BasicSyslogSiembolParserTest {
private SiembolParser syslogBsdParser;
private ParserFactory factory;
/**
* {
* "parser_attributes": {
* "parser_type": "syslog",
* "syslog_config": {
* "syslog_version": "RFC_5424",
* "timezone": "UTC"
* }
* }
* }
**/
@Multiline
public static String syslogConfigRfc_5424;
private final String syslogConfigRfc5424 = """
{
"parser_attributes": {
"parser_type": "syslog",
"syslog_config": {
"syslog_version": "RFC_5424",
"timezone": "UTC"
}
}
}
""";
/**
* {
* "parser_attributes": {
* "parser_type": "syslog",
* "syslog_config": {
* "syslog_version": "RFC_5424",
* "timezone": "UTC",
* "merge_sd_elements" : true
* }
* }
* }
**/
@Multiline
public static String syslogConfigRfc_5424MergeSdElements;
private final String syslogConfigRfc5424MergeSdElements = """
{
"parser_attributes": {
"parser_type": "syslog",
"syslog_config": {
"syslog_version": "RFC_5424",
"timezone": "UTC",
"merge_sd_elements" : true
}
}
}
""";
/**
* {
* "parser_attributes": {
* "parser_type": "syslog",
* "syslog_config": {
* "syslog_version": "RFC_3164",
* "timezone": "UTC"
* }
* }
* }
**/
@Multiline
public static String syslogConfigBsd;
private final String syslogConfigBsd = """
{
"parser_attributes": {
"parser_type": "syslog",
"syslog_config": {
"syslog_version": "RFC_3164",
"timezone": "UTC"
}
}
}
""";
/**
* {
* "parser_attributes": {
* "parser_type": "syslog",
* "syslog_config": {
* "syslog_version": "RFC_3164",
* "timezone": "Europe/London"
* }
* }
* }
**/
@Multiline
public static String syslogConfigBsdLondonTimezone;
private final String syslogConfigBsdLondonTimezone = """
{
"parser_attributes": {
"parser_type": "syslog",
"syslog_config": {
"syslog_version": "RFC_3164",
"timezone": "Europe/London"
}
}
}
""";
/**
* {
* "parser_attributes": {
* "parser_type": "syslog",
* "syslog_config": {
* "syslog_version": "RFC_5424",
* "time_formats": [
* {
* "timezone": "UTC",
* "time_format": "yyyy-MM-dd'T'HH:mm:ss'Z'"
* }]
* }
* },
* "parser_extractors": [
* {
* "extractor_type": "pattern_extractor",
* "name": "dummy",
* "field": "syslog_msg",
* "attributes": {
* "regular_expressions": [
* "^SALscanner INFO TEST:\\s(?<info_msg>.*)$"
* ],
* "should_remove_field": false
* }
* }
* ],
* "transformations" : [
* {
* "transformation_type": "field_name_string_replace_all",
* "attributes": {
* "string_replace_target": "syslog",
* "string_replace_replacement": "dummy"
* }
* }]
* }
**/
@Multiline
public static String syslogConfigCustomTimpestamp;
private final String syslogConfigCustomTimestamp = """
{
"parser_attributes": {
"parser_type": "syslog",
"syslog_config": {
"syslog_version": "RFC_5424",
"time_formats": [
{
"timezone": "UTC",
"time_format": "yyyy-MM-dd'T'HH:mm:ss'Z'"
}]
}
},
"parser_extractors": [
{
"extractor_type": "pattern_extractor",
"name": "dummy",
"field": "syslog_msg",
"attributes": {
"regular_expressions": [
"^SALscanner INFO TEST:\\\\s(?<info_msg>.*)$"
],
"should_remove_field": false
}
}
],
"transformations" : [
{
"transformation_type": "field_name_string_replace_all",
"attributes": {
"string_replace_target": "syslog",
"string_replace_replacement": "dummy"
}
}]
}
""";
/**
* <85>1 2018-05-22T17:07:41+01:00 172.16.18.101 CP-GW - Log [Fields@1.3.6.1.4.1.2620 Action="accept" UUid="{0x5b04404c,0x10004,0x651210ac,0xc0000000}" rule="14" rule_uid="{28F2CB68-9017-442B-8C64-6BD43B8082CD}" rule_name="DNS" src="172.16.16.20" dst="172.16.37.100" proto="17" product="VPN-1 & FireWall-1" service="53" s_port="60349" product_family="Network"]
**/
@Multiline
public static String goodSyslogCheckpoint1;
private final String goodSyslogCheckpoint1 = """
<85>1 2018-05-22T17:07:41+01:00 172.16.18.101 CP-GW - Log [Fields@1.3.6.1.4.1.2620 Action="accept" UUid="{0x5b04404c,0x10004,0x651210ac,0xc0000000}" rule="14" rule_uid="{28F2CB68-9017-442B-8C64-6BD43B8082CD}" rule_name="DNS" src="172.16.16.20" dst="172.16.37.100" proto="17" product="VPN-1 & FireWall-1" service="53" s_port="60349" product_family="Network"]""";
/**
* <81>1 2018-05-22T03:05:37 172.19.34.31 CP-GW - Alert [Fields@1.3.6.1.4.1.2620 Action=" " UUid="{0x0,0x0,0x0,0x0}" Protection Name="Packet Sanity" Severity="2" Confidence Level="5" protection_id="PacketSanity" SmartDefense Profile="Perimeter_Protection" Performance Impact="1" Industry Reference="CAN-2002-1071" Protection Type="anomaly" Attack Info="Invalid TCP flag combination" attack="Malformed Packet" Total logs="24" Suppressed logs="23" proto="6" dst="10.254.101.253" src="10.254.101.12" product="SmartDefense" FollowUp="Not Followed" product_family="Network"]
**/
@Multiline
public static String goodSyslogCheckpoint2;
private final String goodSyslogCheckpoint2 = """
<81>1 2018-05-22T03:05:37 172.19.34.31 CP-GW - Alert [Fields@1.3.6.1.4.1.2620 Action=" " UUid="{0x0,0x0,0x0,0x0}" Protection Name="Packet Sanity" Severity="2" Confidence Level="5" protection_id="PacketSanity" SmartDefense Profile="Perimeter_Protection" Performance Impact="1" Industry Reference="CAN-2002-1071" Protection Type="anomaly" Attack Info="Invalid TCP flag combination" attack="Malformed Packet" Total logs="24" Suppressed logs="23" proto="6" dst="10.254.101.253" src="10.254.101.12" product="SmartDefense" FollowUp="Not Followed" product_family="Network"]""";
private final String goodSyslogEscapedChars = """
<81>1 2018-05-22T03:05:37 172.19.34.31 CP-GW - Alert [Fields@1.3.6.1.4.1.2620 Action=" " UUid="{0x0,0x0,0x0,0x0}" Protection Name="Packet\\" \\] Sanity"]""";
private final String syslogEscapedChars2 = """
<81>1 2018-05-22T03:05:37 172.19.34.31 CP-GW - Alert [Fields@1.3.6.1.4.1.2620 Action=" " Protection Name="Packet" \\] Sanity"][Fields@1.3.6.1.4.1.2620] BOMabcabc""";
private final String goodNilSD = """
<81>1 2018-05-22T03:05:37 172.19.34.31 CP-GW - Alert - BOMabcabc""";
private final String goodBSD = """
<34>Oct 11 22:14:15 mymachine su: 'su root' failed for dummy on /dev/pts/8""";
private final String strangeCheckpoint = """
<85>1 2018-08-01T09:00:24+01:00 10.254.112.76 CP-GW - Log [Fields@1.3.6.1.4.1.2620 Action=" " UUid="{0x0,0x0,0x0,0x0}" default_device_message="<133>xpand[17978]: admin localhost t +installer:packages:Check_Point_R77_30_JUMBO_HF_1_Bundle_T286_FULL.tgz:has_metadata 0 (+)" facility="local use 0" syslog_severity="Notice" product="Syslog" product_family="Network"]""";
/**
* <81>1 2018-05-22T03:05:37 172.19.34.31 CP-GW - Alert [Fields@1.3.6.1.4.1.2620 Action=" " UUid="{0x0,0x0,0x0,0x0}" Protection Name="Packet\" \] Sanity"]
**/
@Multiline
public static String goodSyslogEscapedChars;
private final String customTimeformat = """
<190>1 2019-01-15T12:36:05Z mime1-private.internal.net sal - - - SALscanner INFO TEST: [manlistEmail] applianceupdate.clearswift.com ... [688]""";
/**
* <81>1 2018-05-22T03:05:37 172.19.34.31 CP-GW - Alert [Fields@1.3.6.1.4.1.2620 Action=" " Protection Name="Packet" \] Sanity"][Fields@1.3.6.1.4.1.2620] BOMabcabc
**/
@Multiline
public static String syslogEscapedChars2;
/**
* <81>1 2018-05-22T03:05:37 172.19.34.31 CP-GW - Alert - BOMabcabc
**/
@Multiline
public static String goodNilSD;
/**
* <34>Oct 11 22:14:15 mymachine su: 'su root' failed for dummy on /dev/pts/8
**/
@Multiline
public static String goodBSD;
/**
* <85>1 2018-08-01T09:00:24+01:00 10.254.112.76 CP-GW - Log [Fields@1.3.6.1.4.1.2620 Action=" " UUid="{0x0,0x0,0x0,0x0}" default_device_message="<133>xpand[17978]: admin localhost t +installer:packages:Check_Point_R77_30_JUMBO_HF_1_Bundle_T286_FULL.tgz:has_metadata 0 (+)" facility="local use 0" syslog_severity="Notice" product="Syslog" product_family="Network"]
**/
@Multiline
public static String strangeCheckpoint;
private final String multipleSdElementsDummyCheckpoint1 = """
<85>1 2018-05-22T17:07:41+01:00 172.16.18.101 CP-GW - Log [Fields@1.3.6.1.4.1.2620 Action1="accept"][Fields@1.3.6.1.4.1.2620 Action2="deny"]""";
/**
* <190>1 2019-01-15T12:36:05Z mime1-eqld.uberit.net sal - - - SALscanner INFO TEST: [manlistEmail] applianceupdate.clearswift.com ... [688]
*/
@Multiline
public static String customTimeformat;
private final String multipleSdElementsDummyCheckpoint2 = """
<85>1 2018-05-22T17:07:41Z 172.16.18.101 CP-GW - Log [Fields@1.3.6.1.4.1.2620 syslog1="accept"][Fields@1.3.6.1.4.1.2620 syslog2="deny"]""";
/**
* <85>1 2018-05-22T17:07:41+01:00 172.16.18.101 CP-GW - Log [Fields@1.3.6.1.4.1.2620 Action1="accept"][Fields@1.3.6.1.4.1.2620 Action2="deny"]
**/
@Multiline
public static String multipleSdElementsDummyCheckpoint1;
/**
* <85>1 2018-05-22T17:07:41Z 172.16.18.101 CP-GW - Log [Fields@1.3.6.1.4.1.2620 syslog1="accept"][Fields@1.3.6.1.4.1.2620 syslog2="deny"]
**/
@Multiline
public static String multipleSdElementsDummyCheckpoint2;
public BasicSyslogSiembolParserTest(Locale locale) {
Locale.setDefault(locale);
@@ -191,7 +151,7 @@ public class BasicSyslogSiembolParserTest {
@Before
public void setUp() throws Exception {
factory = ParserFactoryImpl.createParserFactory();
syslogParser = factory.create(syslogConfigRfc_5424).getAttributes().getSiembolParser();
syslogParser = factory.create(syslogConfigRfc5424).getAttributes().getSiembolParser();
syslogBsdParser = factory.create(syslogConfigBsd).getAttributes().getSiembolParser();
}
@@ -299,7 +259,7 @@ public class BasicSyslogSiembolParserTest {
}
@Test
public void goodBSD() throws Exception {
public void goodBSD() {
Map<String, Object> out = syslogBsdParser.parse(goodBSD.trim().getBytes()).get(0);
Assert.assertEquals(0, out.get("syslog_version"));
@@ -312,7 +272,7 @@ public class BasicSyslogSiembolParserTest {
}
@Test
public void goodBSDWithBSDTZ() throws Exception {
public void goodBSDWithBSDTZ() {
syslogBsdParser = factory.create(syslogConfigBsdLondonTimezone.trim()).getAttributes().getSiembolParser();
Map<String, Object> out = syslogBsdParser.parse(goodBSD.trim().getBytes()).get(0);
@@ -325,7 +285,7 @@ public class BasicSyslogSiembolParserTest {
}
@Test
public void goodBSDWithUTCTZ() throws Exception {
public void goodBSDWithUTCTZ() {
Map<String, Object> out = syslogBsdParser.parse(goodBSD.trim().getBytes()).get(0);
Assert.assertEquals(0, out.get("syslog_version"));
@@ -348,12 +308,12 @@ public class BasicSyslogSiembolParserTest {
}
@Test
public void customTimestamp() throws Exception {
syslogParser = factory.create(syslogConfigCustomTimpestamp).getAttributes().getSiembolParser();
public void customTimestamp() {
syslogParser = factory.create(syslogConfigCustomTimestamp).getAttributes().getSiembolParser();
Map<String, Object> out = syslogParser.parse(customTimeformat.trim().getBytes()).get(0);
Assert.assertEquals(1547555765000L, out.get("timestamp"));
Assert.assertEquals("mime1-eqld.uberit.net", out.get("dummy_hostname"));
Assert.assertEquals("mime1-private.internal.net", out.get("dummy_hostname"));
Assert.assertEquals("sal", out.get("dummy_appname"));
Assert.assertEquals(1, out.get("dummy_version"));
Assert.assertEquals(23, out.get("dummy_facility"));
@@ -363,8 +323,8 @@ public class BasicSyslogSiembolParserTest {
}
@Test
public void customTimestampInvalid() throws Exception {
syslogParser = factory.create(syslogConfigCustomTimpestamp).getAttributes().getSiembolParser();
public void customTimestampInvalid() {
syslogParser = factory.create(syslogConfigCustomTimestamp).getAttributes().getSiembolParser();
Map<String, Object> out = syslogParser.parse(customTimeformat.trim()
.replace("2019-01-15T12:36:05Z", "INVALID")
.getBytes())
@@ -372,7 +332,7 @@ public class BasicSyslogSiembolParserTest {
Assert.assertEquals("INVALID", out.get("dummy_timestamp"));
Assert.assertNotNull(out.get("timestamp"));
Assert.assertEquals("mime1-eqld.uberit.net", out.get("dummy_hostname"));
Assert.assertEquals("mime1-private.internal.net", out.get("dummy_hostname"));
Assert.assertEquals("sal", out.get("dummy_appname"));
Assert.assertEquals(1, out.get("dummy_version"));
Assert.assertEquals(23, out.get("dummy_facility"));
@@ -382,8 +342,8 @@ public class BasicSyslogSiembolParserTest {
}
@Test
public void mergingSdparameters(){
syslogParser = factory.create(syslogConfigRfc_5424MergeSdElements).getAttributes().getSiembolParser();
public void mergingSdParameters() {
syslogParser = factory.create(syslogConfigRfc5424MergeSdElements).getAttributes().getSiembolParser();
List<Map<String, Object>> out = syslogParser.parse(multipleSdElementsDummyCheckpoint1.trim().getBytes());
Assert.assertEquals(1, out.size());
Map<String, Object> current = out.get(0);
@@ -394,7 +354,7 @@ public class BasicSyslogSiembolParserTest {
}
@Test
public void noMergeSdparameters(){
public void noMergeSdParameters() {
List<Map<String, Object>> out = syslogParser.parse(multipleSdElementsDummyCheckpoint1.trim().getBytes());
Assert.assertEquals(2, out.size());
Map<String, Object> current1 = out.get(0);
@@ -407,8 +367,8 @@ public class BasicSyslogSiembolParserTest {
}
@Test
public void noMergeSdparametersExtractAndTransform(){
syslogParser = factory.create(syslogConfigCustomTimpestamp).getAttributes().getSiembolParser();
public void noMergeSdParametersExtractAndTransform(){
syslogParser = factory.create(syslogConfigCustomTimestamp).getAttributes().getSiembolParser();
List<Map<String, Object>> out = syslogParser.parse(multipleSdElementsDummyCheckpoint2.trim().getBytes());
Assert.assertEquals(2, out.size());
Map<String, Object> current1 = out.get(0);
@@ -420,4 +380,3 @@ public class BasicSyslogSiembolParserTest {
Assert.assertEquals("Fields@1.3.6.1.4.1.2620", current2.get("dummy_sd_id"));
}
}

297
parsing/parsing-core/src/test/java/uk/co/gresearch/siembol/parsers/transformations/TransformationsTest.java
View File

@@ -3,7 +3,6 @@ package uk.co.gresearch.siembol.parsers.transformations;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -29,181 +28,153 @@ public class TransformationsTest {
log = JSON_LOG_READER.readValue(message);
}
/**
* {
* "transformation_type": "field_name_string_replace",
* "attributes": {
* "string_replace_target": " ",
* "string_replace_replacement": "_"
* }
* }
**/
@Multiline
public static String transformationReplace;
private final String transformationReplace = """
{
"transformation_type": "field_name_string_replace",
"attributes": {
"string_replace_target": " ",
"string_replace_replacement": "_"
}
}
""";
/**
* {
* "transformation_type": "filter_message",
* "attributes": {
* "message_filter" : {
* "matchers" : [
* {
* "field_name" : "dummy field",
* "pattern" : "abc",
* "negated" : false
* },
* {
* "field_name" : "secret_field",
* "pattern" : "secret",
* "negated" : true
* }
* ]
* }
* }
* }
**/
@Multiline
public static String transformationFilter;
private final String transformationFilter = """
{
"transformation_type": "filter_message",
"attributes": {
"message_filter" : {
"matchers" : [
{
"field_name" : "dummy field",
"pattern" : "abc",
"negated" : false
},
{
"field_name" : "secret_field",
"pattern" : "secret",
"negated" : true
}
]
}
}
}
""";
/**
* {
* "transformation_type": "field_name_string_replace_all",
* "attributes": {
* "string_replace_target": " ",
* "string_replace_replacement": "_"
* }
* }
**/
@Multiline
public static String transformationReplaceAll;
private final String transformationReplaceAll = """
{
"transformation_type": "field_name_string_replace_all",
"attributes": {
"string_replace_target": " ",
"string_replace_replacement": "_"
}
}
""";
/**
* {
* "transformation_type": "field_name_string_delete_all",
* "attributes": {
* "string_replace_target": " "
* }
* }
**/
@Multiline
public static String transformationDeleteAll;
private final String transformationDeleteAll = """
{
"transformation_type": "field_name_string_delete_all",
"attributes": {
"string_replace_target": " "
}
}
""";
/**
*{
* "transformation_type": "trim_value",
* "attributes": {
* "fields_filter": {
* "including_fields": ["timestamp", "trim_field"]
* }
* }
* }
**/
@Multiline
public static String transformationTrim;
/**
*{
* "transformation_type": "chomp_value",
* "attributes": {
* "fields_filter": {
* "including_fields": ["timestamp", "chomp_field"]
* }
* }
* }
**/
@Multiline
public static String transformationChomp;
private final String transformationTrim = """
{
"transformation_type": "trim_value",
"attributes": {
"fields_filter": {
"including_fields": ["timestamp", "trim_field"]
}
}
}
""";
/**
*{
* "transformation_type": "delete_fields",
* "attributes": {
* "fields_filter": {
* "including_fields": [".*"],
* "excluding_fields": ["timestamp"]
* }
* }
* }
**/
@Multiline
public static String transformationDelete;
private final String transformationChomp = """
{
"transformation_type": "chomp_value",
"attributes": {
"fields_filter": {
"including_fields": ["timestamp", "chomp_field"]
}
}
}
""";
/**
*{
* "transformation_type": "rename_fields",
* "attributes": {
* "field_rename_map": [
* {
* "field_to_rename": "timestamp",
* "new_name": "timestamp_renamed"
* },
* {
* "field_to_rename": "dummy field",
* "new_name": "dummy_field_renamed"
* }
* ]
* }
* }
**/
@Multiline
public static String transformationRename;
private final String transformationDelete = """
{
"transformation_type": "delete_fields",
"attributes": {
"fields_filter": {
"including_fields": [".*"],
"excluding_fields": ["timestamp"]
}
}
}
""";
/**
*{
* "transformation_type": "field_name_change_case",
* "attributes": {
* "case_type" : "uppercase"
* }
* }
**/
@Multiline
public static String transformationFieldNameUpperCase;
private final String transformationRename = """
{
"transformation_type": "rename_fields",
"attributes": {
"field_rename_map": [
{
"field_to_rename": "timestamp",
"new_name": "timestamp_renamed"
},
{
"field_to_rename": "dummy field",
"new_name": "dummy_field_renamed"
}
]
}
}
""";
/**
*{
* "transformation_type": "field_name_change_case",
* "attributes": {
* "case_type" : "lowercase"
* }
* }
**/
@Multiline
public static String transformationFieldLowerCase;
private final String transformationFieldNameUpperCase = """
{
"transformation_type": "field_name_change_case",
"attributes": {
"case_type" : "uppercase"
}
}
""";
/**
*{
* "transformation_type": "lowercase_value",
* "attributes": {
* "fields_filter": {
* "including_fields": ["timestamp", "chomp_field"]
* }
* }
* }
**/
@Multiline
public static String transformationLowerCase;
private final String transformationFieldLowerCase = """
{
"transformation_type": "field_name_change_case",
"attributes": {
"case_type" : "lowercase"
}
}
""";
/**
*{
* "transformation_type": "uppercase_value",
* "attributes": {
* "fields_filter": {
* "including_fields": ["timestamp", "chomp_field"]
* }
* }
* }
**/
@Multiline
public static String transformationUpperCase;
private final String transformationLowerCase = """
{
"transformation_type": "lowercase_value",
"attributes": {
"fields_filter": {
"including_fields": ["timestamp", "chomp_field"]
}
}
}
""";
private final String transformationUpperCase = """
{
"transformation_type": "uppercase_value",
"attributes": {
"fields_filter": {
"including_fields": ["timestamp", "chomp_field"]
}
}
}
""";
/**
* {"timestamp":12345, "test field a" : "true", "trim_field" : " message ", "dummy field" : "abc", "chomp_field" : "message\n"}
**/
@Multiline
public static String message;
private final String message = """
{"timestamp":12345, "test field a" : "true", "trim_field" : " message ", "dummy field" : "abc", "chomp_field" : "message\\n"}
""";
@Test
public void testGoodReplace() throws IOException {

27
parsing/parsing-storm/pom.xml
View File

@@ -9,9 +9,14 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>23.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
@@ -62,7 +67,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -84,12 +89,6 @@
<version>${junit_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
@@ -227,18 +226,6 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<forceJavacCompilerUse>true</forceJavacCompilerUse>
<source>${java_version}</source>
<compilerArgument>-Xlint:unchecked</compilerArgument>
<target>${java_version}</target>
<showWarnings>true</showWarnings>
</configuration>
</plugin>
</plugins>
</build>
</project>

104
parsing/parsing-storm/src/test/java/uk/co/gresearch/siembol/parsers/storm/ParsingApplicationBoltTest.java
View File

@@ -3,7 +3,6 @@ package uk.co.gresearch.siembol.parsers.storm;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
@@ -30,66 +29,59 @@ import static org.mockito.Mockito.when;
public class ParsingApplicationBoltTest {
private static ObjectReader JSON_READER = new ObjectMapper()
.readerFor(new TypeReference<Map<String, Object>>() {});
/**
*RAW_LOG
**/
@Multiline
public static String log;
String log = """
RAW_LOG
""";
/**
* {"is_metadata" : true}
**/
@Multiline
public static String metadata;
public static String metadata = """
{"is_metadata" : true}
""";
/**
*{
* "parsing_app_name": "test",
* "parsing_app_version": 1,
* "parsing_app_author": "dummy",
* "parsing_app_description": "Description of parser application",
* "parsing_app_settings": {
* "input_topics": [
* "secret"
* ],
* "parse_metadata" : false,
* "error_topic": "error",
* "input_parallelism": 1,
* "parsing_parallelism": 2,
* "output_parallelism": 3,
* "parsing_app_type": "single_parser"
* },
* "parsing_settings": {
* "single_parser": {
* "parser_name": "single",
* "output_topic": "output"
* }
* }
* }
**/
@Multiline
public static String simpleSingleApplicationParser;
/**
* {
* "parsers_version": 1,
* "parsers_configurations": [
* {
* "parser_description": "for testing single app parser",
* "parser_version": 2,
* "parser_name": "single",
* "parser_author": "dummy",
* "parser_attributes": {
* "parser_type": "generic"
* }
* }
* ]
* }
**/
@Multiline
public static String testParsersConfigs;
public static String simpleSingleApplicationParser = """
{
"parsing_app_name": "test",
"parsing_app_version": 1,
"parsing_app_author": "dummy",
"parsing_app_description": "Description of parser application",
"parsing_app_settings": {
"input_topics": [
"secret"
],
"parse_metadata" : false,
"error_topic": "error",
"input_parallelism": 1,
"parsing_parallelism": 2,
"output_parallelism": 3,
"parsing_app_type": "single_parser"
},
"parsing_settings": {
"single_parser": {
"parser_name": "single",
"output_topic": "output"
}
}
}
""";
public static String testParsersConfigs = """
{
"parsers_version": 1,
"parsers_configurations": [
{
"parser_description": "for testing single app parser",
"parser_version": 2,
"parser_name": "single",
"parser_author": "dummy",
"parser_attributes": {
"parser_type": "generic"
}
}
]
}
""";
private Tuple tuple;
private OutputCollector collector;
ParsingApplicationBolt parsingApplicationBolt;

150
parsing/parsing-storm/src/test/java/uk/co/gresearch/siembol/parsers/storm/StormParsingApplicationTest.java
View File

@@ -4,7 +4,6 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.charithe.kafka.EphemeralKafkaBroker;
import com.github.charithe.kafka.KafkaJunitRule;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.generated.StormTopology;
@@ -26,88 +25,81 @@ import static org.mockito.Mockito.when;
import static org.mockito.Mockito.withSettings;
public class StormParsingApplicationTest {
/**
*RAW_LOG
**/
@Multiline
public static String log;
public static String log = """
RAW_LOG
""";
/**
*{
* "parsing_app_name": "test",
* "parsing_app_version": 1,
* "parsing_app_author": "dummy",
* "parsing_app_description": "Description of parser application",
* "parsing_app_settings": {
* "input_topics": [
* "input"
* ],
* "parse_metadata" : false,
* "error_topic": "error",
* "input_parallelism": 1,
* "parsing_parallelism": 1,
* "output_parallelism": 1,
* "parsing_app_type": "single_parser"
* },
* "parsing_settings": {
* "single_parser": {
* "parser_name": "single",
* "output_topic": "output"
* }
* }
* }
**/
@Multiline
public static String simpleSingleApplicationParser = """
{
"parsing_app_name": "test",
"parsing_app_version": 1,
"parsing_app_author": "dummy",
"parsing_app_description": "Description of parser application",
"parsing_app_settings": {
"input_topics": [
"input"
],
"parse_metadata" : false,
"error_topic": "error",
"input_parallelism": 1,
"parsing_parallelism": 1,
"output_parallelism": 1,
"parsing_app_type": "single_parser"
},
"parsing_settings": {
"single_parser": {
"parser_name": "single",
"output_topic": "output"
}
}
}
""";
public static String simpleSingleApplicationParser;
/**
* {
* "parsers_version": 1,
* "parsers_configurations": [
* {
* "parser_description": "for testing single app parser",
* "parser_version": 2,
* "parser_name": "single",
* "parser_author": "dummy",
* "parser_attributes": {
* "parser_type": "generic"
* }
* }
* ]
* }
**/
@Multiline
public static String testParsersConfigs;
public static String testParsersConfigs = """
{
"parsers_version": 1,
"parsers_configurations": [
{
"parser_description": "for testing single app parser",
"parser_version": 2,
"parser_name": "single",
"parser_author": "dummy",
"parser_attributes": {
"parser_type": "generic"
}
}
]
}
""";
/**
* {
* "client.id.prefix": "test_writer",
* "group.id.prefix": "test_reader",
* "zookeeper.attributes": {
* "zk.path": "/parserconfigs",
* "zk.base.sleep.ms": 1000,
* "zk.max.retries": 10
* },
* "kafka.batch.writer.attributes": {
* "batch.size": 1,
* "producer.properties": {
* "security.protocol": "PLAINTEXT"
* }
* },
* "storm.attributes": {
* "first.pool.offset.strategy": "EARLIEST",
* "kafka.spout.properties": {
* "security.protocol": "PLAINTEXT"
* },
* "storm.config": {
* "session.timeout.ms": 100000
* }
* }
* }
**/
@Multiline
public static String stormSettings;
public static String stormSettings = """
{
"client.id.prefix": "test_writer",
"group.id.prefix": "test_reader",
"zookeeper.attributes": {
"zk.path": "/parserconfigs",
"zk.base.sleep.ms": 1000,
"zk.max.retries": 10
},
"kafka.batch.writer.attributes": {
"batch.size": 1,
"producer.properties": {
"security.protocol": "PLAINTEXT"
}
},
"storm.attributes": {
"first.pool.offset.strategy": "EARLIEST",
"kafka.spout.properties": {
"security.protocol": "PLAINTEXT"
},
"storm.config": {
"session.timeout.ms": 100000
}
}
}
""";
@ClassRule
public static KafkaJunitRule kafkaRule = new KafkaJunitRule(EphemeralKafkaBroker.create());

2
parsing/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</parent>
<modules>
<module>parsing-core</module>

25
pom.xml
View File

@@ -6,7 +6,7 @@
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<name>siembol</name>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
<description>A scalable, advanced security analytics framework based on open-source big data technologies.</description>
<inceptionYear>2019</inceptionYear>
<url>https://siembol.io/</url>
@@ -36,9 +36,9 @@
</issueManagement>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<java_version>1.8</java_version>
<java_test_version>13</java_test_version>
<maven.compiler.release>11</maven.compiler.release>
<maven_compiler_version>3.8.1</maven_compiler_version>
<jackson_version>2.13.0</jackson_version>
<junit_version>4.13.2</junit_version>
<mockito_version>4.0.0</mockito_version>
@@ -46,7 +46,7 @@
<curator_version>4.3.0</curator_version>
<curator_test_version>2.13.0</curator_test_version>
<zookeeper_version>3.4.6</zookeeper_version>
<storm_version>1.2.1</storm_version>
<storm_version>2.3.0</storm_version>
<kafka_storm_version>0.10.2.2</kafka_storm_version>
<kafka_version>2.8.0</kafka_version>
<kafka_junit_version>3.1.1</kafka_junit_version>
@@ -112,17 +112,24 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<version>${maven_compiler_version}</version>
<configuration>
<forceJavacCompilerUse>true</forceJavacCompilerUse>
<source>${java_version}</source>
<compilerArgs>
<arg>-Xlint:all</arg>
<arg>-Xlint:-processing</arg>
</compilerArgs>
<target>${java_version}</target>
<showWarnings>true</showWarnings>
</configuration>
<executions>
<execution>
<id>default-testCompile</id>
<configuration>
<release>${java_test_version}</release>
<compilerArgs>--enable-preview</compilerArgs>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -138,6 +145,8 @@
<configuration>
<reuseForks>true</reuseForks>
<forkedProcessExitTimeoutInSeconds>60</forkedProcessExitTimeoutInSeconds>
<argLine>--illegal-access=permit</argLine>
<argLine>--enable-preview</argLine>
</configuration>
</plugin>
<plugin>

2
responding/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</parent>
<modules>
<module>responding-core</module>

12
responding/responding-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -35,24 +35,18 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>1.4.2-SNAPSHOT</version>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId>
<version>2.6.0</version>
</dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>

Some files were not shown because too many files have changed in this diff Show More