scottk/siembol
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/siembol.git synced 2025-12-07 20:45:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Upgrading java to java 11 (java 13 in tests) (#405)

Browse Source 
Upgrading Storm to 2.3.0
Supporting downloading http enrichment tables
Remove multiline string dependency
This commit is contained in:
Marian Novotny
2021-11-04 16:12:54 +00:00
committed by GitHub
parent 2628339ddb
commit 3415ce3909
131 changed files with 4013 additions and 4538 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.github/workflows/ci.yml vendored
View File

@@ -15,10 +15,11 @@ jobs:
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v2 uses: actions/checkout@v2
- name: Set up JDK 1.8 - name: Set up JDK 13
uses: actions/setup-java@v1 uses: actions/setup-java@v2
with: with:
java-version: 1.8 distribution: 'adopt'
java-version: '13'
- name: Cache Maven packages - name: Cache Maven packages
uses: actions/cache@v2 uses: actions/cache@v2
with: with:

7
.github/workflows/codeql-analysis.yml vendored
View File

@@ -65,11 +65,12 @@ jobs:
# uses a compiled language # uses a compiled language
# java build # java build
- name: Set up JDK 1.8 - name: Set up JDK 13
if: ${{ matrix.language == 'java' }} if: ${{ matrix.language == 'java' }}
uses: actions/setup-java@v1 uses: actions/setup-java@v2
with: with:
java-version: 1.8 distribution: 'adopt'
java-version: '13'
- name: Cache Maven packages - name: Cache Maven packages
if: ${{ matrix.language == 'java' }} if: ${{ matrix.language == 'java' }}
uses: actions/cache@v2 uses: actions/cache@v2

10
alerting/alerting-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent> <parent>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting</artifactId> <artifactId>alerting</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</parent> </parent>
<dependencies> <dependencies>
<dependency> <dependency>
@@ -35,13 +35,7 @@
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId> <artifactId>siembol-common</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>junit</groupId> <groupId>junit</groupId>

244
alerting/alerting-core/src/test/java/uk/co/gresearch/siembol/alerts/compiler/AlertingRulesCompilerTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.alerts.compiler; package uk.co.gresearch.siembol.alerts.compiler;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -10,137 +9,128 @@ import uk.co.gresearch.siembol.alerts.common.AlertingResult;
import uk.co.gresearch.siembol.alerts.engine.AlertingEngineImpl; import uk.co.gresearch.siembol.alerts.engine.AlertingEngineImpl;
import java.util.Arrays; import java.util.Arrays;
import java.util.List;
public class AlertingRulesCompilerTest { public class AlertingRulesCompilerTest {
/** private final String alertRules = """
*{ {
* "rules_version" :1, "rules_version" :1,
* "tags" : [ { "tag_name" : "detection_source", "tag_value" : "alerts" } ], "tags" : [ { "tag_name" : "detection_source", "tag_value" : "alerts" } ],
* "rules" : [ { "rules" : [ {
* "rule_name" : "siembol_alert_generic", "rule_name" : "siembol_alert_generic",
* "rule_version" : 1, "rule_version" : 1,
* "rule_author" : "dummy", "rule_author" : "dummy",
* "rule_description": "Test rule - is_alert is equal to true", "rule_description": "Test rule - is_alert is equal to true",
* "source_type" : "*", "source_type" : "*",
* "matchers" : [ { "matchers" : [ {
* "matcher_type" : "REGEX_MATCH", "matcher_type" : "REGEX_MATCH",
* "is_negated" : false, "is_negated" : false,
* "field" : "is_alert", "field" : "is_alert",
* "data" : "(?i)true" }, "data" : "(?i)true" },
* { {
* "matcher_type": "REGEX_MATCH", "matcher_type": "REGEX_MATCH",
* "is_negated": false, "is_negated": false,
* "field": "source_type", "field": "source_type",
* "data": "(?<sensor>.*)" "data": "(?<sensor>.*)"
* } }
* ] ]
* }] }]
*} }
**/ """;
@Multiline
public static String alertRules;
/** private final String alertRule = """
*{ {
* "rule_name" : "siembol_alert_generic", "rule_name" : "siembol_alert_generic",
* "rule_version" : 1, "rule_version" : 1,
* "rule_author" : "dummy", "rule_author" : "dummy",
* "rule_description": "Test rule - is_alert is equal to true", "rule_description": "Test rule - is_alert is equal to true",
* "source_type" : "*", "source_type" : "*",
* "matchers" : [ { "matchers" : [ {
* "matcher_type" : "REGEX_MATCH", "matcher_type" : "REGEX_MATCH",
* "is_negated" : false, "is_negated" : false,
* "field" : "is_alert", "field" : "is_alert",
* "data" : "(?i)true" }, "data" : "(?i)true" },
* { {
* "matcher_type": "REGEX_MATCH", "matcher_type": "REGEX_MATCH",
* "is_negated": false, "is_negated": false,
* "field": "source_type", "field": "source_type",
* "data": "(?<sensor>.*)" "data": "(?<sensor>.*)"
* }] }]
*} }
**/ """;
@Multiline
public static String alertRule;
/** private final String goodAlert = """
*{ {
* "source_type" : "secret", "source_type" : "secret",
* "is_alert" : "TruE", "is_alert" : "TruE",
* "dummy_field_int" : 1, "dummy_field_int" : 1,
* "dummy_field_boolean" : false "dummy_field_boolean" : false
*} }
**/ """;
@Multiline
public static String goodAlert;
/** private final String goodAlertWithSecret = """
*{ {
* "source_type" : "secret", "source_type" : "secret",
* "is_alert" : "TruE", "is_alert" : "TruE",
* "dummy_field_int" : 1, "dummy_field_int" : 1,
* "dummy_field_boolean" : false, "dummy_field_boolean" : false,
* "is_secret" : "true" "is_secret" : "true"
*} }
**/ """;
@Multiline
public static String goodAlertWithSecret;
/** private final String ruleWithCompositeMatchers = """
* { {
* "rule_name": "siembol_alert_generic_with_composite_matchers", "rule_name": "siembol_alert_generic_with_composite_matchers",
* "rule_version": 1, "rule_version": 1,
* "rule_author": "dummy", "rule_author": "dummy",
* "rule_description": "Test rule with composite matchers", "rule_description": "Test rule with composite matchers",
* "source_type": "*", "source_type": "*",
* "matchers": [ "matchers": [
* { {
* "matcher_type": "REGEX_MATCH", "matcher_type": "REGEX_MATCH",
* "is_negated": false, "is_negated": false,
* "field": "is_alert", "field": "is_alert",
* "data": "(?i)true" "data": "(?i)true"
* }, },
* { {
* "matcher_type": "REGEX_MATCH", "matcher_type": "REGEX_MATCH",
* "is_negated": false, "is_negated": false,
* "field": "source_type", "field": "source_type",
* "data": "(?<sensor>.*)" "data": "(?<sensor>.*)"
* }, },
* { {
* "matcher_type": "COMPOSITE_OR", "matcher_type": "COMPOSITE_OR",
* "is_negated": false, "is_negated": false,
* "matchers": [ "matchers": [
* { {
* "matcher_type": "REGEX_MATCH", "matcher_type": "REGEX_MATCH",
* "is_negated": false, "is_negated": false,
* "field": "is_secret", "field": "is_secret",
* "data": "(?i)true" "data": "(?i)true"
* }, },
* { {
* "matcher_type": "COMPOSITE_AND", "matcher_type": "COMPOSITE_AND",
* "is_negated": false, "is_negated": false,
* "matchers": [ "matchers": [
* { {
* "matcher_type": "REGEX_MATCH", "matcher_type": "REGEX_MATCH",
* "is_negated": false, "is_negated": false,
* "field": "is_public", "field": "is_public",
* "data": "(?i)true" "data": "(?i)true"
* }, },
* { {
* "matcher_type": "REGEX_MATCH", "matcher_type": "REGEX_MATCH",
* "is_negated": false, "is_negated": false,
* "field": "is_detected", "field": "is_detected",
* "data": "(?i)yes" "data": "(?i)yes"
* } }
* ] ]
* } }
* ] ]
* } }
* ] ]
* } }
**/ """;
@Multiline
public static String ruleWithCompositeMatchers;
private AlertingCompiler compiler; private AlertingCompiler compiler;
@@ -305,7 +295,7 @@ public class AlertingRulesCompilerTest {
@Test @Test
public void compileRulesListSizeOne() { public void compileRulesListSizeOne() {
AlertingResult compileResult = compiler.compile(Arrays.asList(alertRules)); AlertingResult compileResult = compiler.compile(List.of(alertRules));
Assert.assertEquals(AlertingResult.StatusCode.OK, compileResult.getStatusCode()); Assert.assertEquals(AlertingResult.StatusCode.OK, compileResult.getStatusCode());
Assert.assertNotNull(compileResult.getAttributes().getEngine()); Assert.assertNotNull(compileResult.getAttributes().getEngine());
Assert.assertTrue(compileResult.getAttributes().getEngine() instanceof AlertingEngineImpl); Assert.assertTrue(compileResult.getAttributes().getEngine() instanceof AlertingEngineImpl);

115
alerting/alerting-core/src/test/java/uk/co/gresearch/siembol/alerts/compiler/CorrelationRulesCompilerTest.java
View File

@@ -1,72 +1,67 @@
package uk.co.gresearch.siembol.alerts.compiler; package uk.co.gresearch.siembol.alerts.compiler;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import uk.co.gresearch.siembol.alerts.common.AlertingResult; import uk.co.gresearch.siembol.alerts.common.AlertingResult;
public class CorrelationRulesCompilerTest { public class CorrelationRulesCompilerTest {
/** private final String rulesWithSimpleCorrelationRule = """
*{ {
* "rules_version" :1, "rules_version" :1,
* "tags" : [ { "tag_name" : "detection_source", "tag_value" : "siembol_correlation_alerts" } ], "tags" : [ { "tag_name" : "detection_source", "tag_value" : "siembol_correlation_alerts" } ],
* "rules" : [ { "rules" : [ {
* "tags" : [ { "tag_name" : "test", "tag_value" : "true" } ], "tags" : [ { "tag_name" : "test", "tag_value" : "true" } ],
* "rule_protection": { "rule_protection": {
* "max_per_hour": 500, "max_per_hour": 500,
* "max_per_day": 1000 "max_per_day": 1000
* }, },
* "rule_name" : "test_rule", "rule_name" : "test_rule",
* "rule_version" : 1, "rule_version" : 1,
* "rule_author" : "dummy", "rule_author" : "dummy",
* "rule_description": "Testing rule", "rule_description": "Testing rule",
* "correlation_attributes" : { "correlation_attributes" : {
* "time_unit" : "seconds", "time_unit" : "seconds",
* "time_window" : 500, "time_window" : 500,
* "time_computation_type" : "processing_time", "time_computation_type" : "processing_time",
* "alerts" : [ "alerts" : [
* { {
* "alert" : "alert1", "alert" : "alert1",
* "threshold" : 5 "threshold" : 5
* }, },
* { {
* "alert" : "alert2", "alert" : "alert2",
* "threshold" : 5 "threshold" : 5
* }] }]
* } }
* }] }]
*} }
**/ """;
@Multiline
public static String rulesWithSimpleCorrelationRule;
/**{
* "rule_name" : "test_rule_event_time",
* "rule_version" : 1,
* "rule_author" : "dummy",
* "rule_description": "Testing rule",
* "correlation_attributes" : {
* "time_unit" : "seconds",
* "time_window" : 500,
* "time_computation_type" : "event_time",
* "max_time_lag_in_sec": 30,
* "alerts" : [
* {
* "alert" : "alert1",
* "threshold" : 5
* },
* {
* "mandatory": true,
* "alert" : "alert2",
* "threshold" : 5
* }]
* }
* }
**/
@Multiline
public static String simpleCorrelationRule;
private final String simpleCorrelationRule = """
{
"rule_name" : "test_rule_event_time",
"rule_version" : 1,
"rule_author" : "dummy",
"rule_description": "Testing rule",
"correlation_attributes" : {
"time_unit" : "seconds",
"time_window" : 500,
"time_computation_type" : "event_time",
"max_time_lag_in_sec": 30,
"alerts" : [
{
"alert" : "alert1",
"threshold" : 5
},
{
"mandatory": true,
"alert" : "alert2",
"threshold" : 5
}]
}
}
""";
private AlertingCompiler compiler; private AlertingCompiler compiler;

3
alerting/alerting-core/src/test/java/uk/co/gresearch/siembol/alerts/correlationengine/AlertCounterTest.java
View File

@@ -7,7 +7,7 @@ import org.junit.Test;
import java.util.EnumSet; import java.util.EnumSet;
public class AlertCounterTest { public class AlertCounterTest {
private int threshold = 1000; private final int threshold = 1000;
private AlertCounterMetadata counterMetadata; private AlertCounterMetadata counterMetadata;
private AlertCounter alertCounter; private AlertCounter alertCounter;
@@ -66,5 +66,4 @@ public class AlertCounterTest {
Assert.assertEquals(0, alertCounter.getSize()); Assert.assertEquals(0, alertCounter.getSize());
Assert.assertTrue(alertCounter.isEmpty()); Assert.assertTrue(alertCounter.isEmpty());
} }
} }

5
alerting/alerting-core/src/test/java/uk/co/gresearch/siembol/alerts/correlationengine/CorrelationRuleTest.java
View File

@@ -6,7 +6,6 @@ import org.junit.Test;
import uk.co.gresearch.siembol.alerts.common.AlertingFields; import uk.co.gresearch.siembol.alerts.common.AlertingFields;
import uk.co.gresearch.siembol.alerts.common.AlertingResult; import uk.co.gresearch.siembol.alerts.common.AlertingResult;
import java.util.*; import java.util.*;
import static uk.co.gresearch.siembol.alerts.common.EvaluationResult.MATCH; import static uk.co.gresearch.siembol.alerts.common.EvaluationResult.MATCH;
@@ -24,7 +23,7 @@ public class CorrelationRuleTest {
private final int maxTimeLagInSec = 5; private final int maxTimeLagInSec = 5;
private final String ruleName = "test_rule"; private final String ruleName = "test_rule";
private List<Map<String, Object>> alerts; private List<Map<String, Object>> alerts;
private String correlationKey = "1.2.3.4"; private final String correlationKey = "1.2.3.4";
@Before @Before
@@ -176,7 +175,7 @@ public class CorrelationRuleTest {
rule = builder.flags(ruleFlags).alertsThresholds(1).build(); rule = builder.flags(ruleFlags).alertsThresholds(1).build();
for (int i = 1; i < 100; i++) { for (int i = 1; i < 100; i++) {
alerts = createAlert(2, correlationKey + String.valueOf(i), alerts = createAlert(2, correlationKey + i,
"alert3", "alert3",
30000 + i); 30000 + i);
for (Map<String, Object> alert : alerts) { for (Map<String, Object> alert : alerts) {

22
alerting/alerting-core/src/test/java/uk/co/gresearch/siembol/alerts/engine/AlertingEngineImplTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.alerts.engine; package uk.co.gresearch.siembol.alerts.engine;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Pair;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
@@ -17,14 +16,13 @@ import java.util.*;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
public class AlertingEngineImplTest { public class AlertingEngineImplTest {
/** private final String knownSourceType = """
*{"source_type" : "test_source", "dummy_field" : "true"} { "source_type" : "test_source",
*/ "dummy_field" : "true"
@Multiline }
public static String knownSourceType; """;
private final String sourceType = "test_source";
private String sourceType = "test_source";
private List<Pair<String, String>> constants; private List<Pair<String, String>> constants;
private List<Pair<String, Object>> protections; private List<Pair<String, Object>> protections;
private List<Pair<String, Rule>> rules; private List<Pair<String, Rule>> rules;
@@ -36,8 +34,8 @@ public class AlertingEngineImplTest {
@Before @Before
public void setUp() { public void setUp() {
constants = Arrays.asList(Pair.of("detection_source", "siembol_alerts")); constants = List.of(Pair.of("detection_source", "siembol_alerts"));
protections = Arrays.asList(Pair.of(AlertingFields.MAX_PER_HOUR_FIELD.getAlertingName(), Integer.valueOf(1))); protections = List.of(Pair.of(AlertingFields.MAX_PER_HOUR_FIELD.getAlertingName(), 1));
rule1 = Mockito.mock(Rule.class); rule1 = Mockito.mock(Rule.class);
rule2 = Mockito.mock(Rule.class); rule2 = Mockito.mock(Rule.class);
resultRule1 = AlertingResult.fromEvaluationResult(EvaluationResult.MATCH, new HashMap<>()); resultRule1 = AlertingResult.fromEvaluationResult(EvaluationResult.MATCH, new HashMap<>());
@@ -46,10 +44,10 @@ public class AlertingEngineImplTest {
when(rule1.getRuleName()).thenReturn("rule1"); when(rule1.getRuleName()).thenReturn("rule1");
when(rule1.getFullRuleName()).thenReturn("rule1_v1"); when(rule1.getFullRuleName()).thenReturn("rule1_v1");
when(rule1.match(ArgumentMatchers.<Map<String, Object>>any())).thenReturn(resultRule1); when(rule1.match(ArgumentMatchers.any())).thenReturn(resultRule1);
when(rule2.getRuleName()).thenReturn("rule2"); when(rule2.getRuleName()).thenReturn("rule2");
when(rule2.getFullRuleName()).thenReturn("rule2_v1"); when(rule2.getFullRuleName()).thenReturn("rule2_v1");
when(rule2.match(ArgumentMatchers.<Map<String, Object>>any())).thenReturn(resultRule2); when(rule2.match(ArgumentMatchers.any())).thenReturn(resultRule2);
rules = Arrays.asList(Pair.of(sourceType, rule1), rules = Arrays.asList(Pair.of(sourceType, rule1),
Pair.of("*", rule2)); Pair.of("*", rule2));

47
alerting/alerting-core/src/test/java/uk/co/gresearch/siembol/alerts/engine/IsInSetTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.alerts.engine; package uk.co.gresearch.siembol.alerts.engine;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -14,32 +13,24 @@ public class IsInSetTest {
private Map<String, Object> event; private Map<String, Object> event;
private IsInSetMatcher matcher; private IsInSetMatcher matcher;
/** private final String strings = """
*metron metron
*alerts alerts
*response response
*stoRm stoRm
*123 123""";
**/
@Multiline
private String strings;
/** private final String variableStrings = """
*${variable_field1} ${variable_field1}
*aa ${variable.field2} ${variable:field1} aa ${variable.field2} ${variable:field1}
*http://${host}/${path} http://${host}/${path}""";
**/
@Multiline
private String variableStrings;
/** private final String mixedVariablesConstants = """
*alerts alerts
*${variable:field1} ${variable:field1}
*aa ${variable.field2} b aa ${variable.field2} b
*Metron Metron
**/ """;
@Multiline
private String mixedVariablesConstants;
@Before @Before
public void setUp() { public void setUp() {
@@ -163,8 +154,8 @@ public class IsInSetTest {
event.put(field, "Metron"); event.put(field, "Metron");
EvaluationResult rest = matcher.match(event); EvaluationResult rest = matcher.match(event);
Assert.assertEquals(rest, EvaluationResult.MATCH); Assert.assertEquals(EvaluationResult.MATCH, rest);
Assert.assertEquals(matcher.canModifyEvent(), false); Assert.assertFalse(matcher.canModifyEvent());
} }
@Test @Test
@@ -275,6 +266,4 @@ public class IsInSetTest {
ret = matcher.match(event); ret = matcher.match(event);
Assert.assertEquals(EvaluationResult.MATCH, ret); Assert.assertEquals(EvaluationResult.MATCH, ret);
} }
} }

22
alerting/alerting-core/src/test/java/uk/co/gresearch/siembol/alerts/engine/RegexMatcherTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.alerts.engine; package uk.co.gresearch.siembol.alerts.engine;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -10,22 +9,16 @@ import java.util.Map;
import java.util.HashMap; import java.util.HashMap;
public class RegexMatcherTest { public class RegexMatcherTest {
private String field = "test_field"; private final String field = "test_field";
Map<String, Object> event; Map<String, Object> event;
RegexMatcher matcher; RegexMatcher matcher;
/** private final String goodVofDetail = """
* Threat Level=(?<vof_threat_level>\d) Category=(?<vof_threat_cat>\S+) Type=(?<vof_threat_type>.*?) Threat Level=(?<vof_threat_level>\\d) Category=(?<vof_threat_cat>\\S+) Type=(?<vof_threat_type>.*?)""";
**/
@Multiline
public static String goodVofDetail;
/** private final String vofDetailInstance = """
* Threat Level=1 Category=UNKNOWN Type=a Threat Level=1 Category=UNKNOWN Type=a
*bc bc""";
**/
@Multiline
public static String vofDetailInstance;
@Before @Before
public void setUp() { public void setUp() {
@@ -73,7 +66,7 @@ public class RegexMatcherTest {
EvaluationResult rest = matcher.match(event); EvaluationResult rest = matcher.match(event);
Assert.assertEquals(EvaluationResult.NO_MATCH, rest); Assert.assertEquals(EvaluationResult.NO_MATCH, rest);
Assert.assertEquals(matcher.canModifyEvent(), true); Assert.assertTrue(matcher.canModifyEvent());
} }
@Test @Test
@@ -189,5 +182,4 @@ public class RegexMatcherTest {
.pattern("valid") .pattern("valid")
.build(); .build();
} }
} }

36
alerting/alerting-core/src/test/java/uk/co/gresearch/siembol/alerts/engine/RuleTest.java
View File

@@ -15,9 +15,9 @@ import java.util.*;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
public class RuleTest { public class RuleTest {
private String name = "test_rule"; private final String name = "test_rule";
private Integer version = 1; private final Integer version = 1;
private Map<String, Object> event = new HashMap<>(); private final Map<String, Object> event = new HashMap<>();
private List<Pair<String, String>> constants; private List<Pair<String, String>> constants;
private List<Pair<String, Object>> protections; private List<Pair<String, Object>> protections;
private BasicMatcher matcher; private BasicMatcher matcher;
@@ -25,16 +25,16 @@ public class RuleTest {
@Before @Before
public void setUp() { public void setUp() {
constants = Arrays.asList(Pair.of("detection_source", "alerts")); constants = List.of(Pair.of("detection_source", "alerts"));
protections = Arrays.asList(Pair.of(AlertingFields.MAX_PER_HOUR_FIELD.toString(), Integer.valueOf(1))); protections = List.of(Pair.of(AlertingFields.MAX_PER_HOUR_FIELD.toString(), 1));
matcher = Mockito.mock(BasicMatcher.class); matcher = Mockito.mock(BasicMatcher.class);
when(matcher.match(ArgumentMatchers.<Map<String, Object>>any())).thenReturn(EvaluationResult.MATCH); when(matcher.match(ArgumentMatchers.any())).thenReturn(EvaluationResult.MATCH);
} }
@Test @Test
public void testGoodMetadata() { public void testGoodMetadata() {
rule = Rule.builder() rule = Rule.builder()
.matchers(Arrays.asList(matcher)) .matchers(List.of(matcher))
.name(name) .name(name)
.version(version) .version(version)
.tags(constants) .tags(constants)
@@ -52,7 +52,7 @@ public class RuleTest {
Assert.assertEquals("alerts", event.get("detection_source")); Assert.assertEquals("alerts", event.get("detection_source"));
Assert.assertEquals(Integer.valueOf(1), event.get(AlertingFields.MAX_PER_HOUR_FIELD.toString())); Assert.assertEquals(1, event.get(AlertingFields.MAX_PER_HOUR_FIELD.toString()));
Assert.assertFalse(rule.canModifyEvent()); Assert.assertFalse(rule.canModifyEvent());
} }
@@ -61,7 +61,7 @@ public class RuleTest {
constants = new ArrayList<>(constants); constants = new ArrayList<>(constants);
constants.add(Pair.of("malicious_url", "http://${dummy_host}/${dummy_path}")); constants.add(Pair.of("malicious_url", "http://${dummy_host}/${dummy_path}"));
rule = Rule.builder() rule = Rule.builder()
.matchers(Arrays.asList(matcher)) .matchers(List.of(matcher))
.name(name) .name(name)
.version(version) .version(version)
.tags(constants) .tags(constants)
@@ -82,7 +82,7 @@ public class RuleTest {
Assert.assertEquals("http://alerts.com/about", event.get("malicious_url")); Assert.assertEquals("http://alerts.com/about", event.get("malicious_url"));
Assert.assertEquals(Integer.valueOf(1), event.get(AlertingFields.MAX_PER_HOUR_FIELD.toString())); Assert.assertEquals(1, event.get(AlertingFields.MAX_PER_HOUR_FIELD.toString()));
Assert.assertFalse(rule.canModifyEvent()); Assert.assertFalse(rule.canModifyEvent());
} }
@@ -91,7 +91,7 @@ public class RuleTest {
when(matcher.canModifyEvent()).thenReturn(true); when(matcher.canModifyEvent()).thenReturn(true);
rule = Rule.builder() rule = Rule.builder()
.matchers(Arrays.asList(matcher)) .matchers(List.of(matcher))
.name(name) .name(name)
.version(version) .version(version)
.tags(constants) .tags(constants)
@@ -104,7 +104,7 @@ public class RuleTest {
@Test @Test
public void testGoodMatch() { public void testGoodMatch() {
rule = Rule.builder() rule = Rule.builder()
.matchers(Arrays.asList(matcher)) .matchers(List.of(matcher))
.name(name) .name(name)
.version(version) .version(version)
.tags(constants) .tags(constants)
@@ -118,9 +118,9 @@ public class RuleTest {
@Test @Test
public void testGoodNoMatch() { public void testGoodNoMatch() {
when(matcher.match(ArgumentMatchers.<Map<String, Object>>any())).thenReturn(EvaluationResult.NO_MATCH); when(matcher.match(ArgumentMatchers.any())).thenReturn(EvaluationResult.NO_MATCH);
rule = Rule.builder() rule = Rule.builder()
.matchers(Arrays.asList(matcher)) .matchers(List.of(matcher))
.name(name) .name(name)
.version(version) .version(version)
.tags(constants) .tags(constants)
@@ -134,9 +134,9 @@ public class RuleTest {
@Test(expected = RuntimeException.class) @Test(expected = RuntimeException.class)
public void testThrowsException() throws RuntimeException { public void testThrowsException() throws RuntimeException {
when(matcher.match(ArgumentMatchers.<Map<String, Object>>any())).thenThrow(new RuntimeException()); when(matcher.match(ArgumentMatchers.any())).thenThrow(new RuntimeException());
rule = Rule.builder() rule = Rule.builder()
.matchers(Arrays.asList(matcher)) .matchers(List.of(matcher))
.name(name) .name(name)
.version(version) .version(version)
.tags(constants) .tags(constants)
@@ -149,7 +149,7 @@ public class RuleTest {
@Test(expected = IllegalArgumentException.class) @Test(expected = IllegalArgumentException.class)
public void missingName() { public void missingName() {
rule = Rule.builder() rule = Rule.builder()
.matchers(Arrays.asList(matcher)) .matchers(List.of(matcher))
.version(version) .version(version)
.tags(constants) .tags(constants)
.protections(protections) .protections(protections)
@@ -159,7 +159,7 @@ public class RuleTest {
@Test(expected = IllegalArgumentException.class) @Test(expected = IllegalArgumentException.class)
public void missingVersion() { public void missingVersion() {
Rule.builder() Rule.builder()
.matchers(Arrays.asList(matcher)) .matchers(List.of(matcher))
.name(name) .name(name)
.tags(constants) .tags(constants)
.protections(protections) .protections(protections)

2
alerting/alerting-core/src/test/java/uk/co/gresearch/siembol/alerts/protection/RuleProtectionSystemTest.java
View File

@@ -6,7 +6,7 @@ import uk.co.gresearch.siembol.alerts.common.AlertingResult;
public class RuleProtectionSystemTest { public class RuleProtectionSystemTest {
private RuleProtectionSystem protection; private RuleProtectionSystem protection;
private String ruleName = "test"; private final String ruleName = "test";
@Before @Before
public void setUp() { public void setUp() {

10
alerting/alerting-spark/pom.xml
View File

@@ -11,7 +11,7 @@
<parent> <parent>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting</artifactId> <artifactId>alerting</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</parent> </parent>
<dependencies> <dependencies>
<dependency> <dependency>
@@ -23,7 +23,7 @@
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId> <artifactId>alerting-core</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
<exclusions> <exclusions>
<exclusion> <exclusion>
<artifactId>jackson-databind</artifactId> <artifactId>jackson-databind</artifactId>
@@ -45,12 +45,6 @@
<version>${junit_version}</version> <version>${junit_version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency>
</dependencies> </dependencies>
<build> <build>
<plugins> <plugins>

72
alerting/alerting-spark/src/test/java/uk/co/gresearch/siembol/spark/AlertingSparkEngineTest.java
View File

@@ -1,48 +1,42 @@
package uk.co.gresearch.siembol.spark; package uk.co.gresearch.siembol.spark;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang3.SerializationUtils; import org.apache.commons.lang3.SerializationUtils;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
public class AlertingSparkEngineTest { public class AlertingSparkEngineTest {
/** private final String isAlertRules = """
*{ {
* "rules_version" :1, "rules_version" :1,
* "tags" : [ { "tag_name" : "detection:source", "tag_value" : "alerts" } ], "tags" : [ { "tag_name" : "detection:source", "tag_value" : "alerts" } ],
* "rules" : [ { "rules" : [ {
* "rule_name" : "test_rule", "rule_name" : "test_rule",
* "rule_version" : 1, "rule_version" : 1,
* "rule_author" : "dummy", "rule_author" : "dummy",
* "rule_protection" : { "rule_protection" : {
* "max_per_hour" : 100, "max_per_hour" : 100,
* "max_per_day" : 10000 "max_per_day" : 10000
* }, },
* "rule_description": "test rule - is_alert is equal to true", "rule_description": "test rule - is_alert is equal to true",
* "source_type" : "*", "source_type" : " ",
* "matchers" : [ { "matchers" : [ {
* "matcher_type" : "REGEX_MATCH", "matcher_type" : "REGEX_MATCH",
* "is_negated" : false, "is_negated" : false,
* "field" : "is_alert", "field" : "is_alert",
* "data" : "(?i)true" } "data" : "(?i)true" }
* ] ]
* }] }]
*} }
**/ """;
@Multiline
public static String isAlertRules;
/**
*{
* "source_type" : "secret",
* "is_alert" : "TruE",
* "dummy_field_int" : 1,
* "dummy_field_boolean" : false
*}
**/
@Multiline
public static String goodAlert;
private final String goodAlert = """
{
"source_type" : "secret",
"is_alert" : "TruE",
"dummy_field_int" : 1,
"dummy_field_boolean" : false
}
""";
@Test @Test
public void serializableTest() throws Exception { public void serializableTest() throws Exception {
@@ -57,7 +51,7 @@ public class AlertingSparkEngineTest {
Assert.assertEquals(ret.getMatchesTotal(), retClone.getMatchesTotal()); Assert.assertEquals(ret.getMatchesTotal(), retClone.getMatchesTotal());
Assert.assertEquals(ret.getExceptionsTotal(), retClone.getExceptionsTotal()); Assert.assertEquals(ret.getExceptionsTotal(), retClone.getExceptionsTotal());
Assert.assertTrue(ret.getExceptions().equals(retClone.getExceptions())); Assert.assertEquals(ret.getExceptions(), retClone.getExceptions());
Assert.assertTrue(ret.getMatches().equals(retClone.getMatches())); Assert.assertEquals(ret.getMatches(), retClone.getMatches());
} }
} }

87
alerting/alerting-spark/src/test/java/uk/co/gresearch/siembol/spark/AlertingSparkJobTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.spark; package uk.co.gresearch.siembol.spark;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.JavaSparkContext;
@@ -8,58 +7,52 @@ import org.junit.*;
import java.util.ArrayList; import java.util.ArrayList;
public class AlertingSparkJobTest { public class AlertingSparkJobTest {
/** private final String isAlertRules = """
*{ {
* "rules_version" :1, "rules_version" :1,
* "tags" : [ { "tag_name" : "detection_source", "tag_value" : "siembol_alerts" } ], "tags" : [ { "tag_name" : "detection_source", "tag_value" : "siembol_alerts" } ],
* "rules" : [ { "rules" : [ {
* "rule_name" : "test_rule", "rule_name" : "test_rule",
* "rule_version" : 1, "rule_version" : 1,
* "rule_author" : "dummy", "rule_author" : "dummy",
* "rule_protection" : { "rule_protection" : {
* "max_per_hour" : 100, "max_per_hour" : 100,
* "max_per_day" : 10000 "max_per_day" : 10000
* }, },
* "rule_description": "Testing rule", "rule_description": "Testing rule",
* "source_type" : "*", "source_type" : "*",
* "matchers" : [ { "matchers" : [ {
* "matcher_type" : "REGEX_MATCH", "matcher_type" : "REGEX_MATCH",
* "is_negated" : false, "is_negated" : false,
* "field" : "is_alert", "field" : "is_alert",
* "data" : "(?i)true" } "data" : "(?i)true" }
* ] ]
* }] }]
*} }
**/ """;
@Multiline
public static String isAlertRules;
/** private final String goodAlert = """
*{ {
* "source_type" : "secret", "source_type" : "secret",
* "is_alert" : "TruE", "is_alert" : "TruE",
* "dummy_field_int" : 1, "dummy_field_int" : 1,
* "dummy_field_boolean" : false "dummy_field_boolean" : false
*} }
**/ """;
@Multiline
public static String goodAlert;
/**
*{ private final String eventWithoutAlert = """
* "source_type" : "secret", {
* "dummy_field_int" : 1, "source_type" : "secret",
* "dummy_field_boolean" : false "dummy_field_int" : 1,
*} "dummy_field_boolean" : false
**/ }
@Multiline """;
public static String eventWithoutAlert;
private JavaSparkContext sc; private JavaSparkContext sc;
private AlertingSparkJob job; private AlertingSparkJob job;
private int maxResult = 100; private final int maxResult = 100;
@Before @Before
public void setup() { public void setup() {

59
alerting/alerting-spark/src/test/java/uk/co/gresearch/siembol/spark/AlertingSparkTest.java
View File

@@ -1,42 +1,39 @@
package uk.co.gresearch.siembol.spark; package uk.co.gresearch.siembol.spark;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Ignore; import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import java.util.Base64; import java.util.Base64;
public class AlertingSparkTest { public class AlertingSparkTest {
/** private final String testAttributes = """
*{ {
* "source_type" : "secret", "source_type" : "secret",
* "from_date" : "2019-06-18", "from_date" : "2019-06-18",
* "to_date" : "2019-06-19", "to_date" : "2019-06-19",
* "rules" :{ "rules" :{
* "rules_version" :1, "rules_version" :1,
* "tags" : [ { "tag_name" : "detection_source", "tag_value" : "siembol_alerts" } ], "tags" : [ { "tag_name" : "detection_source", "tag_value" : "siembol_alerts" } ],
* "rules" : [ { "rules" : [ {
* "rule_name" : "test_rule", "rule_name" : "test_rule",
* "rule_version" : 1, "rule_version" : 1,
* "rule_author" : "dummy", "rule_author" : "dummy",
* "rule_protection" : { "rule_protection" : {
* "max_per_hour" : 100, "max_per_hour" : 100,
* "max_per_day" : 10000 "max_per_day" : 10000
* }, },
* "rule_description": "Testing rule", "rule_description": "Testing rule",
* "source_type" : "*", "source_type" : "*",
* "matchers" : [ { "matchers" : [ {
* "matcher_type" : "REGEX_MATCH", "matcher_type" : "REGEX_MATCH",
* "is_negated" : false, "is_negated" : false,
* "field" : "is_alert", "field" : "is_alert",
* "data" : "(?i)true" } "data" : "(?i)true" }
* ] ]
* }] }]
*} }
*} }
**/ """;
@Multiline
public static String testAttributes;
@Test @Test
@Ignore @Ignore

28
alerting/alerting-spark/src/test/java/uk/co/gresearch/siembol/spark/SparkResultTest.java
View File

@@ -7,16 +7,13 @@ import org.junit.Test;
import uk.co.gresearch.siembol.alerts.common.AlertingAttributes; import uk.co.gresearch.siembol.alerts.common.AlertingAttributes;
import uk.co.gresearch.siembol.alerts.common.AlertingResult; import uk.co.gresearch.siembol.alerts.common.AlertingResult;
import java.util.ArrayList; import java.util.*;
import java.util.Arrays;
import java.util.Map;
import java.util.HashMap;
public class SparkResultTest { public class SparkResultTest {
private AlertingAttributes attributes; private AlertingAttributes attributes;
private AlertingResult alertingResult; private AlertingResult alertingResult;
private AlertingSparkResult alertingSparkResult; private AlertingSparkResult alertingSparkResult;
private int maxResult = 100; private final int maxResult = 100;
private Map<String, Object> event; private Map<String, Object> event;
@Before @Before
@@ -27,7 +24,7 @@ public class SparkResultTest {
} }
@Test @Test
public void wrongStatusCodeTest() throws Exception { public void wrongStatusCodeTest() {
alertingResult = new AlertingResult(AlertingResult.StatusCode.ERROR, attributes); alertingResult = new AlertingResult(AlertingResult.StatusCode.ERROR, attributes);
alertingSparkResult = new AlertingSparkResult(alertingResult, maxResult); alertingSparkResult = new AlertingSparkResult(alertingResult, maxResult);
Assert.assertEquals(1, alertingSparkResult.getExceptionsTotal()); Assert.assertEquals(1, alertingSparkResult.getExceptionsTotal());
@@ -38,9 +35,9 @@ public class SparkResultTest {
} }
@Test @Test
public void singleEventTest() throws Exception { public void singleEventTest() {
event.put("test", "true"); event.put("test", "true");
attributes.setOutputEvents(Arrays.asList(event)); attributes.setOutputEvents(List.of(event));
alertingSparkResult = new AlertingSparkResult(alertingResult, maxResult); alertingSparkResult = new AlertingSparkResult(alertingResult, maxResult);
Assert.assertEquals(0, alertingSparkResult.getExceptionsTotal()); Assert.assertEquals(0, alertingSparkResult.getExceptionsTotal());
Assert.assertEquals(1, alertingSparkResult.getMatchesTotal()); Assert.assertEquals(1, alertingSparkResult.getMatchesTotal());
@@ -50,7 +47,7 @@ public class SparkResultTest {
} }
@Test @Test
public void maxResultEventTest() throws Exception { public void maxResultEventTest() {
event.put("test", "true"); event.put("test", "true");
ArrayList<Map<String, Object>> events = new ArrayList<>(); ArrayList<Map<String, Object>> events = new ArrayList<>();
for (int i = 0; i <= maxResult; i++) { for (int i = 0; i <= maxResult; i++) {
@@ -67,9 +64,9 @@ public class SparkResultTest {
} }
@Test @Test
public void singleExceptionTest() throws Exception { public void singleExceptionTest() {
event.put("test", "true"); event.put("test", "true");
attributes.setExceptionEvents(Arrays.asList(event)); attributes.setExceptionEvents(List.of(event));
alertingSparkResult = new AlertingSparkResult(alertingResult, maxResult); alertingSparkResult = new AlertingSparkResult(alertingResult, maxResult);
Assert.assertEquals(1, alertingSparkResult.getExceptionsTotal()); Assert.assertEquals(1, alertingSparkResult.getExceptionsTotal());
Assert.assertEquals(0, alertingSparkResult.getMatchesTotal()); Assert.assertEquals(0, alertingSparkResult.getMatchesTotal());
@@ -79,7 +76,7 @@ public class SparkResultTest {
} }
@Test @Test
public void maxResultExceptionTest() throws Exception { public void maxResultExceptionTest() {
event.put("test", "true"); event.put("test", "true");
ArrayList<Map<String, Object>> events = new ArrayList<>(); ArrayList<Map<String, Object>> events = new ArrayList<>();
for (int i = 0; i <= maxResult; i++) { for (int i = 0; i <= maxResult; i++) {
@@ -96,7 +93,7 @@ public class SparkResultTest {
} }
@Test @Test
public void mergeTest() throws Exception { public void mergeTest() {
event.put("test", "true"); event.put("test", "true");
ArrayList<Map<String, Object>> events = new ArrayList<>(); ArrayList<Map<String, Object>> events = new ArrayList<>();
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
@@ -137,16 +134,15 @@ public class SparkResultTest {
} }
@Test @Test
public void serializableTest() throws Exception { public void serializableTest() {
event.put("test", "true"); event.put("test", "true");
attributes.setOutputEvents(Arrays.asList(event)); attributes.setOutputEvents(List.of(event));
alertingSparkResult = new AlertingSparkResult(alertingResult, maxResult); alertingSparkResult = new AlertingSparkResult(alertingResult, maxResult);
byte[] blob = SerializationUtils.serialize(alertingSparkResult); byte[] blob = SerializationUtils.serialize(alertingSparkResult);
Assert.assertTrue(blob.length > 0); Assert.assertTrue(blob.length > 0);
AlertingSparkResult clone = SerializationUtils.clone(alertingSparkResult); AlertingSparkResult clone = SerializationUtils.clone(alertingSparkResult);
Assert.assertEquals(0, clone.getExceptionsTotal()); Assert.assertEquals(0, clone.getExceptionsTotal());
Assert.assertEquals(1, clone.getMatchesTotal()); Assert.assertEquals(1, clone.getMatchesTotal());
Assert.assertTrue(clone.getExceptions().isEmpty()); Assert.assertTrue(clone.getExceptions().isEmpty());

15
alerting/alerting-storm/pom.xml
View File

@@ -9,9 +9,14 @@
<parent> <parent>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting</artifactId> <artifactId>alerting</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</parent> </parent>
<dependencies> <dependencies>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>23.0</version>
</dependency>
<dependency> <dependency>
<groupId>com.fasterxml.jackson.core</groupId> <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId> <artifactId>jackson-core</artifactId>
@@ -46,7 +51,7 @@
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId> <artifactId>alerting-core</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
<exclusions> <exclusions>
<exclusion> <exclusion>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
@@ -64,12 +69,6 @@
<version>${junit_version}</version> <version>${junit_version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>org.apache.kafka</groupId> <groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.11</artifactId> <artifactId>kafka_2.11</artifactId>

157
alerting/alerting-storm/src/test/java/uk/co/gresearch/siembol/alerts/storm/AlertingEngineBoltTest.java
View File

@@ -3,7 +3,6 @@ package uk.co.gresearch.siembol.alerts.storm;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectReader;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.storm.task.OutputCollector; import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values; import org.apache.storm.tuple.Values;
@@ -33,90 +32,84 @@ import static org.mockito.internal.verification.VerificationModeFactory.times;
public class AlertingEngineBoltTest { public class AlertingEngineBoltTest {
private static final ObjectReader JSON_READER = new ObjectMapper() private static final ObjectReader JSON_READER = new ObjectMapper()
.readerFor(new TypeReference<Map<String, Object>>() {}); .readerFor(new TypeReference<Map<String, Object>>() {
});
/** private final String event = """
*{ {
* "source_type" : "secret", "source_type" : "secret",
* "is_alert" : "TruE", "is_alert" : "TruE",
* "dummy_field_int" : 1, "dummy_field_int" : 1,
* "dummy_field_boolean" : false "dummy_field_boolean" : false
*} }
**/ """;
@Multiline
public static String event;
private final String simpleTestRules = """
{
"rules_version" :1,
"tags" : [ { "tag_name" : "detection_source", "tag_value" : "siembol_alerts" } ],
"rules" : [ {
"rule_name" : "siembol_alert_generic",
"rule_version" : 1,
"rule_author" : "dummy",
"rule_description": "Test rule - is_alert is equal to true",
"source_type" : "*",
"matchers" : [ {
"matcher_type" : "REGEX_MATCH",
"is_negated" : false,
"field" : "is_alert",
"data" : "(?i)true" },
{
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "source_type",
"data": "(?<sensor>.*)"
}
]
}]
}
""";
/** private final String rulesForCorrelation = """
*{ {
* "rules_version" :1, "rules_version": 1,
* "tags" : [ { "tag_name" : "detection_source", "tag_value" : "siembol_alerts" } ], "tags": [
* "rules" : [ { {
* "rule_name" : "siembol_alert_generic", "tag_name": "detection_source",
* "rule_version" : 1, "tag_value": "siembol_alerts"
* "rule_author" : "dummy", }
* "rule_description": "Test rule - is_alert is equal to true", ],
* "source_type" : "*", "rules": [
* "matchers" : [ { {
* "matcher_type" : "REGEX_MATCH", "rule_name": "siembol_alert_generic",
* "is_negated" : false, "rule_version": 1,
* "field" : "is_alert", "rule_author": "dummy",
* "data" : "(?i)true" }, "rule_description": "Test rule - is_alert is equal to true",
* { "source_type": "*",
* "matcher_type": "REGEX_MATCH", "matchers": [
* "is_negated": false, {
* "field": "source_type", "matcher_type": "REGEX_MATCH",
* "data": "(?<sensor>.*)" "is_negated": false,
* } "field": "is_alert",
* ] "data": "(?i)true"
* }] },
*} {
**/ "matcher_type": "REGEX_MATCH",
@Multiline "is_negated": false,
public static String simpleTestRules; "field": "source_type",
"data": "(?<sensor>.*)"
/** }
* { ],
* "rules_version": 1, "tags": [
* "tags": [ {
* { "tag_name": "correlation_key",
* "tag_name": "detection_source", "tag_value": "${dummy_field_int}"
* "tag_value": "siembol_alerts" }
* } ]
* ], }
* "rules": [ ]
* { }
* "rule_name": "siembol_alert_generic", """;
* "rule_version": 1,
* "rule_author": "dummy",
* "rule_description": "Test rule - is_alert is equal to true",
* "source_type": "*",
* "matchers": [
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "is_alert",
* "data": "(?i)true"
* },
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "source_type",
* "data": "(?<sensor>.*)"
* }
* ],
* "tags": [
* {
* "tag_name": "correlation_key",
* "tag_value": "${dummy_field_int}"
* }
* ]
* }
* ]
* }
**/
@Multiline
public static String rulesForCorrelation;
private Tuple tuple; private Tuple tuple;
private OutputCollector collector; private OutputCollector collector;

131
alerting/alerting-storm/src/test/java/uk/co/gresearch/siembol/alerts/storm/AlertingStormApplicationTest.java
View File

@@ -5,7 +5,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectReader;
import com.github.charithe.kafka.EphemeralKafkaBroker; import com.github.charithe.kafka.EphemeralKafkaBroker;
import com.github.charithe.kafka.KafkaJunitRule; import com.github.charithe.kafka.KafkaJunitRule;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.storm.Config; import org.apache.storm.Config;
import org.apache.storm.LocalCluster; import org.apache.storm.LocalCluster;
import org.apache.storm.generated.StormTopology; import org.apache.storm.generated.StormTopology;
@@ -29,77 +28,71 @@ public class AlertingStormApplicationTest {
private static final ObjectReader JSON_PARSERS_CONFIG_READER = new ObjectMapper() private static final ObjectReader JSON_PARSERS_CONFIG_READER = new ObjectMapper()
.readerFor(AlertingStormAttributesDto.class); .readerFor(AlertingStormAttributesDto.class);
private static final ObjectReader JSON_READER = new ObjectMapper() private static final ObjectReader JSON_READER = new ObjectMapper()
.readerFor(new TypeReference<Map<String, Object>>() {}); .readerFor(new TypeReference<Map<String, Object>>() {
});
/** private final String testRules = """
*{ {
* "rules_version" :1, "rules_version" :1,
* "tags" : [ { "tag_name" : "detection_source", "tag_value" : "siembol_alerts" } ], "tags" : [ { "tag_name" : "detection_source", "tag_value" : "siembol_alerts" } ],
* "rules" : [ { "rules" : [ {
* "rule_name" : "test_rule", "rule_name" : "test_rule",
* "rule_version" : 1, "rule_version" : 1,
* "rule_author" : "dummy", "rule_author" : "dummy",
* "rule_protection" : { "rule_protection" : {
* "max_per_hour" : 100, "max_per_hour" : 100,
* "max_per_day" : 10000 "max_per_day" : 10000
* }, },
* "rule_description": "test rule - is_alert is equal to true", "rule_description": "test rule - is_alert is equal to true",
* "source_type" : "*", "source_type" : "*",
* "matchers" : [ { "matchers" : [ {
* "matcher_type" : "REGEX_MATCH", "matcher_type" : "REGEX_MATCH",
* "is_negated" : false, "is_negated" : false,
* "field" : "is_alert", "field" : "is_alert",
* "data" : "(?i)true" } "data" : "(?i)true" }
* ] ]
* }] }]
*} }
**/ """;
@Multiline
private static String testRules;
/** private final String goodAlert = """
*{ {
* "source_type" : "secret", "source_type" : "secret",
* "is_alert" : "TruE", "is_alert" : "TruE",
* "dummy_field_int" : 1, "dummy_field_int" : 1,
* "dummy_field_boolean" : false "dummy_field_boolean" : false
*} }
**/ """;
@Multiline
private static String goodAlert;
private final String testConfig = """
/** {
* { "alerts.engine": "siembol_alerts",
* "alerts.engine": "siembol_alerts", "alerts.input.topics": [ "input" ],
* "alerts.input.topics": [ "input" ], "alerts.correlation.output.topic": "correlation.alerts",
* "alerts.correlation.output.topic": "correlation.alerts", "kafka.error.topic": "errors",
* "kafka.error.topic": "errors", "alerts.output.topic": "alerts",
* "alerts.output.topic": "alerts", "storm.attributes": {
* "storm.attributes": { "first.pool.offset.strategy": "EARLIEST",
* "first.pool.offset.strategy": "EARLIEST", "kafka.spout.properties": {
* "kafka.spout.properties": { "group.id": "alerts.reader",
* "group.id": "alerts.reader", "security.protocol": "PLAINTEXT"
* "security.protocol": "PLAINTEXT" }
* } },
* }, "kafka.spout.num.executors": 1,
* "kafka.spout.num.executors": 1, "alerts.engine.bolt.num.executors": 1,
* "alerts.engine.bolt.num.executors": 1, "kafka.writer.bolt.num.executors": 1,
* "kafka.writer.bolt.num.executors": 1, "kafka.producer.properties": {
* "kafka.producer.properties": { "compression.type": "snappy",
* "compression.type": "snappy", "security.protocol": "PLAINTEXT",
* "security.protocol": "PLAINTEXT", "client.id": "test_producer"
* "client.id": "test_producer" },
* }, "zookeeper.attributes": {
* "zookeeper.attributes": { "zk.path": "rules",
* "zk.path": "rules", "zk.base.sleep.ms": 1000,
* "zk.base.sleep.ms": 1000, "zk.max.retries": 10
* "zk.max.retries": 10 }
* } }
* } """;
**/
@Multiline
public static String testConfig;
@ClassRule @ClassRule
public static KafkaJunitRule kafkaRule = new KafkaJunitRule(EphemeralKafkaBroker.create()); public static KafkaJunitRule kafkaRule = new KafkaJunitRule(EphemeralKafkaBroker.create());

213
alerting/alerting-storm/src/test/java/uk/co/gresearch/siembol/alerts/storm/CorrelationAlertingTest.java
View File

@@ -5,7 +5,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectReader;
import com.github.charithe.kafka.EphemeralKafkaBroker; import com.github.charithe.kafka.EphemeralKafkaBroker;
import com.github.charithe.kafka.KafkaJunitRule; import com.github.charithe.kafka.KafkaJunitRule;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.storm.Config; import org.apache.storm.Config;
import org.apache.storm.LocalCluster; import org.apache.storm.LocalCluster;
import org.apache.storm.generated.StormTopology; import org.apache.storm.generated.StormTopology;
@@ -27,120 +26,112 @@ import static org.mockito.Mockito.withSettings;
public class CorrelationAlertingTest { public class CorrelationAlertingTest {
private static final ObjectReader JSON_PARSERS_CONFIG_READER = new ObjectMapper() private static final ObjectReader JSON_PARSERS_CONFIG_READER = new ObjectMapper()
.readerFor(AlertingStormAttributesDto.class); .readerFor(AlertingStormAttributesDto.class);
private static ObjectReader JSON_READER = new ObjectMapper() private static final ObjectReader JSON_READER = new ObjectMapper()
.readerFor(new TypeReference<Map<String, Object>>() {}); .readerFor(new TypeReference<Map<String, Object>>() {
});
/** private final String alert1 = """
* { {
* "siembol_alerts_full_rule_name": "alert1_v3", "siembol_alerts_full_rule_name": "alert1_v3",
* "siembol_alerts_rule_name": "alert1", "siembol_alerts_rule_name": "alert1",
* "correlation_key": "evil", "correlation_key": "evil",
* "siembol_alerts_max_per_hour": 200, "siembol_alerts_max_per_hour": 200,
* "siembol_alerts_test": "true", "siembol_alerts_test": "true",
* "source_type": "a", "source_type": "a",
* "siembol_alerts_max_per_day": 10000 "siembol_alerts_max_per_day": 10000
* } }
**/ """;
@Multiline
public static String alert1;
/** private final String alert2 = """
* { {
* "siembol_alerts_full_rule_name": "alert1_v3", "siembol_alerts_full_rule_name": "alert1_v3",
* "siembol_alerts_rule_name": "alert2", "siembol_alerts_rule_name": "alert2",
* "correlation_key": "evil", "correlation_key": "evil",
* "sensor": "a", "sensor": "a",
* "siembol_alerts_max_per_hour": 200, "siembol_alerts_max_per_hour": 200,
* "siembol_alerts_test": "true", "siembol_alerts_test": "true",
* "source_type": "a", "source_type": "a",
* "siembol_alerts_max_per_day": 10000 "siembol_alerts_max_per_day": 10000
* } }
**/ """;
@Multiline
public static String alert2; private final String simpleCorrelationRules = """
{
"rules_version": 1,
"tags": [
{
"tag_name": "detection_source",
"tag_value": "siembol_correlation_alerts_instance"
}
],
"rules": [
{
"tags": [
{
"tag_name": "test",
"tag_value": "true"
}
],
"rule_protection": {
"max_per_hour": 500,
"max_per_day": 1000
},
"rule_name": "test_rule",
"rule_version": 1,
"rule_author": "dummy",
"rule_description": "Testing rule",
"correlation_attributes": {
"time_unit": "seconds",
"time_window": 500,
"time_computation_type": "processing_time",
"alerts": [
{
"alert": "alert1",
"threshold": 2
},
{
"alert": "alert2",
"threshold": 1
}
]
}
}
]
}
}
""";
/** private final String testConfig = """
* { {
* "rules_version": 1, "alerts.engine": "siembol_correlation_alerts",
* "tags": [ "alerts.input.topics": [ "input" ],
* { "alerts.correlation.output.topic": "correlation.alerts",
* "tag_name": "detection_source", "kafka.error.topic": "errors",
* "tag_value": "siembol_correlation_alerts_instance" "alerts.output.topic": "alerts",
* } "alerts.engine.clean.interval.sec" : 2,
* ], "storm.attributes": {
* "rules": [ "first.pool.offset.strategy": "EARLIEST",
* { "kafka.spout.properties": {
* "tags": [ "group.id": "alerts.reader",
* { "security.protocol": "PLAINTEXT"
* "tag_name": "test", }
* "tag_value": "true" },
* } "kafka.spout.num.executors": 1,
* ], "alerts.engine.bolt.num.executors": 1,
* "rule_protection": { "kafka.writer.bolt.num.executors": 1,
* "max_per_hour": 500, "kafka.producer.properties": {
* "max_per_day": 1000 "compression.type": "snappy",
* }, "security.protocol": "PLAINTEXT",
* "rule_name": "test_rule", "client.id": "test_producer"
* "rule_version": 1, },
* "rule_author": "dummy", "zookeeper.attributes": {
* "rule_description": "Testing rule", "zk.path": "rules",
* "correlation_attributes": { "zk.base.sleep.ms": 1000,
* "time_unit": "seconds", "zk.max.retries": 10
* "time_window": 500, }
* "time_computation_type": "processing_time", }
* "alerts": [ """;
* {
* "alert": "alert1",
* "threshold": 2
* },
* {
* "alert": "alert2",
* "threshold": 1
* }
* ]
* }
* }
* ]
* }
*}
**/
@Multiline
public static String simpleCorrelationRules;
/**
* {
* "alerts.engine": "siembol_correlation_alerts",
* "alerts.input.topics": [ "input" ],
* "alerts.correlation.output.topic": "correlation.alerts",
* "kafka.error.topic": "errors",
* "alerts.output.topic": "alerts",
* "alerts.engine.clean.interval.sec" : 2,
* "storm.attributes": {
* "first.pool.offset.strategy": "EARLIEST",
* "kafka.spout.properties": {
* "group.id": "alerts.reader",
* "security.protocol": "PLAINTEXT"
* }
* },
* "kafka.spout.num.executors": 1,
* "alerts.engine.bolt.num.executors": 1,
* "kafka.writer.bolt.num.executors": 1,
* "kafka.producer.properties": {
* "compression.type": "snappy",
* "security.protocol": "PLAINTEXT",
* "client.id": "test_producer"
* },
* "zookeeper.attributes": {
* "zk.path": "rules",
* "zk.base.sleep.ms": 1000,
* "zk.max.retries": 10
* }
* }
**/
@Multiline
public static String testConfig;
@ClassRule @ClassRule
public static KafkaJunitRule kafkaRule = new KafkaJunitRule(EphemeralKafkaBroker.create()); public static KafkaJunitRule kafkaRule = new KafkaJunitRule(EphemeralKafkaBroker.create());

148
alerting/alerting-storm/src/test/java/uk/co/gresearch/siembol/alerts/storm/CorrelationEngineBoltTest.java
View File

@@ -3,7 +3,6 @@ package uk.co.gresearch.siembol.alerts.storm;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectReader;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.storm.task.OutputCollector; import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values; import org.apache.storm.tuple.Values;
@@ -33,86 +32,81 @@ import static org.mockito.Mockito.when;
import static org.mockito.internal.verification.VerificationModeFactory.times; import static org.mockito.internal.verification.VerificationModeFactory.times;
public class CorrelationEngineBoltTest { public class CorrelationEngineBoltTest {
private static ObjectReader JSON_READER = new ObjectMapper() private static final ObjectReader JSON_READER = new ObjectMapper()
.readerFor(new TypeReference<Map<String, Object>>() {}); .readerFor(new TypeReference<Map<String, Object>>() {
});
/** private final String alert1 = """
* { {
* "siembol_alerts_full_rule_name": "alert1_v3", "siembol_alerts_full_rule_name": "alert1_v3",
* "siembol_alerts_rule_name": "alert1", "siembol_alerts_rule_name": "alert1",
* "correlation_key": "evil", "correlation_key": "evil",
* "siembol_alerts_max_per_hour": 200, "siembol_alerts_max_per_hour": 200,
* "siembol_alerts_test": "true", "siembol_alerts_test": "true",
* "source_type": "a", "source_type": "a",
* "siembol_alerts_max_per_day": 10000 "siembol_alerts_max_per_day": 10000
* } }
**/ """;
@Multiline
public static String alert1;
/** private final String alert2 = """
* { {
* "siembol_alerts_full_rule_name": "alert1_v3", "siembol_alerts_full_rule_name": "alert1_v3",
* "siembol_alerts_rule_name": "alert2", "siembol_alerts_rule_name": "alert2",
* "correlation_key": "evil", "correlation_key": "evil",
* "sensor": "a", "sensor": "a",
* "siembol_alerts_max_per_hour": 200, "siembol_alerts_max_per_hour": 200,
* "siembol_alerts_test": "true", "siembol_alerts_test": "true",
* "source_type": "a", "source_type": "a",
* "siembol_alerts_max_per_day": 10000 "siembol_alerts_max_per_day": 10000
* } }
**/ """;
@Multiline
public static String alert2;
private final String simpleCorrelationRules = """
{
"rules_version": 1,
"tags": [
{
"tag_name": "detection_source",
"tag_value": "siembol_correlation_alerts_instance"
}
],
"rules": [
{
"tags": [
{
"tag_name": "test",
"tag_value": "true"
}
],
"rule_protection": {
"max_per_hour": 500,
"max_per_day": 1000
},
"rule_name": "test_rule",
"rule_version": 1,
"rule_author": "dummy",
"rule_description": "Testing rule",
"correlation_attributes": {
"time_unit": "seconds",
"time_window": 500,
"time_computation_type": "processing_time",
"alerts": [
{
"alert": "alert1",
"threshold": 2
},
{
"alert": "alert2",
"threshold": 1
}
]
}
}
]
}
}
""";
/**
* {
* "rules_version": 1,
* "tags": [
* {
* "tag_name": "detection_source",
* "tag_value": "siembol_correlation_alerts_instance"
* }
* ],
* "rules": [
* {
* "tags": [
* {
* "tag_name": "test",
* "tag_value": "true"
* }
* ],
* "rule_protection": {
* "max_per_hour": 500,
* "max_per_day": 1000
* },
* "rule_name": "test_rule",
* "rule_version": 1,
* "rule_author": "dummy",
* "rule_description": "Testing rule",
* "correlation_attributes": {
* "time_unit": "seconds",
* "time_window": 500,
* "time_computation_type": "processing_time",
* "alerts": [
* {
* "alert": "alert1",
* "threshold": 2
* },
* {
* "alert": "alert2",
* "threshold": 1
* }
* ]
* }
* }
* ]
* }
*}
**/
@Multiline
public static String simpleCorrelationRules;
private Tuple tuple; private Tuple tuple;
private OutputCollector collector; private OutputCollector collector;

118
alerting/alerting-storm/src/test/java/uk/co/gresearch/siembol/alerts/storm/KafkaWriterBoltTest.java
View File

@@ -5,13 +5,9 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectReader;
import com.github.charithe.kafka.EphemeralKafkaBroker; import com.github.charithe.kafka.EphemeralKafkaBroker;
import com.github.charithe.kafka.KafkaJunitRule; import com.github.charithe.kafka.KafkaJunitRule;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.storm.task.OutputCollector; import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Tuple;
import org.junit.Assert; import org.junit.*;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
import org.mockito.Mockito; import org.mockito.Mockito;
import uk.co.gresearch.siembol.alerts.common.AlertingEngineType; import uk.co.gresearch.siembol.alerts.common.AlertingEngineType;
@@ -32,68 +28,62 @@ public class KafkaWriterBoltTest {
private static final ObjectReader JSON_PARSERS_CONFIG_READER = new ObjectMapper() private static final ObjectReader JSON_PARSERS_CONFIG_READER = new ObjectMapper()
.readerFor(AlertingStormAttributesDto.class); .readerFor(AlertingStormAttributesDto.class);
private static final ObjectReader JSON_MAP_READER = new ObjectMapper() private static final ObjectReader JSON_MAP_READER = new ObjectMapper()
.readerFor(new TypeReference<Map<String, Object>>() {}); .readerFor(new TypeReference<Map<String, Object>>() {
});
/** private final String alertingStormConfig = """
*{ {
* "alerts.engine": "siembol_alerts", "alerts.engine": "siembol_alerts",
* "alerts.input.topics": [ "enrichmnents" ], "alerts.input.topics": [ "enrichmnents" ],
* "alerts.correlation.output.topic": "correlation.alerts", "alerts.correlation.output.topic": "correlation.alerts",
* "kafka.error.topic": "errors", "kafka.error.topic": "errors",
* "alerts.output.topic": "alerts", "alerts.output.topic": "alerts",
* "storm.attributes": { "storm.attributes": {
* "first.pool.offset.strategy": "EARLIEST", "first.pool.offset.strategy": "EARLIEST",
* "kafka.spout.properties": { "kafka.spout.properties": {
* "group.id": "alerts.reader", "group.id": "alerts.reader",
* "security.protocol": "PLAINTEXT" "security.protocol": "PLAINTEXT"
* } }
* }, },
* "kafka.spout.num.executors": 1, "kafka.spout.num.executors": 1,
* "alerts.engine.bolt.num.executors": 1, "alerts.engine.bolt.num.executors": 1,
* "kafka.writer.bolt.num.executors": 1, "kafka.writer.bolt.num.executors": 1,
* "kafka.producer.properties": { "kafka.producer.properties": {
* "compression.type": "snappy", "compression.type": "snappy",
* "security.protocol": "PLAINTEXT", "security.protocol": "PLAINTEXT",
* "client.id": "test_producer" "client.id": "test_producer"
* } }
* } }
**/ """;
@Multiline
public static String alertingStormConfig;
/** private final String AlertMessageStr = """
* { {
* "ip_src_addr": "1.2.3.4", "ip_src_addr": "1.2.3.4",
* "b": 1, "b": 1,
* "is_alert": "true", "is_alert": "true",
* "source_type": "test", "source_type": "test",
* "detection_source": "alerts", "detection_source": "alerts",
* "siembol_alerts_full_rule_name": "alert1_v1", "siembol_alerts_full_rule_name": "alert1_v1",
* "siembol_alerts_rule_name": "alert1", "siembol_alerts_rule_name": "alert1",
* "siembol_alerts_max_per_day": 1, "siembol_alerts_max_per_day": 1,
* "siembol_alerts_max_per_hour": 1 "siembol_alerts_max_per_hour": 1
* } }
**/ """;
@Multiline
public static String AlertMessageStr;
/**
* {
* "ip_src_addr": "1.2.3.4",
* "b": 1,
* "is_alert": "true",
* "source_type": "test",
* "detection_source": "alerts",
* "siembol_alerts_full_rule_name": "alert1_v1",
* "siembol_alerts_rule_name": "alert1",
* "siembol_alerts_max_per_day": 1,
* "siembol_alerts_max_per_hour": 1,
* "correlation_key" : "evil"
* }
**/
@Multiline
public static String AlertMessageCorrelationStr;
private final String AlertMessageCorrelationStr = """
{
"ip_src_addr": "1.2.3.4",
"b": 1,
"is_alert": "true",
"source_type": "test",
"detection_source": "alerts",
"siembol_alerts_full_rule_name": "alert1_v1",
"siembol_alerts_rule_name": "alert1",
"siembol_alerts_max_per_day": 1,
"siembol_alerts_max_per_hour": 1,
"correlation_key" : "evil"
}
""";
@ClassRule @ClassRule
public static KafkaJunitRule kafkaRule = new KafkaJunitRule(EphemeralKafkaBroker.create()); public static KafkaJunitRule kafkaRule = new KafkaJunitRule(EphemeralKafkaBroker.create());

2
alerting/pom.xml
View File

@@ -11,7 +11,7 @@
<parent> <parent>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId> <artifactId>siembol</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</parent> </parent>
<modules> <modules>
<module>alerting-core</module> <module>alerting-core</module>

10
config-editor/config-editor-core/pom.xml
View File

@@ -9,13 +9,13 @@
<parent> <parent>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId> <artifactId>config-editor</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</parent> </parent>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId> <artifactId>siembol-common</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.commons</groupId> <groupId>org.apache.commons</groupId>
@@ -79,12 +79,6 @@
<version>${junit_version}</version> <version>${junit_version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>org.mockito</groupId> <groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId> <artifactId>mockito-core</artifactId>

440
config-editor/config-editor-core/src/test/java/uk/co/gresearch/siembol/configeditor/common/ConfigEditorUtilsTest.java
View File

@@ -4,7 +4,6 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectReader;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
@@ -13,240 +12,229 @@ import java.util.Map;
import java.util.Optional; import java.util.Optional;
public class ConfigEditorUtilsTest { public class ConfigEditorUtilsTest {
/** private final String rulesSchema = """
* { {
* "rules_schema": { "rules_schema": {
* "type": "object", "type": "object",
* "description": "Incident Response Rules", "description": "Incident Response Rules",
* "title": "rules", "title": "rules",
* "properties": { "properties": {
* "rules_version": { "rules_version": {
* "type": "integer", "type": "integer",
* "description": "Incident response rules version", "description": "Incident response rules version",
* "default": 0 "default": 0
* }, },
* "rules": { "rules": {
* "type": "array", "type": "array",
* "items": { "items": {
* "type": "object", "type": "object",
* "description": "Response rule that should handle response to a siembol alert", "description": "Response rule that should handle response to a siembol alert",
* "title": "rule", "title": "rule",
* "properties": { "properties": {
* "rule_name": { "rule_name": {
* "type": "string", "type": "string",
* "description": "ResponseRule name that uniquely identifies the rule" "description": "ResponseRule name that uniquely identifies the rule"
* }, },
* "rule_author": { "rule_author": {
* "type": "string", "type": "string",
* "description": "The owner of the rule" "description": "The owner of the rule"
* }, },
* "rule_version": { "rule_version": {
* "type": "integer", "type": "integer",
* "description": "The version of the rule", "description": "The version of the rule",
* "default": 0 "default": 0
* }, },
* "rule_description": { "rule_description": {
* "type": "string", "type": "string",
* "description": "The description of the rule" "description": "The description of the rule"
* }, },
* "evaluators": { "evaluators": {
* "type": "array", "type": "array",
* "items": { "items": {
* "type": "object", "type": "object",
* "description": "Response evaluator used in response rules", "description": "Response evaluator used in response rules",
* "title": "response evaluator", "title": "response evaluator",
* "oneOf": [ "oneOf": [
* { {
* "type": "object", "type": "object",
* "title": "matching_evaluator", "title": "matching_evaluator",
* "properties": { "properties": {
* "evaluator_type": { "evaluator_type": {
* "enum": [ "enum": [
* "matching_evaluator" "matching_evaluator"
* ], ],
* "default": "matching_evaluator" "default": "matching_evaluator"
* }, },
* "evaluator_attributes": { "evaluator_attributes": {
* "type": "object", "type": "object",
* "description": "Attributes for matching evaluator", "description": "Attributes for matching evaluator",
* "title": "matching evaluator attributes", "title": "matching evaluator attributes",
* "properties": { "properties": {
* "evaluation_result": { "evaluation_result": {
* "enum": [ "enum": [
* "match", "match",
* "filtered" "filtered"
* ], ],
* "type": "string", "type": "string",
* "description": "Evaluation result returned by the evaluator after matching", "description": "Evaluation result returned by the evaluator after matching",
* "default": "match" "default": "match"
* }, },
* "matchers": { "matchers": {
* "type": "array", "type": "array",
* "items": { "items": {
* "type": "object", "type": "object",
* "description": "Matcher for matching fields in response rules", "description": "Matcher for matching fields in response rules",
* "title": "matcher", "title": "matcher",
* "properties": { "properties": {
* "matcher_type": { "matcher_type": {
* "enum": [ "enum": [
* "REGEX_MATCH", "REGEX_MATCH",
* "IS_IN_SET" "IS_IN_SET"
* ], ],
* "type": "string", "type": "string",
* "description": "Type of matcher, either Regex match or list of strings (newline delimited)" "description": "Type of matcher, either Regex match or list of strings (newline delimited)"
* }, },
* "is_negated": { "is_negated": {
* "type": "boolean", "type": "boolean",
* "description": "The matcher is negated", "description": "The matcher is negated",
* "default": false "default": false
* }, },
* "field": { "field": {
* "type": "string", "type": "string",
* "description": "Field on which the matcher will be evaluated" "description": "Field on which the matcher will be evaluated"
* }, },
* "case_insensitive": { "case_insensitive": {
* "type": "boolean", "type": "boolean",
* "description": "Use case insensitive string compare", "description": "Use case insensitive string compare",
* "default": false "default": false
* }, },
* "data": { "data": {
* "type": "string", "type": "string",
* "description": "Matcher expression as defined by matcher type" "description": "Matcher expression as defined by matcher type"
* } }
* }, },
* "required": [ "required": [
* "data", "data",
* "field", "field",
* "matcher_type" "matcher_type"
* ] ]
* }, },
* "description": "Matchers of the evaluator", "description": "Matchers of the evaluator",
* "minItems": 1 "minItems": 1
* } }
* }, },
* "required": [ "required": [
* "evaluation_result", "evaluation_result",
* "matchers" "matchers"
* ] ]
* } }
* }, },
* "required": [ "required": [
* "evaluator_type", "evaluator_type",
* "evaluator_attributes" "evaluator_attributes"
* ] ]
* } }
* ] ]
* }, },
* "description": "Evaluators of the rule", "description": "Evaluators of the rule",
* "minItems": 1 "minItems": 1
* } }
* }, },
* "required": [ "required": [
* "evaluators", "evaluators",
* "rule_author", "rule_author",
* "rule_name", "rule_name",
* "rule_version" "rule_version"
* ] ]
* }, },
* "description": "Response rules", "description": "Response rules",
* "minItems": 1 "minItems": 1
* } }
* }, },
* "required": [ "required": [
* "rules", "rules",
* "rules_version" "rules_version"
* ] ]
* } }
* } }
*/ """;
@Multiline
public static String rulesSchema;
/** private final String layoutConfig = """
* { {
* "$..evaluators": { "$..evaluators": {
* "widget": { "widget": {
* "formlyConfig": { "formlyConfig": {
* "type": "tab-array" "type": "tab-array"
* } }
* } }
* }, },
* "$..rule_description": { "$..rule_description": {
* "widget": { "widget": {
* "formlyConfig": { "formlyConfig": {
* "type": "textarea", "type": "textarea",
* "wrappers": [] "wrappers": []
* } }
* } }
* }, },
* "$..matchers.items": { "$..matchers.items": {
* "widget": { "widget": {
* "formlyConfig": { "formlyConfig": {
* "wrappers": [ "wrappers": [
* "expansion-panel" "expansion-panel"
* ] ]
* } }
* } }
* }, },
* "$..matchers.items.properties.data": { "$..matchers.items.properties.data": {
* "title" : "changed" "title" : "changed"
* } }
* } }
*/ """;
@Multiline
public static String layoutConfig;
/**
* {
* "robots": {
* "type": "tab-array"
* }
* }
*/
@Multiline
public static String unknownKeyConfig;
/** private final String unknownKeyConfig = """
* { {
* "$..items": { "robots": {
* "type": "tab-array" "type": "tab-array"
* } }
* } }
*/ """;
@Multiline
public static String multipleKeyConfig;
/** private final String multipleKeyConfig = """
* { {
* "rules_schema.description": { "$..items": {
* "type": "tab-array" "type": "tab-array"
* } }
* } }
*/ """;
@Multiline
public static String valueWithString;
/** private final String valueWithString = """
* { {
* "$..evaluators": { "rules_schema.description": {
* } "type": "tab-array"
* } }
*/ }
@Multiline """;
public static String valueEmptyObject;
/** private final String valueEmptyObject = """
* { {
* "$..evaluators": "dummy" "$..evaluators": {
* } }
*/ }
@Multiline """;
public static String valueString;
private final String valueString = """
{
"$..evaluators": "dummy"
}
""";
private static final ObjectReader JSON_OBJECT_READER = new ObjectMapper() private static final ObjectReader JSON_OBJECT_READER = new ObjectMapper()
.readerFor(new TypeReference<Map<String, Object>>() {}); .readerFor(new TypeReference<Map<String, Object>>() {
});
private static final ObjectReader FORM_ATTRIBUTES_READER = new ObjectMapper() private static final ObjectReader FORM_ATTRIBUTES_READER = new ObjectMapper()
.readerFor(new TypeReference<Map<String, JsonNode>>() {}); .readerFor(new TypeReference<Map<String, JsonNode>>() {
});
@Test @Test

20
config-editor/config-editor-core/src/test/java/uk/co/gresearch/siembol/configeditor/configinfo/AdminConfigInfoProviderTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.configeditor.configinfo; package uk.co.gresearch.siembol.configeditor.configinfo;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -9,16 +8,14 @@ import uk.co.gresearch.siembol.configeditor.common.UserInfo;
import uk.co.gresearch.siembol.configeditor.common.ConfigInfo; import uk.co.gresearch.siembol.configeditor.common.ConfigInfo;
public class AdminConfigInfoProviderTest { public class AdminConfigInfoProviderTest {
/** private final String testConfig = """
* { {
* "config_version": 1, "config_version": 1,
* "secret": "john", "secret": "john",
* "object": { }, "object": { },
* "actions": [ "test" , "siembol"] "actions": [ "test" , "siembol"]
* } }
**/ """;
@Multiline
public static String testConfig;
private final ConfigInfoProvider infoProvider = new AdminConfigInfoProvider(); private final ConfigInfoProvider infoProvider = new AdminConfigInfoProvider();
private UserInfo steve; private UserInfo steve;
@@ -54,5 +51,4 @@ public class AdminConfigInfoProviderTest {
Assert.assertEquals(infoProvider.isReleaseFile("admin_config.json"), true); Assert.assertEquals(infoProvider.isReleaseFile("admin_config.json"), true);
Assert.assertEquals(infoProvider.isReleaseFile("rules.json"), false); Assert.assertEquals(infoProvider.isReleaseFile("rules.json"), false);
} }
} }

174
config-editor/config-editor-core/src/test/java/uk/co/gresearch/siembol/configeditor/configinfo/JsonRuleConfigInfoProviderTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.configeditor.configinfo; package uk.co.gresearch.siembol.configeditor.configinfo;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -14,102 +13,91 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
public class JsonRuleConfigInfoProviderTest { public class JsonRuleConfigInfoProviderTest {
/** private final String testRule = """
* { {
* "rule_name": "info_provider-test", "rule_name": "info_provider-test",
* "rule_author": "john", "rule_author": "john",
* "rule_version": 12345, "rule_version": 12345,
* "rule_description": "Test rule", "rule_description": "Test rule",
* "enrichments": { }, "enrichments": { },
* "actions": { } "actions": { }
* } }
**/ """;
@Multiline
public static String testRule;
/**
* {
* "rule_name": "info_provider_test",
* "rule_author": "john",
* "rule_version": 0,
* "rule_description": "Test rule",
* "enrichments": { },
* "actions": { }
* }
**/
@Multiline
public static String testNewRule;
/** private final String testNewRule = """
* { {
* "rules_version" : 1, "rule_name": "info_provider_test",
* "rules": [{ "rule_author": "john",
* "rule_name": "info_provider_test", "rule_version": 0,
* "rule_author": "mark", "rule_description": "Test rule",
* "rule_version": 12, "enrichments": { },
* "rule_description": "Test rule", "actions": { }
* "enrichments": { }, }
* "actions": { } """;
* }]
* }
**/
@Multiline
public static String release;
/** private final String release = """
* { {
* "rules_version" : 1, "rules_version" : 1,
* "rules": [] "rules": [{
* } "rule_name": "info_provider_test",
**/ "rule_author": "mark",
@Multiline "rule_version": 12,
public static String releaseNoRules; "rule_description": "Test rule",
"enrichments": { },
"actions": { }
}]
}
""";
/** private final String releaseNoRules = """
* { {
* "rules_version": 1, "rules_version" : 1,
* "rules": [ "rules": []
* { }
* "rule_name": "info_provider_test", """;
* "rule_author": "mark",
* "rule_version": 1,
* "rule_description": "Test rule",
* "enrichments": {},
* "actions": {}
* },
* {
* "rule_name": "info_provider_extra_test",
* "rule_author": "mark",
* "rule_version": 1,
* "rule_description": "Test rule",
* "enrichments": {},
* "actions": {}
* },
* {
* "rule_name": "secret_test",
* "rule_author": "mark",
* "rule_version": 1,
* "rule_description": "Test rule",
* "enrichments": {},
* "actions": {}
* }
* ]
* }
**/
@Multiline
public static String releaseThreeRules;
/** private final String releaseThreeRules = """
* { {
* "rule_name": "../../../test", "rules_version": 1,
* "rule_author": "steve", "rules": [
* "rule_version": 12345, {
* "rule_description": "Test rule", "rule_name": "info_provider_test",
* "enrichments": { }, "rule_author": "mark",
* "actions": { } "rule_version": 1,
* } "rule_description": "Test rule",
**/ "enrichments": {},
@Multiline "actions": {}
public static String maliciousRule; },
{
"rule_name": "info_provider_extra_test",
"rule_author": "mark",
"rule_version": 1,
"rule_description": "Test rule",
"enrichments": {},
"actions": {}
},
{
"rule_name": "secret_test",
"rule_author": "mark",
"rule_version": 1,
"rule_description": "Test rule",
"enrichments": {},
"actions": {}
}
]
}
""";
private final String maliciousRule = """
{
"rule_name": "../../../test",
"rule_author": "steve",
"rule_version": 12345,
"rule_description": "Test rule",
"enrichments": { },
"actions": { }
}
""";
private final ConfigInfoProvider infoProvider = JsonRuleConfigInfoProvider.create(); private final ConfigInfoProvider infoProvider = JsonRuleConfigInfoProvider.create();

188
config-editor/config-editor-core/src/test/java/uk/co/gresearch/siembol/configeditor/configinfo/TestCaseConfigInfoProviderTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.configeditor.configinfo; package uk.co.gresearch.siembol.configeditor.configinfo;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -11,105 +10,98 @@ import uk.co.gresearch.siembol.configeditor.common.ConfigInfoType;
import java.util.ArrayList; import java.util.ArrayList;
public class TestCaseConfigInfoProviderTest { public class TestCaseConfigInfoProviderTest {
/** private final String testCase = """
* { {
* "test_case_name": "test_case", "test_case_name": "test_case",
* "version": 12345, "version": 12345,
* "author": "john", "author": "john",
* "config_name": "syslog", "config_name": "syslog",
* "description": "unitest test case", "description": "unitest test case",
* "test_specification": { "test_specification": {
* "secret": true "secret": true
* }, },
* "assertions": [ "assertions": [
* { {
* "assertion_type": "path_and_value_matches", "assertion_type": "path_and_value_matches",
* "json_path": "$.a", "json_path": "$.a",
* "expected_pattern": "^.*mp$", "expected_pattern": "^.*mp$",
* "negated_pattern": false, "negated_pattern": false,
* "description": "match string", "description": "match string",
* "active": true "active": true
* }, },
* { {
* "assertion_type": "only_if_path_exists", "assertion_type": "only_if_path_exists",
* "json_path": "s", "json_path": "s",
* "expected_pattern": "secret", "expected_pattern": "secret",
* "negated_pattern": true, "negated_pattern": true,
* "description": "skipped assertion", "description": "skipped assertion",
* "active": false "active": false
* } }
* ] ]
* } }
*/ """;
@Multiline
public static String testCase;
/** private final String testCaseNew = """
* { {
* "test_case_name": "test_case", "test_case_name": "test_case",
* "version": 0, "version": 0,
* "author": "john", "author": "john",
* "config_name": "syslog", "config_name": "syslog",
* "description": "unitest test case", "description": "unitest test case",
* "test_specification": { "test_specification": {
* "secret": true "secret": true
* }, },
* "assertions": [ "assertions": [
* { {
* "assertion_type": "path_and_value_matches", "assertion_type": "path_and_value_matches",
* "json_path": "$.a", "json_path": "$.a",
* "expected_pattern": "^.*mp$", "expected_pattern": "^.*mp$",
* "negated_pattern": false, "negated_pattern": false,
* "description": "match string", "description": "match string",
* "active": true "active": true
* }, },
* { {
* "assertion_type": "only_if_path_exists", "assertion_type": "only_if_path_exists",
* "json_path": "s", "json_path": "s",
* "expected_pattern": "secret", "expected_pattern": "secret",
* "negated_pattern": true, "negated_pattern": true,
* "description": "skipped assertion", "description": "skipped assertion",
* "active": false "active": false
* } }
* ] ]
* } }
*/ """;
@Multiline
public static String testCaseNew;
/**
* {
* "test_case_name": "./../../test",
* "version": 1,
* "author": "john",
* "config_name": "syslog",
* "description": "unitest test case",
* "test_specification": {
* "secret": true
* },
* "assertions": [
* {
* "assertion_type": "path_and_value_matches",
* "json_path": "$.a",
* "expected_pattern": "^.*mp$",
* "negated_pattern": false,
* "description": "match string",
* "active": true
* },
* {
* "assertion_type": "only_if_path_exists",
* "json_path": "s",
* "expected_pattern": "secret",
* "negated_pattern": true,
* "description": "skipped assertion",
* "active": false
* }
* ]
* }
*/
@Multiline
public static String maliciousTestCase;
private final String maliciousTestCase = """
{
"test_case_name": "./../../test",
"version": 1,
"author": "john",
"config_name": "syslog",
"description": "unitest test case",
"test_specification": {
"secret": true
},
"assertions": [
{
"assertion_type": "path_and_value_matches",
"json_path": "$.a",
"expected_pattern": "^.*mp$",
"negated_pattern": false,
"description": "match string",
"active": true
},
{
"assertion_type": "only_if_path_exists",
"json_path": "s",
"expected_pattern": "secret",
"negated_pattern": true,
"description": "skipped assertion",
"active": false
}
]
}
""";
private final TestCaseInfoProvider infoProvider = new TestCaseInfoProvider(); private final TestCaseInfoProvider infoProvider = new TestCaseInfoProvider();
private UserInfo steve; private UserInfo steve;

317
config-editor/config-editor-core/src/test/java/uk/co/gresearch/siembol/configeditor/testcase/TestCaseEvaluatorImplTest.java
View File

@@ -1,7 +1,7 @@
package uk.co.gresearch.siembol.configeditor.testcase; package uk.co.gresearch.siembol.configeditor.testcase;
import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -13,173 +13,162 @@ import static uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult.Stat
import static uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult.StatusCode.OK; import static uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult.StatusCode.OK;
public class TestCaseEvaluatorImplTest { public class TestCaseEvaluatorImplTest {
/** private final String testResult = """
* { {
* "a" : "tmp", "a" : "tmp",
* "b" : true, "b" : true,
* "c" : "test", "c" : "test",
* "d" : { "e" : "test"}, "d" : { "e" : "test"},
* "f" : [ "1", "2"] "f" : [ "1", "2"]
* } }
**/ """;
@Multiline
public static String testResult;
/**
* {
* "test_case_name": "test",
* "version": 1,
* "author": "john",
* "config_name": "syslog",
* "description": "unitest test case",
* "test_specification": {
* "secret": true
* },
* "assertions": [
* {
* "assertion_type": "path_and_value_matches",
* "json_path": "$.a",
* "expected_pattern": "^.*mp$",
* "negated_pattern": false,
* "description": "match string",
* "active": true
* },
* {
* "assertion_type": "only_if_path_exists",
* "json_path": "s",
* "expected_pattern": "secret",
* "negated_pattern": true,
* "description": "skipped assertion",
* "active": false
* }
* ]
* }
*/
@Multiline
public static String simpleTestCaseString;
/** private final String simpleTestCaseString = """
* { {
* "test_case_name": "test", "test_case_name": "test",
* "version": 1, "version": 1,
* "author": "john", "author": "john",
* "config_name": "syslog", "config_name": "syslog",
* "description": "unitest test case", "description": "unitest test case",
* "test_specification": { "test_specification": {
* "secret": true "secret": true
* }, },
* "assertions": [ "assertions": [
* { {
* "assertion_type": "only_if_path_exists", "assertion_type": "path_and_value_matches",
* "json_path": "$.b", "json_path": "$.a",
* "expected_pattern": "secret", "expected_pattern": "^.*mp$",
* "negated_pattern": true, "negated_pattern": false,
* "description": "negated match of boolean", "description": "match string",
* "active": true "active": true
* }, },
* { {
* "assertion_type": "only_if_path_exists", "assertion_type": "only_if_path_exists",
* "json_path": "s", "json_path": "s",
* "expected_pattern": "secret", "expected_pattern": "secret",
* "negated_pattern": true, "negated_pattern": true,
* "description": "skipped assertion", "description": "skipped assertion",
* "active": false "active": false
* } }
* ] ]
* } }
*/ """;
@Multiline
public static String simpleTestCaseBoolean;
/** private final String simpleTestCaseBoolean = """
* { {
* "test_case_name": "test", "test_case_name": "test",
* "version": 1, "version": 1,
* "author": "john", "author": "john",
* "config_name": "syslog", "config_name": "syslog",
* "description": "unitest test case", "description": "unitest test case",
* "test_specification": { "test_specification": {
* "secret": true "secret": true
* }, },
* "assertions": [ "assertions": [
* { {
* "assertion_type": "only_if_path_exists", "assertion_type": "only_if_path_exists",
* "json_path": "$.d", "json_path": "$.b",
* "expected_pattern": ".*ask", "expected_pattern": "secret",
* "negated_pattern": false, "negated_pattern": true,
* "description": "fail to match object", "description": "negated match of boolean",
* "active": true "active": true
* }, },
* { {
* "assertion_type": "only_if_path_exists", "assertion_type": "only_if_path_exists",
* "json_path": "s", "json_path": "s",
* "expected_pattern": "secret", "expected_pattern": "secret",
* "negated_pattern": true, "negated_pattern": true,
* "description": "skipped assertion", "description": "skipped assertion",
* "active": false "active": false
* } }
* ] ]
* } }
*/ """;
@Multiline
public static String simpleTestCaseObject;
/** private final String simpleTestCaseObject = """
* { {
* "test_case_name": "test", "test_case_name": "test",
* "version": 1, "version": 1,
* "author": "john", "author": "john",
* "config_name": "syslog", "config_name": "syslog",
* "description": "unitest test case", "description": "unitest test case",
* "test_specification": { "test_specification": {
* "secret": true "secret": true
* }, },
* "assertions": [ "assertions": [
* { {
* "assertion_type": "only_if_path_exists", "assertion_type": "only_if_path_exists",
* "json_path": "$.f", "json_path": "$.d",
* "expected_pattern": ".*1.*", "expected_pattern": ".*ask",
* "negated_pattern": false, "negated_pattern": false,
* "description": "match in array", "description": "fail to match object",
* "active": true "active": true
* } },
* ] {
* } "assertion_type": "only_if_path_exists",
*/ "json_path": "s",
@Multiline "expected_pattern": "secret",
public static String simpleTestCaseArray; "negated_pattern": true,
"description": "skipped assertion",
"active": false
}
]
}
""";
/** private final String simpleTestCaseArray = """
* { {
* "test_case_name": "test", "test_case_name": "test",
* "version": 1, "version": 1,
* "author": "john", "author": "john",
* "config_name": "syslog", "config_name": "syslog",
* "description": "unitest test case", "description": "unitest test case",
* "test_specification": { "test_specification": {
* "secret": true "secret": true
* }, },
* "assertions": [ "assertions": [
* { {
* "assertion_type": "only_if_path_exists", "assertion_type": "only_if_path_exists",
* "json_path": "$.g", "json_path": "$.f",
* "expected_pattern": "secret", "expected_pattern": ".*1.*",
* "negated_pattern": true, "negated_pattern": false,
* "description": "only if path exists test", "description": "match in array",
* "active": true "active": true
* }, }
* { ]
* "assertion_type": "only_if_path_exists", }
* "json_path": "s", """;
* "expected_pattern": "secret",
* "negated_pattern": true, private final String simpleTestCaseMissing = """
* "description": "skipped assertion", {
* "active": false "test_case_name": "test",
* } "version": 1,
* ] "author": "john",
* } "config_name": "syslog",
*/ "description": "unitest test case",
@Multiline "test_specification": {
public static String simpleTestCaseMissing; "secret": true
},
"assertions": [
{
"assertion_type": "only_if_path_exists",
"json_path": "$.g",
"expected_pattern": "secret",
"negated_pattern": true,
"description": "only if path exists test",
"active": true
},
{
"assertion_type": "only_if_path_exists",
"json_path": "s",
"expected_pattern": "secret",
"negated_pattern": true,
"description": "skipped assertion",
"active": false
}
]
}
""";
private TestCaseEvaluator testCaseEvaluator; private TestCaseEvaluator testCaseEvaluator;

18
config-editor/config-editor-rest/pom.xml
View File

@@ -9,7 +9,7 @@
<parent> <parent>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId> <artifactId>config-editor</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</parent> </parent>
<dependencyManagement> <dependencyManagement>
<dependencies> <dependencies>
@@ -56,7 +56,7 @@
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId> <artifactId>siembol-common</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
<exclusions> <exclusions>
<exclusion> <exclusion>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
@@ -67,22 +67,22 @@
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-core</artifactId> <artifactId>config-editor-core</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-services</artifactId> <artifactId>config-editor-services</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-sync</artifactId> <artifactId>config-editor-sync</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId> <artifactId>alerting-core</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
<exclusions> <exclusions>
<exclusion> <exclusion>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
@@ -93,7 +93,7 @@
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId> <artifactId>parsing-app</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
<exclusions> <exclusions>
<exclusion> <exclusion>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
@@ -104,7 +104,7 @@
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-core</artifactId> <artifactId>enriching-core</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
<exclusions> <exclusions>
<exclusion> <exclusion>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
@@ -115,7 +115,7 @@
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding-core</artifactId> <artifactId>responding-core</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
<exclusions> <exclusions>
<exclusion> <exclusion>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>

45
config-editor/config-editor-rest/src/main/java/uk/co/gresearch/siembol/configeditor/rest/application/ConfigEditorConfiguration.java
View File

@@ -7,6 +7,7 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.DependsOn;
import org.springframework.util.ResourceUtils; import org.springframework.util.ResourceUtils;
import uk.co.gresearch.siembol.common.model.ZooKeeperAttributesDto; import uk.co.gresearch.siembol.common.model.ZooKeeperAttributesDto;
import uk.co.gresearch.siembol.common.testing.TestingZooKeeperConnectorFactory; import uk.co.gresearch.siembol.common.testing.TestingZooKeeperConnectorFactory;
@@ -50,7 +51,7 @@ public class ConfigEditorConfiguration implements DisposableBean {
private ServiceAggregator serviceAggregator; private ServiceAggregator serviceAggregator;
@Bean @Bean("serviceAggregator")
ServiceAggregator serviceAggregator() throws Exception { ServiceAggregator serviceAggregator() throws Exception {
Map<String, ConfigStoreProperties> configStorePropertiesMap = ConfigEditorHelper Map<String, ConfigStoreProperties> configStorePropertiesMap = ConfigEditorHelper
.getConfigStoreProperties(this.properties); .getConfigStoreProperties(this.properties);
@@ -76,33 +77,28 @@ public class ConfigEditorConfiguration implements DisposableBean {
return serviceAggregator; return serviceAggregator;
} }
@Bean @Bean("testCaseEvaluator")
TestCaseEvaluator testCaseEvaluator() throws Exception { TestCaseEvaluator testCaseEvaluator() throws Exception {
ConfigEditorUiLayout uiLayout = ConfigEditorUtils.readUiLayoutFile(properties.getTestCasesUiConfigFileName()); ConfigEditorUiLayout uiLayout = ConfigEditorUtils.readUiLayoutFile(properties.getTestCasesUiConfigFileName());
return new TestCaseEvaluatorImpl(uiLayout); return new TestCaseEvaluatorImpl(uiLayout);
} }
@Bean @Bean("stormApplicationProvider")
@ConditionalOnProperty(prefix = "config-editor", value = "synchronisation") @ConditionalOnProperty(prefix = "config-editor", value = "synchronisation")
StormApplicationProvider stormApplicationProvider() throws Exception { @DependsOn("zooKeeperConnectorFactory")
return StormApplicationProviderImpl.create(zooKeeperConnectorFactory(), properties.getStormTopologiesZooKeeper()); StormApplicationProvider stormApplicationProvider(
@Autowired ZooKeeperConnectorFactory zooKeeperConnectorFactory) throws Exception {
return StormApplicationProviderImpl.create(zooKeeperConnectorFactory, properties.getStormTopologiesZooKeeper());
} }
@Bean @Bean("synchronisationService")
@ConditionalOnProperty(prefix = "config-editor", value = "synchronisation") @ConditionalOnProperty(prefix = "config-editor", value = "synchronisation")
SynchronisationService synchronisationService() throws Exception { @DependsOn({"zooKeeperConnectorFactory", "stormApplicationProvider"})
SynchronisationService synchronisationService(
@Autowired ZooKeeperConnectorFactory zooKeeperConnectorFactory,
@Autowired StormApplicationProvider stormApplicationProvider) throws Exception {
serviceAggregator = serviceAggregator(); serviceAggregator = serviceAggregator();
ZooKeeperConnectorFactory zooKeeperConnectorFactory = zooKeeperConnectorFactory();
Map<String, ZooKeeperConnector> zooKeeperConnectorMap = new HashMap<>();
if (properties.getEnrichmentTablesZooKeeper() != null) {
for (Map.Entry<String, ZooKeeperAttributesDto> entry : properties.getEnrichmentTablesZooKeeper().entrySet()) {
zooKeeperConnectorMap.put(entry.getKey(),
zooKeeperConnectorFactory.createZookeeperConnector(entry.getValue()));
}
}
enrichmentTablesProvider(zooKeeperConnectorMap);
StormApplicationProvider stormApplicationProvider = stormApplicationProvider();
List<ConfigServiceHelper> aggregatorServices = serviceAggregator List<ConfigServiceHelper> aggregatorServices = serviceAggregator
.getAggregatorServices() .getAggregatorServices()
.stream() .stream()
@@ -117,7 +113,7 @@ public class ConfigEditorConfiguration implements DisposableBean {
return ret; return ret;
} }
@Bean @Bean("zooKeeperConnectorFactory")
@ConditionalOnProperty(prefix = "config-editor", value = "synchronisation") @ConditionalOnProperty(prefix = "config-editor", value = "synchronisation")
ZooKeeperConnectorFactory zooKeeperConnectorFactory() throws Exception { ZooKeeperConnectorFactory zooKeeperConnectorFactory() throws Exception {
if (properties.getTestingZookeeperFiles() == null) { if (properties.getTestingZookeeperFiles() == null) {
@@ -134,9 +130,18 @@ public class ConfigEditorConfiguration implements DisposableBean {
return ret; return ret;
} }
@Bean @Bean("enrichmentTablesProvider")
@ConditionalOnProperty(prefix = "config-editor", value = "synchronisation") @ConditionalOnProperty(prefix = "config-editor", value = "synchronisation")
EnrichmentTablesProvider enrichmentTablesProvider(Map<String, ZooKeeperConnector> zooKeeperConnectorMap) { @DependsOn("zooKeeperConnectorFactory")
EnrichmentTablesProvider enrichmentTablesProvider(
@Autowired ZooKeeperConnectorFactory zooKeeperConnectorFactory) throws Exception {
Map<String, ZooKeeperConnector> zooKeeperConnectorMap = new HashMap<>();
if (properties.getEnrichmentTablesZooKeeper() != null) {
for (Map.Entry<String, ZooKeeperAttributesDto> entry : properties.getEnrichmentTablesZooKeeper().entrySet()) {
zooKeeperConnectorMap.put(entry.getKey(),
zooKeeperConnectorFactory.createZookeeperConnector(entry.getValue()));
}
}
return new EnrichmentTablesProviderImpl(zooKeeperConnectorMap); return new EnrichmentTablesProviderImpl(zooKeeperConnectorMap);
} }

26
config-editor/config-editor-services/pom.xml
View File

@@ -10,53 +10,53 @@
<parent> <parent>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId> <artifactId>config-editor</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</parent> </parent>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId> <artifactId>siembol-common</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-core</artifactId> <artifactId>config-editor-core</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId> <artifactId>alerting-core</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-storm</artifactId> <artifactId>alerting-storm</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-storm</artifactId> <artifactId>parsing-storm</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-storm</artifactId> <artifactId>enriching-storm</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId> <artifactId>parsing-app</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-core</artifactId> <artifactId>enriching-core</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding-core</artifactId> <artifactId>responding-core</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>junit</groupId> <groupId>junit</groupId>
@@ -70,12 +70,6 @@
<version>${mockito_version}</version> <version>${mockito_version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId> <groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-yaml</artifactId> <artifactId>jackson-dataformat-yaml</artifactId>

34
config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/alerts/AlertingRuleSchemaServiceTest.java
View File

@@ -1,6 +1,6 @@
package uk.co.gresearch.siembol.configeditor.service.alerts; package uk.co.gresearch.siembol.configeditor.service.alerts;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -29,27 +29,23 @@ import static uk.co.gresearch.siembol.common.result.SiembolResult.StatusCode.OK;
public class AlertingRuleSchemaServiceTest { public class AlertingRuleSchemaServiceTest {
/** private final String testEvent = """
* {"test_event":"true"} {"test_event":"true"}""";
**/
@Multiline private final String testSpecification = """
public static String testEvent; {
/** "event" : {"test_event":"true"}
* { }
* "event" : {"test_event":"true"} """;
* }
**/
@Multiline
public static String testSpecification;
private AlertingRuleSchemaService alertingRuleSchemaService; private AlertingRuleSchemaService alertingRuleSchemaService;
private final String ruleSchema = "dummmy schema"; private final String ruleSchema = "dummy schema";
private final String testSchema = "dummmy test schema"; private final String testSchema = "dummy test schema";
private final String adminSchema = "dummmy admin config schema"; private final String adminSchema = "dummy admin config schema";
private final String testRule = "dummmy rule"; private final String testRule = "dummy rule";
private final String testRules = "dummmy rules"; private final String testRules = "dummy rules";
private final String testResultOutput = "test output"; private final String testResultOutput = "test output";
private final String testConfig = "dummmy config"; private final String testConfig = "dummy config";
private AlertingCompiler alertingCompiler; private AlertingCompiler alertingCompiler;
private AlertingResult alertingResult; private AlertingResult alertingResult;
private AlertingAttributes alertingAttributes; private AlertingAttributes alertingAttributes;

30
config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/alerts/sigma/SigmaConditionTokenTest.java
View File

@@ -1,6 +1,6 @@
package uk.co.gresearch.siembol.configeditor.service.alerts.sigma; package uk.co.gresearch.siembol.configeditor.service.alerts.sigma;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Pair;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
@@ -14,29 +14,17 @@ import java.util.*;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
public class SigmaConditionTokenTest { public class SigmaConditionTokenTest {
/** private final String condition1 = """
* not 1 of filter* not 1 of filter*""";
**/
@Multiline
private static String condition1;
/** private final String condition2 = """
* ( 1 of filter* and not secret) or long ( 1 of filter* and not secret) or long""";
**/
@Multiline
private static String condition2;
/** private final String conditionUnsupported = """
* 1 of A | 1 of B 1 of A | 1 of B""";
**/
@Multiline
private static String conditionUnsupported;
/** private final String conditionUnknownToken = """
* 1 of A $ 1 of B 1 of A $ 1 of B""";
**/
@Multiline
private static String conditionUnknownToken;
private SigmaConditionTokenNode node; private SigmaConditionTokenNode node;
Map<String, SigmaSearch> searches; Map<String, SigmaSearch> searches;

123
config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/alerts/sigma/SigmaRuleImporterTest.java
View File

@@ -3,7 +3,7 @@ package uk.co.gresearch.siembol.configeditor.service.alerts.sigma;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectReader;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -18,69 +18,64 @@ import static uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult.Stat
public class SigmaRuleImporterTest { public class SigmaRuleImporterTest {
private static final ObjectReader ALERTING_RULE_READER = new ObjectMapper().readerFor(RuleDto.class); private static final ObjectReader ALERTING_RULE_READER = new ObjectMapper().readerFor(RuleDto.class);
/** private final String importerAttributes = """
* { {
* "field_mapping": [ "field_mapping": [
* { {
* "sigma_field": "sigma_user", "sigma_field": "sigma_user",
* "siembol_field": "siembol_user" "siembol_field": "siembol_user"
* } }
* ], ],
* "rule_metadata_mapping": { "rule_metadata_mapping": {
* "rule_name": "based_on_${title}", "rule_name": "based_on_${title}",
* "rule_description": "generated from ${description} and id: ${id}", "rule_description": "generated from ${description} and id: ${id}",
* "source_type": "secret_data", "source_type": "secret_data",
* "tags": [ "tags": [
* { {
* "tag_name": "sigma_tags", "tag_name": "sigma_tags",
* "tag_value": "${tags}" "tag_value": "${tags}"
* } }
* ] ]
* } }
* } }
**/ """;
@Multiline
private static String importerAttributes;
/** private final String sigmaRuleExample = """
* title: Sigma Title( Experimental??? title: Sigma Title( Experimental???
* id: d06be400-8045-4200-0067-740a2009db25 id: d06be400-8045-4200-0067-740a2009db25
* status: experimental status: experimental
* description: Detects secret description: Detects secret
* references: references:
* - https://github.com/siembol - https://github.com/siembol
* author: Joe author: Joe
* date: 2021/10/09 date: 2021/10/09
* logsource: logsource:
* category: process_creation category: process_creation
* product: windows product: windows
* detection: detection:
* image_path: image_path:
* Image|endswith: 'secret.exe' Image|endswith: 'secret.exe'
* cmd_s: cmd_s:
* CommandLine|contains: '/S' CommandLine|contains: '/S'
* cmd_c: cmd_c:
* CommandLine|contains: '/C' CommandLine|contains: '/C'
* net_utility: net_utility:
* Image|endswith: Image|endswith:
* - '\net.exe' - '\\net.exe'
* - '\net1.exe' - '\\net1.exe'
* CommandLine|contains: CommandLine|contains:
* - ' user ' - ' user '
* - ' use ' - ' use '
* - ' group ' - ' group '
* condition: image_path and cmd_c and (cmd_s or not net_utility) condition: image_path and cmd_c and (cmd_s or not net_utility)
* fields: fields:
* - CommandLine - CommandLine
* falsepositives: falsepositives:
* - Unknown - Unknown
* level: medium level: medium
* tags: tags:
* - attack.defense_evasion - attack.defense_evasion
* - attack.example - attack.example""";
*/
@Multiline
private static String sigmaRuleExample;
private SigmaRuleImporter importer; private SigmaRuleImporter importer;
@@ -117,7 +112,7 @@ public class SigmaRuleImporterTest {
@Test @Test
public void validateAttributesMissingRequired() { public void validateAttributesMissingRequired() {
ConfigEditorResult result = importer.validateImporterAttributes( ConfigEditorResult result = importer.validateImporterAttributes(
importerAttributes.replace("source_type", "uknown")); importerAttributes.replace("source_type", "unknown"));
Assert.assertEquals(BAD_REQUEST, result.getStatusCode()); Assert.assertEquals(BAD_REQUEST, result.getStatusCode());
Assert.assertNotNull(result.getAttributes().getMessage()); Assert.assertNotNull(result.getAttributes().getMessage());
} }

61
config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/alerts/sigma/SigmaSearchTest.java
View File

@@ -3,7 +3,7 @@ package uk.co.gresearch.siembol.configeditor.service.alerts.sigma;
import com.fasterxml.jackson.core.*; import com.fasterxml.jackson.core.*;
import com.fasterxml.jackson.databind.*; import com.fasterxml.jackson.databind.*;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -16,40 +16,34 @@ import uk.co.gresearch.siembol.configeditor.service.alerts.sigma.model.SigmaDete
import java.util.*; import java.util.*;
public class SigmaSearchTest { public class SigmaSearchTest {
/** private final String sigmaDetectionExample = """
* iptables: iptables:
* Image|endswith: '/service' Image|endswith: '/service'
* CommandLine|contains|all: CommandLine|contains|all:
* - 'iptables' - 'iptables'
* - 'stop' - 'stop'
* - 1 - 1
* keywords: keywords:
* - entered promiscuous mode - entered promiscuous mode
* - 1 - 1
* - test secret word - test secret word
* keyword: keyword:
* - single - single
*/ """;
@Multiline
private static String sigmaDetectionExample;
/** private final String sigmaDetectionExampleEmptyValues = """
* iptables: iptables:
* Image: null Image: null
* test: '' test: ''
*/ """;
@Multiline
private static String sigmaDetectionExampleEmptyValues;
/** private final String sigmaDetectionExampleBooleanValue = """
* iptables: iptables:
* Image: 'abc' Image: 'abc'
* CommandLine|contains|all: CommandLine|contains|all:
* - true - true
* - 1 - 1
*/ """;
@Multiline
private static String sigmaDetectionExampleBooleanValue;
private static final ObjectReader SIGMA_DETECTION_READER = new ObjectMapper(new YAMLFactory()) private static final ObjectReader SIGMA_DETECTION_READER = new ObjectMapper(new YAMLFactory())
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
@@ -209,5 +203,4 @@ public class SigmaSearchTest {
.fieldNames() .fieldNames()
.forEachRemaining(x -> builder.addMapEntry(x, searchesMap.get("iptables").get(x))); .forEachRemaining(x -> builder.addMapEntry(x, searchesMap.get("iptables").get(x)));
} }
} }

150
config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/parserconfig/ParserConfigConfigInfoProviderTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.configeditor.service.parserconfig; package uk.co.gresearch.siembol.configeditor.service.parserconfig;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -13,84 +12,77 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
public class ParserConfigConfigInfoProviderTest { public class ParserConfigConfigInfoProviderTest {
/** private final String testParser = """
* { {
* "parser_name": "test_parser", "parser_name": "test_parser",
* "parser_author": "john", "parser_author": "john",
* "parser_version": 12345, "parser_version": 12345,
* "parser_config": { "parser_config": {
* "parser_attributes": { "parser_attributes": {
* "parser_type": "syslog", "parser_type": "syslog",
* "syslog_config": { "syslog_config": {
* "syslog_version": "RFC_3164", "syslog_version": "RFC_3164",
* "timezone": "UTC" "timezone": "UTC"
* } }
* } }
* } }
* } }
**/ """;
@Multiline
public static String testParser;
/**
* {
* "parser_name": "test_parser",
* "parser_author": "john",
* "parser_version": 0,
* "parser_config": {
* "parser_attributes": {
* "parser_type": "syslog",
* "syslog_config": {
* "syslog_version": "RFC_3164",
* "timezone": "UTC"
* }
* }
* }
* }
**/
@Multiline
public static String testNewParser;
/** private final String testNewParser = """
* { {
* "parsers_version" : 1, "parser_name": "test_parser",
* "parser_configurations": [ "parser_author": "john",
* { "parser_version": 0,
* "parser_name": "test_parser", "parser_config": {
* "parser_author": "john", "parser_attributes": {
* "parser_version": 1, "parser_type": "syslog",
* "parser_config": { "syslog_config": {
* "parser_attributes": { "syslog_version": "RFC_3164",
* "parser_type": "syslog", "timezone": "UTC"
* "syslog_config": { }
* "syslog_version": "RFC_3164", }
* "timezone": "UTC" }
* } }
* } """;
* }
* }]
* }
**/
@Multiline
public static String release;
/** private final String release = """
* { {
* "parser_name": "../../../test_parser", "parsers_version" : 1,
* "parser_author": "john", "parser_configurations": [
* "parser_version": 12345, {
* "parser_config": { "parser_name": "test_parser",
* "parser_attributes": { "parser_author": "john",
* "parser_type": "syslog", "parser_version": 1,
* "syslog_config": { "parser_config": {
* "syslog_version": "RFC_3164", "parser_attributes": {
* "timezone": "UTC" "parser_type": "syslog",
* } "syslog_config": {
* } "syslog_version": "RFC_3164",
* } "timezone": "UTC"
* } }
**/ }
@Multiline }
public static String maliciousConfig; }]
}
""";
private final String maliciousConfig = """
{
"parser_name": "../../../test_parser",
"parser_author": "john",
"parser_version": 12345,
"parser_config": {
"parser_attributes": {
"parser_type": "syslog",
"syslog_config": {
"syslog_version": "RFC_3164",
"timezone": "UTC"
}
}
}
}
""";
private final ConfigInfoProvider infoProvider = ParserConfigConfigInfoProvider.create(); private final ConfigInfoProvider infoProvider = ParserConfigConfigInfoProvider.create();
@@ -121,6 +113,7 @@ public class ParserConfigConfigInfoProviderTest {
Assert.assertEquals(1, info.getFilesContent().size()); Assert.assertEquals(1, info.getFilesContent().size());
Assert.assertTrue(info.getFilesContent().containsKey("test_parser.json")); Assert.assertTrue(info.getFilesContent().containsKey("test_parser.json"));
Assert.assertTrue(info.getFilesContent().get("test_parser.json").isPresent());
Assert.assertTrue(info.getFilesContent() Assert.assertTrue(info.getFilesContent()
.get("test_parser.json").get().indexOf("\"parser_version\": 12346,") > 0); .get("test_parser.json").get().indexOf("\"parser_version\": 12346,") > 0);
Assert.assertTrue(info.getFilesContent() Assert.assertTrue(info.getFilesContent()
@@ -137,6 +130,7 @@ public class ParserConfigConfigInfoProviderTest {
Assert.assertEquals("john@secret.net", info.getCommitterEmail()); Assert.assertEquals("john@secret.net", info.getCommitterEmail());
Assert.assertEquals(1, info.getFilesContent().size()); Assert.assertEquals(1, info.getFilesContent().size());
Assert.assertTrue(info.getFilesContent().containsKey("test_parser.json")); Assert.assertTrue(info.getFilesContent().containsKey("test_parser.json"));
Assert.assertTrue(info.getFilesContent().get("test_parser.json").isPresent());
Assert.assertTrue(info.getFilesContent() Assert.assertTrue(info.getFilesContent()
.get("test_parser.json").get().indexOf("\"parser_version\": 12346,") > 0); .get("test_parser.json").get().indexOf("\"parser_version\": 12346,") > 0);
Assert.assertTrue(info.getFilesContent() Assert.assertTrue(info.getFilesContent()
@@ -153,6 +147,7 @@ public class ParserConfigConfigInfoProviderTest {
Assert.assertEquals(info.getCommitterEmail(), steve.getEmail()); Assert.assertEquals(info.getCommitterEmail(), steve.getEmail());
Assert.assertEquals(1, info.getFilesContent().size()); Assert.assertEquals(1, info.getFilesContent().size());
Assert.assertTrue(info.getFilesContent().containsKey("test_parser.json")); Assert.assertTrue(info.getFilesContent().containsKey("test_parser.json"));
Assert.assertTrue(info.getFilesContent().get("test_parser.json").isPresent());
Assert.assertTrue(info.getFilesContent() Assert.assertTrue(info.getFilesContent()
.get("test_parser.json").get().indexOf("\"parser_version\": 1,") > 0); .get("test_parser.json").get().indexOf("\"parser_version\": 1,") > 0);
Assert.assertTrue(info.isNewConfig()); Assert.assertTrue(info.isNewConfig());
@@ -192,6 +187,7 @@ public class ParserConfigConfigInfoProviderTest {
Assert.assertEquals(1, info.getFilesContent().size()); Assert.assertEquals(1, info.getFilesContent().size());
Assert.assertTrue(info.getFilesContent().containsKey("parsers.json")); Assert.assertTrue(info.getFilesContent().containsKey("parsers.json"));
Assert.assertTrue(info.getFilesContent().get("parsers.json").isPresent());
Assert.assertTrue(info.getFilesContent() Assert.assertTrue(info.getFilesContent()
.get("parsers.json").get().indexOf("\"parsers_version\": 2,") > 0); .get("parsers.json").get().indexOf("\"parsers_version\": 2,") > 0);

29
config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/parserconfig/ParserConfigSchemaServiceTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.configeditor.service.parserconfig; package uk.co.gresearch.siembol.configeditor.service.parserconfig;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -22,23 +21,19 @@ import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.times; import static org.mockito.Mockito.times;
public class ParserConfigSchemaServiceTest { public class ParserConfigSchemaServiceTest {
/** private final String logUtf8 = """
* { {
* "encoding" : "utf8_string", "encoding" : "utf8_string",
* "log" : "dummy log" "log" : "dummy log"
* } }
**/ """;
@Multiline
public static String logUtf8;
/** private final String logHex = """
* { {
* "encoding" : "hex_string", "encoding" : "hex_string",
* "log" : "64756D6D79206C6F67" "log" : "64756D6D79206C6F67"
* } }
**/ """;
@Multiline
public static String logHex;
private ParserConfigSchemaService parserConfigSchemaService; private ParserConfigSchemaService parserConfigSchemaService;
private final String schema = "dummmy schema"; private final String schema = "dummmy schema";

167
config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/parsingapp/ParsingAppConfigInfoProviderTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.configeditor.service.parsingapp; package uk.co.gresearch.siembol.configeditor.service.parsingapp;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -10,93 +9,85 @@ import uk.co.gresearch.siembol.configeditor.common.ConfigInfoProvider;
public class ParsingAppConfigInfoProviderTest { public class ParsingAppConfigInfoProviderTest {
/** private final String simpleSingleApplicationParser = """
*{ {
* "parsing_app_name": "test", "parsing_app_name": "test",
* "parsing_app_version": 12345, "parsing_app_version": 12345,
* "parsing_app_author": "dummy", "parsing_app_author": "dummy",
* "parsing_app_description": "Description of parser application", "parsing_app_description": "Description of parser application",
* "parsing_app_settings": { "parsing_app_settings": {
* "input_topics": [ "input_topics": [
* "secret" "secret"
* ], ],
* "error_topic": "error", "error_topic": "error",
* "input_parallelism": 1, "input_parallelism": 1,
* "parsing_parallelism": 2, "parsing_parallelism": 2,
* "output_parallelism": 3, "output_parallelism": 3,
* "parsing_app_type": "single_parser" "parsing_app_type": "single_parser"
* }, },
* "parsing_settings": { "parsing_settings": {
* "single_parser": { "single_parser": {
* "parser_name": "single", "parser_name": "single",
* "output_topic": "output" "output_topic": "output"
* } }
* } }
* } }
**/ """;
@Multiline
static String simpleSingleApplicationParser;
/** private final String simpleSingleApplicationParserNew = """
*{ {
* "parsing_app_name": "test", "parsing_app_name": "test",
* "parsing_app_version": 0, "parsing_app_version": 0,
* "parsing_app_author": "dummy", "parsing_app_author": "dummy",
* "parsing_app_description": "Description of parser application", "parsing_app_description": "Description of parser application",
* "parsing_app_settings": { "parsing_app_settings": {
* "input_topics": [ "input_topics": [
* "secret" "secret"
* ], ],
* "error_topic": "error", "error_topic": "error",
* "input_parallelism": 1, "input_parallelism": 1,
* "parsing_parallelism": 2, "parsing_parallelism": 2,
* "output_parallelism": 3, "output_parallelism": 3,
* "parsing_app_type": "single_parser" "parsing_app_type": "single_parser"
* }, },
* "parsing_settings": { "parsing_settings": {
* "single_parser": { "single_parser": {
* "parser_name": "single", "parser_name": "single",
* "output_topic": "output" "output_topic": "output"
* } }
* } }
* } }
**/ """;
@Multiline
static String simpleSingleApplicationParserNew;
/**
*{
* "parsing_applications_version" : 1,
* "parsing_applications" : [
* {
* "parsing_app_name": "test",
* "parsing_app_version": 12345,
* "parsing_app_author": "dummy",
* "parsing_app_description": "Description of parser application",
* "parsing_app_settings": {
* "input_topics": [
* "secret"
* ],
* "error_topic": "error",
* "input_parallelism": 1,
* "parsing_parallelism": 2,
* "output_parallelism": 3,
* "parsing_app_type": "single_parser"
* },
* "parsing_settings": {
* "single_parser": {
* "parser_name": "single",
* "output_topic": "output"
* }
* }
* }
* ]
* }
*
**/
@Multiline
static String release;
private final String release = """
{
"parsing_applications_version" : 1,
"parsing_applications" : [
{
"parsing_app_name": "test",
"parsing_app_version": 12345,
"parsing_app_author": "dummy",
"parsing_app_description": "Description of parser application",
"parsing_app_settings": {
"input_topics": [
"secret"
],
"error_topic": "error",
"input_parallelism": 1,
"parsing_parallelism": 2,
"output_parallelism": 3,
"parsing_app_type": "single_parser"
},
"parsing_settings": {
"single_parser": {
"parser_name": "single",
"output_topic": "output"
}
}
}
]
}
""";
static String user = "unknown@secret.net"; static String user = "unknown@secret.net";
private final ConfigInfoProvider infoProvider = ParsingAppConfigInfoProvider.create(); private final ConfigInfoProvider infoProvider = ParsingAppConfigInfoProvider.create();
@@ -127,6 +118,7 @@ public class ParsingAppConfigInfoProviderTest {
Assert.assertEquals(1, info.getFilesContent().size()); Assert.assertEquals(1, info.getFilesContent().size());
Assert.assertTrue(info.getFilesContent().containsKey("test.json")); Assert.assertTrue(info.getFilesContent().containsKey("test.json"));
Assert.assertTrue(info.getFilesContent().get("test.json").isPresent());
Assert.assertTrue(info.getFilesContent() Assert.assertTrue(info.getFilesContent()
.get("test.json").get().indexOf("\"parsing_app_version\": 12346,") > 0); .get("test.json").get().indexOf("\"parsing_app_version\": 12346,") > 0);
Assert.assertTrue(info.getFilesContent() Assert.assertTrue(info.getFilesContent()
@@ -143,6 +135,7 @@ public class ParsingAppConfigInfoProviderTest {
Assert.assertEquals("dummy@secret.net", info.getCommitterEmail()); Assert.assertEquals("dummy@secret.net", info.getCommitterEmail());
Assert.assertEquals(1, info.getFilesContent().size()); Assert.assertEquals(1, info.getFilesContent().size());
Assert.assertTrue(info.getFilesContent().containsKey("test.json")); Assert.assertTrue(info.getFilesContent().containsKey("test.json"));
Assert.assertTrue(info.getFilesContent().get("test.json").isPresent());
Assert.assertTrue(info.getFilesContent() Assert.assertTrue(info.getFilesContent()
.get("test.json").get().indexOf("\"parsing_app_version\": 12346,") > 0); .get("test.json").get().indexOf("\"parsing_app_version\": 12346,") > 0);
Assert.assertTrue(info.getFilesContent() Assert.assertTrue(info.getFilesContent()
@@ -159,6 +152,7 @@ public class ParsingAppConfigInfoProviderTest {
Assert.assertEquals(user, info.getCommitterEmail()); Assert.assertEquals(user, info.getCommitterEmail());
Assert.assertEquals(1, info.getFilesContent().size()); Assert.assertEquals(1, info.getFilesContent().size());
Assert.assertTrue(info.getFilesContent().containsKey("test.json")); Assert.assertTrue(info.getFilesContent().containsKey("test.json"));
Assert.assertTrue(info.getFilesContent().get("test.json").isPresent());
Assert.assertTrue(info.getFilesContent() Assert.assertTrue(info.getFilesContent()
.get("test.json").get().indexOf("\"parsing_app_version\": 1,") > 0); .get("test.json").get().indexOf("\"parsing_app_version\": 1,") > 0);
Assert.assertTrue(info.getFilesContent() Assert.assertTrue(info.getFilesContent()
@@ -195,6 +189,7 @@ public class ParsingAppConfigInfoProviderTest {
Assert.assertEquals(1, info.getFilesContent().size()); Assert.assertEquals(1, info.getFilesContent().size());
Assert.assertTrue(info.getFilesContent().containsKey("parsing_applications.json")); Assert.assertTrue(info.getFilesContent().containsKey("parsing_applications.json"));
Assert.assertTrue(info.getFilesContent().get("parsing_applications.json").isPresent());
Assert.assertTrue(info.getFilesContent() Assert.assertTrue(info.getFilesContent()
.get("parsing_applications.json").get().indexOf("\"parsing_applications_version\": 2,") > 0); .get("parsing_applications.json").get().indexOf("\"parsing_applications_version\": 2,") > 0);
} }

2
config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/parsingapp/ParsingAppConfigSchemaServiceTest.java
View File

@@ -129,7 +129,7 @@ public class ParsingAppConfigSchemaServiceTest {
} }
@Test @Test
public void getImportersEmpty() throws Exception { public void getImportersEmpty() {
ConfigEditorResult ret = parsingAppConfigSchemaService.getImporters(); ConfigEditorResult ret = parsingAppConfigSchemaService.getImporters();
Assert.assertEquals(ConfigEditorResult.StatusCode.OK, ret.getStatusCode()); Assert.assertEquals(ConfigEditorResult.StatusCode.OK, ret.getStatusCode());
Assert.assertNotNull(ret.getAttributes().getConfigImporters()); Assert.assertNotNull(ret.getAttributes().getConfigImporters());

341
config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/response/ResponseSchemaServiceTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.configeditor.service.response; package uk.co.gresearch.siembol.configeditor.service.response;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -18,187 +17,179 @@ import static uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult.Stat
import static uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult.StatusCode.OK; import static uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult.StatusCode.OK;
public class ResponseSchemaServiceTest { public class ResponseSchemaServiceTest {
/** private final String rulesSchema = """
* { {
* "statusCode": "OK", "statusCode": "OK",
* "attributes": { "attributes": {
* "rules_schema": { "rules_schema": {
* "type": "object", "type": "object",
* "description": "Incident Response Rules", "description": "Incident Response Rules",
* "title": "rules", "title": "rules",
* "properties": { "properties": {
* "rules_version": { "rules_version": {
* "type": "integer", "type": "integer",
* "description": "Incident response rules version", "description": "Incident response rules version",
* "default": 0 "default": 0
* }, },
* "rules": { "rules": {
* "type": "array", "type": "array",
* "items": { "items": {
* "type": "object", "type": "object",
* "description": "Response rule that should handle response to a siembol alert", "description": "Response rule that should handle response to a siembol alert",
* "title": "rule", "title": "rule",
* "properties": { "properties": {
* "rule_name": { "rule_name": {
* "type": "string", "type": "string",
* "description": "ResponseRule name that uniquely identifies the rule" "description": "ResponseRule name that uniquely identifies the rule"
* }, },
* "rule_author": { "rule_author": {
* "type": "string", "type": "string",
* "description": "The owner of the rule" "description": "The owner of the rule"
* }, },
* "rule_version": { "rule_version": {
* "type": "integer", "type": "integer",
* "description": "The version of the rule", "description": "The version of the rule",
* "default": 0 "default": 0
* }, },
* "rule_description": { "rule_description": {
* "type": "string", "type": "string",
* "description": "The description of the rule" "description": "The description of the rule"
* }, },
* "evaluators": { "evaluators": {
* "type": "array", "type": "array",
* "items": { "items": {
* "type": "object", "type": "object",
* "description": "Response evaluator used in response rules", "description": "Response evaluator used in response rules",
* "title": "response evaluator", "title": "response evaluator",
* "oneOf": [ "oneOf": [
* { {
* "type": "object", "type": "object",
* "title": "matching_evaluator", "title": "matching_evaluator",
* "properties": { "properties": {
* "evaluator_type": { "evaluator_type": {
* "enum": [ "enum": [
* "matching_evaluator" "matching_evaluator"
* ], ],
* "default": "matching_evaluator" "default": "matching_evaluator"
* }, },
* "evaluator_attributes": { "evaluator_attributes": {
* "type": "object", "type": "object",
* "description": "Attributes for matching evaluator", "description": "Attributes for matching evaluator",
* "title": "matching evaluator attributes", "title": "matching evaluator attributes",
* "properties": { "properties": {
* "evaluation_result": { "evaluation_result": {
* "enum": [ "enum": [
* "match", "match",
* "filtered" "filtered"
* ], ],
* "type": "string", "type": "string",
* "description": "Evaluation result returned by the evaluator after matching", "description": "Evaluation result returned by the evaluator after matching",
* "default": "match" "default": "match"
* }, },
* "matchers": { "matchers": {
* "type": "array", "type": "array",
* "items": { "items": {
* "type": "object", "type": "object",
* "description": "Matcher for matching fields in response rules", "description": "Matcher for matching fields in response rules",
* "title": "matcher", "title": "matcher",
* "properties": { "properties": {
* "matcher_type": { "matcher_type": {
* "enum": [ "enum": [
* "REGEX_MATCH", "REGEX_MATCH",
* "IS_IN_SET" "IS_IN_SET"
* ], ],
* "type": "string", "type": "string",
* "description": "Type of matcher, either Regex match or list of strings (newline delimited)" "description": "Type of matcher, either Regex match or list of strings (newline delimited)"
* }, },
* "is_negated": { "is_negated": {
* "type": "boolean", "type": "boolean",
* "description": "The matcher is negated", "description": "The matcher is negated",
* "default": false "default": false
* }, },
* "field": { "field": {
* "type": "string", "type": "string",
* "description": "Field on which the matcher will be evaluated" "description": "Field on which the matcher will be evaluated"
* }, },
* "case_insensitive": { "case_insensitive": {
* "type": "boolean", "type": "boolean",
* "description": "Use case insensitive string compare", "description": "Use case insensitive string compare",
* "default": false "default": false
* }, },
* "data": { "data": {
* "type": "string", "type": "string",
* "description": "Matcher expression as defined by matcher type" "description": "Matcher expression as defined by matcher type"
* } }
* }, },
* "required": [ "required": [
* "data", "data",
* "field", "field",
* "matcher_type" "matcher_type"
* ] ]
* }, },
* "description": "Matchers of the evaluator", "description": "Matchers of the evaluator",
* "minItems": 1 "minItems": 1
* } }
* }, },
* "required": [ "required": [
* "evaluation_result", "evaluation_result",
* "matchers" "matchers"
* ] ]
* } }
* }, },
* "required": [ "required": [
* "evaluator_type", "evaluator_type",
* "evaluator_attributes" "evaluator_attributes"
* ] ]
* } }
* ] ]
* }, },
* "description": "Evaluators of the rule", "description": "Evaluators of the rule",
* "minItems": 1 "minItems": 1
* } }
* }, },
* "required": [ "required": [
* "evaluators", "evaluators",
* "rule_author", "rule_author",
* "rule_name", "rule_name",
* "rule_version" "rule_version"
* ] ]
* }, },
* "description": "Response rules", "description": "Response rules",
* "minItems": 1 "minItems": 1
* } }
* }, },
* "required": [ "required": [
* "rules", "rules",
* "rules_version" "rules_version"
* ] ]
* } }
* } }
* } }
*/ """;
@Multiline
public static String rulesSchema;
/** private final String testSchema = """
* {"statusCode":"OK","attributes":{"test_schema":{ "type" : "object", "description" : "Specification for testing responding rules", "title" : "response test specification", "properties" : { "event" : { "type" : "object", "description" : "Alert for response alerts evaluation", "title" : "json raw string" } }, "required" : [ "event" ]}}} {"statusCode":"OK","attributes":{"test_schema":{ "type" : "object", "description" : "Specification for testing responding rules", "title" : "response test specification", "properties" : { "event" : { "type" : "object", "description" : "Alert for response alerts evaluation", "title" : "json raw string" } }, "required" : [ "event" ]}}}
*/ """;
@Multiline
public static String testSchema;
/** private final String errorMessage = """
* { {
* "statusCode":"ERROR", "statusCode":"ERROR",
* "attributes":{ "message" : "dummy"}} "attributes":{ "message" : "dummy"}}
* } }
*/ """;
@Multiline
public static String errorMessage;
/** private final String okMessage = """
* { {
* "statusCode":"OK", "statusCode":"OK",
* "attributes":{ "message" : "dummy"}} "attributes":{ "message" : "dummy"}}
* } }
*/ """;
@Multiline
public static String okMessage;
private ResponseSchemaService.Builder builder; private ResponseSchemaService.Builder builder;
private ResponseSchemaService responseSchemaService; private ResponseSchemaService responseSchemaService;
private HttpProvider httpProvider; private HttpProvider httpProvider;
private String dummyJsonObject = "{ \"dummy\" : true }"; private final String dummyJsonObject = "{ \"dummy\" : true }";
private String dummyJsonObject2 = "{ \"dummy2\" : true }"; private final String dummyJsonObject2 = "{ \"dummy2\" : true }";
@Before @Before
public void setup() throws Exception { public void setup() throws Exception {

14
config-editor/config-editor-sync/pom.xml
View File

@@ -9,7 +9,7 @@
<parent> <parent>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId> <artifactId>config-editor</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</parent> </parent>
<dependencies> <dependencies>
<dependency> <dependency>
@@ -20,17 +20,17 @@
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId> <artifactId>siembol-common</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-core</artifactId> <artifactId>config-editor-core</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId> <artifactId>parsing-app</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
<dependency> <dependency>
@@ -126,12 +126,6 @@
<version>${junit_version}</version> <version>${junit_version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>org.mockito</groupId> <groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId> <artifactId>mockito-core</artifactId>

4
config-editor/config-editor-sync/src/test/java/uk/co/gresearch/siembol/configeditor/sync/actions/GetAdminConfigActionTest.java
View File

@@ -17,10 +17,10 @@ import static uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult.Stat
public class GetAdminConfigActionTest { public class GetAdminConfigActionTest {
private ConfigServiceHelper serviceHelper; private ConfigServiceHelper serviceHelper;
private String adminConfig = "ADMIN_CONFIG"; private final String adminConfig = "ADMIN_CONFIG";
private GetAdminConfigAction getAdminConfigAction; private GetAdminConfigAction getAdminConfigAction;
private ConfigEditorServiceContext context; private ConfigEditorServiceContext context;
private int version = 1; private final int version = 1;
@Before @Before
public void setUp() { public void setUp() {

292
config-editor/config-editor-sync/src/test/java/uk/co/gresearch/siembol/configeditor/sync/actions/GetParsingAppStormTopologyActionTest.java
View File

@@ -2,7 +2,7 @@ package uk.co.gresearch.siembol.configeditor.sync.actions;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectReader;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -25,156 +25,150 @@ public class GetParsingAppStormTopologyActionTest {
private static final ObjectReader ADMIN_CONFIG_READER = new ObjectMapper() private static final ObjectReader ADMIN_CONFIG_READER = new ObjectMapper()
.readerFor(StormParsingApplicationAttributesDto.class); .readerFor(StormParsingApplicationAttributesDto.class);
/** private final String adminConfig = """
*{ {
* "config_version": 1, "config_version": 1,
* "client.id.prefix": "siembol.writer", "client.id.prefix": "siembol.writer",
* "group.id.prefix": "siembol.reader", "group.id.prefix": "siembol.reader",
* "zookeeper.attributes": { "zookeeper.attributes": {
* "zk.url": "global_url", "zk.url": "global_url",
* "zk.path": "global_path", "zk.path": "global_path",
* "zk.base.sleep.ms": 1000, "zk.base.sleep.ms": 1000,
* "zk.max.retries": 3 "zk.max.retries": 3
* }, },
* "kafka.batch.writer.attributes": { "kafka.batch.writer.attributes": {
* "batch.size": 50, "batch.size": 50,
* "producer.properties": { "producer.properties": {
* "bootstrap.servers": "global_servers", "bootstrap.servers": "global_servers",
* "security.protocol": "SASL_PLAINTEXT" "security.protocol": "SASL_PLAINTEXT"
* } }
* }, },
* "storm.attributes": { "storm.attributes": {
* "bootstrap.servers": "dummy", "bootstrap.servers": "dummy",
* "first.pool.offset.strategy": "UNCOMMITTED_LATEST", "first.pool.offset.strategy": "UNCOMMITTED_LATEST",
* "kafka.spout.properties": { "kafka.spout.properties": {
* "session.timeout.ms": 300000, "session.timeout.ms": 300000,
* "security.protocol": "SASL_PLAINTEXT" "security.protocol": "SASL_PLAINTEXT"
* }, },
* "storm.config": { "storm.config": {
* "num.workers": 1 "num.workers": 1
* } }
* }, },
* "overridden.applications": [ "overridden.applications": [
* { {
* "application.name": "secret", "application.name": "secret",
* "kafka.batch.writer.attributes": { "kafka.batch.writer.attributes": {
* "batch.size": 1, "batch.size": 1,
* "producer.properties": { "producer.properties": {
* "bootstrap.servers": "dummy", "bootstrap.servers": "dummy",
* "security.protocol": "SASL_PLAINTEXT" "security.protocol": "SASL_PLAINTEXT"
* } }
* }, },
* "storm.attributes": { "storm.attributes": {
* "bootstrap.servers": "dummy", "bootstrap.servers": "dummy",
* "first.pool.offset.strategy": "UNCOMMITTED_LATEST", "first.pool.offset.strategy": "UNCOMMITTED_LATEST",
* "kafka.spout.properties": { "kafka.spout.properties": {
* "session.timeout.ms": 300000, "session.timeout.ms": 300000,
* "security.protocol": "SASL_PLAINTEXT" "security.protocol": "SASL_PLAINTEXT"
* }, },
* "storm.config": { "storm.config": {
* "num.workers": 2 "num.workers": 2
* } }
* } }
* } }
* ] ]
* } }
**/ """;
@Multiline
public static String adminConfig;
/** private final String adminConfigNoOverriddenApplications = """
*{ {
* "config_version": 1, "config_version": 1,
* "client.id.prefix": "siembol.writer", "client.id.prefix": "siembol.writer",
* "group.id.prefix": "siembol.reader", "group.id.prefix": "siembol.reader",
* "zookeeper.attributes": { "zookeeper.attributes": {
* "zk.url": "global_url", "zk.url": "global_url",
* "zk.path": "global_path", "zk.path": "global_path",
* "zk.base.sleep.ms": 1000, "zk.base.sleep.ms": 1000,
* "zk.max.retries": 3 "zk.max.retries": 3
* }, },
* "kafka.batch.writer.attributes": { "kafka.batch.writer.attributes": {
* "batch.size": 50, "batch.size": 50,
* "producer.properties": { "producer.properties": {
* "bootstrap.servers": "global_servers", "bootstrap.servers": "global_servers",
* "security.protocol": "SASL_PLAINTEXT" "security.protocol": "SASL_PLAINTEXT"
* } }
* }, },
* "storm.attributes": { "storm.attributes": {
* "bootstrap.servers": "dummy", "bootstrap.servers": "dummy",
* "first.pool.offset.strategy": "UNCOMMITTED_LATEST", "first.pool.offset.strategy": "UNCOMMITTED_LATEST",
* "kafka.spout.properties": { "kafka.spout.properties": {
* "session.timeout.ms": 300000, "session.timeout.ms": 300000,
* "security.protocol": "SASL_PLAINTEXT" "security.protocol": "SASL_PLAINTEXT"
* }, },
* "storm.config": { "storm.config": {
* "num.workers": 1 "num.workers": 1
* } }
* } }
* } }
**/ """;
@Multiline
public static String adminConfigNoOverriddenApplications;
/** private final String release = """
*{ {
* "parsing_applications_version": 0, "parsing_applications_version": 0,
* "parsing_applications": [ "parsing_applications": [
* { {
* "parsing_app_name": "secret", "parsing_app_name": "secret",
* "parsing_app_version": 1, "parsing_app_version": 1,
* "parsing_app_author": "siembol", "parsing_app_author": "siembol",
* "parsing_app_settings": { "parsing_app_settings": {
* "parsing_app_type": "single_parser", "parsing_app_type": "single_parser",
* "input_topics": [ "input_topics": [
* "test" "test"
* ], ],
* "error_topic": "test", "error_topic": "test",
* "input_parallelism": 4, "input_parallelism": 4,
* "parsing_parallelism": 4, "parsing_parallelism": 4,
* "output_parallelism": 4, "output_parallelism": 4,
* "parse_metadata": false "parse_metadata": false
* }, },
* "parsing_settings": { "parsing_settings": {
* "single_parser": { "single_parser": {
* "output_topic": "test", "output_topic": "test",
* "parser_name": "test" "parser_name": "test"
* } }
* } }
* }, },
* { {
* "parsing_app_name": "public", "parsing_app_name": "public",
* "parsing_app_version": 1, "parsing_app_version": 1,
* "parsing_app_author": "siembol", "parsing_app_author": "siembol",
* "parsing_app_settings": { "parsing_app_settings": {
* "parsing_app_type": "single_parser", "parsing_app_type": "single_parser",
* "input_topics": [ "input_topics": [
* "test" "test"
* ], ],
* "error_topic": "test", "error_topic": "test",
* "input_parallelism": 4, "input_parallelism": 4,
* "parsing_parallelism": 4, "parsing_parallelism": 4,
* "output_parallelism": 4, "output_parallelism": 4,
* "parse_metadata": false "parse_metadata": false
* }, },
* "parsing_settings": { "parsing_settings": {
* "single_parser": { "single_parser": {
* "output_topic": "test", "output_topic": "test",
* "parser_name": "test" "parser_name": "test"
* } }
* } }
* } }
* ] ]
* } }
**/ """;
@Multiline
public static String release;
private ConfigServiceHelper serviceHelper; private ConfigServiceHelper serviceHelper;
private GetParsingAppStormTopologyAction getStormTopologyAction; private GetParsingAppStormTopologyAction getStormTopologyAction;
private ConfigEditorServiceContext context; private ConfigEditorServiceContext context;
private String topologyImage = "dummyImage"; private final String topologyImage = "dummyImage";
private String serviceName = "dummyService"; private final String serviceName = "dummyService";
@Before @Before
public void setUp() { public void setUp() {
@@ -229,9 +223,9 @@ public class GetParsingAppStormTopologyActionTest {
Assert.assertEquals(1, adminConfigSecret.getKafkaBatchWriterAttributes().getBatchSize().intValue()); Assert.assertEquals(1, adminConfigSecret.getKafkaBatchWriterAttributes().getBatchSize().intValue());
Assert.assertEquals(50, adminConfigPublic.getKafkaBatchWriterAttributes().getBatchSize().intValue()); Assert.assertEquals(50, adminConfigPublic.getKafkaBatchWriterAttributes().getBatchSize().intValue());
Assert.assertEquals(Integer.valueOf(2), Assert.assertEquals(2,
adminConfigSecret.getStormAttributes().getStormConfig().getRawMap().get("num.workers")); adminConfigSecret.getStormAttributes().getStormConfig().getRawMap().get("num.workers"));
Assert.assertEquals(Integer.valueOf(1), Assert.assertEquals(1,
adminConfigPublic.getStormAttributes().getStormConfig().getRawMap().get("num.workers")); adminConfigPublic.getStormAttributes().getStormConfig().getRawMap().get("num.workers"));
} }
@@ -279,9 +273,9 @@ public class GetParsingAppStormTopologyActionTest {
Assert.assertEquals(50, adminConfigSecret.getKafkaBatchWriterAttributes().getBatchSize().intValue()); Assert.assertEquals(50, adminConfigSecret.getKafkaBatchWriterAttributes().getBatchSize().intValue());
Assert.assertEquals(50, adminConfigPublic.getKafkaBatchWriterAttributes().getBatchSize().intValue()); Assert.assertEquals(50, adminConfigPublic.getKafkaBatchWriterAttributes().getBatchSize().intValue());
Assert.assertEquals(Integer.valueOf(1), Assert.assertEquals(1,
adminConfigSecret.getStormAttributes().getStormConfig().getRawMap().get("num.workers")); adminConfigSecret.getStormAttributes().getStormConfig().getRawMap().get("num.workers"));
Assert.assertEquals(Integer.valueOf(1), Assert.assertEquals(1,
adminConfigPublic.getStormAttributes().getStormConfig().getRawMap().get("num.workers")); adminConfigPublic.getStormAttributes().getStormConfig().getRawMap().get("num.workers"));
} }

6
config-editor/config-editor-sync/src/test/java/uk/co/gresearch/siembol/configeditor/sync/actions/GetReleaseActionTest.java
View File

@@ -18,16 +18,18 @@ import static uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult.Stat
public class GetReleaseActionTest { public class GetReleaseActionTest {
private ConfigServiceHelper serviceHelper; private ConfigServiceHelper serviceHelper;
private String release = "RELEASE"; private final String release = "RELEASE";
private GetReleaseAction getReleaseAction; private GetReleaseAction getReleaseAction;
private ConfigEditorServiceContext context; private ConfigEditorServiceContext context;
private int version = 1; private final int version = 1;
@Before @Before
public void setUp() { public void setUp() {
context = new ConfigEditorServiceContext(); context = new ConfigEditorServiceContext();
serviceHelper = Mockito.mock(ConfigServiceHelper.class); serviceHelper = Mockito.mock(ConfigServiceHelper.class);
when(serviceHelper.getConfigsRelease()).thenReturn(Optional.of(release)); when(serviceHelper.getConfigsRelease()).thenReturn(Optional.of(release));
when(serviceHelper.getReleaseVersion(eq(release))).thenReturn(version); when(serviceHelper.getReleaseVersion(eq(release))).thenReturn(version);
when(serviceHelper.validateConfigurations(eq(release))).thenReturn(true); when(serviceHelper.validateConfigurations(eq(release))).thenReturn(true);
getReleaseAction = new GetReleaseAction(serviceHelper); getReleaseAction = new GetReleaseAction(serviceHelper);

8
config-editor/config-editor-sync/src/test/java/uk/co/gresearch/siembol/configeditor/sync/actions/GetStormTopologyActionTest.java
View File

@@ -19,12 +19,12 @@ import static uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult.Stat
public class GetStormTopologyActionTest { public class GetStormTopologyActionTest {
private ConfigServiceHelper serviceHelper; private ConfigServiceHelper serviceHelper;
private String adminConfig = "ADMIN_CONFIG"; private final String adminConfig = "ADMIN_CONFIG";
private GetStormTopologyAction getStormTopologyAction; private GetStormTopologyAction getStormTopologyAction;
private ConfigEditorServiceContext context; private ConfigEditorServiceContext context;
private String topologyName = "dummyTopologyName"; private final String topologyName = "dummyTopologyName";
private String topologyImage = "dummyImage"; private final String topologyImage = "dummyImage";
private String serviceName = "dummyService"; private final String serviceName = "dummyService";
@Before @Before
public void setUp() { public void setUp() {

4
config-editor/config-editor-sync/src/test/java/uk/co/gresearch/siembol/configeditor/sync/actions/UpdateRulesInZookeeperActionTest.java
View File

@@ -19,11 +19,11 @@ import static uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult.Stat
public class UpdateRulesInZookeeperActionTest { public class UpdateRulesInZookeeperActionTest {
private ConfigServiceHelper serviceHelper; private ConfigServiceHelper serviceHelper;
private String release = "RELEASE"; private final String release = "RELEASE";
private UpdateReleaseInZookeeperAction updateReleaseInZookeeperAction; private UpdateReleaseInZookeeperAction updateReleaseInZookeeperAction;
private ZooKeeperConnector zooKeeperConnector; private ZooKeeperConnector zooKeeperConnector;
private ConfigEditorServiceContext context; private ConfigEditorServiceContext context;
private String currentRelease = "ZK_RELEASE"; private final String currentRelease = "ZK_RELEASE";
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {

75
config-editor/config-editor-sync/src/test/java/uk/co/gresearch/siembol/configeditor/sync/service/EnrichmentTablesProviderTest.java
View File

@@ -1,67 +1,57 @@
package uk.co.gresearch.siembol.configeditor.sync.service; package uk.co.gresearch.siembol.configeditor.sync.service;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.mockito.Mockito; import org.mockito.Mockito;
import uk.co.gresearch.siembol.common.model.EnrichmentTableDto; import uk.co.gresearch.siembol.common.model.EnrichmentTableDto;
import uk.co.gresearch.siembol.common.model.StormTopologyDto;
import uk.co.gresearch.siembol.common.zookeeper.ZooKeeperConnector; import uk.co.gresearch.siembol.common.zookeeper.ZooKeeperConnector;
import uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult; import uk.co.gresearch.siembol.configeditor.model.ConfigEditorResult;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
public class EnrichmentTablesProviderTest { public class EnrichmentTablesProviderTest {
/** private final String tablesUpdateServiceA = """
*{ {
* "enrichment_tables": [ "enrichment_tables": [
* { {
* "name": "test_table", "name": "test_table",
* "path": "/siembol/tables/enrichment/test.json" "path": "/siembol/tables/enrichment/test.json"
* }, },
* { {
* "name": "dns_table", "name": "dns_table",
* "path": "/siembol/tables/enrichment/dns.json" "path": "/siembol/tables/enrichment/dns.json"
* } }
* ] ]
* } }
**/ """;
@Multiline
public static String tablesUpdateServiceA;
/** private final String tablesUpdateServiceB = """
*{ {
* "enrichment_tables": [ "enrichment_tables": [
* { {
* "name": "users_table", "name": "users_table",
* "path": "/siembol/tables/enrichment/users.json" "path": "/siembol/tables/enrichment/users.json"
* }, },
* { {
* "name": "url_table", "name": "url_table",
* "path": "/siembol/tables/enrichment/url.json" "path": "/siembol/tables/enrichment/url.json"
* }, },
* { {
* "name": "ioc_table", "name": "ioc_table",
* "path": "/siembol/tables/enrichment/ioc.json" "path": "/siembol/tables/enrichment/ioc.json"
* } }
* ] ]
* } }
**/ """;
@Multiline
public static String tablesUpdateServiceB;
private Map<String, ZooKeeperConnector> zooKeeperConnectorMap; private Map<String, ZooKeeperConnector> zooKeeperConnectorMap;
private EnrichmentTablesProviderImpl enrichmentTablesProvider; private EnrichmentTablesProviderImpl enrichmentTablesProvider;
private Set<String> services;
private List<StormTopologyDto> topologiesToUpdate;
private final String serviceA = "a"; private final String serviceA = "a";
private final String serviceB = "b"; private final String serviceB = "b";
@@ -184,5 +174,4 @@ public class EnrichmentTablesProviderTest {
Assert.assertEquals(ConfigEditorResult.StatusCode.BAD_REQUEST, result.getStatusCode()); Assert.assertEquals(ConfigEditorResult.StatusCode.BAD_REQUEST, result.getStatusCode());
Assert.assertNotNull(result.getAttributes().getMessage()); Assert.assertNotNull(result.getAttributes().getMessage());
} }
} }

183
config-editor/config-editor-sync/src/test/java/uk/co/gresearch/siembol/configeditor/sync/service/StormApplicationProviderTest.java
View File

@@ -2,7 +2,7 @@ package uk.co.gresearch.siembol.configeditor.sync.service;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectReader;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -23,93 +23,88 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.*; import static org.mockito.Mockito.*;
public class StormApplicationProviderTest { public class StormApplicationProviderTest {
private static ObjectReader TOPOLOGIES_READER = new ObjectMapper() private static final ObjectReader TOPOLOGIES_READER = new ObjectMapper()
.readerFor(StormTopologiesDto.class); .readerFor(StormTopologiesDto.class);
/** private final String initTopologies = """
* { {
* "timestamp": 1, "timestamp": 1,
* "topologies": [ "topologies": [
* { {
* "topology_name": "a", "topology_name": "a",
* "topology_id": "1", "topology_id": "1",
* "image": "secret", "image": "secret",
* "service_name": "alert", "service_name": "alert",
* "attributes": [ "attributes": [
* "a1", "a1",
* "a2" "a2"
* ] ]
* }, },
* { {
* "topology_name": "b", "topology_name": "b",
* "topology_id": "2", "topology_id": "2",
* "image": "secret", "image": "secret",
* "service_name": "parsing", "service_name": "parsing",
* "attributes": [ "attributes": [
* "a1", "a1",
* "a2", "a2",
* "a3" "a3"
* ] ]
* }, },
* { {
* "topology_name": "c", "topology_name": "c",
* "topology_id": "3", "topology_id": "3",
* "image": "secret", "image": "secret",
* "service_name": "parsing", "service_name": "parsing",
* "attributes": [ "attributes": [
* "a1", "a1",
* "a2", "a2",
* "a3" "a3"
* ] ]
* } }
* ] ]
* } }
**/ """;
@Multiline
public static String initTopologies;
/**
* {
* "timestamp": 1,
* "topologies": [
* {
* "topology_name": "a",
* "topology_id": "1",
* "image": "secret",
* "service_name": "alert",
* "attributes": [
* "a1",
* "a3"
* ]
* },
* {
* "topology_name": "b",
* "topology_id": "2",
* "image": "secret",
* "service_name": "parsing",
* "attributes": [
* "a1",
* "a2",
* "a3"
* ]
* },
* {
* "topology_name": "c",
* "topology_id": "3",
* "image": "secret",
* "service_name": "parsing",
* "attributes": [
* "a1",
* "a2",
* "a4"
* ]
* }
* ]
* }
**/
@Multiline
public static String updatedTopologies;
private final String updatedTopologies = """
{
"timestamp": 1,
"topologies": [
{
"topology_name": "a",
"topology_id": "1",
"image": "secret",
"service_name": "alert",
"attributes": [
"a1",
"a3"
]
},
{
"topology_name": "b",
"topology_id": "2",
"image": "secret",
"service_name": "parsing",
"attributes": [
"a1",
"a2",
"a3"
]
},
{
"topology_name": "c",
"topology_id": "3",
"image": "secret",
"service_name": "parsing",
"attributes": [
"a1",
"a2",
"a4"
]
}
]
}
""";
private ZooKeeperConnector zooKeeperConnector; private ZooKeeperConnector zooKeeperConnector;
private StormApplicationProviderImpl stormApplicationProvider; private StormApplicationProviderImpl stormApplicationProvider;
@@ -164,13 +159,13 @@ public class StormApplicationProviderTest {
Assert.assertEquals(ConfigEditorResult.StatusCode.OK, result.getStatusCode()); Assert.assertEquals(ConfigEditorResult.StatusCode.OK, result.getStatusCode());
Assert.assertNotNull(result.getAttributes().getTopologies()); Assert.assertNotNull(result.getAttributes().getTopologies());
Assert.assertEquals(3, result.getAttributes().getTopologies().size()); Assert.assertEquals(3, result.getAttributes().getTopologies().size());
StormTopologyDto restarted = result.getAttributes().getTopologies().stream() Optional<StormTopologyDto> restarted = result.getAttributes().getTopologies().stream()
.filter(x -> x.getTopologyName().equals("a")) .filter(x -> x.getTopologyName().equals("a"))
.findFirst().orElseGet(null); .findFirst();
Assert.assertNotNull(restarted); Assert.assertTrue(restarted.isPresent());
Assert.assertNotNull(restarted.getTopologyId()); Assert.assertNotNull(restarted.get().getTopologyId());
Assert.assertNotEquals("1", restarted.getTopologyId()); Assert.assertNotEquals("1", restarted.get().getTopologyId());
} }
@Test @Test
@@ -203,7 +198,7 @@ public class StormApplicationProviderTest {
@Test @Test
public void removeTopologiesFromOneService() throws Exception { public void removeTopologiesFromOneService() throws Exception {
services.addAll(Arrays.asList("parsing")); services.addAll(List.of("parsing"));
ConfigEditorResult result = stormApplicationProvider.updateStormTopologies(new ArrayList<>(), services); ConfigEditorResult result = stormApplicationProvider.updateStormTopologies(new ArrayList<>(), services);
Assert.assertEquals(ConfigEditorResult.StatusCode.OK, result.getStatusCode()); Assert.assertEquals(ConfigEditorResult.StatusCode.OK, result.getStatusCode());
Assert.assertNotNull(result.getAttributes().getTopologies()); Assert.assertNotNull(result.getAttributes().getTopologies());
@@ -236,20 +231,20 @@ public class StormApplicationProviderTest {
} }
@Test(expected = IllegalStateException.class) @Test(expected = IllegalStateException.class)
public void wrongInitTopologies() throws Exception { public void wrongInitTopologies() {
when(zooKeeperConnector.getData()).thenReturn("INVALID"); when(zooKeeperConnector.getData()).thenReturn("INVALID");
doNothing().when(zooKeeperConnector).addCacheListener(any()); doNothing().when(zooKeeperConnector).addCacheListener(any());
stormApplicationProvider = new StormApplicationProviderImpl(zooKeeperConnector); stormApplicationProvider = new StormApplicationProviderImpl(zooKeeperConnector);
} }
@Test @Test
public void testHealth() throws Exception { public void testHealth() {
Health health = stormApplicationProvider.checkHealth(); Health health = stormApplicationProvider.checkHealth();
Assert.assertEquals(Status.UP, health.getStatus()); Assert.assertEquals(Status.UP, health.getStatus());
} }
@Test @Test
public void updateDuplicatesError() throws Exception { public void updateDuplicatesError() {
topologiesToUpdate.get(0).setTopologyName("b"); topologiesToUpdate.get(0).setTopologyName("b");
services.addAll(Arrays.asList("alert", "parsing")); services.addAll(Arrays.asList("alert", "parsing"));
ConfigEditorResult result = stormApplicationProvider.updateStormTopologies(topologiesToUpdate, services); ConfigEditorResult result = stormApplicationProvider.updateStormTopologies(topologiesToUpdate, services);

2
config-editor/pom.xml
View File

@@ -11,7 +11,7 @@
<parent> <parent>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId> <artifactId>siembol</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</parent> </parent>
<modules> <modules>
<module>config-editor-core</module> <module>config-editor-core</module>

27
config/config-editor-rest/application.properties
View File

@@ -30,6 +30,10 @@ config-editor.test-cases-ui-config-file-name=ui-config/test-cases-layout-config.
config-editor.synchronisation=ALL config-editor.synchronisation=ALL
config-editor.storm-topologies-zookeeper.zk-path=/siembol/synchronise config-editor.storm-topologies-zookeeper.zk-path=/siembol/synchronise
config-editor.storm-topologies-zookeeper.zk-url=siembol-zookeeper:2181 config-editor.storm-topologies-zookeeper.zk-url=siembol-zookeeper:2181
config-editor.storm-topologies-zookeeper.init-value-if-not-exists={}
config-editor.enrichment-tables-zookeeper.enrichment.zk-path=/siembol/enrichment_tables
config-editor.enrichment-tables-zookeeper.enrichment.zk-url=siembol-zookeeper:2181
config-editor.enrichment-tables-zookeeper.enrichment.init-value-if-not-exists={}
config-editor.services.alert.type=alert config-editor.services.alert.type=alert
config-editor.services.alert.config-store.git-user-name=${GITHUB_USER} config-editor.services.alert.config-store.git-user-name=${GITHUB_USER}
@@ -47,6 +51,7 @@ config-editor.services.alert.ui-config-file-name=ui-config/alert-layout-config.j
config-editor.services.alert.synchronisation=ALL config-editor.services.alert.synchronisation=ALL
config-editor.services.alert.release-zookeeper.zk-path=/siembol/alerts config-editor.services.alert.release-zookeeper.zk-path=/siembol/alerts
config-editor.services.alert.release-zookeeper.zk-url=siembol-zookeeper:2181 config-editor.services.alert.release-zookeeper.zk-url=siembol-zookeeper:2181
config-editor.services.alert.release-zookeeper.init-value-if-not-exists={}
config-editor.services.alert.topology-image=gresearchdev/siembol-alerting-storm:latest config-editor.services.alert.topology-image=gresearchdev/siembol-alerting-storm:latest
config-editor.services.correlation-alert.type=correlationalert config-editor.services.correlation-alert.type=correlationalert
@@ -65,6 +70,7 @@ config-editor.services.correlation-alert.ui-config-file-name=ui-config/correlati
config-editor.services.correlation-alert.synchronisation=ALL config-editor.services.correlation-alert.synchronisation=ALL
config-editor.services.correlation-alert.release-zookeeper.zk-path=/siembol/correlation_alerts config-editor.services.correlation-alert.release-zookeeper.zk-path=/siembol/correlation_alerts
config-editor.services.correlation-alert.release-zookeeper.zk-url=siembol-zookeeper:2181 config-editor.services.correlation-alert.release-zookeeper.zk-url=siembol-zookeeper:2181
config-editor.services.correlation-alert.release-zookeeper.init-value-if-not-exists={}
config-editor.services.correlation-alert.topology-image=gresearchdev/siembol-alerting-storm:latest config-editor.services.correlation-alert.topology-image=gresearchdev/siembol-alerting-storm:latest
config-editor.services.parser-config.type=parserconfig config-editor.services.parser-config.type=parserconfig
@@ -82,6 +88,7 @@ config-editor.services.parser-config.ui-config-file-name=ui-config/parser-config
config-editor.services.parser-config.synchronisation=RELEASE config-editor.services.parser-config.synchronisation=RELEASE
config-editor.services.parser-config.release-zookeeper.zk-path=/siembol/parser_configs config-editor.services.parser-config.release-zookeeper.zk-path=/siembol/parser_configs
config-editor.services.parser-config.release-zookeeper.zk-url=siembol-zookeeper:2181 config-editor.services.parser-config.release-zookeeper.zk-url=siembol-zookeeper:2181
config-editor.services.parser-config.release-zookeeper.init-value-if-not-exists={}
config-editor.services.parsing-application.type=parsingapp config-editor.services.parsing-application.type=parsingapp
config-editor.services.parsing-application.config-store.git-user-name=${GITHUB_USER} config-editor.services.parsing-application.config-store.git-user-name=${GITHUB_USER}
@@ -100,3 +107,23 @@ config-editor.services.parsing-application.ui-config-file-name=ui-config/parsing
config-editor.services.parsing-application.synchronisation=ALL config-editor.services.parsing-application.synchronisation=ALL
config-editor.services.parsing-application.topology-image=gresearchdev/siembol-parsing-storm:latest config-editor.services.parsing-application.topology-image=gresearchdev/siembol-parsing-storm:latest
config-editor.services.enrichment.type=enrichment
config-editor.services.enrichment.config-store.git-user-name=${GITHUB_USER}
config-editor.services.enrichment.config-store.git-password=${GITHUB_TOKEN}
config-editor.services.enrichment.config-store.github-url=${GITHUB_URL}
config-editor.services.enrichment.config-store.store-repository-name=${GITHUB_REPO_NAME}
config-editor.services.enrichment.config-store.release-repository-name=${GITHUB_REPO_NAME}
config-editor.services.enrichment.config-store.admin-config-repository-name=${GITHUB_REPO_NAME}
config-editor.services.enrichment.config-store.store-repository-path=/tmp/siembol-config
config-editor.services.enrichment.config-store.store-directory=enrichment/rules
config-editor.services.enrichment.config-store.release-directory=enrichment/release
config-editor.services.enrichment.config-store.test-case-directory=enrichment/testcases
config-editor.services.enrichment.config-store.admin-config-directory=enrichment/adminconfig
config-editor.services.enrichment.ui-config-file-name=ui-config/enrichment-layout-config.json
config-editor.services.enrichment.synchronisation=ALL
config-editor.services.enrichment.release-zookeeper.zk-path=/siembol/enrichment_rules
config-editor.services.enrichment.release-zookeeper.zk-url=siembol-zookeeper:2181
config-editor.services.enrichment.release-zookeeper.init-value-if-not-exists={}
config-editor.services.enrichment.topology-image=gresearchdev/siembol-enriching-storm:latest

1
config/storm-topology-manager/application.properties
View File

@@ -31,6 +31,7 @@ topology-manager.desired-state.zk-url=siembol-zookeeper:2181
topology-manager.saved-state.zk-path=/siembol/cache topology-manager.saved-state.zk-path=/siembol/cache
topology-manager.saved-state.zk-url=siembol-zookeeper:2181 topology-manager.saved-state.zk-url=siembol-zookeeper:2181
topology-manager.saved-state.init-value-if-not-exists={}
topology-manager.k8s.namespace=siembol topology-manager.k8s.namespace=siembol
topology-manager.k8s.storm-submit-job-template-file=storm-submit.yaml topology-manager.k8s.storm-submit-job-template-file=storm-submit.yaml

6
config/storm-topology-manager/storm-submit.yaml
View File

@@ -4,6 +4,7 @@ metadata:
name: --name-- name: --name--
namespace: --namespace-- namespace: --namespace--
spec: spec:
ttlSecondsAfterFinished: 5
template: template:
spec: spec:
restartPolicy: Never restartPolicy: Never
@@ -14,5 +15,10 @@ spec:
args: --args-- args: --args--
image: --image-- image: --image--
name: --name-- name: --name--
resources:
requests:
memory: "64Mi"
limits:
memory: "128Mi"
securityContext: securityContext:
runAsUser: 101 runAsUser: 101

2
deployment/docker/Dockerfile.java
View File

@@ -1,4 +1,4 @@
FROM openjdk:8-jre-slim FROM openjdk:11-jre-slim
ARG APP ARG APP
ENV APP=$APP ENV APP=$APP

2
deployment/docker/Dockerfile.storm
View File

@@ -1,4 +1,4 @@
FROM storm:1.2.3 FROM storm:2.2.0
ARG JAR ARG JAR
ARG CLASS ARG CLASS

2
deployment/docker/storm-entrypoint.sh
View File

@@ -2,7 +2,7 @@
# if command starts with something that is not executable, prepend our deploy command # if command starts with something that is not executable, prepend our deploy command
if ! which "${1}" >/dev/null; then if ! which "${1}" >/dev/null; then
set -- storm -c nimbus.seeds="${NIMBUS_SEEDS:-"[\"nimbus\"]"}" -c nimbus.thrift.port=${NIMBUS_PORT:-6627} jar $TOPOLOGY_JAR $TOPOLOGY_CLASS "$@" set -- storm jar $TOPOLOGY_JAR $TOPOLOGY_CLASS "$@" -c nimbus.seeds="${NIMBUS_SEEDS:-"[\"nimbus\"]"}" -c nimbus.thrift.port=${NIMBUS_PORT:-6627}
fi fi
exec "$@" exec "$@"

10
deployment/storm-topology-manager/pom.xml
View File

@@ -9,7 +9,7 @@
<parent> <parent>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId> <artifactId>siembol</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath> <relativePath>../../pom.xml</relativePath>
</parent> </parent>
<dependencyManagement> <dependencyManagement>
@@ -43,7 +43,7 @@
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId> <artifactId>siembol-common</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
<exclusions> <exclusions>
<exclusion> <exclusion>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
@@ -159,12 +159,6 @@
<version>${junit_version}</version> <version>${junit_version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>org.junit.platform</groupId> <groupId>org.junit.platform</groupId>
<artifactId>junit-platform-launcher</artifactId> <artifactId>junit-platform-launcher</artifactId>

69
deployment/storm-topology-manager/src/test/java/uk/co/gresearch/siembol/deployment/storm/providers/KubernetesProviderImplTest.java
View File

@@ -3,7 +3,6 @@ package uk.co.gresearch.siembol.deployment.storm.providers;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectReader;
import io.fabric8.kubernetes.client.server.mock.KubernetesServer; import io.fabric8.kubernetes.client.server.mock.KubernetesServer;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Before; import org.junit.Before;
import org.junit.Ignore; import org.junit.Ignore;
import org.junit.Rule; import org.junit.Rule;
@@ -18,43 +17,39 @@ public class KubernetesProviderImplTest {
private static final ObjectReader READER = new ObjectMapper() private static final ObjectReader READER = new ObjectMapper()
.readerFor(StormTopologyDto.class); .readerFor(StormTopologyDto.class);
/** private final String topologyConfig = """
* { {
* "topology_name": "parsing-heartbeat", "topology_name": "parsing-heartbeat",
* "topology_id": "id1234", "topology_id": "id1234",
* "image": "gr/siembol-parsing-storm:1.72-SNAPSHOT", "image": "gr/siembol-parsing-storm:1.72-SNAPSHOT",
* "service_name": "parsing", "service_name": "parsing",
* "attributes": [ "attributes": [
* "testattributes1", "testattributes1",
* "testattributes2" "testattributes2"
* ] ]
* } }
**/ """;
@Multiline
private static String topologyConfig;
/** private final String expectedYaml = """
*apiVersion: batch/v1 apiVersion: batch/v1
*kind: Job kind: Job
*metadata: metadata:
* name: parsing-heartbeat name: parsing-heartbeat
* namespace: siembol namespace: siembol
*spec: spec:
* template: template:
* spec: spec:
* restartPolicy: Never restartPolicy: Never
* containers: containers:
* - env: - env:
* - name: NIMBUS_SEEDS - name: NIMBUS_SEEDS
* value: '["nimbus"]' value: '["nimbus"]'
* args: ["testattributes1", "testattributes2"] args: ["testattributes1", "testattributes2"]
* image: gr/siembol-parsing-storm:1.72-SNAPSHOT image: gr/siembol-parsing-storm:1.72-SNAPSHOT
* name: parsing-heartbeat name: parsing-heartbeat
* securityContext: securityContext:
* runAsUser: 1000 runAsUser: 1000
*/ """;
@Multiline
private static String expectedYaml;
KubernetesProvider provider; KubernetesProvider provider;

9
deployment/storm-topology-manager/src/test/java/uk/co/gresearch/siembol/deployment/storm/providers/StormProviderImplTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.deployment.storm.providers; package uk.co.gresearch.siembol.deployment.storm.providers;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -17,11 +16,9 @@ import static org.mockito.Mockito.when;
public class StormProviderImplTest { public class StormProviderImplTest {
/** private final String runningTopologies = """
* {"topologies":[{"id":"id1","name":"topology1"}]} {"topologies":[{"id":"id1","name":"topology1"}]}
*/ """;
@Multiline
private static String runningTopologies;
HttpProvider httpProvider = mock(HttpProvider.class); HttpProvider httpProvider = mock(HttpProvider.class);
StormProvider stormProvider; StormProvider stormProvider;

140
deployment/storm-topology-manager/src/test/java/uk/co/gresearch/siembol/deployment/storm/service/TopologyManagerServiceImplTests.java
View File

@@ -5,7 +5,6 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectReader;
import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.ObjectWriter;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.EqualsBuilder;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
@@ -34,100 +33,73 @@ public class TopologyManagerServiceImplTests {
.setSerializationInclusion(JsonInclude.Include.NON_NULL) .setSerializationInclusion(JsonInclude.Include.NON_NULL)
.writerFor(StormTopologiesDto.class); .writerFor(StormTopologiesDto.class);
/** private final String runningTopologies = """
* {"topologies":[{"id":"id1","name":"topology1"}]} {"topologies":[{"id":"id1","name":"topology1"}]}
*/ """;
@Multiline
private static String runningTopologies;
/**
* {"topologies":[{"id":"id1","name":"topology1"},{"id":"id2","name":"topology2"},{"id":"id3","name":"topology3"},{"id":"id4","name":"topology4"}]}
*/
@Multiline
private static String runningTopologies2;
/** private final String runningTopologies2 = """
* {"topologies": []} {"topologies":[{"id":"id1","name":"topology1"},{"id":"id2","name":"topology2"},{"id":"id3","name":"topology3"},{"id":"id4","name":"topology4"}]}
* """;
**/
@Multiline
private static String empty;
/** private final String empty = """
* {"topologies": [ {"topologies": []}
* { "topology_id": "t1", "topology_name": "topology1", "image": "", "attributes": ["", ""] } """;
* ]}
*
**/
@Multiline
private static String topologies1;
/** private final String topologies1 = """
* {"topologies": [ {"topologies": [
* { "topology_id": "t2", "topology_name": "topology1", "image": "", "attributes": ["", ""] } { "topology_id": "t1", "topology_name": "topology1", "image": "", "attributes": ["", ""] }
* ]} ]}
* """;
**/
@Multiline
private static String topologies1changed;
/**
* {"topologies": [
* { "topology_id": "t1", "topology_name": "topology1", "image": "", "attributes": ["", ""] },
* { "topology_id": "t2", "topology_name": "topology2", "image": "", "attributes": ["", ""] }
* ]}
*
**/
@Multiline
private static String topologies2;
/** private final String topologies1changed = """
* {"topologies": [ {"topologies": [
* { "topology_id": "t3", "topology_name": "topology1", "image": "", "attributes": ["", ""] }, { "topology_id": "t2", "topology_name": "topology1", "image": "", "attributes": ["", ""] }
* { "topology_id": "t4", "topology_name": "topology2", "image": "", "attributes": ["", ""] } ]}
* ]} """;
*
**/
@Multiline
private static String topologies2changed;
/** private final String topologies2 = """
* {"topologies": [ {"topologies": [
* { "topology_id": "t1", "topology_name": "topology1", "image": "", "attributes": ["", ""] }, { "topology_id": "t1", "topology_name": "topology1", "image": "", "attributes": ["", ""] },
* { "topology_id": "t2", "topology_name": "topology2", "image": "", "attributes": ["", ""] }, { "topology_id": "t2", "topology_name": "topology2", "image": "", "attributes": ["", ""] }
* { "topology_id": "t3", "topology_name": "topology3", "image": "", "attributes": ["", ""] } ]}
* ]} """;
*
**/
@Multiline
private static String topologies3;
/** private final String topologies2changed = """
* {"topologies": [ {"topologies": [
* { "topology_id": "t1", "topology_name": "topology1", "image": "", "attributes": ["", ""] }, { "topology_id": "t3", "topology_name": "topology1", "image": "", "attributes": ["", ""] },
* { "topology_id": "t2", "topology_name": "topology2", "image": "", "attributes": ["", ""] }, { "topology_id": "t4", "topology_name": "topology2", "image": "", "attributes": ["", ""] }
* { "topology_id": "t3", "topology_name": "topology3", "image": "", "attributes": ["", ""] }, ]}
* { "topology_id": "t4", "topology_name": "topology4", "image": "", "attributes": ["", ""] } """;
* ]}
*
**/
@Multiline
private static String topologies4;
/** private final String topologies3 = """
* {"topologies": [ {"topologies": [
* { "topology_id": "t8", "topology_name": "topology1", "image": "", "attributes": ["", ""] }, { "topology_id": "t1", "topology_name": "topology1", "image": "", "attributes": ["", ""] },
* { "topology_id": "t2", "topology_name": "topology2", "image": "", "attributes": ["", ""] }, { "topology_id": "t2", "topology_name": "topology2", "image": "", "attributes": ["", ""] },
* { "topology_id": "t3", "topology_name": "topology3", "image": "", "attributes": ["", ""] }, { "topology_id": "t3", "topology_name": "topology3", "image": "", "attributes": ["", ""] }
* { "topology_id": "t4", "topology_name": "topology4", "image": "", "attributes": ["", ""] } ]}
* ]} """;
*
**/ private final String topologies4 = """
@Multiline {"topologies": [
private static String topologies4changed; { "topology_id": "t1", "topology_name": "topology1", "image": "", "attributes": ["", ""] },
{ "topology_id": "t2", "topology_name": "topology2", "image": "", "attributes": ["", ""] },
{ "topology_id": "t3", "topology_name": "topology3", "image": "", "attributes": ["", ""] },
{ "topology_id": "t4", "topology_name": "topology4", "image": "", "attributes": ["", ""] }
]}
""";
private final String topologies4changed = """
{"topologies": [
{ "topology_id": "t8", "topology_name": "topology1", "image": "", "attributes": ["", ""] },
{ "topology_id": "t2", "topology_name": "topology2", "image": "", "attributes": ["", ""] },
{ "topology_id": "t3", "topology_name": "topology3", "image": "", "attributes": ["", ""] },
{ "topology_id": "t4", "topology_name": "topology4", "image": "", "attributes": ["", ""] }
]}
""";
StormResponseDto stormTopologies = READER_STORM.readValue(runningTopologies); StormResponseDto stormTopologies = READER_STORM.readValue(runningTopologies);
TopologyManagerService service; TopologyManagerService service;
KubernetesProvider kubernetesProvider = mock(KubernetesProvider.class); KubernetesProvider kubernetesProvider = mock(KubernetesProvider.class);
StormProvider stormProvider = mock(StormProvider.class); StormProvider stormProvider = mock(StormProvider.class);

2
docs/services/how-tos/how_to_set_up_enrichment_table.md
View File

@@ -11,7 +11,7 @@ An enrichment table is defined in a JSON file, it only supports mappings of `str
After updating an enrichment table it is necessary to inform the enrichment topology of the changes, this is done using Zookeeper. The Zookeeper node is configured in the admin config of enrichment ([see here](../siembol_enrichment_service.md)). This message is stored in a Zookeeper node in JSON format. In it the tables are in a list in the "hdfs_tables" key and each table has a name and the path to the latest table. Here is an example with two tables: After updating an enrichment table it is necessary to inform the enrichment topology of the changes, this is done using Zookeeper. The Zookeeper node is configured in the admin config of enrichment ([see here](../siembol_enrichment_service.md)). This message is stored in a Zookeeper node in JSON format. In it the tables are in a list in the "hdfs_tables" key and each table has a name and the path to the latest table. Here is an example with two tables:
{ {
"hdfs_tables":[ "enrichment_tables":[
{ {
"name":"employees", "name":"employees",
"path":"/siembol-enrichment/employees/1.json" "path":"/siembol-enrichment/employees/1.json"

12
enriching/enriching-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent> <parent>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching</artifactId> <artifactId>enriching</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</parent> </parent>
<dependencies> <dependencies>
<dependency> <dependency>
@@ -35,18 +35,12 @@
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId> <artifactId>siembol-common</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId> <artifactId>alerting-core</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>junit</groupId> <groupId>junit</groupId>

414
enriching/enriching-core/src/test/java/uk/co/gresearch/siembol/enrichments/compiler/EnrichmentCompilerTest.java
View File

@@ -1,10 +1,9 @@
package uk.co.gresearch.siembol.enrichments.compiler; package uk.co.gresearch.siembol.enrichments.compiler;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectReader;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -20,226 +19,209 @@ import static uk.co.gresearch.siembol.enrichments.compiler.EnrichmentCompilerImp
public class EnrichmentCompilerTest { public class EnrichmentCompilerTest {
private static final ObjectReader JSON_MAP_READER = new ObjectMapper() private static final ObjectReader JSON_MAP_READER = new ObjectMapper()
.readerFor(new TypeReference<Map<String, Object>>() { }); .readerFor(new TypeReference<Map<String, Object>>() {
});
/** private final String testRules = """
*{ {
* "rules_version": 1, "rules_version": 1,
* "rules": [ "rules": [
* { {
* "rule_name": "siembol_enrichments_test", "rule_name": "siembol_enrichments_test",
* "rule_version": 1, "rule_version": 1,
* "rule_author": "dummy", "rule_author": "dummy",
* "rule_description": "Test rule", "rule_description": "Test rule",
* "source_type": "*", "source_type": "*",
* "matchers": [ "matchers": [
* { {
* "matcher_type": "REGEX_MATCH", "matcher_type": "REGEX_MATCH",
* "is_negated": false, "is_negated": false,
* "field": "is_alert", "field": "is_alert",
* "data": "(?i)true" "data": "(?i)true"
* } }
* ], ],
* "table_mapping": { "table_mapping": {
* "table_name": "test_table", "table_name": "test_table",
* "joining_key": "${ip_src_addr}", "joining_key": "${ip_src_addr}",
* "tags": [ "tags": [
* { {
* "tag_name": "is_test_tag", "tag_name": "is_test_tag",
* "tag_value": "true" "tag_value": "true"
* } }
* ], ],
* "enriching_fields": [ "enriching_fields": [
* { {
* "table_field_name": "dns_name", "table_field_name": "dns_name",
* "event_field_name": "siembol:enrichments:dns" "event_field_name": "siembol:enrichments:dns"
* } }
* ] ]
* } }
* } }
* ] ]
* } }
* """;
**/
@Multiline
public static String testRules;
/** private final String testRulesTagsOnly = """
*{ {
* "rules_version": 1, "rules_version": 1,
* "rules": [ "rules": [
* { {
* "rule_name": "siembol_enrichments_test", "rule_name": "siembol_enrichments_test",
* "rule_version": 1, "rule_version": 1,
* "rule_author": "dummy", "rule_author": "dummy",
* "rule_description": "Test rule", "rule_description": "Test rule",
* "source_type": "*", "source_type": "*",
* "matchers": [ "matchers": [
* { {
* "matcher_type": "REGEX_MATCH", "matcher_type": "REGEX_MATCH",
* "is_negated": false, "is_negated": false,
* "field": "is_alert", "field": "is_alert",
* "data": "(?i)true" "data": "(?i)true"
* } }
* ], ],
* "table_mapping": { "table_mapping": {
* "table_name": "test_table", "table_name": "test_table",
* "joining_key": "${ip_src_addr}", "joining_key": "${ip_src_addr}",
* "tags": [ "tags": [
* { {
* "tag_name": "is_test_tag", "tag_name": "is_test_tag",
* "tag_value": "true" "tag_value": "true"
* } }
* ] ]
* } }
* } }
* ] ]
* } }
* """;
**/
@Multiline
public static String testRulesTagsOnly;
/**
*{
* "rules_version": 1,
* "rules": [
* {
* "rule_name": "siembol_enrichments_test",
* "rule_version": 1,
* "rule_author": "dummy",
* "rule_description": "Test rule",
* "source_type": "*",
* "matchers": [
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "is_alert",
* "data": "(?i)true"
* }
* ],
* "table_mapping": {
* "table_name": "test_table",
* "joining_key": "${ip_src_addr}",
* "enriching_fields": [
* {
* "table_field_name": "dns_name",
* "event_field_name": "siembol:enrichments:dns"
* }
* ]
* }
* }
* ]
* }
*
**/
@Multiline
public static String testRulesEnrichingFieldsOnly;
/** private final String testRulesEnrichingFieldsOnly = """
*{ {
* "rules_version": 1, "rules_version": 1,
* "rules": [ "rules": [
* { {
* "rule_name": "siembol_enrichments_test", "rule_name": "siembol_enrichments_test",
* "rule_version": 1, "rule_version": 1,
* "rule_author": "dummy", "rule_author": "dummy",
* "rule_description": "Test rule", "rule_description": "Test rule",
* "source_type": "*", "source_type": "*",
* "matchers": [ "matchers": [
* { {
* "matcher_type": "REGEX_MATCH", "matcher_type": "REGEX_MATCH",
* "is_negated": false, "is_negated": false,
* "field": "is_alert", "field": "is_alert",
* "data": "(?i)true" "data": "(?i)true"
* } }
* ], ],
* "table_mapping": { "table_mapping": {
* "table_name": "test_table", "table_name": "test_table",
* "joining_key": "${ip_src_addr}" "joining_key": "${ip_src_addr}",
* } "enriching_fields": [
* } {
* ] "table_field_name": "dns_name",
* } "event_field_name": "siembol:enrichments:dns"
* }
**/ ]
@Multiline }
public static String testRulesMissingTagsAndEnrichingFields; }
]
}
""";
/**
*{
* "rule_name": "siembol_enrichments_test",
* "rule_version": 1,
* "rule_author": "dummy",
* "rule_description": "Test rule",
* "source_type": "*",
* "matchers": [
* {
* "matcher_type": "IS_IN_SET",
* "is_negated": false,
* "field": "is_alert",
* "data": "true"
* }
* ],
* "table_mapping": {
* "table_name": "test_table",
* "joining_key": "${ip_src_addr}",
* "tags": [
* {
* "tag_name": "is_test_tag",
* "tag_value": "true"
* }
* ],
* "enriching_fields": [
* {
* "table_field_name": "dns_name",
* "event_field_name": "siembol:enrichments:dns"
* }
* ]
* }
* }
*
*
**/
@Multiline
public static String testRule;
/** private final String testRulesMissingTagsAndEnrichingFields = """
* { {
* "event": { "rules_version": 1,
* "source_type": "secret", "rules": [
* "is_alert": "true", {
* "ip_src_addr": "1.2.3.4" "rule_name": "siembol_enrichments_test",
* }, "rule_version": 1,
* "testing_table_name": "test_table", "rule_author": "dummy",
* "testing_table_mapping": { "rule_description": "Test rule",
* "1.2.3.4": { "source_type": "*",
* "dns_name": "secret.abc" "matchers": [
* } {
* } "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* } "field": "is_alert",
**/ "data": "(?i)true"
@Multiline }
public static String testSpecification; ],
"table_mapping": {
"table_name": "test_table",
"joining_key": "${ip_src_addr}"
}
}
]
}
/** """;
* {
* "event": { private final String testRule = """
* "source_type": "secret", {
* "is_alert": "false", "rule_name": "siembol_enrichments_test",
* "ip_src_addr": "1.2.3.4" "rule_version": 1,
* }, "rule_author": "dummy",
* "testing_table_name": "test_table", "rule_description": "Test rule",
* "testing_table_mapping": { "source_type": "*",
* "1.2.3.4": { "matchers": [
* "dns_name": "secret.abc" {
* } "matcher_type": "IS_IN_SET",
* } "is_negated": false,
* } "field": "is_alert",
**/ "data": "true"
@Multiline }
public static String testSpecificationNoMatch; ],
"table_mapping": {
"table_name": "test_table",
"joining_key": "${ip_src_addr}",
"tags": [
{
"tag_name": "is_test_tag",
"tag_value": "true"
}
],
"enriching_fields": [
{
"table_field_name": "dns_name",
"event_field_name": "siembol:enrichments:dns"
}
]
}
}
""";
private final String testSpecification = """
{
"event": {
"source_type": "secret",
"is_alert": "true",
"ip_src_addr": "1.2.3.4"
},
"testing_table_name": "test_table",
"testing_table_mapping": {
"1.2.3.4": {
"dns_name": "secret.abc"
}
}
}
""";
private final String testSpecificationNoMatch = """
{
"event": {
"source_type": "secret",
"is_alert": "false",
"ip_src_addr": "1.2.3.4"
},
"testing_table_name": "test_table",
"testing_table_mapping": {
"1.2.3.4": {
"dns_name": "secret.abc"
}
}
}
""";
private EnrichmentCompiler enrichmentCompiler; private EnrichmentCompiler enrichmentCompiler;
@@ -380,7 +362,7 @@ public class EnrichmentCompilerTest {
} }
@Test @Test
public void testRulesMissingTagsAndEnrichingFields() throws IOException { public void testRulesMissingTagsAndEnrichingFields() {
EnrichmentResult result = enrichmentCompiler.testConfigurations(testRulesMissingTagsAndEnrichingFields, EnrichmentResult result = enrichmentCompiler.testConfigurations(testRulesMissingTagsAndEnrichingFields,
testSpecification); testSpecification);
Assert.assertEquals(ERROR, result.getStatusCode()); Assert.assertEquals(ERROR, result.getStatusCode());

20
enriching/enriching-core/src/test/java/uk/co/gresearch/siembol/enrichments/evaluation/AlertingEnrichmentEvaluatorTest.java
View File

@@ -1,7 +1,5 @@
package uk.co.gresearch.siembol.enrichments.evaluation; package uk.co.gresearch.siembol.enrichments.evaluation;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Pair;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
@@ -23,16 +21,14 @@ import static uk.co.gresearch.siembol.enrichments.common.EnrichmentResult.Status
import static uk.co.gresearch.siembol.enrichments.common.EnrichmentResult.StatusCode.OK; import static uk.co.gresearch.siembol.enrichments.common.EnrichmentResult.StatusCode.OK;
public class AlertingEnrichmentEvaluatorTest { public class AlertingEnrichmentEvaluatorTest {
/** private final String simpleEvent = """
* { {
* "timestamp" : 1, "timestamp" : 1,
* "dummy_bool" : true, "dummy_bool" : true,
* "dummy_str" : "test", "dummy_str" : "test",
* "a" : "conflict" "a" : "conflict"
* } }
**/ """;
@Multiline
public static String simpleEvent;
private AlertingAttributes alertingAttributes; private AlertingAttributes alertingAttributes;
private AlertingResult alertingResult; private AlertingResult alertingResult;

8
enriching/enriching-core/src/test/java/uk/co/gresearch/siembol/enrichments/evaluation/EnrichingRuleTest.java
View File

@@ -21,9 +21,9 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
public class EnrichingRuleTest { public class EnrichingRuleTest {
private String name = "test_rule"; private final String name = "test_rule";
private Integer version = 1; private final Integer version = 1;
private String tableName = "test_table"; private final String tableName = "test_table";
private String key = "test_key"; private String key = "test_key";
private Map<String, Object> event; private Map<String, Object> event;
@@ -40,7 +40,7 @@ public class EnrichingRuleTest {
enrichmentTags.add(Pair.of("is_test", "true")); enrichmentTags.add(Pair.of("is_test", "true"));
matcher = Mockito.mock(BasicMatcher.class); matcher = Mockito.mock(BasicMatcher.class);
event = new HashMap<>(); event = new HashMap<>();
when(matcher.match(ArgumentMatchers.<Map<String, Object>>any())).thenReturn(EvaluationResult.MATCH); when(matcher.match(ArgumentMatchers.any())).thenReturn(EvaluationResult.MATCH);
} }
@Test @Test

20
enriching/enriching-core/src/test/java/uk/co/gresearch/siembol/enrichments/evaluation/EnrichmentEvaluatorLibraryTest.java
View File

@@ -3,7 +3,7 @@ package uk.co.gresearch.siembol.enrichments.evaluation;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectReader;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Pair;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
@@ -20,16 +20,14 @@ import static org.mockito.Mockito.when;
public class EnrichmentEvaluatorLibraryTest { public class EnrichmentEvaluatorLibraryTest {
private static final ObjectReader JSON_MAP_READER = private static final ObjectReader JSON_MAP_READER =
new ObjectMapper().readerFor(new TypeReference<Map<String, Object>>() {}); new ObjectMapper().readerFor(new TypeReference<Map<String, Object>>() {});
/** private final String simpleEvent = """
* { {
* "timestamp" : 1, "timestamp" : 1,
* "dummy_bool" : true, "dummy_bool" : true,
* "dummy_str" : "test", "dummy_str" : "test",
* "a" : "conflict" "a" : "conflict"
* } }
**/ """;
@Multiline
public static String simpleEvent;
private List<Pair<String, String>> enrichments; private List<Pair<String, String>> enrichments;
private List<EnrichmentCommand> commands; private List<EnrichmentCommand> commands;

79
enriching/enriching-core/src/test/java/uk/co/gresearch/siembol/enrichments/table/EnrichmentsMemoryTableTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.enrichments.table; package uk.co.gresearch.siembol.enrichments.table;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Pair;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
@@ -13,53 +12,42 @@ import java.util.*;
public class EnrichmentsMemoryTableTest { public class EnrichmentsMemoryTableTest {
/** private final String simpleEmptyFields = """
* {
* { "1.2.3.1" : {},
* "1.2.3.1" : {}, "1.2.3.2" : {},
* "1.2.3.2" : {}, "1.2.3.3" : {},
* "1.2.3.3" : {}, "1.2.3.4" : {},
* "1.2.3.4" : {}, "1.2.3.5" : {}
* "1.2.3.5" : {} }
* } """;
**/
@Multiline
public static String simpleEmptyFields;
/** private final String unsupportedFieldType = """
* {
* { "1.2.3.1" : {"is_ioc" : 1}
* "1.2.3.1" : {"is_ioc" : 1} }
* } """;
**/
@Multiline
public static String unsupportedFieldType;
/** private final String simpleOneField = """
* {
* { "1.2.3.1" : { "is_malicious" : "true" },
* "1.2.3.1" : { "is_malicious" : "true" }, "1.2.3.2" : { "is_malicious" : "true"},
* "1.2.3.2" : { "is_malicious" : "true"}, "1.2.3.3" : {"is_malicious" : "false"},
* "1.2.3.3" : {"is_malicious" : "false"}, "1.2.3.4" : {"is_malicious" : "true"},
* "1.2.3.4" : {"is_malicious" : "true"}, "1.2.3.5" : {"is_malicious" : "true"}
* "1.2.3.5" : {"is_malicious" : "true"} }
* } """;
**/
@Multiline private final String simpleMixedFields = """
public static String simpleOneField; {
"1.2.3.1" : { "is_malicious" : "true", "is_ioc" : "false" },
"1.2.3.2" : {},
"1.2.3.3" : {"is_malicious" : "true", "is_ioc" : "false", "is_alert" : "true"},
"1.2.3.4" : {},
"1.2.3.5" : {"is_malicious" : "true"}
}
""";
/**
*
* {
* "1.2.3.1" : { "is_malicious" : "true", "is_ioc" : "false" },
* "1.2.3.2" : {},
* "1.2.3.3" : {"is_malicious" : "true", "is_ioc" : "false", "is_alert" : "true"},
* "1.2.3.4" : {},
* "1.2.3.5" : {"is_malicious" : "true"}
* }
**/
@Multiline
public static String simpleMixedFields;
private EnrichmentMemoryTable table; private EnrichmentMemoryTable table;
@Test @Test
@@ -186,5 +174,4 @@ public class EnrichmentsMemoryTableTest {
EnrichmentMemoryTable.fromJsonStream(is); EnrichmentMemoryTable.fromJsonStream(is);
} }
} }
} }

15
enriching/enriching-storm/pom.xml
View File

@@ -9,9 +9,14 @@
<parent> <parent>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching</artifactId> <artifactId>enriching</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</parent> </parent>
<dependencies> <dependencies>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>23.0</version>
</dependency>
<dependency> <dependency>
<groupId>com.fasterxml.jackson.core</groupId> <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId> <artifactId>jackson-core</artifactId>
@@ -62,7 +67,7 @@
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-core</artifactId> <artifactId>enriching-core</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
<exclusions> <exclusions>
<exclusion> <exclusion>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
@@ -80,12 +85,6 @@
<version>${junit_version}</version> <version>${junit_version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>org.apache.zookeeper</groupId> <groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId> <artifactId>zookeeper</artifactId>

8
enriching/enriching-storm/src/main/java/uk/co/gresearch/siembol/enrichments/storm/EnrichmentEvaluatorBolt.java
View File

@@ -49,11 +49,11 @@ public class EnrichmentEvaluatorBolt extends BaseRichBolt {
private OutputCollector collector; private OutputCollector collector;
private ZooKeeperConnector zooKeeperConnector; private ZooKeeperConnector zooKeeperConnector;
private final ZooKeeperAttributesDto zookeperAttributes; private final ZooKeeperAttributesDto zooKeeperAttributes;
private final ZooKeeperConnectorFactory zooKeeperConnectorFactory; private final ZooKeeperConnectorFactory zooKeeperConnectorFactory;
EnrichmentEvaluatorBolt(StormEnrichmentAttributesDto attributes, ZooKeeperConnectorFactory zooKeeperConnectorFactory) { EnrichmentEvaluatorBolt(StormEnrichmentAttributesDto attributes, ZooKeeperConnectorFactory zooKeeperConnectorFactory) {
this.zookeperAttributes = attributes.getEnrichingRulesZookeperAttributes(); this.zooKeeperAttributes = attributes.getEnrichingRulesZookeperAttributes();
this.zooKeeperConnectorFactory = zooKeeperConnectorFactory; this.zooKeeperConnectorFactory = zooKeeperConnectorFactory;
} }
@@ -61,13 +61,12 @@ public class EnrichmentEvaluatorBolt extends BaseRichBolt {
this(attributes, new ZooKeeperConnectorFactoryImpl()); this(attributes, new ZooKeeperConnectorFactoryImpl());
} }
@SuppressWarnings("rawtypes")
@Override @Override
public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) { public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
this.collector = outputCollector; this.collector = outputCollector;
try { try {
LOG.info(ENGINE_INIT_START); LOG.info(ENGINE_INIT_START);
zooKeeperConnector = zooKeeperConnectorFactory.createZookeeperConnector(zookeperAttributes); zooKeeperConnector = zooKeeperConnectorFactory.createZookeeperConnector(zooKeeperAttributes);
updateRules(); updateRules();
if (enrichmentEvaluator.get() == null) { if (enrichmentEvaluator.get() == null) {
@@ -96,7 +95,6 @@ public class EnrichmentEvaluatorBolt extends BaseRichBolt {
LOG.info(ENGINE_UPDATE_COMPLETED); LOG.info(ENGINE_UPDATE_COMPLETED);
} catch (Exception e) { } catch (Exception e) {
LOG.error(UPDATE_EXCEPTION_LOG, ExceptionUtils.getStackTrace(e)); LOG.error(UPDATE_EXCEPTION_LOG, ExceptionUtils.getStackTrace(e));
return;
} }
} }

17
enriching/enriching-storm/src/main/java/uk/co/gresearch/siembol/enrichments/storm/MemoryTableEnrichmentBolt.java
View File

@@ -15,9 +15,9 @@ import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values; import org.apache.storm.tuple.Values;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import uk.co.gresearch.siembol.common.filesystem.HdfsFileSystemFactory;
import uk.co.gresearch.siembol.common.filesystem.SiembolFileSystem; import uk.co.gresearch.siembol.common.filesystem.SiembolFileSystem;
import uk.co.gresearch.siembol.common.filesystem.SiembolFileSystemFactory; import uk.co.gresearch.siembol.common.filesystem.SiembolFileSystemFactory;
import uk.co.gresearch.siembol.common.filesystem.SupportedFileSystem;
import uk.co.gresearch.siembol.common.model.EnrichmentTableDto; import uk.co.gresearch.siembol.common.model.EnrichmentTableDto;
import uk.co.gresearch.siembol.common.model.EnrichmentTablesUpdateDto; import uk.co.gresearch.siembol.common.model.EnrichmentTablesUpdateDto;
import uk.co.gresearch.siembol.common.model.StormEnrichmentAttributesDto; import uk.co.gresearch.siembol.common.model.StormEnrichmentAttributesDto;
@@ -58,7 +58,7 @@ public class MemoryTableEnrichmentBolt extends BaseRichBolt {
private static final String INVALID_TYPE_IN_TUPLES = "Invalid type in tuple provided"; private static final String INVALID_TYPE_IN_TUPLES = "Invalid type in tuple provided";
private final AtomicReference<Map<String, EnrichmentTable>> enrichmentTables = new AtomicReference<>(); private final AtomicReference<Map<String, EnrichmentTable>> enrichmentTables = new AtomicReference<>();
private final ZooKeeperAttributesDto zookeperAttributes; private final ZooKeeperAttributesDto zooKeeeperAttributes;
private final ZooKeeperConnectorFactory zooKeeperConnectorFactory; private final ZooKeeperConnectorFactory zooKeeperConnectorFactory;
private final SiembolFileSystemFactory fileSystemFactory; private final SiembolFileSystemFactory fileSystemFactory;
@@ -68,7 +68,7 @@ public class MemoryTableEnrichmentBolt extends BaseRichBolt {
MemoryTableEnrichmentBolt(StormEnrichmentAttributesDto attributes, MemoryTableEnrichmentBolt(StormEnrichmentAttributesDto attributes,
ZooKeeperConnectorFactory zooKeeperConnectorFactory, ZooKeeperConnectorFactory zooKeeperConnectorFactory,
SiembolFileSystemFactory fileSystemFactory) { SiembolFileSystemFactory fileSystemFactory) {
this.zookeperAttributes = attributes.getEnrichingTablesAttributes(); this.zooKeeeperAttributes = attributes.getEnrichingTablesAttributes();
this.zooKeeperConnectorFactory = zooKeeperConnectorFactory; this.zooKeeperConnectorFactory = zooKeeperConnectorFactory;
this.fileSystemFactory = fileSystemFactory; this.fileSystemFactory = fileSystemFactory;
} }
@@ -76,17 +76,16 @@ public class MemoryTableEnrichmentBolt extends BaseRichBolt {
public MemoryTableEnrichmentBolt(StormEnrichmentAttributesDto attributes) { public MemoryTableEnrichmentBolt(StormEnrichmentAttributesDto attributes) {
this(attributes, this(attributes,
new ZooKeeperConnectorFactoryImpl(), new ZooKeeperConnectorFactoryImpl(),
new HdfsFileSystemFactory(attributes.getEnrichingTablesHdfsUri())); SupportedFileSystem.fromUri(attributes.getEnrichingTablesUri()));
} }
@SuppressWarnings("rawtypes")
@Override @Override
public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) { public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
this.collector = outputCollector; this.collector = outputCollector;
try { try {
LOG.info(TABLES_INIT_START); LOG.info(TABLES_INIT_START);
zooKeeperConnector = zooKeeperConnectorFactory.createZookeeperConnector(zookeperAttributes); zooKeeperConnector = zooKeeperConnectorFactory.createZookeeperConnector(zooKeeeperAttributes);
updateTables(); updateTables();
if (enrichmentTables.get() == null) { if (enrichmentTables.get() == null) {
@@ -124,7 +123,6 @@ public class MemoryTableEnrichmentBolt extends BaseRichBolt {
LOG.info(TABLES_UPDATES_COMPLETED); LOG.info(TABLES_UPDATES_COMPLETED);
} catch (Exception e) { } catch (Exception e) {
LOG.error(TABLES_UPDATE_EXCEPTION_FORMAT, ExceptionUtils.getStackTrace(e)); LOG.error(TABLES_UPDATE_EXCEPTION_FORMAT, ExceptionUtils.getStackTrace(e));
return;
} }
} }
@@ -155,10 +153,7 @@ public class MemoryTableEnrichmentBolt extends BaseRichBolt {
} }
Optional<List<Pair<String, String>>> result = table.getValues(command); Optional<List<Pair<String, String>>> result = table.getValues(command);
if (result.isPresent()) { result.ifPresent(enrichments::addAll);
enrichments.addAll(result.get());
}
} }
collector.emit(tuple, new Values(event, enrichments, exceptions)); collector.emit(tuple, new Values(event, enrichments, exceptions));
collector.ack(tuple); collector.ack(tuple);

6
enriching/enriching-storm/src/main/java/uk/co/gresearch/siembol/enrichments/storm/StormEnrichingApplication.java
View File

@@ -12,8 +12,8 @@ import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values; import org.apache.storm.tuple.Values;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import uk.co.gresearch.siembol.common.filesystem.HdfsFileSystemFactory;
import uk.co.gresearch.siembol.common.filesystem.SiembolFileSystemFactory; import uk.co.gresearch.siembol.common.filesystem.SiembolFileSystemFactory;
import uk.co.gresearch.siembol.common.filesystem.SupportedFileSystem;
import uk.co.gresearch.siembol.common.storm.KafkaBatchWriterBolt; import uk.co.gresearch.siembol.common.storm.KafkaBatchWriterBolt;
import uk.co.gresearch.siembol.common.model.StormAttributesDto; import uk.co.gresearch.siembol.common.model.StormAttributesDto;
import uk.co.gresearch.siembol.common.storm.StormHelper; import uk.co.gresearch.siembol.common.storm.StormHelper;
@@ -101,9 +101,9 @@ public class StormEnrichingApplication {
config.putAll(attributes.getStormAttributes().getStormConfig().getRawMap()); config.putAll(attributes.getStormAttributes().getStormConfig().getRawMap());
StormTopology topology = createTopology(attributes, StormTopology topology = createTopology(attributes,
new ZooKeeperConnectorFactoryImpl(), new ZooKeeperConnectorFactoryImpl(),
new HdfsFileSystemFactory(attributes.getEnrichingTablesHdfsUri())); SupportedFileSystem.fromUri(attributes.getEnrichingTablesUri()));
LOG.info(SUBMIT_INFO_MSG, attributesStr); LOG.info(SUBMIT_INFO_MSG, attributes.getTopologyName(), attributesStr);
StormSubmitter.submitTopology(attributes.getTopologyName(), config, topology); StormSubmitter.submitTopology(attributes.getTopologyName(), config, topology);
} }
} }

96
enriching/enriching-storm/src/test/java/uk/co/gresearch/siembol/enrichments/storm/EnrichmentEvaluatorBoltTest.java
View File

@@ -1,6 +1,6 @@
package uk.co.gresearch.siembol.enrichments.storm; package uk.co.gresearch.siembol.enrichments.storm;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.storm.task.OutputCollector; import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values; import org.apache.storm.tuple.Values;
@@ -20,57 +20,53 @@ import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
public class EnrichmentEvaluatorBoltTest { public class EnrichmentEvaluatorBoltTest {
/** private final String event = """
* {"a" : "tmp_string", "b" : 1, "is_alert" : "true", "source_type" : "test"} {"a" : "tmp_string", "b" : 1, "is_alert" : "true", "source_type" : "test"}
**/ """;
@Multiline
public static String event;
/** private final String testRules = """
* { {
* "rules_version": 1, "rules_version": 1,
* "rules": [ "rules": [
* { {
* "rule_name": "test_rule", "rule_name": "test_rule",
* "rule_version": 1, "rule_version": 1,
* "rule_author": "john", "rule_author": "john",
* "rule_description": "Test rule", "rule_description": "Test rule",
* "source_type": "*", "source_type": "*",
* "matchers": [ "matchers": [
* { {
* "matcher_type": "REGEX_MATCH", "matcher_type": "REGEX_MATCH",
* "is_negated": false, "is_negated": false,
* "field": "is_alert", "field": "is_alert",
* "data": "(?i)true" "data": "(?i)true"
* } }
* ], ],
* "table_mapping": { "table_mapping": {
* "table_name": "test_table", "table_name": "test_table",
* "joining_key": "${a}", "joining_key": "${a}",
* "tags": [ "tags": [
* { {
* "tag_name": "is_test_tag", "tag_name": "is_test_tag",
* "tag_value": "true" "tag_value": "true"
* } }
* ], ],
* "enriching_fields": [ "enriching_fields": [
* { {
* "table_field_name": "dns_name", "table_field_name": "dns_name",
* "event_field_name": "siembol:enrichments:dns" "event_field_name": "siembol:enrichments:dns"
* } }
* ] ]
* } }
* } }
* ] ]
* } }
**/ """;
@Multiline
public static String testRules;
private Tuple tuple; private Tuple tuple;
private OutputCollector collector; private OutputCollector collector;
EnrichmentEvaluatorBolt enrichmentEvaluatorBolt; EnrichmentEvaluatorBolt enrichmentEvaluatorBolt;
ZooKeeperAttributesDto zookeperAttributes; ZooKeeperAttributesDto zooKeeperAttributes;
StormEnrichmentAttributesDto attributes; StormEnrichmentAttributesDto attributes;
ZooKeeperConnector zooKeeperConnector; ZooKeeperConnector zooKeeperConnector;
ZooKeeperConnectorFactory zooKeeperConnectorFactory; ZooKeeperConnectorFactory zooKeeperConnectorFactory;
@@ -78,9 +74,9 @@ public class EnrichmentEvaluatorBoltTest {
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
zookeperAttributes = new ZooKeeperAttributesDto(); zooKeeperAttributes = new ZooKeeperAttributesDto();
attributes = new StormEnrichmentAttributesDto(); attributes = new StormEnrichmentAttributesDto();
attributes.setEnrichingRulesZookeperAttributes(zookeperAttributes); attributes.setEnrichingRulesZookeperAttributes(zooKeeperAttributes);
tuple = Mockito.mock(Tuple.class); tuple = Mockito.mock(Tuple.class);
collector = Mockito.mock(OutputCollector.class); collector = Mockito.mock(OutputCollector.class);
@@ -89,7 +85,7 @@ public class EnrichmentEvaluatorBoltTest {
zooKeeperConnector = Mockito.mock(ZooKeeperConnector.class); zooKeeperConnector = Mockito.mock(ZooKeeperConnector.class);
when(zooKeeperConnectorFactory.createZookeeperConnector(zookeperAttributes)).thenReturn(zooKeeperConnector); when(zooKeeperConnectorFactory.createZookeeperConnector(zooKeeperAttributes)).thenReturn(zooKeeperConnector);
when(zooKeeperConnector.getData()).thenReturn(testRules); when(zooKeeperConnector.getData()).thenReturn(testRules);
when(tuple.getStringByField(eq(EnrichmentTuples.EVENT.toString()))).thenReturn(event); when(tuple.getStringByField(eq(EnrichmentTuples.EVENT.toString()))).thenReturn(event);

24
enriching/enriching-storm/src/test/java/uk/co/gresearch/siembol/enrichments/storm/EnrichmentMergerBoltTest.java
View File

@@ -1,6 +1,6 @@
package uk.co.gresearch.siembol.enrichments.storm; package uk.co.gresearch.siembol.enrichments.storm;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Pair;
import org.apache.storm.task.OutputCollector; import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Tuple;
@@ -24,20 +24,16 @@ import static org.mockito.Mockito.when;
public class EnrichmentMergerBoltTest { public class EnrichmentMergerBoltTest {
/** private final String event = """
* {"a": "string","b": 1,"c": true} {"a": "string","b": 1,"c": true}
**/ """;
@Multiline
public static String event;
/** private final String enrichedEventPrefix = """
* {"a":"string","b":1,"c":true,"siembol_enriching_ts": {"a":"string","b":1,"c":true,"siembol_enriching_ts":
**/ """;
@Multiline
public static String enrichedEventPrefix;
private String errorTopic = "error"; private final String errorTopic = "error";
private String outputTopic = "output"; private final String outputTopic = "output";
private Tuple tuple; private Tuple tuple;
private OutputCollector collector; private OutputCollector collector;
@@ -91,7 +87,7 @@ public class EnrichmentMergerBoltTest {
KafkaBatchWriterMessages messages = (KafkaBatchWriterMessages)values.get(0); KafkaBatchWriterMessages messages = (KafkaBatchWriterMessages)values.get(0);
Assert.assertEquals(outputTopic, messages.get(0).getTopic()); Assert.assertEquals(outputTopic, messages.get(0).getTopic());
Assert.assertTrue(messages.get(0).getMessage().contains("")); Assert.assertFalse(messages.get(0).getMessage().isEmpty());
Assert.assertEquals(errorTopic, messages.get(1).getTopic()); Assert.assertEquals(errorTopic, messages.get(1).getTopic());
Assert.assertEquals("dummy1", messages.get(1).getMessage()); Assert.assertEquals("dummy1", messages.get(1).getMessage());

51
enriching/enriching-storm/src/test/java/uk/co/gresearch/siembol/enrichments/storm/MemoryTableEnrichmentBoltTest.java
View File

@@ -1,6 +1,6 @@
package uk.co.gresearch.siembol.enrichments.storm; package uk.co.gresearch.siembol.enrichments.storm;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Pair;
import org.apache.storm.task.OutputCollector; import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Tuple;
@@ -28,36 +28,29 @@ import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
public class MemoryTableEnrichmentBoltTest { public class MemoryTableEnrichmentBoltTest {
/** private final String event = """
* {"a": "string", "b": 1, "c": true} {"a": "string", "b": 1, "c": true}
**/ """;
@Multiline
public static String event;
/** private final String tablesUpdate = """
* { {
* "enrichment_tables" : [ "enrichment_tables" : [
* { {
* "name" : "test_table", "name" : "test_table",
* "path": "/siembol/tables/enrichment/test.json" "path": "/siembol/tables/enrichment/test.json"
* }] }]
* } }
**/ """;
@Multiline
public static String tablesUpdate;
/** private final String simpleOneField = """
* {
* { "1.2.3.1" : { "is_malicious" : "true" },
* "1.2.3.1" : { "is_malicious" : "true" }, "1.2.3.2" : { "is_malicious" : "true"},
* "1.2.3.2" : { "is_malicious" : "true"}, "1.2.3.3" : { "is_malicious" : "false"},
* "1.2.3.3" : {"is_malicious" : "false"}, "1.2.3.4" : { "is_malicious" : "true"},
* "1.2.3.4" : {"is_malicious" : "true"}, "1.2.3.5" : { "is_malicious" : "true"}
* "1.2.3.5" : {"is_malicious" : "true"} }
* } """;
**/
@Multiline
public static String simpleOneField;
private Tuple tuple; private Tuple tuple;
private OutputCollector collector; private OutputCollector collector;

277
enriching/enriching-storm/src/test/java/uk/co/gresearch/siembol/enrichments/storm/StormEnrichingApplicationTest.java
View File

@@ -4,7 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectReader;
import com.github.charithe.kafka.EphemeralKafkaBroker; import com.github.charithe.kafka.EphemeralKafkaBroker;
import com.github.charithe.kafka.KafkaJunitRule; import com.github.charithe.kafka.KafkaJunitRule;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.storm.Config; import org.apache.storm.Config;
import org.apache.storm.LocalCluster; import org.apache.storm.LocalCluster;
@@ -29,156 +29,141 @@ public class StormEnrichingApplicationTest {
private static final ObjectReader JSON_PARSERS_CONFIG_READER = new ObjectMapper() private static final ObjectReader JSON_PARSERS_CONFIG_READER = new ObjectMapper()
.readerFor(StormEnrichmentAttributesDto.class); .readerFor(StormEnrichmentAttributesDto.class);
/** private final String testEnrichmentStormConfig = """
* { {
* "topology.name": "testing", "topology.name": "testing",
* "kafka.spout.num.executors": 1, "kafka.spout.num.executors": 1,
* "enriching.engine.bolt.num.executors": 1, "enriching.engine.bolt.num.executors": 1,
* "memory.enriching.bolt.num.executors": 1, "memory.enriching.bolt.num.executors": 1,
* "merging.bolt.num.executors": 1, "merging.bolt.num.executors": 1,
* "kafka.writer.bolt.num.executors": 1, "kafka.writer.bolt.num.executors": 1,
* "enriching.input.topics" : [ "enrichments" ], "enriching.input.topics" : [ "enrichments" ],
* "enriching.output.topic": "output", "enriching.output.topic": "output",
* "enriching.error.topic": "error", "enriching.error.topic": "error",
* "enriching.tables.hdfs.uri": "hdfs://secret", "enriching.tables.uri": "hdfs://secret",
* "enriching.rules.zookeeper.attributes": { "enriching.rules.zookeeper.attributes": {
* "zk.path": "/enrichment/rules", "zk.path": "/enrichment/rules",
* "zk.base.sleep.ms": 1000, "zk.base.sleep.ms": 1000,
* "zk.max.retries": 10 "zk.max.retries": 10
* }, },
* "enriching.tables.zookeeper.attributes": { "enriching.tables.zookeeper.attributes": {
* "zk.path": "/enrichment/tables", "zk.path": "/enrichment/tables",
* "zk.base.sleep.ms": 1000, "zk.base.sleep.ms": 1000,
* "zk.max.retries": 10 "zk.max.retries": 10
* }, },
* "kafka.batch.writer.attributes": { "kafka.batch.writer.attributes": {
* "batch.size": 1, "batch.size": 1,
* "producer.properties": { "producer.properties": {
* "client.id": "writer", "client.id": "writer",
* "compression.type": "snappy", "compression.type": "snappy",
* "security.protocol": "PLAINTEXT" "security.protocol": "PLAINTEXT"
* } }
* }, },
* "storm.attributes": { "storm.attributes": {
* "first.pool.offset.strategy": "EARLIEST", "first.pool.offset.strategy": "EARLIEST",
* "kafka.spout.properties": { "kafka.spout.properties": {
* "security.protocol": "PLAINTEXT" "security.protocol": "PLAINTEXT"
* }, },
* "storm.config": { "storm.config": {
* "session.timeout.ms": 100000 "session.timeout.ms": 100000
* } }
* } }
* } }
**/ """;
@Multiline
public static String testEnrichmentStormConfig;
/** private final String tablesUpdate = """
* { {
* "enrichment_tables" : [ "enrichment_tables" : [
* { {
* "name" : "test_table", "name" : "test_table",
* "path": "/siembol/tables/enrichment/test.json" "path": "/siembol/tables/enrichment/test.json"
* }] }]
* } }
**/ """;
@Multiline
public static String tablesUpdate;
/** private final String simpleOneField = """
* {
* { "1.2.3.4" : { "dns_name" : "secret.unknown" },
* "1.2.3.4" : { "dns_name" : "secret.unknown" }, "1.2.3.5" : { "dns_name" : "secret.known" }
* "1.2.3.5" : { "dns_name" : "secret.known" } }
* } """;
**/
@Multiline
public static String simpleOneField;
/** private final String expectedEvent = """
*{"ip_src_addr":"1.2.3.4","ip_dst_addr":"1.2.3.5","b":1,"is_alert":"true","source_type":"test","is_test_tag_first":"true","src_dns_name":"secret.unknown","is_test_tag_second":"true","dst_dns_name":"secret.known","siembol_enriching_ts": {"ip_src_addr":"1.2.3.4","ip_dst_addr":"1.2.3.5","b":1,"is_alert":"true","source_type":"test","is_test_tag_first":"true","src_dns_name":"secret.unknown","is_test_tag_second":"true","dst_dns_name":"secret.known","siembol_enriching_ts":""";
**/
@Multiline
public static String expectedEvent;
/** private final String event = """
* {"ip_src_addr" : "1.2.3.4", "ip_dst_addr" : "1.2.3.5", "b" : 1, "is_alert" : "true", "source_type" : "test"} {"ip_src_addr" : "1.2.3.4", "ip_dst_addr" : "1.2.3.5", "b" : 1, "is_alert" : "true", "source_type" : "test"}
**/ """;
@Multiline
public static String event;
/**
* {
* "rules_version": 1,
* "rules": [
* {
* "rule_name": "test_rule_first",
* "rule_version": 1,
* "rule_author": "john",
* "rule_description": "Test rule",
* "source_type": "*",
* "matchers": [
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "is_alert",
* "data": "(?i)true"
* }
* ],
* "table_mapping": {
* "table_name": "test_table",
* "joining_key": "${ip_src_addr}",
* "tags": [
* {
* "tag_name": "is_test_tag_first",
* "tag_value": "true"
* }
* ],
* "enriching_fields": [
* {
* "table_field_name": "dns_name",
* "event_field_name": "src_dns_name"
* }
* ]
* }
* },
* {
* "rule_name": "test_rule_second",
* "rule_version": 1,
* "rule_author": "john",
* "rule_description": "Test rule",
* "source_type": "*",
* "matchers": [
* {
* "matcher_type": "REGEX_MATCH",
* "is_negated": false,
* "field": "is_alert",
* "data": "(?i)true"
* }
* ],
* "table_mapping": {
* "table_name": "test_table",
* "joining_key": "${ip_dst_addr}",
* "tags": [
* {
* "tag_name": "is_test_tag_second",
* "tag_value": "true"
* }
* ],
* "enriching_fields": [
* {
* "table_field_name": "dns_name",
* "event_field_name": "dst_dns_name"
* }
* ]
* }
* }
* ]
* }
**/
@Multiline
public static String testRules;
private final String testRules = """
{
"rules_version": 1,
"rules": [
{
"rule_name": "test_rule_first",
"rule_version": 1,
"rule_author": "john",
"rule_description": "Test rule",
"source_type": "*",
"matchers": [
{
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_alert",
"data": "(?i)true"
}
],
"table_mapping": {
"table_name": "test_table",
"joining_key": "${ip_src_addr}",
"tags": [
{
"tag_name": "is_test_tag_first",
"tag_value": "true"
}
],
"enriching_fields": [
{
"table_field_name": "dns_name",
"event_field_name": "src_dns_name"
}
]
}
},
{
"rule_name": "test_rule_second",
"rule_version": 1,
"rule_author": "john",
"rule_description": "Test rule",
"source_type": "*",
"matchers": [
{
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_alert",
"data": "(?i)true"
}
],
"table_mapping": {
"table_name": "test_table",
"joining_key": "${ip_dst_addr}",
"tags": [
{
"tag_name": "is_test_tag_second",
"tag_value": "true"
}
],
"enriching_fields": [
{
"table_field_name": "dns_name",
"event_field_name": "dst_dns_name"
}
]
}
}
]
}
""";
@ClassRule @ClassRule
public static KafkaJunitRule kafkaRule = new KafkaJunitRule(EphemeralKafkaBroker.create()); public static KafkaJunitRule kafkaRule = new KafkaJunitRule(EphemeralKafkaBroker.create());

2
enriching/pom.xml
View File

@@ -11,7 +11,7 @@
<parent> <parent>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId> <artifactId>siembol</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</parent> </parent>
<modules> <modules>
<module>enriching-core</module> <module>enriching-core</module>

12
parsing/parsing-app/pom.xml
View File

@@ -11,7 +11,7 @@
<parent> <parent>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing</artifactId> <artifactId>parsing</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</parent> </parent>
<dependencies> <dependencies>
<dependency> <dependency>
@@ -39,18 +39,12 @@
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId> <artifactId>siembol-common</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-core</artifactId> <artifactId>parsing-core</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>junit</groupId> <groupId>junit</groupId>

243
parsing/parsing-app/src/test/java/uk/co/gresearch/siembol/parsers/application/factory/ParsingApplicationFactoryImplTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.parsers.application.factory; package uk.co.gresearch.siembol.parsers.application.factory;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
@@ -10,136 +9,128 @@ public class ParsingApplicationFactoryImplTest {
factory = new ParsingApplicationFactoryImpl(); factory = new ParsingApplicationFactoryImpl();
} }
/** private final String simpleSingleApplicationParser = """
*{ {
* "parsing_app_name": "test", "parsing_app_name": "test",
* "parsing_app_version": 1, "parsing_app_version": 1,
* "parsing_app_author": "dummy", "parsing_app_author": "dummy",
* "parsing_app_description": "Description of parser application", "parsing_app_description": "Description of parser application",
* "parsing_app_settings": { "parsing_app_settings": {
* "input_topics": [ "input_topics": [
* "secret" "secret"
* ], ],
* "error_topic": "error", "error_topic": "error",
* "input_parallelism": 1, "input_parallelism": 1,
* "parsing_parallelism": 2, "parsing_parallelism": 2,
* "output_parallelism": 3, "output_parallelism": 3,
* "parsing_app_type": "single_parser" "parsing_app_type": "single_parser"
* }, },
* "parsing_settings": { "parsing_settings": {
* "single_parser": { "single_parser": {
* "parser_name": "single", "parser_name": "single",
* "output_topic": "output" "output_topic": "output"
* } }
* } }
* } }
**/ """;
@Multiline
public static String simpleSingleApplicationParser; private final String simpleRoutingApplicationParser = """
{
"parsing_app_name": "test",
"parsing_app_version": 1,
"parsing_app_author": "dummy",
"parsing_app_description": "Description of parser application",
"parsing_app_settings": {
"input_topics": [
"secret"
],
"error_topic": "error",
"input_parallelism": 1,
"parsing_parallelism": 2,
"output_parallelism": 3,
"parsing_app_type": "router_parsing"
},
"parsing_settings": {
"routing_parser": {
"router_parser_name": "router",
"routing_field": "host",
"routing_message": "msg",
"merged_fields": [
"timestamp",
"syslog_host"
],
"default_parser": {
"parser_name": "default",
"output_topic": "output_default"
},
"parsers": [
{
"routing_field_pattern": "secret",
"parser_properties": {
"parser_name": "single",
"output_topic": "out_secret"
}
}
]
}
}
}
""";
/** private final String testParsersConfigs = """
* { {
* "parsing_app_name": "test", "parsers_version": 1,
* "parsing_app_version": 1, "parsers_configurations": [
* "parsing_app_author": "dummy", {
* "parsing_app_description": "Description of parser application", "parser_description": "for testing single app parser",
* "parsing_app_settings": { "parser_version": 2,
* "input_topics": [ "parser_name": "single",
* "secret" "parser_author": "dummy",
* ], "parser_attributes": {
* "error_topic": "error", "parser_type": "generic"
* "input_parallelism": 1, }
* "parsing_parallelism": 2, },
* "output_parallelism": 3, {
* "parsing_app_type": "router_parsing" "parser_description": "for testing routing app parser",
* }, "parser_version": 2,
* "parsing_settings": { "parser_name": "router",
* "routing_parser": { "parser_author": "dummy",
* "router_parser_name": "router", "parser_attributes": {
* "routing_field": "host", "parser_type": "generic"
* "routing_message": "msg", }
* "merged_fields": [ },
* "timestamp", {
* "syslog_host" "parser_description": "for testing routing app parser",
* ], "parser_version": 2,
* "default_parser": { "parser_name": "default",
* "parser_name": "default", "parser_author": "dummy",
* "output_topic": "output_default" "parser_attributes": {
* }, "parser_type": "generic"
* "parsers": [ }
* { }
* "routing_field_pattern": "secret", ]
* "parser_properties": { }
* "parser_name": "single", """;
* "output_topic": "out_secret"
* }
* }
* ]
* }
* }
* }
**/
@Multiline
public static String simpleRoutingApplicationParser;
/**
* {
* "parsers_version": 1,
* "parsers_configurations": [
* {
* "parser_description": "for testing single app parser",
* "parser_version": 2,
* "parser_name": "single",
* "parser_author": "dummy",
* "parser_attributes": {
* "parser_type": "generic"
* }
* },
* {
* "parser_description": "for testing routing app paerser",
* "parser_version": 2,
* "parser_name": "router",
* "parser_author": "dummy",
* "parser_attributes": {
* "parser_type": "generic"
* }
* },
* {
* "parser_description": "for testing routing app parser",
* "parser_version": 2,
* "parser_name": "default",
* "parser_author": "dummy",
* "parser_attributes": {
* "parser_type": "generic"
* }
* }
* ]
* }
**/
@Multiline
public static String testParsersConfigs;
@Test @Test
public void testGetSchema() { public void testGetSchema() {
ParsingApplicationFactoryResult schemaResult = factory.getSchema(); ParsingApplicationFactoryResult schemaResult = factory.getSchema();
Assert.assertTrue(schemaResult.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.OK); Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.OK, schemaResult.getStatusCode());
Assert.assertFalse(schemaResult.getAttributes().getJsonSchema().isEmpty()); Assert.assertFalse(schemaResult.getAttributes().getJsonSchema().isEmpty());
} }
@Test @Test
public void testValidationSingleGood() { public void testValidationSingleGood() {
ParsingApplicationFactoryResult result = factory.validateConfiguration(simpleSingleApplicationParser); ParsingApplicationFactoryResult result = factory.validateConfiguration(simpleSingleApplicationParser);
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.OK); Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.OK, result.getStatusCode());
} }
@Test @Test
public void testValidationSingleFail() { public void testValidationSingleFail() {
ParsingApplicationFactoryResult result = factory.validateConfiguration(simpleSingleApplicationParser ParsingApplicationFactoryResult result = factory.validateConfiguration(simpleSingleApplicationParser
.replace("error_topic", "dummy")); .replace("error_topic", "dummy"));
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR); Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("error_topic")); Assert.assertTrue(result.getAttributes().getMessage().contains("error_topic"));
} }
@@ -147,7 +138,7 @@ public class ParsingApplicationFactoryImplTest {
public void testValidationSingleFail2() { public void testValidationSingleFail2() {
ParsingApplicationFactoryResult result = factory.validateConfiguration(simpleSingleApplicationParser ParsingApplicationFactoryResult result = factory.validateConfiguration(simpleSingleApplicationParser
.replace("\"parsing_parallelism\": 2,", "")); .replace("\"parsing_parallelism\": 2,", ""));
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR); Assert.assertSame( ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage() Assert.assertTrue(result.getAttributes().getMessage()
.contains("missing required properties ([\"parsing_parallelism\"])")); .contains("missing required properties ([\"parsing_parallelism\"])"));
} }
@@ -155,7 +146,7 @@ public class ParsingApplicationFactoryImplTest {
@Test @Test
public void testCreationSingleGood() { public void testCreationSingleGood() {
ParsingApplicationFactoryResult result = factory.create(simpleSingleApplicationParser, testParsersConfigs); ParsingApplicationFactoryResult result = factory.create(simpleSingleApplicationParser, testParsersConfigs);
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.OK); Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.OK, result.getStatusCode());
Assert.assertEquals("test", result.getAttributes().getName()); Assert.assertEquals("test", result.getAttributes().getName());
Assert.assertEquals(1, result.getAttributes().getInputParallelism().intValue()); Assert.assertEquals(1, result.getAttributes().getInputParallelism().intValue());
Assert.assertEquals(2, result.getAttributes().getParsingParallelism().intValue()); Assert.assertEquals(2, result.getAttributes().getParsingParallelism().intValue());
@@ -169,14 +160,14 @@ public class ParsingApplicationFactoryImplTest {
ParsingApplicationFactoryResult result = factory.create( ParsingApplicationFactoryResult result = factory.create(
simpleSingleApplicationParser.replace("error_topic", "dummy"), simpleSingleApplicationParser.replace("error_topic", "dummy"),
testParsersConfigs); testParsersConfigs);
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR); Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("error_topic")); Assert.assertTrue(result.getAttributes().getMessage().contains("error_topic"));
} }
@Test @Test
public void testCreationSingleWrongParserConfigs() { public void testCreationSingleWrongParserConfigs() {
ParsingApplicationFactoryResult result = factory.create(simpleSingleApplicationParser, "INVALID"); ParsingApplicationFactoryResult result = factory.create(simpleSingleApplicationParser, "INVALID");
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR); Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("INVALID")); Assert.assertTrue(result.getAttributes().getMessage().contains("INVALID"));
} }
@@ -184,28 +175,28 @@ public class ParsingApplicationFactoryImplTest {
public void testCreationSingleMissingParserConfigs() { public void testCreationSingleMissingParserConfigs() {
ParsingApplicationFactoryResult result = factory.create(simpleSingleApplicationParser, ParsingApplicationFactoryResult result = factory.create(simpleSingleApplicationParser,
testParsersConfigs.replace("single", "unwanted")); testParsersConfigs.replace("single", "unwanted"));
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR); Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("Missing parser: single")); Assert.assertTrue(result.getAttributes().getMessage().contains("Missing parser: single"));
} }
@Test @Test
public void testValidationRoutingGood() { public void testValidationRoutingGood() {
ParsingApplicationFactoryResult result = factory.validateConfiguration(simpleRoutingApplicationParser); ParsingApplicationFactoryResult result = factory.validateConfiguration(simpleRoutingApplicationParser);
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.OK); Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.OK, result.getStatusCode());
} }
@Test @Test
public void testValidationRoutingFail() { public void testValidationRoutingFail() {
ParsingApplicationFactoryResult result = factory.validateConfiguration(simpleRoutingApplicationParser ParsingApplicationFactoryResult result = factory.validateConfiguration(simpleRoutingApplicationParser
.replace("error_topic", "dummy")); .replace("error_topic", "dummy"));
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR); Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("error_topic")); Assert.assertTrue(result.getAttributes().getMessage().contains("error_topic"));
} }
@Test @Test
public void testCreationRoutingGood() { public void testCreationRoutingGood() {
ParsingApplicationFactoryResult result = factory.create(simpleRoutingApplicationParser, testParsersConfigs); ParsingApplicationFactoryResult result = factory.create(simpleRoutingApplicationParser, testParsersConfigs);
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.OK); Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.OK, result.getStatusCode());
Assert.assertEquals("test", result.getAttributes().getName()); Assert.assertEquals("test", result.getAttributes().getName());
Assert.assertEquals(1, result.getAttributes().getInputParallelism().intValue()); Assert.assertEquals(1, result.getAttributes().getInputParallelism().intValue());
Assert.assertEquals(2, result.getAttributes().getParsingParallelism().intValue()); Assert.assertEquals(2, result.getAttributes().getParsingParallelism().intValue());
@@ -219,14 +210,14 @@ public class ParsingApplicationFactoryImplTest {
ParsingApplicationFactoryResult result = factory.create( ParsingApplicationFactoryResult result = factory.create(
simpleRoutingApplicationParser.replace("error_topic", "dummy"), simpleRoutingApplicationParser.replace("error_topic", "dummy"),
testParsersConfigs); testParsersConfigs);
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR); Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("error_topic")); Assert.assertTrue(result.getAttributes().getMessage().contains("error_topic"));
} }
@Test @Test
public void testCreationRoutingWrongParserConfigs() { public void testCreationRoutingWrongParserConfigs() {
ParsingApplicationFactoryResult result = factory.create(simpleRoutingApplicationParser, "INVALID"); ParsingApplicationFactoryResult result = factory.create(simpleRoutingApplicationParser, "INVALID");
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR); Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("INVALID")); Assert.assertTrue(result.getAttributes().getMessage().contains("INVALID"));
} }
@@ -234,7 +225,7 @@ public class ParsingApplicationFactoryImplTest {
public void testCreationRoutingMissingRouterParserConfigs() { public void testCreationRoutingMissingRouterParserConfigs() {
ParsingApplicationFactoryResult result = factory.create(simpleRoutingApplicationParser, ParsingApplicationFactoryResult result = factory.create(simpleRoutingApplicationParser,
testParsersConfigs.replace("router", "unwanted")); testParsersConfigs.replace("router", "unwanted"));
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR); Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("Missing parser: router")); Assert.assertTrue(result.getAttributes().getMessage().contains("Missing parser: router"));
} }
@@ -242,7 +233,7 @@ public class ParsingApplicationFactoryImplTest {
public void testCreationRoutingMissingDefaultParserConfigs() { public void testCreationRoutingMissingDefaultParserConfigs() {
ParsingApplicationFactoryResult result = factory.create(simpleRoutingApplicationParser, ParsingApplicationFactoryResult result = factory.create(simpleRoutingApplicationParser,
testParsersConfigs.replace("default", "unwanted")); testParsersConfigs.replace("default", "unwanted"));
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR); Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("Missing parser: default")); Assert.assertTrue(result.getAttributes().getMessage().contains("Missing parser: default"));
} }
@@ -250,7 +241,7 @@ public class ParsingApplicationFactoryImplTest {
public void testCreationRoutingMissingParserConfigs() { public void testCreationRoutingMissingParserConfigs() {
ParsingApplicationFactoryResult result = factory.create(simpleRoutingApplicationParser, ParsingApplicationFactoryResult result = factory.create(simpleRoutingApplicationParser,
testParsersConfigs.replace("single", "unwanted")); testParsersConfigs.replace("single", "unwanted"));
Assert.assertTrue(result.getStatusCode() == ParsingApplicationFactoryResult.StatusCode.ERROR); Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("Missing parser: single")); Assert.assertTrue(result.getAttributes().getMessage().contains("Missing parser: single"));
} }
} }

69
parsing/parsing-app/src/test/java/uk/co/gresearch/siembol/parsers/application/parsing/RoutingParsingApplicationParserTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.parsers.application.parsing; package uk.co.gresearch.siembol.parsers.application.parsing;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -17,38 +16,34 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
public class RoutingParsingApplicationParserTest { public class RoutingParsingApplicationParserTest {
/** private final String metadata = """
* { {
* "a": "string",
* "a": "string", "b": 1,
* "b": 1, "c": true
* "c": true }
* } """;
**/
@Multiline
public static String metadata;
private SerializableSiembolParser routerParser; private SerializableSiembolParser routerParser;
private SerializableSiembolParser defaultParser; private SerializableSiembolParser defaultParser;
private SerializableSiembolParser routedParser1; private SerializableSiembolParser routedParser1;
private SerializableSiembolParser routedParser2; private SerializableSiembolParser routedParser2;
private String routingConditionField = "test_field"; private final String routingConditionField = "test_field";
private String routingMessageField = "original_string"; private final String routingMessageField = "original_string";
private RoutingParsingApplicationParser appParser; private RoutingParsingApplicationParser appParser;
private Map<String, Object> message1; private Map<String, Object> message1;
private Map<String, Object> message2; private Map<String, Object> message2;
private List<Map<String, Object>> parsed; private List<Map<String, Object>> parsed;
private String errorTopic = "error"; private final String errorTopic = "error";
private String outputTopic = "output"; private final String outputTopic = "output";
private byte[] input = "test".getBytes(); private final byte[] input = "test".getBytes();
private ParserResult routerParserResult; private ParserResult routerParserResult;
private ParserResult routedParserResult1; private ParserResult routedParserResult1;
private ParserResult routedParserResult2; private ParserResult routedParserResult2;
TimeProvider timeProvider; TimeProvider timeProvider;
long currentTime = 1L; long currentTime = 1L;
@Before @Before
public void setUp() { public void setUp() {
timeProvider = Mockito.mock(TimeProvider.class); timeProvider = Mockito.mock(TimeProvider.class);
@@ -95,7 +90,7 @@ public class RoutingParsingApplicationParserTest {
} }
@Test(expected = IllegalArgumentException.class) @Test(expected = IllegalArgumentException.class)
public void testMissingARguments() { public void testMissingArguments() {
appParser = RoutingParsingApplicationParser.builder() appParser = RoutingParsingApplicationParser.builder()
.errorTopic(errorTopic) .errorTopic(errorTopic)
.build(); .build();
@@ -138,13 +133,13 @@ public class RoutingParsingApplicationParserTest {
Assert.assertEquals(outputTopic, result.get(0).getTopic()); Assert.assertEquals(outputTopic, result.get(0).getTopic());
Assert.assertEquals(1, result.size()); Assert.assertEquals(1, result.size());
Assert.assertEquals(1, result.get(0).getMessages().size()); Assert.assertEquals(1, result.get(0).getMessages().size());
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("output_field" + "\":\"routed")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("output_field" + "\":\"routed"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("original_string" + "\":\"test")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("original_string" + "\":\"test"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":3")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":3"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains( Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"default-parser\"")); SiembolMessageFields.SENSOR_TYPE + "\":\"default-parser\""));
} }
@Test @Test
@@ -194,14 +189,14 @@ public class RoutingParsingApplicationParserTest {
Assert.assertEquals(outputTopic, result.get(0).getTopic()); Assert.assertEquals(outputTopic, result.get(0).getTopic());
Assert.assertEquals(1, result.size()); Assert.assertEquals(1, result.size());
Assert.assertEquals(1, result.get(0).getMessages().size()); Assert.assertEquals(1, result.get(0).getMessages().size());
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("output_field" + "\":\"routed")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("output_field" + "\":\"routed"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("original_string" + "\":\"test")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("original_string" + "\":\"test"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":3")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":3"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("guid" + "\":")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("guid" + "\":"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains( Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"default-parser\"")); SiembolMessageFields.SENSOR_TYPE + "\":\"default-parser\""));
} }
@Test @Test
@@ -285,13 +280,13 @@ public class RoutingParsingApplicationParserTest {
Assert.assertEquals(outputTopic, result.get(0).getTopic()); Assert.assertEquals(outputTopic, result.get(0).getTopic());
Assert.assertEquals(1, result.size()); Assert.assertEquals(1, result.size());
Assert.assertEquals(1, result.get(0).getMessages().size()); Assert.assertEquals(1, result.get(0).getMessages().size());
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("output_field" + "\":\"routed")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("output_field" + "\":\"routed"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("original_string" + "\":\"test")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("original_string" + "\":\"test"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":3")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":3"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains( Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"default-parser\"")); SiembolMessageFields.SENSOR_TYPE + "\":\"default-parser\""));
} }
@Test @Test
@@ -320,10 +315,10 @@ public class RoutingParsingApplicationParserTest {
Assert.assertEquals(2, result.size()); Assert.assertEquals(2, result.size());
Assert.assertEquals("dummy1", result.get(0).getTopic()); Assert.assertEquals("dummy1", result.get(0).getTopic());
Assert.assertTrue(result.get(0).getMessages().get(0).contains( Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"routed-parser1\"")); SiembolMessageFields.SENSOR_TYPE + "\":\"routed-parser1\""));
Assert.assertEquals("dummy2", result.get(1).getTopic()); Assert.assertEquals("dummy2", result.get(1).getTopic());
Assert.assertTrue(result.get(1).getMessages().get(0).contains( Assert.assertTrue(result.get(1).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"routed-parser2\"")); SiembolMessageFields.SENSOR_TYPE + "\":\"routed-parser2\""));
} }
@Test @Test
@@ -335,7 +330,7 @@ public class RoutingParsingApplicationParserTest {
.routingMessageField(routingMessageField) .routingMessageField(routingMessageField)
.addParser("dummy1", routedParser1, "a") .addParser("dummy1", routedParser1, "a")
.addParser("dummy2", routedParser2, "b") .addParser("dummy2", routedParser2, "b")
.mergedFields(Arrays.asList("timestamp")) .mergedFields(List.of("timestamp"))
.name("test") .name("test")
.errorTopic(errorTopic) .errorTopic(errorTopic)
.timeProvider(timeProvider) .timeProvider(timeProvider)
@@ -356,9 +351,9 @@ public class RoutingParsingApplicationParserTest {
Assert.assertEquals("dummy2", result.get(1).getTopic()); Assert.assertEquals("dummy2", result.get(1).getTopic());
Assert.assertTrue(result.get(1).getMessages().get(0).contains("timestamp" + "\":2")); Assert.assertTrue(result.get(1).getMessages().get(0).contains("timestamp" + "\":2"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains( Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"routed-parser1\"")); SiembolMessageFields.SENSOR_TYPE + "\":\"routed-parser1\""));
Assert.assertTrue(result.get(1).getMessages().get(0).contains( Assert.assertTrue(result.get(1).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"routed-parser2\"")); SiembolMessageFields.SENSOR_TYPE + "\":\"routed-parser2\""));
} }
@Test @Test
@@ -370,7 +365,7 @@ public class RoutingParsingApplicationParserTest {
.routingMessageField(routingMessageField) .routingMessageField(routingMessageField)
.addParser("dummy1", routedParser1, "a") .addParser("dummy1", routedParser1, "a")
.addParser("dummy2", routedParser2, "b") .addParser("dummy2", routedParser2, "b")
.mergedFields(Arrays.asList("timestamp")) .mergedFields(List.of("timestamp"))
.name("test") .name("test")
.errorTopic(errorTopic) .errorTopic(errorTopic)
.timeProvider(timeProvider) .timeProvider(timeProvider)
@@ -392,9 +387,9 @@ public class RoutingParsingApplicationParserTest {
Assert.assertEquals("dummy2", result.get(1).getTopic()); Assert.assertEquals("dummy2", result.get(1).getTopic());
Assert.assertTrue(result.get(1).getMessages().get(0).contains("timestamp" + "\":2")); Assert.assertTrue(result.get(1).getMessages().get(0).contains("timestamp" + "\":2"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains( Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"routed-parser1\"")); SiembolMessageFields.SENSOR_TYPE + "\":\"routed-parser1\""));
Assert.assertTrue(result.get(1).getMessages().get(0).contains( Assert.assertTrue(result.get(1).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"routed-parser2\"")); SiembolMessageFields.SENSOR_TYPE + "\":\"routed-parser2\""));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("guid" + "\":")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("guid" + "\":"));
Assert.assertTrue(result.get(1).getMessages().get(0).contains("guid" + "\":")); Assert.assertTrue(result.get(1).getMessages().get(0).contains("guid" + "\":"));
} }
@@ -408,7 +403,7 @@ public class RoutingParsingApplicationParserTest {
.routingMessageField(routingMessageField) .routingMessageField(routingMessageField)
.addParser("dummy1", routedParser1, "a") .addParser("dummy1", routedParser1, "a")
.addParser("dummy2", routedParser2, "b") .addParser("dummy2", routedParser2, "b")
.mergedFields(Arrays.asList("timestamp")) .mergedFields(List.of("timestamp"))
.name("test") .name("test")
.errorTopic(errorTopic) .errorTopic(errorTopic)
.timeProvider(timeProvider) .timeProvider(timeProvider)
@@ -431,6 +426,6 @@ public class RoutingParsingApplicationParserTest {
Assert.assertEquals("dummy2", result.get(1).getTopic()); Assert.assertEquals("dummy2", result.get(1).getTopic());
Assert.assertTrue(result.get(1).getMessages().get(0).contains("timestamp" + "\":2")); Assert.assertTrue(result.get(1).getMessages().get(0).contains("timestamp" + "\":2"));
Assert.assertTrue(result.get(1).getMessages().get(0).contains( Assert.assertTrue(result.get(1).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"routed-parser2\"")); SiembolMessageFields.SENSOR_TYPE + "\":\"routed-parser2\""));
} }
} }

71
parsing/parsing-app/src/test/java/uk/co/gresearch/siembol/parsers/application/parsing/SingleApplicationParserTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.parsers.application.parsing; package uk.co.gresearch.siembol.parsers.application.parsing;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -17,26 +16,24 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
public class SingleApplicationParserTest { public class SingleApplicationParserTest {
/** private final String metadata = """
* { {
*
* "a": "string", "a": "string",
* "b": 1, "b": 1,
* "c": true "c": true
* } }
**/ """;
@Multiline
public static String metadata;
private SerializableSiembolParser siembolParser; private SerializableSiembolParser siembolParser;
private String sourceType = "test_type"; private final String sourceType = "test_type";
private SingleApplicationParser appParser; private SingleApplicationParser appParser;
private Map<String, Object> message1; private Map<String, Object> message1;
private Map<String, Object> message2; private Map<String, Object> message2;
private List<Map<String, Object>> parsed; private List<Map<String, Object>> parsed;
private String errorTopic = "error"; private final String errorTopic = "error";
private String outputTopic = "output"; private final String outputTopic = "output";
private byte[] input = "test".getBytes(); private final byte[] input = "test".getBytes();
private ParserResult parserResult; private ParserResult parserResult;
TimeProvider timeProvider; TimeProvider timeProvider;
long currentTime = 1L; long currentTime = 1L;
@@ -65,14 +62,14 @@ public class SingleApplicationParserTest {
} }
@Test(expected = IllegalArgumentException.class) @Test(expected = IllegalArgumentException.class)
public void testMissingARguments() { public void testMissingArguments() {
appParser = SingleApplicationParser.builder() appParser = SingleApplicationParser.builder()
.errorTopic(errorTopic) .errorTopic(errorTopic)
.build(); .build();
} }
@Test(expected = IllegalArgumentException.class) @Test(expected = IllegalArgumentException.class)
public void testMissingARguments2() throws Exception { public void testMissingArguments2() throws Exception {
appParser = SingleApplicationParser.builder() appParser = SingleApplicationParser.builder()
.parser(outputTopic, siembolParser) .parser(outputTopic, siembolParser)
.name("test") .name("test")
@@ -97,16 +94,16 @@ public class SingleApplicationParserTest {
Assert.assertEquals(1, result.size()); Assert.assertEquals(1, result.size());
Assert.assertEquals(2, result.get(0).getMessages().size()); Assert.assertEquals(2, result.get(0).getMessages().size());
Assert.assertEquals(outputTopic, result.get(0).getTopic()); Assert.assertEquals(outputTopic, result.get(0).getTopic());
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_field" + "\":\"a")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_field" + "\":\"a"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains( Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"test_type\"")); SiembolMessageFields.SENSOR_TYPE + "\":\"test_type\""));
Assert.assertTrue(result.get(0).getMessages().get(1).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(1).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_field" + "\":\"b")); Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_field" + "\":\"b"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("timestamp" + "\":2")); Assert.assertTrue(result.get(0).getMessages().get(1).contains("timestamp" + "\":2"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains( Assert.assertTrue(result.get(0).getMessages().get(1).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"test_type\"")); SiembolMessageFields.SENSOR_TYPE + "\":\"test_type\""));
} }
@Test @Test
@@ -126,12 +123,12 @@ public class SingleApplicationParserTest {
Assert.assertEquals(1, result.size()); Assert.assertEquals(1, result.size());
Assert.assertEquals(1, result.get(0).getMessages().size()); Assert.assertEquals(1, result.get(0).getMessages().size());
Assert.assertEquals(outputTopic, result.get(0).getTopic()); Assert.assertEquals(outputTopic, result.get(0).getTopic());
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_field" + "\":\"a")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_field" + "\":\"a"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":1"));
Assert.assertFalse(result.get(0).getMessages().get(0).contains("guid" + "\":")); Assert.assertFalse(result.get(0).getMessages().get(0).contains("guid" + "\":"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains( Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"test_type\"")); SiembolMessageFields.SENSOR_TYPE + "\":\"test_type\""));
} }
@Test @Test
@@ -152,12 +149,12 @@ public class SingleApplicationParserTest {
Assert.assertEquals(1, result.size()); Assert.assertEquals(1, result.size());
Assert.assertEquals(1, result.get(0).getMessages().size()); Assert.assertEquals(1, result.get(0).getMessages().size());
Assert.assertEquals(outputTopic, result.get(0).getTopic()); Assert.assertEquals(outputTopic, result.get(0).getTopic());
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_field" + "\":\"a")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_field" + "\":\"a"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("guid" + "\":")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("guid" + "\":"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains( Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"test_type\"")); SiembolMessageFields.SENSOR_TYPE + "\":\"test_type\""));
} }
@Test @Test
@@ -256,22 +253,22 @@ public class SingleApplicationParserTest {
Assert.assertEquals(1, result.size()); Assert.assertEquals(1, result.size());
Assert.assertEquals(2, result.get(0).getMessages().size()); Assert.assertEquals(2, result.get(0).getMessages().size());
Assert.assertEquals(outputTopic, result.get(0).getTopic()); Assert.assertEquals(outputTopic, result.get(0).getTopic());
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_field" + "\":\"a")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_field" + "\":\"a"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_metadata:a" + "\":\"string\"")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_metadata:a" + "\":\"string\""));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_metadata:b" + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_metadata:b" + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_metadata:c" + "\":true")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_metadata:c" + "\":true"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains( Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"test_type\"")); SiembolMessageFields.SENSOR_TYPE + "\":\"test_type\""));
Assert.assertTrue(result.get(0).getMessages().get(1).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(1).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_field" + "\":\"b")); Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_field" + "\":\"b"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("timestamp" + "\":2")); Assert.assertTrue(result.get(0).getMessages().get(1).contains("timestamp" + "\":2"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_metadata:a" + "\":\"string\"")); Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_metadata:a" + "\":\"string\""));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_metadata:b" + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_metadata:b" + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_metadata:c" + "\":true")); Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_metadata:c" + "\":true"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains( Assert.assertTrue(result.get(0).getMessages().get(1).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"test_type\"")); SiembolMessageFields.SENSOR_TYPE + "\":\"test_type\""));
} }
@Test @Test
@@ -293,7 +290,7 @@ public class SingleApplicationParserTest {
Assert.assertEquals(1, result.size()); Assert.assertEquals(1, result.size());
Assert.assertEquals(2, result.get(0).getMessages().size()); Assert.assertEquals(2, result.get(0).getMessages().size());
Assert.assertEquals(outputTopic, result.get(0).getTopic()); Assert.assertEquals(outputTopic, result.get(0).getTopic());
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_field" + "\":\"a")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_field" + "\":\"a"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_metadata:a" + "\":\"string\"")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_metadata:a" + "\":\"string\""));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_metadata:b" + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_metadata:b" + "\":1"));
@@ -301,15 +298,15 @@ public class SingleApplicationParserTest {
Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("guid" + "\":")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("guid" + "\":"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains( Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"test_type\"")); SiembolMessageFields.SENSOR_TYPE + "\":\"test_type\""));
Assert.assertTrue(result.get(0).getMessages().get(1).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(1).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_field" + "\":\"b")); Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_field" + "\":\"b"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("timestamp" + "\":2")); Assert.assertTrue(result.get(0).getMessages().get(1).contains("timestamp" + "\":2"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_metadata:a" + "\":\"string\"")); Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_metadata:a" + "\":\"string\""));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_metadata:b" + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_metadata:b" + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_metadata:c" + "\":true")); Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_metadata:c" + "\":true"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains( Assert.assertTrue(result.get(0).getMessages().get(1).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"test_type\"")); SiembolMessageFields.SENSOR_TYPE + "\":\"test_type\""));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("guid" + "\":")); Assert.assertTrue(result.get(0).getMessages().get(1).contains("guid" + "\":"));
} }
@@ -330,21 +327,21 @@ public class SingleApplicationParserTest {
Assert.assertEquals(1, result.size()); Assert.assertEquals(1, result.size());
Assert.assertEquals(2, result.get(0).getMessages().size()); Assert.assertEquals(2, result.get(0).getMessages().size());
Assert.assertEquals(outputTopic, result.get(0).getTopic()); Assert.assertEquals(outputTopic, result.get(0).getTopic());
Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(0).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_field" + "\":\"a")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("test_field" + "\":\"a"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("a" + "\":\"string\"")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("a" + "\":\"string\""));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("b" + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("b" + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("c" + "\":true")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("c" + "\":true"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(0).contains("timestamp" + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(0).contains( Assert.assertTrue(result.get(0).getMessages().get(0).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"test_type\"")); SiembolMessageFields.SENSOR_TYPE + "\":\"test_type\""));
Assert.assertTrue(result.get(0).getMessages().get(1).contains(SiembolMessageFields.PARSING_TIME.toString() + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(1).contains(SiembolMessageFields.PARSING_TIME + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_field" + "\":\"b")); Assert.assertTrue(result.get(0).getMessages().get(1).contains("test_field" + "\":\"b"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("timestamp" + "\":2")); Assert.assertTrue(result.get(0).getMessages().get(1).contains("timestamp" + "\":2"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("a" + "\":\"string\"")); Assert.assertTrue(result.get(0).getMessages().get(1).contains("a" + "\":\"string\""));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("b" + "\":1")); Assert.assertTrue(result.get(0).getMessages().get(1).contains("b" + "\":1"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains("c" + "\":true")); Assert.assertTrue(result.get(0).getMessages().get(1).contains("c" + "\":true"));
Assert.assertTrue(result.get(0).getMessages().get(1).contains( Assert.assertTrue(result.get(0).getMessages().get(1).contains(
SiembolMessageFields.SENSOR_TYPE.toString() + "\":\"test_type\"")); SiembolMessageFields.SENSOR_TYPE + "\":\"test_type\""));
} }
} }

10
parsing/parsing-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent> <parent>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing</artifactId> <artifactId>parsing</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</parent> </parent>
<dependencies> <dependencies>
<dependency> <dependency>
@@ -45,19 +45,13 @@
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId> <artifactId>siembol-common</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>joda-time</groupId> <groupId>joda-time</groupId>
<artifactId>joda-time</artifactId> <artifactId>joda-time</artifactId>
<version>2.10.13</version> <version>2.10.13</version>
</dependency> </dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>junit</groupId> <groupId>junit</groupId>
<artifactId>junit</artifactId> <artifactId>junit</artifactId>

71
parsing/parsing-core/src/test/java/uk/co/gresearch/siembol/parsers/common/SerializableSiembolParserTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.parsers.common; package uk.co.gresearch.siembol.parsers.common;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang3.SerializationUtils; import org.apache.commons.lang3.SerializationUtils;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
@@ -9,43 +8,39 @@ import java.util.List;
import java.util.Map; import java.util.Map;
public class SerializableSiembolParserTest { public class SerializableSiembolParserTest {
/** private final String simpleGenericParser = """
* { {
* "parser_name" : "test", "parser_name" : "test",
* "parser_version" : 1, "parser_version" : 1,
* "parser_author" : "dummy", "parser_author" : "dummy",
* "parser_attributes": { "parser_attributes": {
* "parser_type": "generic" "parser_type": "generic"
* }, },
* "parser_extractors": [ "parser_extractors": [
* { {
* "extractor_type": "json_extractor", "extractor_type": "json_extractor",
* "name": "test", "name": "test",
* "field": "original_string", "field": "original_string",
* "post_processing_functions": [ "post_processing_functions": [
* "format_timestamp" "format_timestamp"
* ], ],
* "attributes": { "attributes": {
* "should_overwrite_fields": true, "should_overwrite_fields": true,
* "should_remove_field": false, "should_remove_field": false,
* "time_formats": [ "time_formats": [
* { {
* "time_format": "yyyy-MM-dd HH:mm:ss.SSS 'Z'" "time_format": "yyyy-MM-dd HH:mm:ss.SSS 'Z'"
* } }
* ] ]
* } }
* } }
* ] ]
* } }
**/ """;
@Multiline
public static String simpleGenericParser;
/** private final String message = """
* {"timestamp":"2019-03-27 18:52:02.732 Z"} {"timestamp":"2019-03-27 18:52:02.732 Z"}
**/ """;
@Multiline
public static String message;
@Test @Test
public void serializableTest() throws Exception { public void serializableTest() throws Exception {
@@ -57,7 +52,7 @@ public class SerializableSiembolParserTest {
SerializableSiembolParser clone = SerializationUtils.clone(original); SerializableSiembolParser clone = SerializationUtils.clone(original);
List<Map<String, Object>> parsedClone = clone.parse(message.getBytes()); List<Map<String, Object>> parsedClone = clone.parse(message.getBytes());
Assert.assertTrue(parsedOriginal.equals(parsedClone)); Assert.assertEquals(parsedOriginal, parsedClone);
Assert.assertEquals(1553712722732L, parsedClone.get(0).get("timestamp")); Assert.assertEquals(1553712722732L, parsedClone.get(0).get("timestamp"));
} }

53
parsing/parsing-core/src/test/java/uk/co/gresearch/siembol/parsers/extractors/CSVExtractorTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.parsers.extractors; package uk.co.gresearch.siembol.parsers.extractors;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -8,46 +7,34 @@ import org.junit.Test;
import java.util.*; import java.util.*;
public class CSVExtractorTest { public class CSVExtractorTest {
private String name = "test_name"; private final String name = "test_name";
private String field = "test_field"; private final String field = "test_field";
private List<ColumnNames> columnNamesList; private List<ColumnNames> columnNamesList;
private EnumSet<ParserExtractor.ParserExtractorFlags> extractorFlags; private EnumSet<ParserExtractor.ParserExtractorFlags> extractorFlags;
/** private final String simpleNoQuotes = """
* a,bb,ccc,,ee a,bb,ccc,,ee
**/ """;
@Multiline
public static String simpleNoQuotes;
/** private final String simpleEmptyLastColumn = """
* a,bb,ccc,,ee, a,bb,ccc,,ee,
**/ """;
@Multiline
public static String simpleEmptyLastColumn;
/** private final String stringDelimiterEmptyLastColumn = """
* a||bb||ccc||||ee|| a||bb||ccc||||ee||
**/ """;
@Multiline
public static String stringDelimiterEmptyLastColumn;
/** private final String simpleChangedDelimiter = """
* a;bb;ccc;;ee a;bb;ccc;;ee
**/ """;
@Multiline
public static String simpleChangedDelimiter;
/** private final String simpleQuotes = """
* a,"b,,,b",cc""c,"","ee a,"b,,,b",cc""c,"","ee
**/ """;
@Multiline
public static String simpleQuotes;
/** private final String noQuotesStringDelimiter = """
* a||bb||ccc||||ee a||bb||ccc||||ee
**/ """;
@Multiline
public static String noQuotesStringDelimiter;
@Before @Before
public void setUp() { public void setUp() {

24
parsing/parsing-core/src/test/java/uk/co/gresearch/siembol/parsers/extractors/JsonExtractorTest.java
View File

@@ -1,30 +1,24 @@
package uk.co.gresearch.siembol.parsers.extractors; package uk.co.gresearch.siembol.parsers.extractors;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import java.util.EnumSet; import java.util.EnumSet;
import java.util.Map; import java.util.Map;
public class JsonExtractorTest { public class JsonExtractorTest {
private String name = "test_name"; private final String name = "test_name";
private String field = "test_field"; private final String field = "test_field";
private EnumSet<ParserExtractor.ParserExtractorFlags> extractorFlags = private final EnumSet<ParserExtractor.ParserExtractorFlags> extractorFlags =
EnumSet.of(ParserExtractor.ParserExtractorFlags.SHOULD_REMOVE_FIELD); EnumSet.of(ParserExtractor.ParserExtractorFlags.SHOULD_REMOVE_FIELD);
/** private final String simpleJson = """
* {"key1":"bbb", "key2":2, "key3": true, "key4": {"nested1": { "neste21" : 1, "nested22" : true, "nested23" : {}, "nested24": []} }} {"ignored": "hopefully"} {"key1":"bbb", "key2":2, "key3": true, "key4": {"nested1": { "neste21" : 1, "nested22" : true, "nested23" : {}, "nested24": []} }} {"ignored": "hopefully"}
**/ """;
@Multiline
public static String simpleJson;
private final String simpleArrayJson = """
/** {"key1":"bbb", "key2": {"nested1": [{"order" : 1}, {"order" : 2}]}} {"ignored": "hopefully"}
* {"key1":"bbb", "key2": {"nested1": [{"order" : 1}, {"order" : 2}]}} {"ignored": "hopefully"} """;
**/
@Multiline
public static String simpleArrayJson;
@Test @Test
public void testGoodNested() { public void testGoodNested() {

49
parsing/parsing-core/src/test/java/uk/co/gresearch/siembol/parsers/extractors/KeyValueExtractorTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.parsers.extractors; package uk.co.gresearch.siembol.parsers.extractors;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -10,33 +9,27 @@ import java.util.Map;
public class KeyValueExtractorTest { public class KeyValueExtractorTest {
private String name = "test_name"; private final String name = "test_name";
private String field = "test_field"; private final String field = "test_field";
private EnumSet<ParserExtractor.ParserExtractorFlags> extractorFlags; private EnumSet<ParserExtractor.ParserExtractorFlags> extractorFlags;
private EnumSet<KeyValueExtractor.KeyValueExtractorFlags> keyValueFlags; private EnumSet<KeyValueExtractor.KeyValueExtractorFlags> keyValueFlags;
/**
* Level=1 Category=UNKNOWN Type=abc
**/
@Multiline
public static String simpleNoQuotas;
/** private final String simpleNoQuotas = """
* Threat=Evil Level='A' Category="UN =KNOWN" Level=1 Category=UNKNOWN Type=abc
**/ """;
@Multiline
public static String simpleQuotes;
/** private final String simpleQuotes = """
* Threat|Evil,Level|'A',Category|"UN,|KNOWN" Threat=Evil Level='A' Category="UN =KNOWN"
**/ """;
@Multiline
public static String nonStandardDelimiters;
/** private final String nonStandardDelimiters = """
* Threat|Evil,Level|'\'A',Category|"UN,|KN\"OWN" Threat|Evil,Level|'A',Category|"UN,|KNOWN"
**/ """;
@Multiline
public static String nonStandartDelimitersEscaping;
private final String nonStandardDelimitersEscaping = """
Threat|Evil,Level|'\\'A',Category|"UN,|KN\\"OWN"
""";
@Before @Before
public void setUp() { public void setUp() {
@@ -120,7 +113,7 @@ public class KeyValueExtractorTest {
Assert.assertEquals("'A'", out.get("Level")); Assert.assertEquals("'A'", out.get("Level"));
} }
@Test @Test
public void testGoodNonStandartsDelimiter() { public void testGoodNonStandardsDelimiter() {
extractorFlags.add( extractorFlags.add(
ParserExtractor.ParserExtractorFlags.SHOULD_OVERWRITE_FIELDS); ParserExtractor.ParserExtractorFlags.SHOULD_OVERWRITE_FIELDS);
keyValueFlags.add(KeyValueExtractor.KeyValueExtractorFlags.QUOTA_VALUE_HANDLING); keyValueFlags.add(KeyValueExtractor.KeyValueExtractorFlags.QUOTA_VALUE_HANDLING);
@@ -147,7 +140,7 @@ public class KeyValueExtractorTest {
} }
@Test @Test
public void testGoodNonStandartsDelimiterEscaping() { public void testGoodNonStandardsDelimiterEscaping() {
extractorFlags.add( extractorFlags.add(
ParserExtractor.ParserExtractorFlags.SHOULD_OVERWRITE_FIELDS); ParserExtractor.ParserExtractorFlags.SHOULD_OVERWRITE_FIELDS);
keyValueFlags.add(KeyValueExtractor.KeyValueExtractorFlags.QUOTA_VALUE_HANDLING); keyValueFlags.add(KeyValueExtractor.KeyValueExtractorFlags.QUOTA_VALUE_HANDLING);
@@ -168,7 +161,7 @@ public class KeyValueExtractorTest {
Assert.assertFalse(extractor.shouldRemoveField()); Assert.assertFalse(extractor.shouldRemoveField());
Assert.assertTrue(extractor.shouldOverwiteFields()); Assert.assertTrue(extractor.shouldOverwiteFields());
Map<String, Object> out = extractor.extract(nonStandartDelimitersEscaping.trim()); Map<String, Object> out = extractor.extract(nonStandardDelimitersEscaping.trim());
Assert.assertEquals(3, out.size()); Assert.assertEquals(3, out.size());
Assert.assertEquals("Evil", out.get("Threat")); Assert.assertEquals("Evil", out.get("Threat"));
Assert.assertEquals("\"UN,|KN\\\"OWN\"", out.get("Category")); Assert.assertEquals("\"UN,|KN\\\"OWN\"", out.get("Category"));
@@ -176,7 +169,7 @@ public class KeyValueExtractorTest {
} }
@Test @Test
public void testGoodNonStandartsDelimiterEscapingNextKey() { public void testGoodNonStandardsDelimiterEscapingNextKey() {
extractorFlags.add( extractorFlags.add(
ParserExtractor.ParserExtractorFlags.SHOULD_OVERWRITE_FIELDS); ParserExtractor.ParserExtractorFlags.SHOULD_OVERWRITE_FIELDS);
keyValueFlags.add(KeyValueExtractor.KeyValueExtractorFlags.QUOTA_VALUE_HANDLING); keyValueFlags.add(KeyValueExtractor.KeyValueExtractorFlags.QUOTA_VALUE_HANDLING);
@@ -198,7 +191,7 @@ public class KeyValueExtractorTest {
Assert.assertFalse(extractor.shouldRemoveField()); Assert.assertFalse(extractor.shouldRemoveField());
Assert.assertTrue(extractor.shouldOverwiteFields()); Assert.assertTrue(extractor.shouldOverwiteFields());
Map<String, Object> out = extractor.extract(nonStandartDelimitersEscaping.trim()); Map<String, Object> out = extractor.extract(nonStandardDelimitersEscaping.trim());
Assert.assertEquals(3, out.size()); Assert.assertEquals(3, out.size());
Assert.assertEquals("Evil", out.get("Threat")); Assert.assertEquals("Evil", out.get("Threat"));
Assert.assertEquals("\"UN,|KN\\\"OWN\"", out.get("Category")); Assert.assertEquals("\"UN,|KN\\\"OWN\"", out.get("Category"));

47
parsing/parsing-core/src/test/java/uk/co/gresearch/siembol/parsers/extractors/PatternExtractorTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.parsers.extractors; package uk.co.gresearch.siembol.parsers.extractors;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -12,41 +11,25 @@ import java.util.Map;
public class PatternExtractorTest { public class PatternExtractorTest {
private String name = "test_name"; private final String name = "test_name";
private String field = "test_field"; private final String field = "test_field";
private List<String> stringPatterns; private List<String> stringPatterns;
/** private final String goodMid = """
* MID (?<my_mid>\d+) MID (?<my_mid>\\d+)""";
**/
@Multiline
public static String goodMid;
private final String goodVofDetail = """
Threat Level=(?<vof_threat_level>\\d) Category=(?<vof_threat_cat>\\S+) Type=(?<vof_threat_type>.*?)""";
/** private final String vofDetailInstance = """
* Threat Level=(?<vof_threat_level>\d) Category=(?<vof_threat_cat>\S+) Type=(?<vof_threat_type>.*?) Threat Level=1 Category=UNKNOWN Type=a
**/ bc""";
@Multiline
public static String goodVofDetail;
/** private final String vofDetailInstanceWrong1 = """
* Threat Level=1 Category=UNKNOWN Type=a Threat Level=A Category=UNKNOWN Type=abc""";
*bc
**/
@Multiline
public static String vofDetailInstance;
/** private final String vofDetailInstanceWrong2 = """
* Threat Level=A Category=UNKNOWN Type=abc Threat Level=1 Category= Type=abc""";
**/
@Multiline
public static String vofDetailInstanceWrong1;
/**
* Threat Level=1 Category= Type=abc
**/
@Multiline
public static String vofDetailInstanceWrong2;
@Before @Before
public void setUp() { public void setUp() {
@@ -55,7 +38,6 @@ public class PatternExtractorTest {
@Test @Test
public void testGoodMid() { public void testGoodMid() {
stringPatterns.add(goodMid.trim()); stringPatterns.add(goodMid.trim());
PatternExtractor extractor = PatternExtractor.builder() PatternExtractor extractor = PatternExtractor.builder()
.patterns(stringPatterns) .patterns(stringPatterns)
@@ -82,7 +64,6 @@ public class PatternExtractorTest {
@Test @Test
public void testGoodMid2() { public void testGoodMid2() {
stringPatterns.add(goodMid.trim()); stringPatterns.add(goodMid.trim());
stringPatterns.add(".*" + goodMid.trim()); stringPatterns.add(".*" + goodMid.trim());
@@ -110,7 +91,6 @@ public class PatternExtractorTest {
@Test @Test
public void testGoodVofDetail() { public void testGoodVofDetail() {
stringPatterns.add(goodVofDetail.trim()); stringPatterns.add(goodVofDetail.trim());
PatternExtractor extractor = PatternExtractor.builder() PatternExtractor extractor = PatternExtractor.builder()
.patterns(stringPatterns) .patterns(stringPatterns)
@@ -137,7 +117,6 @@ public class PatternExtractorTest {
@Test @Test
public void testGoodVofDetail2() { public void testGoodVofDetail2() {
stringPatterns.add(goodVofDetail.trim()); stringPatterns.add(goodVofDetail.trim());
stringPatterns.add(".*" + goodVofDetail.trim()); stringPatterns.add(".*" + goodVofDetail.trim());
PatternExtractor extractor = PatternExtractor.builder() PatternExtractor extractor = PatternExtractor.builder()

51
parsing/parsing-core/src/test/java/uk/co/gresearch/siembol/parsers/extractors/RegexSelectExtractorTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.parsers.extractors; package uk.co.gresearch.siembol.parsers.extractors;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Pair;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
@@ -9,45 +8,29 @@ import org.junit.Test;
import java.util.*; import java.util.*;
public class RegexSelectExtractorTest { public class RegexSelectExtractorTest {
private String name = "test_name"; private final String name = "test_name";
private String field = "test_field"; private final String field = "test_field";
private EnumSet<ParserExtractor.ParserExtractorFlags> extractorFlags; private EnumSet<ParserExtractor.ParserExtractorFlags> extractorFlags;
private List<Pair<String, String>> patterns = new ArrayList<>(); private final List<Pair<String, String>> patterns = new ArrayList<>();
/** private final String auditdMessage = """
* node=nqptick1 type=EOE msg=audit(1526397806.509:3436363485): node=nqptick1 type=EOE msg=audit(1526397806.509:3436363485):""";
**/ private final String auditdRegex = "^node=";
@Multiline
private String auditdMessage;
private String auditdRegex = "^node=";
/** private final String k8sMessage = """
* <13>Jan 14 13:26:58 prod-1.k8s.abc fluentd: stream:stdout docker:{"container_id"=>"88751a072197197da7fa50987c485c04fdd7325a98831a533291ac113b558278"} kubernetes:{"container_name"=>"dummy", "namespace_name"=>"dev", "pod_name"=>"dummy", "container_image"=>"unknown.net/service:cwh10r-gb4ys-km514euza-3azyc-niutqq", "container_image_id"=>"docker-pullable://docker.artifactory.net", "pod_id"=>"416af93b-15c3-11e9-add7-48df3701a2c4", "labels"=>{"app"=>"abc", "master"=>"false", "pod-template-hash"=>"1710645932"}, "host"=>"abc", "master_url"=>"https://1.2.3.4:443/api", "namespace_id"=>"3993c1be-b01f-11e8-bee0-30e1716064fc", "namespace_labels"=>{"istio-injection"=>"disabled", "opa-validating-webhook"=>"enabled", "spooning"=>"dev"}} <13>Jan 14 13:26:58 prod-1.k8s.abc fluentd: stream:stdout docker:{"container_id"=>"88751a072197197da7fa50987c485c04fdd7325a98831a533291ac113b558278"} kubernetes:{"container_name"=>"dummy", "namespace_name"=>"dev", "pod_name"=>"dummy", "container_image"=>"unknown.net/service:cwh10r-gb4ys-km514euza-3azyc-niutqq", "container_image_id"=>"docker-pullable://docker.artifactory.net", "pod_id"=>"416af93b-15c3-11e9-add7-48df3701a2c4", "labels"=>{"app"=>"abc", "master"=>"false", "pod-template-hash"=>"1710645932"}, "host"=>"abc", "master_url"=>"https://1.2.3.4:443/api", "namespace_id"=>"3993c1be-b01f-11e8-bee0-30e1716064fc", "namespace_labels"=>{"istio-injection"=>"disabled", "opa-validating-webhook"=>"enabled", "spooning"=>"dev"}}""";
**/ private final String k8sRegex = "^<\\d+>\\w+\\s+\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}\\s+[\\w-\\.]+\\.k8s";
@Multiline
private String k8sMessage;
private String k8sRegex = "^<\\d+>\\w+\\s+\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}\\s+[\\w-\\.]+\\.k8s";
/** private final String checkpointMessage = """
* <85>1 2019-02-18T17:37:47 10.18.9.141 CP-GW - Log [Fields@1.3.6.1.4.1.2620 Action="accept" UUid="{0x5c6aed6a,0x67,0x8d09120a,0xc0000002}" rule="313" rule_uid="{DB44DDE8-CD96-4B37-8A14-3E978D5BC322}" rule_name="RZ Splunk Fwd and Dep" src="192.168.55.11" dst="192.168.41.43" proto="6" product="VPN-1 & FireWall-1" service="9997" s_port="50172" product_family="Network"] <85>1 2019-02-18T17:37:47 10.18.9.141 CP-GW - Log [Fields@1.3.6.1.4.1.2620 Action="accept" UUid="{0x5c6aed6a,0x67,0x8d09120a,0xc0000002}" rule="313" rule_uid="{DB44DDE8-CD96-4B37-8A14-3E978D5BC322}" rule_name="RZ Splunk Fwd and Dep" src="192.168.55.11" dst="192.168.41.43" proto="6" product="VPN-1 & FireWall-1" service="9997" s_port="50172" product_family="Network"]""";
**/ private final String checkpointRegex = "^<\\d+>\\d+\\s+\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(.\\d{6})?([+-]\\d{2}:\\d{2})?\\s+[^\\s]+\\sCP-GW";
@Multiline
private String checkpointMessage;
private String checkpointRegex = "^<\\d+>\\d+\\s+\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(.\\d{6})?([+-]\\d{2}:\\d{2})?\\s+[^\\s]+\\sCP-GW";
/** private final String cyberarkMessage = """
* CEF:0|2019-02-18T17:37:47Z|GSPARK3|Cyber-Ark|Vault|9.99.0000|295|Retrieve password|5|act="Retrieve password" duser="dummmy" fname="Root\abc.net" src="172.22.1.1" cs1Label="Affected User Name" cs1="" cs2Label="Safe Name" cs2="AD" cs3Label="Location" cs3="" cs4Label="Property Name" cs4="" cs5Label="Target User Name" cs5="" cs6Label="Gateway Address" cs6="" cn1Label="Request Id" cn1="" msg="[AppID: Switches] Getting account password for Switches", , Retrieve password CEF:0|2019-02-18T17:37:47Z|GSPARK3|Cyber-Ark|Vault|9.99.0000|295|Retrieve password|5|act="Retrieve password" duser="dummmy" fname="Root\\abc.net" src="172.22.1.1" cs1Label="Affected User Name" cs1="" cs2Label="Safe Name" cs2="AD" cs3Label="Location" cs3="" cs4Label="Property Name" cs4="" cs5Label="Target User Name" cs5="" cs6Label="Gateway Address" cs6="" cn1Label="Request Id" cn1="" msg="[AppID: Switches] Getting account password for Switches", , Retrieve password""";
**/ private final String cyberarkRegex = "(?i)(\\w+\\s+\\d{1,2}\\s+\\d{2}:\\d{2}:\\d{2}|.+?\\|.+?\\|)\\s*...ark\\d";
@Multiline
private String cyberarkMessage;
private String cyberarkRegex = "(?i)(\\w+\\s+\\d{1,2}\\s+\\d{2}:\\d{2}:\\d{2}|.+?\\|.+?\\|)\\s*...ark\\d";
/**
* this is a simple message
**/
@Multiline
String simpleMessage;
private final String simpleMessage = """
this is a simple message""";
@Before @Before
public void setUp() { public void setUp() {

100
parsing/parsing-core/src/test/java/uk/co/gresearch/siembol/parsers/factory/ParserFactoryImplTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.parsers.factory; package uk.co.gresearch.siembol.parsers.factory;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import uk.co.gresearch.siembol.common.constants.SiembolMessageFields; import uk.co.gresearch.siembol.common.constants.SiembolMessageFields;
@@ -14,65 +13,60 @@ public class ParserFactoryImplTest {
factory = ParserFactoryImpl.createParserFactory(); factory = ParserFactoryImpl.createParserFactory();
} }
/** private final String simpleGenericParser = """
* { {
* "parser_name" : "test", "parser_name" : "test",
* "parser_version" : 1, "parser_version" : 1,
* "parser_author" : "dummy", "parser_author" : "dummy",
* "parser_attributes": { "parser_attributes": {
* "parser_type": "generic" "parser_type": "generic"
* }, },
* "parser_extractors": [ "parser_extractors": [
* { {
* "extractor_type": "json_extractor", "extractor_type": "json_extractor",
* "name": "test", "name": "test",
* "field": "original_string", "field": "original_string",
* "post_processing_functions": [ "post_processing_functions": [
* "format_timestamp" "format_timestamp"
* ], ],
* "attributes": { "attributes": {
* "should_overwrite_fields": true, "should_overwrite_fields": true,
* "should_remove_field": false, "should_remove_field": false,
* "time_formats": [ "time_formats": [
* { {
* "time_format": "yyyy-MM-dd HH:mm:ss.SSS 'Z'" "time_format": "yyyy-MM-dd HH:mm:ss.SSS 'Z'"
* } }
* ] ]
* } }
* } }
* ], ],
* "transformations": [ "transformations": [
* { {
* "transformation_type": "field_name_string_replace", "transformation_type": "field_name_string_replace",
* "attributes": { "attributes": {
* "string_replace_target": " ", "string_replace_target": " ",
* "string_replace_replacement": "_" "string_replace_replacement": "_"
* } }
* } }
* ] ]
* } }
**/ """;
@Multiline
public static String simpleGenericParser;
/** private final String message = """
* {"timestamp":"2019-03-27 18:52:02.732 Z", "test field" : true, "test_field1" : " message ", "test_field2" : " message "} {"timestamp":"2019-03-27 18:52:02.732 Z", "test field" : true, "test_field1" : " message ", "test_field2" : " message "}""";
**/
@Multiline
public static String message;
@Test @Test
public void testGetSchema() { public void testGetSchema() {
ParserFactoryResult schemaResult = factory.getSchema(); ParserFactoryResult schemaResult = factory.getSchema();
Assert.assertTrue(schemaResult.getStatusCode() == ParserFactoryResult.StatusCode.OK); Assert.assertSame(ParserFactoryResult.StatusCode.OK, schemaResult.getStatusCode());
Assert.assertFalse(schemaResult.getAttributes().getJsonSchema().isEmpty()); Assert.assertFalse(schemaResult.getAttributes().getJsonSchema().isEmpty());
} }
@Test @Test
public void testGoodCreate() { public void testGoodCreate() {
ParserFactoryResult result = factory.create(simpleGenericParser); ParserFactoryResult result = factory.create(simpleGenericParser);
Assert.assertTrue(result.getStatusCode() == ParserFactoryResult.StatusCode.OK); Assert.assertSame(ParserFactoryResult.StatusCode.OK, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getSiembolParser() != null); Assert.assertNotNull(result.getAttributes().getSiembolParser());
List<Map<String, Object>> parsed = result.getAttributes().getSiembolParser().parse(message.getBytes()); List<Map<String, Object>> parsed = result.getAttributes().getSiembolParser().parse(message.getBytes());
Assert.assertEquals(1553712722732L, parsed.get(0).get("timestamp")); Assert.assertEquals(1553712722732L, parsed.get(0).get("timestamp"));
@@ -83,20 +77,20 @@ public class ParserFactoryImplTest {
@Test @Test
public void testInvalidCreate() { public void testInvalidCreate() {
ParserFactoryResult result = factory.create("INVALID"); ParserFactoryResult result = factory.create("INVALID");
Assert.assertTrue(result.getStatusCode() == ParserFactoryResult.StatusCode.ERROR); Assert.assertSame(ParserFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage() != null); Assert.assertNotNull(result.getAttributes().getMessage());
} }
@Test @Test
public void testValidationGood() { public void testValidationGood() {
ParserFactoryResult result = factory.validateConfiguration(simpleGenericParser); ParserFactoryResult result = factory.validateConfiguration(simpleGenericParser);
Assert.assertTrue(result.getStatusCode() == ParserFactoryResult.StatusCode.OK); Assert.assertSame(ParserFactoryResult.StatusCode.OK, result.getStatusCode());
} }
@Test @Test
public void testTestingGood() { public void testTestingGood() {
ParserFactoryResult result = factory.test(simpleGenericParser, null, message.getBytes()); ParserFactoryResult result = factory.test(simpleGenericParser, null, message.getBytes());
Assert.assertTrue(result.getStatusCode() == ParserFactoryResult.StatusCode.OK); Assert.assertSame(result.getStatusCode(), ParserFactoryResult.StatusCode.OK);
List<Map<String, Object>> parsed = result.getAttributes().getParserResult().getParsedMessages(); List<Map<String, Object>> parsed = result.getAttributes().getParserResult().getParsedMessages();
Assert.assertEquals(1553712722732L, parsed.get(0).get("timestamp")); Assert.assertEquals(1553712722732L, parsed.get(0).get("timestamp"));
Assert.assertEquals(true, parsed.get(0).get("test_field")); Assert.assertEquals(true, parsed.get(0).get("test_field"));

131
parsing/parsing-core/src/test/java/uk/co/gresearch/siembol/parsers/generic/SiembolGenericParserTest.java
View File

@@ -1,5 +1,4 @@
package uk.co.gresearch.siembol.parsers.generic; package uk.co.gresearch.siembol.parsers.generic;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -14,77 +13,69 @@ public class SiembolGenericParserTest {
private SiembolParser genericParser; private SiembolParser genericParser;
private ParserFactory factory; private ParserFactory factory;
/** private final String simpleGenericParserConfig = """
* { {
* "parser_attributes": { "parser_attributes": {
* "parser_type": "generic" "parser_type": "generic"
* }, },
* "parser_extractors" : [ "parser_extractors" : [
* { {
* "extractor_type": "pattern_extractor", "extractor_type": "pattern_extractor",
* "name": "simple_message", "name": "simple_message",
* "field": "original_string", "field": "original_string",
* "attributes": { "attributes": {
* "regular_expressions": [ "regular_expressions": [
* "^msg:\\s(?<secret_msg>.*)$", "^msg:\\\\s(?<secret_msg>.*)$",
* "^msg2:\\s(?<timestamp>.*)$" "^msg2:\\\\s(?<timestamp>.*)$"
* ], ],
* "should_remove_field" : false "should_remove_field" : false
* } }
* }], }],
* "transformations" : [ "transformations" : [
* { {
* "transformation_type": "field_name_string_replace_all", "transformation_type": "field_name_string_replace_all",
* "attributes": { "attributes": {
* "string_replace_target": "secret_msg", "string_replace_target": "secret_msg",
* "string_replace_replacement": "dummy" "string_replace_replacement": "dummy"
* } }
* }] }]
* } }
**/ """;
@Multiline
public static String simpleGenericParserConfig;
/** private final String simpleGenericParserFiltered = """
* { {
* "parser_attributes": { "parser_attributes": {
* "parser_type": "generic" "parser_type": "generic"
* }, },
* "parser_extractors" : [ "parser_extractors" : [
* { {
* "extractor_type": "pattern_extractor", "extractor_type": "pattern_extractor",
* "name": "simple_message", "name": "simple_message",
* "field": "original_string", "field": "original_string",
* "attributes": { "attributes": {
* "regular_expressions": [ "regular_expressions": [
* "^msg:\\s(?<secret_msg>.*)$" "^msg:\\\\s(?<secret_msg>.*)$"
* ], ],
* "should_remove_field" : false "should_remove_field" : false
* } }
* }], }],
* "transformations" : [ "transformations" : [
* { {
* "transformation_type": "filter_message", "transformation_type": "filter_message",
* "attributes": { "attributes": {
* "message_filter" : { "message_filter" : {
* "matchers" : [ "matchers" : [
* { {
* "field_name" : "secret_msg", "field_name" : "secret_msg",
* "pattern" : "secret", "pattern" : "secret",
* "negated" : false "negated" : false
* }] }]
* }}}] }}}]
* } }
**/ """;
@Multiline
public static String simpleGenericParserFiltered;
/**
* msg: secret
**/
@Multiline
public static String simpleMessage;
private final String simpleMessage = """
msg: secret""";
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {

283
parsing/parsing-core/src/test/java/uk/co/gresearch/siembol/parsers/syslog/BasicSyslogSiembolParserTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.parsers.syslog; package uk.co.gresearch.siembol.parsers.syslog;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -26,163 +25,124 @@ public class BasicSyslogSiembolParserTest {
private SiembolParser syslogBsdParser; private SiembolParser syslogBsdParser;
private ParserFactory factory; private ParserFactory factory;
/** private final String syslogConfigRfc5424 = """
* { {
* "parser_attributes": { "parser_attributes": {
* "parser_type": "syslog", "parser_type": "syslog",
* "syslog_config": { "syslog_config": {
* "syslog_version": "RFC_5424", "syslog_version": "RFC_5424",
* "timezone": "UTC" "timezone": "UTC"
* } }
* } }
* } }
**/ """;
@Multiline
public static String syslogConfigRfc_5424;
/** private final String syslogConfigRfc5424MergeSdElements = """
* { {
* "parser_attributes": { "parser_attributes": {
* "parser_type": "syslog", "parser_type": "syslog",
* "syslog_config": { "syslog_config": {
* "syslog_version": "RFC_5424", "syslog_version": "RFC_5424",
* "timezone": "UTC", "timezone": "UTC",
* "merge_sd_elements" : true "merge_sd_elements" : true
* } }
* } }
* } }
**/ """;
@Multiline
public static String syslogConfigRfc_5424MergeSdElements;
/** private final String syslogConfigBsd = """
* { {
* "parser_attributes": { "parser_attributes": {
* "parser_type": "syslog", "parser_type": "syslog",
* "syslog_config": { "syslog_config": {
* "syslog_version": "RFC_3164", "syslog_version": "RFC_3164",
* "timezone": "UTC" "timezone": "UTC"
* } }
* } }
* } }
**/ """;
@Multiline
public static String syslogConfigBsd;
/** private final String syslogConfigBsdLondonTimezone = """
* { {
* "parser_attributes": { "parser_attributes": {
* "parser_type": "syslog", "parser_type": "syslog",
* "syslog_config": { "syslog_config": {
* "syslog_version": "RFC_3164", "syslog_version": "RFC_3164",
* "timezone": "Europe/London" "timezone": "Europe/London"
* } }
* } }
* } }
**/ """;
@Multiline
public static String syslogConfigBsdLondonTimezone;
/** private final String syslogConfigCustomTimestamp = """
* { {
* "parser_attributes": { "parser_attributes": {
* "parser_type": "syslog", "parser_type": "syslog",
* "syslog_config": { "syslog_config": {
* "syslog_version": "RFC_5424", "syslog_version": "RFC_5424",
* "time_formats": [ "time_formats": [
* { {
* "timezone": "UTC", "timezone": "UTC",
* "time_format": "yyyy-MM-dd'T'HH:mm:ss'Z'" "time_format": "yyyy-MM-dd'T'HH:mm:ss'Z'"
* }] }]
* } }
* }, },
* "parser_extractors": [ "parser_extractors": [
* { {
* "extractor_type": "pattern_extractor", "extractor_type": "pattern_extractor",
* "name": "dummy", "name": "dummy",
* "field": "syslog_msg", "field": "syslog_msg",
* "attributes": { "attributes": {
* "regular_expressions": [ "regular_expressions": [
* "^SALscanner INFO TEST:\\s(?<info_msg>.*)$" "^SALscanner INFO TEST:\\\\s(?<info_msg>.*)$"
* ], ],
* "should_remove_field": false "should_remove_field": false
* } }
* } }
* ], ],
* "transformations" : [ "transformations" : [
* { {
* "transformation_type": "field_name_string_replace_all", "transformation_type": "field_name_string_replace_all",
* "attributes": { "attributes": {
* "string_replace_target": "syslog", "string_replace_target": "syslog",
* "string_replace_replacement": "dummy" "string_replace_replacement": "dummy"
* } }
* }] }]
* } }
**/ """;
@Multiline
public static String syslogConfigCustomTimpestamp;
/** private final String goodSyslogCheckpoint1 = """
* <85>1 2018-05-22T17:07:41+01:00 172.16.18.101 CP-GW - Log [Fields@1.3.6.1.4.1.2620 Action="accept" UUid="{0x5b04404c,0x10004,0x651210ac,0xc0000000}" rule="14" rule_uid="{28F2CB68-9017-442B-8C64-6BD43B8082CD}" rule_name="DNS" src="172.16.16.20" dst="172.16.37.100" proto="17" product="VPN-1 & FireWall-1" service="53" s_port="60349" product_family="Network"] <85>1 2018-05-22T17:07:41+01:00 172.16.18.101 CP-GW - Log [Fields@1.3.6.1.4.1.2620 Action="accept" UUid="{0x5b04404c,0x10004,0x651210ac,0xc0000000}" rule="14" rule_uid="{28F2CB68-9017-442B-8C64-6BD43B8082CD}" rule_name="DNS" src="172.16.16.20" dst="172.16.37.100" proto="17" product="VPN-1 & FireWall-1" service="53" s_port="60349" product_family="Network"]""";
**/
@Multiline
public static String goodSyslogCheckpoint1;
/** private final String goodSyslogCheckpoint2 = """
* <81>1 2018-05-22T03:05:37 172.19.34.31 CP-GW - Alert [Fields@1.3.6.1.4.1.2620 Action=" " UUid="{0x0,0x0,0x0,0x0}" Protection Name="Packet Sanity" Severity="2" Confidence Level="5" protection_id="PacketSanity" SmartDefense Profile="Perimeter_Protection" Performance Impact="1" Industry Reference="CAN-2002-1071" Protection Type="anomaly" Attack Info="Invalid TCP flag combination" attack="Malformed Packet" Total logs="24" Suppressed logs="23" proto="6" dst="10.254.101.253" src="10.254.101.12" product="SmartDefense" FollowUp="Not Followed" product_family="Network"] <81>1 2018-05-22T03:05:37 172.19.34.31 CP-GW - Alert [Fields@1.3.6.1.4.1.2620 Action=" " UUid="{0x0,0x0,0x0,0x0}" Protection Name="Packet Sanity" Severity="2" Confidence Level="5" protection_id="PacketSanity" SmartDefense Profile="Perimeter_Protection" Performance Impact="1" Industry Reference="CAN-2002-1071" Protection Type="anomaly" Attack Info="Invalid TCP flag combination" attack="Malformed Packet" Total logs="24" Suppressed logs="23" proto="6" dst="10.254.101.253" src="10.254.101.12" product="SmartDefense" FollowUp="Not Followed" product_family="Network"]""";
**/
@Multiline private final String goodSyslogEscapedChars = """
public static String goodSyslogCheckpoint2; <81>1 2018-05-22T03:05:37 172.19.34.31 CP-GW - Alert [Fields@1.3.6.1.4.1.2620 Action=" " UUid="{0x0,0x0,0x0,0x0}" Protection Name="Packet\\" \\] Sanity"]""";
private final String syslogEscapedChars2 = """
<81>1 2018-05-22T03:05:37 172.19.34.31 CP-GW - Alert [Fields@1.3.6.1.4.1.2620 Action=" " Protection Name="Packet" \\] Sanity"][Fields@1.3.6.1.4.1.2620] BOMabcabc""";
private final String goodNilSD = """
<81>1 2018-05-22T03:05:37 172.19.34.31 CP-GW - Alert - BOMabcabc""";
private final String goodBSD = """
<34>Oct 11 22:14:15 mymachine su: 'su root' failed for dummy on /dev/pts/8""";
private final String strangeCheckpoint = """
<85>1 2018-08-01T09:00:24+01:00 10.254.112.76 CP-GW - Log [Fields@1.3.6.1.4.1.2620 Action=" " UUid="{0x0,0x0,0x0,0x0}" default_device_message="<133>xpand[17978]: admin localhost t +installer:packages:Check_Point_R77_30_JUMBO_HF_1_Bundle_T286_FULL.tgz:has_metadata 0 (+)" facility="local use 0" syslog_severity="Notice" product="Syslog" product_family="Network"]""";
/** private final String customTimeformat = """
* <81>1 2018-05-22T03:05:37 172.19.34.31 CP-GW - Alert [Fields@1.3.6.1.4.1.2620 Action=" " UUid="{0x0,0x0,0x0,0x0}" Protection Name="Packet\" \] Sanity"] <190>1 2019-01-15T12:36:05Z mime1-private.internal.net sal - - - SALscanner INFO TEST: [manlistEmail] applianceupdate.clearswift.com ... [688]""";
**/
@Multiline
public static String goodSyslogEscapedChars;
/** private final String multipleSdElementsDummyCheckpoint1 = """
* <81>1 2018-05-22T03:05:37 172.19.34.31 CP-GW - Alert [Fields@1.3.6.1.4.1.2620 Action=" " Protection Name="Packet" \] Sanity"][Fields@1.3.6.1.4.1.2620] BOMabcabc <85>1 2018-05-22T17:07:41+01:00 172.16.18.101 CP-GW - Log [Fields@1.3.6.1.4.1.2620 Action1="accept"][Fields@1.3.6.1.4.1.2620 Action2="deny"]""";
**/
@Multiline
public static String syslogEscapedChars2;
/**
* <81>1 2018-05-22T03:05:37 172.19.34.31 CP-GW - Alert - BOMabcabc
**/
@Multiline
public static String goodNilSD;
/**
* <34>Oct 11 22:14:15 mymachine su: 'su root' failed for dummy on /dev/pts/8
**/
@Multiline
public static String goodBSD;
/**
* <85>1 2018-08-01T09:00:24+01:00 10.254.112.76 CP-GW - Log [Fields@1.3.6.1.4.1.2620 Action=" " UUid="{0x0,0x0,0x0,0x0}" default_device_message="<133>xpand[17978]: admin localhost t +installer:packages:Check_Point_R77_30_JUMBO_HF_1_Bundle_T286_FULL.tgz:has_metadata 0 (+)" facility="local use 0" syslog_severity="Notice" product="Syslog" product_family="Network"]
**/
@Multiline
public static String strangeCheckpoint;
/** private final String multipleSdElementsDummyCheckpoint2 = """
* <190>1 2019-01-15T12:36:05Z mime1-eqld.uberit.net sal - - - SALscanner INFO TEST: [manlistEmail] applianceupdate.clearswift.com ... [688] <85>1 2018-05-22T17:07:41Z 172.16.18.101 CP-GW - Log [Fields@1.3.6.1.4.1.2620 syslog1="accept"][Fields@1.3.6.1.4.1.2620 syslog2="deny"]""";
*/
@Multiline
public static String customTimeformat;
/**
* <85>1 2018-05-22T17:07:41+01:00 172.16.18.101 CP-GW - Log [Fields@1.3.6.1.4.1.2620 Action1="accept"][Fields@1.3.6.1.4.1.2620 Action2="deny"]
**/
@Multiline
public static String multipleSdElementsDummyCheckpoint1;
/**
* <85>1 2018-05-22T17:07:41Z 172.16.18.101 CP-GW - Log [Fields@1.3.6.1.4.1.2620 syslog1="accept"][Fields@1.3.6.1.4.1.2620 syslog2="deny"]
**/
@Multiline
public static String multipleSdElementsDummyCheckpoint2;
public BasicSyslogSiembolParserTest(Locale locale) { public BasicSyslogSiembolParserTest(Locale locale) {
Locale.setDefault(locale); Locale.setDefault(locale);
@@ -191,7 +151,7 @@ public class BasicSyslogSiembolParserTest {
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
factory = ParserFactoryImpl.createParserFactory(); factory = ParserFactoryImpl.createParserFactory();
syslogParser = factory.create(syslogConfigRfc_5424).getAttributes().getSiembolParser(); syslogParser = factory.create(syslogConfigRfc5424).getAttributes().getSiembolParser();
syslogBsdParser = factory.create(syslogConfigBsd).getAttributes().getSiembolParser(); syslogBsdParser = factory.create(syslogConfigBsd).getAttributes().getSiembolParser();
} }
@@ -299,7 +259,7 @@ public class BasicSyslogSiembolParserTest {
} }
@Test @Test
public void goodBSD() throws Exception { public void goodBSD() {
Map<String, Object> out = syslogBsdParser.parse(goodBSD.trim().getBytes()).get(0); Map<String, Object> out = syslogBsdParser.parse(goodBSD.trim().getBytes()).get(0);
Assert.assertEquals(0, out.get("syslog_version")); Assert.assertEquals(0, out.get("syslog_version"));
@@ -312,7 +272,7 @@ public class BasicSyslogSiembolParserTest {
} }
@Test @Test
public void goodBSDWithBSDTZ() throws Exception { public void goodBSDWithBSDTZ() {
syslogBsdParser = factory.create(syslogConfigBsdLondonTimezone.trim()).getAttributes().getSiembolParser(); syslogBsdParser = factory.create(syslogConfigBsdLondonTimezone.trim()).getAttributes().getSiembolParser();
Map<String, Object> out = syslogBsdParser.parse(goodBSD.trim().getBytes()).get(0); Map<String, Object> out = syslogBsdParser.parse(goodBSD.trim().getBytes()).get(0);
@@ -325,7 +285,7 @@ public class BasicSyslogSiembolParserTest {
} }
@Test @Test
public void goodBSDWithUTCTZ() throws Exception { public void goodBSDWithUTCTZ() {
Map<String, Object> out = syslogBsdParser.parse(goodBSD.trim().getBytes()).get(0); Map<String, Object> out = syslogBsdParser.parse(goodBSD.trim().getBytes()).get(0);
Assert.assertEquals(0, out.get("syslog_version")); Assert.assertEquals(0, out.get("syslog_version"));
@@ -348,12 +308,12 @@ public class BasicSyslogSiembolParserTest {
} }
@Test @Test
public void customTimestamp() throws Exception { public void customTimestamp() {
syslogParser = factory.create(syslogConfigCustomTimpestamp).getAttributes().getSiembolParser(); syslogParser = factory.create(syslogConfigCustomTimestamp).getAttributes().getSiembolParser();
Map<String, Object> out = syslogParser.parse(customTimeformat.trim().getBytes()).get(0); Map<String, Object> out = syslogParser.parse(customTimeformat.trim().getBytes()).get(0);
Assert.assertEquals(1547555765000L, out.get("timestamp")); Assert.assertEquals(1547555765000L, out.get("timestamp"));
Assert.assertEquals("mime1-eqld.uberit.net", out.get("dummy_hostname")); Assert.assertEquals("mime1-private.internal.net", out.get("dummy_hostname"));
Assert.assertEquals("sal", out.get("dummy_appname")); Assert.assertEquals("sal", out.get("dummy_appname"));
Assert.assertEquals(1, out.get("dummy_version")); Assert.assertEquals(1, out.get("dummy_version"));
Assert.assertEquals(23, out.get("dummy_facility")); Assert.assertEquals(23, out.get("dummy_facility"));
@@ -363,8 +323,8 @@ public class BasicSyslogSiembolParserTest {
} }
@Test @Test
public void customTimestampInvalid() throws Exception { public void customTimestampInvalid() {
syslogParser = factory.create(syslogConfigCustomTimpestamp).getAttributes().getSiembolParser(); syslogParser = factory.create(syslogConfigCustomTimestamp).getAttributes().getSiembolParser();
Map<String, Object> out = syslogParser.parse(customTimeformat.trim() Map<String, Object> out = syslogParser.parse(customTimeformat.trim()
.replace("2019-01-15T12:36:05Z", "INVALID") .replace("2019-01-15T12:36:05Z", "INVALID")
.getBytes()) .getBytes())
@@ -372,7 +332,7 @@ public class BasicSyslogSiembolParserTest {
Assert.assertEquals("INVALID", out.get("dummy_timestamp")); Assert.assertEquals("INVALID", out.get("dummy_timestamp"));
Assert.assertNotNull(out.get("timestamp")); Assert.assertNotNull(out.get("timestamp"));
Assert.assertEquals("mime1-eqld.uberit.net", out.get("dummy_hostname")); Assert.assertEquals("mime1-private.internal.net", out.get("dummy_hostname"));
Assert.assertEquals("sal", out.get("dummy_appname")); Assert.assertEquals("sal", out.get("dummy_appname"));
Assert.assertEquals(1, out.get("dummy_version")); Assert.assertEquals(1, out.get("dummy_version"));
Assert.assertEquals(23, out.get("dummy_facility")); Assert.assertEquals(23, out.get("dummy_facility"));
@@ -382,8 +342,8 @@ public class BasicSyslogSiembolParserTest {
} }
@Test @Test
public void mergingSdparameters(){ public void mergingSdParameters() {
syslogParser = factory.create(syslogConfigRfc_5424MergeSdElements).getAttributes().getSiembolParser(); syslogParser = factory.create(syslogConfigRfc5424MergeSdElements).getAttributes().getSiembolParser();
List<Map<String, Object>> out = syslogParser.parse(multipleSdElementsDummyCheckpoint1.trim().getBytes()); List<Map<String, Object>> out = syslogParser.parse(multipleSdElementsDummyCheckpoint1.trim().getBytes());
Assert.assertEquals(1, out.size()); Assert.assertEquals(1, out.size());
Map<String, Object> current = out.get(0); Map<String, Object> current = out.get(0);
@@ -394,7 +354,7 @@ public class BasicSyslogSiembolParserTest {
} }
@Test @Test
public void noMergeSdparameters(){ public void noMergeSdParameters() {
List<Map<String, Object>> out = syslogParser.parse(multipleSdElementsDummyCheckpoint1.trim().getBytes()); List<Map<String, Object>> out = syslogParser.parse(multipleSdElementsDummyCheckpoint1.trim().getBytes());
Assert.assertEquals(2, out.size()); Assert.assertEquals(2, out.size());
Map<String, Object> current1 = out.get(0); Map<String, Object> current1 = out.get(0);
@@ -407,8 +367,8 @@ public class BasicSyslogSiembolParserTest {
} }
@Test @Test
public void noMergeSdparametersExtractAndTransform(){ public void noMergeSdParametersExtractAndTransform(){
syslogParser = factory.create(syslogConfigCustomTimpestamp).getAttributes().getSiembolParser(); syslogParser = factory.create(syslogConfigCustomTimestamp).getAttributes().getSiembolParser();
List<Map<String, Object>> out = syslogParser.parse(multipleSdElementsDummyCheckpoint2.trim().getBytes()); List<Map<String, Object>> out = syslogParser.parse(multipleSdElementsDummyCheckpoint2.trim().getBytes());
Assert.assertEquals(2, out.size()); Assert.assertEquals(2, out.size());
Map<String, Object> current1 = out.get(0); Map<String, Object> current1 = out.get(0);
@@ -420,4 +380,3 @@ public class BasicSyslogSiembolParserTest {
Assert.assertEquals("Fields@1.3.6.1.4.1.2620", current2.get("dummy_sd_id")); Assert.assertEquals("Fields@1.3.6.1.4.1.2620", current2.get("dummy_sd_id"));
} }
} }

297
parsing/parsing-core/src/test/java/uk/co/gresearch/siembol/parsers/transformations/TransformationsTest.java
View File

@@ -3,7 +3,6 @@ package uk.co.gresearch.siembol.parsers.transformations;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectReader;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -29,181 +28,153 @@ public class TransformationsTest {
log = JSON_LOG_READER.readValue(message); log = JSON_LOG_READER.readValue(message);
} }
/** private final String transformationReplace = """
* { {
* "transformation_type": "field_name_string_replace", "transformation_type": "field_name_string_replace",
* "attributes": { "attributes": {
* "string_replace_target": " ", "string_replace_target": " ",
* "string_replace_replacement": "_" "string_replace_replacement": "_"
* } }
* } }
**/ """;
@Multiline
public static String transformationReplace;
/** private final String transformationFilter = """
* { {
* "transformation_type": "filter_message", "transformation_type": "filter_message",
* "attributes": { "attributes": {
* "message_filter" : { "message_filter" : {
* "matchers" : [ "matchers" : [
* { {
* "field_name" : "dummy field", "field_name" : "dummy field",
* "pattern" : "abc", "pattern" : "abc",
* "negated" : false "negated" : false
* }, },
* { {
* "field_name" : "secret_field", "field_name" : "secret_field",
* "pattern" : "secret", "pattern" : "secret",
* "negated" : true "negated" : true
* } }
* ] ]
* } }
* } }
* } }
**/ """;
@Multiline
public static String transformationFilter;
/** private final String transformationReplaceAll = """
* { {
* "transformation_type": "field_name_string_replace_all", "transformation_type": "field_name_string_replace_all",
* "attributes": { "attributes": {
* "string_replace_target": " ", "string_replace_target": " ",
* "string_replace_replacement": "_" "string_replace_replacement": "_"
* } }
* } }
**/ """;
@Multiline
public static String transformationReplaceAll;
/** private final String transformationDeleteAll = """
* { {
* "transformation_type": "field_name_string_delete_all", "transformation_type": "field_name_string_delete_all",
* "attributes": { "attributes": {
* "string_replace_target": " " "string_replace_target": " "
* } }
* } }
**/ """;
@Multiline
public static String transformationDeleteAll;
/** private final String transformationTrim = """
*{ {
* "transformation_type": "trim_value", "transformation_type": "trim_value",
* "attributes": { "attributes": {
* "fields_filter": { "fields_filter": {
* "including_fields": ["timestamp", "trim_field"] "including_fields": ["timestamp", "trim_field"]
* } }
* } }
* } }
**/ """;
@Multiline
public static String transformationTrim;
/**
*{
* "transformation_type": "chomp_value",
* "attributes": {
* "fields_filter": {
* "including_fields": ["timestamp", "chomp_field"]
* }
* }
* }
**/
@Multiline
public static String transformationChomp;
/** private final String transformationChomp = """
*{ {
* "transformation_type": "delete_fields", "transformation_type": "chomp_value",
* "attributes": { "attributes": {
* "fields_filter": { "fields_filter": {
* "including_fields": [".*"], "including_fields": ["timestamp", "chomp_field"]
* "excluding_fields": ["timestamp"] }
* } }
* } }
* } """;
**/
@Multiline
public static String transformationDelete;
/** private final String transformationDelete = """
*{ {
* "transformation_type": "rename_fields", "transformation_type": "delete_fields",
* "attributes": { "attributes": {
* "field_rename_map": [ "fields_filter": {
* { "including_fields": [".*"],
* "field_to_rename": "timestamp", "excluding_fields": ["timestamp"]
* "new_name": "timestamp_renamed" }
* }, }
* { }
* "field_to_rename": "dummy field", """;
* "new_name": "dummy_field_renamed"
* }
* ]
* }
* }
**/
@Multiline
public static String transformationRename;
/** private final String transformationRename = """
*{ {
* "transformation_type": "field_name_change_case", "transformation_type": "rename_fields",
* "attributes": { "attributes": {
* "case_type" : "uppercase" "field_rename_map": [
* } {
* } "field_to_rename": "timestamp",
**/ "new_name": "timestamp_renamed"
@Multiline },
public static String transformationFieldNameUpperCase; {
"field_to_rename": "dummy field",
"new_name": "dummy_field_renamed"
}
]
}
}
""";
/** private final String transformationFieldNameUpperCase = """
*{ {
* "transformation_type": "field_name_change_case", "transformation_type": "field_name_change_case",
* "attributes": { "attributes": {
* "case_type" : "lowercase" "case_type" : "uppercase"
* } }
* } }
**/ """;
@Multiline
public static String transformationFieldLowerCase;
/** private final String transformationFieldLowerCase = """
*{ {
* "transformation_type": "lowercase_value", "transformation_type": "field_name_change_case",
* "attributes": { "attributes": {
* "fields_filter": { "case_type" : "lowercase"
* "including_fields": ["timestamp", "chomp_field"] }
* } }
* } """;
* }
**/
@Multiline
public static String transformationLowerCase;
/** private final String transformationLowerCase = """
*{ {
* "transformation_type": "uppercase_value", "transformation_type": "lowercase_value",
* "attributes": { "attributes": {
* "fields_filter": { "fields_filter": {
* "including_fields": ["timestamp", "chomp_field"] "including_fields": ["timestamp", "chomp_field"]
* } }
* } }
* } }
**/ """;
@Multiline
public static String transformationUpperCase;
private final String transformationUpperCase = """
{
"transformation_type": "uppercase_value",
"attributes": {
"fields_filter": {
"including_fields": ["timestamp", "chomp_field"]
}
}
}
""";
private final String message = """
/** {"timestamp":12345, "test field a" : "true", "trim_field" : " message ", "dummy field" : "abc", "chomp_field" : "message\\n"}
* {"timestamp":12345, "test field a" : "true", "trim_field" : " message ", "dummy field" : "abc", "chomp_field" : "message\n"} """;
**/
@Multiline
public static String message;
@Test @Test
public void testGoodReplace() throws IOException { public void testGoodReplace() throws IOException {

27
parsing/parsing-storm/pom.xml
View File

@@ -9,9 +9,14 @@
<parent> <parent>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing</artifactId> <artifactId>parsing</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</parent> </parent>
<dependencies> <dependencies>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>23.0</version>
</dependency>
<dependency> <dependency>
<groupId>com.fasterxml.jackson.core</groupId> <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId> <artifactId>jackson-core</artifactId>
@@ -62,7 +67,7 @@
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId> <artifactId>parsing-app</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
<exclusions> <exclusions>
<exclusion> <exclusion>
<groupId>org.slf4j</groupId> <groupId>org.slf4j</groupId>
@@ -84,12 +89,6 @@
<version>${junit_version}</version> <version>${junit_version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>org.apache.zookeeper</groupId> <groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId> <artifactId>zookeeper</artifactId>
@@ -227,18 +226,6 @@
</execution> </execution>
</executions> </executions>
</plugin> </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<forceJavacCompilerUse>true</forceJavacCompilerUse>
<source>${java_version}</source>
<compilerArgument>-Xlint:unchecked</compilerArgument>
<target>${java_version}</target>
<showWarnings>true</showWarnings>
</configuration>
</plugin>
</plugins> </plugins>
</build> </build>
</project> </project>

104
parsing/parsing-storm/src/test/java/uk/co/gresearch/siembol/parsers/storm/ParsingApplicationBoltTest.java
View File

@@ -3,7 +3,6 @@ package uk.co.gresearch.siembol.parsers.storm;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectReader;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.storm.task.OutputCollector; import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values; import org.apache.storm.tuple.Values;
@@ -30,66 +29,59 @@ import static org.mockito.Mockito.when;
public class ParsingApplicationBoltTest { public class ParsingApplicationBoltTest {
private static ObjectReader JSON_READER = new ObjectMapper() private static ObjectReader JSON_READER = new ObjectMapper()
.readerFor(new TypeReference<Map<String, Object>>() {}); .readerFor(new TypeReference<Map<String, Object>>() {});
/** String log = """
*RAW_LOG RAW_LOG
**/ """;
@Multiline
public static String log;
/** public static String metadata = """
* {"is_metadata" : true} {"is_metadata" : true}
**/ """;
@Multiline
public static String metadata;
/**
*{
* "parsing_app_name": "test",
* "parsing_app_version": 1,
* "parsing_app_author": "dummy",
* "parsing_app_description": "Description of parser application",
* "parsing_app_settings": {
* "input_topics": [
* "secret"
* ],
* "parse_metadata" : false,
* "error_topic": "error",
* "input_parallelism": 1,
* "parsing_parallelism": 2,
* "output_parallelism": 3,
* "parsing_app_type": "single_parser"
* },
* "parsing_settings": {
* "single_parser": {
* "parser_name": "single",
* "output_topic": "output"
* }
* }
* }
**/
@Multiline
public static String simpleSingleApplicationParser;
/** public static String simpleSingleApplicationParser = """
* { {
* "parsers_version": 1, "parsing_app_name": "test",
* "parsers_configurations": [ "parsing_app_version": 1,
* { "parsing_app_author": "dummy",
* "parser_description": "for testing single app parser", "parsing_app_description": "Description of parser application",
* "parser_version": 2, "parsing_app_settings": {
* "parser_name": "single", "input_topics": [
* "parser_author": "dummy", "secret"
* "parser_attributes": { ],
* "parser_type": "generic" "parse_metadata" : false,
* } "error_topic": "error",
* } "input_parallelism": 1,
* ] "parsing_parallelism": 2,
* } "output_parallelism": 3,
**/ "parsing_app_type": "single_parser"
@Multiline },
public static String testParsersConfigs; "parsing_settings": {
"single_parser": {
"parser_name": "single",
"output_topic": "output"
}
}
}
""";
public static String testParsersConfigs = """
{
"parsers_version": 1,
"parsers_configurations": [
{
"parser_description": "for testing single app parser",
"parser_version": 2,
"parser_name": "single",
"parser_author": "dummy",
"parser_attributes": {
"parser_type": "generic"
}
}
]
}
""";
private Tuple tuple; private Tuple tuple;
private OutputCollector collector; private OutputCollector collector;
ParsingApplicationBolt parsingApplicationBolt; ParsingApplicationBolt parsingApplicationBolt;

150
parsing/parsing-storm/src/test/java/uk/co/gresearch/siembol/parsers/storm/StormParsingApplicationTest.java
View File

@@ -4,7 +4,6 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.charithe.kafka.EphemeralKafkaBroker; import com.github.charithe.kafka.EphemeralKafkaBroker;
import com.github.charithe.kafka.KafkaJunitRule; import com.github.charithe.kafka.KafkaJunitRule;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.storm.Config; import org.apache.storm.Config;
import org.apache.storm.LocalCluster; import org.apache.storm.LocalCluster;
import org.apache.storm.generated.StormTopology; import org.apache.storm.generated.StormTopology;
@@ -26,88 +25,81 @@ import static org.mockito.Mockito.when;
import static org.mockito.Mockito.withSettings; import static org.mockito.Mockito.withSettings;
public class StormParsingApplicationTest { public class StormParsingApplicationTest {
/** public static String log = """
*RAW_LOG RAW_LOG
**/ """;
@Multiline
public static String log;
/** public static String simpleSingleApplicationParser = """
*{ {
* "parsing_app_name": "test", "parsing_app_name": "test",
* "parsing_app_version": 1, "parsing_app_version": 1,
* "parsing_app_author": "dummy", "parsing_app_author": "dummy",
* "parsing_app_description": "Description of parser application", "parsing_app_description": "Description of parser application",
* "parsing_app_settings": { "parsing_app_settings": {
* "input_topics": [ "input_topics": [
* "input" "input"
* ], ],
* "parse_metadata" : false, "parse_metadata" : false,
* "error_topic": "error", "error_topic": "error",
* "input_parallelism": 1, "input_parallelism": 1,
* "parsing_parallelism": 1, "parsing_parallelism": 1,
* "output_parallelism": 1, "output_parallelism": 1,
* "parsing_app_type": "single_parser" "parsing_app_type": "single_parser"
* }, },
* "parsing_settings": { "parsing_settings": {
* "single_parser": { "single_parser": {
* "parser_name": "single", "parser_name": "single",
* "output_topic": "output" "output_topic": "output"
* } }
* } }
* } }
**/ """;
@Multiline
public static String simpleSingleApplicationParser;
/** public static String testParsersConfigs = """
* { {
* "parsers_version": 1, "parsers_version": 1,
* "parsers_configurations": [ "parsers_configurations": [
* { {
* "parser_description": "for testing single app parser", "parser_description": "for testing single app parser",
* "parser_version": 2, "parser_version": 2,
* "parser_name": "single", "parser_name": "single",
* "parser_author": "dummy", "parser_author": "dummy",
* "parser_attributes": { "parser_attributes": {
* "parser_type": "generic" "parser_type": "generic"
* } }
* } }
* ] ]
* } }
**/ """;
@Multiline
public static String testParsersConfigs;
/**
* { public static String stormSettings = """
* "client.id.prefix": "test_writer", {
* "group.id.prefix": "test_reader", "client.id.prefix": "test_writer",
* "zookeeper.attributes": { "group.id.prefix": "test_reader",
* "zk.path": "/parserconfigs", "zookeeper.attributes": {
* "zk.base.sleep.ms": 1000, "zk.path": "/parserconfigs",
* "zk.max.retries": 10 "zk.base.sleep.ms": 1000,
* }, "zk.max.retries": 10
* "kafka.batch.writer.attributes": { },
* "batch.size": 1, "kafka.batch.writer.attributes": {
* "producer.properties": { "batch.size": 1,
* "security.protocol": "PLAINTEXT" "producer.properties": {
* } "security.protocol": "PLAINTEXT"
* }, }
* "storm.attributes": { },
* "first.pool.offset.strategy": "EARLIEST", "storm.attributes": {
* "kafka.spout.properties": { "first.pool.offset.strategy": "EARLIEST",
* "security.protocol": "PLAINTEXT" "kafka.spout.properties": {
* }, "security.protocol": "PLAINTEXT"
* "storm.config": { },
* "session.timeout.ms": 100000 "storm.config": {
* } "session.timeout.ms": 100000
* } }
* } }
**/ }
@Multiline """;
public static String stormSettings;
@ClassRule @ClassRule
public static KafkaJunitRule kafkaRule = new KafkaJunitRule(EphemeralKafkaBroker.create()); public static KafkaJunitRule kafkaRule = new KafkaJunitRule(EphemeralKafkaBroker.create());

2
parsing/pom.xml
View File

@@ -11,7 +11,7 @@
<parent> <parent>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId> <artifactId>siembol</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</parent> </parent>
<modules> <modules>
<module>parsing-core</module> <module>parsing-core</module>

25
pom.xml
View File

@@ -6,7 +6,7 @@
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId> <artifactId>siembol</artifactId>
<name>siembol</name> <name>siembol</name>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
<description>A scalable, advanced security analytics framework based on open-source big data technologies.</description> <description>A scalable, advanced security analytics framework based on open-source big data technologies.</description>
<inceptionYear>2019</inceptionYear> <inceptionYear>2019</inceptionYear>
<url>https://siembol.io/</url> <url>https://siembol.io/</url>
@@ -36,9 +36,9 @@
</issueManagement> </issueManagement>
<properties> <properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.8</maven.compiler.source> <java_test_version>13</java_test_version>
<maven.compiler.target>1.8</maven.compiler.target> <maven.compiler.release>11</maven.compiler.release>
<java_version>1.8</java_version> <maven_compiler_version>3.8.1</maven_compiler_version>
<jackson_version>2.13.0</jackson_version> <jackson_version>2.13.0</jackson_version>
<junit_version>4.13.2</junit_version> <junit_version>4.13.2</junit_version>
<mockito_version>4.0.0</mockito_version> <mockito_version>4.0.0</mockito_version>
@@ -46,7 +46,7 @@
<curator_version>4.3.0</curator_version> <curator_version>4.3.0</curator_version>
<curator_test_version>2.13.0</curator_test_version> <curator_test_version>2.13.0</curator_test_version>
<zookeeper_version>3.4.6</zookeeper_version> <zookeeper_version>3.4.6</zookeeper_version>
<storm_version>1.2.1</storm_version> <storm_version>2.3.0</storm_version>
<kafka_storm_version>0.10.2.2</kafka_storm_version> <kafka_storm_version>0.10.2.2</kafka_storm_version>
<kafka_version>2.8.0</kafka_version> <kafka_version>2.8.0</kafka_version>
<kafka_junit_version>3.1.1</kafka_junit_version> <kafka_junit_version>3.1.1</kafka_junit_version>
@@ -112,17 +112,24 @@
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId> <artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version> <version>${maven_compiler_version}</version>
<configuration> <configuration>
<forceJavacCompilerUse>true</forceJavacCompilerUse> <forceJavacCompilerUse>true</forceJavacCompilerUse>
<source>${java_version}</source>
<compilerArgs> <compilerArgs>
<arg>-Xlint:all</arg> <arg>-Xlint:all</arg>
<arg>-Xlint:-processing</arg> <arg>-Xlint:-processing</arg>
</compilerArgs> </compilerArgs>
<target>${java_version}</target>
<showWarnings>true</showWarnings> <showWarnings>true</showWarnings>
</configuration> </configuration>
<executions>
<execution>
<id>default-testCompile</id>
<configuration>
<release>${java_test_version}</release>
<compilerArgs>--enable-preview</compilerArgs>
</configuration>
</execution>
</executions>
</plugin> </plugin>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
@@ -138,6 +145,8 @@
<configuration> <configuration>
<reuseForks>true</reuseForks> <reuseForks>true</reuseForks>
<forkedProcessExitTimeoutInSeconds>60</forkedProcessExitTimeoutInSeconds> <forkedProcessExitTimeoutInSeconds>60</forkedProcessExitTimeoutInSeconds>
<argLine>--illegal-access=permit</argLine>
<argLine>--enable-preview</argLine>
</configuration> </configuration>
</plugin> </plugin>
<plugin> <plugin>

2
responding/pom.xml
View File

@@ -11,7 +11,7 @@
<parent> <parent>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId> <artifactId>siembol</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</parent> </parent>
<modules> <modules>
<module>responding-core</module> <module>responding-core</module>

12
responding/responding-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent> <parent>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding</artifactId> <artifactId>responding</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</parent> </parent>
<dependencies> <dependencies>
<dependency> <dependency>
@@ -35,24 +35,18 @@
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId> <artifactId>siembol-common</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>uk.co.gresearch.siembol</groupId> <groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId> <artifactId>alerting-core</artifactId>
<version>1.4.2-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.jayway.jsonpath</groupId> <groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId> <artifactId>json-path</artifactId>
<version>2.6.0</version> <version>2.6.0</version>
</dependency> </dependency>
<dependency>
<groupId>org.adrianwalker</groupId>
<artifactId>multiline-string</artifactId>
<version>${multiline_string_version}</version>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>junit</groupId> <groupId>junit</groupId>
<artifactId>junit</artifactId> <artifactId>junit</artifactId>

215
responding/responding-core/src/test/java/uk/co/gresearch/siembol/response/compiler/RespondingCompilerImplTest.java
View File

@@ -1,6 +1,5 @@
package uk.co.gresearch.siembol.response.compiler; package uk.co.gresearch.siembol.response.compiler;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@@ -13,119 +12,110 @@ import static org.mockito.Mockito.when;
import static uk.co.gresearch.siembol.response.common.RespondingResult.StatusCode.OK; import static uk.co.gresearch.siembol.response.common.RespondingResult.StatusCode.OK;
public class RespondingCompilerImplTest { public class RespondingCompilerImplTest {
/** private final String evaluatorAttributes = """
* { {
* "type" : "object", "type" : "object",
* "description" : "Attributes for fixed evaluator", "description" : "Attributes for fixed evaluator",
* "title" : "fixed evaluator attributes", "title" : "fixed evaluator attributes",
* "properties" : { "properties" : {
* "evaluation_result" : { "evaluation_result" : {
* "enum" : [ "match", "no_match", "filtered" ], "enum" : [ "match", "no_match", "filtered" ],
* "type" : "string", "type" : "string",
* "description" : "Evaluation result returned by the evaluator" "description" : "Evaluation result returned by the evaluator"
* } }
* }, },
* "required" : [ "evaluation_result" ] "required" : [ "evaluation_result" ]
* } }
*/ """;
@Multiline
public static String evaluatorAttributes;
/** private final String evaluatorNextAttributes = """
* { {
* "type" : "object", "type" : "object",
* "description" : "json path assignment", "description" : "json path assignment",
* "title" : "json path assignment", "title" : "json path assignment",
* "properties" : { "properties" : {
* "assignment_type" : { "assignment_type" : {
* "enum" : [ "match_always", "no_match_when_empty", "error_match_when_empty" ], "enum" : [ "match_always", "no_match_when_empty", "error_match_when_empty" ],
* "type" : "string", "type" : "string",
* "description" : "The type of the assigment based on json path evaluation" "description" : "The type of the assignment based on json path evaluation"
* }, },
* "field_name" : { "field_name" : {
* "type" : "string", "type" : "string",
* "description" : "The name of the field in which the non empty result of the json path will be stored" "description" : "The name of the field in which the non empty result of the json path will be stored"
* }, },
* "json_path" : { "json_path" : {
* "type" : "string", "type" : "string",
* "description" : "Json path ", "description" : "Json path ",
* "minItems" : 1 "minItems" : 1
* } }
* }, },
* "required" : [ "assignment_type", "field_name", "json_path" ] "required" : [ "assignment_type", "field_name", "json_path" ]
* } }
*/ """;
@Multiline
public static String evaluatorNextAttributes;
/**
* {
* "rules_version": 1,
* "rules": [
* {
* "rule_name": "test_rule",
* "rule_version": 1,
* "rule_author": "john",
* "rule_description": "Test rule",
* "evaluators": [
* {
* "evaluator_type": "b_first_evaluator",
* "evaluator_attributes": {
* "evaluation_result": "match"
* }
* },
* {
* "evaluator_type": "a_second_evaluator",
* "evaluator_attributes": {
* "assignment_type": "match_always",
* "field_name": "test_field",
* "json_path": "$..a"
* }
* }
* ]
* }
* ]
* }
*/
@Multiline
public static String testingRules;
/** private final String testingRules = """
* { {
* "rule_name": "test_rule", "rules_version": 1,
* "rule_version": 1, "rules": [
* "rule_author": "john", {
* "rule_description": "Test rule", "rule_name": "test_rule",
* "evaluators": [ "rule_version": 1,
* { "rule_author": "john",
* "evaluator_type": "b_first_evaluator", "rule_description": "Test rule",
* "evaluator_attributes": { "evaluators": [
* "evaluation_result": "match" {
* } "evaluator_type": "b_first_evaluator",
* }, "evaluator_attributes": {
* { "evaluation_result": "match"
* "evaluator_type": "a_second_evaluator", }
* "evaluator_attributes": { },
* "assignment_type": "match_always", {
* "field_name": "test_field", "evaluator_type": "a_second_evaluator",
* "json_path": "$..a" "evaluator_attributes": {
* } "assignment_type": "match_always",
* } "field_name": "test_field",
* ] "json_path": "$..a"
* } }
*/ }
@Multiline ]
public static String testingRule; }
]
}
""";
/** private final String testingRule = """
* { {
* "event": { "rule_name": "test_rule",
* "is_test": true "rule_version": 1,
* } "rule_author": "john",
* } "rule_description": "Test rule",
*/ "evaluators": [
@Multiline {
public static String testSpecification; "evaluator_type": "b_first_evaluator",
"evaluator_attributes": {
"evaluation_result": "match"
}
},
{
"evaluator_type": "a_second_evaluator",
"evaluator_attributes": {
"assignment_type": "match_always",
"field_name": "test_field",
"json_path": "$..a"
}
}
]
}
""";
private final String testSpecification = """
{
"event": {
"is_test": true
}
}
""";
private RespondingCompilerImpl compiler; private RespondingCompilerImpl compiler;
private RespondingCompilerImpl.Builder builder; private RespondingCompilerImpl.Builder builder;
@@ -171,7 +161,7 @@ public class RespondingCompilerImplTest {
} }
@Test @Test
public void testgetSchemaOneEvaluatorFactory() throws Exception { public void testGetSchemaOneEvaluatorFactory() throws Exception {
builder.addRespondingEvaluatorFactory(evaluatorFactory); builder.addRespondingEvaluatorFactory(evaluatorFactory);
compiler = builder.build(); compiler = builder.build();
RespondingResult result = compiler.getSchema(); RespondingResult result = compiler.getSchema();
@@ -181,7 +171,7 @@ public class RespondingCompilerImplTest {
} }
@Test @Test
public void testgetSchemaTwoEvaluatorFactories() throws Exception { public void testGetSchemaTwoEvaluatorFactories() throws Exception {
builder.addRespondingEvaluatorFactory(evaluatorFactory); builder.addRespondingEvaluatorFactory(evaluatorFactory);
builder.addRespondingEvaluatorFactory(evaluatorFactoryNext); builder.addRespondingEvaluatorFactory(evaluatorFactoryNext);
compiler = builder.build(); compiler = builder.build();
@@ -205,7 +195,6 @@ public class RespondingCompilerImplTest {
Assert.assertEquals(2, result.getAttributes().getRespondingEvaluatorFactories().size()); Assert.assertEquals(2, result.getAttributes().getRespondingEvaluatorFactories().size());
} }
@Test @Test
public void testCompileRules() throws Exception { public void testCompileRules() throws Exception {
builder.addRespondingEvaluatorFactory(evaluatorFactory); builder.addRespondingEvaluatorFactory(evaluatorFactory);

Some files were not shown because too many files have changed in this diff Show More