Siembol services: supporting optional items (#613)

This commit is contained in:
Marian Novotny
2022-05-03 15:07:34 +01:00
committed by GitHub
parent 5e280ae316
commit d1489d8b3b
37 changed files with 388 additions and 122 deletions

View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -35,7 +35,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>

View File

@@ -79,6 +79,7 @@ public class AlertingRulesCompiler implements AlertingCompiler {
throw new IllegalArgumentException(MISSING_MATCHERS_IN_COMPOSITE_MATCHER);
}
List<Matcher> matchers = matcherDto.getMatchers().stream()
.filter(x -> x.isEnabled())
.map(this::createMatcher)
.collect(Collectors.toList());
return CompositeMatcher.builder()
@@ -120,6 +121,7 @@ public class AlertingRulesCompiler implements AlertingCompiler {
for (RuleDto ruleDto : rulesDto.getRules()) {
List<Matcher> matchers = ruleDto.getMatchers()
.stream()
.filter(x -> x.isEnabled())
.map(this::createMatcher)
.collect(Collectors.toList());

View File

@@ -54,6 +54,7 @@ public class CompositeMatcher implements Matcher {
}
public static class Builder {
private static final String EMPTY_MATCHERS = "Empty matchers in the composite matcher";
private static final String WRONG_ARGUMENTS = "wrong arguments in the composite matcher";
private static final String COMPOSITE_OR_MODIFY_EVENT_MSG = "COMPOSITE_OR matcher includes a matcher " +
"that can modify an event";
@@ -82,9 +83,11 @@ public class CompositeMatcher implements Matcher {
}
public CompositeMatcher build() {
if (negated == null
|| matchers == null || matchers.isEmpty()
|| matcherType == null) {
if (matchers == null || matchers.isEmpty()) {
throw new IllegalArgumentException(EMPTY_MATCHERS);
}
if (negated == null || matcherType == null) {
throw new IllegalArgumentException(WRONG_ARGUMENTS);
}

View File

@@ -43,7 +43,7 @@ public class Rule extends AbstractRule {
}
public static abstract class Builder<T extends Rule> extends AbstractRule.Builder<T>{
protected static final String MISSING_MATCHERS = "Missing matchers in alerts rule builder";
protected static final String MISSING_MATCHERS = "Empty matchers in a rule";
protected List<Matcher> matchers;
protected EnumSet<RuleFlags> flags = EnumSet.noneOf(RuleFlags.class);

View File

@@ -7,6 +7,13 @@ import java.util.List;
@Attributes(title = "matcher", description = "Matcher for matching fields")
public class MatcherDto {
@Attributes(description = "The matcher is enabled", required = false)
@JsonProperty("is_enabled")
private boolean enabled = true;
@Attributes(description = "Description of the matcher", required = false)
@JsonProperty("description")
private String description;
@JsonProperty("matcher_type")
@Attributes(required = true, description = "Type of matcher, either Regex match or list of strings " +
"(newline delimited) or a composite matcher composing several matchers")
@@ -100,5 +107,21 @@ public class MatcherDto {
public void setEndsWith(Boolean endsWith) {
this.endsWith = endsWith;
}
public boolean isEnabled() {
return enabled;
}
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
}

View File

@@ -23,39 +23,47 @@ public class AlertingRulesCompilerTest {
"rule_description": "Test rule - is_alert is equal to true",
"source_type" : "*",
"matchers" : [ {
"matcher_type" : "REGEX_MATCH",
"is_negated" : false,
"field" : "is_alert",
"data" : "(?i)true" },
"is_enabled" : true,
"matcher_type" : "REGEX_MATCH",
"is_negated" : false,
"field" : "is_alert",
"data" : "(?i)true"
},
{
"is_enabled" : true,
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "source_type",
"data": "(?<sensor>.*)"
}
]
]
}]
}
""";
private final String alertRule = """
{
"rule_name" : "siembol_alert_generic",
"rule_version" : 1,
"rule_author" : "dummy",
"rule_description": "Test rule - is_alert is equal to true",
"source_type" : "*",
"matchers" : [ {
"matcher_type" : "REGEX_MATCH",
"is_negated" : false,
"field" : "is_alert",
"data" : "(?i)true" },
{
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "source_type",
"data": "(?<sensor>.*)"
}]
"rule_name": "siembol_alert_generic",
"rule_version": 1,
"rule_author": "dummy",
"rule_description": "Test rule - is_alert is equal to true",
"source_type": "*",
"matchers": [
{
"is_enabled": true,
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "source_type",
"data": "(?<sensor>.*)"
},
{
"is_enabled": true,
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_alert",
"data": "(?i)true"
}
]
}
""";
@@ -87,6 +95,7 @@ public class AlertingRulesCompilerTest {
"source_type": "*",
"matchers": [
{
"is_enabled" : true,
"matcher_type": "CONTAINS",
"is_negated": false,
"field": "is_alert",
@@ -96,32 +105,38 @@ public class AlertingRulesCompilerTest {
"ends_with" : true
},
{
"is_enabled" : true,
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "source_type",
"data": "(?<sensor>.*)"
},
{
"is_enabled" : true,
"matcher_type": "COMPOSITE_OR",
"is_negated": false,
"matchers": [
{
"is_enabled" : true,
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_secret",
"data": "(?i)true"
},
{
"is_enabled" : true,
"matcher_type": "COMPOSITE_AND",
"is_negated": false,
"matchers": [
{
"is_enabled":true ,
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_public",
"data": "(?i)true"
},
{
"is_enabled":true,
"matcher_type": "IS_IN_SET",
"is_negated": false,
"field": "is_detected",
@@ -179,7 +194,19 @@ public class AlertingRulesCompilerTest {
}
@Test
public void validationRuleOK() {
public void validationRuleDisabledFirstMatcherOk() {
AlertingResult ret = compiler.validateRule(alertRule.replaceFirst("\"is_enabled\": true",
"\"is_enabled\" : false"));
Assert.assertEquals(AlertingResult.StatusCode.OK, ret.getStatusCode());
AlertingResult matchResult = ret.getAttributes().getEngine().evaluate(goodAlert);
Assert.assertEquals(AlertingResult.StatusCode.OK, matchResult.getStatusCode());
Assert.assertEquals(EvaluationResult.MATCH, matchResult.getAttributes().getEvaluationResult());
Assert.assertEquals(1, matchResult.getAttributes().getOutputEvents().size());
Assert.assertNull(matchResult.getAttributes().getOutputEvents().get(0).get("sensor"));
}
@Test
public void validationRuleOk() {
AlertingResult ret = compiler.validateRule(alertRule);
Assert.assertEquals(AlertingResult.StatusCode.OK, ret.getStatusCode());
AlertingResult matchResult = ret.getAttributes().getEngine().evaluate(goodAlert);
@@ -196,6 +223,14 @@ public class AlertingRulesCompilerTest {
Assert.assertTrue(ret.getAttributes().getException().contains("PatternSyntaxException"));
}
@Test
public void validationRuleDisabledMatchers() {
AlertingResult ret = compiler.validateRule(alertRule.replace("\"is_enabled\": true",
"\"is_enabled\" : false"));
Assert.assertEquals(AlertingResult.StatusCode.ERROR, ret.getStatusCode());
Assert.assertTrue(ret.getAttributes().getException().contains("Empty matchers in a rule"));
}
@Test
public void validationRuleInvalidJson() {
AlertingResult ret = compiler.validateRule("INVALID JSON");
@@ -248,6 +283,14 @@ public class AlertingRulesCompilerTest {
Assert.assertEquals(AlertingResult.StatusCode.ERROR, ret.getStatusCode());
}
@Test
public void testingRuleDisabledMatchers() {
AlertingResult ret = compiler.testRule(alertRule.replace("\"is_enabled\": true",
"\"is_enabled\" : false"), goodAlert);
Assert.assertEquals(AlertingResult.StatusCode.ERROR, ret.getStatusCode());
Assert.assertTrue(ret.getAttributes().getException().contains("Empty matchers in a rule"));
}
@Test
public void testRulesOK() {
AlertingResult ret = compiler.testRules(alertRules, goodAlert);
@@ -291,6 +334,14 @@ public class AlertingRulesCompilerTest {
Assert.assertEquals(AlertingResult.StatusCode.ERROR, ret.getStatusCode());
}
@Test
public void validationRuleWithCompositeDisabled() {
AlertingResult ret = compiler.validateRule(ruleWithCompositeMatchers.replace("\"is_enabled\":true",
"\"is_enabled\" : false"));
Assert.assertEquals(AlertingResult.StatusCode.ERROR, ret.getStatusCode());
Assert.assertTrue(ret.getAttributes().getException().contains("Empty matchers in the composite matcher"));
}
@Test
public void compileRulesWithCompositeMatcher() {
String rules = "{ \"rules_version\" : 1, \"tags\" : [], \"rules\" : [" + ruleWithCompositeMatchers + "]}";

View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -23,7 +23,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
<exclusions>
<exclusion>
<artifactId>jackson-databind</artifactId>

View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -51,7 +51,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</parent>
<modules>
<module>alerting-core</module>

View File

@@ -9,13 +9,13 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>

View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</parent>
<dependencyManagement>
<dependencies>
@@ -56,7 +56,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -67,22 +67,22 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-core</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-services</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-sync</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -93,7 +93,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -104,7 +104,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-core</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -115,7 +115,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding-core</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -126,7 +126,7 @@
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-core</artifactId>
<version>3.4.17</version>
<version>${reactor_core_version}</version>
</dependency>
<dependency>
<groupId>org.springframework.plugin</groupId>
@@ -151,7 +151,7 @@
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
<version>1.8.5</version>
<version>${io_micrometer_version}</version>
</dependency>
<dependency>
<groupId>commons-beanutils</groupId>
@@ -164,6 +164,12 @@
<version>${junit_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>net.bytebuddy</groupId>
<artifactId>byte-buddy</artifactId>
<version>${byte_buddy_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>

View File

@@ -10,7 +10,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -41,32 +41,32 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-core</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-core</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding-core</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>

View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -20,17 +20,17 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-core</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>

View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</parent>
<modules>
<module>config-editor-core</module>

View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<dependencyManagement>
@@ -43,7 +43,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -81,7 +81,7 @@
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-core</artifactId>
<version>3.4.17</version>
<version>${reactor_core_version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
@@ -113,10 +113,16 @@
<artifactId>snakeyaml</artifactId>
<version>1.30</version>
</dependency>
<dependency>
<groupId>net.bytebuddy</groupId>
<artifactId>byte-buddy</artifactId>
<version>${byte_buddy_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
<version>1.8.5</version>
<version>${io_micrometer_version}</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>

View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -35,12 +35,12 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>

View File

@@ -87,6 +87,7 @@ public class EnrichmentCompilerImpl implements EnrichmentCompiler {
}
List<Matcher> matchers = ruleDto.getMatchers().stream()
.filter(x -> x.isEnabled())
.map(x -> createMatcher(x))
.collect(Collectors.toList());

View File

@@ -5,6 +5,13 @@ import com.github.reinert.jjschema.Attributes;
@Attributes(title = "matcher", description = "Matcher for matching fields in enrichment rules")
public class MatcherDto {
@Attributes(description = "The matcher is enabled", required = false)
@JsonProperty("is_enabled")
private boolean enabled = true;
@Attributes(description = "Description of the matcher", required = false)
@JsonProperty("description")
private String description;
@JsonProperty("matcher_type")
@Attributes(required = true,
description = "Type of matcher, either Regex match or list of strings (newline delimited)")
@@ -63,4 +70,20 @@ public class MatcherDto {
public void setType(MatcherTypeDto type) {
this.type = type;
}
public boolean isEnabled() {
return enabled;
}
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
}

View File

@@ -34,6 +34,7 @@ public class EnrichmentCompilerTest {
"source_type": "*",
"matchers": [
{
"is_enabled" : true,
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_alert",
@@ -73,6 +74,7 @@ public class EnrichmentCompilerTest {
"source_type": "*",
"matchers": [
{
"is_enabled" : true,
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_alert",
@@ -107,6 +109,7 @@ public class EnrichmentCompilerTest {
"source_type": "*",
"matchers": [
{
"is_enabled" : true,
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_alert",
@@ -141,6 +144,7 @@ public class EnrichmentCompilerTest {
"source_type": "*",
"matchers": [
{
"is_enabled" : true,
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_alert",
@@ -165,10 +169,11 @@ public class EnrichmentCompilerTest {
"source_type": "*",
"matchers": [
{
"is_enabled" : true,
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "is_alert",
"data": "true"
"data": "(?i)true"
}
],
"table_mapping": {
@@ -257,7 +262,7 @@ public class EnrichmentCompilerTest {
}
@Test
public void compileeRulesInvalidJson() {
public void compileRulesInvalidJson() {
EnrichmentResult result = enrichmentCompiler.compile("INVALID");
Assert.assertEquals(ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("INVALID"));
@@ -287,11 +292,19 @@ public class EnrichmentCompilerTest {
@Test
public void validateRuleInvalidRegex() {
EnrichmentResult result = enrichmentCompiler.validateConfiguration(
testRule.replaceAll( "true", "(?<"));
testRule.replace( "(?i)true", "["));
Assert.assertEquals(ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("PatternSyntaxException"));
}
@Test
public void validateRuleDisabledMatchers() {
EnrichmentResult result = enrichmentCompiler.validateConfiguration(
testRule.replace( "\"is_enabled\" : true", "\"is_enabled\" : false"));
Assert.assertEquals(ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("Empty matchers in a rule"));
}
@Test
public void validateRuleMissingRequiredFields() {
EnrichmentResult result = enrichmentCompiler.validateConfiguration(testRule.replace("table_mapping",
@@ -381,8 +394,17 @@ public class EnrichmentCompilerTest {
@Test
public void testingRuleInvalidRegex() {
EnrichmentResult result = enrichmentCompiler.testConfiguration(
testRule.replaceAll( "true", "(?<"), testSpecification);
testRule.replace( "(?i)true", "(?<"), testSpecification);
Assert.assertEquals(ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("PatternSyntaxException"));
}
@Test
public void testingRuleDisabledMatchers() {
EnrichmentResult result = enrichmentCompiler.testConfiguration(
testRule.replace( "\"is_enabled\" : true", "\"is_enabled\" : false"),
testSpecification);
Assert.assertEquals(ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("Empty matchers in a rule"));
}
}

View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -75,7 +75,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-core</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</parent>
<modules>
<module>enriching-core</module>

View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -39,12 +39,12 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-core</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>

View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -45,7 +45,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>

View File

@@ -25,6 +25,7 @@ import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
import static java.util.stream.Collectors.toCollection;
import static uk.co.gresearch.siembol.parsers.model.PreProcessingFunctionDto.STRING_REPLACE;
public class ParserFactoryImpl implements ParserFactory {
@@ -398,12 +399,12 @@ public class ParserFactoryImpl implements ParserFactory {
return Optional.empty();
}
ArrayList<ParserExtractor> ret = new ArrayList<>();
for (ParserExtractorDto extractor : parserConfig.getParserExtractors()) {
ret.add(createParserExtractor(extractor));
}
return ret.isEmpty() ? Optional.empty() : Optional.of(ret);
ArrayList<ParserExtractor> ret = parserConfig.getParserExtractors().stream()
.filter(x -> x.isEnabled())
.map(x -> createParserExtractor(x))
.collect(toCollection(ArrayList::new));
return ret.isEmpty() ? Optional.empty() : Optional.of(ret);
}
private String wrapParserConfigToParsersConfig(String configStr) throws IOException {
@@ -420,13 +421,12 @@ public class ParserFactoryImpl implements ParserFactory {
return Optional.empty();
}
ArrayList<Transformation> ret = new ArrayList<>();
for (TransformationDto transformation : parserConfig.getParserTransformations()) {
ret.add(transformationFactory.create(transformation));
}
ArrayList<Transformation> ret = parserConfig.getParserTransformations().stream()
.filter(x -> x.isEnabled())
.map(x -> transformationFactory.create(x))
.collect(toCollection(ArrayList::new));
return ret.isEmpty() ? Optional.empty() : Optional.of(ret);
}
private SiembolParser createSyslogParser(SyslogParserConfigDto syslogConfig,

View File

@@ -8,6 +8,13 @@ import java.util.List;
@Attributes(title = "parser extractor", description = "Parser extractor specification")
public class ParserExtractorDto {
@Attributes(description = "The extractor is enabled", required = false)
@JsonProperty("is_enabled")
private boolean enabled = true;
@Attributes(description = "Description of the extractor", required = false)
@JsonProperty("description")
private String description;
@Attributes(required = true, description = "The name of the extractor")
private String name;
@@ -77,4 +84,20 @@ public class ParserExtractorDto {
public void setPreProcessingFunction(PreProcessingFunctionDto preProcessingFunction) {
this.preProcessingFunction = preProcessingFunction;
}
public boolean isEnabled() {
return enabled;
}
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
}

View File

@@ -4,6 +4,13 @@ import com.github.reinert.jjschema.Attributes;
@Attributes(title = "transformation", description = "The specification of transformation")
public class TransformationDto {
@Attributes(description = "The transformation is enabled", required = false)
@JsonProperty("is_enabled")
private boolean enabled = true;
@Attributes(description = "Description of the transformation", required = false)
@JsonProperty("description")
private String description;
@JsonProperty("transformation_type")
@Attributes(required = true, description = "The type of the transformation")
private TransformationTypeDto type;
@@ -25,4 +32,20 @@ public class TransformationDto {
public void setAttributes(TransformationAttributesDto attributes) {
this.attributes = attributes;
}
public boolean isEnabled() {
return enabled;
}
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
}

View File

@@ -23,6 +23,7 @@ public class ParserFactoryImplTest {
},
"parser_extractors": [
{
"is_enabled": true,
"extractor_type": "json_extractor",
"name": "test",
"field": "original_string",
@@ -41,6 +42,7 @@ public class ParserFactoryImplTest {
}
},
{
"is_enabled" : true,
"extractor_type": "pattern_extractor",
"name": "pattern",
"field": "dummy_field",
@@ -58,6 +60,7 @@ public class ParserFactoryImplTest {
],
"transformations": [
{
"is_enabled":true,
"transformation_type": "field_name_string_replace",
"attributes": {
"string_replace_target": " ",
@@ -72,14 +75,14 @@ public class ParserFactoryImplTest {
{"timestamp":"2019-03-27 18:52:02.732 Z", "test field" : true, "test_field1" : " message ", "test_field2" : " message ", "empty" : ""}""";
@Test
public void testGetSchema() {
public void getSchema() {
ParserFactoryResult schemaResult = factory.getSchema();
Assert.assertSame(ParserFactoryResult.StatusCode.OK, schemaResult.getStatusCode());
Assert.assertFalse(schemaResult.getAttributes().getJsonSchema().isEmpty());
}
@Test
public void testGoodCreate() {
public void createOk() {
ParserFactoryResult result = factory.create(simpleGenericParser);
Assert.assertSame(ParserFactoryResult.StatusCode.OK, result.getStatusCode());
Assert.assertNotNull(result.getAttributes().getSiembolParser());
@@ -89,23 +92,48 @@ public class ParserFactoryImplTest {
Assert.assertEquals(true, parsed.get(0).get("test_field"));
Assert.assertNull(parsed.get(0).get("source_type"));
Assert.assertNull(parsed.get(0).get("empty"));
}
@Test
public void testInvalidCreate() {
public void createDisabledExtractor() {
ParserFactoryResult result = factory.create(simpleGenericParser.replace("\"is_enabled\": true",
"\"is_enabled\": false"));
Assert.assertSame(ParserFactoryResult.StatusCode.OK, result.getStatusCode());
Assert.assertNotNull(result.getAttributes().getSiembolParser());
List<Map<String, Object>> parsed = result.getAttributes().getSiembolParser().parse(message.getBytes());
Assert.assertNotEquals(1553712722732L, parsed.get(0).get("timestamp"));
Assert.assertNull(parsed.get(0).get("test_field"));
}
@Test
public void createDisabledTransformation() {
ParserFactoryResult result = factory.create(simpleGenericParser.replace("\"is_enabled\":true",
"\"is_enabled\": false"));
Assert.assertSame(ParserFactoryResult.StatusCode.OK, result.getStatusCode());
Assert.assertNotNull(result.getAttributes().getSiembolParser());
List<Map<String, Object>> parsed = result.getAttributes().getSiembolParser().parse(message.getBytes());
Assert.assertEquals(1553712722732L, parsed.get(0).get("timestamp"));
Assert.assertEquals(true, parsed.get(0).get("test field"));
}
@Test
public void invalidCreate() {
ParserFactoryResult result = factory.create("INVALID");
Assert.assertSame(ParserFactoryResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertNotNull(result.getAttributes().getMessage());
}
@Test
public void testValidationGood() {
public void validationOk() {
ParserFactoryResult result = factory.validateConfiguration(simpleGenericParser);
Assert.assertSame(ParserFactoryResult.StatusCode.OK, result.getStatusCode());
}
@Test
public void testValidationInvalidPatternExtractor() {
public void validationInvalidPatternExtractor() {
ParserFactoryResult result = factory.validateConfiguration(simpleGenericParser.replace("<test_match>",
"<test_match"));
Assert.assertSame(ParserFactoryResult.StatusCode.ERROR, result.getStatusCode());
@@ -113,7 +141,7 @@ public class ParserFactoryImplTest {
}
@Test
public void testTestingGood() {
public void testingOk() {
ParserFactoryResult result = factory.test(simpleGenericParser, null, message.getBytes());
Assert.assertSame(result.getStatusCode(), ParserFactoryResult.StatusCode.OK);
List<Map<String, Object>> parsed = result.getAttributes().getParserResult().getParsedMessages();
@@ -123,7 +151,7 @@ public class ParserFactoryImplTest {
}
@Test
public void testTestingInvalidPatternExtractor() {
public void testingInvalidPatternExtractor() {
ParserFactoryResult result = factory.test(simpleGenericParser.replace("<test_match>",
"<test_match"), null, message.getBytes());
Assert.assertSame(ParserFactoryResult.StatusCode.ERROR, result.getStatusCode());

View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -75,7 +75,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</parent>
<modules>
<module>parsing-core</module>

View File

@@ -6,7 +6,7 @@
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<name>siembol</name>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
<description>A scalable, advanced security analytics framework based on open-source big data technologies.</description>
<inceptionYear>2019</inceptionYear>
<url>https://siembol.io/</url>
@@ -41,7 +41,7 @@
<maven_compiler_version>3.10.1</maven_compiler_version>
<jackson_version>2.13.2</jackson_version>
<junit_version>4.13.2</junit_version>
<mockito_version>4.4.0</mockito_version>
<mockito_version>4.5.1</mockito_version>
<shade_version>3.3.0</shade_version>
<curator_version>5.2.0</curator_version>
<curator_test_version>2.13.0</curator_test_version>
@@ -65,6 +65,9 @@
<commons_io_version>2.11.0</commons_io_version>
<gson_version>2.9.0</gson_version>
<kubernetes_client_version>5.12.2</kubernetes_client_version>
<reactor_core_version>3.4.17</reactor_core_version>
<io_micrometer_version>1.8.5</io_micrometer_version>
<byte_buddy_version>1.12.10</byte_buddy_version>
</properties>
<modules>
<module>siembol-common</module>

View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</parent>
<modules>
<module>responding-core</module>

View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -35,12 +35,12 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.jayway.jsonpath</groupId>

View File

@@ -15,13 +15,19 @@ import java.util.List;
import java.util.stream.Collectors;
public class MatchingEvaluator implements Evaluable {
private static final String EMPTY_MATCHERS = "Empty matchers in matching evaluator";
private final List<BasicMatcher> matchers;
private final MatchingEvaluatorResultDto matchingResult;
public MatchingEvaluator(MatchingEvaluatorAttributesDto attributesDto) {
matchers = attributesDto.getMatchers().stream()
.filter(x -> x.isEnabled())
.map(this::createMatcher)
.collect(Collectors.toList());
if (matchers.isEmpty()) {
throw new IllegalArgumentException(EMPTY_MATCHERS);
}
matchingResult = attributesDto.getEvaluationResult();
}

View File

@@ -5,6 +5,13 @@ import com.github.reinert.jjschema.Attributes;
@Attributes(title = "matcher", description = "Matcher for matching fields in response rules")
public class MatcherDto {
@Attributes(description = "The matcher is enabled", required = false)
@JsonProperty("is_enabled")
private boolean enabled = true;
@Attributes(description = "Description of the matcher", required = false)
@JsonProperty("description")
private String description;
@JsonProperty("matcher_type")
@Attributes(required = true,
description = "Type of matcher, either Regex match or list of strings (newline delimited)")
@@ -63,4 +70,20 @@ public class MatcherDto {
public void setType(MatcherTypeDto type) {
this.type = type;
}
public boolean isEnabled() {
return enabled;
}
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
}

View File

@@ -12,12 +12,14 @@ public class MatchingEvaluatorFactoryTest {
"evaluation_result": "match",
"matchers": [
{
"is_enabled" : true,
"matcher_type": "IS_IN_SET",
"is_negated": false,
"field": "is_alert",
"data": "true"
},
{
"is_enabled" : true,
"matcher_type": "REGEX_MATCH",
"is_negated": false,
"field": "to_copy",
@@ -35,7 +37,7 @@ public class MatchingEvaluatorFactoryTest {
}
@Test
public void testGetType() {
public void getType() {
RespondingResult result = factory.getType();
Assert.assertEquals(RespondingResult.StatusCode.OK, result.getStatusCode());
Assert.assertNotNull(result.getAttributes());
@@ -44,7 +46,7 @@ public class MatchingEvaluatorFactoryTest {
}
@Test
public void testGetAttributesJsonSchema() {
public void getAttributesJsonSchema() {
RespondingResult result = factory.getAttributesJsonSchema();
Assert.assertEquals(RespondingResult.StatusCode.OK, result.getStatusCode());
Assert.assertNotNull(result.getAttributes());
@@ -52,7 +54,7 @@ public class MatchingEvaluatorFactoryTest {
}
@Test
public void testCreateInstance() {
public void createInstance() {
RespondingResult result = factory.createInstance(attributes);
Assert.assertEquals(RespondingResult.StatusCode.OK, result.getStatusCode());
Assert.assertNotNull(result.getAttributes());
@@ -60,19 +62,34 @@ public class MatchingEvaluatorFactoryTest {
}
@Test
public void testValidateAttributesOk() {
public void validateAttributesOk() {
RespondingResult result = factory.validateAttributes(attributes);
Assert.assertEquals(RespondingResult.StatusCode.OK, result.getStatusCode());
}
@Test
public void testValidateAttributesInvalidJson() {
public void validateAttributesInvalidJson() {
RespondingResult result = factory.validateAttributes("INVALID");
Assert.assertEquals(RespondingResult.StatusCode.ERROR, result.getStatusCode());
}
@Test
public void testValidateAttributesInvalid() {
public void validateAttributesInvalidRegex() {
RespondingResult result = factory.validateAttributes(attributes.replace("<new_field>", "["));
Assert.assertEquals(RespondingResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("PatternSyntaxException"));
}
@Test
public void validateAttributesDisabledMatchers() {
RespondingResult result = factory.validateAttributes(attributes.replace("\"is_enabled\" : true",
"\"is_enabled\" : false"));
Assert.assertEquals(RespondingResult.StatusCode.ERROR, result.getStatusCode());
Assert.assertTrue(result.getAttributes().getMessage().contains("Empty matchers in matching evaluator"));
}
@Test
public void validateAttributesInvalid() {
RespondingResult result = factory.validateAttributes(
attributes.replace("\"match\"", "\"unsupported\""));
Assert.assertEquals(RespondingResult.StatusCode.ERROR, result.getStatusCode());

View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</parent>
<dependencyManagement>
<dependencies>
@@ -51,7 +51,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -62,7 +62,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding-core</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
@@ -92,7 +92,7 @@
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-core</artifactId>
<version>3.4.17</version>
<version>${reactor_core_version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
@@ -137,13 +137,7 @@
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
<version>1.8.5</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>${mockito_version}</version>
<scope>test</scope>
<version>${io_micrometer_version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
@@ -156,6 +150,18 @@
<artifactId>guava</artifactId>
<version>${guava_version}</version>
</dependency>
<dependency>
<groupId>net.bytebuddy</groupId>
<artifactId>byte-buddy</artifactId>
<version>${byte_buddy_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>${mockito_version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-streams-test-utils</artifactId>

View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.5.0-SNAPSHOT</version>
<version>2.5.1-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -187,7 +187,7 @@
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-core</artifactId>
<version>1.8.5</version>
<version>${io_micrometer_version}</version>
<scope>provided</scope>
</dependency>
</dependencies>