scottk/siembol
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/siembol.git synced 2025-11-02 03:18:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Config-editor-rest: compress json when writing to zk (#602)

Browse Source
This commit is contained in:
Celie Valentiny
2022-04-26 10:48:55 +01:00
committed by GitHub
parent f30bdb27a8
commit d150577d48
24 changed files with 119 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
alerting/alerting-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -35,7 +35,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>

4
alerting/alerting-spark/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -23,7 +23,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
<exclusions>
<exclusion>
<artifactId>jackson-databind</artifactId>

4
alerting/alerting-storm/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -51,7 +51,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

2
alerting/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</parent>
<modules>
<module>alerting-core</module>

4
config-editor/config-editor-core/pom.xml
View File

@@ -9,13 +9,13 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>

18
config-editor/config-editor-rest/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</parent>
<dependencyManagement>
<dependencies>
@@ -56,7 +56,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -67,22 +67,22 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-core</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-services</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-sync</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -93,7 +93,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -104,7 +104,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-core</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -115,7 +115,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding-core</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

14
config-editor/config-editor-services/pom.xml
View File

@@ -10,7 +10,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -41,32 +41,32 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-core</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-core</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding-core</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>

8
config-editor/config-editor-sync/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -20,17 +20,17 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-core</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>

2
config-editor/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</parent>
<modules>
<module>config-editor-core</module>

4
deployment/storm-topology-manager/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<dependencyManagement>
@@ -43,7 +43,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

6
enriching/enriching-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -35,12 +35,12 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>

4
enriching/enriching-storm/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -75,7 +75,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-core</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

2
enriching/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</parent>
<modules>
<module>enriching-core</module>

6
parsing/parsing-app/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -39,12 +39,12 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-core</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>

4
parsing/parsing-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -45,7 +45,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>

4
parsing/parsing-storm/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -75,7 +75,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

2
parsing/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</parent>
<modules>
<module>parsing-core</module>

2
pom.xml
View File

@@ -6,7 +6,7 @@
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<name>siembol</name>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
<description>A scalable, advanced security analytics framework based on open-source big data technologies.</description>
<inceptionYear>2019</inceptionYear>
<url>https://siembol.io/</url>

2
responding/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</parent>
<modules>
<module>responding-core</module>

6
responding/responding-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -35,12 +35,12 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.jayway.jsonpath</groupId>

6
responding/responding-stream/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</parent>
<dependencyManagement>
<dependencies>
@@ -51,7 +51,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -62,7 +62,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding-core</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>

2
siembol-common/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.4.8-SNAPSHOT</version>
<version>2.4.9-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>

16
siembol-common/src/main/java/uk/co/gresearch/siembol/common/zookeeper/ZooKeeperConnectorImpl.java
View File

@@ -1,5 +1,9 @@
package uk.co.gresearch.siembol.common.zookeeper;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.recipes.cache.*;
@@ -21,8 +25,11 @@ import static java.nio.charset.StandardCharsets.UTF_8;
public class ZooKeeperConnectorImpl implements ZooKeeperConnector {
private static final int SLEEP_TIME_MS = 100;
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final String EMPTY_GET_DATA_MSG = "Trying to read form empty cache from zk path: %s";
private static final String INIT_TIMEOUT_MSG = "Initialisation of zk path: %s exceeded timeout ";
private static final String NON_JSON_DATA_MSG = "Data set in zk path: {} is not JSON";
private static final ObjectMapper JSON_MAPPER = new ObjectMapper();
private final CuratorFramework client;
private final CuratorCache cache;
@@ -50,7 +57,13 @@ public class ZooKeeperConnectorImpl implements ZooKeeperConnector {
@Override
public void setData(String data) throws Exception {
client.setData().forPath(this.path, data.getBytes(UTF_8));
try {
var json = JSON_MAPPER.readValue(data, JsonNode.class);
client.setData().forPath(this.path, JSON_MAPPER.writeValueAsBytes(json));
} catch (JsonParseException e) {
LOG.warn(NON_JSON_DATA_MSG, this.path);
client.setData().forPath(this.path, data.getBytes(UTF_8));
}
}
@Override
@@ -79,7 +92,6 @@ public class ZooKeeperConnectorImpl implements ZooKeeperConnector {
}
public static class Builder {
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final String WRONG_ATTRIBUTES_LOG_MSG = "Missing ZooKeeper connector attributes, zkServer: {}, " +
"path: {}, baseSleepTimeMs: {}, maxRetries: {}";
private static final String WRONG_ATTRIBUTES_EXCEPTION_MSG = "Missing required parameters to initialise " +

50
siembol-common/src/test/java/uk/co/gresearch/siembol/common/zookeeper/ZooKeeperConnectorTest.java
View File

@@ -12,6 +12,24 @@ import java.util.concurrent.atomic.AtomicReference;
public class ZooKeeperConnectorTest {
private static TestingServer TESTING_SERVER;
private final String testJson = """
{
"rules_version" :1,
"tags" : [ { "tag_name" : "detection_source", "tag_value" : "siembol_alerts" } ],
"rules" : [ {
"rule_name" : "test_rule",
"rule_version" : 1,
"rule_author" : "dummy",
"rule_protection" : {
"max_per_hour" : 100,
"max_per_day" : 10000
}
}]
}
""";
private final String testJsonOneLine = "{\"rules_version\":1,\"tags\":[{\"tag_name\":\"detection_source\",\"tag_value\":\"siembol_alerts\"}],\"rules\":[{\"rule_name\":\"test_rule\",\"rule_version\":1,\"rule_author\":\"dummy\",\"rule_protection\":{\"max_per_hour\":100,\"max_per_day\":10000}}]}";
@BeforeClass
public static void setUp() throws Exception {
TESTING_SERVER = new TestingServer();
@@ -68,6 +86,38 @@ public class ZooKeeperConnectorTest {
Thread.sleep(1000);
}
@Test
public void setJsonDataOK() throws Exception {
final String path = "/c/d/f";
var zooKeeperConnector = new ZooKeeperConnectorImpl.Builder()
.zkServer(TESTING_SERVER.getConnectString())
.path(path)
.initValueIfNotExists("dummy_value")
.build();
zooKeeperConnector.initialise();
zooKeeperConnector.setData(testJson);
Thread.sleep(1000);
Assert.assertEquals(testJsonOneLine, zooKeeperConnector.getData());
zooKeeperConnector.close();
Thread.sleep(1000);
}
@Test
public void setNotJsonDataOK() throws Exception {
final String path = "/c/d/g";
var zooKeeperConnector = new ZooKeeperConnectorImpl.Builder()
.zkServer(TESTING_SERVER.getConnectString())
.path(path)
.initValueIfNotExists("dummy_value")
.build();
zooKeeperConnector.initialise();
zooKeeperConnector.setData("test string");
Thread.sleep(1000);
Assert.assertEquals("test string", zooKeeperConnector.getData());
zooKeeperConnector.close();
Thread.sleep(1000);
}
@Test(expected = IllegalArgumentException.class)
public void builderMissingArguments() throws Exception {
final String path = "/c/d/e";