scottk/siembol
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/siembol.git synced 2025-11-03 03:47:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Siembol release 2.2.0 (#526)

Browse Source
This commit is contained in:
Marian Novotny
2022-02-04 13:37:51 +00:00
committed by GitHub
parent 8d2905ad5b
commit a17ebe742a
28 changed files with 85 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
CHANGELOG.md
View File

@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [2.2.0] - 2022-02-04
- Reworking kafka writing in Storm applications to use async sending in order to improve performance
- Deprecating custom batching in kafka writer and use Storm and Kafka batching instead
- Upgrading Apache Kafka dependency to 3.1.0 and Spring Boot applications to 2.6.3
- Upgrading Angular to 13.2.0 in Siembol UI
- Adding management page into Siembol UI including management links and restarting all Siembol Storm applications
- Adding topic routing parsing and header routing parsing application types
- Various fixes required for Siembol k8s deployment
## [2.1.0] - 2021-11-19
- Supporting autocreation of ZooKeeper nodes in ZooKeeper connector

4
alerting/alerting-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</parent>
<dependencies>
<dependency>
@@ -35,7 +35,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>junit</groupId>

4
alerting/alerting-spark/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</parent>
<dependencies>
<dependency>
@@ -23,7 +23,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
<exclusions>
<exclusion>
<artifactId>jackson-databind</artifactId>

4
alerting/alerting-storm/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</parent>
<dependencies>
<dependency>
@@ -51,7 +51,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

2
alerting/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</parent>
<modules>
<module>alerting-core</module>

4
config-editor/config-editor-core/pom.xml
View File

@@ -9,13 +9,13 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</parent>
<dependencies>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>

18
config-editor/config-editor-rest/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</parent>
<dependencyManagement>
<dependencies>
@@ -56,7 +56,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -67,22 +67,22 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-core</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-services</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-sync</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -93,7 +93,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -104,7 +104,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-core</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -115,7 +115,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding-core</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

14
config-editor/config-editor-services/pom.xml
View File

@@ -10,7 +10,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</parent>
<dependencies>
<dependency>
@@ -41,32 +41,32 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-core</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-core</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding-core</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>junit</groupId>

8
config-editor/config-editor-sync/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</parent>
<dependencies>
<dependency>
@@ -20,17 +20,17 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-core</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
<scope>provided</scope>
</dependency>
<dependency>

2
config-editor/config-editor-ui/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "rule-editor.ui",
"version": "2.2.8-dev",
"version": "2.2.0",
"license": "MIT",
"scripts": {
"ng": "ng",

2
config-editor/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</parent>
<modules>
<module>config-editor-core</module>

4
deployment/storm-topology-manager/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<dependencyManagement>
@@ -43,7 +43,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

1
docs/services/images/header_router_parsing.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 13 KiB

1
docs/services/images/topic_router_parsing.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 12 KiB

1
docs/services/siembol_enrichment_service.md
View File

@@ -67,7 +67,6 @@ Note: you can only enrich from one table per rule. If you want to enrich the sam
- `zk.url` - Zookeeper servers url. Multiple servers are separated by comma
- `zk.path` - Path to a zookeeper node
- `kafka.batch.writer.attributes` - Kafka batch writer attributes for producing output messages
- `batch.size` - The max size of batch used for producing messages
- `producer.properties` - Defines kafka producer properties, see [https://kafka.apache.org/0102/documentation.html#producerconfigs](https://kafka.apache.org/0102/documentation.html#producerconfigs)
- `storm.attributes` - Storm attributes for the enrichment topology
- `bootstrap.servers` - Kafka brokers servers url. Multiple servers are separated by comma

18
docs/services/siembol_parsing_services.md
View File

@@ -107,7 +107,7 @@ Parsers are integrated in a stream application (storm topology) that combines on
- `parsing_app_autho` - The author of the parsing application
- `parsing_app_description`- Description of the parsing application
- `parsing_app_settings` - Parsing application settings
- `parsing_app_type`- The type of the parsing application - `router_parsing` or `single_parser`
- `parsing_app_type`- The type of the parsing application - `single_parser`, `router_parsing`, `topic_routing_parsing` or `header_routing_parsing`
- `input_topics` - The kafka topics for reading messages for parsing
- `error_topic`- The kafka topic for publishing error messages
- `input_parallelism` - The number of parallel executors for reading messages from the input kafka topics
@@ -130,6 +130,21 @@ The application integrates multiple parsers. First, the router parser parses the
- `parsers` - The list of parsers for further parsing
- `routing_field_pattern` - The pattern for selecting the parser
- `parser_properties` - The properties of the selected parser with `parser_name` and `output_topic`
### Topic routing parsing
![topic_routing_parsing](images/topic_router_parsing.svg)
The application integrates multiple parsers and reads logs from multiple topics. The parser is selected based on the topic name on which the log was received.
- `default_parser` - The parser that should be used if no other parsers is selected with `parser_name` and `output_topic`
- `parsers` - The list of parsers for further parsing
- `topic_name` - The name of the topic for selecting the parser
- `parser_properties` - The properties of the selected parser with `parser_name` and `output_topic`
### Header routing parsing
![header_routing_parsing](images/header_router_parsing.svg)
The application integrates multiple parsers and uses a kafka message header for routing. The parser is selected based on the dedicated header value.
- `default_parser` - The parser that should be used if no other parsers is selected with `parser_name` and `output_topic`
- `header_name` - The name of the header used for routing
- `parsers` - The list of parsers for further parsing
- `source_header_value` - The value in the header for selecting the parser
- `parser_properties` - The properties of the selected parser with `parser_name` and `output_topic`
## Admin Config
- `topology.name.prefix` - The prefix that will be used to create a topology name using the application name, by default `parsing`
- `client.id.prefix` - The prefix that will be used to create a kafka producer client id using the application name
@@ -138,7 +153,6 @@ The application integrates multiple parsers. First, the router parser parses the
- `zk.url` - Zookeeper servers url. Multiple servers are separated by a comma
- `zk.path` - Path to a zookeeper node
- `kafka.batch.writer.attributes` - Global settings for the kafka batch writer used if they are not overridden
- `batch.size` - The max size of batch used for producing messages
- `producer.properties` - Defines kafka producer properties, see [https://kafka.apache.org/0102/documentation.html#producerconfigs](https://kafka.apache.org/0102/documentation.html#producerconfigs)
- `storm.attributes` - Global settings for storm attributes used if they are not overridden
- `bootstrap.servers` - Kafka brokers servers url. Multiple servers are separated by a comma

6
enriching/enriching-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</parent>
<dependencies>
<dependency>
@@ -35,12 +35,12 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>junit</groupId>

4
enriching/enriching-storm/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</parent>
<dependencies>
<dependency>
@@ -75,7 +75,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-core</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

2
enriching/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</parent>
<modules>
<module>enriching-core</module>

6
parsing/parsing-app/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</parent>
<dependencies>
<dependency>
@@ -39,12 +39,12 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-core</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>junit</groupId>

4
parsing/parsing-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</parent>
<dependencies>
<dependency>
@@ -45,7 +45,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>

4
parsing/parsing-storm/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</parent>
<dependencies>
<dependency>
@@ -75,7 +75,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

2
parsing/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</parent>
<modules>
<module>parsing-core</module>

2
pom.xml
View File

@@ -6,7 +6,7 @@
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<name>siembol</name>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
<description>A scalable, advanced security analytics framework based on open-source big data technologies.</description>
<inceptionYear>2019</inceptionYear>
<url>https://siembol.io/</url>

2
responding/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</parent>
<modules>
<module>responding-core</module>

6
responding/responding-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</parent>
<dependencies>
<dependency>
@@ -35,12 +35,12 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>com.jayway.jsonpath</groupId>

6
responding/responding-stream/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</parent>
<dependencyManagement>
<dependencies>
@@ -51,7 +51,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -62,7 +62,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding-core</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>

2
siembol-common/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.2.13-SNAPSHOT</version>
<version>2.2.0</version>
</parent>
<dependencies>
<dependency>