scottk/siembol
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/siembol.git synced 2025-12-02 02:03:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

moving num workers into parsing application (#689)

Browse Source 
fixing header reoutingapp with null headers
This commit is contained in:
Marian Novotny
2022-06-21 12:33:56 +01:00
committed by GitHub
parent 71830b8c43
commit 6111335983
30 changed files with 113 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
alerting/alerting-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -35,7 +35,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>

4
alerting/alerting-spark/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -23,7 +23,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
<exclusions>
<exclusion>
<artifactId>jackson-databind</artifactId>

4
alerting/alerting-storm/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -51,7 +51,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

2
alerting/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</parent>
<modules>
<module>alerting-core</module>

4
config-editor/config-editor-core/pom.xml
View File

@@ -9,13 +9,13 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>

18
config-editor/config-editor-rest/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</parent>
<dependencyManagement>
<dependencies>
@@ -56,7 +56,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -67,22 +67,22 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-core</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-services</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-sync</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -93,7 +93,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -104,7 +104,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-core</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -115,7 +115,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding-core</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

14
config-editor/config-editor-services/pom.xml
View File

@@ -10,7 +10,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -41,32 +41,32 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-core</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-core</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding-core</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>

8
config-editor/config-editor-sync/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -20,17 +20,17 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>config-editor-core</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>

2
config-editor/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</parent>
<modules>
<module>config-editor-core</module>

4
deployment/siembol-monitoring/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<dependencyManagement>
@@ -37,7 +37,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

4
deployment/storm-topology-manager/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<dependencyManagement>
@@ -43,7 +43,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

7
docs/services/siembol_parsing_services.md
View File

@@ -120,9 +120,10 @@ Parsers are integrated in a stream application (storm topology) that combines on
- `parsing_app_type`- The type of the parsing application - `single_parser`, `router_parsing`, `topic_routing_parsing` or `header_routing_parsing`
- `input_topics` - The kafka topics for reading messages for parsing
- `error_topic`- The kafka topic for publishing error messages
- `input_parallelism` - The number of parallel executors for reading messages from the input kafka topics
- `parsing_parallelism` - The number of parallel executors for parsing messages
- `output_parallelism` - The number of parallel executors for publishing parsed messages to kafka
- `num_workers` - The number of workers for the parsing application
- `input_parallelism` - The number of parallel executors per worker for reading messages from the input kafka topics
- `parsing_parallelism` - The number of parallel executors per worker for parsing messages
- `output_parallelism` - The number of parallel executors per worker for publishing parsed messages to kafka
- `parse_metadata` - Parsing json metadata from input key records using `metadata_prefix` added to metadata field names, by default `metadata_`
- `max_num_fields` - Maximum number of fields after parsing the message
- `max_field_size` - Maximum field size after parsing the message in bytes

6
enriching/enriching-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -35,12 +35,12 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>

4
enriching/enriching-storm/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -75,7 +75,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>enriching-core</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

2
enriching/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</parent>
<modules>
<module>enriching-core</module>

6
parsing/parsing-app/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -39,12 +39,12 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-core</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>

9
parsing/parsing-app/src/main/java/uk/co/gresearch/siembol/parsers/application/factory/ParsingApplicationFactoryAttributes.java
View File

@@ -9,6 +9,7 @@ public class ParsingApplicationFactoryAttributes {
private String jsonSchema;
private String name;
private String applicationParserSpecification;
private Integer numWorkers;
private Integer inputParallelism;
private Integer outputParallelism;
private Integer parsingParallelism;
@@ -105,4 +106,12 @@ public class ParsingApplicationFactoryAttributes {
public void setApplicationType(ParsingApplicationTypeDto applicationType) {
this.applicationType = applicationType;
}
public Integer getNumWorkers() {
return numWorkers;
}
public void setNumWorkers(Integer numWorkers) {
this.numWorkers = numWorkers;
}
}

1
parsing/parsing-app/src/main/java/uk/co/gresearch/siembol/parsers/application/factory/ParsingApplicationFactoryImpl.java
View File

@@ -330,6 +330,7 @@ public class ParsingApplicationFactoryImpl implements ParsingApplicationFactory
attributes.setSourceHeaderName(
application.getParsingSettingsDto().getHeaderRoutingParserDto().getHeaderName());
}
attributes.setNumWorkers(application.getParsingApplicationSettingsDto().getNumWorkers());
}
private String getParserFromMap(String parserName, Map<String, String> parserMap) {

17
parsing/parsing-app/src/main/java/uk/co/gresearch/siembol/parsers/application/model/ParsingApplicationSettingsDto.java
View File

@@ -19,18 +19,21 @@ public class ParsingApplicationSettingsDto {
@Attributes(description = "The kafka topic for publishing error messages", required = true)
private String errorTopic;
@JsonProperty("num_workers")
@Attributes(description = "The number of workers for the parsing application", minimum = 1, required = true)
private Integer numWorkers = 1;
@JsonProperty("input_parallelism")
@Attributes(description = "The number of parallel executors for reading messages from the input kafka topics",
@Attributes(description = "The number of parallel executors per worker for reading messages from the input topics",
required = true, minimum = 1)
private Integer inputParallelism;
@JsonProperty("parsing_parallelism")
@Attributes(description = "The number of parallel executors for parsing messages",
@Attributes(description = "The number of parallel executors per worker for parsing messages",
required = true, minimum = 1)
private Integer parsingParallelism;
@JsonProperty("output_parallelism")
@Attributes(description = "The number of parallel executors for publishing to kafka",
@Attributes(description = "The number of parallel executors per worker for publishing to kafka",
required = true, minimum = 1)
private Integer outputParallelism;
@@ -141,4 +144,12 @@ public class ParsingApplicationSettingsDto {
public void setOriginalStringTopic(String originalStringTopic) {
this.originalStringTopic = originalStringTopic;
}
public Integer getNumWorkers() {
return numWorkers;
}
public void setNumWorkers(Integer numWorkers) {
this.numWorkers = numWorkers;
}
}

8
parsing/parsing-app/src/test/java/uk/co/gresearch/siembol/parsers/application/factory/ParsingApplicationFactoryImplTest.java
View File

@@ -24,6 +24,7 @@ public class ParsingApplicationFactoryImplTest {
"max_num_fields" : 100,
"max_field_size" : 40000,
"original_string_topic" : "truncated",
"num_workers" : 3,
"input_parallelism": 1,
"parsing_parallelism": 2,
"output_parallelism": 3,
@@ -49,6 +50,7 @@ public class ParsingApplicationFactoryImplTest {
"secret"
],
"error_topic": "error",
"num_workers" : 3,
"input_parallelism": 1,
"parsing_parallelism": 2,
"output_parallelism": 3,
@@ -96,6 +98,7 @@ public class ParsingApplicationFactoryImplTest {
"public"
],
"error_topic": "error",
"num_workers" : 3,
"input_parallelism": 1,
"parsing_parallelism": 2,
"output_parallelism": 3,
@@ -144,6 +147,7 @@ public class ParsingApplicationFactoryImplTest {
"public"
],
"error_topic": "error",
"num_workers" : 3,
"input_parallelism": 1,
"parsing_parallelism": 2,
"output_parallelism": 3,
@@ -260,6 +264,7 @@ public class ParsingApplicationFactoryImplTest {
ParsingApplicationFactoryResult result = factory.create(simpleSingleApplicationParser, testParsersConfigs);
Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.OK, result.getStatusCode());
Assert.assertEquals("test", result.getAttributes().getName());
Assert.assertEquals(3, result.getAttributes().getNumWorkers().intValue());
Assert.assertEquals(1, result.getAttributes().getInputParallelism().intValue());
Assert.assertEquals(2, result.getAttributes().getParsingParallelism().intValue());
Assert.assertEquals(3, result.getAttributes().getOutputParallelism().intValue());
@@ -312,6 +317,7 @@ public class ParsingApplicationFactoryImplTest {
ParsingApplicationFactoryResult result = factory.create(simpleRoutingApplicationParser, testParsersConfigs);
Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.OK, result.getStatusCode());
Assert.assertEquals("test", result.getAttributes().getName());
Assert.assertEquals(3, result.getAttributes().getNumWorkers().intValue());
Assert.assertEquals(1, result.getAttributes().getInputParallelism().intValue());
Assert.assertEquals(2, result.getAttributes().getParsingParallelism().intValue());
Assert.assertEquals(3, result.getAttributes().getOutputParallelism().intValue());
@@ -397,6 +403,7 @@ public class ParsingApplicationFactoryImplTest {
ParsingApplicationFactoryResult result = factory.create(headerRoutingApplicationParser, testParsersConfigs);
Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.OK, result.getStatusCode());
Assert.assertEquals("test", result.getAttributes().getName());
Assert.assertEquals(3, result.getAttributes().getNumWorkers().intValue());
Assert.assertEquals(1, result.getAttributes().getInputParallelism().intValue());
Assert.assertEquals(2, result.getAttributes().getParsingParallelism().intValue());
Assert.assertEquals(3, result.getAttributes().getOutputParallelism().intValue());
@@ -452,6 +459,7 @@ public class ParsingApplicationFactoryImplTest {
ParsingApplicationFactoryResult result = factory.create(topicRoutingApplicationParser, testParsersConfigs);
Assert.assertSame(ParsingApplicationFactoryResult.StatusCode.OK, result.getStatusCode());
Assert.assertEquals("test", result.getAttributes().getName());
Assert.assertEquals(3, result.getAttributes().getNumWorkers().intValue());
Assert.assertEquals(1, result.getAttributes().getInputParallelism().intValue());
Assert.assertEquals(2, result.getAttributes().getParsingParallelism().intValue());
Assert.assertEquals(3, result.getAttributes().getOutputParallelism().intValue());

4
parsing/parsing-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -50,7 +50,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>

4
parsing/parsing-storm/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -75,7 +75,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>parsing-app</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>

25
parsing/parsing-storm/src/main/java/uk/co/gresearch/siembol/parsers/storm/StormParsingApplication.java
View File

@@ -2,6 +2,7 @@ package uk.co.gresearch.siembol.parsers.storm;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.common.header.Header;
import org.apache.kafka.common.serialization.ByteArrayDeserializer;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.storm.Config;
@@ -51,6 +52,8 @@ public class StormParsingApplication {
"with storm attributes: {}\nparsing application attributes: {}";
private static final String UNKNOWN_SOURCE = "unknown";
private static final String UNKNOWN_SOURCE_HEADER = "unknown_header";
private static KafkaSpoutConfig<String, byte[]> createKafkaSpoutConfig(
StormParsingApplicationAttributesDto parsingApplicationAttributes,
ParsingApplicationFactoryAttributes parsingAttributes) {
@@ -77,9 +80,16 @@ public class StormParsingApplication {
return r -> new Values(r.topic(), r.key(), r.value());
case HEADER_ROUTING_PARSING:
final String headerName = parsingAttributes.getSourceHeaderName();
return r -> new Values(new String(r.headers().lastHeader(headerName).value(), StandardCharsets.UTF_8),
r.key(),
r.value());
return r -> {
Header header = r.headers() != null
? r.headers().lastHeader(headerName)
: null;
String headerValue = header != null && header.value() != null
? new String(header.value(), StandardCharsets.UTF_8)
: UNKNOWN_SOURCE_HEADER;
return new Values(headerValue, r.key(), r.value());
};
default:
throw new IllegalArgumentException();
}
@@ -95,14 +105,15 @@ public class StormParsingApplication {
.put(CLIENT_ID_CONFIG, stormAppAttributes.getClientId(parsingAttributes.getName()));
stormAppAttributes.getStormAttributes().setKafkaTopics(parsingAttributes.getInputTopics());
var numWorkers = parsingAttributes.getNumWorkers();
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout(KAFKA_SPOUT,
new KafkaSpout<>(createKafkaSpoutConfig(stormAppAttributes, parsingAttributes)),
parsingAttributes.getInputParallelism());
parsingAttributes.getInputParallelism() * numWorkers);
builder.setBolt(parsingAttributes.getName(),
new ParsingApplicationBolt(stormAppAttributes, parsingAttributes, zooKeeperConnectorFactory, metricsFactory),
parsingAttributes.getParsingParallelism())
parsingAttributes.getParsingParallelism() * numWorkers)
.localOrShuffleGrouping(KAFKA_SPOUT);
builder.setBolt(KAFKA_WRITER,
@@ -110,7 +121,7 @@ public class StormParsingApplication {
ParsingApplicationTuples.PARSING_MESSAGES.toString(),
ParsingApplicationTuples.COUNTERS.toString(),
metricsFactory),
parsingAttributes.getOutputParallelism())
parsingAttributes.getOutputParallelism() * numWorkers)
.localOrShuffleGrouping(parsingAttributes.getName());
return builder.createTopology();
@@ -137,6 +148,8 @@ public class StormParsingApplication {
ParsingApplicationFactoryAttributes parsingAttributes = result.getAttributes();
Config config = new Config();
config.putAll(stormAttributes.getStormAttributes().getStormConfig().getRawMap());
config.put(Config.TOPOLOGY_WORKERS, parsingAttributes.getNumWorkers());
StormTopology topology = createTopology(stormAttributes,
parsingAttributes,
new ZooKeeperConnectorFactoryImpl(),

1
parsing/parsing-storm/src/test/java/uk/co/gresearch/siembol/parsers/storm/StormParsingApplicationTest.java
View File

@@ -42,6 +42,7 @@ public class StormParsingApplicationTest {
],
"parse_metadata" : false,
"error_topic": "error",
"num_workers" : 1,
"input_parallelism": 1,
"parsing_parallelism": 1,
"output_parallelism": 1,

2
parsing/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</parent>
<modules>
<module>parsing-core</module>

2
pom.xml
View File

@@ -6,7 +6,7 @@
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<name>siembol</name>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
<description>A scalable, advanced security analytics framework based on open-source big data technologies.</description>
<inceptionYear>2019</inceptionYear>
<url>https://siembol.io/</url>

2
responding/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</parent>
<modules>
<module>responding-core</module>

6
responding/responding-core/pom.xml
View File

@@ -11,7 +11,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
@@ -35,12 +35,12 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>alerting-core</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.jayway.jsonpath</groupId>

6
responding/responding-stream/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</parent>
<dependencyManagement>
<dependencies>
@@ -51,7 +51,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol-common</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -62,7 +62,7 @@
<dependency>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>responding-core</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>

2
siembol-common/pom.xml
View File

@@ -9,7 +9,7 @@
<parent>
<groupId>uk.co.gresearch.siembol</groupId>
<artifactId>siembol</artifactId>
<version>2.5.13-SNAPSHOT</version>
<version>2.5.14-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>