scottk/wlan-cloud-lib-cppkafka
1
0
Fork 0
mirror of https://github.com/Telecominfraproject/wlan-cloud-lib-cppkafka.git synced 2025-11-01 02:57:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add MessageBuilder and use it on producer classes

Browse Source
This commit is contained in:
Matias Fontanini
2017-04-15 16:46:37 -07:00
parent c8154ac6cb
commit e26d7e7db2
10 changed files with 224 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
include/cppkafka/message_builder.h Normal file
View File

@@ -0,0 +1,126 @@
/*
* Copyright (c) 2016, Matias Fontanini
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#ifndef CPPKAFKA_MESSAGE_BUILDER_H
#define CPPKAFKA_MESSAGE_BUILDER_H
#include <boost/optional.hpp>
#include "buffer.h"
#include "topic.h"
#include "partition.h"
namespace cppkafka {
/**
* \brief Message builder class
*
* Allows building a message including topic, partition, key, payload, etc.
*
* Example:
*
* \code
* Producer producer(...);
* Topic topic = producer.get_topic("test");
*
* string payload = "hello world";
* producer.produce(MessageBuilder(topic).partition(5).payload(payload));
* \endcode
*/
class MessageBuilder {
public:
/**
* Construct a MessageBuilder
*
* \param topic The topic into which this message would be produced
*/
MessageBuilder(const Topic& topic);
/**
* Sets the partition into which this message will be produced
*
* \param value The partition to be used
*/
MessageBuilder& partition(Partition value);
/**
* Sets the message's key
*
* \param value The key to be used
*/
MessageBuilder& key(const Buffer& value);
/**
* Sets the message's payload
*
* \param value The payload to be used
*/
MessageBuilder& payload(const Buffer& value);
/**
* Sets the message's user data pointer
*
* \param value Pointer to the user data to be used on the produce call
*/
MessageBuilder& user_data(void* value);
/**
* Gets the topic this message will be produced into
*/
const Topic& topic() const;
/**
* Gets the partition this message will be produced into
*/
const Partition& partition() const;
/**
* Gets the message's key
*/
const Buffer& key() const;
/**
* Gets the message's payload
*/
const Buffer& payload() const;
/**
* Gets the message's user data pointer
*/
void* user_data() const;
private:
const Topic& topic_;
Partition partition_;
Buffer key_;
Buffer payload_;
void* user_data_;
};
} // cppkafka
#endif // CPPKAFKA_MESSAGE_BUILDER_H

30
include/cppkafka/producer.h
View File

@@ -37,6 +37,7 @@
#include "topic.h"
#include "partition.h"
#include "macros.h"
#include "message_builder.h"
namespace cppkafka {
@@ -72,10 +73,10 @@ class TopicConfiguration;
* string payload = "some payload";
*
* // Write a message into an unassigned partition
* producer.produce(topic, Partition(), payload);
* producer.produce(MessageBuilder(topic).payload(payload));
*
* // Write using a key on a fixed partition (42)
* producer.produce(topic, 42, key, payload);
* producer.produce(MessageBuilder(topic).partition(42).key(key).payload(payload));
*
* \endcode
*/
@@ -112,30 +113,7 @@ public:
* \param partition The partition to write the message to
* \param payload The message payload
*/
void produce(const Topic& topic, const Partition& partition, const Buffer& payload);
/**
* Produces a message
*
* \param topic The topic to write the message to
* \param partition The partition to write the message to
* \param key The message key
* \param payload The message payload
*/
void produce(const Topic& topic, const Partition& partition, const Buffer& key,
const Buffer& payload);
/**
* Produces a message
*
* \param topic The topic to write the message to
* \param partition The partition to write the message to
* \param key The message key
* \param payload The message payload
* \param user_data The opaque data pointer to be used (accesible via Message::private_data)
*/
void produce(const Topic& topic, const Partition& partition, const Buffer& key,
const Buffer& payload, void* user_data);
void produce(const MessageBuilder& builder);
/**
* \brief Polls on this handle

91
include/cppkafka/utils/buffered_producer.h
View File

@@ -29,24 +29,9 @@ public:
*
* The message won't be sent until flush is called.
*
* \param topic The topic in which this message should be written to
* \param partition The partition in which this message should be written to
* \param payload The message's payload
* \param builder The builder that contains the message to be added
*/
void add_message(const std::string& topic, const Partition& partition, BufferType payload);
/**
* \brief Adds a message to the producer's buffer.
*
* The message won't be sent until flush is called.
*
* \param topic The topic in which this message should be written to
* \param partition The partition in which this message should be written to
* \param key The message's key
* \param payload The message's payload
*/
void add_message(const std::string& topic, const Partition& partition, BufferType key,
BufferType payload);
void add_message(const MessageBuilder& builder);
/**
* \brief Flushes the buffered messages.
@@ -68,34 +53,13 @@ private:
// Pick the most appropriate index type depending on the platform we're using
using IndexType = std::conditional<sizeof(void*) == 8, uint64_t, uint32_t>::type;
struct BufferedMessage {
BufferedMessage(unsigned topic_index, Partition partition, BufferType key,
BufferType payload)
: key(std::move(key)), payload(std::move(payload)), topic_index(topic_index),
partition(partition) {
}
BufferedMessage(unsigned topic_index, Partition partition, BufferType payload)
: payload(std::move(payload)), topic_index(topic_index), partition(partition) {
}
boost::optional<BufferType> key;
BufferType payload;
unsigned topic_index;
Partition partition;
};
template <typename... Args>
void buffer_message(const std::string& topic, Args&&... args);
unsigned get_topic_index(const std::string& topic);
void produce_message(IndexType index, const BufferedMessage& message);
const Topic& get_topic(const std::string& topic);
void produce_message(IndexType index, MessageBuilder& message);
Configuration prepare_configuration(Configuration config);
void on_delivery_report(const Message& message);
Producer producer_;
std::map<IndexType, BufferedMessage> messages_;
std::map<IndexType, MessageBuilder> messages_;
std::vector<IndexType> failed_indexes_;
IndexType current_index_{0};
std::vector<Topic> topics_;
@@ -109,15 +73,19 @@ BufferedProducer<BufferType>::BufferedProducer(Configuration config)
}
template <typename BufferType>
void BufferedProducer<BufferType>::add_message(const std::string& topic,
const Partition& partition,
BufferType payload) {
buffer_message(topic, partition, payload);
void BufferedProducer<BufferType>::add_message(const MessageBuilder& builder) {
MessageBuilder local_builder(get_topic(builder.topic().get_name()));
local_builder.partition(builder.partition());
local_builder.key(builder.key());
local_builder.payload(builder.payload());
IndexType index = messages_.size();
messages_.emplace(index, std::move(local_builder));
}
template <typename BufferType>
void BufferedProducer<BufferType>::flush() {
for (const auto& message_pair : messages_) {
for (auto& message_pair : messages_) {
produce_message(message_pair.first, message_pair.second);
}
@@ -143,45 +111,24 @@ const Producer& BufferedProducer<BufferType>::get_producer() const {
}
template <typename BufferType>
void BufferedProducer<BufferType>::add_message(const std::string& topic,
const Partition& partition,
BufferType key, BufferType payload) {
buffer_message(topic, partition, key, payload);
}
template <typename BufferType>
template <typename... Args>
void BufferedProducer<BufferType>::buffer_message(const std::string& topic, Args&&... args) {
IndexType index = messages_.size();
BufferedMessage message{get_topic_index(topic), std::forward<Args>(args)...};
messages_.emplace(index, std::move(message));
}
template <typename BufferType>
unsigned BufferedProducer<BufferType>::get_topic_index(const std::string& topic) {
const Topic& BufferedProducer<BufferType>::get_topic(const std::string& topic) {
auto iter = topic_mapping_.find(topic);
if (iter == topic_mapping_.end()) {
unsigned index = topics_.size();
topics_.push_back(producer_.get_topic(topic));
iter = topic_mapping_.emplace(topic, index).first;
}
return iter->second;
return topics_[iter->second];
}
template <typename BufferType>
void BufferedProducer<BufferType>::produce_message(IndexType index,
const BufferedMessage& message) {
MessageBuilder& builder) {
bool sent = false;
builder.user_data(reinterpret_cast<void*>(index));
while (!sent) {
try {
if (message.key) {
producer_.produce(topics_[message.topic_index], message.partition, *message.key,
message.payload, reinterpret_cast<void*>(index));
}
else {
producer_.produce(topics_[message.topic_index], message.partition, {} /*key*/,
message.payload, reinterpret_cast<void*>(index));
}
producer_.produce(builder);
sent = true;
}
catch (const HandleException& ex) {