scottk/wlan-cloud-lib-cppkafka
1
0
Fork 0
mirror of https://github.com/Telecominfraproject/wlan-cloud-lib-cppkafka.git synced 2025-11-01 19:18:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Buffered producer thread safe (#72)

Browse Source 
* Thread safe buffered producer

* Using single mutex version

* Changed based on feedback

* Changes based on latest review

* Added flush counter
This commit is contained in:
Alex Damian
2018-05-28 21:33:36 -04:00
committed by Matias Fontanini
parent f543810515
commit 429ec92369
8 changed files with 387 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

255
include/cppkafka/utils/buffered_producer.h
View File

@@ -31,12 +31,14 @@
#define CPPKAFKA_BUFFERED_PRODUCER_H
#include <string>
#include <queue>
#include <deque>
#include <cstdint>
#include <algorithm>
#include <unordered_set>
#include <unordered_map>
#include <map>
#include <mutex>
#include <atomic>
#include <boost/optional.hpp>
#include "../producer.h"
#include "../message.h"
@@ -50,13 +52,34 @@ namespace cppkafka {
* to produce them just as you would using the Producer class.
*
* When calling either flush or wait_for_acks, the buffered producer will block until all
* produced messages (either in a buffer or non buffered way) are acknowledged by the kafka
* brokers.
* produced messages (either buffered or sent directly) are acknowledged by the kafka brokers.
*
* When producing messages, this class will handle cases where the producer's queue is full so it\
* When producing messages, this class will handle cases where the producer's queue is full so it
* will poll until the production is successful.
*
* This class is not thread safe.
* \remark This class is thread safe.
*
* \remark Releasing buffers: For high-performance applications preferring a zero-copy approach
* (using PayloadPolicy::PASSTHROUGH_PAYLOAD - see warning below) it is very important to know when
* to safely release owned message buffers. One way is to perform individual cleanup when
* ProduceSuccessCallback is called. If the application produces messages in batches or has a
* bursty behavior another way is to check when flush operations have fully completed with
* get_buffer_size()==0 && get_flushes_in_progress()==0. Note that get_pending_acks()==0
* is not always a guarantee as there is very small window when flush() starts where
* get_buffer_size()==0 && get_pending_acks()==0 but messages have not yet been sent to the
* remote broker. For applications producing messages w/o buffering, get_pending_acks()==0
* is sufficient.
*
* \warning Delivery Report Callback: This class makes internal use of this function and will
* overwrite anything the user has supplied as part of the configuration options. Instead user
* should call set_produce_success_callback() and set_produce_failure_callback() respectively.
*
* \warning Payload Policy: For payload-owning BufferTypes such as std::string or std::vector<char>
* the default policy is set to Producer::PayloadPolicy::COPY_PAYLOAD. For the specific non-payload owning type
* cppkafka::Buffer the policy is Producer::PayloadPolicy::PASSTHROUGH_PAYLOAD. In this case, librdkafka
* shall not make any internal copies of the message and it is the application's responsability to free
* the messages *after* the ProduceSuccessCallback has reported a successful delivery to avoid memory
* corruptions.
*/
template <typename BufferType>
class CPPKAFKA_API BufferedProducer {
@@ -65,11 +88,21 @@ public:
* Concrete builder
*/
using Builder = ConcreteMessageBuilder<BufferType>;
/**
* Callback to indicate a message was delivered to the broker
*/
using ProduceSuccessCallback = std::function<void(const Message&)>;
/**
* Callback to indicate a message failed to be produced.
* Callback to indicate a message failed to be produced by the broker
*/
using ProduceFailureCallback = std::function<bool(const Message&)>;
/**
* Callback to indicate a message failed to be flushed
*/
using FlushFailureCallback = std::function<bool(const Builder&, Error error)>;
/**
* \brief Constructs a buffered producer using the provided configuration
@@ -106,6 +139,8 @@ public:
* wait for it to be acknowledged.
*
* \param builder The builder that contains the message to be produced
*
* \remark This method throws cppkafka::HandleException on failure
*/
void produce(const MessageBuilder& builder);
@@ -116,6 +151,8 @@ public:
* wait for it to be acknowledged.
*
* \param message The message to be produced
*
* \remark This method throws cppkafka::HandleException on failure
*/
void produce(const Message& message);
@@ -124,6 +161,10 @@ public:
*
* This will send all messages and keep waiting until all of them are acknowledged (this is
* done by calling wait_for_acks).
*
* \remark Although it is possible to call flush from multiple threads concurrently, better
* performance is achieved when called from the same thread or when serialized
* with respect to other threads.
*/
void flush();
@@ -159,11 +200,35 @@ public:
ssize_t get_max_buffer_size() const;
/**
* \brief Get the number of messages in the buffer
* \brief Get the number of unsent messages in the buffer
*
* \return The number of messages
*/
size_t get_buffer_size() const;
/**
* \brief Get the number of messages not yet acked by the broker
*
* \return The number of messages
*/
size_t get_pending_acks() const;
/**
* \brief Get the total number of messages successfully produced since the beginning
*
* \return The number of messages
*/
size_t get_total_messages_produced() const;
/**
* \brief Get the total outstanding flush operations in progress
*
* Since flush can be called from multiple threads concurrently, this counter indicates
* how many operations are curretnly in progress.
*
* \return The number of outstanding flush operations.
*/
size_t get_flushes_in_progress() const;
/**
* Gets the Producer object
@@ -188,69 +253,131 @@ public:
* false. Note that if the callback return false, then the message will be discarded.
*
* \param callback The callback to be set
*
* \remark It is *highly* recommended to set this callback as your message may be produced
* indefinitely if there's a remote error.
*
* \warning Do not call any method on the BufferedProducer while inside this callback.
*/
void set_produce_failure_callback(ProduceFailureCallback callback);
/**
* \brief Sets the successful delivery callback
*
* The user can use this function to cleanup any application-owned message buffers.
*
* \param callback The callback to be set
*/
void set_produce_success_callback(ProduceSuccessCallback callback);
/**
* \brief Sets the local message produce failure callback
*
* This callback will be called when local message production fails during a flush() operation.
* Failure errors are typically payload too large, unknown topic or unknown partition.
* Note that if the callback returns false, the message will be dropped from the buffer,
* otherwise it will be re-enqueued for later retry.
*
* \param callback
*
* \warning Do not call any method on the BufferedProducer while inside this callback
*/
void set_flush_failure_callback(FlushFailureCallback callback);
private:
using QueueType = std::queue<Builder>;
using QueueType = std::deque<Builder>;
enum class MessagePriority { Low, High };
template <typename T>
struct CounterGuard{
CounterGuard(std::atomic<T>& counter) : counter_(counter) { ++counter_; }
~CounterGuard() { --counter_; }
std::atomic<T>& counter_;
};
template <typename BuilderType>
void do_add_message(BuilderType&& builder);
void do_add_message(BuilderType&& builder, MessagePriority priority, bool do_flush);
template <typename MessageType>
void produce_message(const MessageType& message);
Configuration prepare_configuration(Configuration config);
void on_delivery_report(const Message& message);
Configuration::DeliveryReportCallback delivery_report_callback_;
// Members
Producer producer_;
QueueType messages_;
mutable std::mutex mutex_;
ProduceSuccessCallback produce_success_callback_;
ProduceFailureCallback produce_failure_callback_;
size_t expected_acks_{0};
size_t messages_acked_{0};
FlushFailureCallback flush_failure_callback_;
ssize_t max_buffer_size_{-1};
std::atomic<size_t> pending_acks_{0};
std::atomic<size_t> flushes_in_progress_{0};
std::atomic<size_t> total_messages_produced_{0};
};
template <typename BufferType>
BufferedProducer<BufferType>::BufferedProducer(Configuration config)
: delivery_report_callback_(config.get_delivery_report_callback()),
producer_(prepare_configuration(std::move(config))) {
Producer::PayloadPolicy get_default_payload_policy() {
return Producer::PayloadPolicy::COPY_PAYLOAD;
}
template <> inline
Producer::PayloadPolicy get_default_payload_policy<Buffer>() {
return Producer::PayloadPolicy::PASSTHROUGH_PAYLOAD;
}
template <typename BufferType>
BufferedProducer<BufferType>::BufferedProducer(Configuration config)
: producer_(prepare_configuration(std::move(config))) {
producer_.set_payload_policy(get_default_payload_policy<BufferType>());
}
template <typename BufferType>
void BufferedProducer<BufferType>::add_message(const MessageBuilder& builder) {
do_add_message(builder);
do_add_message(builder, MessagePriority::Low, true);
}
template <typename BufferType>
void BufferedProducer<BufferType>::add_message(Builder builder) {
do_add_message(move(builder));
do_add_message(move(builder), MessagePriority::Low, true);
}
template <typename BufferType>
void BufferedProducer<BufferType>::produce(const MessageBuilder& builder) {
produce_message(builder);
expected_acks_++;
}
template <typename BufferType>
void BufferedProducer<BufferType>::produce(const Message& message) {
produce_message(message);
expected_acks_++;
}
template <typename BufferType>
void BufferedProducer<BufferType>::flush() {
while (!messages_.empty()) {
produce_message(messages_.front());
messages_.pop();
CounterGuard<size_t> counter_guard(flushes_in_progress_);
QueueType flush_queue; // flush from temporary queue
{
std::lock_guard<std::mutex> lock(mutex_);
std::swap(messages_, flush_queue);
}
while (!flush_queue.empty()) {
try {
produce_message(flush_queue.front());
}
catch (const HandleException& ex) {
if (flush_failure_callback_ &&
flush_failure_callback_(flush_queue.front(), ex.get_error())) {
// retry again later
do_add_message(std::move(flush_queue.front()), MessagePriority::Low, false);
}
}
flush_queue.pop_front();
}
wait_for_acks();
}
template <typename BufferType>
void BufferedProducer<BufferType>::wait_for_acks() {
while (messages_acked_ < expected_acks_) {
while (pending_acks_ > 0) {
try {
producer_.flush();
}
@@ -264,16 +391,13 @@ void BufferedProducer<BufferType>::wait_for_acks() {
}
}
}
expected_acks_ = 0;
messages_acked_ = 0;
}
template <typename BufferType>
void BufferedProducer<BufferType>::clear() {
std::lock_guard<std::mutex> lock(mutex_);
QueueType tmp;
std::swap(tmp, messages_);
expected_acks_ = 0;
messages_acked_ = 0;
}
template <typename BufferType>
@@ -296,10 +420,19 @@ size_t BufferedProducer<BufferType>::get_buffer_size() const {
template <typename BufferType>
template <typename BuilderType>
void BufferedProducer<BufferType>::do_add_message(BuilderType&& builder) {
expected_acks_++;
messages_.push(std::forward<BuilderType>(builder));
if ((max_buffer_size_ >= 0) && (max_buffer_size_ <= (ssize_t)messages_.size())) {
void BufferedProducer<BufferType>::do_add_message(BuilderType&& builder,
MessagePriority priority,
bool do_flush) {
{
std::lock_guard<std::mutex> lock(mutex_);
if (priority == MessagePriority::High) {
messages_.emplace_front(std::move(builder));
}
else {
messages_.emplace_back(std::move(builder));
}
}
if (do_flush && (max_buffer_size_ >= 0) && (max_buffer_size_ <= (ssize_t)messages_.size())) {
flush();
}
}
@@ -314,6 +447,21 @@ const Producer& BufferedProducer<BufferType>::get_producer() const {
return producer_;
}
template <typename BufferType>
size_t BufferedProducer<BufferType>::get_pending_acks() const {
return pending_acks_;
}
template <typename BufferType>
size_t BufferedProducer<BufferType>::get_total_messages_produced() const {
return total_messages_produced_;
}
template <typename BufferType>
size_t BufferedProducer<BufferType>::get_flushes_in_progress() const {
return flushes_in_progress_;
}
template <typename BufferType>
typename BufferedProducer<BufferType>::Builder
BufferedProducer<BufferType>::make_builder(std::string topic) {
@@ -325,18 +473,28 @@ void BufferedProducer<BufferType>::set_produce_failure_callback(ProduceFailureCa
produce_failure_callback_ = std::move(callback);
}
template <typename BufferType>
void BufferedProducer<BufferType>::set_produce_success_callback(ProduceSuccessCallback callback) {
produce_success_callback_ = std::move(callback);
}
template <typename BufferType>
void BufferedProducer<BufferType>::set_flush_failure_callback(FlushFailureCallback callback) {
flush_failure_callback_ = std::move(callback);
}
template <typename BufferType>
template <typename MessageType>
void BufferedProducer<BufferType>::produce_message(const MessageType& message) {
bool sent = false;
while (!sent) {
while (true) {
try {
producer_.produce(message);
sent = true;
// Sent successfully
++pending_acks_;
break;
}
catch (const HandleException& ex) {
const Error error = ex.get_error();
if (error == RD_KAFKA_RESP_ERR__QUEUE_FULL) {
if (ex.get_error() == RD_KAFKA_RESP_ERR__QUEUE_FULL) {
// If the output queue is full, then just poll
producer_.poll();
}
@@ -357,21 +515,26 @@ Configuration BufferedProducer<BufferType>::prepare_configuration(Configuration
template <typename BufferType>
void BufferedProducer<BufferType>::on_delivery_report(const Message& message) {
// Call the user-supplied delivery report callback if any
if (delivery_report_callback_) {
delivery_report_callback_(producer_, message);
}
// Decrement the expected acks
--pending_acks_;
assert(pending_acks_ != (size_t)-1); // Prevent underflow
// We should produce this message again if it has an error and we either don't have a
// produce failure callback or we have one but it returns true
bool should_produce = message.get_error() &&
(!produce_failure_callback_ || produce_failure_callback_(message));
if (should_produce) {
produce_message(message);
return;
// Re-enqueue for later retransmission with higher priority (i.e. front of the queue)
do_add_message(Builder(message), MessagePriority::High, false);
}
else {
// Successful delivery
if (produce_success_callback_) {
produce_success_callback_(message);
}
// Increment the total successful transmissions
++total_messages_produced_;
}
// If production was successful or the produce failure callback returned false, then
// let's consider it to be acked
messages_acked_++;
}
} // cppkafka