scottk/wlan-cloud-lib-cppkafka
1
0
Fork 0
mirror of https://github.com/Telecominfraproject/wlan-cloud-lib-cppkafka.git synced 2025-12-01 01:23:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove zookeeper support

Browse Source
This commit is contained in:
Matias Fontanini
2016-06-16 20:22:36 -07:00
parent e1c2ee34fe
commit 0fb0afc4f4
27 changed files with 42 additions and 800 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/CMakeLists.txt
View File

@@ -15,17 +15,6 @@ set(SOURCES
consumer.cpp
)
if (ENABLE_ZOOKEEPER)
add_definitions("-DCPPKAFKA_ENABLE_ZOOKEEPER=1")
set(ZOOKEEPER_SOURCES
${CMAKE_CURRENT_SOURCE_DIR}/zookeeper/zookeeper_watcher.cpp
${CMAKE_CURRENT_SOURCE_DIR}/zookeeper/zookeeper_pool.cpp
${CMAKE_CURRENT_SOURCE_DIR}/zookeeper/zookeeper_subscription.cpp)
# Add json library as include dir (header only)
include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/json)
set(SOURCES ${SOURCES} ${ZOOKEEPER_SOURCES})
endif()
add_library(cppkafka ${SOURCES})
install(

57
src/configuration.cpp
View File

@@ -34,7 +34,6 @@
#include "message.h"
#include "producer.h"
#include "consumer.h"
#include "config.h"
using std::string;
using std::move;
@@ -47,12 +46,6 @@ using std::chrono::milliseconds;
namespace cppkafka {
const unordered_set<string> Configuration::VALID_EXTENSIONS = {
#ifdef CPPKAFKA_HAVE_ZOOKEEPER
"zookeeper", "zookeeper.receive.timeout.ms"
#endif // CPPKAFKA_HAVE_ZOOKEEPER
};
// Callback proxies
void delivery_report_callback_proxy(rd_kafka_t*, const rd_kafka_message_t* msg, void *opaque) {
@@ -133,18 +126,12 @@ Configuration::Configuration(rd_kafka_conf_t* ptr)
}
void Configuration::set(const string& name, const string& value) {
if (VALID_EXTENSIONS.count(name)) {
// This is one of cppkafka's extensions
extension_properties_.emplace(name, value);
}
else {
char error_buffer[512];
rd_kafka_conf_res_t result;
result = rd_kafka_conf_set(handle_.get(), name.data(), value.data(), error_buffer,
sizeof(error_buffer));
if (result != RD_KAFKA_CONF_OK) {
throw ConfigException(name, error_buffer);
}
char error_buffer[512];
rd_kafka_conf_res_t result;
result = rd_kafka_conf_set(handle_.get(), name.data(), value.data(), error_buffer,
sizeof(error_buffer));
if (result != RD_KAFKA_CONF_OK) {
throw ConfigException(name, error_buffer);
}
}
@@ -188,13 +175,8 @@ void Configuration::set_default_topic_configuration(optional<TopicConfiguration>
}
bool Configuration::has_property(const string& name) const {
if (VALID_EXTENSIONS.count(name)) {
return extension_properties_.count(name) == 1;
}
else {
size_t size = 0;
return rd_kafka_conf_get(handle_.get(), name.data(), nullptr, &size) == RD_KAFKA_CONF_OK;
}
size_t size = 0;
return rd_kafka_conf_get(handle_.get(), name.data(), nullptr, &size) == RD_KAFKA_CONF_OK;
}
rd_kafka_conf_t* Configuration::get_handle() const {
@@ -202,23 +184,14 @@ rd_kafka_conf_t* Configuration::get_handle() const {
}
string Configuration::get(const string& name) const {
if (VALID_EXTENSIONS.count(name)) {
auto iter = extension_properties_.find(name);
if (iter == extension_properties_.end()) {
throw ConfigOptionNotFound(name);
}
return iter->second;
}
else {
size_t size = 0;
auto result = rd_kafka_conf_get(handle_.get(), name.data(), nullptr, &size);
if (result != RD_KAFKA_CONF_OK) {
throw ConfigOptionNotFound(name);
}
vector<char> buffer(size);
rd_kafka_conf_get(handle_.get(), name.data(), buffer.data(), &size);
return string(buffer.data());
size_t size = 0;
auto result = rd_kafka_conf_get(handle_.get(), name.data(), nullptr, &size);
if (result != RD_KAFKA_CONF_OK) {
throw ConfigOptionNotFound(name);
}
vector<char> buffer(size);
rd_kafka_conf_get(handle_.get(), name.data(), buffer.data(), &size);
return string(buffer.data());
}
const Configuration::DeliveryReportCallback& Configuration::get_delivery_report_callback() const {

29
src/kafka_handle_base.cpp
View File

@@ -31,9 +31,6 @@
#include "exceptions.h"
#include "topic.h"
#include "topic_partition_list.h"
#ifdef CPPKAFKA_HAVE_ZOOKEEPER
#include "zookeeper/zookeeper_pool.h"
#endif // CPPKAFKA_HAVE_ZOOKEEPER
using std::string;
using std::vector;
@@ -126,32 +123,6 @@ const Configuration& KafkaHandleBase::get_configuration() const {
void KafkaHandleBase::set_handle(rd_kafka_t* handle) {
handle_ = HandlePtr(handle, &rd_kafka_destroy);
#ifdef CPPKAFKA_HAVE_ZOOKEEPER
if (config_.has_property("zookeeper")) {
string endpoint = config_.get("zookeeper");
milliseconds timeout = ZookeeperWatcher::DEFAULT_RECEIVE_TIMEOUT;
if (config_.has_property("zookeeper.receive.timeout.ms")) {
try {
timeout = milliseconds(stoi(config_.get("zookeeper.receive.timeout.ms")));
}
catch (exception&) {
throw ZookeeperException("Invalid zookeeper receive timeout");
}
}
auto& pool = ZookeeperPool::instance();
auto callback = [&](const string& brokers) {
// Add the brokers and poll
rd_kafka_brokers_add(handle_.get(), brokers.data());
rd_kafka_poll(handle_.get(), 10);
};
ZookeeperSubscription subscriber = pool.subscribe(endpoint, timeout, callback);
zookeeper_subscription_.reset(new ZookeeperSubscription(move(subscriber)));
callback(pool.get_brokers(endpoint));
}
#endif // CPPKAFKA_HAVE_ZOOKEEPER
}
Topic KafkaHandleBase::get_topic(const string& name, rd_kafka_topic_conf_t* conf) {

89
src/zookeeper/zookeeper_pool.cpp
View File

@@ -1,89 +0,0 @@
/*
* Copyright (c) 2016, Matias Fontanini
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#include "zookeeper/zookeeper_pool.h"
#include "zookeeper/zookeeper_subscription.h"
#include "exceptions.h"
using std::string;
using std::mutex;
using std::lock_guard;
using std::forward_as_tuple;
using std::piecewise_construct;
using std::chrono::milliseconds;
namespace cppkafka {
ZookeeperPool& ZookeeperPool::instance() {
static ZookeeperPool the_instance;
return the_instance;
}
ZookeeperSubscription ZookeeperPool::subscribe(const string& endpoint,
milliseconds receive_timeout,
ZookeeperWatcher::WatcherCallback callback) {
lock_guard<mutex> _(watchers_mutex_);
auto iter = watchers_.find(endpoint);
if (iter == watchers_.end()) {
iter = watchers_.emplace(piecewise_construct, forward_as_tuple(endpoint),
forward_as_tuple(endpoint, receive_timeout)).first;
}
string id = iter->second.subscribe(move(callback));
return ZookeeperSubscription(endpoint, id);
}
void ZookeeperPool::unsubscribe(const ZookeeperSubscription& subscriber) {
lock_guard<mutex> _(watchers_mutex_);
auto iter = watchers_.find(subscriber.get_endpoint());
if (iter != watchers_.end()) {
iter->second.unsubscribe(subscriber.get_subscription_id());
}
}
string ZookeeperPool::get_brokers(const string& endpoint) {
lock_guard<mutex> _(watchers_mutex_);
auto iter = watchers_.find(endpoint);
if (iter == watchers_.end()) {
throw ZookeeperException("No zookeeper watcher for given endpoint");
}
return iter->second.get_brokers();
}
size_t ZookeeperPool::get_subscriber_count(const string& endpoint) const {
lock_guard<mutex> _(watchers_mutex_);
auto iter = watchers_.find(endpoint);
if (iter == watchers_.end()) {
return 0;
}
else {
return iter->second.get_subscriber_count();
}
}
} // cppkafka

25
src/zookeeper/zookeeper_subscription.cpp
View File

@@ -1,25 +0,0 @@
#include "zookeeper/zookeeper_subscription.h"
#include "zookeeper/zookeeper_pool.h"
using std::string;
namespace cppkafka {
ZookeeperSubscription::ZookeeperSubscription(string endpoint, string subscription_id)
: endpoint_(move(endpoint)), subscription_id_(move(subscription_id)) {
}
ZookeeperSubscription::~ZookeeperSubscription() {
ZookeeperPool::instance().unsubscribe(*this);
}
const string& ZookeeperSubscription::get_endpoint() const {
return endpoint_;
}
const string& ZookeeperSubscription::get_subscription_id() const {
return subscription_id_;
}
} // cppkafka

150
src/zookeeper/zookeeper_watcher.cpp
View File

@@ -1,150 +0,0 @@
/*
* Copyright (c) 2016, Matias Fontanini
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#include <stdexcept>
#include <sstream>
#include <chrono>
#include <picojson.h>
#include "zookeeper/zookeeper_watcher.h"
#include "exceptions.h"
using std::string;
using std::lock_guard;
using std::mutex;
using std::unique_ptr;
using std::ostringstream;
using std::runtime_error;
using std::make_shared;
using std::chrono::milliseconds;
using std::chrono::system_clock;
using picojson::value;
using picojson::object;
namespace cppkafka {
const milliseconds ZookeeperWatcher::DEFAULT_RECEIVE_TIMEOUT{10000};
const string ZookeeperWatcher::BROKERS_PATH = "/brokers/ids";
ZookeeperWatcher::ZookeeperWatcher(const string& endpoint)
: ZookeeperWatcher(endpoint, DEFAULT_RECEIVE_TIMEOUT) {
}
ZookeeperWatcher::ZookeeperWatcher(const string& endpoint, milliseconds receive_timeout)
: handle_(nullptr, nullptr) {
auto raw_handle = zookeeper_init(endpoint.data(), &ZookeeperWatcher::handle_event_proxy,
receive_timeout.count(), nullptr, this, 0);
if (!raw_handle) {
// TODO: make this a proper exception
throw runtime_error("Failed to create zookeeper handle");
}
handle_ = HandlePtr(raw_handle, &zookeeper_close);
}
string ZookeeperWatcher::subscribe(WatcherCallback callback) {
lock_guard<mutex> _(callbacks_mutex_);
string id = generate_id();
callbacks_.emplace(id, move(callback));
return id;
}
void ZookeeperWatcher::unsubscribe(const string& id) {
lock_guard<mutex> _(callbacks_mutex_);
callbacks_.erase(id);
}
string ZookeeperWatcher::get_brokers() {
auto handle = handle_.get();
using VectorPtr = unique_ptr<String_vector, decltype(&deallocate_String_vector)>;
String_vector brokers;
if (zoo_get_children(handle, BROKERS_PATH.data(), 1, &brokers) != ZOK) {
throw ZookeeperException("Failed to get broker list from zookeeper");
}
// RAII up this pointer
ostringstream oss;
VectorPtr _(&brokers, &deallocate_String_vector);
for (int i = 0; i < brokers.count; i++) {
char config_line[1024];
string path = "/brokers/ids/" + string(brokers.data[i]);
int config_len = sizeof(config_line);
zoo_get(handle, path.data(), 0, config_line, &config_len, NULL);
if (config_len > 0) {
config_line[config_len] = 0;
value root;
string error = picojson::parse(root, config_line);
if (!error.empty() || !root.is<object>()) {
throw ZookeeperException("Failed to parse zookeeper json: " + error);
}
const value::object& json_object = root.get<object>();
const value& host_json = json_object.at("host");
const value& port_json = json_object.at("port");
if (!host_json.is<string>() || !port_json.is<double>()) {
throw ZookeeperException("Invalid JSON received from zookeeper");
}
string host = host_json.get<string>();
int port = port_json.get<double>();
if (i != 0) {
oss << ",";
}
oss << host << ":" << port;
}
}
return oss.str();
}
size_t ZookeeperWatcher::get_subscriber_count() const {
lock_guard<mutex> _(callbacks_mutex_);
return callbacks_.size();
}
void ZookeeperWatcher::handle_event_proxy(zhandle_t*, int type, int state, const char* path,
void* ctx) {
auto self = static_cast<ZookeeperWatcher*>(ctx);
self->handle_event(type, state, path);
}
void ZookeeperWatcher::handle_event(int type, int state, const char* path) {
if (type == ZOO_CHILD_EVENT && path == BROKERS_PATH) {
string brokers = get_brokers();
lock_guard<mutex> _(callbacks_mutex_);
for (const auto& callbackPair : callbacks_) {
callbackPair.second(brokers);
}
}
}
string ZookeeperWatcher::generate_id() {
ostringstream oss;
oss << id_counter_++ << "-" << system_clock::now().time_since_epoch().count();
return oss.str();
}
} // cppkafka