From ccb17f89cd3c7086c4c70f10cf7605814f9b01db Mon Sep 17 00:00:00 2001 From: Nikolai Vladimirov Date: Fri, 27 Jun 2025 19:37:37 +0000 Subject: [PATCH 01/37] Draft version without consume_one implementation --- .../boost/redis/adapter/detail/adapters.hpp | 28 +++++++ .../redis/adapter/detail/response_traits.hpp | 7 ++ .../redis/adapter/detail/result_traits.hpp | 8 ++ include/boost/redis/resp3/node.hpp | 8 ++ include/boost/redis/response.hpp | 77 +++++++++++++++++++ 5 files changed, 128 insertions(+) diff --git a/include/boost/redis/adapter/detail/adapters.hpp b/include/boost/redis/adapter/detail/adapters.hpp index 818cf6c02..83de233b6 100644 --- a/include/boost/redis/adapter/detail/adapters.hpp +++ b/include/boost/redis/adapter/detail/adapters.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -176,6 +177,33 @@ class general_aggregate { } }; +template <> +class general_aggregate> { +private: + result* result_; + +public: + explicit general_aggregate(result* c = nullptr) + : result_(c) + { } + template + void operator()(resp3::basic_node const& nd, system::error_code&) + { + BOOST_ASSERT_MSG(!!result_, "Unexpected null pointer"); + switch (nd.data_type) { + case resp3::type::blob_error: + case resp3::type::simple_error: + *result_ = error{ + nd.data_type, + std::string{std::cbegin(nd.value), std::cend(nd.value)} + }; + break; + default: + result_->value().push_back(nd); + } + } +}; + template class general_simple { private: diff --git a/include/boost/redis/adapter/detail/response_traits.hpp b/include/boost/redis/adapter/detail/response_traits.hpp index f24dc8917..1ab2c5893 100644 --- a/include/boost/redis/adapter/detail/response_traits.hpp +++ b/include/boost/redis/adapter/detail/response_traits.hpp @@ -107,6 +107,13 @@ struct response_traits> { static auto adapt(response_type& r) noexcept { return adapter_type{r}; } }; +template <> +struct response_traits { + using response_type = generic_flat_response; + using adapter_type = vector_adapter; + + static auto adapt(response_type& v) noexcept { return adapter_type{v}; } +}; } // namespace boost::redis::adapter::detail #endif // BOOST_REDIS_ADAPTER_DETAIL_RESPONSE_TRAITS_HPP diff --git a/include/boost/redis/adapter/detail/result_traits.hpp b/include/boost/redis/adapter/detail/result_traits.hpp index da625c999..2e158a399 100644 --- a/include/boost/redis/adapter/detail/result_traits.hpp +++ b/include/boost/redis/adapter/detail/result_traits.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include @@ -62,6 +63,13 @@ struct result_traits, Allocator>>> static auto adapt(response_type& v) noexcept { return adapter_type{&v}; } }; +template <> +struct result_traits { + using response_type = generic_flat_response; + using adapter_type = adapter::detail::general_aggregate; + static auto adapt(response_type& v) noexcept { return adapter_type{&v}; } +}; + template using adapter_t = typename result_traits>::adapter_type; diff --git a/include/boost/redis/resp3/node.hpp b/include/boost/redis/resp3/node.hpp index 6e3f4aac2..d55957a36 100644 --- a/include/boost/redis/resp3/node.hpp +++ b/include/boost/redis/resp3/node.hpp @@ -59,6 +59,14 @@ using node = basic_node; /// A node in the response tree that does not own its data. using node_view = basic_node; +/** + * TODO: documentation + */ +struct offset_node : node_view { + std::size_t offset{}; + std::size_t size{}; +}; + } // namespace boost::redis::resp3 #endif // BOOST_REDIS_RESP3_NODE_HPP diff --git a/include/boost/redis/response.hpp b/include/boost/redis/response.hpp index 94fdd63d5..251dcd8ca 100644 --- a/include/boost/redis/response.hpp +++ b/include/boost/redis/response.hpp @@ -31,6 +31,83 @@ using response = std::tuple...>; */ using generic_response = adapter::result>; +struct flat_response_impl { +private: + class iterator { + public: + using value_type = resp3::node_view; + using difference_type = std::ptrdiff_t; + using pointer = void; + using reference = value_type; + using iterator_category = std::forward_iterator_tag; + + explicit iterator(flat_response_impl* owner, std::size_t i) noexcept + : owner_(owner) + , index_(i) + { } + + value_type operator*() const { return owner_->operator[](index_); } + + iterator& operator++() + { + ++index_; + return *this; + } + + bool operator==(const iterator& other) const { return index_ == other.index_; } + bool operator!=(const iterator& other) const { return !(*this == other); } + + private: + flat_response_impl* owner_; + std::size_t index_; + }; + +public: + resp3::node_view at(std::size_t index) { return make_node_view(view_.at(index)); } + + std::size_t size() { return view_.size(); } + + resp3::node_view operator[](std::size_t index) { return make_node_view(view_[index]); } + + iterator begin() { return iterator{this, 0}; } + + iterator end() { return iterator{this, view_.size()}; } + + template + void push_back(const resp3::basic_node& nd) + { + resp3::offset_node new_node; + new_node.data_type = nd.data_type; + new_node.aggregate_size = nd.aggregate_size; + new_node.depth = nd.depth; + new_node.offset = data_.size(); + new_node.size = nd.value.size(); + + data_ += std::string{std::cbegin(nd.value), std::cend(nd.value)}; + + view_.push_back(std::move(new_node)); + } + +private: + resp3::node_view make_node_view(const resp3::offset_node& n) + { + return resp3::node_view{ + .data_type = n.data_type, + .aggregate_size = n.aggregate_size, + .depth = n.depth, + .value = std::string_view{data_.data() + n.offset, n.size} + }; + } + + std::string data_; + std::vector view_; +}; + +/** + * TODO: documentation + */ +using generic_flat_response = adapter::result; + /** @brief Consume on response from a generic response * * This function rotates the elements so that the start of the next From 7d959c1039946d3e4676f412459766a8046de816 Mon Sep 17 00:00:00 2001 From: Nikolai Vladimirov Date: Fri, 27 Jun 2025 23:28:48 +0000 Subject: [PATCH 02/37] Addressed some comments --- .../boost/redis/adapter/detail/adapters.hpp | 9 +++-- include/boost/redis/resp3/node.hpp | 8 ----- include/boost/redis/response.hpp | 34 +++++++++++-------- 3 files changed, 23 insertions(+), 28 deletions(-) diff --git a/include/boost/redis/adapter/detail/adapters.hpp b/include/boost/redis/adapter/detail/adapters.hpp index 83de233b6..0884632c2 100644 --- a/include/boost/redis/adapter/detail/adapters.hpp +++ b/include/boost/redis/adapter/detail/adapters.hpp @@ -178,12 +178,12 @@ class general_aggregate { }; template <> -class general_aggregate> { +class general_aggregate> { private: - result* result_; + result* result_; public: - explicit general_aggregate(result* c = nullptr) + explicit general_aggregate(result* c = nullptr) : result_(c) { } template @@ -198,8 +198,7 @@ class general_aggregate> { std::string{std::cbegin(nd.value), std::cend(nd.value)} }; break; - default: - result_->value().push_back(nd); + default: result_->value().push_back(nd); } } }; diff --git a/include/boost/redis/resp3/node.hpp b/include/boost/redis/resp3/node.hpp index d55957a36..6e3f4aac2 100644 --- a/include/boost/redis/resp3/node.hpp +++ b/include/boost/redis/resp3/node.hpp @@ -59,14 +59,6 @@ using node = basic_node; /// A node in the response tree that does not own its data. using node_view = basic_node; -/** - * TODO: documentation - */ -struct offset_node : node_view { - std::size_t offset{}; - std::size_t size{}; -}; - } // namespace boost::redis::resp3 #endif // BOOST_REDIS_RESP3_NODE_HPP diff --git a/include/boost/redis/response.hpp b/include/boost/redis/response.hpp index 251dcd8ca..106a3fd58 100644 --- a/include/boost/redis/response.hpp +++ b/include/boost/redis/response.hpp @@ -31,7 +31,7 @@ using response = std::tuple...>; */ using generic_response = adapter::result>; -struct flat_response_impl { +struct flat_response_value { private: class iterator { public: @@ -41,7 +41,7 @@ struct flat_response_impl { using reference = value_type; using iterator_category = std::forward_iterator_tag; - explicit iterator(flat_response_impl* owner, std::size_t i) noexcept + explicit iterator(flat_response_value* owner, std::size_t i) noexcept : owner_(owner) , index_(i) { } @@ -58,10 +58,15 @@ struct flat_response_impl { bool operator!=(const iterator& other) const { return !(*this == other); } private: - flat_response_impl* owner_; + flat_response_value* owner_; std::size_t index_; }; + struct offset_node : resp3::node_view { + std::size_t offset{}; + std::size_t size{}; + }; + public: resp3::node_view at(std::size_t index) { return make_node_view(view_.at(index)); } @@ -76,37 +81,36 @@ struct flat_response_impl { template void push_back(const resp3::basic_node& nd) { - resp3::offset_node new_node; + offset_node new_node; new_node.data_type = nd.data_type; new_node.aggregate_size = nd.aggregate_size; new_node.depth = nd.depth; new_node.offset = data_.size(); new_node.size = nd.value.size(); - data_ += std::string{std::cbegin(nd.value), std::cend(nd.value)}; - + data_.append(nd.value.data()); view_.push_back(std::move(new_node)); } private: - resp3::node_view make_node_view(const resp3::offset_node& n) + resp3::node_view make_node_view(const offset_node& nd) { - return resp3::node_view{ - .data_type = n.data_type, - .aggregate_size = n.aggregate_size, - .depth = n.depth, - .value = std::string_view{data_.data() + n.offset, n.size} - }; + resp3::node_view result; + result.data_type = nd.data_type; + result.aggregate_size = nd.aggregate_size; + result.depth = nd.depth; + result.value = std::string_view{data_.data() + nd.offset, nd.size}; + return result; } std::string data_; - std::vector view_; + std::vector view_; }; /** * TODO: documentation */ -using generic_flat_response = adapter::result; +using generic_flat_response = adapter::result; /** @brief Consume on response from a generic response * From b78cf818e0d61df9f4866ec969546d3e6d9763e8 Mon Sep 17 00:00:00 2001 From: Nikolai Vladimirov Date: Sat, 28 Jun 2025 04:59:37 +0000 Subject: [PATCH 03/37] reserve function --- include/boost/redis/response.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/boost/redis/response.hpp b/include/boost/redis/response.hpp index 106a3fd58..d54a38376 100644 --- a/include/boost/redis/response.hpp +++ b/include/boost/redis/response.hpp @@ -68,6 +68,12 @@ struct flat_response_value { }; public: + void reserve(std::size_t new_cap, std::size_t elem_length) + { + data_.reserve(new_cap * elem_length); + view_.reserve(new_cap); + } + resp3::node_view at(std::size_t index) { return make_node_view(view_.at(index)); } std::size_t size() { return view_.size(); } From 9419c857fdbcf54de60fc0fa450d7aaa6af2bdf6 Mon Sep 17 00:00:00 2001 From: Nikolai Vladimirov Date: Wed, 2 Jul 2025 18:39:43 +0000 Subject: [PATCH 04/37] Corrected and cleaner implementation with comments --- example/cpp20_subscriber.cpp | 3 + include/boost/redis/impl/response.ipp | 37 ++++++--- include/boost/redis/resp3/node.hpp | 8 ++ include/boost/redis/response.hpp | 105 +++++++++----------------- 4 files changed, 71 insertions(+), 82 deletions(-) diff --git a/example/cpp20_subscriber.cpp b/example/cpp20_subscriber.cpp index d2ec462f6..09536dd17 100644 --- a/example/cpp20_subscriber.cpp +++ b/example/cpp20_subscriber.cpp @@ -23,6 +23,7 @@ namespace asio = boost::asio; using namespace std::chrono_literals; using boost::redis::request; using boost::redis::generic_response; +using boost::redis::generic_flat_response; using boost::redis::consume_one; using boost::redis::logger; using boost::redis::config; @@ -54,6 +55,8 @@ auto receiver(std::shared_ptr conn) -> asio::awaitable request req; req.push("SUBSCRIBE", "channel"); + // Alternatively, you can use generic_flat_response here, but keep in mind + // that to access elements you need to call .view() on resp.value() generic_response resp; conn->set_receive_response(resp); diff --git a/include/boost/redis/impl/response.ipp b/include/boost/redis/impl/response.ipp index a4b09a6e7..19194c681 100644 --- a/include/boost/redis/impl/response.ipp +++ b/include/boost/redis/impl/response.ipp @@ -11,15 +11,30 @@ namespace boost::redis { -void consume_one(generic_response& r, system::error_code& ec) +namespace { +template +auto& get_value(Container& c) +{ + return c; +} + +template <> +auto& get_value(flat_response_value& c) +{ + return c.view(); +} + +template +void consume_one_impl(Response& r, system::error_code& ec) { if (r.has_error()) return; // Nothing to consume. - if (std::empty(r.value())) + auto& value = get_value(r.value()); + if (std::empty(value)) return; // Nothing to consume. - auto const depth = r.value().front().depth; + auto const depth = value.front().depth; // To simplify we will refuse to consume any data-type that is not // a root node. I think there is no use for that and it is complex @@ -33,17 +48,15 @@ void consume_one(generic_response& r, system::error_code& ec) return e.depth == depth; }; - auto match = std::find_if(std::next(std::cbegin(r.value())), std::cend(r.value()), f); + auto match = std::find_if(std::next(std::cbegin(value)), std::cend(value), f); - r.value().erase(std::cbegin(r.value()), match); + value.erase(std::cbegin(value), match); } -void consume_one(generic_response& r) -{ - system::error_code ec; - consume_one(r, ec); - if (ec) - throw system::system_error(ec); -} +} // namespace + +void consume_one(generic_response& r, system::error_code& ec) { consume_one_impl(r, ec); } + +void consume_one(generic_flat_response& r, system::error_code& ec) { consume_one_impl(r, ec); } } // namespace boost::redis diff --git a/include/boost/redis/resp3/node.hpp b/include/boost/redis/resp3/node.hpp index 6e3f4aac2..85147b1cd 100644 --- a/include/boost/redis/resp3/node.hpp +++ b/include/boost/redis/resp3/node.hpp @@ -59,6 +59,14 @@ using node = basic_node; /// A node in the response tree that does not own its data. using node_view = basic_node; +struct offset_string { + std::string_view data; + std::size_t offset{}; + std::size_t size{}; +}; + +using offset_node = basic_node; + } // namespace boost::redis::resp3 #endif // BOOST_REDIS_RESP3_NODE_HPP diff --git a/include/boost/redis/response.hpp b/include/boost/redis/response.hpp index d54a38376..a8a8a4141 100644 --- a/include/boost/redis/response.hpp +++ b/include/boost/redis/response.hpp @@ -32,89 +32,48 @@ using response = std::tuple...>; using generic_response = adapter::result>; struct flat_response_value { -private: - class iterator { - public: - using value_type = resp3::node_view; - using difference_type = std::ptrdiff_t; - using pointer = void; - using reference = value_type; - using iterator_category = std::forward_iterator_tag; - - explicit iterator(flat_response_value* owner, std::size_t i) noexcept - : owner_(owner) - , index_(i) - { } - - value_type operator*() const { return owner_->operator[](index_); } - - iterator& operator++() - { - ++index_; - return *this; - } - - bool operator==(const iterator& other) const { return index_ == other.index_; } - bool operator!=(const iterator& other) const { return !(*this == other); } - - private: - flat_response_value* owner_; - std::size_t index_; - }; - - struct offset_node : resp3::node_view { - std::size_t offset{}; - std::size_t size{}; - }; - public: - void reserve(std::size_t new_cap, std::size_t elem_length) + /// Reserve capacity for nodes and data storage. + void reserve(std::size_t num_nodes, std::size_t string_size) { - data_.reserve(new_cap * elem_length); - view_.reserve(new_cap); + data_.reserve(num_nodes * string_size); + view_.reserve(num_nodes); } - resp3::node_view at(std::size_t index) { return make_node_view(view_.at(index)); } - - std::size_t size() { return view_.size(); } - - resp3::node_view operator[](std::size_t index) { return make_node_view(view_[index]); } - - iterator begin() { return iterator{this, 0}; } - - iterator end() { return iterator{this, view_.size()}; } + std::vector const& view() const { return view_; } + std::vector& view() { return view_; } template void push_back(const resp3::basic_node& nd) { - offset_node new_node; + resp3::offset_string offset_string; + offset_string.offset = data_.size(); + offset_string.size = nd.value.size(); + + data_.append(nd.value.data()); + + offset_string.data = std::string_view{ + data_.data() + offset_string.offset, + offset_string.size}; + + resp3::offset_node new_node; new_node.data_type = nd.data_type; new_node.aggregate_size = nd.aggregate_size; new_node.depth = nd.depth; - new_node.offset = data_.size(); - new_node.size = nd.value.size(); + new_node.value = std::move(offset_string); - data_.append(nd.value.data()); view_.push_back(std::move(new_node)); } private: - resp3::node_view make_node_view(const offset_node& nd) - { - resp3::node_view result; - result.data_type = nd.data_type; - result.aggregate_size = nd.aggregate_size; - result.depth = nd.depth; - result.value = std::string_view{data_.data() + nd.offset, nd.size}; - return result; - } - std::string data_; - std::vector view_; + std::vector view_; }; -/** - * TODO: documentation +/** @brief A memory-efficient generic response to a request. + * @ingroup high-level-api + * + * Uses a compact buffer to store RESP3 data with reduced allocations. */ using generic_flat_response = adapter::result; @@ -159,12 +118,18 @@ using generic_flat_response = adapter::result; */ void consume_one(generic_response& r, system::error_code& ec); -/** - * @brief Throwing overload of `consume_one`. - * - * @param r The response to modify. - */ -void consume_one(generic_response& r); +/// Consume on response from a generic flat response +void consume_one(generic_flat_response& r, system::error_code& ec); + +/// Throwing overload of `consume_one`. +template +void consume_one(Response& r) +{ + system::error_code ec; + consume_one(r, ec); + if (ec) + throw system::system_error(ec); +} } // namespace boost::redis From ecd15732579c5a6d0ed107b0103e2d39ebb8ec66 Mon Sep 17 00:00:00 2001 From: Nikolai Vladimirov Date: Sat, 5 Jul 2025 18:30:04 +0000 Subject: [PATCH 05/37] Call prepare_done function to form response --- include/boost/redis/adapter/any_adapter.hpp | 1 + .../boost/redis/adapter/detail/adapters.hpp | 18 ++++++++++++++++++ include/boost/redis/response.hpp | 16 +++++++++++----- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/include/boost/redis/adapter/any_adapter.hpp b/include/boost/redis/adapter/any_adapter.hpp index a32e25450..0311a28df 100644 --- a/include/boost/redis/adapter/any_adapter.hpp +++ b/include/boost/redis/adapter/any_adapter.hpp @@ -53,6 +53,7 @@ class any_adapter { static auto create_impl(T& resp) -> impl_t { using namespace boost::redis::adapter; + return [adapter2 = boost_redis_adapt(resp)]( any_adapter::parse_event ev, resp3::node_view const& nd, diff --git a/include/boost/redis/adapter/detail/adapters.hpp b/include/boost/redis/adapter/detail/adapters.hpp index 0884632c2..fe97b6770 100644 --- a/include/boost/redis/adapter/detail/adapters.hpp +++ b/include/boost/redis/adapter/detail/adapters.hpp @@ -139,6 +139,24 @@ void boost_redis_from_bulk(T& t, resp3::basic_node const& node, system:: //================================================ +template +auto prepare_done(T&) noexcept -> std::function +{ + return [] { }; +} + +template +auto prepare_done(generic_flat_response& resp) noexcept -> std::function +{ + return [resp]() mutable { + if (resp.has_value()) { + resp.value().set_view(); + } + }; +} + +//================================================ + template class general_aggregate { private: diff --git a/include/boost/redis/response.hpp b/include/boost/redis/response.hpp index a8a8a4141..f236faf3b 100644 --- a/include/boost/redis/response.hpp +++ b/include/boost/redis/response.hpp @@ -50,11 +50,7 @@ struct flat_response_value { offset_string.offset = data_.size(); offset_string.size = nd.value.size(); - data_.append(nd.value.data()); - - offset_string.data = std::string_view{ - data_.data() + offset_string.offset, - offset_string.size}; + data_.append(nd.value.data(), nd.value.size()); resp3::offset_node new_node; new_node.data_type = nd.data_type; @@ -65,6 +61,16 @@ struct flat_response_value { view_.push_back(std::move(new_node)); } + void set_view() + { + for (auto& node : view_) { + auto& offset_string = node.value; + offset_string.data = std::string_view{ + data_.data() + offset_string.offset, + offset_string.size}; + } + } + private: std::string data_; std::vector view_; From 5444e077f9b000a61f321ed5be217e97bf57e586 Mon Sep 17 00:00:00 2001 From: Nikolai Vladimirov Date: Sat, 5 Jul 2025 19:31:23 +0000 Subject: [PATCH 06/37] Defined done_fn_type in adapters.hpp --- include/boost/redis/adapter/detail/adapters.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/include/boost/redis/adapter/detail/adapters.hpp b/include/boost/redis/adapter/detail/adapters.hpp index fe97b6770..5113c41e0 100644 --- a/include/boost/redis/adapter/detail/adapters.hpp +++ b/include/boost/redis/adapter/detail/adapters.hpp @@ -139,14 +139,16 @@ void boost_redis_from_bulk(T& t, resp3::basic_node const& node, system:: //================================================ +using done_fn_type = std::function; + template -auto prepare_done(T&) noexcept -> std::function +auto prepare_done(T&) noexcept -> done_fn_type { return [] { }; } template -auto prepare_done(generic_flat_response& resp) noexcept -> std::function +auto prepare_done(generic_flat_response& resp) noexcept -> done_fn_type { return [resp]() mutable { if (resp.has_value()) { From 1d9f9ab0e3a1819f7dded3da328d656ed688b087 Mon Sep 17 00:00:00 2001 From: Nikolai Vladimirov Date: Sat, 5 Jul 2025 20:18:28 +0000 Subject: [PATCH 07/37] Moved implementation of push_back to general_aggregate --- .../boost/redis/adapter/detail/adapters.hpp | 17 ++++++++++- include/boost/redis/response.hpp | 30 ++++++++----------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/include/boost/redis/adapter/detail/adapters.hpp b/include/boost/redis/adapter/detail/adapters.hpp index 5113c41e0..554a5a23a 100644 --- a/include/boost/redis/adapter/detail/adapters.hpp +++ b/include/boost/redis/adapter/detail/adapters.hpp @@ -218,7 +218,22 @@ class general_aggregate> { std::string{std::cbegin(nd.value), std::cend(nd.value)} }; break; - default: result_->value().push_back(nd); + default: + auto& data = result_->value().data_; + + resp3::offset_string offset_string; + offset_string.offset = data.size(); + offset_string.size = nd.value.size(); + + data.append(nd.value.data(), nd.value.size()); + + resp3::offset_node new_node; + new_node.data_type = nd.data_type; + new_node.aggregate_size = nd.aggregate_size; + new_node.depth = nd.depth; + new_node.value = std::move(offset_string); + + result_->value().view_.push_back(std::move(new_node)); } } }; diff --git a/include/boost/redis/response.hpp b/include/boost/redis/response.hpp index f236faf3b..cd664e71f 100644 --- a/include/boost/redis/response.hpp +++ b/include/boost/redis/response.hpp @@ -31,6 +31,15 @@ using response = std::tuple...>; */ using generic_response = adapter::result>; +/** + * Forward declaration to allow friendship with the template class + * that manages filling of flat_response_value. + */ +namespace adapter::detail { +template +class general_aggregate; +} + struct flat_response_value { public: /// Reserve capacity for nodes and data storage. @@ -43,24 +52,6 @@ struct flat_response_value { std::vector const& view() const { return view_; } std::vector& view() { return view_; } - template - void push_back(const resp3::basic_node& nd) - { - resp3::offset_string offset_string; - offset_string.offset = data_.size(); - offset_string.size = nd.value.size(); - - data_.append(nd.value.data(), nd.value.size()); - - resp3::offset_node new_node; - new_node.data_type = nd.data_type; - new_node.aggregate_size = nd.aggregate_size; - new_node.depth = nd.depth; - new_node.value = std::move(offset_string); - - view_.push_back(std::move(new_node)); - } - void set_view() { for (auto& node : view_) { @@ -72,6 +63,9 @@ struct flat_response_value { } private: + template + friend class adapter::detail::general_aggregate; + std::string data_; std::vector view_; }; From 53e5ae0cd4bca609e818164f6aadaeb3c8f364d6 Mon Sep 17 00:00:00 2001 From: Nikolai Vladimirov Date: Sat, 12 Jul 2025 14:23:42 +0000 Subject: [PATCH 08/37] Avoid turning throwing consume_one into template --- include/boost/redis/impl/response.ipp | 16 ++++++++++++++++ include/boost/redis/response.hpp | 16 +++++++--------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/include/boost/redis/impl/response.ipp b/include/boost/redis/impl/response.ipp index 19194c681..21e7b287a 100644 --- a/include/boost/redis/impl/response.ipp +++ b/include/boost/redis/impl/response.ipp @@ -59,4 +59,20 @@ void consume_one(generic_response& r, system::error_code& ec) { consume_one_impl void consume_one(generic_flat_response& r, system::error_code& ec) { consume_one_impl(r, ec); } +void consume_one(generic_response& r) +{ + system::error_code ec; + consume_one(r, ec); + if (ec) + throw system::system_error(ec); +} + +void consume_one(generic_flat_response& r) +{ + system::error_code ec; + consume_one(r, ec); + if (ec) + throw system::system_error(ec); +} + } // namespace boost::redis diff --git a/include/boost/redis/response.hpp b/include/boost/redis/response.hpp index cd664e71f..9a39d7130 100644 --- a/include/boost/redis/response.hpp +++ b/include/boost/redis/response.hpp @@ -121,15 +121,13 @@ void consume_one(generic_response& r, system::error_code& ec); /// Consume on response from a generic flat response void consume_one(generic_flat_response& r, system::error_code& ec); -/// Throwing overload of `consume_one`. -template -void consume_one(Response& r) -{ - system::error_code ec; - consume_one(r, ec); - if (ec) - throw system::system_error(ec); -} +/** + * @brief Throwing overloads of `consume_one`. + * + * @param r The response to modify. + */ +void consume_one(generic_response& r); +void consume_one(generic_flat_response& r); } // namespace boost::redis From 019a080b651d5e19d1e28fb464d5b9a1f57cf781 Mon Sep 17 00:00:00 2001 From: Nikolai Vladimirov Date: Sat, 12 Jul 2025 15:42:42 +0000 Subject: [PATCH 09/37] Get rid of type erasion in details::prepare_done --- include/boost/redis/adapter/detail/adapters.hpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/include/boost/redis/adapter/detail/adapters.hpp b/include/boost/redis/adapter/detail/adapters.hpp index 554a5a23a..f80391194 100644 --- a/include/boost/redis/adapter/detail/adapters.hpp +++ b/include/boost/redis/adapter/detail/adapters.hpp @@ -139,16 +139,14 @@ void boost_redis_from_bulk(T& t, resp3::basic_node const& node, system:: //================================================ -using done_fn_type = std::function; - template -auto prepare_done(T&) noexcept -> done_fn_type +auto prepare_done(T&) noexcept { return [] { }; } template -auto prepare_done(generic_flat_response& resp) noexcept -> done_fn_type +auto prepare_done(generic_flat_response& resp) noexcept { return [resp]() mutable { if (resp.has_value()) { From 84fa39918fdb517a2192ebf53132be71556b9dad Mon Sep 17 00:00:00 2001 From: Nikolai Vladimirov Date: Sat, 12 Jul 2025 16:22:00 +0000 Subject: [PATCH 10/37] Added API for generic_flat_response, replaced generic_response with new response type in tests --- .../boost/redis/adapter/detail/adapters.hpp | 13 +-- include/boost/redis/resp3/node.hpp | 4 + include/boost/redis/response.hpp | 12 +++ test/test_any_adapter.cpp | 3 + test/test_conn_check_health.cpp | 2 +- test/test_conn_exec.cpp | 2 +- test/test_conn_exec_cancel.cpp | 2 +- test/test_conn_exec_cancel2.cpp | 95 +++++++++++++++++++ test/test_conn_exec_error.cpp | 4 +- 9 files changed, 126 insertions(+), 11 deletions(-) create mode 100644 test/test_conn_exec_cancel2.cpp diff --git a/include/boost/redis/adapter/detail/adapters.hpp b/include/boost/redis/adapter/detail/adapters.hpp index f80391194..fa3fa4824 100644 --- a/include/boost/redis/adapter/detail/adapters.hpp +++ b/include/boost/redis/adapter/detail/adapters.hpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -140,17 +141,17 @@ void boost_redis_from_bulk(T& t, resp3::basic_node const& node, system:: //================================================ template -auto prepare_done(T&) noexcept +inline auto prepare_done(T*) noexcept { return [] { }; } -template -auto prepare_done(generic_flat_response& resp) noexcept +template <> +inline auto prepare_done(generic_flat_response* resp) noexcept { - return [resp]() mutable { - if (resp.has_value()) { - resp.value().set_view(); + return [resp] { + if (resp->has_value()) { + resp->value().set_view(); } }; } diff --git a/include/boost/redis/resp3/node.hpp b/include/boost/redis/resp3/node.hpp index 85147b1cd..34a99aa6a 100644 --- a/include/boost/redis/resp3/node.hpp +++ b/include/boost/redis/resp3/node.hpp @@ -63,8 +63,12 @@ struct offset_string { std::string_view data; std::size_t offset{}; std::size_t size{}; + + operator std::string() const { return std::string{data}; } }; +inline std::ostream& operator<<(std::ostream& os, offset_string const& s) { return os << s.data; } + using offset_node = basic_node; } // namespace boost::redis::resp3 diff --git a/include/boost/redis/response.hpp b/include/boost/redis/response.hpp index 9a39d7130..3053bbaef 100644 --- a/include/boost/redis/response.hpp +++ b/include/boost/redis/response.hpp @@ -49,6 +49,18 @@ struct flat_response_value { view_.reserve(num_nodes); } + void clear() + { + data_.clear(); + view_.clear(); + } + + std::size_t size() const noexcept { return view_.size(); } + bool empty() noexcept { return view_.empty(); } + + resp3::offset_node& at(std::size_t index) { return view_.at(index); } + resp3::offset_node const& at(std::size_t index) const { return view_.at(index); } + std::vector const& view() const { return view_; } std::vector& view() { return view_; } diff --git a/test/test_any_adapter.cpp b/test/test_any_adapter.cpp index f0345ac1e..6d4ee82e3 100644 --- a/test/test_any_adapter.cpp +++ b/test/test_any_adapter.cpp @@ -13,6 +13,7 @@ #include using boost::redis::generic_response; +using boost::redis::generic_flat_response; using boost::redis::response; using boost::redis::ignore; using boost::redis::any_adapter; @@ -24,10 +25,12 @@ BOOST_AUTO_TEST_CASE(any_adapter_response_types) response r1; response r2; generic_response r3; + generic_flat_response r4; BOOST_CHECK_NO_THROW(any_adapter{r1}); BOOST_CHECK_NO_THROW(any_adapter{r2}); BOOST_CHECK_NO_THROW(any_adapter{r3}); + BOOST_CHECK_NO_THROW(any_adapter{r4}); BOOST_CHECK_NO_THROW(any_adapter{ignore}); } diff --git a/test/test_conn_check_health.cpp b/test/test_conn_check_health.cpp index 78062e3b5..d101a52a2 100644 --- a/test/test_conn_check_health.cpp +++ b/test/test_conn_check_health.cpp @@ -262,4 +262,4 @@ int main() test_flexible().run(); return boost::report_errors(); -} \ No newline at end of file +} diff --git a/test/test_conn_exec.cpp b/test/test_conn_exec.cpp index 28a23497b..c2d4fc420 100644 --- a/test/test_conn_exec.cpp +++ b/test/test_conn_exec.cpp @@ -26,7 +26,7 @@ namespace net = boost::asio; using boost::redis::config; using boost::redis::connection; -using boost::redis::generic_response; +using boost::redis::generic_flat_response; using boost::redis::ignore; using boost::redis::operation; using boost::redis::request; diff --git a/test/test_conn_exec_cancel.cpp b/test/test_conn_exec_cancel.cpp index 939cd3617..1870d7842 100644 --- a/test/test_conn_exec_cancel.cpp +++ b/test/test_conn_exec_cancel.cpp @@ -32,7 +32,7 @@ using boost::redis::operation; using boost::redis::error; using boost::redis::request; using boost::redis::response; -using boost::redis::generic_response; +using boost::redis::generic_flat_response; using boost::redis::ignore; using boost::redis::ignore_t; using boost::redis::logger; diff --git a/test/test_conn_exec_cancel2.cpp b/test/test_conn_exec_cancel2.cpp new file mode 100644 index 000000000..083374619 --- /dev/null +++ b/test/test_conn_exec_cancel2.cpp @@ -0,0 +1,95 @@ +/* Copyright (c) 2018-2022 Marcelo Zimbres Silva (mzimbres@gmail.com) + * + * Distributed under the Boost Software License, Version 1.0. (See + * accompanying file LICENSE.txt) + */ + +#include + +#include +#define BOOST_TEST_MODULE conn_exec_cancel +#include + +#include "common.hpp" + +#include + +#ifdef BOOST_ASIO_HAS_CO_AWAIT + +// NOTE1: Sends hello separately. I have observed that if hello and +// blpop are sent toguether, Redis will send the response of hello +// right away, not waiting for blpop. That is why we have to send it +// separately. + +namespace net = boost::asio; +using error_code = boost::system::error_code; +using boost::redis::operation; +using boost::redis::request; +using boost::redis::response; +using boost::redis::generic_flat_response; +using boost::redis::ignore; +using boost::redis::ignore_t; +using boost::redis::config; +using boost::redis::logger; +using boost::redis::connection; +using namespace std::chrono_literals; + +namespace { + +auto async_ignore_explicit_cancel_of_req_written() -> net::awaitable +{ + auto ex = co_await net::this_coro::executor; + + generic_flat_response gresp; + auto conn = std::make_shared(ex); + + run(conn); + + net::steady_timer st{ex}; + st.expires_after(std::chrono::seconds{1}); + + // See NOTE1. + request req0; + req0.push("PING", "async_ignore_explicit_cancel_of_req_written"); + co_await conn->async_exec(req0, gresp); + + request req1; + req1.push("BLPOP", "any", 3); + + bool seen = false; + conn->async_exec(req1, gresp, [&](error_code ec, std::size_t) { + // No error should occur since the cancellation should be ignored + std::cout << "async_exec (1): " << ec.message() << std::endl; + BOOST_TEST(ec == error_code()); + seen = true; + }); + + // Will complete while BLPOP is pending. + error_code ec; + co_await st.async_wait(net::redirect_error(ec)); + conn->cancel(operation::exec); + + BOOST_TEST(ec == error_code()); + + request req2; + req2.push("PING"); + + // Test whether the connection remains usable after a call to + // cancel(exec). + co_await conn->async_exec(req2, gresp, net::redirect_error(ec)); + conn->cancel(); + + BOOST_TEST(ec == error_code()); + BOOST_TEST(seen); +} + +BOOST_AUTO_TEST_CASE(test_ignore_explicit_cancel_of_req_written) +{ + run_coroutine_test(async_ignore_explicit_cancel_of_req_written()); +} + +} // namespace + +#else +BOOST_AUTO_TEST_CASE(dummy) { } +#endif diff --git a/test/test_conn_exec_error.cpp b/test/test_conn_exec_error.cpp index 5a121929d..19638e2d6 100644 --- a/test/test_conn_exec_error.cpp +++ b/test/test_conn_exec_error.cpp @@ -21,7 +21,7 @@ using error_code = boost::system::error_code; using boost::redis::connection; using boost::redis::request; using boost::redis::response; -using boost::redis::generic_response; +using boost::redis::generic_flat_response; using boost::redis::ignore; using boost::redis::ignore_t; using boost::redis::error; @@ -266,7 +266,7 @@ BOOST_AUTO_TEST_CASE(subscriber_wrong_syntax) conn->async_exec(req1, ignore, c1); - generic_response gresp; + generic_flat_response gresp; conn->set_receive_response(gresp); auto c3 = [&](error_code ec, std::size_t) { From b1420d3d1d45335f4ba9979c46848d4fc7ddf211 Mon Sep 17 00:00:00 2001 From: Nikolai Vladimirov Date: Mon, 18 Aug 2025 17:17:47 +0000 Subject: [PATCH 11/37] Adjust code base to new changes --- example/cpp20_chat_room.cpp | 4 +-- example/cpp20_streams.cpp | 6 ++-- .../boost/redis/adapter/detail/adapters.hpp | 29 ++++++------------- include/boost/redis/response.hpp | 2 +- 4 files changed, 15 insertions(+), 26 deletions(-) diff --git a/example/cpp20_chat_room.cpp b/example/cpp20_chat_room.cpp index e765865eb..4a820d521 100644 --- a/example/cpp20_chat_room.cpp +++ b/example/cpp20_chat_room.cpp @@ -33,7 +33,7 @@ using boost::asio::redirect_error; using boost::asio::use_awaitable; using boost::redis::config; using boost::redis::connection; -using boost::redis::generic_response; +using boost::redis::generic_flat_response; using boost::redis::ignore; using boost::redis::request; using boost::system::error_code; @@ -47,7 +47,7 @@ auto receiver(std::shared_ptr conn) -> awaitable request req; req.push("SUBSCRIBE", "channel"); - generic_response resp; + generic_flat_response resp; conn->set_receive_response(resp); while (conn->will_reconnect()) { diff --git a/example/cpp20_streams.cpp b/example/cpp20_streams.cpp index 1cef143f8..14c14087f 100644 --- a/example/cpp20_streams.cpp +++ b/example/cpp20_streams.cpp @@ -23,7 +23,7 @@ namespace net = boost::asio; using boost::redis::config; -using boost::redis::generic_response; +using boost::redis::generic_flat_response; using boost::redis::operation; using boost::redis::request; using boost::redis::connection; @@ -33,7 +33,7 @@ auto stream_reader(std::shared_ptr conn) -> net::awaitable { std::string redisStreamKey_; request req; - generic_response resp; + generic_flat_response resp; std::string stream_id{"$"}; std::string const field = "myfield"; @@ -51,7 +51,7 @@ auto stream_reader(std::shared_ptr conn) -> net::awaitable // The following approach was taken in order to be able to // deal with the responses, as generated by redis in the case // that there are multiple stream 'records' within a single - // generic_response. The nesting and number of values in + // generic_flat_response. The nesting and number of values in // resp.value() are different, depending on the contents // of the stream in redis. Uncomment the above commented-out // code for examples while running the XADD command. diff --git a/include/boost/redis/adapter/detail/adapters.hpp b/include/boost/redis/adapter/detail/adapters.hpp index fa3fa4824..37727cd20 100644 --- a/include/boost/redis/adapter/detail/adapters.hpp +++ b/include/boost/redis/adapter/detail/adapters.hpp @@ -138,26 +138,6 @@ void boost_redis_from_bulk(T& t, resp3::basic_node const& node, system:: from_bulk_impl::apply(t, node, ec); } -//================================================ - -template -inline auto prepare_done(T*) noexcept -{ - return [] { }; -} - -template <> -inline auto prepare_done(generic_flat_response* resp) noexcept -{ - return [resp] { - if (resp->has_value()) { - resp->value().set_view(); - } - }; -} - -//================================================ - template class general_aggregate { private: @@ -205,6 +185,15 @@ class general_aggregate> { explicit general_aggregate(result* c = nullptr) : result_(c) { } + + void on_init() { } + void on_done() + { + if (result_->has_value()) { + result_->value().set_view(); + } + } + template void operator()(resp3::basic_node const& nd, system::error_code&) { diff --git a/include/boost/redis/response.hpp b/include/boost/redis/response.hpp index 3053bbaef..350d864f5 100644 --- a/include/boost/redis/response.hpp +++ b/include/boost/redis/response.hpp @@ -64,6 +64,7 @@ struct flat_response_value { std::vector const& view() const { return view_; } std::vector& view() { return view_; } +private: void set_view() { for (auto& node : view_) { @@ -74,7 +75,6 @@ struct flat_response_value { } } -private: template friend class adapter::detail::general_aggregate; From c2849605494a55dec08a949fefb9aca0911cae30 Mon Sep 17 00:00:00 2001 From: Nikolai Vladimirov Date: Mon, 18 Aug 2025 19:35:48 +0000 Subject: [PATCH 12/37] Replaced generic_response with generic_flat_response in tests --- example/cpp20_subscriber.cpp | 5 +---- test/test_low_level.cpp | 5 +++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/example/cpp20_subscriber.cpp b/example/cpp20_subscriber.cpp index 09536dd17..400178350 100644 --- a/example/cpp20_subscriber.cpp +++ b/example/cpp20_subscriber.cpp @@ -22,7 +22,6 @@ namespace asio = boost::asio; using namespace std::chrono_literals; using boost::redis::request; -using boost::redis::generic_response; using boost::redis::generic_flat_response; using boost::redis::consume_one; using boost::redis::logger; @@ -55,9 +54,7 @@ auto receiver(std::shared_ptr conn) -> asio::awaitable request req; req.push("SUBSCRIBE", "channel"); - // Alternatively, you can use generic_flat_response here, but keep in mind - // that to access elements you need to call .view() on resp.value() - generic_response resp; + generic_flat_response resp; conn->set_receive_response(resp); // Loop while reconnection is enabled diff --git a/test/test_low_level.cpp b/test/test_low_level.cpp index dc68e0ec9..d26a377a0 100644 --- a/test/test_low_level.cpp +++ b/test/test_low_level.cpp @@ -29,6 +29,7 @@ using boost::system::error_code; using boost::redis::request; using boost::redis::response; using boost::redis::generic_response; +using boost::redis::generic_flat_response; using boost::redis::ignore; using boost::redis::ignore_t; using boost::redis::adapter::result; @@ -634,7 +635,7 @@ BOOST_AUTO_TEST_CASE(cancel_one_1) BOOST_AUTO_TEST_CASE(cancel_one_empty) { - generic_response resp; + generic_flat_response resp; BOOST_TEST(resp.has_value()); consume_one(resp); @@ -643,7 +644,7 @@ BOOST_AUTO_TEST_CASE(cancel_one_empty) BOOST_AUTO_TEST_CASE(cancel_one_has_error) { - generic_response resp = boost::redis::adapter::error{resp3::type::simple_string, {}}; + generic_flat_response resp = boost::redis::adapter::error{resp3::type::simple_string, {}}; BOOST_TEST(resp.has_error()); consume_one(resp); From bd799aff963810a56da9136bafb72ec3f2762a60 Mon Sep 17 00:00:00 2001 From: Nikolai Vladimirov Date: Mon, 18 Aug 2025 19:48:09 +0000 Subject: [PATCH 13/37] create flat_response_value::add_node() --- .../boost/redis/adapter/detail/adapters.hpp | 17 +--------------- include/boost/redis/response.hpp | 20 ++++++++++++++++++- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/include/boost/redis/adapter/detail/adapters.hpp b/include/boost/redis/adapter/detail/adapters.hpp index 37727cd20..9c485430e 100644 --- a/include/boost/redis/adapter/detail/adapters.hpp +++ b/include/boost/redis/adapter/detail/adapters.hpp @@ -206,22 +206,7 @@ class general_aggregate> { std::string{std::cbegin(nd.value), std::cend(nd.value)} }; break; - default: - auto& data = result_->value().data_; - - resp3::offset_string offset_string; - offset_string.offset = data.size(); - offset_string.size = nd.value.size(); - - data.append(nd.value.data(), nd.value.size()); - - resp3::offset_node new_node; - new_node.data_type = nd.data_type; - new_node.aggregate_size = nd.aggregate_size; - new_node.depth = nd.depth; - new_node.value = std::move(offset_string); - - result_->value().view_.push_back(std::move(new_node)); + default: result_->value().add_node(nd); } } }; diff --git a/include/boost/redis/response.hpp b/include/boost/redis/response.hpp index 350d864f5..f22e672a2 100644 --- a/include/boost/redis/response.hpp +++ b/include/boost/redis/response.hpp @@ -75,7 +75,25 @@ struct flat_response_value { } } - template + template + void add_node(resp3::basic_node const& nd) + { + resp3::offset_string offset_string; + offset_string.offset = data_.size(); + offset_string.size = nd.value.size(); + + data_.append(nd.value.data(), nd.value.size()); + + resp3::offset_node new_node; + new_node.data_type = nd.data_type; + new_node.aggregate_size = nd.aggregate_size; + new_node.depth = nd.depth; + new_node.value = std::move(offset_string); + + view_.push_back(std::move(new_node)); + } + + template friend class adapter::detail::general_aggregate; std::string data_; From 6ff474008f202e7405c6191ac18a90ebd23c88e1 Mon Sep 17 00:00:00 2001 From: Nikolai Vladimirov Date: Mon, 18 Aug 2025 19:57:51 +0000 Subject: [PATCH 14/37] Cosmetic changes --- include/boost/redis/resp3/node.hpp | 7 +++++-- include/boost/redis/response.hpp | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/include/boost/redis/resp3/node.hpp b/include/boost/redis/resp3/node.hpp index 34a99aa6a..84fb5fca9 100644 --- a/include/boost/redis/resp3/node.hpp +++ b/include/boost/redis/resp3/node.hpp @@ -65,9 +65,12 @@ struct offset_string { std::size_t size{}; operator std::string() const { return std::string{data}; } -}; -inline std::ostream& operator<<(std::ostream& os, offset_string const& s) { return os << s.data; } + friend std::ostream& operator<<(std::ostream& os, offset_string const& s) + { + return os << s.data; + } +}; using offset_node = basic_node; diff --git a/include/boost/redis/response.hpp b/include/boost/redis/response.hpp index f22e672a2..304ca1e65 100644 --- a/include/boost/redis/response.hpp +++ b/include/boost/redis/response.hpp @@ -36,7 +36,7 @@ using generic_response = adapter::result>; * that manages filling of flat_response_value. */ namespace adapter::detail { -template +template class general_aggregate; } From 2c1f1c4c50f41c0ec0993d56e1dc107ebab58e61 Mon Sep 17 00:00:00 2001 From: Marcelo Zimbres Date: Fri, 3 Oct 2025 15:24:09 +0200 Subject: [PATCH 15/37] Concludes the work started by Nikolai Vladimirov on the generic_flat_response --- README.md | 17 +- doc/modules/ROOT/pages/index.adoc | 17 +- example/cpp20_chat_room.cpp | 22 +- example/cpp20_streams.cpp | 6 +- example/cpp20_subscriber.cpp | 34 +- include/boost/redis/adapter/any_adapter.hpp | 1 - .../boost/redis/adapter/detail/adapters.hpp | 62 ++- .../redis/adapter/detail/response_traits.hpp | 27 +- .../redis/adapter/detail/result_traits.hpp | 18 +- include/boost/redis/connection.hpp | 126 +++++- include/boost/redis/error.hpp | 2 +- include/boost/redis/impl/flat_tree.ipp | 125 ++++++ include/boost/redis/impl/response.ipp | 39 +- include/boost/redis/resp3/flat_tree.hpp | 131 ++++++ include/boost/redis/resp3/node.hpp | 24 +- include/boost/redis/resp3/tree.hpp | 29 ++ include/boost/redis/response.hpp | 89 +--- include/boost/redis/src.hpp | 1 + test/CMakeLists.txt | 1 + test/test_any_adapter.cpp | 4 +- test/test_conn_check_health.cpp | 2 +- test/test_conn_echo_stress.cpp | 86 ++-- test/test_conn_exec.cpp | 2 +- test/test_conn_exec_cancel.cpp | 2 +- test/test_conn_exec_cancel2.cpp | 95 ----- test/test_conn_exec_error.cpp | 12 +- test/test_conn_monitor.cpp | 4 +- test/test_conn_push2.cpp | 392 ++++++++++++++++++ test/test_issue_50.cpp | 15 +- test/test_low_level.cpp | 5 +- test/test_low_level_sync_sans_io.cpp | 151 ++++++- 31 files changed, 1135 insertions(+), 406 deletions(-) create mode 100644 include/boost/redis/impl/flat_tree.ipp create mode 100644 include/boost/redis/resp3/flat_tree.hpp create mode 100644 include/boost/redis/resp3/tree.hpp delete mode 100644 test/test_conn_exec_cancel2.cpp create mode 100644 test/test_conn_push2.cpp diff --git a/README.md b/README.md index 9b2db656d..af0b575b7 100644 --- a/README.md +++ b/README.md @@ -87,9 +87,9 @@ them are: * [Client-side caching](https://redis.io/docs/manual/client-side-caching/). The connection class supports server pushes by means of the -`connection::async_receive` function, which can be +`connection::async_receive2` function, which can be called in the same connection that is being used to execute commands. -The coroutine below shows how to use it: +The coroutine below shows how to use it ```cpp @@ -99,26 +99,25 @@ receiver(std::shared_ptr conn) -> net::awaitable request req; req.push("SUBSCRIBE", "channel"); - generic_response resp; + flat_tree resp; conn->set_receive_response(resp); // Loop while reconnection is enabled while (conn->will_reconnect()) { // Reconnect to channels. - co_await conn->async_exec(req, ignore); + co_await conn->async_exec(req); // Loop reading Redis pushes. - for (;;) { - error_code ec; - co_await conn->async_receive(resp, net::redirect_error(net::use_awaitable, ec)); + for (error_code ec;;) { + co_await conn->async_receive2(resp, redirect_error(ec)); if (ec) break; // Connection lost, break so we can reconnect to channels. // Use the response resp in some way and then clear it. ... - consume_one(resp); + resp.clear(); } } } @@ -126,4 +125,4 @@ receiver(std::shared_ptr conn) -> net::awaitable ## Further reading -Full documentation is [here](https://www.boost.org/doc/libs/master/libs/redis/index.html). \ No newline at end of file +Full documentation is [here](https://www.boost.org/doc/libs/master/libs/redis/index.html). diff --git a/doc/modules/ROOT/pages/index.adoc b/doc/modules/ROOT/pages/index.adoc index fe9f04f18..36b83d493 100644 --- a/doc/modules/ROOT/pages/index.adoc +++ b/doc/modules/ROOT/pages/index.adoc @@ -97,9 +97,9 @@ them are: * https://redis.io/docs/manual/client-side-caching/[Client-side caching]. The connection class supports server pushes by means of the -xref:reference:boost/redis/basic_connection/async_receive.adoc[`connection::async_receive`] function, which can be +xref:reference:boost/redis/basic_connection/async_receive.adoc[`connection::async_receive2`] function, which can be called in the same connection that is being used to execute commands. -The coroutine below shows how to use it: +The coroutine below shows how to use it [source,cpp] @@ -110,26 +110,25 @@ receiver(std::shared_ptr conn) -> net::awaitable request req; req.push("SUBSCRIBE", "channel"); - generic_response resp; + flat_tree resp; conn->set_receive_response(resp); // Loop while reconnection is enabled while (conn->will_reconnect()) { // Reconnect to channels. - co_await conn->async_exec(req, ignore); + co_await conn->async_exec(req); // Loop reading Redis pushes. - for (;;) { - error_code ec; - co_await conn->async_receive(resp, net::redirect_error(net::use_awaitable, ec)); + for (error_code ec;;) { + co_await conn->async_receive2(resp, redirect_error(ec)); if (ec) break; // Connection lost, break so we can reconnect to channels. - // Use the response resp in some way and then clear it. + // Use the response here and then clear it. ... - consume_one(resp); + resp.clear(); } } } diff --git a/example/cpp20_chat_room.cpp b/example/cpp20_chat_room.cpp index 4a820d521..89ae78696 100644 --- a/example/cpp20_chat_room.cpp +++ b/example/cpp20_chat_room.cpp @@ -30,11 +30,9 @@ using boost::asio::consign; using boost::asio::detached; using boost::asio::dynamic_buffer; using boost::asio::redirect_error; -using boost::asio::use_awaitable; using boost::redis::config; using boost::redis::connection; -using boost::redis::generic_flat_response; -using boost::redis::ignore; +using boost::redis::resp3::flat_tree; using boost::redis::request; using boost::system::error_code; using namespace std::chrono_literals; @@ -47,21 +45,25 @@ auto receiver(std::shared_ptr conn) -> awaitable request req; req.push("SUBSCRIBE", "channel"); - generic_flat_response resp; + flat_tree resp; conn->set_receive_response(resp); while (conn->will_reconnect()) { // Subscribe to channels. - co_await conn->async_exec(req, ignore); + co_await conn->async_exec(req); // Loop reading Redis push messages. for (error_code ec;;) { - co_await conn->async_receive(redirect_error(use_awaitable, ec)); + co_await conn->async_receive2(redirect_error(ec)); if (ec) break; // Connection lost, break so we can reconnect to channels. - std::cout << resp.value().at(1).value << " " << resp.value().at(2).value << " " - << resp.value().at(3).value << std::endl; - resp.value().clear(); + + for (auto const& elem: resp.get_view()) + std::cout << elem.value << "\n"; + + std::cout << std::endl; + + resp.clear(); } } } @@ -74,7 +76,7 @@ auto publisher(std::shared_ptr in, std::shared_ptrasync_exec(req, ignore); + co_await conn->async_exec(req); msg.erase(0, n); } } diff --git a/example/cpp20_streams.cpp b/example/cpp20_streams.cpp index 14c14087f..1cef143f8 100644 --- a/example/cpp20_streams.cpp +++ b/example/cpp20_streams.cpp @@ -23,7 +23,7 @@ namespace net = boost::asio; using boost::redis::config; -using boost::redis::generic_flat_response; +using boost::redis::generic_response; using boost::redis::operation; using boost::redis::request; using boost::redis::connection; @@ -33,7 +33,7 @@ auto stream_reader(std::shared_ptr conn) -> net::awaitable { std::string redisStreamKey_; request req; - generic_flat_response resp; + generic_response resp; std::string stream_id{"$"}; std::string const field = "myfield"; @@ -51,7 +51,7 @@ auto stream_reader(std::shared_ptr conn) -> net::awaitable // The following approach was taken in order to be able to // deal with the responses, as generated by redis in the case // that there are multiple stream 'records' within a single - // generic_flat_response. The nesting and number of values in + // generic_response. The nesting and number of values in // resp.value() are different, depending on the contents // of the stream in redis. Uncomment the above commented-out // code for examples while running the XADD command. diff --git a/example/cpp20_subscriber.cpp b/example/cpp20_subscriber.cpp index 400178350..531fdaf9b 100644 --- a/example/cpp20_subscriber.cpp +++ b/example/cpp20_subscriber.cpp @@ -1,11 +1,10 @@ -/* Copyright (c) 2018-2022 Marcelo Zimbres Silva (mzimbres@gmail.com) +/* Copyright (c) 2018-2025 Marcelo Zimbres Silva (mzimbres@gmail.com) * * Distributed under the Boost Software License, Version 1.0. (See * accompanying file LICENSE.txt) */ #include -#include #include #include @@ -22,12 +21,8 @@ namespace asio = boost::asio; using namespace std::chrono_literals; using boost::redis::request; -using boost::redis::generic_flat_response; -using boost::redis::consume_one; -using boost::redis::logger; +using boost::redis::resp3::flat_tree; using boost::redis::config; -using boost::redis::ignore; -using boost::redis::error; using boost::system::error_code; using boost::redis::connection; using asio::signal_set; @@ -54,30 +49,29 @@ auto receiver(std::shared_ptr conn) -> asio::awaitable request req; req.push("SUBSCRIBE", "channel"); - generic_flat_response resp; + flat_tree resp; conn->set_receive_response(resp); // Loop while reconnection is enabled while (conn->will_reconnect()) { // Reconnect to the channels. - co_await conn->async_exec(req, ignore); + co_await conn->async_exec(req); - // Loop reading Redis pushs messages. + // Loop to read Redis push messages. for (error_code ec;;) { - // First tries to read any buffered pushes. - conn->receive(ec); - if (ec == error::sync_receive_push_failed) { - ec = {}; - co_await conn->async_receive(asio::redirect_error(asio::use_awaitable, ec)); - } - + // Wait for pushes + co_await conn->async_receive2(asio::redirect_error(ec)); if (ec) break; // Connection lost, break so we can reconnect to channels. - std::cout << resp.value().at(1).value << " " << resp.value().at(2).value << " " - << resp.value().at(3).value << std::endl; + // The response must be consumed without suspending the + // coroutine i.e. without the use of async operations. + for (auto const& elem: resp.get_view()) + std::cout << elem.value << "\n"; + + std::cout << std::endl; - consume_one(resp); + resp.clear(); } } } diff --git a/include/boost/redis/adapter/any_adapter.hpp b/include/boost/redis/adapter/any_adapter.hpp index 0311a28df..a32e25450 100644 --- a/include/boost/redis/adapter/any_adapter.hpp +++ b/include/boost/redis/adapter/any_adapter.hpp @@ -53,7 +53,6 @@ class any_adapter { static auto create_impl(T& resp) -> impl_t { using namespace boost::redis::adapter; - return [adapter2 = boost_redis_adapt(resp)]( any_adapter::parse_event ev, resp3::node_view const& nd, diff --git a/include/boost/redis/adapter/detail/adapters.hpp b/include/boost/redis/adapter/detail/adapters.hpp index 9c485430e..432fa9bd4 100644 --- a/include/boost/redis/adapter/detail/adapters.hpp +++ b/include/boost/redis/adapter/detail/adapters.hpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2024 Marcelo Zimbres Silva (mzimbres@gmail.com) +/* Copyright (c) 2018-2025 Marcelo Zimbres Silva (mzimbres@gmail.com) * * Distributed under the Boost Software License, Version 1.0. (See * accompanying file LICENSE.txt) @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -138,6 +137,8 @@ void boost_redis_from_bulk(T& t, resp3::basic_node const& node, system:: from_bulk_impl::apply(t, node, ec); } +//================================================ + template class general_aggregate { private: @@ -177,37 +178,54 @@ class general_aggregate { }; template <> -class general_aggregate> { +class general_aggregate { private: - result* result_; + resp3::tree* tree_ = nullptr; public: - explicit general_aggregate(result* c = nullptr) - : result_(c) + explicit general_aggregate(resp3::tree* c = nullptr) + : tree_(c) + { } + + void on_init() { } + void on_done() { } + + template + void on_node(resp3::basic_node const& nd, system::error_code&) + { + BOOST_ASSERT_MSG(!!tree_, "Unexpected null pointer"); + + resp3::node tmp; + tmp.data_type = nd.data_type; + tmp.aggregate_size = nd.aggregate_size; + tmp.depth = nd.depth; + tmp.value = std::string{std::cbegin(nd.value), std::cend(nd.value)}; + + tree_->push_back(std::move(tmp)); + } +}; + +template <> +class general_aggregate { +private: + resp3::flat_tree* tree_ = nullptr; + +public: + explicit general_aggregate(resp3::flat_tree* c = nullptr) + : tree_(c) { } void on_init() { } void on_done() { - if (result_->has_value()) { - result_->value().set_view(); - } + tree_->notify_done(); } template - void operator()(resp3::basic_node const& nd, system::error_code&) + void on_node(resp3::basic_node const& nd, system::error_code&) { - BOOST_ASSERT_MSG(!!result_, "Unexpected null pointer"); - switch (nd.data_type) { - case resp3::type::blob_error: - case resp3::type::simple_error: - *result_ = error{ - nd.data_type, - std::string{std::cbegin(nd.value), std::cend(nd.value)} - }; - break; - default: result_->value().add_node(nd); - } + BOOST_ASSERT_MSG(!!tree_, "Unexpected null pointer"); + tree_->push(nd); } }; diff --git a/include/boost/redis/adapter/detail/response_traits.hpp b/include/boost/redis/adapter/detail/response_traits.hpp index 1ab2c5893..493858127 100644 --- a/include/boost/redis/adapter/detail/response_traits.hpp +++ b/include/boost/redis/adapter/detail/response_traits.hpp @@ -92,8 +92,24 @@ struct response_traits> { }; template -struct response_traits, Allocator>>> { - using response_type = result, Allocator>>; +struct response_traits>> { + using response_type = result>; + using adapter_type = general_aggregate; + + static auto adapt(response_type& v) noexcept { return adapter_type{&v}; } +}; + +template +struct response_traits> { + using response_type = resp3::basic_tree; + using adapter_type = general_aggregate; + + static auto adapt(response_type& v) noexcept { return adapter_type{&v}; } +}; + +template <> +struct response_traits { + using response_type = resp3::flat_tree; using adapter_type = general_aggregate; static auto adapt(response_type& v) noexcept { return adapter_type{&v}; } @@ -107,13 +123,6 @@ struct response_traits> { static auto adapt(response_type& r) noexcept { return adapter_type{r}; } }; -template <> -struct response_traits { - using response_type = generic_flat_response; - using adapter_type = vector_adapter; - - static auto adapt(response_type& v) noexcept { return adapter_type{v}; } -}; } // namespace boost::redis::adapter::detail #endif // BOOST_REDIS_ADAPTER_DETAIL_RESPONSE_TRAITS_HPP diff --git a/include/boost/redis/adapter/detail/result_traits.hpp b/include/boost/redis/adapter/detail/result_traits.hpp index 2e158a399..671318197 100644 --- a/include/boost/redis/adapter/detail/result_traits.hpp +++ b/include/boost/redis/adapter/detail/result_traits.hpp @@ -13,7 +13,8 @@ #include #include #include -#include +#include +#include #include @@ -57,19 +58,26 @@ struct result_traits>> { }; template -struct result_traits, Allocator>>> { +struct result_traits>> { using response_type = result, Allocator>>; using adapter_type = adapter::detail::general_aggregate; static auto adapt(response_type& v) noexcept { return adapter_type{&v}; } }; -template <> -struct result_traits { - using response_type = generic_flat_response; +template +struct result_traits> { + using response_type = resp3::basic_tree; using adapter_type = adapter::detail::general_aggregate; static auto adapt(response_type& v) noexcept { return adapter_type{&v}; } }; +template <> +struct result_traits { + using response_type = resp3::flat_tree; + using adapter_type = general_aggregate; + static auto adapt(response_type& v) noexcept { return adapter_type{&v}; } +}; + template using adapter_t = typename result_traits>::adapter_type; diff --git a/include/boost/redis/connection.hpp b/include/boost/redis/connection.hpp index e83a89462..25791feaa 100644 --- a/include/boost/redis/connection.hpp +++ b/include/boost/redis/connection.hpp @@ -207,6 +207,50 @@ struct connection_impl { { st_.mpx.set_receive_adapter(std::move(adapter)); } + + std::size_t receive(system::error_code& ec) + { + std::size_t size = 0; + + auto f = [&](system::error_code const& ec2, std::size_t n) { + ec = ec2; + size = n; + }; + + auto const res = receive_channel_.try_receive(f); + if (ec) + return 0; + + if (!res) + ec = error::sync_receive_push_failed; + + return size; + } + + template + auto async_receive2(CompletionToken&& token) + { + return + receive_channel_.async_receive( + asio::deferred( + [this](system::error_code ec, std::size_t) + { + if (!ec) { + auto f = [](system::error_code, std::size_t) { + // There is no point in checking for errors + // here since async_receive just completed + // without errors. + }; + + // We just want to drain the channel. + while (receive_channel_.try_receive(f)); + } + + return asio::deferred.values(ec); + } + ) + )(std::forward(token)); + } }; template @@ -593,7 +637,7 @@ class basic_connection { return async_run(config{}, std::forward(token)); } - /** @brief Receives server side pushes asynchronously. + /** @brief (Deprecated) Receives server side pushes asynchronously. * * When pushes arrive and there is no `async_receive` operation in * progress, pushed data, requests, and responses will be paused @@ -623,12 +667,57 @@ class basic_connection { * @param token Completion token. */ template > + BOOST_DEPRECATED("Please use async_receive2 instead.") auto async_receive(CompletionToken&& token = {}) { return impl_->receive_channel_.async_receive(std::forward(token)); } - /** @brief Receives server pushes synchronously without blocking. + /** @brief Wait for server pushes asynchronously + * + * This function suspends until a server push is received by the + * connection. On completion an unspecified number of pushes will + * have been added to the response object set with @ref + * boost::redis::connection::set_receive_response. + * + * To prevent receiving an unbound number of pushes the connection + * blocks further read operations on the socket when 256 pushes + * accumulate internally (we don't make any commitment to this + * exact number). When that happens any `async_exec`s and + * health-checks won't make any progress and the connection may + * eventually timeout. To avoid that Apps should call + * `async_receive2` continuously in a loop. + * + * @Note To avoid deadlocks the task (e.g. coroutine) calling + * `async_receive2` should not call `async_exec` in a way where + * they could block each other. + * + * For an example see cpp20_subscriber.cpp. The completion token + * must have the following signature + * + * @code + * void f(system::error_code); + * @endcode + * + * @par Per-operation cancellation + * This operation supports the following cancellation types: + * + * @li `asio::cancellation_type_t::terminal`. + * @li `asio::cancellation_type_t::partial`. + * @li `asio::cancellation_type_t::total`. + * + * Calling `basic_connection::cancel(operation::receive)` will + * also cancel any ongoing receive operations. + * + * @param token Completion token. + */ + template > + auto async_receive2(CompletionToken&& token = {}) + { + return impl_->async_receive2(std::forward(token)); + } + + /** @brief (Deprecated) Receives server pushes synchronously without blocking. * * Receives a server push synchronously by calling `try_receive` on * the underlying channel. If the operation fails because @@ -638,23 +727,10 @@ class basic_connection { * @param ec Contains the error if any occurred. * @returns The number of bytes read from the socket. */ + BOOST_DEPRECATED("Please, use async_receive2 instead.") std::size_t receive(system::error_code& ec) { - std::size_t size = 0; - - auto f = [&](system::error_code const& ec2, std::size_t n) { - ec = ec2; - size = n; - }; - - auto const res = impl_->receive_channel_.try_receive(f); - if (ec) - return 0; - - if (!res) - ec = error::sync_receive_push_failed; - - return size; + return impl_->receive(ec); } /** @brief Executes commands on the Redis server asynchronously. @@ -837,7 +913,7 @@ class basic_connection { "the other member functions to interact with the connection.") auto const& next_layer() const noexcept { return impl_->stream_.next_layer(); } - /// Sets the response object of @ref async_receive operations. + /// Sets the response object of @ref async_receive2 operations. template void set_receive_response(Response& resp) { @@ -1028,13 +1104,25 @@ class connection { /// @copydoc basic_connection::async_receive template + BOOST_DEPRECATED("Please use async_receive2 instead.") auto async_receive(CompletionToken&& token = {}) { return impl_.async_receive(std::forward(token)); } + /// @copydoc basic_connection::async_receive2 + template + auto async_receive2(CompletionToken&& token = {}) + { + return impl_.async_receive2(std::forward(token)); + } + /// @copydoc basic_connection::receive - std::size_t receive(system::error_code& ec) { return impl_.receive(ec); } + BOOST_DEPRECATED("Please use async_receive2 instead.") + std::size_t receive(system::error_code& ec) + { + return impl_.impl_->receive(ec); + } /** * @brief Calls @ref boost::redis::basic_connection::async_exec. diff --git a/include/boost/redis/error.hpp b/include/boost/redis/error.hpp index 506d0139d..f39699f37 100644 --- a/include/boost/redis/error.hpp +++ b/include/boost/redis/error.hpp @@ -74,7 +74,7 @@ enum class error /// SSL handshake timeout ssl_handshake_timeout, - /// Can't receive push synchronously without blocking + /// (Deprecated) Can't receive push synchronously without blocking sync_receive_push_failed, /// Incompatible node depth. diff --git a/include/boost/redis/impl/flat_tree.ipp b/include/boost/redis/impl/flat_tree.ipp new file mode 100644 index 000000000..48be5040c --- /dev/null +++ b/include/boost/redis/impl/flat_tree.ipp @@ -0,0 +1,125 @@ +// +// Copyright (c) 2025 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Nikolai Vladimirov (nvladimirov.work@gmail.com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#include +#include + +namespace boost::redis::resp3 { + +flat_tree::flat_tree(flat_tree const& other) +: data_{other.data_} +, view_tree_{other.view_tree_} +, ranges_{other.ranges_} +, pos_{0u} +, reallocs_{0u} +, total_msgs_{other.total_msgs_} +{ + view_tree_.resize(ranges_.size()); + set_views(); +} + +flat_tree& +flat_tree::operator=(flat_tree other) +{ + swap(*this, other); + return *this; +} + +void flat_tree::reserve(std::size_t bytes, std::size_t nodes) +{ + data_.reserve(bytes); + view_tree_.reserve(nodes); + ranges_.reserve(nodes); +} + +void flat_tree::clear() +{ + pos_ = 0u; + total_msgs_ = 0u; + reallocs_ = 0u; + data_.clear(); + view_tree_.clear(); + ranges_.clear(); +} + +void flat_tree::set_views() +{ + BOOST_ASSERT_MSG(pos_ < view_tree_.size(), "notify_done called but no nodes added."); + BOOST_ASSERT_MSG(view_tree_.size() == ranges_.size(), "Incompatible sizes."); + + for (; pos_ < view_tree_.size(); ++pos_) { + auto const& r = ranges_.at(pos_); + view_tree_.at(pos_).value = std::string_view{data_.data() + r.offset, r.size}; + } +} + +void flat_tree::notify_done() +{ + total_msgs_ += 1; + set_views(); +} + +void flat_tree::push(node_view const& node) +{ + auto data_before = data_.data(); + add_node_impl(node); + auto data_after = data_.data(); + + if (data_after != data_before) { + pos_ = 0; + reallocs_ += 1; + } +} + +void flat_tree::add_node_impl(node_view const& node) +{ + ranges_.push_back({data_.size(), node.value.size()}); + + // This must come after setting the offset above. + data_.append(node.value.data(), node.value.size()); + + view_tree_.push_back(node); +} + +void swap(flat_tree& a, flat_tree& b) +{ + using std::swap; + + swap(a.data_, b.data_); + swap(a.view_tree_, b.view_tree_); + swap(a.ranges_, b.ranges_); + swap(a.pos_, b.pos_); + swap(a.reallocs_, b.reallocs_); + swap(a.total_msgs_, b.total_msgs_); +} + +bool +operator==( + flat_tree::range const& a, + flat_tree::range const& b) +{ + return a.offset == b.offset && a.size == b.size; +} + +bool operator==(flat_tree const& a, flat_tree const& b) +{ + return + a.data_ == b.data_ && + a.view_tree_ == b.view_tree_ && + a.ranges_ == b.ranges_ && + a.pos_ == b.pos_ && + //a.reallocs_ == b.reallocs_ && + a.total_msgs_ == b.total_msgs_; +} + +bool operator!=(flat_tree const& a, flat_tree const& b) +{ + return !(a == b); +} + +} // namespace boost::redis diff --git a/include/boost/redis/impl/response.ipp b/include/boost/redis/impl/response.ipp index 21e7b287a..a4b09a6e7 100644 --- a/include/boost/redis/impl/response.ipp +++ b/include/boost/redis/impl/response.ipp @@ -11,30 +11,15 @@ namespace boost::redis { -namespace { -template -auto& get_value(Container& c) -{ - return c; -} - -template <> -auto& get_value(flat_response_value& c) -{ - return c.view(); -} - -template -void consume_one_impl(Response& r, system::error_code& ec) +void consume_one(generic_response& r, system::error_code& ec) { if (r.has_error()) return; // Nothing to consume. - auto& value = get_value(r.value()); - if (std::empty(value)) + if (std::empty(r.value())) return; // Nothing to consume. - auto const depth = value.front().depth; + auto const depth = r.value().front().depth; // To simplify we will refuse to consume any data-type that is not // a root node. I think there is no use for that and it is complex @@ -48,17 +33,11 @@ void consume_one_impl(Response& r, system::error_code& ec) return e.depth == depth; }; - auto match = std::find_if(std::next(std::cbegin(value)), std::cend(value), f); + auto match = std::find_if(std::next(std::cbegin(r.value())), std::cend(r.value()), f); - value.erase(std::cbegin(value), match); + r.value().erase(std::cbegin(r.value()), match); } -} // namespace - -void consume_one(generic_response& r, system::error_code& ec) { consume_one_impl(r, ec); } - -void consume_one(generic_flat_response& r, system::error_code& ec) { consume_one_impl(r, ec); } - void consume_one(generic_response& r) { system::error_code ec; @@ -67,12 +46,4 @@ void consume_one(generic_response& r) throw system::system_error(ec); } -void consume_one(generic_flat_response& r) -{ - system::error_code ec; - consume_one(r, ec); - if (ec) - throw system::system_error(ec); -} - } // namespace boost::redis diff --git a/include/boost/redis/resp3/flat_tree.hpp b/include/boost/redis/resp3/flat_tree.hpp new file mode 100644 index 000000000..90b232143 --- /dev/null +++ b/include/boost/redis/resp3/flat_tree.hpp @@ -0,0 +1,131 @@ +// +// Copyright (c) 2025 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Nikolai Vladimirov (nvladimirov.work@gmail.com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#ifndef BOOST_REDIS_RESP3_FLAT_TREE_HPP +#define BOOST_REDIS_RESP3_FLAT_TREE_HPP + +#include +#include + +#include +#include + +namespace boost::redis { + +namespace adapter::detail { + template class general_aggregate; +} + +namespace resp3 { + +/** @brief A generic-response that stores data contiguously + * + * Similar to the @ref boost::redis::resp3::tree but data is + * stored contiguously. + */ +struct flat_tree { +public: + /// Default constructor + flat_tree() = default; + + /// Move constructor + flat_tree(flat_tree&&) noexcept = default; + + /// Copy constructor + flat_tree(flat_tree const& other); + + /// Copy assignment + flat_tree& operator=(flat_tree other); + + friend void swap(flat_tree&, flat_tree&); + + friend + bool operator==(flat_tree const&, flat_tree const&); + + friend + bool operator!=(flat_tree const&, flat_tree const&); + + /** @brief Reserve capacity + * + * Reserve memory for incoming data. + * + * @param bytes Number of bytes to reserve for data. + * @param nodes Number of nodes to reserve. + */ + void reserve(std::size_t bytes, std::size_t nodes); + + /** @brief Clear both the data and the node buffers + * + * @Note: A `boost::redis:.flat_tree` can contain the + * response to multiple Redis commands and server pushes. Calling + * this function will erase everything contained in it. + */ + void clear(); + + /// Returns the size of the data buffer + auto data_size() const noexcept -> std::size_t + { return data_.size(); } + + /// Returns the RESP3 response + auto get_view() const -> view_tree const& + { return view_tree_; } + + /** @brief Returns the number of times reallocation took place + * + * This function returns how many reallocations were performed and + * can be useful to determine how much memory to reserve upfront. + */ + auto get_reallocs() const noexcept -> std::size_t + { return reallocs_; } + + /// Returns the number of complete RESP3 messages contained in this object. + std::size_t get_total_msgs() const noexcept + { return total_msgs_; } + +private: + template friend class adapter::detail::general_aggregate; + + // Notify the object that all nodes were pushed. + void notify_done(); + + // Push a new node to the response + void push(node_view const& node); + + void add_node_impl(node_view const& node); + + void set_views(); + + // Range into the data buffer. + struct range { + std::size_t offset; + std::size_t size; + }; + + friend bool operator==(range const&, range const&); + + std::string data_; + view_tree view_tree_; + std::vector ranges_; + std::size_t pos_ = 0u; + std::size_t reallocs_ = 0u; + std::size_t total_msgs_ = 0u; +}; + +/// Swaps two responses +void swap(flat_tree&, flat_tree&); + +/// Equality operator +bool operator==(flat_tree const&, flat_tree const&); + +/// Inequality operator +bool operator!=(flat_tree const&, flat_tree const&); + +} // resp3 +} // namespace boost::redis + +#endif // BOOST_REDIS_RESP3_FLAT_TREE_HPP diff --git a/include/boost/redis/resp3/node.hpp b/include/boost/redis/resp3/node.hpp index 84fb5fca9..1e284dab3 100644 --- a/include/boost/redis/resp3/node.hpp +++ b/include/boost/redis/resp3/node.hpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2024 Marcelo Zimbres Silva (mzimbres@gmail.com) +/* Copyright (c) 2018-2025 Marcelo Zimbres Silva (mzimbres@gmail.com) * * Distributed under the Boost Software License, Version 1.0. (See * accompanying file LICENSE.txt) @@ -53,27 +53,19 @@ auto operator==(basic_node const& a, basic_node const& b) // clang-format on }; +/// Inequality operator for RESP3 nodes +template +auto operator!=(basic_node const& a, basic_node const& b) +{ + return !(a == b); +}; + /// A node in the response tree that owns its data. using node = basic_node; /// A node in the response tree that does not own its data. using node_view = basic_node; -struct offset_string { - std::string_view data; - std::size_t offset{}; - std::size_t size{}; - - operator std::string() const { return std::string{data}; } - - friend std::ostream& operator<<(std::ostream& os, offset_string const& s) - { - return os << s.data; - } -}; - -using offset_node = basic_node; - } // namespace boost::redis::resp3 #endif // BOOST_REDIS_RESP3_NODE_HPP diff --git a/include/boost/redis/resp3/tree.hpp b/include/boost/redis/resp3/tree.hpp new file mode 100644 index 000000000..fa4f4c66b --- /dev/null +++ b/include/boost/redis/resp3/tree.hpp @@ -0,0 +1,29 @@ +/* Copyright (c) 2018-2025 Marcelo Zimbres Silva (mzimbres@gmail.com) + * + * Distributed under the Boost Software License, Version 1.0. (See + * accompanying file LICENSE.txt) + */ + +#ifndef BOOST_REDIS_RESP3_TREE_HPP +#define BOOST_REDIS_RESP3_TREE_HPP + +#include + +#include +#include + +namespace boost::redis::resp3 { + +/// A RESP3 tree that owns its data. +template >> +using basic_tree = std::vector, Allocator>; + +/// A RESP3 tree that owns its data. +using tree = basic_tree; + +/// A RESP3 tree whose data are `std::string_views`. +using view_tree = basic_tree; + +} + +#endif // BOOST_REDIS_RESP3_RESPONSE_HPP diff --git a/include/boost/redis/response.hpp b/include/boost/redis/response.hpp index 304ca1e65..5622f3a83 100644 --- a/include/boost/redis/response.hpp +++ b/include/boost/redis/response.hpp @@ -9,6 +9,7 @@ #include #include +#include #include @@ -29,85 +30,9 @@ using response = std::tuple...>; * [pre-order](https://en.wikipedia.org/wiki/Tree_traversal#Pre-order,_NLR) * view of the response tree. */ -using generic_response = adapter::result>; +using generic_response = adapter::result; -/** - * Forward declaration to allow friendship with the template class - * that manages filling of flat_response_value. - */ -namespace adapter::detail { -template -class general_aggregate; -} - -struct flat_response_value { -public: - /// Reserve capacity for nodes and data storage. - void reserve(std::size_t num_nodes, std::size_t string_size) - { - data_.reserve(num_nodes * string_size); - view_.reserve(num_nodes); - } - - void clear() - { - data_.clear(); - view_.clear(); - } - - std::size_t size() const noexcept { return view_.size(); } - bool empty() noexcept { return view_.empty(); } - - resp3::offset_node& at(std::size_t index) { return view_.at(index); } - resp3::offset_node const& at(std::size_t index) const { return view_.at(index); } - - std::vector const& view() const { return view_; } - std::vector& view() { return view_; } - -private: - void set_view() - { - for (auto& node : view_) { - auto& offset_string = node.value; - offset_string.data = std::string_view{ - data_.data() + offset_string.offset, - offset_string.size}; - } - } - - template - void add_node(resp3::basic_node const& nd) - { - resp3::offset_string offset_string; - offset_string.offset = data_.size(); - offset_string.size = nd.value.size(); - - data_.append(nd.value.data(), nd.value.size()); - - resp3::offset_node new_node; - new_node.data_type = nd.data_type; - new_node.aggregate_size = nd.aggregate_size; - new_node.depth = nd.depth; - new_node.value = std::move(offset_string); - - view_.push_back(std::move(new_node)); - } - - template - friend class adapter::detail::general_aggregate; - - std::string data_; - std::vector view_; -}; - -/** @brief A memory-efficient generic response to a request. - * @ingroup high-level-api - * - * Uses a compact buffer to store RESP3 data with reduced allocations. - */ -using generic_flat_response = adapter::result; - -/** @brief Consume on response from a generic response +/** @brief (Deprecated) Consume on response from a generic response * * This function rotates the elements so that the start of the next * response becomes the new front element. For example the output of @@ -146,18 +71,16 @@ using generic_flat_response = adapter::result; * @param r The response to modify. * @param ec Will be populated in case of error. */ +//BOOST_DEPRECATED("This function is not needed anymore to consume server pushes.") void consume_one(generic_response& r, system::error_code& ec); -/// Consume on response from a generic flat response -void consume_one(generic_flat_response& r, system::error_code& ec); - /** - * @brief Throwing overloads of `consume_one`. + * @brief (Deprecated) Throwing overload of `consume_one`. * * @param r The response to modify. */ +//BOOST_DEPRECATED("This function is not needed anymore to consume server pushes.") void consume_one(generic_response& r); -void consume_one(generic_flat_response& r); } // namespace boost::redis diff --git a/include/boost/redis/src.hpp b/include/boost/redis/src.hpp index 093344215..06c2dcd3b 100644 --- a/include/boost/redis/src.hpp +++ b/include/boost/redis/src.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 78a65ee16..4a9bf03c2 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -56,6 +56,7 @@ make_test(test_conn_run_cancel) make_test(test_conn_check_health) make_test(test_conn_exec) make_test(test_conn_push) +make_test(test_conn_push2) make_test(test_conn_monitor) make_test(test_conn_reconnect) make_test(test_conn_exec_cancel) diff --git a/test/test_any_adapter.cpp b/test/test_any_adapter.cpp index 6d4ee82e3..ca123a49a 100644 --- a/test/test_any_adapter.cpp +++ b/test/test_any_adapter.cpp @@ -13,7 +13,7 @@ #include using boost::redis::generic_response; -using boost::redis::generic_flat_response; +using boost::redis::resp3::flat_tree; using boost::redis::response; using boost::redis::ignore; using boost::redis::any_adapter; @@ -25,7 +25,7 @@ BOOST_AUTO_TEST_CASE(any_adapter_response_types) response r1; response r2; generic_response r3; - generic_flat_response r4; + flat_tree r4; BOOST_CHECK_NO_THROW(any_adapter{r1}); BOOST_CHECK_NO_THROW(any_adapter{r2}); diff --git a/test/test_conn_check_health.cpp b/test/test_conn_check_health.cpp index d101a52a2..78062e3b5 100644 --- a/test/test_conn_check_health.cpp +++ b/test/test_conn_check_health.cpp @@ -262,4 +262,4 @@ int main() test_flexible().run(); return boost::report_errors(); -} +} \ No newline at end of file diff --git a/test/test_conn_echo_stress.cpp b/test/test_conn_echo_stress.cpp index 5b37489d4..48157b0be 100644 --- a/test/test_conn_echo_stress.cpp +++ b/test/test_conn_echo_stress.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2022 Marcelo Zimbres Silva (mzimbres@gmail.com) +/* Copyright (c) 2018-2025 Marcelo Zimbres Silva (mzimbres@gmail.com) * * Distributed under the Boost Software License, Version 1.0. (See * accompanying file LICENSE.txt) @@ -27,6 +27,7 @@ using error_code = boost::system::error_code; using boost::redis::operation; using boost::redis::request; using boost::redis::response; +using boost::redis::resp3::flat_tree; using boost::redis::ignore; using boost::redis::ignore_t; using boost::redis::logger; @@ -54,39 +55,44 @@ std::ostream& operator<<(std::ostream& os, usage const& u) namespace { -auto push_consumer(connection& conn, int expected) -> net::awaitable +auto +receiver( + connection& conn, + flat_tree& resp, + std::size_t expected) -> net::awaitable { - int c = 0; - for (error_code ec;;) { - conn.receive(ec); - if (ec == error::sync_receive_push_failed) { - ec = {}; - co_await conn.async_receive(net::redirect_error(ec)); - } else if (!ec) { - //std::cout << "Skipping suspension." << std::endl; - } - - if (ec) { - BOOST_TEST(false, "push_consumer error: " << ec.message()); - co_return; - } - if (++c == expected) - break; + std::size_t push_counter = 0; + while (push_counter != expected) { + co_await conn.async_receive2(); + push_counter += resp.get_total_msgs(); + resp.clear(); } conn.cancel(); } -auto echo_session(connection& conn, const request& pubs, int n) -> net::awaitable +auto echo_session(connection& conn, const request& req, std::size_t n) -> net::awaitable { - for (auto i = 0; i < n; ++i) - co_await conn.async_exec(pubs); + for (auto i = 0u; i < n; ++i) + co_await conn.async_exec(req); } void rethrow_on_error(std::exception_ptr exc) { - if (exc) + if (exc) { + BOOST_TEST(false); std::rethrow_exception(exc); + } +} + +request make_pub_req(std::size_t n_pubs) +{ + request req; + req.push("PING"); + for (std::size_t i = 0u; i < n_pubs; ++i) + req.push("PUBLISH", "channel", "payload"); + + return req; } BOOST_AUTO_TEST_CASE(echo_stress) @@ -98,22 +104,22 @@ BOOST_AUTO_TEST_CASE(echo_stress) // Number of coroutines that will send pings sharing the same // connection to redis. - constexpr int sessions = 150; + constexpr std::size_t sessions = 150u; // The number of pings that will be sent by each session. - constexpr int msgs = 200; + constexpr std::size_t msgs = 200u; // The number of publishes that will be sent by each session with // each message. - constexpr int n_pubs = 25; + constexpr std::size_t n_pubs = 25u; // This is the total number of pushes we will receive. - constexpr int total_pushes = sessions * msgs * n_pubs + 1; + constexpr std::size_t total_pushes = sessions * msgs * n_pubs + 1; + + flat_tree resp; + conn.set_receive_response(resp); - request pubs; - pubs.push("PING"); - for (int i = 0; i < n_pubs; ++i) - pubs.push("PUBLISH", "channel", "payload"); + request const pub_req = make_pub_req(n_pubs); // Run the connection bool run_finished = false, subscribe_finished = false; @@ -123,6 +129,10 @@ BOOST_AUTO_TEST_CASE(echo_stress) std::clog << "async_run finished" << std::endl; }); + // Op that will consume the pushes counting down until all expected + // pushes have been received. + net::co_spawn(ctx, receiver(conn, resp, total_pushes), rethrow_on_error); + // Subscribe, then launch the coroutines request req; req.push("SUBSCRIBE", "channel"); @@ -130,12 +140,8 @@ BOOST_AUTO_TEST_CASE(echo_stress) subscribe_finished = true; BOOST_TEST(ec == error_code()); - // Op that will consume the pushes counting down until all expected - // pushes have been received. - net::co_spawn(ctx, push_consumer(conn, total_pushes), rethrow_on_error); - - for (int i = 0; i < sessions; ++i) - net::co_spawn(ctx, echo_session(conn, pubs, msgs), rethrow_on_error); + for (std::size_t i = 0; i < sessions; ++i) + net::co_spawn(ctx, echo_session(conn, pub_req, msgs), rethrow_on_error); }); // Run the test @@ -144,7 +150,13 @@ BOOST_AUTO_TEST_CASE(echo_stress) BOOST_TEST(subscribe_finished); // Print statistics - std::cout << "-------------------\n" << conn.get_usage() << std::endl; + std::cout + << "-------------------\n" + << "Usage data: \n" + << conn.get_usage() << "\n" + << "-------------------\n" + << "Reallocations: " << resp.get_reallocs() + << std::endl; } } // namespace diff --git a/test/test_conn_exec.cpp b/test/test_conn_exec.cpp index c2d4fc420..28a23497b 100644 --- a/test/test_conn_exec.cpp +++ b/test/test_conn_exec.cpp @@ -26,7 +26,7 @@ namespace net = boost::asio; using boost::redis::config; using boost::redis::connection; -using boost::redis::generic_flat_response; +using boost::redis::generic_response; using boost::redis::ignore; using boost::redis::operation; using boost::redis::request; diff --git a/test/test_conn_exec_cancel.cpp b/test/test_conn_exec_cancel.cpp index 1870d7842..939cd3617 100644 --- a/test/test_conn_exec_cancel.cpp +++ b/test/test_conn_exec_cancel.cpp @@ -32,7 +32,7 @@ using boost::redis::operation; using boost::redis::error; using boost::redis::request; using boost::redis::response; -using boost::redis::generic_flat_response; +using boost::redis::generic_response; using boost::redis::ignore; using boost::redis::ignore_t; using boost::redis::logger; diff --git a/test/test_conn_exec_cancel2.cpp b/test/test_conn_exec_cancel2.cpp deleted file mode 100644 index 083374619..000000000 --- a/test/test_conn_exec_cancel2.cpp +++ /dev/null @@ -1,95 +0,0 @@ -/* Copyright (c) 2018-2022 Marcelo Zimbres Silva (mzimbres@gmail.com) - * - * Distributed under the Boost Software License, Version 1.0. (See - * accompanying file LICENSE.txt) - */ - -#include - -#include -#define BOOST_TEST_MODULE conn_exec_cancel -#include - -#include "common.hpp" - -#include - -#ifdef BOOST_ASIO_HAS_CO_AWAIT - -// NOTE1: Sends hello separately. I have observed that if hello and -// blpop are sent toguether, Redis will send the response of hello -// right away, not waiting for blpop. That is why we have to send it -// separately. - -namespace net = boost::asio; -using error_code = boost::system::error_code; -using boost::redis::operation; -using boost::redis::request; -using boost::redis::response; -using boost::redis::generic_flat_response; -using boost::redis::ignore; -using boost::redis::ignore_t; -using boost::redis::config; -using boost::redis::logger; -using boost::redis::connection; -using namespace std::chrono_literals; - -namespace { - -auto async_ignore_explicit_cancel_of_req_written() -> net::awaitable -{ - auto ex = co_await net::this_coro::executor; - - generic_flat_response gresp; - auto conn = std::make_shared(ex); - - run(conn); - - net::steady_timer st{ex}; - st.expires_after(std::chrono::seconds{1}); - - // See NOTE1. - request req0; - req0.push("PING", "async_ignore_explicit_cancel_of_req_written"); - co_await conn->async_exec(req0, gresp); - - request req1; - req1.push("BLPOP", "any", 3); - - bool seen = false; - conn->async_exec(req1, gresp, [&](error_code ec, std::size_t) { - // No error should occur since the cancellation should be ignored - std::cout << "async_exec (1): " << ec.message() << std::endl; - BOOST_TEST(ec == error_code()); - seen = true; - }); - - // Will complete while BLPOP is pending. - error_code ec; - co_await st.async_wait(net::redirect_error(ec)); - conn->cancel(operation::exec); - - BOOST_TEST(ec == error_code()); - - request req2; - req2.push("PING"); - - // Test whether the connection remains usable after a call to - // cancel(exec). - co_await conn->async_exec(req2, gresp, net::redirect_error(ec)); - conn->cancel(); - - BOOST_TEST(ec == error_code()); - BOOST_TEST(seen); -} - -BOOST_AUTO_TEST_CASE(test_ignore_explicit_cancel_of_req_written) -{ - run_coroutine_test(async_ignore_explicit_cancel_of_req_written()); -} - -} // namespace - -#else -BOOST_AUTO_TEST_CASE(dummy) { } -#endif diff --git a/test/test_conn_exec_error.cpp b/test/test_conn_exec_error.cpp index 19638e2d6..b77f12997 100644 --- a/test/test_conn_exec_error.cpp +++ b/test/test_conn_exec_error.cpp @@ -21,7 +21,7 @@ using error_code = boost::system::error_code; using boost::redis::connection; using boost::redis::request; using boost::redis::response; -using boost::redis::generic_flat_response; +using boost::redis::generic_response; using boost::redis::ignore; using boost::redis::ignore_t; using boost::redis::error; @@ -266,12 +266,12 @@ BOOST_AUTO_TEST_CASE(subscriber_wrong_syntax) conn->async_exec(req1, ignore, c1); - generic_flat_response gresp; + generic_response gresp; conn->set_receive_response(gresp); - auto c3 = [&](error_code ec, std::size_t) { + auto c3 = [&](error_code ec) { c3_called = true; - std::cout << "async_receive" << std::endl; + std::cout << "async_receive2" << std::endl; BOOST_TEST(!ec); BOOST_TEST(gresp.has_error()); BOOST_CHECK_EQUAL(gresp.error().data_type, resp3::type::simple_error); @@ -281,7 +281,7 @@ BOOST_AUTO_TEST_CASE(subscriber_wrong_syntax) conn->cancel(operation::reconnection); }; - conn->async_receive(c3); + conn->async_receive2(c3); run(conn); @@ -326,4 +326,4 @@ BOOST_AUTO_TEST_CASE(issue_287_generic_response_error_then_success) BOOST_TEST(resp.error().diagnostic == "ERR wrong number of arguments for 'set' command"); } -} // namespace \ No newline at end of file +} // namespace diff --git a/test/test_conn_monitor.cpp b/test/test_conn_monitor.cpp index 24d22ee11..090c54557 100644 --- a/test/test_conn_monitor.cpp +++ b/test/test_conn_monitor.cpp @@ -41,7 +41,7 @@ class test_monitor { void start_receive() { - conn.async_receive([this](error_code ec, std::size_t) { + conn.async_receive2([this](error_code ec) { // We should expect one push entry, at least BOOST_TEST_EQ(ec, error_code()); BOOST_TEST(monitor_resp.has_value()); @@ -118,4 +118,4 @@ int main() test_monitor{}.run(); return boost::report_errors(); -} \ No newline at end of file +} diff --git a/test/test_conn_push2.cpp b/test/test_conn_push2.cpp new file mode 100644 index 000000000..9e3d7f623 --- /dev/null +++ b/test/test_conn_push2.cpp @@ -0,0 +1,392 @@ +/* Copyright (c) 2018-2025 Marcelo Zimbres Silva (mzimbres@gmail.com) + * + * Distributed under the Boost Software License, Version 1.0. (See + * accompanying file LICENSE.txt) + */ + +#include +#include +#include +#include + +#include +#include + +#include + +#define BOOST_TEST_MODULE conn_push +#include + +#include "common.hpp" + +#include +#include + +namespace net = boost::asio; +namespace redis = boost::redis; + +using boost::redis::operation; +using boost::redis::connection; +using boost::system::error_code; +using boost::redis::request; +using boost::redis::response; +using boost::redis::resp3::flat_tree; +using boost::redis::ignore; +using boost::redis::ignore_t; +using boost::system::error_code; +using boost::redis::logger; +using namespace std::chrono_literals; + +namespace { + +BOOST_AUTO_TEST_CASE(receives_push_waiting_resps) +{ + request req1; + req1.push("HELLO", 3); + req1.push("PING", "Message1"); + + request req2; + req2.push("SUBSCRIBE", "channel"); + + request req3; + req3.push("PING", "Message2"); + req3.push("QUIT"); + + net::io_context ioc; + + auto conn = std::make_shared(ioc); + + bool push_received = false, c1_called = false, c2_called = false, c3_called = false; + + auto c3 = [&](error_code ec, std::size_t) { + c3_called = true; + std::cout << "c3: " << ec.message() << std::endl; + }; + + auto c2 = [&, conn](error_code ec, std::size_t) { + c2_called = true; + BOOST_TEST(ec == error_code()); + conn->async_exec(req3, ignore, c3); + }; + + auto c1 = [&, conn](error_code ec, std::size_t) { + c1_called = true; + BOOST_TEST(ec == error_code()); + conn->async_exec(req2, ignore, c2); + }; + + conn->async_exec(req1, ignore, c1); + + run(conn, make_test_config(), {}); + + conn->async_receive2([&, conn](error_code ec) { + std::cout << "async_receive2" << std::endl; + BOOST_TEST(ec == error_code()); + push_received = true; + conn->cancel(); + }); + + ioc.run_for(test_timeout); + + BOOST_TEST(push_received); + BOOST_TEST(c1_called); + BOOST_TEST(c2_called); + BOOST_TEST(c3_called); +} + +BOOST_AUTO_TEST_CASE(push_received1) +{ + net::io_context ioc; + auto conn = std::make_shared(ioc); + + flat_tree resp; + conn->set_receive_response(resp); + + // Trick: Uses SUBSCRIBE because this command has no response or + // better said, its response is a server push, which is what we + // want to test. + request req; + req.push("SUBSCRIBE", "channel1"); + req.push("SUBSCRIBE", "channel2"); + + bool push_received = false, exec_finished = false; + + conn->async_exec(req, ignore, [&, conn](error_code ec, std::size_t) { + exec_finished = true; + std::cout << "async_exec" << std::endl; + BOOST_TEST(ec == error_code()); + }); + + conn->async_receive2([&, conn](error_code ec) { + push_received = true; + std::cout << "async_receive2" << std::endl; + + BOOST_TEST(ec == error_code()); + + BOOST_CHECK_EQUAL(resp.get_total_msgs(), 2u); + + conn->cancel(); + }); + + run(conn); + ioc.run_for(test_timeout); + + BOOST_TEST(exec_finished); + BOOST_TEST(push_received); +} + +BOOST_AUTO_TEST_CASE(push_filtered_out) +{ + net::io_context ioc; + auto conn = std::make_shared(ioc); + + request req; + req.push("HELLO", 3); + req.push("PING"); + req.push("SUBSCRIBE", "channel"); + req.push("QUIT"); + + response resp; + + bool exec_finished = false, push_received = false; + + conn->async_exec(req, resp, [conn, &exec_finished](error_code ec, std::size_t) { + exec_finished = true; + BOOST_TEST(ec == error_code()); + }); + + conn->async_receive2([&, conn](error_code ec) { + push_received = true; + BOOST_TEST(ec == error_code()); + conn->cancel(operation::reconnection); + }); + + run(conn); + + ioc.run_for(test_timeout); + BOOST_TEST(exec_finished); + BOOST_TEST(push_received); + + BOOST_CHECK_EQUAL(std::get<1>(resp).value(), "PONG"); + BOOST_CHECK_EQUAL(std::get<2>(resp).value(), "OK"); +} + +struct response_error_tag { }; +response_error_tag error_tag_obj; + +struct response_error_adapter { + void on_init() { } + void on_done() { } + + void on_node( + boost::redis::resp3::basic_node const&, + boost::system::error_code& ec) + { + ec = boost::redis::error::incompatible_size; + } +}; + +auto boost_redis_adapt(response_error_tag&) { return response_error_adapter{}; } + +BOOST_AUTO_TEST_CASE(test_push_adapter) +{ + net::io_context ioc; + auto conn = std::make_shared(ioc); + + request req; + req.push("HELLO", 3); + req.push("PING"); + req.push("SUBSCRIBE", "channel"); + req.push("PING"); + + conn->set_receive_response(error_tag_obj); + + bool push_received = false, exec_finished = false, run_finished = false; + + conn->async_receive2([&, conn](error_code ec) { + BOOST_CHECK_EQUAL(ec, boost::asio::experimental::error::channel_cancelled); + push_received = true; + }); + + conn->async_exec(req, ignore, [&exec_finished](error_code ec, std::size_t) { + BOOST_CHECK_EQUAL(ec, boost::system::errc::errc_t::operation_canceled); + exec_finished = true; + }); + + auto cfg = make_test_config(); + cfg.reconnect_wait_interval = 0s; + conn->async_run(cfg, [&run_finished](error_code ec) { + BOOST_CHECK_EQUAL(ec, redis::error::incompatible_size); + run_finished = true; + }); + + ioc.run_for(test_timeout); + BOOST_TEST(push_received); + BOOST_TEST(exec_finished); + BOOST_TEST(run_finished); + + // TODO: Reset the ioc reconnect and send a quit to ensure + // reconnection is possible after an error. +} + +void launch_push_consumer(std::shared_ptr conn) +{ + conn->async_receive2([conn](error_code ec) { + if (ec) { + BOOST_TEST(ec == net::experimental::error::channel_cancelled); + return; + } + launch_push_consumer(conn); + }); +} + +BOOST_AUTO_TEST_CASE(many_subscribers) +{ + request req0; + req0.get_config().cancel_on_connection_lost = false; + req0.push("HELLO", 3); + + request req1; + req1.get_config().cancel_on_connection_lost = false; + req1.push("PING", "Message1"); + + request req2; + req2.get_config().cancel_on_connection_lost = false; + req2.push("SUBSCRIBE", "channel"); + + request req3; + req3.get_config().cancel_on_connection_lost = false; + req3.push("QUIT"); + + net::io_context ioc; + auto conn = std::make_shared(ioc); + + bool finished = false; + + auto c11 = [&](error_code ec, std::size_t) { + BOOST_TEST(ec == error_code()); + conn->cancel(operation::reconnection); + finished = true; + }; + auto c10 = [&](error_code ec, std::size_t) { + BOOST_TEST(ec == error_code()); + conn->async_exec(req3, ignore, c11); + }; + auto c9 = [&](error_code ec, std::size_t) { + BOOST_TEST(ec == error_code()); + conn->async_exec(req2, ignore, c10); + }; + auto c8 = [&](error_code ec, std::size_t) { + BOOST_TEST(ec == error_code()); + conn->async_exec(req1, ignore, c9); + }; + auto c7 = [&](error_code ec, std::size_t) { + BOOST_TEST(ec == error_code()); + conn->async_exec(req2, ignore, c8); + }; + auto c6 = [&](error_code ec, std::size_t) { + BOOST_TEST(ec == error_code()); + conn->async_exec(req2, ignore, c7); + }; + auto c5 = [&](error_code ec, std::size_t) { + BOOST_TEST(ec == error_code()); + conn->async_exec(req1, ignore, c6); + }; + auto c4 = [&](error_code ec, std::size_t) { + BOOST_TEST(ec == error_code()); + conn->async_exec(req2, ignore, c5); + }; + auto c3 = [&](error_code ec, std::size_t) { + BOOST_TEST(ec == error_code()); + conn->async_exec(req1, ignore, c4); + }; + auto c2 = [&](error_code ec, std::size_t) { + BOOST_TEST(ec == error_code()); + conn->async_exec(req2, ignore, c3); + }; + auto c1 = [&](error_code ec, std::size_t) { + BOOST_TEST(ec == error_code()); + conn->async_exec(req2, ignore, c2); + }; + auto c0 = [&](error_code ec, std::size_t) { + BOOST_TEST(ec == error_code()); + conn->async_exec(req1, ignore, c1); + }; + + conn->async_exec(req0, ignore, c0); + launch_push_consumer(conn); + + run(conn, make_test_config(), {}); + + ioc.run_for(test_timeout); + BOOST_TEST(finished); +} + +BOOST_AUTO_TEST_CASE(test_unsubscribe) +{ + net::io_context ioc; + connection conn{ioc}; + + // Subscribe to 3 channels and 2 patterns. Use CLIENT INFO to verify this took effect + request req_subscribe; + req_subscribe.push("SUBSCRIBE", "ch1", "ch2", "ch3"); + req_subscribe.push("PSUBSCRIBE", "ch1*", "ch2*"); + req_subscribe.push("CLIENT", "INFO"); + + // Then, unsubscribe from some of them, and verify again + request req_unsubscribe; + req_unsubscribe.push("UNSUBSCRIBE", "ch1"); + req_unsubscribe.push("PUNSUBSCRIBE", "ch2*"); + req_unsubscribe.push("CLIENT", "INFO"); + + // Finally, ping to verify that the connection is still usable + request req_ping; + req_ping.push("PING", "test_unsubscribe"); + + response resp_subscribe, resp_unsubscribe, resp_ping; + + bool subscribe_finished = false, unsubscribe_finished = false, ping_finished = false, + run_finished = false; + + auto on_ping = [&](error_code ec, std::size_t) { + BOOST_TEST(ec == error_code()); + ping_finished = true; + BOOST_TEST(std::get<0>(resp_ping).has_value()); + BOOST_TEST(std::get<0>(resp_ping).value() == "test_unsubscribe"); + conn.cancel(); + }; + + auto on_unsubscribe = [&](error_code ec, std::size_t) { + unsubscribe_finished = true; + BOOST_TEST(ec == error_code()); + BOOST_TEST(std::get<0>(resp_unsubscribe).has_value()); + BOOST_TEST(find_client_info(std::get<0>(resp_unsubscribe).value(), "sub") == "2"); + BOOST_TEST(find_client_info(std::get<0>(resp_unsubscribe).value(), "psub") == "1"); + conn.async_exec(req_ping, resp_ping, on_ping); + }; + + auto on_subscribe = [&](error_code ec, std::size_t) { + subscribe_finished = true; + BOOST_TEST(ec == error_code()); + BOOST_TEST(std::get<0>(resp_subscribe).has_value()); + BOOST_TEST(find_client_info(std::get<0>(resp_subscribe).value(), "sub") == "3"); + BOOST_TEST(find_client_info(std::get<0>(resp_subscribe).value(), "psub") == "2"); + conn.async_exec(req_unsubscribe, resp_unsubscribe, on_unsubscribe); + }; + + conn.async_exec(req_subscribe, resp_subscribe, on_subscribe); + + conn.async_run(make_test_config(), [&run_finished](error_code ec) { + BOOST_TEST(ec == net::error::operation_aborted); + run_finished = true; + }); + + ioc.run_for(test_timeout); + + BOOST_TEST(subscribe_finished); + BOOST_TEST(unsubscribe_finished); + BOOST_TEST(ping_finished); + BOOST_TEST(run_finished); +} + +} // namespace diff --git a/test/test_issue_50.cpp b/test/test_issue_50.cpp index 98a4c76be..00c04cad8 100644 --- a/test/test_issue_50.cpp +++ b/test/test_issue_50.cpp @@ -42,16 +42,15 @@ namespace { // Push consumer auto receiver(std::shared_ptr conn) -> net::awaitable { - std::cout << "uuu" << std::endl; + std::cout << "Entering receiver" << std::endl; while (conn->will_reconnect()) { - std::cout << "dddd" << std::endl; - // Loop reading Redis pushs messages. - for (;;) { - std::cout << "aaaa" << std::endl; - error_code ec; - co_await conn->async_receive(net::redirect_error(ec)); + std::cout << "Reconnect loop" << std::endl; + // Loop reading Redis pushes. + for (error_code ec;;) { + std::cout << "Receive loop" << std::endl; + co_await conn->async_receive2(net::redirect_error(ec)); if (ec) { - std::cout << "Error in async_receive" << std::endl; + std::cout << "Error in async_receive2" << std::endl; break; } } diff --git a/test/test_low_level.cpp b/test/test_low_level.cpp index d26a377a0..dc68e0ec9 100644 --- a/test/test_low_level.cpp +++ b/test/test_low_level.cpp @@ -29,7 +29,6 @@ using boost::system::error_code; using boost::redis::request; using boost::redis::response; using boost::redis::generic_response; -using boost::redis::generic_flat_response; using boost::redis::ignore; using boost::redis::ignore_t; using boost::redis::adapter::result; @@ -635,7 +634,7 @@ BOOST_AUTO_TEST_CASE(cancel_one_1) BOOST_AUTO_TEST_CASE(cancel_one_empty) { - generic_flat_response resp; + generic_response resp; BOOST_TEST(resp.has_value()); consume_one(resp); @@ -644,7 +643,7 @@ BOOST_AUTO_TEST_CASE(cancel_one_empty) BOOST_AUTO_TEST_CASE(cancel_one_has_error) { - generic_flat_response resp = boost::redis::adapter::error{resp3::type::simple_string, {}}; + generic_response resp = boost::redis::adapter::error{resp3::type::simple_string, {}}; BOOST_TEST(resp.has_error()); consume_one(resp); diff --git a/test/test_low_level_sync_sans_io.cpp b/test/test_low_level_sync_sans_io.cpp index 5ec3e92fb..800d304f9 100644 --- a/test/test_low_level_sync_sans_io.cpp +++ b/test/test_low_level_sync_sans_io.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2024 Marcelo Zimbres Silva (mzimbres@gmail.com) +/* Copyright (c) 2018-2025 Marcelo Zimbres Silva (mzimbres@gmail.com) * * Distributed under the Boost Software License, Version 1.0. (See * accompanying file LICENSE.txt) @@ -22,15 +22,19 @@ using boost::redis::request; using boost::redis::adapter::adapt2; using boost::redis::adapter::result; -using boost::redis::generic_response; +using boost::redis::resp3::tree; +using boost::redis::resp3::flat_tree; using boost::redis::ignore_t; using boost::redis::resp3::detail::deserialize; using boost::redis::resp3::node; +using boost::redis::resp3::node_view; using boost::redis::resp3::to_string; using boost::redis::response; using boost::redis::any_adapter; using boost::system::error_code; +namespace resp3 = boost::redis::resp3; + #define RESP3_SET_PART1 "~6\r\n+orange\r" #define RESP3_SET_PART2 "\n+apple\r\n+one" #define RESP3_SET_PART3 "\r\n+two\r" @@ -42,7 +46,9 @@ BOOST_AUTO_TEST_CASE(low_level_sync_sans_io) try { result> resp; - deserialize(resp3_set, adapt2(resp)); + error_code ec; + deserialize(resp3_set, adapt2(resp), ec); + BOOST_CHECK_EQUAL(ec, error_code{}); for (auto const& e : resp.value()) std::cout << e << std::endl; @@ -65,7 +71,9 @@ BOOST_AUTO_TEST_CASE(issue_210_empty_set) char const* wire = "*4\r\n:1\r\n~0\r\n$25\r\nthis_should_not_be_in_set\r\n:2\r\n"; - deserialize(wire, adapt2(resp)); + error_code ec; + deserialize(wire, adapt2(resp), ec); + BOOST_CHECK_EQUAL(ec, error_code{}); BOOST_CHECK_EQUAL(std::get<0>(resp.value()).value(), 1); BOOST_CHECK(std::get<1>(resp.value()).value().empty()); @@ -91,7 +99,9 @@ BOOST_AUTO_TEST_CASE(issue_210_non_empty_set_size_one) char const* wire = "*4\r\n:1\r\n~1\r\n$3\r\nfoo\r\n$25\r\nthis_should_not_be_in_set\r\n:2\r\n"; - deserialize(wire, adapt2(resp)); + error_code ec; + deserialize(wire, adapt2(resp), ec); + BOOST_CHECK_EQUAL(ec, error_code{}); BOOST_CHECK_EQUAL(std::get<0>(resp.value()).value(), 1); BOOST_CHECK_EQUAL(std::get<1>(resp.value()).value().size(), 1u); @@ -118,7 +128,9 @@ BOOST_AUTO_TEST_CASE(issue_210_non_empty_set_size_two) char const* wire = "*4\r\n:1\r\n~2\r\n$3\r\nfoo\r\n$3\r\nbar\r\n$25\r\nthis_should_not_be_in_set\r\n:2\r\n"; - deserialize(wire, adapt2(resp)); + error_code ec; + deserialize(wire, adapt2(resp), ec); + BOOST_CHECK_EQUAL(ec, error_code{}); BOOST_CHECK_EQUAL(std::get<0>(resp.value()).value(), 1); BOOST_CHECK_EQUAL(std::get<1>(resp.value()).value().at(0), std::string{"foo"}); @@ -140,7 +152,9 @@ BOOST_AUTO_TEST_CASE(issue_210_no_nested) char const* wire = "*4\r\n:1\r\n$3\r\nfoo\r\n$3\r\nbar\r\n$25\r\nthis_should_not_be_in_set\r\n"; - deserialize(wire, adapt2(resp)); + error_code ec; + deserialize(wire, adapt2(resp), ec); + BOOST_CHECK_EQUAL(ec, error_code{}); BOOST_CHECK_EQUAL(std::get<0>(resp.value()).value(), 1); BOOST_CHECK_EQUAL(std::get<1>(resp.value()).value(), std::string{"foo"}); @@ -159,7 +173,10 @@ BOOST_AUTO_TEST_CASE(issue_233_array_with_null) result>> resp; char const* wire = "*3\r\n+one\r\n_\r\n+two\r\n"; - deserialize(wire, adapt2(resp)); + + error_code ec; + deserialize(wire, adapt2(resp), ec); + BOOST_CHECK_EQUAL(ec, error_code{}); BOOST_CHECK_EQUAL(resp.value().at(0).value(), "one"); BOOST_TEST(!resp.value().at(1).has_value()); @@ -177,7 +194,10 @@ BOOST_AUTO_TEST_CASE(issue_233_optional_array_with_null) result>>> resp; char const* wire = "*3\r\n+one\r\n_\r\n+two\r\n"; - deserialize(wire, adapt2(resp)); + + error_code ec; + deserialize(wire, adapt2(resp), ec); + BOOST_CHECK_EQUAL(ec, error_code{}); BOOST_CHECK_EQUAL(resp.value().value().at(0).value(), "one"); BOOST_TEST(!resp.value().value().at(1).has_value()); @@ -313,3 +333,116 @@ BOOST_AUTO_TEST_CASE(check_counter_adapter) BOOST_CHECK_EQUAL(node, 7); BOOST_CHECK_EQUAL(done, 1); } + +namespace boost::redis::resp3 { + +template +std::ostream& operator<<(std::ostream& os, basic_node const& nd) +{ + os << "type: " << to_string(nd.data_type) << "\n" + << "aggregate_size: " << nd.aggregate_size << "\n" + << "depth: " << nd.depth << "\n" + << "value: " << nd.value << "\n"; + return os; +} + +template +std::ostream& operator<<(std::ostream& os, basic_tree const& resp) +{ + for (auto const& e: resp) + os << e << ","; + return os; +} + +} + +node from_node_view(node_view const& v) +{ + node ret; + ret.data_type = v.data_type; + ret.aggregate_size = v.aggregate_size; + ret.depth = v.depth; + ret.value = v.value; + return ret; +} + +tree from_flat(flat_tree const& resp) +{ + tree ret; + for (auto const& e: resp.get_view()) + ret.push_back(from_node_view(e)); + + return ret; +} + +// Parses the same data into a tree and a +// flat_tree, they should be equal to each other. +BOOST_AUTO_TEST_CASE(flat_tree_views_are_set) +{ + tree resp1; + flat_tree fresp; + + error_code ec; + deserialize(resp3_set, adapt2(resp1), ec); + BOOST_CHECK_EQUAL(ec, error_code{}); + + deserialize(resp3_set, adapt2(fresp), ec); + BOOST_CHECK_EQUAL(ec, error_code{}); + + BOOST_CHECK_EQUAL(fresp.get_reallocs(), 1u); + BOOST_CHECK_EQUAL(fresp.get_total_msgs(), 1u); + + auto const resp2 = from_flat(fresp); + BOOST_CHECK_EQUAL(resp1, resp2); +} + +// The response should be reusable. +BOOST_AUTO_TEST_CASE(flat_tree_reuse) +{ + flat_tree tmp; + + // First use + error_code ec; + deserialize(resp3_set, adapt2(tmp), ec); + BOOST_CHECK_EQUAL(ec, error_code{}); + + BOOST_CHECK_EQUAL(tmp.get_reallocs(), 1u); + BOOST_CHECK_EQUAL(tmp.get_total_msgs(), 1u); + + // Copy to compare after the reuse. + auto const resp1 = tmp.get_view(); + tmp.clear(); + + // Second use + deserialize(resp3_set, adapt2(tmp), ec); + BOOST_CHECK_EQUAL(ec, error_code{}); + + // No reallocation this time + BOOST_CHECK_EQUAL(tmp.get_reallocs(), 0u); + BOOST_CHECK_EQUAL(tmp.get_total_msgs(), 1u); + + BOOST_CHECK_EQUAL(resp1, tmp.get_view()); +} + +BOOST_AUTO_TEST_CASE(flat_tree_copy_assign) +{ + flat_tree resp; + + error_code ec; + deserialize(resp3_set, adapt2(resp), ec); + BOOST_CHECK_EQUAL(ec, error_code{}); + + // Copy + resp3::flat_tree copy1{resp}; + + // Copy assignment + resp3::flat_tree copy2 = resp; + + // Assignment + resp3::flat_tree copy3; + copy3 = resp; + + BOOST_TEST((copy1 == resp)); + BOOST_TEST((copy2 == resp)); + BOOST_TEST((copy3 == resp)); +} From b365b96228ab3ddc88d23232f58b8eb7b0a37c5f Mon Sep 17 00:00:00 2001 From: Marcelo Zimbres Date: Mon, 10 Nov 2025 23:21:16 +0100 Subject: [PATCH 16/37] Fixup Adds generic_flat_response typedef --- example/cpp20_chat_room.cpp | 10 +-- example/cpp20_subscriber.cpp | 8 +-- .../boost/redis/adapter/detail/adapters.hpp | 40 +++++++++++ .../redis/adapter/detail/response_traits.hpp | 8 +++ .../redis/adapter/detail/result_traits.hpp | 7 ++ include/boost/redis/response.hpp | 4 ++ test/test_low_level_sync_sans_io.cpp | 69 +++++++++++++++++-- 7 files changed, 131 insertions(+), 15 deletions(-) diff --git a/example/cpp20_chat_room.cpp b/example/cpp20_chat_room.cpp index 89ae78696..562225f56 100644 --- a/example/cpp20_chat_room.cpp +++ b/example/cpp20_chat_room.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2022 Marcelo Zimbres Silva (mzimbres@gmail.com) +/* Copyright (c) 2018-2025 Marcelo Zimbres Silva (mzimbres@gmail.com) * * Distributed under the Boost Software License, Version 1.0. (See * accompanying file LICENSE.txt) @@ -32,7 +32,7 @@ using boost::asio::dynamic_buffer; using boost::asio::redirect_error; using boost::redis::config; using boost::redis::connection; -using boost::redis::resp3::flat_tree; +using boost::redis::generic_flat_response; using boost::redis::request; using boost::system::error_code; using namespace std::chrono_literals; @@ -45,7 +45,7 @@ auto receiver(std::shared_ptr conn) -> awaitable request req; req.push("SUBSCRIBE", "channel"); - flat_tree resp; + generic_flat_response resp; conn->set_receive_response(resp); while (conn->will_reconnect()) { @@ -58,12 +58,12 @@ auto receiver(std::shared_ptr conn) -> awaitable if (ec) break; // Connection lost, break so we can reconnect to channels. - for (auto const& elem: resp.get_view()) + for (auto const& elem: resp.value().get_view()) std::cout << elem.value << "\n"; std::cout << std::endl; - resp.clear(); + resp.value().clear(); } } } diff --git a/example/cpp20_subscriber.cpp b/example/cpp20_subscriber.cpp index 531fdaf9b..1f0dd86fe 100644 --- a/example/cpp20_subscriber.cpp +++ b/example/cpp20_subscriber.cpp @@ -21,7 +21,7 @@ namespace asio = boost::asio; using namespace std::chrono_literals; using boost::redis::request; -using boost::redis::resp3::flat_tree; +using boost::redis::generic_flat_response; using boost::redis::config; using boost::system::error_code; using boost::redis::connection; @@ -49,7 +49,7 @@ auto receiver(std::shared_ptr conn) -> asio::awaitable request req; req.push("SUBSCRIBE", "channel"); - flat_tree resp; + generic_flat_response resp; conn->set_receive_response(resp); // Loop while reconnection is enabled @@ -66,12 +66,12 @@ auto receiver(std::shared_ptr conn) -> asio::awaitable // The response must be consumed without suspending the // coroutine i.e. without the use of async operations. - for (auto const& elem: resp.get_view()) + for (auto const& elem: resp.value().get_view()) std::cout << elem.value << "\n"; std::cout << std::endl; - resp.clear(); + resp.value().clear(); } } } diff --git a/include/boost/redis/adapter/detail/adapters.hpp b/include/boost/redis/adapter/detail/adapters.hpp index 432fa9bd4..7830f0ed3 100644 --- a/include/boost/redis/adapter/detail/adapters.hpp +++ b/include/boost/redis/adapter/detail/adapters.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include @@ -205,6 +206,45 @@ class general_aggregate { } }; +template <> +class general_aggregate { +private: + generic_flat_response* tree_ = nullptr; + +public: + explicit general_aggregate(generic_flat_response* c = nullptr) + : tree_(c) + { } + + void on_init() { } + void on_done() + { + BOOST_ASSERT_MSG(!!tree_, "Unexpected null pointer"); + if (tree_->has_value()) { + tree_->value().notify_done(); + } + } + + template + void on_node(resp3::basic_node const& nd, system::error_code&) + { + BOOST_ASSERT_MSG(!!tree_, "Unexpected null pointer"); + switch (nd.data_type) { + case resp3::type::blob_error: + case resp3::type::simple_error: + *tree_ = error{ + nd.data_type, + std::string{std::cbegin(nd.value), std::cend(nd.value)} + }; + break; + default: + if (tree_->has_value()) { + (**tree_).push(nd); + } + } + } +}; + template <> class general_aggregate { private: diff --git a/include/boost/redis/adapter/detail/response_traits.hpp b/include/boost/redis/adapter/detail/response_traits.hpp index 493858127..d4d8a3049 100644 --- a/include/boost/redis/adapter/detail/response_traits.hpp +++ b/include/boost/redis/adapter/detail/response_traits.hpp @@ -115,6 +115,14 @@ struct response_traits { static auto adapt(response_type& v) noexcept { return adapter_type{&v}; } }; +template <> +struct response_traits { + using response_type = generic_flat_response; + using adapter_type = general_aggregate; + + static auto adapt(response_type& v) noexcept { return adapter_type{&v}; } +}; + template struct response_traits> { using response_type = response; diff --git a/include/boost/redis/adapter/detail/result_traits.hpp b/include/boost/redis/adapter/detail/result_traits.hpp index 671318197..83eddcc22 100644 --- a/include/boost/redis/adapter/detail/result_traits.hpp +++ b/include/boost/redis/adapter/detail/result_traits.hpp @@ -71,6 +71,13 @@ struct result_traits> { static auto adapt(response_type& v) noexcept { return adapter_type{&v}; } }; +template <> +struct result_traits { + using response_type = generic_flat_response; + using adapter_type = general_aggregate; + static auto adapt(response_type& v) noexcept { return adapter_type{&v}; } +}; + template <> struct result_traits { using response_type = resp3::flat_tree; diff --git a/include/boost/redis/response.hpp b/include/boost/redis/response.hpp index 5622f3a83..c39dc632d 100644 --- a/include/boost/redis/response.hpp +++ b/include/boost/redis/response.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include @@ -32,6 +33,9 @@ using response = std::tuple...>; */ using generic_response = adapter::result; +/// Similar to @ref boost::redis::generic_response but stores data contiguously. +using generic_flat_response = adapter::result; + /** @brief (Deprecated) Consume on response from a generic response * * This function rotates the elements so that the start of the next diff --git a/test/test_low_level_sync_sans_io.cpp b/test/test_low_level_sync_sans_io.cpp index 800d304f9..246cc0f30 100644 --- a/test/test_low_level_sync_sans_io.cpp +++ b/test/test_low_level_sync_sans_io.cpp @@ -24,6 +24,7 @@ using boost::redis::adapter::adapt2; using boost::redis::adapter::result; using boost::redis::resp3::tree; using boost::redis::resp3::flat_tree; +using boost::redis::generic_flat_response; using boost::redis::ignore_t; using boost::redis::resp3::detail::deserialize; using boost::redis::resp3::node; @@ -375,25 +376,45 @@ tree from_flat(flat_tree const& resp) return ret; } +tree from_flat(generic_flat_response const& resp) +{ + tree ret; + for (auto const& e: resp.value().get_view()) + ret.push_back(from_node_view(e)); + + return ret; +} + + // Parses the same data into a tree and a // flat_tree, they should be equal to each other. BOOST_AUTO_TEST_CASE(flat_tree_views_are_set) { tree resp1; - flat_tree fresp; + flat_tree resp2; + generic_flat_response resp3; error_code ec; deserialize(resp3_set, adapt2(resp1), ec); BOOST_CHECK_EQUAL(ec, error_code{}); - deserialize(resp3_set, adapt2(fresp), ec); + deserialize(resp3_set, adapt2(resp2), ec); BOOST_CHECK_EQUAL(ec, error_code{}); - BOOST_CHECK_EQUAL(fresp.get_reallocs(), 1u); - BOOST_CHECK_EQUAL(fresp.get_total_msgs(), 1u); + deserialize(resp3_set, adapt2(resp3), ec); + BOOST_CHECK_EQUAL(ec, error_code{}); + + BOOST_CHECK_EQUAL(resp2.get_reallocs(), 1u); + BOOST_CHECK_EQUAL(resp2.get_total_msgs(), 1u); + + BOOST_CHECK_EQUAL(resp3.value().get_reallocs(), 1u); + BOOST_CHECK_EQUAL(resp3.value().get_total_msgs(), 1u); + + auto const tmp2 = from_flat(resp2); + BOOST_CHECK_EQUAL(resp1, tmp2); - auto const resp2 = from_flat(fresp); - BOOST_CHECK_EQUAL(resp1, resp2); + auto const tmp3 = from_flat(resp3); + BOOST_CHECK_EQUAL(resp1, tmp3); } // The response should be reusable. @@ -446,3 +467,39 @@ BOOST_AUTO_TEST_CASE(flat_tree_copy_assign) BOOST_TEST((copy2 == resp)); BOOST_TEST((copy3 == resp)); } + +BOOST_AUTO_TEST_CASE(generic_flat_response_simple_error) +{ + generic_flat_response resp; + + char const* wire = "-Error\r\n"; + + error_code ec; + deserialize(wire, adapt2(resp), ec); + BOOST_CHECK_EQUAL(ec, error_code{}); + + BOOST_TEST(!resp.has_value()); + BOOST_TEST(resp.has_error()); + auto const error = resp.error(); + + BOOST_CHECK_EQUAL(error.data_type, boost::redis::resp3::type::simple_error); + BOOST_CHECK_EQUAL(error.diagnostic, std::string{"Error"}); +} + +BOOST_AUTO_TEST_CASE(generic_flat_response_blob_error) +{ + generic_flat_response resp; + + char const* wire = "!5\r\nError\r\n"; + + error_code ec; + deserialize(wire, adapt2(resp), ec); + BOOST_CHECK_EQUAL(ec, error_code{}); + + BOOST_TEST(!resp.has_value()); + BOOST_TEST(resp.has_error()); + auto const error = resp.error(); + + BOOST_CHECK_EQUAL(error.data_type, boost::redis::resp3::type::blob_error); + BOOST_CHECK_EQUAL(error.diagnostic, std::string{"Error"}); +} From bdd9c327c107d2da0fe13e696900fb773aacb6a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anarthal=20=28Rub=C3=A9n=20P=C3=A9rez=29?= <34971811+anarthal@users.noreply.github.com> Date: Wed, 19 Nov 2025 22:31:19 +0100 Subject: [PATCH 17/37] Adds Sentinel support (#345) close #237 close #269 close #268 close #229 --- doc/modules/ROOT/nav.adoc | 1 + doc/modules/ROOT/pages/examples.adoc | 2 +- doc/modules/ROOT/pages/reference.adoc | 4 + doc/modules/ROOT/pages/sentinel.adoc | 152 ++++ example/CMakeLists.txt | 2 +- example/cpp20_resolve_with_sentinel.cpp | 77 -- example/cpp20_sentinel.cpp | 60 ++ include/boost/redis/adapter/any_adapter.hpp | 43 +- include/boost/redis/config.hpp | 184 ++++- include/boost/redis/connection.hpp | 121 ++- include/boost/redis/detail/connect_fsm.hpp | 10 +- include/boost/redis/detail/connect_params.hpp | 65 ++ .../boost/redis/detail/connection_state.hpp | 28 +- include/boost/redis/detail/exec_one_fsm.hpp | 69 ++ include/boost/redis/detail/multiplexer.hpp | 6 + include/boost/redis/detail/redis_stream.hpp | 29 +- include/boost/redis/detail/run_fsm.hpp | 5 + .../redis/detail/sentinel_resolve_fsm.hpp | 93 +++ include/boost/redis/error.hpp | 18 + include/boost/redis/impl/connect_fsm.ipp | 37 +- include/boost/redis/impl/error.ipp | 15 +- include/boost/redis/impl/exec_one_fsm.ipp | 95 +++ include/boost/redis/impl/log_utils.hpp | 11 + include/boost/redis/impl/run_fsm.ipp | 166 ++-- .../boost/redis/impl/sentinel_resolve_fsm.ipp | 182 +++++ include/boost/redis/impl/sentinel_utils.hpp | 277 +++++++ .../boost/redis/impl/setup_request_utils.hpp | 73 +- include/boost/redis/src.hpp | 2 + test/CMakeLists.txt | 8 +- test/Jamfile | 7 +- test/common.cpp | 48 +- test/common.hpp | 7 + test/print_node.hpp | 28 + test/sansio_utils.cpp | 22 + test/sansio_utils.hpp | 9 + test/test_conn_sentinel.cpp | 491 ++++++++++++ test/test_conn_setup.cpp | 65 +- test/test_conn_tls.cpp | 2 +- test/test_connect_fsm.cpp | 107 ++- test/test_exec_one_fsm.cpp | 365 +++++++++ test/test_low_level.cpp | 5 + test/test_low_level_sync_sans_io.cpp | 21 +- test/test_multiplexer.cpp | 12 +- test/test_parse_sentinel_response.cpp | 727 ++++++++++++++++++ test/test_run_fsm.cpp | 339 +++++++- test/test_sentinel_resolve_fsm.cpp | 682 ++++++++++++++++ test/test_setup_adapter.cpp | 349 +++++++++ test/test_setup_request_utils.cpp | 48 ++ test/test_unix_sockets.cpp | 4 +- test/test_update_sentinel_list.cpp | 212 +++++ tools/docker-compose.yml | 137 +++- tools/docker/entrypoint.sh | 16 - 52 files changed, 5117 insertions(+), 421 deletions(-) create mode 100644 doc/modules/ROOT/pages/sentinel.adoc delete mode 100644 example/cpp20_resolve_with_sentinel.cpp create mode 100644 example/cpp20_sentinel.cpp create mode 100644 include/boost/redis/detail/connect_params.hpp create mode 100644 include/boost/redis/detail/exec_one_fsm.hpp create mode 100644 include/boost/redis/detail/sentinel_resolve_fsm.hpp create mode 100644 include/boost/redis/impl/exec_one_fsm.ipp create mode 100644 include/boost/redis/impl/sentinel_resolve_fsm.ipp create mode 100644 include/boost/redis/impl/sentinel_utils.hpp create mode 100644 test/print_node.hpp create mode 100644 test/test_conn_sentinel.cpp create mode 100644 test/test_exec_one_fsm.cpp create mode 100644 test/test_parse_sentinel_response.cpp create mode 100644 test/test_sentinel_resolve_fsm.cpp create mode 100644 test/test_setup_adapter.cpp create mode 100644 test/test_update_sentinel_list.cpp delete mode 100755 tools/docker/entrypoint.sh diff --git a/doc/modules/ROOT/nav.adoc b/doc/modules/ROOT/nav.adoc index 7c10e4ae8..8fd44b67f 100644 --- a/doc/modules/ROOT/nav.adoc +++ b/doc/modules/ROOT/nav.adoc @@ -3,6 +3,7 @@ * xref:cancellation.adoc[] * xref:serialization.adoc[] * xref:logging.adoc[] +* xref:sentinel.adoc[] * xref:benchmarks.adoc[] * xref:comparison.adoc[] * xref:examples.adoc[] diff --git a/doc/modules/ROOT/pages/examples.adoc b/doc/modules/ROOT/pages/examples.adoc index 6073cfb48..454e9b378 100644 --- a/doc/modules/ROOT/pages/examples.adoc +++ b/doc/modules/ROOT/pages/examples.adoc @@ -15,7 +15,7 @@ The examples below show how to use the features discussed throughout this docume * {site-url}/example/cpp20_containers.cpp[cpp20_containers.cpp]: Shows how to send and receive STL containers and how to use transactions. * {site-url}/example/cpp20_json.cpp[cpp20_json.cpp]: Shows how to serialize types using Boost.Json. * {site-url}/example/cpp20_protobuf.cpp[cpp20_protobuf.cpp]: Shows how to serialize types using protobuf. -* {site-url}/example/cpp20_resolve_with_sentinel.cpp[cpp20_resolve_with_sentinel.cpp]: Shows how to resolve a master address using sentinels. +* {site-url}/example/cpp20_sentinel.cpp[cpp20_sentinel.cpp]: Shows how to use the library with a Sentinel deployment. * {site-url}/example/cpp20_subscriber.cpp[cpp20_subscriber.cpp]: Shows how to implement pubsub with reconnection re-subscription. * {site-url}/example/cpp20_echo_server.cpp[cpp20_echo_server.cpp]: A simple TCP echo server. * {site-url}/example/cpp20_chat_room.cpp[cpp20_chat_room.cpp]: A command line chat built on Redis pubsub. diff --git a/doc/modules/ROOT/pages/reference.adoc b/doc/modules/ROOT/pages/reference.adoc index 5e1976c27..e87ee7cd8 100644 --- a/doc/modules/ROOT/pages/reference.adoc +++ b/doc/modules/ROOT/pages/reference.adoc @@ -25,8 +25,12 @@ xref:reference:boost/redis/basic_connection.adoc[`basic_connection`] xref:reference:boost/redis/address.adoc[`address`] +xref:reference:boost/redis/role.adoc[`role`] + xref:reference:boost/redis/config.adoc[`config`] +xref:reference:boost/redis/sentinel_config.adoc[`sentinel_config`] + xref:reference:boost/redis/error.adoc[`error`] xref:reference:boost/redis/logger.adoc[`logger`] diff --git a/doc/modules/ROOT/pages/sentinel.adoc b/doc/modules/ROOT/pages/sentinel.adoc new file mode 100644 index 000000000..bd03faff8 --- /dev/null +++ b/doc/modules/ROOT/pages/sentinel.adoc @@ -0,0 +1,152 @@ +// +// Copyright (c) 2025 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + += Sentinel + +Boost.Redis supports Redis Sentinel deployments. Sentinel handling +in `connection` is built-in: xref:reference:boost/redis/basic_connection/async_run-04.adoc[`async_run`] +automatically connects to Sentinels, resolves the master's address, and connects to the master. + +Configuration is done using xref:reference:boost/redis/sentinel_config.adoc[`config::sentinel`]: + +[source,cpp] +---- +config cfg; + +// To enable Sentinel, set this field to a non-empty list +// of (hostname, port) pairs where Sentinels are listening +cfg.sentinel.addresses = { + {"sentinel1.example.com", "26379"}, + {"sentinel2.example.com", "26379"}, + {"sentinel3.example.com", "26379"}, +}; + +// Set master_name to the identifier that you configured +// in the "sentinel monitor" statement of your sentinel.conf file +cfg.sentinel.master_name = "mymaster"; +---- + +Once set, the connection object can be used normally. See our +our {site-url}/example/cpp20_sentinel.cpp[Sentinel example] +for a full program. + +== Connecting to replicas + +By default, the library connects to the Redis master. +You can connect to one of its replicas by using +xref:reference:boost/redis/sentinel_config/server_role.adoc[`config::sentinel::server_role`]. +This can be used to balance load, if all your commands read data from +the server and never write to it. The particular replica will be chosen randomly. + +[source,cpp] +---- +config cfg; + +// Set up Sentinel +cfg.sentinel.addresses = { + {"sentinel1.example.com", "26379"}, + {"sentinel2.example.com", "26379"}, + {"sentinel3.example.com", "26379"}, +}; +cfg.sentinel.master_name = "mymaster"; + +// Ask the library to connect to a random replica of 'mymaster', rather than the master node +cfg.sentinel.server_role = role::replica; +---- + + +== Sentinel authentication + +If your Sentinels require authentication, +you can use xref:reference:boost/redis/sentinel_config/setup.adoc[`config::sentinel::setup`] +to provide credentials. +This request is executed immediately after connecting to Sentinels, and +before any other command: + +[source,cpp] +---- +// Set up Sentinel +config cfg; +cfg.sentinel.addresses = { + {"sentinel1.example.com", "26379"}, + {"sentinel2.example.com", "26379"}, + {"sentinel3.example.com", "26379"}, +}; +cfg.sentinel.master_name = "mymaster"; + +// By default, setup contains a 'HELLO 3' command. +// Override it to add an AUTH clause to it with out credentials. +cfg.sentinel.setup.clear(); +cfg.sentinel.setup.push("HELLO", 3, "AUTH", "sentinel_user", "sentinel_password"); + +// cfg.sentinel.setup applies to Sentinels, only. +// Use cfg.setup to authenticate to masters/replicas. +cfg.use_setup = true; // Required for cfg.setup to be used, for historic reasons +cfg.setup.clear(); +cfg.setup.push("HELLO", 3, "AUTH", "master_user", "master_password"); +---- + +== Using TLS with Sentinels + +You might use TLS with Sentinels only, masters/replicas only, or both by adjusting +xref:reference:boost/redis/sentinel_config/use_ssl.adoc[`config::sentinel::use_ssl`] +and xref:reference:boost/redis/config/use_ssl.adoc[`config::use_ssl`]: + +[source,cpp] +---- +// Set up Sentinel +config cfg; +cfg.sentinel.addresses = { + {"sentinel1.example.com", "26379"}, + {"sentinel2.example.com", "26379"}, + {"sentinel3.example.com", "26379"}, +}; +cfg.sentinel.master_name = "mymaster"; + +// Adjust these switches to enable/disable TLS +cfg.use_ssl = true; // Applies to masters and replicas +cfg.sentinel.use_ssl = true; // Applies to Sentinels +---- + +== Sentinel algorithm + +This section details how `async_run` interacts with Sentinel. +Most of the algorithm follows +https://redis.io/docs/latest/develop/reference/sentinel-clients/[the official Sentinel client guidelines]. +Some of these details may vary between library versions. + +* Connections maintain an internal list of Sentinels, bootstrapped from + xref:reference:boost/redis/sentinel_config/addresses.adoc[`config::sentinel::addresses`]. +* The first Sentinel in the list is contacted by performing the following: +** A physical connection is established. +** The setup request is executed. +** The master's address is resolved using + https://redis.io/docs/latest/operate/oss_and_stack/management/sentinel/#sentinel-api[`SENTINEL GET-MASTER-NAME-BY-ADDR`]. +** If `config::sentinel::server_role` is `role::replica`, replica addresses are obtained using + https://redis.io/docs/latest/operate/oss_and_stack/management/sentinel/#sentinel-api[`SENTINEL REPLICAS`]. + One replica is chosen randomly. +** The address of other Sentinels also monitoring this master are retrieved using + https://redis.io/docs/latest/operate/oss_and_stack/management/sentinel/#sentinel-api[`SENTINEL SENTINELS`]. +* If a Sentinel is unreachable, doesn't know about the configured master, + or returns an error while executing the above requests, the next Sentinel in the list is tried. +* If all Sentinels have been tried without success, `config::reconnect_wait_interval` + is waited, and the process starts again. +* After a successful Sentinel response, the internal Sentinel list is updated + with any newly discovered Sentinels. + Sentinels in `config::sentinel::addresses` are always kept in the list, + even if they weren't present in the output of `SENTINEL SENTINELS`. +* The retrieved address is used + to establish a connection with the master or replica. + A `ROLE` command is added at the end of the setup request. + This is used to detect situations where a Sentinel returns outdated + information due to a failover in process. If `ROLE` doesn't output + the expected role (`"master"` or `"slave"`, depending on `config::sentinel::server_role`) + `config::reconnect_wait_interval` is waited and Sentinel is contacted again. +* The connection to the master/replica is run like any other connection. + If network errors or timeouts happen, `config::reconnect_wait_interval` + is waited and Sentinel is contacted again. diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index c9b182357..7f77b4f10 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -28,11 +28,11 @@ make_testable_example(cpp20_containers 20) make_testable_example(cpp20_json 20) make_testable_example(cpp20_unix_sockets 20) make_testable_example(cpp20_timeouts 20) +make_testable_example(cpp20_sentinel 20) make_example(cpp20_subscriber 20) make_example(cpp20_streams 20) make_example(cpp20_echo_server 20) -make_example(cpp20_resolve_with_sentinel 20) make_example(cpp20_intro_tls 20) # We test the protobuf example only on gcc. diff --git a/example/cpp20_resolve_with_sentinel.cpp b/example/cpp20_resolve_with_sentinel.cpp deleted file mode 100644 index 3d4434026..000000000 --- a/example/cpp20_resolve_with_sentinel.cpp +++ /dev/null @@ -1,77 +0,0 @@ -/* Copyright (c) 2018-2022 Marcelo Zimbres Silva (mzimbres@gmail.com) - * - * Distributed under the Boost Software License, Version 1.0. (See - * accompanying file LICENSE.txt) - */ - -#include - -#include -#include -#include -#include - -#include - -#if defined(BOOST_ASIO_HAS_CO_AWAIT) - -namespace asio = boost::asio; -using endpoints = asio::ip::tcp::resolver::results_type; -using boost::redis::request; -using boost::redis::response; -using boost::redis::ignore_t; -using boost::redis::config; -using boost::redis::address; -using boost::redis::connection; - -auto redir(boost::system::error_code& ec) { return asio::redirect_error(asio::use_awaitable, ec); } - -// For more info see -// - https://redis.io/docs/manual/sentinel. -// - https://redis.io/docs/reference/sentinel-clients. -auto resolve_master_address(std::vector
const& addresses) -> asio::awaitable
-{ - request req; - req.push("SENTINEL", "get-master-addr-by-name", "mymaster"); - req.push("QUIT"); - - auto conn = std::make_shared(co_await asio::this_coro::executor); - - response>, ignore_t> resp; - for (auto addr : addresses) { - boost::system::error_code ec; - config cfg; - cfg.addr = addr; - // TODO: async_run and async_exec should be lauched in - // parallel here so we can wait for async_run completion - // before eventually calling it again. - conn->async_run(cfg, asio::consign(asio::detached, conn)); - co_await conn->async_exec(req, resp, redir(ec)); - conn->cancel(); - if (!ec && std::get<0>(resp)) - co_return address{ - std::get<0>(resp).value().value().at(0), - std::get<0>(resp).value().value().at(1)}; - } - - co_return address{}; -} - -auto co_main(config cfg) -> asio::awaitable -{ - // A list of sentinel addresses from which only one is responsive. - // This simulates sentinels that are down. - std::vector
const addresses{ - address{"foo", "26379"}, - address{"bar", "26379"}, - cfg.addr - }; - - auto const ep = co_await resolve_master_address(addresses); - - std::clog << "Host: " << ep.host << "\n" - << "Port: " << ep.port << "\n" - << std::flush; -} - -#endif // defined(BOOST_ASIO_HAS_CO_AWAIT) diff --git a/example/cpp20_sentinel.cpp b/example/cpp20_sentinel.cpp new file mode 100644 index 000000000..5e51413ba --- /dev/null +++ b/example/cpp20_sentinel.cpp @@ -0,0 +1,60 @@ +// +// Copyright (c) 2025 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#include + +#include +#include +#include + +#include + +#if defined(BOOST_ASIO_HAS_CO_AWAIT) + +namespace asio = boost::asio; +using boost::redis::request; +using boost::redis::response; +using boost::redis::config; +using boost::redis::connection; + +// Called from the main function (see main.cpp) +auto co_main(config cfg) -> asio::awaitable +{ + // Boost.Redis has built-in support for Sentinel deployments. + // To enable it, set the fields in config shown here. + // sentinel.addresses should contain a list of (hostname, port) pairs + // where Sentinels are listening. IPs can also be used. + cfg.sentinel.addresses = { + {"localhost", "26379"}, + {"localhost", "26380"}, + {"localhost", "26381"}, + }; + + // Set master_name to the identifier that you configured + // in the "sentinel monitor" statement of your sentinel.conf file + cfg.sentinel.master_name = "mymaster"; + + // async_run will contact the Sentinels, obtain the master address, + // connect to it and keep the connection healthy. If a failover happens, + // the address will be resolved again and the new elected master will be contacted. + auto conn = std::make_shared(co_await asio::this_coro::executor); + conn->async_run(cfg, asio::consign(asio::detached, conn)); + + // You can now use the connection normally, as you would use a connection to a single master. + request req; + req.push("PING", "Hello world"); + response resp; + + // Execute the request. + co_await conn->async_exec(req, resp); + conn->cancel(); + + std::cout << "PING: " << std::get<0>(resp).value() << std::endl; +} + +#endif // defined(BOOST_ASIO_HAS_CO_AWAIT) diff --git a/include/boost/redis/adapter/any_adapter.hpp b/include/boost/redis/adapter/any_adapter.hpp index a32e25450..05764ab72 100644 --- a/include/boost/redis/adapter/any_adapter.hpp +++ b/include/boost/redis/adapter/any_adapter.hpp @@ -12,9 +12,7 @@ #include -#include #include -#include #include namespace boost::redis { @@ -50,20 +48,7 @@ class any_adapter { using impl_t = std::function; template - static auto create_impl(T& resp) -> impl_t - { - using namespace boost::redis::adapter; - return [adapter2 = boost_redis_adapt(resp)]( - any_adapter::parse_event ev, - resp3::node_view const& nd, - system::error_code& ec) mutable { - switch (ev) { - case parse_event::init: adapter2.on_init(); break; - case parse_event::node: adapter2.on_node(nd, ec); break; - case parse_event::done: adapter2.on_done(); break; - } - }; - } + static auto create_impl(T& resp) -> impl_t; /// Contructs from a type erased adaper any_adapter(impl_t fn = [](parse_event, resp3::node_view const&, system::error_code&) { }) @@ -109,6 +94,32 @@ class any_adapter { impl_t impl_; }; +namespace detail { + +template +any_adapter::impl_t make_any_adapter_impl(Adapter&& value) +{ + return [adapter = std::move(value)]( + any_adapter::parse_event ev, + resp3::node_view const& nd, + system::error_code& ec) mutable { + switch (ev) { + case any_adapter::parse_event::init: adapter.on_init(); break; + case any_adapter::parse_event::node: adapter.on_node(nd, ec); break; + case any_adapter::parse_event::done: adapter.on_done(); break; + } + }; +} + +} // namespace detail + } // namespace boost::redis +template +auto boost::redis::any_adapter::create_impl(T& resp) -> impl_t +{ + using adapter::boost_redis_adapt; + return detail::make_any_adapter_impl(boost_redis_adapt(resp)); +} + #endif diff --git a/include/boost/redis/config.hpp b/include/boost/redis/config.hpp index 5a812141d..4e9806794 100644 --- a/include/boost/redis/config.hpp +++ b/include/boost/redis/config.hpp @@ -13,6 +13,7 @@ #include #include #include +#include namespace boost::redis { @@ -24,12 +25,130 @@ struct address { std::string port = "6379"; }; +/** @brief Compares two addresses for equality. + * @relates address + * + * @param a Left hand side address. + * @param b Right hand side address. + */ +inline bool operator==(address const& a, address const& b) +{ + return a.host == b.host && a.port == b.port; +} + +/** @brief Compares two addresses for inequality. + * @relates address + * + * @param a Left hand side address. + * @param b Right hand side address. + */ +inline bool operator!=(address const& a, address const& b) { return !(a == b); } + +/// Identifies the possible roles of a Redis server. +enum class role +{ + /// The server is a master. + master, + + /// The server is a replica. + replica, +}; + +/// Configuration values to use when using Sentinel. +struct sentinel_config { + /** + * @brief A list of (hostname, port) pairs where the Sentinels are listening. + * + * Sentinels in this list will be contacted in order, until a successful + * connection is made. At this point, the `SENTINEL SENTINELS` command + * will be used to retrieve any additional Sentinels monitoring the configured master. + * Thus, it is not required to keep this list comprehensive - if Sentinels are added + * later, they will be detected at runtime. + * + * Sentinel will only be used if this value is not empty. + * + * Numeric IP addresses are also allowed as hostnames. + */ + std::vector
addresses{}; + + /** + * @brief The name of the master to connect to, as configured in the + * `sentinel monitor` statement in `sentinel.conf`. + * + * This field is required even when connecting to replicas. + */ + std::string master_name{}; + + /** + * @brief Whether connections to Sentinels should use TLS or not. + * Does not affect connections to masters. + * + * When set to `true`, physical connections to Sentinels will be established + * using TLS. This setting does *not* influence how masters and replicas are contacted. + * To use TLS when connecting to these, set @ref config::use_ssl to `true`. + */ + bool use_ssl = false; + + /** + * @brief A request to be sent to Sentinels upon connection establishment. + * + * This request is executed every time a Sentinel is contacted, and before + * commands like `SENTINEL GET-MASTER-NAME-BY-ADDR` are run. + * By default, this field contains a `HELLO 3` command. + * You can use this request to set up any authorization required by Sentinels. + * + * This request should ensure that the connection is upgraded to RESP3 + * by executing `HELLO 3` or similar. RESP2 is not supported yet. + */ + request setup = detail::make_hello_request(); + + /** + * @brief Time span that the Sentinel resolve operation is allowed to elapse. + * Does not affect connections to masters and replicas, controlled by @ref config::resolve_timeout. + */ + std::chrono::steady_clock::duration resolve_timeout = std::chrono::milliseconds{500}; + + /** + * @brief Time span that the Sentinel connect operation is allowed to elapse. + * Does not affect connections to masters and replicas, controlled by @ref config::connect_timeout. + */ + std::chrono::steady_clock::duration connect_timeout = std::chrono::milliseconds{500}; + + /** + * @brief Time span that the Sentinel TLS handshake operation is allowed to elapse. + * Does not affect connections to masters and replicas, controlled by @ref config::ssl_handshake_timeout. + */ + std::chrono::steady_clock::duration ssl_handshake_timeout = std::chrono::seconds{5}; + + /** + * @brief Time span that the Sentinel request/response exchange is allowed to elapse. + * Includes executing the commands in @ref setup and the commands required to + * resolve the server's address. + */ + std::chrono::steady_clock::duration request_timeout = std::chrono::seconds{5}; + + /** + * @brief Whether to connect to a Redis master or to a replica. + * + * The library resolves and connects to the Redis master, by default. + * Set this value to @ref role::replica to connect to one of the replicas + * of the master identified by @ref master_name. + * The particular replica will be chosen randomly. + */ + role server_role = role::master; +}; + /// Configure parameters used by the connection classes. struct config { - /// Uses SSL instead of a plain connection. + /** + * @brief Whether to use TLS instead of plaintext connections. + * + * When using Sentinel, configures whether to use TLS when connecting to masters and replicas. + * Use @ref sentinel_config::use_ssl to control TLS for Sentinels. + */ bool use_ssl = false; - /// For TCP connections, hostname and port of the Redis server. + /// For TCP connections, hostname and port of the Redis server. Ignored when using Sentinel. address addr = address{"127.0.0.1", "6379"}; /** @@ -37,8 +156,11 @@ struct config { * * If non-empty, communication with the server will happen using * UNIX domain sockets, and @ref addr will be ignored. + * * UNIX domain sockets can't be used with SSL: if `unix_socket` is non-empty, - * @ref use_ssl must be `false`. + * @ref use_ssl must be `false`. UNIX domain sockets can't be used with Sentinel, either. + * + * UNIX domain sockets can't be used with Sentinel. */ std::string unix_socket; @@ -51,6 +173,9 @@ struct config { * If the username equals the literal `"default"` (the default) * and no password is specified, the `HELLO` command is sent * without authentication parameters. + * + * When using Sentinel, this setting applies to masters and replicas. + * Use @ref sentinel_config::setup to configure authorization for Sentinels. */ std::string username = "default"; @@ -63,6 +188,9 @@ struct config { * If the username equals the literal `"default"` (the default) * and no password is specified, the `HELLO` command is sent * without authentication parameters. + * + * When using Sentinel, this setting applies to masters and replicas. + * Use @ref sentinel_config::setup to configure authorization for Sentinels. */ std::string password; @@ -71,6 +199,9 @@ struct config { * If @ref use_setup is false (the default), during connection establishment, * a `HELLO` command is sent. If this field is not empty, the `HELLO` command * will contain a `SETNAME` subcommand containing this value. + * + * When using Sentinel, this setting applies to masters and replicas. + * Use @ref sentinel_config::setup to configure this value for Sentinels. */ std::string clientname = "Boost.Redis"; @@ -80,6 +211,8 @@ struct config { * non-empty optional, and its value is different than zero, * a `SELECT` command will be issued during connection establishment to set the logical * database index. By default, no `SELECT` command is sent. + * + * When using Sentinel, this setting applies to masters and replicas. */ std::optional database_index = 0; @@ -95,13 +228,22 @@ struct config { */ std::string log_prefix = "(Boost.Redis) "; - /// Time span that the resolve operation is allowed to elapse. + /** + * @brief Time span that the resolve operation is allowed to elapse. + * When using Sentinel, this setting applies to masters and replicas. + */ std::chrono::steady_clock::duration resolve_timeout = std::chrono::seconds{10}; - /// Time span that the connect operation is allowed to elapse. + /** + * @brief Time span that the connect operation is allowed to elapse. + * When using Sentinel, this setting applies to masters and replicas. + */ std::chrono::steady_clock::duration connect_timeout = std::chrono::seconds{10}; - /// Time span that the SSL handshake operation is allowed to elapse. + /** + * @brief Time span that the SSL handshake operation is allowed to elapse. + * When using Sentinel, this setting applies to masters and replicas. + */ std::chrono::steady_clock::duration ssl_handshake_timeout = std::chrono::seconds{10}; /** @brief Time span between successive health checks. @@ -123,18 +265,28 @@ struct config { * * The exact timeout values are *not* part of the interface, and might change * in future versions. + * + * When using Sentinel, this setting applies to masters and replicas. + * Sentinels are not health-checked. */ std::chrono::steady_clock::duration health_check_interval = std::chrono::seconds{2}; /** @brief Time span to wait between successive connection retries. - * Set to zero to disable reconnection. + * Set to zero to disable reconnection. + * + * When using Sentinel, this setting applies to masters, replicas and Sentinels. + * If none of the configured Sentinels can be contacted, this time span will + * be waited before trying again. After a connection error with a master or replica + * is encountered, this time span will be waited before contacting Sentinels again. */ std::chrono::steady_clock::duration reconnect_wait_interval = std::chrono::seconds{1}; /** @brief Maximum size of the socket read-buffer in bytes. * - * Sets a limit on how much data is allowed to be read into the - * read buffer. It can be used to prevent DDOS. + * Sets a limit on how much data is allowed to be read into the + * read buffer. It can be used to prevent DDOS. + * + * When using Sentinel, this setting applies to masters, replicas and Sentinels. */ std::size_t max_read_size = (std::numeric_limits::max)(); @@ -144,6 +296,8 @@ struct config { * needed. This can help avoiding some memory allocations. Once the * maximum size is reached no more memory allocations are made * since the buffer is reused. + * + * When using Sentinel, this setting applies to masters, replicas and Sentinels. */ std::size_t read_buffer_append_size = 4096; @@ -168,6 +322,9 @@ struct config { * systems that don't support `HELLO`. * * By default, this field is false, and @ref setup will not be used. + * + * When using Sentinel, this setting applies to masters and replicas. + * Use @ref sentinel_config::setup for Sentinels. */ bool use_setup = false; @@ -177,8 +334,17 @@ struct config { * @ref use_setup docs for more info. * * By default, `setup` contains a `"HELLO 3"` command. + * + * When using Sentinel, this setting applies to masters and replicas. + * Use @ref sentinel_config::setup for Sentinels. */ request setup = detail::make_hello_request(); + + /** + * @brief Configuration values for Sentinel. Sentinel is enabled only if + * @ref sentinel_config::addresses is not empty. + */ + sentinel_config sentinel{}; }; } // namespace boost::redis diff --git a/include/boost/redis/connection.hpp b/include/boost/redis/connection.hpp index 25791feaa..3e8298f39 100644 --- a/include/boost/redis/connection.hpp +++ b/include/boost/redis/connection.hpp @@ -12,10 +12,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -32,9 +34,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -45,6 +49,7 @@ #include #include #include +#include #include #include @@ -230,6 +235,7 @@ struct connection_impl { template auto async_receive2(CompletionToken&& token) { + // clang-format off return receive_channel_.async_receive( asio::deferred( @@ -250,9 +256,104 @@ struct connection_impl { } ) )(std::forward(token)); + // clang-format on } }; +template +struct exec_one_op { + connection_impl* conn_; + const request* req_; + exec_one_fsm fsm_; + + explicit exec_one_op(connection_impl& conn, const request& req, any_adapter resp) + : conn_(&conn) + , req_(&req) + , fsm_(std::move(resp), req.get_expected_responses()) + { } + + template + void operator()(Self& self, system::error_code ec = {}, std::size_t bytes_written = 0u) + { + exec_one_action act = fsm_.resume( + conn_->st_.mpx.get_read_buffer(), + ec, + bytes_written, + self.get_cancellation_state().cancelled()); + + switch (act.type) { + case exec_one_action_type::done: self.complete(act.ec); return; + case exec_one_action_type::write: + asio::async_write(conn_->stream_, asio::buffer(req_->payload()), std::move(self)); + return; + case exec_one_action_type::read_some: + conn_->stream_.async_read_some( + conn_->st_.mpx.get_read_buffer().get_prepared(), + std::move(self)); + return; + } + } +}; + +template +auto async_exec_one( + connection_impl& conn, + const request& req, + any_adapter resp, + CompletionToken&& token) +{ + return asio::async_compose( + exec_one_op{conn, req, std::move(resp)}, + token, + conn); +} + +template +struct sentinel_resolve_op { + connection_impl* conn_; + sentinel_resolve_fsm fsm_; + + explicit sentinel_resolve_op(connection_impl& conn) + : conn_(&conn) + { } + + template + void operator()(Self& self, system::error_code ec = {}) + { + auto* conn = conn_; // Prevent potential use-after-move errors with cancel_after + sentinel_action act = fsm_.resume(conn_->st_, ec, self.get_cancellation_state().cancelled()); + + switch (act.get_type()) { + case sentinel_action::type::done: self.complete(act.error()); return; + case sentinel_action::type::connect: + conn->stream_.async_connect( + make_sentinel_connect_params(conn->st_.cfg, act.connect_addr()), + conn->st_.logger, + std::move(self)); + return; + case sentinel_action::type::request: + async_exec_one( + *conn, + conn->st_.cfg.sentinel.setup, + make_sentinel_adapter(conn->st_), + asio::cancel_after( + conn->reconnect_timer_, // should be safe to re-use this + conn->st_.cfg.sentinel.request_timeout, + std::move(self))); + return; + } + } +}; + +template +auto async_sentinel_resolve(connection_impl& conn, CompletionToken&& token) +{ + return asio::async_compose( + sentinel_resolve_op{conn}, + token, + conn); +} + template struct writer_op { connection_impl* conn_; @@ -379,8 +480,14 @@ class run_op { case run_action_type::immediate: asio::async_immediate(self.get_io_executor(), std::move(self)); return; + case run_action_type::sentinel_resolve: + async_sentinel_resolve(*conn_, std::move(self)); + return; case run_action_type::connect: - conn_->stream_.async_connect(conn_->st_.cfg, conn_->st_.logger, std::move(self)); + conn_->stream_.async_connect( + make_run_connect_params(conn_->st_), + conn_->st_.logger, + std::move(self)); return; case run_action_type::parallel_group: asio::experimental::make_parallel_group( @@ -531,6 +638,8 @@ class basic_connection { * This function establishes a connection to the Redis server and keeps * it healthy by performing the following operations: * + * @li For Sentinel deployments (`config::sentinel::addresses` is not empty), + * contacts Sentinels to obtain the address of the configured master. * @li For TCP connections, resolves the server hostname passed in * @ref boost::redis::config::addr. * @li Establishes a physical connection to the server. For TCP connections, @@ -728,10 +837,7 @@ class basic_connection { * @returns The number of bytes read from the socket. */ BOOST_DEPRECATED("Please, use async_receive2 instead.") - std::size_t receive(system::error_code& ec) - { - return impl_->receive(ec); - } + std::size_t receive(system::error_code& ec) { return impl_->receive(ec); } /** @brief Executes commands on the Redis server asynchronously. * @@ -1119,10 +1225,7 @@ class connection { /// @copydoc basic_connection::receive BOOST_DEPRECATED("Please use async_receive2 instead.") - std::size_t receive(system::error_code& ec) - { - return impl_.impl_->receive(ec); - } + std::size_t receive(system::error_code& ec) { return impl_.impl_->receive(ec); } /** * @brief Calls @ref boost::redis::basic_connection::async_exec. diff --git a/include/boost/redis/detail/connect_fsm.hpp b/include/boost/redis/detail/connect_fsm.hpp index 176276159..352355667 100644 --- a/include/boost/redis/detail/connect_fsm.hpp +++ b/include/boost/redis/detail/connect_fsm.hpp @@ -9,8 +9,6 @@ #ifndef BOOST_REDIS_CONNECT_FSM_HPP #define BOOST_REDIS_CONNECT_FSM_HPP -#include - #include #include #include @@ -62,17 +60,13 @@ struct connect_action { class connect_fsm { int resume_point_{0}; - const config* cfg_{nullptr}; buffered_logger* lgr_{nullptr}; public: - connect_fsm(const config& cfg, buffered_logger& lgr) noexcept - : cfg_(&cfg) - , lgr_(&lgr) + connect_fsm(buffered_logger& lgr) noexcept + : lgr_(&lgr) { } - const config& get_config() const { return *cfg_; } - connect_action resume( system::error_code ec, const asio::ip::tcp::resolver::results_type& resolver_results, diff --git a/include/boost/redis/detail/connect_params.hpp b/include/boost/redis/detail/connect_params.hpp new file mode 100644 index 000000000..e0dfa01d0 --- /dev/null +++ b/include/boost/redis/detail/connect_params.hpp @@ -0,0 +1,65 @@ +// +// Copyright (c) 2025 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#ifndef BOOST_REDIS_CONNECT_PARAMS_HPP +#define BOOST_REDIS_CONNECT_PARAMS_HPP + +// Parameters used by redis_stream::async_connect + +#include +#include + +#include +#include + +namespace boost::redis::detail { + +// Fully identifies where a server is listening. Reference type. +class any_address_view { + transport_type type_; + union { + const address* tcp_; + std::string_view unix_; + }; + +public: + any_address_view(const address& addr, bool use_ssl) noexcept + : type_(use_ssl ? transport_type::tcp_tls : transport_type::tcp) + , tcp_(&addr) + { } + + explicit any_address_view(std::string_view unix_socket) noexcept + : type_(transport_type::unix_socket) + , unix_(unix_socket) + { } + + transport_type type() const { return type_; } + + const address& tcp_address() const + { + BOOST_ASSERT(type_ == transport_type::tcp || type_ == transport_type::tcp_tls); + return *tcp_; + } + + std::string_view unix_socket() const + { + BOOST_ASSERT(type_ == transport_type::unix_socket); + return unix_; + } +}; + +struct connect_params { + any_address_view addr; + std::chrono::steady_clock::duration resolve_timeout; + std::chrono::steady_clock::duration connect_timeout; + std::chrono::steady_clock::duration ssl_handshake_timeout; +}; + +} // namespace boost::redis::detail + +#endif // BOOST_REDIS_CONNECTOR_HPP diff --git a/include/boost/redis/detail/connection_state.hpp b/include/boost/redis/detail/connection_state.hpp index f908c5d14..b4d2e1a01 100644 --- a/include/boost/redis/detail/connection_state.hpp +++ b/include/boost/redis/detail/connection_state.hpp @@ -13,20 +13,46 @@ #include #include #include +#include #include +#include #include +#include namespace boost::redis::detail { +// A random engine that gets seeded lazily. +// Seeding with std::random_device is not trivial and might fail. +class lazy_random_engine { + bool seeded_{}; + std::minstd_rand eng_; + +public: + lazy_random_engine() = default; + std::minstd_rand& get() + { + if (!seeded_) { + eng_.seed(static_cast(std::random_device{}())); + seeded_ = true; + } + return eng_; + } +}; + // Contains all the members in connection that don't depend on the Executor. // Makes implementing sans-io algorithms easier struct connection_state { buffered_logger logger; config cfg{}; multiplexer mpx{}; - std::string setup_diagnostic{}; + std::string diagnostic{}; // Used by the setup request and Sentinel request ping_req{}; + + // Sentinel stuff + lazy_random_engine eng{}; + std::vector
sentinels{}; + std::vector sentinel_resp_nodes{}; // for parsing }; } // namespace boost::redis::detail diff --git a/include/boost/redis/detail/exec_one_fsm.hpp b/include/boost/redis/detail/exec_one_fsm.hpp new file mode 100644 index 000000000..4ad3bd3ec --- /dev/null +++ b/include/boost/redis/detail/exec_one_fsm.hpp @@ -0,0 +1,69 @@ +// +// Copyright (c) 2025 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#ifndef BOOST_REDIS_EXEC_ONE_FSM_HPP +#define BOOST_REDIS_EXEC_ONE_FSM_HPP + +#include +#include + +#include +#include + +#include + +// Sans-io algorithm for async_exec_one, as a finite state machine + +namespace boost::redis::detail { + +class read_buffer; + +// What should we do next? +enum class exec_one_action_type +{ + done, // Call the final handler + write, // Write the request + read_some, // Read into the read buffer +}; + +struct exec_one_action { + exec_one_action_type type; + system::error_code ec; + + exec_one_action(exec_one_action_type type) noexcept + : type{type} + { } + + exec_one_action(system::error_code ec) noexcept + : type{exec_one_action_type::done} + , ec{ec} + { } +}; + +class exec_one_fsm { + int resume_point_{0}; + any_adapter adapter_; + std::size_t remaining_responses_; + resp3::parser parser_; + +public: + exec_one_fsm(any_adapter resp, std::size_t expected_responses) + : adapter_(std::move(resp)) + , remaining_responses_(expected_responses) + { } + + exec_one_action resume( + read_buffer& buffer, + system::error_code ec, + std::size_t bytes_transferred, + asio::cancellation_type_t cancel_state); +}; + +} // namespace boost::redis::detail + +#endif // BOOST_REDIS_CONNECTOR_HPP diff --git a/include/boost/redis/detail/multiplexer.hpp b/include/boost/redis/detail/multiplexer.hpp index 14afc6b50..797c9493a 100644 --- a/include/boost/redis/detail/multiplexer.hpp +++ b/include/boost/redis/detail/multiplexer.hpp @@ -181,6 +181,12 @@ class multiplexer { return std::string_view{write_buffer_}.substr(write_offset_); } + [[nodiscard]] + auto get_read_buffer() noexcept -> read_buffer& + { + return read_buffer_; + } + [[nodiscard]] auto get_prepared_read_buffer() noexcept -> read_buffer::span_type; diff --git a/include/boost/redis/detail/redis_stream.hpp b/include/boost/redis/detail/redis_stream.hpp index 7c51624cd..349cc650d 100644 --- a/include/boost/redis/detail/redis_stream.hpp +++ b/include/boost/redis/detail/redis_stream.hpp @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -24,6 +25,7 @@ #include #include #include +#include #include #include @@ -48,12 +50,14 @@ class redis_stream { struct connect_op { redis_stream& obj_; connect_fsm fsm_; + connect_params params_; template void execute_action(Self& self, connect_action act) { - auto& obj = this->obj_; // prevent use-after-move errors - const auto& cfg = fsm_.get_config(); + // Prevent use-after-move errors + auto& obj = this->obj_; + auto params = this->params_; switch (act.type) { case connect_action_type::unix_socket_close: @@ -70,8 +74,8 @@ class redis_stream { case connect_action_type::unix_socket_connect: #ifdef BOOST_ASIO_HAS_LOCAL_SOCKETS obj.unix_socket_.async_connect( - cfg.unix_socket, - asio::cancel_after(obj.timer_, cfg.connect_timeout, std::move(self))); + params.addr.unix_socket(), + asio::cancel_after(obj.timer_, params.connect_timeout, std::move(self))); #else BOOST_ASSERT(false); #endif @@ -79,9 +83,9 @@ class redis_stream { case connect_action_type::tcp_resolve: obj.resolv_.async_resolve( - cfg.addr.host, - cfg.addr.port, - asio::cancel_after(obj.timer_, cfg.resolve_timeout, std::move(self))); + params.addr.tcp_address().host, + params.addr.tcp_address().port, + asio::cancel_after(obj.timer_, params.resolve_timeout, std::move(self))); return; case connect_action_type::ssl_stream_reset: obj.reset_stream(); @@ -91,7 +95,7 @@ class redis_stream { case connect_action_type::ssl_handshake: obj.stream_.async_handshake( asio::ssl::stream_base::client, - asio::cancel_after(obj.timer_, cfg.ssl_handshake_timeout, std::move(self))); + asio::cancel_after(obj.timer_, params.ssl_handshake_timeout, std::move(self))); return; case connect_action_type::done: self.complete(act.ec); break; // Connect should use the specialized handler, where resolver results are available @@ -124,11 +128,11 @@ class redis_stream { { auto act = fsm_.resume(ec, endpoints, obj_.st_, self.get_cancellation_state().cancelled()); if (act.type == connect_action_type::tcp_connect) { - auto& obj = this->obj_; // prevent use-after-free errors + auto& obj = this->obj_; // prevent use-after-move errors asio::async_connect( obj.stream_.next_layer(), std::move(endpoints), - asio::cancel_after(obj.timer_, fsm_.get_config().connect_timeout, std::move(self))); + asio::cancel_after(obj.timer_, params_.connect_timeout, std::move(self))); } else { execute_action(self, act); } @@ -172,10 +176,11 @@ class redis_stream { // I/O template - auto async_connect(const config& cfg, buffered_logger& l, CompletionToken&& token) + auto async_connect(const connect_params& params, buffered_logger& l, CompletionToken&& token) { + this->st_.type = params.addr.type(); return asio::async_compose( - connect_op{*this, connect_fsm(cfg, l)}, + connect_op{*this, connect_fsm{l}, params}, token); } diff --git a/include/boost/redis/detail/run_fsm.hpp b/include/boost/redis/detail/run_fsm.hpp index c2a17d236..b125fa746 100644 --- a/include/boost/redis/detail/run_fsm.hpp +++ b/include/boost/redis/detail/run_fsm.hpp @@ -9,6 +9,8 @@ #ifndef BOOST_REDIS_RUN_FSM_HPP #define BOOST_REDIS_RUN_FSM_HPP +#include + #include #include @@ -25,6 +27,7 @@ enum class run_action_type done, // Call the final handler immediate, // Call asio::async_immediate connect, // Transport connection establishment + sentinel_resolve, // Contact Sentinels to resolve the master's address parallel_group, // Run the reader, writer and friends cancel_receive, // Cancel the receiver channel wait_for_reconnection, // Sleep for the reconnection period @@ -57,6 +60,8 @@ class run_fsm { asio::cancellation_type_t cancel_state); }; +connect_params make_run_connect_params(const connection_state& st); + } // namespace boost::redis::detail #endif // BOOST_REDIS_CONNECTOR_HPP diff --git a/include/boost/redis/detail/sentinel_resolve_fsm.hpp b/include/boost/redis/detail/sentinel_resolve_fsm.hpp new file mode 100644 index 000000000..de8d4db69 --- /dev/null +++ b/include/boost/redis/detail/sentinel_resolve_fsm.hpp @@ -0,0 +1,93 @@ +// +// Copyright (c) 2025 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#ifndef BOOST_REDIS_SENTINEL_RESOLVE_FSM_HPP +#define BOOST_REDIS_SENTINEL_RESOLVE_FSM_HPP + +#include +#include +#include + +#include +#include +#include + +// Sans-io algorithm for async_sentinel_resolve, as a finite state machine + +namespace boost::redis::detail { + +// Forward decls +struct connection_state; + +class sentinel_action { +public: + enum class type + { + done, // Call the final handler + connect, // Transport connection establishment + request, // Send the Sentinel request + }; + + sentinel_action(system::error_code ec) noexcept + : type_(type::done) + , ec_(ec) + { } + + sentinel_action(const address& addr) noexcept + : type_(type::connect) + , connect_(&addr) + { } + + static sentinel_action request() { return {type::request}; } + + type get_type() const { return type_; } + + [[nodiscard]] + system::error_code error() const + { + BOOST_ASSERT(type_ == type::done); + return ec_; + } + + const address& connect_addr() const + { + BOOST_ASSERT(type_ == type::connect); + return *connect_; + } + +private: + type type_; + union { + system::error_code ec_; + const address* connect_; + }; + + sentinel_action(type type) noexcept + : type_(type) + { } +}; + +class sentinel_resolve_fsm { + int resume_point_{0}; + std::size_t idx_{0u}; + +public: + sentinel_resolve_fsm() = default; + + sentinel_action resume( + connection_state& st, + system::error_code ec, + asio::cancellation_type_t cancel_state); +}; + +connect_params make_sentinel_connect_params(const config& cfg, const address& sentinel_addr); +any_adapter make_sentinel_adapter(connection_state& st); + +} // namespace boost::redis::detail + +#endif // BOOST_REDIS_CONNECTOR_HPP diff --git a/include/boost/redis/error.hpp b/include/boost/redis/error.hpp index f39699f37..346e3ad1d 100644 --- a/include/boost/redis/error.hpp +++ b/include/boost/redis/error.hpp @@ -94,6 +94,24 @@ enum class error /// Timeout while writing data to the server. write_timeout, + + /// The configuration specified UNIX sockets with Sentinel, which is not supported. + sentinel_unix_sockets_unsupported, + + /// No Sentinel could be used to obtain the address of the Redis server. + /// Sentinels might be unreachable, have authentication misconfigured or may not know about + /// the configured master. Turn logging on for details. + sentinel_resolve_failed, + + /// The contacted server is not a master as expected. + /// This is likely a transient failure caused by a Sentinel failover in progress. + role_check_failed, + + /// Expects a RESP3 string, but got a different data type. + expects_resp3_string, + + /// Expects a RESP3 array, but got a different data type. + expects_resp3_array, }; /** diff --git a/include/boost/redis/impl/connect_fsm.ipp b/include/boost/redis/impl/connect_fsm.ipp index 827245888..f2699dda8 100644 --- a/include/boost/redis/impl/connect_fsm.ipp +++ b/include/boost/redis/impl/connect_fsm.ipp @@ -61,20 +61,6 @@ struct log_traits { } }; -inline transport_type transport_from_config(const config& cfg) -{ - if (cfg.unix_socket.empty()) { - if (cfg.use_ssl) { - return transport_type::tcp_tls; - } else { - return transport_type::tcp; - } - } else { - BOOST_ASSERT(!cfg.use_ssl); - return transport_type::unix_socket; - } -} - inline system::error_code translate_timeout_error( system::error_code io_ec, asio::cancellation_type_t cancel_state, @@ -105,9 +91,9 @@ connect_action connect_fsm::resume( // Log it if (ec) { - log_info(*lgr_, "Error resolving the server hostname: ", ec); + log_info(*lgr_, "Connect: hostname resolution failed: ", ec); } else { - log_info(*lgr_, "Resolve results: ", resolver_results); + log_debug(*lgr_, "Connect: hostname resolution results: ", resolver_results); } // Delegate to the regular resume function @@ -125,9 +111,9 @@ connect_action connect_fsm::resume( // Log it if (ec) { - log_info(*lgr_, "Failed to connect to the server: ", ec); + log_info(*lgr_, "Connect: TCP connect failed: ", ec); } else { - log_info(*lgr_, "Connected to ", selected_endpoint); + log_debug(*lgr_, "Connect: TCP connect succeeded. Selected endpoint: ", selected_endpoint); } // Delegate to the regular resume function @@ -142,9 +128,6 @@ connect_action connect_fsm::resume( switch (resume_point_) { BOOST_REDIS_CORO_INITIAL - // Record the transport that we will be using - st.type = transport_from_config(*cfg_); - if (st.type == transport_type::unix_socket) { // Reset the socket, to discard any previous state. Ignore any errors BOOST_REDIS_YIELD(resume_point_, 1, connect_action_type::unix_socket_close) @@ -160,9 +143,9 @@ connect_action connect_fsm::resume( // Log it if (ec) { - log_info(*lgr_, "Failed to connect to the server: ", ec); + log_info(*lgr_, "Connect: UNIX socket connect failed: ", ec); } else { - log_info(*lgr_, "Connected to ", cfg_->unix_socket); + log_debug(*lgr_, "Connect: UNIX socket connect succeeded"); } // If this failed, we can't continue @@ -178,7 +161,7 @@ connect_action connect_fsm::resume( // Must be done before anything else is done on the stream. // We don't need to close the TCP socket if using plaintext TCP // because range-connect closes open sockets, while individual connect doesn't - if (cfg_->use_ssl && st.ssl_stream_used) { + if (st.type == transport_type::tcp_tls && st.ssl_stream_used) { BOOST_REDIS_YIELD(resume_point_, 3, connect_action_type::ssl_stream_reset) } @@ -200,7 +183,7 @@ connect_action connect_fsm::resume( return ec; } - if (cfg_->use_ssl) { + if (st.type == transport_type::tcp_tls) { // Mark the SSL stream as used st.ssl_stream_used = true; @@ -212,9 +195,9 @@ connect_action connect_fsm::resume( // Log it if (ec) { - log_info(*lgr_, "Failed to perform SSL handshake: ", ec); + log_info(*lgr_, "Connect: SSL handshake failed: ", ec); } else { - log_info(*lgr_, "Successfully performed SSL handshake"); + log_debug(*lgr_, "Connect: SSL handshake succeeded"); } // If this failed, we can't continue diff --git a/include/boost/redis/impl/error.ipp b/include/boost/redis/impl/error.ipp index 0ac39952c..f70715064 100644 --- a/include/boost/redis/impl/error.ipp +++ b/include/boost/redis/impl/error.ipp @@ -55,8 +55,19 @@ struct error_category_impl : system::error_category { case error::exceeds_maximum_read_buffer_size: return "Reading data from the socket would exceed the maximum size allowed of the read " "buffer."; - case error::write_timeout: - return "Timeout while writing data to the server."; + case error::write_timeout: return "Timeout while writing data to the server."; + case error::sentinel_unix_sockets_unsupported: + return "The configuration specified UNIX sockets with Sentinel, which is not " + "supported."; + case error::sentinel_resolve_failed: + return "No Sentinel could be used to obtain the address of the Redis server."; + case error::role_check_failed: + return "The contacted server does not have the expected role. " + "This is likely a transient failure caused by a Sentinel failover in progress."; + case error::expects_resp3_string: + return "Expects a RESP3 string, but got a different data type."; + case error::expects_resp3_array: + return "Expects a RESP3 array, but got a different data type."; default: BOOST_ASSERT(false); return "Boost.Redis error."; } } diff --git a/include/boost/redis/impl/exec_one_fsm.ipp b/include/boost/redis/impl/exec_one_fsm.ipp new file mode 100644 index 000000000..b4a7250b2 --- /dev/null +++ b/include/boost/redis/impl/exec_one_fsm.ipp @@ -0,0 +1,95 @@ +// +// Copyright (c) 2025 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#ifndef BOOST_REDIS_EXEC_ONE_FSM_IPP +#define BOOST_REDIS_EXEC_ONE_FSM_IPP + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +namespace boost::redis::detail { + +exec_one_action exec_one_fsm::resume( + read_buffer& buffer, + system::error_code ec, + std::size_t bytes_transferred, + asio::cancellation_type_t cancel_state) +{ + switch (resume_point_) { + BOOST_REDIS_CORO_INITIAL + + // Send the request to the server + BOOST_REDIS_YIELD(resume_point_, 1, exec_one_action_type::write) + + // Errors and cancellations + if (is_terminal_cancel(cancel_state)) + return system::error_code{asio::error::operation_aborted}; + if (ec) + return ec; + + // If the request didn't expect any response, we're done + if (remaining_responses_ == 0u) + return system::error_code{}; + + // Read responses until we're done + buffer.clear(); + while (true) { + // Prepare the buffer to read some data + ec = buffer.prepare(); + if (ec) + return ec; + + // Read data + BOOST_REDIS_YIELD(resume_point_, 2, exec_one_action_type::read_some) + + // Errors and cancellations + if (is_terminal_cancel(cancel_state)) + return system::error_code{asio::error::operation_aborted}; + if (ec) + return ec; + + // Commit the data into the buffer + buffer.commit(bytes_transferred); + + // Consume the data until we run out or all the responses have been read + while (resp3::parse(parser_, buffer.get_commited(), adapter_, ec)) { + // Check for errors + if (ec) + return ec; + + // We've finished parsing a response + buffer.consume(parser_.get_consumed()); + parser_.reset(); + + // When no more responses remain, we're done. + // Don't read ahead, even if more data is available + if (--remaining_responses_ == 0u) + return system::error_code{}; + } + } + } + + BOOST_ASSERT(false); + return system::error_code(); +} + +} // namespace boost::redis::detail + +#endif diff --git a/include/boost/redis/impl/log_utils.hpp b/include/boost/redis/impl/log_utils.hpp index e4b7349ca..5515319d2 100644 --- a/include/boost/redis/impl/log_utils.hpp +++ b/include/boost/redis/impl/log_utils.hpp @@ -7,6 +7,7 @@ #ifndef BOOST_REDIS_LOG_UTILS_HPP #define BOOST_REDIS_LOG_UTILS_HPP +#include #include #include @@ -48,6 +49,16 @@ struct log_traits { } }; +template <> +struct log_traits
{ + static inline void log(std::string& to, const address& value) + { + to += value.host; + to += ':'; + to += value.port; + } +}; + template void format_log_args(std::string& to, const Args&... args) { diff --git a/include/boost/redis/impl/run_fsm.ipp b/include/boost/redis/impl/run_fsm.ipp index d3d08cdd2..845f9d202 100644 --- a/include/boost/redis/impl/run_fsm.ipp +++ b/include/boost/redis/impl/run_fsm.ipp @@ -8,12 +8,15 @@ #include #include +#include #include #include #include #include +#include #include #include +#include #include #include @@ -28,6 +31,8 @@ inline system::error_code check_config(const config& cfg) if (!cfg.unix_socket.empty()) { if (cfg.use_ssl) return error::unix_sockets_ssl_unsupported; + if (use_sentinel(cfg)) + return error::sentinel_unix_sockets_unsupported; #ifndef BOOST_ASIO_HAS_LOCAL_SOCKETS return error::unix_sockets_unsupported; #endif @@ -41,45 +46,44 @@ inline void compose_ping_request(const config& cfg, request& to) to.push("PING", cfg.health_check_id); } -inline void process_setup_node( - connection_state& st, - resp3::basic_node const& nd, - system::error_code& ec) -{ - switch (nd.data_type) { - case resp3::type::simple_error: - case resp3::type::blob_error: - case resp3::type::null: - ec = redis::error::resp3_hello; - st.setup_diagnostic = nd.value; - break; - default:; - } -} - -inline any_adapter make_setup_adapter(connection_state& st) -{ - return any_adapter{ - [&st](any_adapter::parse_event evt, resp3::node_view const& nd, system::error_code& ec) { - if (evt == any_adapter::parse_event::node) - process_setup_node(st, nd, ec); - }}; -} - inline void on_setup_done(const multiplexer::elem& elm, connection_state& st) { const auto ec = elm.get_error(); if (ec) { - if (st.setup_diagnostic.empty()) { + if (st.diagnostic.empty()) { log_info(st.logger, "Setup request execution: ", ec); } else { - log_info(st.logger, "Setup request execution: ", ec, " (", st.setup_diagnostic, ")"); + log_info(st.logger, "Setup request execution: ", ec, " (", st.diagnostic, ")"); } } else { log_info(st.logger, "Setup request execution: success"); } } +inline any_address_view get_server_address(const connection_state& st) +{ + if (st.cfg.unix_socket.empty()) { + return {st.cfg.addr, st.cfg.use_ssl}; + } else { + return any_address_view{st.cfg.unix_socket}; + } +} + +template <> +struct log_traits { + static inline void log(std::string& to, any_address_view value) + { + if (value.type() == transport_type::unix_socket) { + to += '\''; + to += value.unix_socket(); + to += '\''; + } else { + log_traits
::log(to, value.tcp_address()); + to += value.type() == transport_type::tcp_tls ? " (TLS enabled)" : " (TLS disabled)"; + } + } +}; + run_action run_fsm::resume( connection_state& st, system::error_code ec, @@ -103,9 +107,34 @@ run_action run_fsm::resume( // Compose the PING request. Same as above compose_ping_request(st.cfg, st.ping_req); + if (use_sentinel(st.cfg)) { + // Sentinel request. Same as above + compose_sentinel_request(st.cfg); + + // Bootstrap the sentinel list with the ones configured by the user + st.sentinels = st.cfg.sentinel.addresses; + } + for (;;) { + // Sentinel resolve, if required. This leaves the address in st.cfg.address + if (use_sentinel(st.cfg)) { + // This operation does the logging for us. + BOOST_REDIS_YIELD(resume_point_, 2, run_action_type::sentinel_resolve) + + // Check for cancellations + if (is_terminal_cancel(cancel_state)) { + log_debug(st.logger, "Run: cancelled (4)"); + return {asio::error::operation_aborted}; + } + + // Check for errors + if (ec) + goto sleep_and_reconnect; + } + // Try to connect - BOOST_REDIS_YIELD(resume_point_, 2, run_action_type::connect) + log_info(st.logger, "Trying to connect to Redis server at ", get_server_address(st)); + BOOST_REDIS_YIELD(resume_point_, 4, run_action_type::connect) // Check for cancellations if (is_terminal_cancel(cancel_state)) { @@ -113,39 +142,50 @@ run_action run_fsm::resume( return system::error_code(asio::error::operation_aborted); } - // If we were successful, run all the connection tasks - if (!ec) { - // Initialization - st.mpx.reset(); - st.setup_diagnostic.clear(); - - // Add the setup request to the multiplexer - if (st.cfg.setup.get_commands() != 0u) { - auto elm = make_elem(st.cfg.setup, make_setup_adapter(st)); - elm->set_done_callback([&elem_ref = *elm, &st] { - on_setup_done(elem_ref, st); - }); - st.mpx.add(elm); - } + if (ec) { + // There was an error. Skip to the reconnection loop + log_info( + st.logger, + "Failed to connect to Redis server at ", + get_server_address(st), + ": ", + ec); + goto sleep_and_reconnect; + } + + // We were successful + log_info(st.logger, "Connected to Redis server at ", get_server_address(st)); - // Run the tasks - BOOST_REDIS_YIELD(resume_point_, 3, run_action_type::parallel_group) + // Initialization + st.mpx.reset(); + st.diagnostic.clear(); - // Store any error yielded by the tasks for later - stored_ec_ = ec; + // Add the setup request to the multiplexer + if (st.cfg.setup.get_commands() != 0u) { + auto elm = make_elem(st.cfg.setup, make_any_adapter_impl(setup_adapter{st})); + elm->set_done_callback([&elem_ref = *elm, &st] { + on_setup_done(elem_ref, st); + }); + st.mpx.add(elm); + } - // We've lost connection or otherwise been cancelled. - // Remove from the multiplexer the required requests. - st.mpx.cancel_on_conn_lost(); + // Run the tasks + BOOST_REDIS_YIELD(resume_point_, 5, run_action_type::parallel_group) - // The receive operation must be cancelled because channel - // subscription does not survive a reconnection but requires - // re-subscription. - BOOST_REDIS_YIELD(resume_point_, 4, run_action_type::cancel_receive) + // Store any error yielded by the tasks for later + stored_ec_ = ec; - // Restore the error - ec = stored_ec_; - } + // We've lost connection or otherwise been cancelled. + // Remove from the multiplexer the required requests. + st.mpx.cancel_on_conn_lost(); + + // The receive operation must be cancelled because channel + // subscription does not survive a reconnection but requires + // re-subscription. + BOOST_REDIS_YIELD(resume_point_, 6, run_action_type::cancel_receive) + + // Restore the error + ec = stored_ec_; // Check for cancellations if (is_terminal_cancel(cancel_state)) { @@ -153,13 +193,15 @@ run_action run_fsm::resume( return system::error_code(asio::error::operation_aborted); } +sleep_and_reconnect: + // If we are not going to try again, we're done if (st.cfg.reconnect_wait_interval.count() == 0) { return ec; } // Wait for the reconnection interval - BOOST_REDIS_YIELD(resume_point_, 5, run_action_type::wait_for_reconnection) + BOOST_REDIS_YIELD(resume_point_, 7, run_action_type::wait_for_reconnection) // Check for cancellations if (is_terminal_cancel(cancel_state)) { @@ -174,4 +216,14 @@ run_action run_fsm::resume( return system::error_code(); } +connect_params make_run_connect_params(const connection_state& st) +{ + return { + get_server_address(st), + st.cfg.resolve_timeout, + st.cfg.connect_timeout, + st.cfg.ssl_handshake_timeout, + }; +} + } // namespace boost::redis::detail diff --git a/include/boost/redis/impl/sentinel_resolve_fsm.ipp b/include/boost/redis/impl/sentinel_resolve_fsm.ipp new file mode 100644 index 000000000..96cf28f92 --- /dev/null +++ b/include/boost/redis/impl/sentinel_resolve_fsm.ipp @@ -0,0 +1,182 @@ +// +// Copyright (c) 2025 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#ifndef BOOST_REDIS_SENTINEL_RESOLVE_FSM_IPP +#define BOOST_REDIS_SENTINEL_RESOLVE_FSM_IPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +namespace boost::redis::detail { + +// Logs an error at info level, and also stores it in the state, +// so it can be logged at error level if all Sentinels fail. +template +void log_sentinel_error(connection_state& st, std::size_t current_idx, const Args&... args) +{ + st.diagnostic += "\n "; + std::size_t size_before = st.diagnostic.size(); + format_log_args(st.diagnostic, "Sentinel at ", st.sentinels[current_idx], ": ", args...); + log_info(st.logger, std::string_view{st.diagnostic}.substr(size_before)); +} + +sentinel_action sentinel_resolve_fsm::resume( + connection_state& st, + system::error_code ec, + asio::cancellation_type_t cancel_state) +{ + switch (resume_point_) { + BOOST_REDIS_CORO_INITIAL + + st.diagnostic.clear(); + + log_info( + st.logger, + "Trying to resolve the address of ", + st.cfg.sentinel.server_role == role::master ? "master" : "a replica of master", + " '", + st.cfg.sentinel.master_name, + "' using Sentinel"); + + // Try all Sentinels in order. Upon any errors, save the diagnostic and try with the next one. + // If none of them are available, print an error diagnostic and fail. + for (idx_ = 0u; idx_ < st.sentinels.size(); ++idx_) { + log_debug(st.logger, "Trying to contact Sentinel at ", st.sentinels[idx_]); + + // Try to connect + BOOST_REDIS_YIELD(resume_point_, 1, st.sentinels[idx_]) + + // Check for cancellations + if (is_terminal_cancel(cancel_state)) { + log_debug(st.logger, "Sentinel resolve: cancelled (1)"); + return system::error_code(asio::error::operation_aborted); + } + + // Check for errors + if (ec) { + log_sentinel_error(st, idx_, "connection establishment error: ", ec); + continue; + } + + // Execute the Sentinel request + log_debug(st.logger, "Executing Sentinel request at ", st.sentinels[idx_]); + st.sentinel_resp_nodes.clear(); + BOOST_REDIS_YIELD(resume_point_, 2, sentinel_action::request()) + + // Check for cancellations + if (is_terminal_cancel(cancel_state)) { + log_debug(st.logger, "Sentinel resolve: cancelled (2)"); + return system::error_code(asio::error::operation_aborted); + } + + // Check for errors + if (ec) { + log_sentinel_error(st, idx_, "error while executing request: ", ec); + continue; + } + + // Parse the response + sentinel_response resp; + ec = parse_sentinel_response(st.sentinel_resp_nodes, st.cfg.sentinel.server_role, resp); + + if (ec) { + if (ec == error::resp3_simple_error || ec == error::resp3_blob_error) { + log_sentinel_error(st, idx_, "responded with an error: ", resp.diagnostic); + } else if (ec == error::resp3_null) { + log_sentinel_error(st, idx_, "doesn't know about the configured master"); + } else { + log_sentinel_error( + st, + idx_, + "error parsing response (maybe forgot to upgrade to RESP3?): ", + ec); + } + + continue; + } + + // When asking for replicas, we might get no replicas + if (st.cfg.sentinel.server_role == role::replica && resp.replicas.empty()) { + log_sentinel_error(st, idx_, "the configured master has no replicas"); + continue; + } + + // Store the resulting address in a well-known place + if (st.cfg.sentinel.server_role == role::master) { + st.cfg.addr = resp.master_addr; + } else { + // Choose a random replica + std::uniform_int_distribution dist{0u, resp.replicas.size() - 1u}; + const auto idx = dist(st.eng.get()); + st.cfg.addr = resp.replicas[idx]; + } + + // Sentinel knows about this master. Log and update our config + log_info( + st.logger, + "Sentinel at ", + st.sentinels[idx_], + " resolved the server address to ", + st.cfg.addr); + + update_sentinel_list(st.sentinels, idx_, resp.sentinels, st.cfg.sentinel.addresses); + + st.sentinel_resp_nodes.clear(); // reduce memory consumption + return system::error_code(); + } + + // No Sentinel resolved our address + log_err( + st.logger, + "Failed to resolve the address of ", + st.cfg.sentinel.server_role == role::master ? "master" : "a replica of master", + " '", + st.cfg.sentinel.master_name, + "'. Tried the following Sentinels:", + st.diagnostic); + return {error::sentinel_resolve_failed}; + } + + // We should never get here + BOOST_ASSERT(false); + return system::error_code(); +} + +connect_params make_sentinel_connect_params(const config& cfg, const address& addr) +{ + return { + any_address_view{addr, cfg.sentinel.use_ssl}, + cfg.sentinel.resolve_timeout, + cfg.sentinel.connect_timeout, + cfg.sentinel.ssl_handshake_timeout, + }; +} + +any_adapter make_sentinel_adapter(connection_state& st) +{ + return any_adapter(st.sentinel_resp_nodes); +} + +} // namespace boost::redis::detail + +#endif diff --git a/include/boost/redis/impl/sentinel_utils.hpp b/include/boost/redis/impl/sentinel_utils.hpp new file mode 100644 index 000000000..b3df430ba --- /dev/null +++ b/include/boost/redis/impl/sentinel_utils.hpp @@ -0,0 +1,277 @@ +// +// Copyright (c) 2025 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#ifndef BOOST_REDIS_SENTINEL_UTILS_HPP +#define BOOST_REDIS_SENTINEL_UTILS_HPP + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace boost::redis::detail { + +// Returns true if Sentinel should be used +inline bool use_sentinel(const config& cfg) { return !cfg.sentinel.addresses.empty(); } + +// Composes the request to send to Sentinel modifying cfg.sentinel.setup +inline void compose_sentinel_request(config& cfg) +{ + // These commands should go after the user-supplied setup, as this might involve authentication. + // We ask for the master even when connecting to replicas to correctly detect when the master doesn't exist + cfg.sentinel.setup.push("SENTINEL", "GET-MASTER-ADDR-BY-NAME", cfg.sentinel.master_name); + if (cfg.sentinel.server_role == role::replica) + cfg.sentinel.setup.push("SENTINEL", "REPLICAS", cfg.sentinel.master_name); + cfg.sentinel.setup.push("SENTINEL", "SENTINELS", cfg.sentinel.master_name); + + // Note that we don't care about request flags because this is a one-time request +} + +// Parses a list of replicas or sentinels +inline system::error_code parse_server_list( + const resp3::node*& first, + const resp3::node* last, + std::vector
& out) +{ + const auto* it = first; + ignore_unused(last); + + // The root node must be an array + BOOST_ASSERT(it != last); + BOOST_ASSERT(it->depth == 0u); + if (it->data_type != resp3::type::array) + return {error::expects_resp3_array}; + const std::size_t num_servers = it->aggregate_size; + ++it; + + // Each element in the array represents a server + out.resize(num_servers); + for (std::size_t i = 0u; i < num_servers; ++i) { + // A server is a map (resp3) or array (resp2, currently unsupported) + BOOST_ASSERT(it != last); + BOOST_ASSERT(it->depth == 1u); + if (it->data_type != resp3::type::map) + return {error::expects_resp3_map}; + const std::size_t num_key_values = it->aggregate_size; + ++it; + + // The server object is composed by a set of key/value pairs. + // Skip everything except for the ones we care for. + bool ip_seen = false, port_seen = false; + for (std::size_t j = 0; j < num_key_values; ++j) { + // Key. It should be a string + BOOST_ASSERT(it != last); + BOOST_ASSERT(it->depth == 2u); + if (it->data_type != resp3::type::blob_string) + return {error::expects_resp3_string}; + const std::string_view key = it->value; + ++it; + + // Value. All values seem to be strings, too. + BOOST_ASSERT(it != last); + BOOST_ASSERT(it->depth == 2u); + if (it->data_type != resp3::type::blob_string) + return {error::expects_resp3_string}; + + // Record it + if (key == "ip") { + ip_seen = true; + out[i].host = it->value; + } else if (key == "port") { + port_seen = true; + out[i].port = it->value; + } + + ++it; + } + + // Check that the response actually contained the fields we wanted + if (!ip_seen || !port_seen) + return {error::empty_field}; + } + + // Done + first = it; + return system::error_code(); +} + +// The output type of parse_sentinel_response +struct sentinel_response { + std::string diagnostic; // In case the server returned an error + address master_addr; // Always populated + std::vector
replicas; // Populated only when connecting to replicas + std::vector
sentinels; +}; + +// Parses an array of nodes into a sentinel_response. +// The request originating this response should be: +// +// SENTINEL GET-MASTER-ADDR-BY-NAME +// SENTINEL REPLICAS (only if server_role is replica) +// SENTINEL SENTINELS +// SENTINEL SENTINELS and SENTINEL REPLICAS error when the master name is unknown. Error nodes +// should be allowed in the node array. +// This means that we can't use generic_response, since its adapter errors on error nodes. +// SENTINEL GET-MASTER-ADDR-BY-NAME is sent even when connecting to replicas +// for better diagnostics when the master name is unknown. +// Preconditions: +// * There are at least 2 (master)/3 (replica) root nodes. +// * The node array originates from parsing a valid RESP3 message. +// E.g. we won't check that the first node has depth 0. +inline system::error_code parse_sentinel_response( + span nodes, + role server_role, + sentinel_response& out) +{ + auto check_errors = [&out](const resp3::node& nd) { + switch (nd.data_type) { + case resp3::type::simple_error: + out.diagnostic = nd.value; + return system::error_code(error::resp3_simple_error); + case resp3::type::blob_error: + out.diagnostic = nd.value; + return system::error_code(error::resp3_blob_error); + default: return system::error_code(); + } + }; + + // Clear the output + out.diagnostic.clear(); + out.sentinels.clear(); + out.replicas.clear(); + + // Find the first root node of interest. It's the 2nd or 3rd, starting with the end + auto find_first = [nodes, server_role] { + const std::size_t expected_roots = server_role == role::master ? 2u : 3u; + std::size_t roots_seen = 0u; + for (auto it = nodes.rbegin();; ++it) { + BOOST_ASSERT(it != nodes.rend()); + if (it->depth == 0u && ++roots_seen == expected_roots) + return &*it; + } + }; + const resp3::node* lib_first = find_first(); + + // Iterators + const resp3::node* it = nodes.begin(); + const resp3::node* last = nodes.end(); + ignore_unused(last); + + // Go through all the responses to user-supplied requests checking for errors + for (; it != lib_first; ++it) { + if (auto ec = check_errors(*it)) + return ec; + } + + // SENTINEL GET-MASTER-ADDR-BY-NAME + + // Check for errors + if (auto ec = check_errors(*it)) + return ec; + + // If the root node is NULL, Sentinel doesn't know about this master. + // We use resp3_null to signal this fact. This doesn't reach the end user. + if (it->data_type == resp3::type::null) { + return {error::resp3_null}; + } + + // If the root node is an array, an IP and port follow + if (it->data_type != resp3::type::array) + return {error::expects_resp3_array}; + if (it->aggregate_size != 2u) + return {error::incompatible_size}; + ++it; + + // IP + BOOST_ASSERT(it != last); + BOOST_ASSERT(it->depth == 1u); + if (it->data_type != resp3::type::blob_string) + return {error::expects_resp3_string}; + out.master_addr.host = it->value; + ++it; + + // Port + BOOST_ASSERT(it != last); + BOOST_ASSERT(it->depth == 1u); + if (it->data_type != resp3::type::blob_string) + return {error::expects_resp3_string}; + out.master_addr.port = it->value; + ++it; + + if (server_role == role::replica) { + // SENTINEL REPLICAS + + // This request fails if Sentinel doesn't know about this master. + // However, that's not the case if we got here. + // Check for other errors. + if (auto ec = check_errors(*it)) + return ec; + + // Actual parsing + if (auto ec = parse_server_list(it, last, out.replicas)) + return ec; + } + + // SENTINEL SENTINELS + + // This request fails if Sentinel doesn't know about this master. + // However, that's not the case if we got here. + // Check for other errors. + if (auto ec = check_errors(*it)) + return ec; + + // Actual parsing + if (auto ec = parse_server_list(it, last, out.sentinels)) + return ec; + + // Done + return system::error_code(); +} + +// Updates the internal Sentinel list. +// to should never be empty +inline void update_sentinel_list( + std::vector
& to, + std::size_t current_index, // the one to maintain and place first + span gossip_sentinels, // the ones that SENTINEL SENTINELS returned + span bootstrap_sentinels // the ones the user supplied +) +{ + BOOST_ASSERT(!to.empty()); + + // Remove everything, except the Sentinel that succeeded + if (current_index != 0u) + std::swap(to.front(), to[current_index]); + to.resize(1u); + + // Add one group. These Sentinels are always unique and don't include the one we're currently connected to. + to.insert(to.end(), gossip_sentinels.begin(), gossip_sentinels.end()); + + // Insert any user-supplied sentinels, if not already present. + // This is O(n^2), but is okay because n will be small. + // The list can't be sorted, anyway + for (const auto& sentinel : bootstrap_sentinels) { + if (std::find(to.begin(), to.end(), sentinel) == to.end()) + to.push_back(sentinel); + } +} + +} // namespace boost::redis::detail + +#endif diff --git a/include/boost/redis/impl/setup_request_utils.hpp b/include/boost/redis/impl/setup_request_utils.hpp index 97053f2f1..c220d98e9 100644 --- a/include/boost/redis/impl/setup_request_utils.hpp +++ b/include/boost/redis/impl/setup_request_utils.hpp @@ -8,19 +8,27 @@ #define BOOST_REDIS_SETUP_REQUEST_UTILS_HPP #include +#include +#include +#include // use_sentinel #include +#include +#include #include +#include + namespace boost::redis::detail { // Modifies config::setup to make a request suitable to be sent // to the server using async_exec inline void compose_setup_request(config& cfg) { + auto& req = cfg.setup; + if (!cfg.use_setup) { // We're not using the setup request as-is, but should compose one based on // the values passed by the user - auto& req = cfg.setup; req.clear(); // Which parts of the command should we send? @@ -46,14 +54,71 @@ inline void compose_setup_request(config& cfg) req.push("SELECT", cfg.database_index.value()); } + // When using Sentinel, we should add a role check. + // This must happen after the other commands, as it requires authentication. + if (use_sentinel(cfg)) + req.push("ROLE"); + // In any case, the setup request should have the priority // flag set so it's executed before any other request. // The setup request should never be retried. - request_access::set_priority(cfg.setup, true); - cfg.setup.get_config().cancel_if_unresponded = true; - cfg.setup.get_config().cancel_on_connection_lost = true; + request_access::set_priority(req, true); + req.get_config().cancel_if_unresponded = true; + req.get_config().cancel_on_connection_lost = true; } +class setup_adapter { + connection_state* st_; + std::size_t response_idx_{0u}; + bool role_seen_{false}; + + system::error_code on_node_impl(const resp3::node_view& nd) + { + // An error node is always an error + switch (nd.data_type) { + case resp3::type::simple_error: + case resp3::type::blob_error: st_->diagnostic = nd.value; return error::resp3_hello; + default: ; + } + + // When using Sentinel, we add a ROLE command at the end. + // We need to ensure that this instance is a master. + if (use_sentinel(st_->cfg) && response_idx_ == st_->cfg.setup.get_expected_responses() - 1u) { + // ROLE's response should be an array of at least 1 element + if (nd.depth == 0u) { + if (nd.data_type != resp3::type::array) + return error::invalid_data_type; + if (nd.aggregate_size == 0u) + return error::incompatible_size; + } + + // The first node should be 'master' if we're connecting to a primary, + // 'slave' if we're connecting to a replica + if (nd.depth == 1u && !role_seen_) { + role_seen_ = true; + if (nd.data_type != resp3::type::blob_string) + return error::invalid_data_type; + + const char* expected_role = st_->cfg.sentinel.server_role == role::master ? "master" + : "slave"; + if (nd.value != expected_role) + return error::role_check_failed; + } + } + + return system::error_code(); + } + +public: + explicit setup_adapter(connection_state& st) noexcept + : st_(&st) + { } + + void on_init() { } + void on_done() { ++response_idx_; } + void on_node(const resp3::node_view& node, system::error_code& ec) { ec = on_node_impl(node); } +}; + } // namespace boost::redis::detail #endif // BOOST_REDIS_RUNNER_HPP diff --git a/include/boost/redis/src.hpp b/include/boost/redis/src.hpp index 06c2dcd3b..eb48981b9 100644 --- a/include/boost/redis/src.hpp +++ b/include/boost/redis/src.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -16,6 +17,7 @@ #include #include #include +#include #include #include #include diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 4a9bf03c2..58ad78a4f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -37,15 +37,20 @@ make_test(test_low_level) make_test(test_request) make_test(test_low_level_sync_sans_io) make_test(test_any_adapter) -make_test(test_exec_fsm) make_test(test_log_to_file) make_test(test_conn_logging) +make_test(test_exec_fsm) +make_test(test_exec_one_fsm) make_test(test_writer_fsm) make_test(test_reader_fsm) make_test(test_connect_fsm) +make_test(test_sentinel_resolve_fsm) make_test(test_run_fsm) make_test(test_setup_request_utils) +make_test(test_setup_adapter) make_test(test_multiplexer) +make_test(test_parse_sentinel_response) +make_test(test_update_sentinel_list) # Tests that require a real Redis server make_test(test_conn_quit) @@ -68,6 +73,7 @@ make_test(test_conversions) make_test(test_conn_tls) make_test(test_unix_sockets) make_test(test_conn_cancel_after) +make_test(test_conn_sentinel) # Coverage set( diff --git a/test/Jamfile b/test/Jamfile index 8e607279b..4ac93900a 100644 --- a/test/Jamfile +++ b/test/Jamfile @@ -54,15 +54,20 @@ local tests = test_request test_low_level_sync_sans_io test_any_adapter - test_exec_fsm test_log_to_file test_conn_logging + test_exec_fsm + test_exec_one_fsm test_writer_fsm test_reader_fsm + test_sentinel_resolve_fsm test_run_fsm test_connect_fsm test_setup_request_utils + test_setup_adapter test_multiplexer + test_parse_sentinel_response + test_update_sentinel_list ; # Build and run the tests diff --git a/test/common.cpp b/test/common.cpp index f0533bf39..cb7e1891d 100644 --- a/test/common.cpp +++ b/test/common.cpp @@ -1,11 +1,16 @@ +#include +#include + #include #include +#include #include "common.hpp" #include #include #include +#include namespace net = boost::asio; @@ -71,7 +76,6 @@ void run_coroutine_test(net::awaitable op, std::chrono::steady_clock::dura // Finds a value in the output of the CLIENT INFO command // format: key1=value1 key2=value2 -// TODO: duplicated std::string_view find_client_info(std::string_view client_info, std::string_view key) { std::string prefix{key}; @@ -84,3 +88,45 @@ std::string_view find_client_info(std::string_view client_info, std::string_view auto const pos_end = client_info.find(' ', pos_begin); return client_info.substr(pos_begin, pos_end - pos_begin); } + +void create_user(std::string_view port, std::string_view username, std::string_view password) +{ + // Setup + net::io_context ioc; + boost::redis::connection conn{ioc}; + + boost::redis::config cfg; + cfg.addr.port = port; + + // Enable the user and grant them permissions on everything + boost::redis::request req; + req.push("ACL", "SETUSER", username, "on", ">" + std::string(password), "~*", "&*", "+@all"); + + bool run_finished = false, exec_finished = false; + + conn.async_run(cfg, [&](boost::system::error_code ec) { + run_finished = true; + BOOST_TEST_EQ(ec, net::error::operation_aborted); + }); + + conn.async_exec(req, boost::redis::ignore, [&](boost::system::error_code ec, std::size_t) { + exec_finished = true; + BOOST_TEST_EQ(ec, boost::system::error_code()); + conn.cancel(); + }); + + ioc.run_for(test_timeout); + + BOOST_TEST(run_finished); + BOOST_TEST(exec_finished); +} + +boost::redis::logger make_string_logger(std::string& to) +{ + return { + boost::redis::logger::level::info, + [&to](boost::redis::logger::level, std::string_view msg) { + to += msg; + to += '\n'; + }}; +} diff --git a/test/common.hpp b/test/common.hpp index af1905806..569a7bcd5 100644 --- a/test/common.hpp +++ b/test/common.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -11,6 +12,7 @@ #include #include +#include #include // The timeout for tests involving communication to a real server. @@ -40,3 +42,8 @@ void run( // Finds a value in the output of the CLIENT INFO command // format: key1=value1 key2=value2 std::string_view find_client_info(std::string_view client_info, std::string_view key); + +// Connects to the Redis server at the given port and creates a user +void create_user(std::string_view port, std::string_view username, std::string_view password); + +boost::redis::logger make_string_logger(std::string& to); diff --git a/test/print_node.hpp b/test/print_node.hpp new file mode 100644 index 000000000..9c762cb35 --- /dev/null +++ b/test/print_node.hpp @@ -0,0 +1,28 @@ + +/* Copyright (c) 2018-2025 Marcelo Zimbres Silva (mzimbres@gmail.com) + * + * Distributed under the Boost Software License, Version 1.0. (See + * accompanying file LICENSE.txt) + */ + +#ifndef BOOST_REDIS_TEST_PRINT_NODE_HPP +#define BOOST_REDIS_TEST_PRINT_NODE_HPP + +#include +#include + +#include + +namespace boost::redis::resp3 { + +template +std::ostream& operator<<(std::ostream& os, basic_node const& nd) +{ + return os << "node{ .data_type=" << to_string(nd.data_type) + << ", .aggregate_size=" << nd.aggregate_size << ", .depth=" << nd.depth + << ", .value=" << nd.value << "}"; +} + +} // namespace boost::redis::resp3 + +#endif // BOOST_REDIS_TEST_SANSIO_UTILS_HPP diff --git a/test/sansio_utils.cpp b/test/sansio_utils.cpp index 294763e50..b3e446ac0 100644 --- a/test/sansio_utils.cpp +++ b/test/sansio_utils.cpp @@ -4,8 +4,10 @@ * accompanying file LICENSE.txt) */ +#include #include +#include #include #include @@ -72,4 +74,24 @@ logger log_fixture::make_logger() }); } +std::vector nodes_from_resp3( + const std::vector& msgs, + source_location loc) +{ + std::vector nodes; + any_adapter adapter{nodes}; + + for (std::string_view resp : msgs) { + resp3::parser p; + system::error_code ec; + bool done = resp3::parse(p, resp, adapter, ec); + if (!BOOST_TEST(done)) + std::cerr << "Called from " << loc << std::endl; + if (!BOOST_TEST_EQ(ec, system::error_code())) + std::cerr << "Called from " << loc << std::endl; + } + + return nodes; +} + } // namespace boost::redis::detail diff --git a/test/sansio_utils.hpp b/test/sansio_utils.hpp index 47a49ff32..cd4d7ebc6 100644 --- a/test/sansio_utils.hpp +++ b/test/sansio_utils.hpp @@ -8,6 +8,7 @@ #define BOOST_REDIS_TEST_SANSIO_UTILS_HPP #include +#include #include @@ -15,6 +16,7 @@ #include #include #include +#include namespace boost::redis::detail { @@ -50,6 +52,13 @@ constexpr auto to_milliseconds(std::chrono::steady_clock::duration d) return std::chrono::duration_cast(d).count(); } +// Creates a vector of nodes from a set of RESP3 messages. +// Using the raw RESP values ensures that the correct +// node tree is built, which is not always obvious +std::vector nodes_from_resp3( + const std::vector& msgs, + source_location loc = BOOST_CURRENT_LOCATION); + } // namespace boost::redis::detail #endif // BOOST_REDIS_TEST_SANSIO_UTILS_HPP diff --git a/test/test_conn_sentinel.cpp b/test/test_conn_sentinel.cpp new file mode 100644 index 000000000..7caec74ec --- /dev/null +++ b/test/test_conn_sentinel.cpp @@ -0,0 +1,491 @@ +// +// Copyright (c) 2025 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "common.hpp" +#include "print_node.hpp" + +#include + +namespace net = boost::asio; +using namespace boost::redis; +using namespace std::chrono_literals; +using boost::system::error_code; + +namespace { + +// We can execute requests normally when using Sentinel run +void test_exec() +{ + // Setup + net::io_context ioc; + connection conn{ioc}; + + config cfg; + cfg.sentinel.addresses = { + {"localhost", "26379"}, + {"localhost", "26380"}, + {"localhost", "26381"}, + }; + cfg.sentinel.master_name = "mymaster"; + + // Verify that we're connected to the master + request req; + req.push("ROLE"); + + generic_response resp; + + bool exec_finished = false, run_finished = false; + + conn.async_exec(req, resp, [&](error_code ec, std::size_t) { + exec_finished = true; + BOOST_TEST_EQ(ec, error_code()); + + // ROLE outputs an array, 1st element should be 'master' + BOOST_TEST(resp.has_value()); + BOOST_TEST_GE(resp.value().size(), 2u); + BOOST_TEST_EQ(resp.value().at(1u).value, "master"); + + conn.cancel(); + }); + + conn.async_run(cfg, [&](error_code ec) { + run_finished = true; + BOOST_TEST_EQ(ec, net::error::operation_aborted); + }); + + ioc.run_for(test_timeout); + + BOOST_TEST(exec_finished); + BOOST_TEST(run_finished); +} + +// We can use receive normally when using Sentinel run +void test_receive() +{ + // Setup + net::io_context ioc; + connection conn{ioc}; + + config cfg; + cfg.sentinel.addresses = { + {"localhost", "26379"}, + {"localhost", "26380"}, + {"localhost", "26381"}, + }; + cfg.sentinel.master_name = "mymaster"; + + resp3::tree resp; + conn.set_receive_response(resp); + + // Subscribe to a channel. This produces a push message on itself + request req; + req.push("SUBSCRIBE", "sentinel_channel"); + + bool exec_finished = false, receive_finished = false, run_finished = false; + + conn.async_exec(req, resp, [&](error_code ec, std::size_t) { + exec_finished = true; + BOOST_TEST_EQ(ec, error_code()); + }); + + conn.async_receive2([&](error_code ec2) { + receive_finished = true; + BOOST_TEST_EQ(ec2, error_code()); + conn.cancel(); + }); + + conn.async_run(cfg, [&](error_code ec) { + run_finished = true; + BOOST_TEST_EQ(ec, net::error::operation_aborted); + }); + + ioc.run_for(test_timeout); + + BOOST_TEST(exec_finished); + BOOST_TEST(receive_finished); + BOOST_TEST(run_finished); + + // We subscribed to channel 'sentinel_channel', and have 1 active subscription + const resp3::node expected[] = { + {resp3::type::push, 3u, 0u, "" }, + {resp3::type::blob_string, 1u, 1u, "subscribe" }, + {resp3::type::blob_string, 1u, 1u, "sentinel_channel"}, + {resp3::type::number, 1u, 1u, "1" }, + }; + + BOOST_TEST_ALL_EQ(resp.begin(), resp.end(), std::begin(expected), std::end(expected)); +} + +// If connectivity to the Redis master fails, we can reconnect +void test_reconnect() +{ + // Setup + net::io_context ioc; + connection conn{ioc}; + + config cfg; + cfg.sentinel.addresses = { + {"localhost", "26379"}, + {"localhost", "26380"}, + {"localhost", "26381"}, + }; + cfg.sentinel.master_name = "mymaster"; + + // Will cause the connection to fail + request req_quit; + req_quit.push("QUIT"); + + // Will succeed if the reconnection succeeds + request req_ping; + req_ping.push("PING", "sentinel_reconnect"); + req_ping.get_config().cancel_if_unresponded = false; + + bool quit_finished = false, ping_finished = false, run_finished = false; + + conn.async_exec(req_quit, ignore, [&](error_code ec1, std::size_t) { + quit_finished = true; + BOOST_TEST_EQ(ec1, error_code()); + conn.async_exec(req_ping, ignore, [&](error_code ec2, std::size_t) { + ping_finished = true; + BOOST_TEST_EQ(ec2, error_code()); + conn.cancel(); + }); + }); + + conn.async_run(cfg, [&](error_code ec) { + run_finished = true; + BOOST_TEST_EQ(ec, net::error::operation_aborted); + }); + + ioc.run_for(test_timeout); + + BOOST_TEST(quit_finished); + BOOST_TEST(ping_finished); + BOOST_TEST(run_finished); +} + +// If a Sentinel is not reachable, we try the next one +void test_sentinel_not_reachable() +{ + // Setup + net::io_context ioc; + connection conn{ioc}; + + config cfg; + cfg.sentinel.addresses = { + {"localhost", "45678"}, // invalid + {"localhost", "26381"}, + }; + cfg.sentinel.master_name = "mymaster"; + + // Verify that we're connected to the master, listening at port 6380 + request req; + req.push("PING", "test_sentinel_not_reachable"); + + bool exec_finished = false, run_finished = false; + + conn.async_exec(req, ignore, [&](error_code ec, std::size_t) { + exec_finished = true; + BOOST_TEST_EQ(ec, error_code()); + conn.cancel(); + }); + + conn.async_run(cfg, [&](error_code ec) { + run_finished = true; + BOOST_TEST_EQ(ec, net::error::operation_aborted); + }); + + ioc.run_for(test_timeout); + + BOOST_TEST(exec_finished); + BOOST_TEST(run_finished); +} + +// Both Sentinels and masters may be protected with authorization +void test_auth() +{ + // Setup + net::io_context ioc; + connection conn{ioc}; + + config cfg; + cfg.sentinel.addresses = { + {"localhost", "26379"}, + }; + cfg.sentinel.master_name = "mymaster"; + cfg.sentinel.setup.push("HELLO", 3, "AUTH", "sentinel_user", "sentinel_pass"); + + cfg.use_setup = true; + cfg.setup.clear(); + cfg.setup.push("HELLO", 3, "AUTH", "redis_user", "redis_pass"); + + // Verify that we're authenticated correctly + request req; + req.push("ACL", "WHOAMI"); + + response resp; + + bool exec_finished = false, run_finished = false; + + conn.async_exec(req, resp, [&](error_code ec, std::size_t) { + exec_finished = true; + BOOST_TEST_EQ(ec, error_code()); + BOOST_TEST(std::get<0>(resp).has_value()); + BOOST_TEST_EQ(std::get<0>(resp).value(), "redis_user"); + conn.cancel(); + }); + + conn.async_run(cfg, [&](error_code ec) { + run_finished = true; + BOOST_TEST_EQ(ec, net::error::operation_aborted); + }); + + ioc.run_for(test_timeout); + + BOOST_TEST(exec_finished); + BOOST_TEST(run_finished); +} + +// TLS might be used with Sentinels. In our setup, nodes don't use TLS, +// but this setting is independent from Sentinel. +void test_tls() +{ + // Setup + net::io_context ioc; + net::ssl::context ssl_ctx{net::ssl::context::tlsv13_client}; + + // The custom server uses a certificate signed by a CA + // that is not trusted by default - skip verification. + ssl_ctx.set_verify_mode(net::ssl::verify_none); + + connection conn{ioc, std::move(ssl_ctx)}; + + config cfg; + cfg.sentinel.addresses = { + {"localhost", "36379"}, + {"localhost", "36380"}, + {"localhost", "36381"}, + }; + cfg.sentinel.master_name = "mymaster"; + cfg.sentinel.use_ssl = true; + + request req; + req.push("PING", "test_sentinel_tls"); + + bool exec_finished = false, run_finished = false; + + conn.async_exec(req, ignore, [&](error_code ec, std::size_t) { + exec_finished = true; + BOOST_TEST(ec == error_code()); + conn.cancel(); + }); + + conn.async_run(cfg, {}, [&](error_code ec) { + run_finished = true; + BOOST_TEST(ec == net::error::operation_aborted); + }); + + ioc.run_for(test_timeout); + + BOOST_TEST(exec_finished); + BOOST_TEST(run_finished); +} + +// We can also connect to replicas +void test_replica() +{ + // Setup + net::io_context ioc; + connection conn{ioc}; + + config cfg; + cfg.sentinel.addresses = { + {"localhost", "26379"}, + {"localhost", "26380"}, + {"localhost", "26381"}, + }; + cfg.sentinel.master_name = "mymaster"; + cfg.sentinel.server_role = role::replica; + + // Verify that we're connected to a replica + request req; + req.push("ROLE"); + + generic_response resp; + + bool exec_finished = false, run_finished = false; + + conn.async_exec(req, resp, [&](error_code ec, std::size_t) { + exec_finished = true; + BOOST_TEST_EQ(ec, error_code()); + + // ROLE outputs an array, 1st element should be 'slave' + BOOST_TEST(resp.has_value()); + BOOST_TEST_GE(resp.value().size(), 2u); + BOOST_TEST_EQ(resp.value().at(1u).value, "slave"); + + conn.cancel(); + }); + + conn.async_run(cfg, [&](error_code ec) { + run_finished = true; + BOOST_TEST_EQ(ec, net::error::operation_aborted); + }); + + ioc.run_for(test_timeout); + + BOOST_TEST(exec_finished); + BOOST_TEST(run_finished); +} + +// If no Sentinel is reachable, an error is issued. +// This tests disabling reconnection with Sentinel, too. +void test_error_no_sentinel_reachable() +{ + // Setup + std::string logs; + net::io_context ioc; + connection conn{ioc, make_string_logger(logs)}; + + config cfg; + cfg.sentinel.addresses = { + {"localhost", "43210"}, + {"localhost", "43211"}, + }; + cfg.sentinel.master_name = "mymaster"; + cfg.reconnect_wait_interval = 0s; // disable reconnection so we can verify the error + + bool run_finished = false; + + conn.async_run(cfg, [&](error_code ec) { + run_finished = true; + BOOST_TEST_EQ(ec, error::sentinel_resolve_failed); + }); + + ioc.run_for(test_timeout); + + BOOST_TEST(run_finished); + + if ( + !BOOST_TEST_NE( + logs.find("Sentinel at localhost:43210: connection establishment error"), + std::string::npos) || + !BOOST_TEST_NE( + logs.find("Sentinel at localhost:43211: connection establishment error"), + std::string::npos)) { + std::cerr << "Log was:\n" << logs << std::endl; + } +} + +// If Sentinel doesn't know about the configured master, +// the appropriate error is returned +void test_error_unknown_master() +{ + // Setup + std::string logs; + net::io_context ioc; + connection conn{ioc, make_string_logger(logs)}; + + config cfg; + cfg.sentinel.addresses = { + {"localhost", "26380"}, + }; + cfg.sentinel.master_name = "unknown_master"; + cfg.reconnect_wait_interval = 0s; // disable reconnection so we can verify the error + + bool run_finished = false; + + conn.async_run(cfg, [&](error_code ec) { + run_finished = true; + BOOST_TEST_EQ(ec, error::sentinel_resolve_failed); + }); + + ioc.run_for(test_timeout); + + BOOST_TEST(run_finished); + + if (!BOOST_TEST_NE( + logs.find("Sentinel at localhost:26380: doesn't know about the configured master"), + std::string::npos)) { + std::cerr << "Log was:\n" << logs << std::endl; + } +} + +// The same applies when connecting to replicas, too +void test_error_unknown_master_replica() +{ + // Setup + std::string logs; + net::io_context ioc; + connection conn{ioc, make_string_logger(logs)}; + + config cfg; + cfg.sentinel.addresses = { + {"localhost", "26380"}, + }; + cfg.sentinel.master_name = "unknown_master"; + cfg.reconnect_wait_interval = 0s; // disable reconnection so we can verify the error + cfg.sentinel.server_role = role::replica; + + bool run_finished = false; + + conn.async_run(cfg, [&](error_code ec) { + run_finished = true; + BOOST_TEST_EQ(ec, error::sentinel_resolve_failed); + }); + + ioc.run_for(test_timeout); + + BOOST_TEST(run_finished); + + if (!BOOST_TEST_NE( + logs.find("Sentinel at localhost:26380: doesn't know about the configured master"), + std::string::npos)) { + std::cerr << "Log was:\n" << logs << std::endl; + } +} + +} // namespace + +int main() +{ + // Create the required users in the master, replicas and sentinels + create_user("6379", "redis_user", "redis_pass"); + create_user("6380", "redis_user", "redis_pass"); + create_user("6381", "redis_user", "redis_pass"); + create_user("26379", "sentinel_user", "sentinel_pass"); + create_user("26380", "sentinel_user", "sentinel_pass"); + create_user("26381", "sentinel_user", "sentinel_pass"); + + // Actual tests + test_exec(); + test_receive(); + test_reconnect(); + test_sentinel_not_reachable(); + test_auth(); + test_tls(); + test_replica(); + + test_error_no_sentinel_reachable(); + test_error_unknown_master(); + test_error_unknown_master_replica(); + + return boost::report_errors(); +} diff --git a/test/test_conn_setup.cpp b/test/test_conn_setup.cpp index 52d7b017f..9997cb6b9 100644 --- a/test/test_conn_setup.cpp +++ b/test/test_conn_setup.cpp @@ -18,7 +18,6 @@ #include "common.hpp" #include -#include #include #include @@ -29,37 +28,6 @@ using boost::system::error_code; namespace { -// Creates a user with a known password. Harmless if the user already exists -void setup_password() -{ - // Setup - asio::io_context ioc; - redis::connection conn{ioc}; - - // Enable the user and grant them permissions on everything - redis::request req; - req.push("ACL", "SETUSER", "myuser", "on", ">mypass", "~*", "&*", "+@all"); - redis::generic_response resp; - - bool run_finished = false, exec_finished = false; - conn.async_run(make_test_config(), [&](error_code ec) { - run_finished = true; - BOOST_TEST_EQ(ec, asio::error::operation_aborted); - }); - - conn.async_exec(req, resp, [&](error_code ec, std::size_t) { - exec_finished = true; - BOOST_TEST_EQ(ec, error_code()); - conn.cancel(); - }); - - ioc.run_for(test_timeout); - - BOOST_TEST(run_finished); - BOOST_TEST(exec_finished); - BOOST_TEST(resp.has_value()); -} - void test_auth_success() { // Setup @@ -96,17 +64,13 @@ void test_auth_success() BOOST_TEST_EQ(std::get<0>(resp).value(), "myuser"); } +// Verify that we log appropriately (see https://github.com/boostorg/redis/issues/297) void test_auth_failure() { - // Verify that we log appropriately (see https://github.com/boostorg/redis/issues/297) - std::ostringstream oss; - redis::logger lgr(redis::logger::level::info, [&](redis::logger::level, std::string_view msg) { - oss << msg << '\n'; - }); - // Setup + std::string logs; asio::io_context ioc; - redis::connection conn{ioc, std::move(lgr)}; + redis::connection conn{ioc, make_string_logger(logs)}; // Disable reconnection so the hello error causes the connection to exit auto cfg = make_test_config(); @@ -126,9 +90,8 @@ void test_auth_failure() BOOST_TEST(run_finished); // Check the log - auto log = oss.str(); - if (!BOOST_TEST_NE(log.find("WRONGPASS"), std::string::npos)) { - std::cerr << "Log was: " << log << std::endl; + if (!BOOST_TEST_NE(logs.find("WRONGPASS"), std::string::npos)) { + std::cerr << "Log was: \n" << logs << std::endl; } } @@ -275,17 +238,13 @@ void test_setup_no_hello() BOOST_TEST_EQ(find_client_info(std::get<0>(resp).value(), "db"), "8"); } +// Verify that we log appropriately (see https://github.com/boostorg/redis/issues/297) void test_setup_failure() { - // Verify that we log appropriately (see https://github.com/boostorg/redis/issues/297) - std::ostringstream oss; - redis::logger lgr(redis::logger::level::info, [&](redis::logger::level, std::string_view msg) { - oss << msg << '\n'; - }); - // Setup + std::string logs; asio::io_context ioc; - redis::connection conn{ioc, std::move(lgr)}; + redis::connection conn{ioc, make_string_logger(logs)}; // Disable reconnection so the hello error causes the connection to exit auto cfg = make_test_config(); @@ -306,9 +265,8 @@ void test_setup_failure() BOOST_TEST(run_finished); // Check the log - auto log = oss.str(); - if (!BOOST_TEST_NE(log.find("wrong number of arguments"), std::string::npos)) { - std::cerr << "Log was: " << log << std::endl; + if (!BOOST_TEST_NE(logs.find("wrong number of arguments"), std::string::npos)) { + std::cerr << "Log was:\n" << logs << std::endl; } } @@ -316,7 +274,8 @@ void test_setup_failure() int main() { - setup_password(); + create_user("6379", "myuser", "mypass"); + test_auth_success(); test_auth_failure(); test_database_index(); diff --git a/test/test_conn_tls.cpp b/test/test_conn_tls.cpp index 6832a5e6d..43939b40a 100644 --- a/test/test_conn_tls.cpp +++ b/test/test_conn_tls.cpp @@ -55,7 +55,7 @@ static config make_tls_config() config cfg; cfg.use_ssl = true; cfg.addr.host = get_server_hostname(); - cfg.addr.port = "6380"; + cfg.addr.port = "16380"; return cfg; } diff --git a/test/test_connect_fsm.cpp b/test/test_connect_fsm.cpp index d3fde0913..5c5f9aac5 100644 --- a/test/test_connect_fsm.cpp +++ b/test/test_connect_fsm.cpp @@ -6,7 +6,6 @@ // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // -#include #include #include #include @@ -103,30 +102,15 @@ auto resolver_data = [] { // Reduce duplication struct fixture : detail::log_fixture { - config cfg; buffered_logger lgr{make_logger()}; - connect_fsm fsm{cfg, lgr}; - redis_stream_state st{}; + connect_fsm fsm{lgr}; + redis_stream_state st; - fixture(config&& cfg = {}) - : cfg{std::move(cfg)} + fixture(transport_type type = transport_type::tcp) + : st{type, false} { } }; -config make_ssl_config() -{ - config cfg; - cfg.use_ssl = true; - return cfg; -} - -config make_unix_config() -{ - config cfg; - cfg.unix_socket = "/run/redis.sock"; - return cfg; -} - void test_tcp_success() { // Setup @@ -141,20 +125,21 @@ void test_tcp_success() BOOST_TEST_EQ(act, connect_action_type::done); // The transport type was appropriately set - BOOST_TEST_EQ(fix.st.type, transport_type::tcp); BOOST_TEST_NOT(fix.st.ssl_stream_used); // Check logging fix.check_log({ - {logger::level::info, "Resolve results: 192.168.10.1:1234, 192.168.10.2:1235"}, - {logger::level::info, "Connected to 192.168.10.1:1234" }, + // clang-format off + {logger::level::debug, "Connect: hostname resolution results: 192.168.10.1:1234, 192.168.10.2:1235"}, + {logger::level::debug, "Connect: TCP connect succeeded. Selected endpoint: 192.168.10.1:1234" }, + // clang-format on }); } void test_tcp_tls_success() { // Setup - fixture fix{make_ssl_config()}; + fixture fix{transport_type::tcp_tls}; // Run the algorithm. No SSL stream reset is performed here auto act = fix.fsm.resume(error_code(), fix.st, cancellation_type_t::none); @@ -167,21 +152,22 @@ void test_tcp_tls_success() BOOST_TEST_EQ(act, connect_action_type::done); // The transport type was appropriately set - BOOST_TEST_EQ(fix.st.type, transport_type::tcp_tls); BOOST_TEST(fix.st.ssl_stream_used); // Check logging fix.check_log({ - {logger::level::info, "Resolve results: 192.168.10.1:1234, 192.168.10.2:1235"}, - {logger::level::info, "Connected to 192.168.10.1:1234" }, - {logger::level::info, "Successfully performed SSL handshake" }, + // clang-format off + {logger::level::debug, "Connect: hostname resolution results: 192.168.10.1:1234, 192.168.10.2:1235"}, + {logger::level::debug, "Connect: TCP connect succeeded. Selected endpoint: 192.168.10.1:1234" }, + {logger::level::debug, "Connect: SSL handshake succeeded" }, + // clang-format on }); } void test_tcp_tls_success_reconnect() { // Setup - fixture fix{make_ssl_config()}; + fixture fix{transport_type::tcp_tls}; fix.st.ssl_stream_used = true; // Run the algorithm. The stream is used, so it needs to be reset @@ -197,21 +183,22 @@ void test_tcp_tls_success_reconnect() BOOST_TEST_EQ(act, connect_action_type::done); // The transport type was appropriately set - BOOST_TEST_EQ(fix.st.type, transport_type::tcp_tls); BOOST_TEST(fix.st.ssl_stream_used); // Check logging fix.check_log({ - {logger::level::info, "Resolve results: 192.168.10.1:1234, 192.168.10.2:1235"}, - {logger::level::info, "Connected to 192.168.10.1:1234" }, - {logger::level::info, "Successfully performed SSL handshake" }, + // clang-format off + {logger::level::debug, "Connect: hostname resolution results: 192.168.10.1:1234, 192.168.10.2:1235"}, + {logger::level::debug, "Connect: TCP connect succeeded. Selected endpoint: 192.168.10.1:1234" }, + {logger::level::debug, "Connect: SSL handshake succeeded" }, + // clang-format on }); } void test_unix_success() { // Setup - fixture fix{make_unix_config()}; + fixture fix{transport_type::unix_socket}; // Run the algorithm auto act = fix.fsm.resume(error_code(), fix.st, cancellation_type_t::none); @@ -222,12 +209,11 @@ void test_unix_success() BOOST_TEST_EQ(act, connect_action_type::done); // The transport type was appropriately set - BOOST_TEST_EQ(fix.st.type, transport_type::unix_socket); BOOST_TEST_NOT(fix.st.ssl_stream_used); // Check logging fix.check_log({ - {logger::level::info, "Connected to /run/redis.sock"}, + {logger::level::debug, "Connect: UNIX socket connect succeeded"}, }); } @@ -235,7 +221,7 @@ void test_unix_success() void test_unix_success_close_error() { // Setup - fixture fix{make_unix_config()}; + fixture fix{transport_type::unix_socket}; // Run the algorithm auto act = fix.fsm.resume(error_code(), fix.st, cancellation_type_t::none); @@ -246,12 +232,11 @@ void test_unix_success_close_error() BOOST_TEST_EQ(act, connect_action_type::done); // The transport type was appropriately set - BOOST_TEST_EQ(fix.st.type, transport_type::unix_socket); BOOST_TEST_NOT(fix.st.ssl_stream_used); // Check logging fix.check_log({ - {logger::level::info, "Connected to /run/redis.sock"}, + {logger::level::debug, "Connect: UNIX socket connect succeeded"}, }); } @@ -270,7 +255,7 @@ void test_tcp_resolve_error() // Check logging fix.check_log({ // clang-format off - {logger::level::info, "Error resolving the server hostname: Expected field value is empty. [boost.redis:5]"}, + {logger::level::info, "Connect: hostname resolution failed: Expected field value is empty. [boost.redis:5]"}, // clang-format on }); } @@ -293,7 +278,7 @@ void test_tcp_resolve_timeout() // Check logging fix.check_log({ // clang-format off - {logger::level::info, "Error resolving the server hostname: Resolve timeout. [boost.redis:17]"}, + {logger::level::info, "Connect: hostname resolution failed: Resolve timeout. [boost.redis:17]"}, // clang-format on }); } @@ -349,8 +334,8 @@ void test_tcp_connect_error() // Check logging fix.check_log({ // clang-format off - {logger::level::info, "Resolve results: 192.168.10.1:1234, 192.168.10.2:1235"}, - {logger::level::info, "Failed to connect to the server: Expected field value is empty. [boost.redis:5]"}, + {logger::level::debug, "Connect: hostname resolution results: 192.168.10.1:1234, 192.168.10.2:1235"}, + {logger::level::info, "Connect: TCP connect failed: Expected field value is empty. [boost.redis:5]"}, // clang-format on }); } @@ -375,8 +360,8 @@ void test_tcp_connect_timeout() // Check logging fix.check_log({ // clang-format off - {logger::level::info, "Resolve results: 192.168.10.1:1234, 192.168.10.2:1235"}, - {logger::level::info, "Failed to connect to the server: Connect timeout. [boost.redis:18]"}, + {logger::level::debug, "Connect: hostname resolution results: 192.168.10.1:1234, 192.168.10.2:1235"}, + {logger::level::info, "Connect: TCP connect failed: Connect timeout. [boost.redis:18]"}, // clang-format on }); } @@ -423,7 +408,7 @@ void test_tcp_connect_cancel_edge() void test_ssl_handshake_error() { // Setup - fixture fix{make_ssl_config()}; + fixture fix{transport_type::tcp_tls}; // Run the algorithm. No SSL stream reset is performed here auto act = fix.fsm.resume(error_code(), fix.st, cancellation_type_t::none); @@ -441,9 +426,9 @@ void test_ssl_handshake_error() // Check logging fix.check_log({ // clang-format off - {logger::level::info, "Resolve results: 192.168.10.1:1234, 192.168.10.2:1235"}, - {logger::level::info, "Connected to 192.168.10.1:1234" }, - {logger::level::info, "Failed to perform SSL handshake: Expected field value is empty. [boost.redis:5]"}, + {logger::level::debug, "Connect: hostname resolution results: 192.168.10.1:1234, 192.168.10.2:1235"}, + {logger::level::debug, "Connect: TCP connect succeeded. Selected endpoint: 192.168.10.1:1234"}, + {logger::level::info, "Connect: SSL handshake failed: Expected field value is empty. [boost.redis:5]"}, // clang-format on }); } @@ -451,7 +436,7 @@ void test_ssl_handshake_error() void test_ssl_handshake_timeout() { // Setup - fixture fix{make_ssl_config()}; + fixture fix{transport_type::tcp_tls}; // Run the algorithm. Timeout = operation_aborted without the cancel type set auto act = fix.fsm.resume(error_code(), fix.st, cancellation_type_t::none); @@ -469,9 +454,9 @@ void test_ssl_handshake_timeout() // Check logging fix.check_log({ // clang-format off - {logger::level::info, "Resolve results: 192.168.10.1:1234, 192.168.10.2:1235"}, - {logger::level::info, "Connected to 192.168.10.1:1234" }, - {logger::level::info, "Failed to perform SSL handshake: SSL handshake timeout. [boost.redis:20]"}, + {logger::level::debug, "Connect: hostname resolution results: 192.168.10.1:1234, 192.168.10.2:1235"}, + {logger::level::debug, "Connect: TCP connect succeeded. Selected endpoint: 192.168.10.1:1234"}, + {logger::level::info, "Connect: SSL handshake failed: SSL handshake timeout. [boost.redis:20]"}, // clang-format on }); } @@ -479,7 +464,7 @@ void test_ssl_handshake_timeout() void test_ssl_handshake_cancel() { // Setup - fixture fix{make_ssl_config()}; + fixture fix{transport_type::tcp_tls}; // Run the algorithm. Cancel = operation_aborted with the cancel type set auto act = fix.fsm.resume(error_code(), fix.st, cancellation_type_t::none); @@ -501,7 +486,7 @@ void test_ssl_handshake_cancel() void test_ssl_handshake_cancel_edge() { // Setup - fixture fix{make_ssl_config()}; + fixture fix{transport_type::tcp_tls}; // Run the algorithm. No error, but the cancel state is set auto act = fix.fsm.resume(error_code(), fix.st, cancellation_type_t::none); @@ -524,7 +509,7 @@ void test_ssl_handshake_cancel_edge() void test_unix_connect_error() { // Setup - fixture fix{make_unix_config()}; + fixture fix{transport_type::unix_socket}; // Run the algorithm auto act = fix.fsm.resume(error_code(), fix.st, cancellation_type_t::none); @@ -537,7 +522,7 @@ void test_unix_connect_error() // Check logging fix.check_log({ // clang-format off - {logger::level::info, "Failed to connect to the server: Expected field value is empty. [boost.redis:5]"}, + {logger::level::info, "Connect: UNIX socket connect failed: Expected field value is empty. [boost.redis:5]"}, // clang-format on }); } @@ -545,7 +530,7 @@ void test_unix_connect_error() void test_unix_connect_timeout() { // Setup - fixture fix{make_unix_config()}; + fixture fix{transport_type::unix_socket}; // Run the algorithm. Timeout = operation_aborted without a cancel state auto act = fix.fsm.resume(error_code(), fix.st, cancellation_type_t::none); @@ -558,7 +543,7 @@ void test_unix_connect_timeout() // Check logging fix.check_log({ // clang-format off - {logger::level::info, "Failed to connect to the server: Connect timeout. [boost.redis:18]"}, + {logger::level::info, "Connect: UNIX socket connect failed: Connect timeout. [boost.redis:18]"}, // clang-format on }); } @@ -566,7 +551,7 @@ void test_unix_connect_timeout() void test_unix_connect_cancel() { // Setup - fixture fix{make_unix_config()}; + fixture fix{transport_type::unix_socket}; // Run the algorithm. Cancel = operation_aborted with a cancel state auto act = fix.fsm.resume(error_code(), fix.st, cancellation_type_t::none); @@ -583,7 +568,7 @@ void test_unix_connect_cancel() void test_unix_connect_cancel_edge() { // Setup - fixture fix{make_unix_config()}; + fixture fix{transport_type::unix_socket}; // Run the algorithm. No error, but cancel state is set auto act = fix.fsm.resume(error_code(), fix.st, cancellation_type_t::none); diff --git a/test/test_exec_one_fsm.cpp b/test/test_exec_one_fsm.cpp new file mode 100644 index 000000000..1e5d7c024 --- /dev/null +++ b/test/test_exec_one_fsm.cpp @@ -0,0 +1,365 @@ +// +// Copyright (c) 2025 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "print_node.hpp" + +#include +#include +#include +#include + +using namespace boost::redis; +namespace asio = boost::asio; +using detail::exec_one_fsm; +using detail::exec_one_action; +using detail::exec_one_action_type; +using detail::read_buffer; +using boost::system::error_code; +using boost::asio::cancellation_type_t; +using parse_event = any_adapter::parse_event; +using resp3::type; + +// Operators +static const char* to_string(exec_one_action_type value) +{ + switch (value) { + case exec_one_action_type::done: return "done"; + case exec_one_action_type::write: return "write"; + case exec_one_action_type::read_some: return "read_some"; + default: return ""; + } +} + +namespace boost::redis::detail { + +bool operator==(const exec_one_action& lhs, const exec_one_action& rhs) noexcept +{ + return lhs.type == rhs.type && lhs.ec == rhs.ec; +} + +std::ostream& operator<<(std::ostream& os, const exec_one_action& act) +{ + os << "exec_one_action{ .type=" << to_string(act.type); + if (act.type == exec_one_action_type::done) + os << ", ec=" << act.ec; + return os << " }"; +} + +} // namespace boost::redis::detail + +namespace { + +struct adapter_event { + parse_event type; + resp3::node node{}; + + friend bool operator==(const adapter_event& lhs, const adapter_event& rhs) noexcept + { + return lhs.type == rhs.type && lhs.node == rhs.node; + } + + friend std::ostream& operator<<(std::ostream& os, const adapter_event& value) + { + switch (value.type) { + case parse_event::init: return os << "adapter_event{ .type=init }"; + case parse_event::done: return os << "adapter_event{ .type=done }"; + case parse_event::node: + return os << "adapter_event{ .type=node, .node=" << value.node << " }"; + default: return os << "adapter_event{ .type=unknown }"; + } + } +}; + +any_adapter make_snoop_adapter(std::vector& events) +{ + return any_adapter::impl_t{[&](parse_event ev, resp3::node_view const& nd, error_code&) { + events.push_back({ + ev, + {nd.data_type, nd.aggregate_size, nd.depth, std::string(nd.value)} + }); + }}; +} + +void copy_to(read_buffer& buff, std::string_view data) +{ + auto const buffer = buff.get_prepared(); + BOOST_TEST_GE(buffer.size(), data.size()); + std::copy(data.cbegin(), data.cend(), buffer.begin()); +} + +void test_success() +{ + // Setup + std::vector events; + exec_one_fsm fsm{make_snoop_adapter(events), 2u}; + read_buffer buff; + + // Write the request + auto act = fsm.resume(buff, error_code(), 0u, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_one_action_type::write); + + // FSM should now ask for data + act = fsm.resume(buff, error_code(), 25u, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_one_action_type::read_some); + + // Read the entire response in one go + constexpr std::string_view payload = "$5\r\nhello\r\n*1\r\n+goodbye\r\n"; + copy_to(buff, payload); + act = fsm.resume(buff, error_code(), payload.size(), cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_one_action_type::done); + + // Verify the adapter calls + const adapter_event expected[] = { + {parse_event::init}, + {parse_event::node, {type::blob_string, 1u, 0u, "hello"}}, + {parse_event::done}, + {parse_event::init}, + {parse_event::node, {type::array, 1u, 0u, ""}}, + {parse_event::node, {type::simple_string, 1u, 1u, "goodbye"}}, + {parse_event::done}, + }; + BOOST_TEST_ALL_EQ(events.begin(), events.end(), std::begin(expected), std::end(expected)); +} + +// The request didn't have any expected response (e.g. SUBSCRIBE) +void test_no_expected_response() +{ + // Setup + std::vector events; + exec_one_fsm fsm{make_snoop_adapter(events), 0u}; + read_buffer buff; + + // Write the request + auto act = fsm.resume(buff, error_code(), 0u, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_one_action_type::write); + + // FSM shouldn't ask for data + act = fsm.resume(buff, error_code(), 25u, cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code()); + + // No adapter calls should be done + BOOST_TEST_EQ(events.size(), 0u); +} + +// The response is scattered in several smaller fragments +void test_short_reads() +{ + // Setup + std::vector events; + exec_one_fsm fsm{make_snoop_adapter(events), 2u}; + read_buffer buff; + + // Write the request + auto act = fsm.resume(buff, error_code(), 0u, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_one_action_type::write); + + // FSM should now ask for data + act = fsm.resume(buff, error_code(), 25u, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_one_action_type::read_some); + + // Read fragments + constexpr std::string_view payload = "$5\r\nhello\r\n*1\r\n+goodbye\r\n"; + copy_to(buff, payload.substr(0, 6u)); + act = fsm.resume(buff, error_code(), 6u, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_one_action_type::read_some); + + copy_to(buff, payload.substr(6, 10u)); + act = fsm.resume(buff, error_code(), 10u, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_one_action_type::read_some); + + copy_to(buff, payload.substr(16)); + act = fsm.resume(buff, error_code(), payload.substr(16).size(), cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_one_action_type::done); + + // Verify the adapter calls + const adapter_event expected[] = { + {parse_event::init}, + {parse_event::node, {type::blob_string, 1u, 0u, "hello"}}, + {parse_event::done}, + {parse_event::init}, + {parse_event::node, {type::array, 1u, 0u, ""}}, + {parse_event::node, {type::simple_string, 1u, 1u, "goodbye"}}, + {parse_event::done}, + }; + BOOST_TEST_ALL_EQ(events.begin(), events.end(), std::begin(expected), std::end(expected)); +} + +// Errors in write +void test_write_error() +{ + // Setup + std::vector events; + exec_one_fsm fsm{make_snoop_adapter(events), 2u}; + read_buffer buff; + + // Write the request + auto act = fsm.resume(buff, error_code(), 0u, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_one_action_type::write); + + // Write error + act = fsm.resume(buff, asio::error::connection_reset, 10u, cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code(asio::error::connection_reset)); +} + +void test_write_cancel() +{ + // Setup + std::vector events; + exec_one_fsm fsm{make_snoop_adapter(events), 2u}; + read_buffer buff; + + // Write the request + auto act = fsm.resume(buff, error_code(), 0u, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_one_action_type::write); + + // Edge case where the operation finished successfully but with the cancellation state set + act = fsm.resume(buff, error_code(), 10u, cancellation_type_t::terminal); + BOOST_TEST_EQ(act, error_code(asio::error::operation_aborted)); +} + +// Errors in read +void test_read_error() +{ + // Setup + std::vector events; + exec_one_fsm fsm{make_snoop_adapter(events), 2u}; + read_buffer buff; + + // Write the request + auto act = fsm.resume(buff, error_code(), 0u, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_one_action_type::write); + + // FSM should now ask for data + act = fsm.resume(buff, error_code(), 25u, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_one_action_type::read_some); + + // Read error + act = fsm.resume(buff, asio::error::network_reset, 0u, cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code(asio::error::network_reset)); +} + +void test_read_cancelled() +{ + // Setup + std::vector events; + exec_one_fsm fsm{make_snoop_adapter(events), 2u}; + read_buffer buff; + + // Write the request + auto act = fsm.resume(buff, error_code(), 0u, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_one_action_type::write); + + // FSM should now ask for data + act = fsm.resume(buff, error_code(), 25u, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_one_action_type::read_some); + + // Edge case where the operation finished successfully but with the cancellation state set + copy_to(buff, "$5\r\n"); + act = fsm.resume(buff, error_code(), 4u, cancellation_type_t::terminal); + BOOST_TEST_EQ(act, error_code(asio::error::operation_aborted)); +} + +// Buffer too small +void test_buffer_prepare_error() +{ + // Setup + std::vector events; + exec_one_fsm fsm{make_snoop_adapter(events), 2u}; + read_buffer buff; + buff.set_config({4096u, 8u}); // max size is 8 bytes + + // Write the request + auto act = fsm.resume(buff, error_code(), 0u, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_one_action_type::write); + + // When preparing the buffer, we encounter an error + act = fsm.resume(buff, error_code(), 25u, cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code(error::exceeds_maximum_read_buffer_size)); +} + +// An invalid RESP3 message +void test_parse_error() +{ + // Setup + std::vector events; + exec_one_fsm fsm{make_snoop_adapter(events), 2u}; + read_buffer buff; + + // Write the request + auto act = fsm.resume(buff, error_code(), 0u, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_one_action_type::write); + + // FSM should now ask for data + act = fsm.resume(buff, error_code(), 25u, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_one_action_type::read_some); + + // The response contains an invalid message + constexpr std::string_view payload = "$bad\r\n"; + copy_to(buff, payload); + act = fsm.resume(buff, error_code(), payload.size(), cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code(error::not_a_number)); +} + +// Adapter signals an error +void test_adapter_error() +{ + // Setup. The adapter will fail in the 2nd node + any_adapter adapter{[](parse_event ev, resp3::node_view const&, error_code& ec) { + if (ev == parse_event::node) + ec = error::empty_field; + }}; + exec_one_fsm fsm{std::move(adapter), 2u}; + read_buffer buff; + + // Write the request + auto act = fsm.resume(buff, error_code(), 0u, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_one_action_type::write); + + // FSM should now ask for data + act = fsm.resume(buff, error_code(), 25u, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_one_action_type::read_some); + + // Read the entire response in one go + constexpr std::string_view payload = "$5\r\nhello\r\n*1\r\n+goodbye\r\n"; + copy_to(buff, payload); + act = fsm.resume(buff, error_code(), payload.size(), cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code(error::empty_field)); +} + +} // namespace + +int main() +{ + test_success(); + test_no_expected_response(); + test_short_reads(); + + test_write_error(); + test_write_cancel(); + + test_read_error(); + test_read_cancelled(); + + test_buffer_prepare_error(); + test_parse_error(); + test_adapter_error(); + + return boost::report_errors(); +} diff --git a/test/test_low_level.cpp b/test/test_low_level.cpp index dc68e0ec9..0b208d9d5 100644 --- a/test/test_low_level.cpp +++ b/test/test_low_level.cpp @@ -530,6 +530,11 @@ BOOST_AUTO_TEST_CASE(cover_error) check_error("boost.redis", boost::redis::error::resp3_hello); check_error("boost.redis", boost::redis::error::exceeds_maximum_read_buffer_size); check_error("boost.redis", boost::redis::error::write_timeout); + check_error("boost.redis", boost::redis::error::sentinel_unix_sockets_unsupported); + check_error("boost.redis", boost::redis::error::sentinel_resolve_failed); + check_error("boost.redis", boost::redis::error::role_check_failed); + check_error("boost.redis", boost::redis::error::expects_resp3_string); + check_error("boost.redis", boost::redis::error::expects_resp3_array); } std::string get_type_as_str(boost::redis::resp3::type t) diff --git a/test/test_low_level_sync_sans_io.cpp b/test/test_low_level_sync_sans_io.cpp index 246cc0f30..f60b17113 100644 --- a/test/test_low_level_sync_sans_io.cpp +++ b/test/test_low_level_sync_sans_io.cpp @@ -13,6 +13,8 @@ #include #include +#include "print_node.hpp" + #define BOOST_TEST_MODULE low_level_sync_sans_io #include @@ -337,25 +339,15 @@ BOOST_AUTO_TEST_CASE(check_counter_adapter) namespace boost::redis::resp3 { -template -std::ostream& operator<<(std::ostream& os, basic_node const& nd) -{ - os << "type: " << to_string(nd.data_type) << "\n" - << "aggregate_size: " << nd.aggregate_size << "\n" - << "depth: " << nd.depth << "\n" - << "value: " << nd.value << "\n"; - return os; -} - template std::ostream& operator<<(std::ostream& os, basic_tree const& resp) { - for (auto const& e: resp) + for (auto const& e : resp) os << e << ","; return os; } -} +} // namespace boost::redis::resp3 node from_node_view(node_view const& v) { @@ -370,7 +362,7 @@ node from_node_view(node_view const& v) tree from_flat(flat_tree const& resp) { tree ret; - for (auto const& e: resp.get_view()) + for (auto const& e : resp.get_view()) ret.push_back(from_node_view(e)); return ret; @@ -379,13 +371,12 @@ tree from_flat(flat_tree const& resp) tree from_flat(generic_flat_response const& resp) { tree ret; - for (auto const& e: resp.value().get_view()) + for (auto const& e : resp.value().get_view()) ret.push_back(from_node_view(e)); return ret; } - // Parses the same data into a tree and a // flat_tree, they should be equal to each other. BOOST_AUTO_TEST_CASE(flat_tree_views_are_set) diff --git a/test/test_multiplexer.cpp b/test/test_multiplexer.cpp index 52db5e5cd..dbb6c0e09 100644 --- a/test/test_multiplexer.cpp +++ b/test/test_multiplexer.cpp @@ -15,6 +15,7 @@ #include #include +#include "print_node.hpp" #include "sansio_utils.hpp" #include @@ -33,17 +34,6 @@ using boost::redis::response; using boost::redis::any_adapter; using boost::system::error_code; -namespace boost::redis::resp3 { - -std::ostream& operator<<(std::ostream& os, node const& nd) -{ - return os << "node{ .data_type=" << to_string(nd.data_type) - << ", .aggregate_size=" << nd.aggregate_size << ", .depth=" << nd.depth - << ", .value=" << nd.value << "}"; -} - -} // namespace boost::redis::resp3 - namespace boost::redis::detail { std::ostream& operator<<(std::ostream& os, consume_result v) diff --git a/test/test_parse_sentinel_response.cpp b/test/test_parse_sentinel_response.cpp new file mode 100644 index 000000000..00a83d752 --- /dev/null +++ b/test/test_parse_sentinel_response.cpp @@ -0,0 +1,727 @@ +// +// Copyright (c) 2025 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "sansio_utils.hpp" + +#include +#include +#include +#include + +using namespace boost::redis; +using detail::nodes_from_resp3; +using detail::parse_sentinel_response; +using detail::sentinel_response; +using boost::system::error_code; + +// Operators +namespace boost::redis { + +std::ostream& operator<<(std::ostream& os, const address& addr) +{ + return os << "address{ .host=" << addr.host << ", .port=" << addr.port << " }"; +} + +} // namespace boost::redis + +namespace { + +struct fixture { + sentinel_response resp{ + "leftover", + {"leftover_host", "6543"}, + {address()}, + {address()}, + }; + + void check_response( + const address& expected_master_addr, + boost::span expected_replicas, + boost::span expected_sentinels, + boost::source_location loc = BOOST_CURRENT_LOCATION) const + { + if (!BOOST_TEST_EQ(resp.diagnostic, "")) + std::cerr << "Called from " << loc << std::endl; + if (!BOOST_TEST_EQ(resp.master_addr, expected_master_addr)) + std::cerr << "Called from " << loc << std::endl; + if (!BOOST_TEST_ALL_EQ( + resp.replicas.begin(), + resp.replicas.end(), + expected_replicas.begin(), + expected_replicas.end())) + std::cerr << "Called from " << loc << std::endl; + if (!BOOST_TEST_ALL_EQ( + resp.sentinels.begin(), + resp.sentinels.end(), + expected_sentinels.begin(), + expected_sentinels.end())) + std::cerr << "Called from " << loc << std::endl; + } +}; + +// Usual response when asking for a master +void test_master() +{ + // Setup + fixture fix; + auto nodes = nodes_from_resp3({ + // clang-format off + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*2\r\n" + "%14\r\n" + "$4\r\nname\r\n$40\r\nf14ef06a8a478cdd66ded467ec18accd2a24b731\r\n$2\r\nip\r\n$8\r\nhost.one\r\n$4\r\nport\r\n$5\r\n26380\r\n" + "$5\r\nrunid\r\n$40\r\nf14ef06a8a478cdd66ded467ec18accd2a24b731\r\n$5\r\nflags\r\n$8\r\nsentinel\r\n" + "$21\r\nlink-pending-commands\r\n$1\r\n0\r\n$13\r\nlink-refcount\r\n$1\r\n1\r\n$14\r\nlast-ping-sent\r\n$1\r\n0\r\n" + "$18\r\nlast-ok-ping-reply\r\n$3\r\n696\r\n$15\r\nlast-ping-reply\r\n$3\r\n696\r\n$23\r\ndown-after-milliseconds\r\n$5\r\n10000\r\n" + "$18\r\nlast-hello-message\r\n$3\r\n334\r\n$12\r\nvoted-leader\r\n$1\r\n?\r\n$18\r\nvoted-leader-epoch\r\n$1\r\n0\r\n" + "%14\r\n" + "$4\r\nname\r\n$40\r\nf9b54e79e2e7d3f17ad60527504191ec8a861f27\r\n$2\r\nip\r\n$8\r\nhost.two\r\n$4\r\nport\r\n$5\r\n26381\r\n" + "$5\r\nrunid\r\n$40\r\nf9b54e79e2e7d3f17ad60527504191ec8a861f27\r\n$5\r\nflags\r\n$8\r\nsentinel\r\n" + "$21\r\nlink-pending-commands\r\n$1\r\n0\r\n$13\r\nlink-refcount\r\n$1\r\n1\r\n$14\r\nlast-ping-sent\r\n$1\r\n0\r\n" + "$18\r\nlast-ok-ping-reply\r\n$3\r\n696\r\n$15\r\nlast-ping-reply\r\n$3\r\n696\r\n$23\r\ndown-after-milliseconds\r\n$5\r\n10000\r\n" + "$18\r\nlast-hello-message\r\n$3\r\n134\r\n$12\r\nvoted-leader\r\n$1\r\n?\r\n$18\r\nvoted-leader-epoch\r\n$1\r\n0\r\n", + // clang-format on + }); + + // Call the function + auto ec = parse_sentinel_response(nodes, role::master, fix.resp); + BOOST_TEST_EQ(ec, error_code()); + + // Check + const address expected_sentinels[] = { + {"host.one", "26380"}, + {"host.two", "26381"}, + }; + fix.check_response({"localhost", "6380"}, {}, expected_sentinels); +} + +// Works correctly even if no Sentinels are present +void test_master_no_sentinels() +{ + // Setup + fixture fix; + auto nodes = nodes_from_resp3({ + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*0\r\n", + }); + + // Call the function + auto ec = parse_sentinel_response(nodes, role::master, fix.resp); + BOOST_TEST_EQ(ec, error_code()); + fix.check_response({"localhost", "6380"}, {}, {}); +} + +// The responses corresponding to the user-defined setup request are ignored +void test_master_setup_request() +{ + // Setup + fixture fix; + auto nodes = nodes_from_resp3({ + // clang-format off + "+OK\r\n", + "%6\r\n$6\r\nserver\r\n$5\r\nredis\r\n$7\r\nversion\r\n$5\r\n7.4.2\r\n$5\r\nproto\r\n:3\r\n$2\r\nid\r\n:3\r\n$4\r\nmode\r\n$8\r\nsentinel\r\n$7\r\nmodules\r\n*0\r\n", + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*2\r\n" + "%14\r\n" + "$4\r\nname\r\n$40\r\nf14ef06a8a478cdd66ded467ec18accd2a24b731\r\n$2\r\nip\r\n$8\r\nhost.one\r\n$4\r\nport\r\n$5\r\n26380\r\n" + "$5\r\nrunid\r\n$40\r\nf14ef06a8a478cdd66ded467ec18accd2a24b731\r\n$5\r\nflags\r\n$8\r\nsentinel\r\n" + "$21\r\nlink-pending-commands\r\n$1\r\n0\r\n$13\r\nlink-refcount\r\n$1\r\n1\r\n$14\r\nlast-ping-sent\r\n$1\r\n0\r\n" + "$18\r\nlast-ok-ping-reply\r\n$3\r\n696\r\n$15\r\nlast-ping-reply\r\n$3\r\n696\r\n$23\r\ndown-after-milliseconds\r\n$5\r\n10000\r\n" + "$18\r\nlast-hello-message\r\n$3\r\n334\r\n$12\r\nvoted-leader\r\n$1\r\n?\r\n$18\r\nvoted-leader-epoch\r\n$1\r\n0\r\n" + "%14\r\n" + "$4\r\nname\r\n$40\r\nf9b54e79e2e7d3f17ad60527504191ec8a861f27\r\n$2\r\nip\r\n$8\r\nhost.two\r\n$4\r\nport\r\n$5\r\n26381\r\n" + "$5\r\nrunid\r\n$40\r\nf9b54e79e2e7d3f17ad60527504191ec8a861f27\r\n$5\r\nflags\r\n$8\r\nsentinel\r\n" + "$21\r\nlink-pending-commands\r\n$1\r\n0\r\n$13\r\nlink-refcount\r\n$1\r\n1\r\n$14\r\nlast-ping-sent\r\n$1\r\n0\r\n" + "$18\r\nlast-ok-ping-reply\r\n$3\r\n696\r\n$15\r\nlast-ping-reply\r\n$3\r\n696\r\n$23\r\ndown-after-milliseconds\r\n$5\r\n10000\r\n" + "$18\r\nlast-hello-message\r\n$3\r\n134\r\n$12\r\nvoted-leader\r\n$1\r\n?\r\n$18\r\nvoted-leader-epoch\r\n$1\r\n0\r\n", + // clang-format on + }); + + // Call the function + auto ec = parse_sentinel_response(nodes, role::master, fix.resp); + BOOST_TEST_EQ(ec, error_code()); + + // Check + const address expected_sentinels[] = { + {"host.one", "26380"}, + {"host.two", "26381"}, + }; + fix.check_response({"localhost", "6380"}, {}, expected_sentinels); +} + +// IP and port can be out of order +void test_master_ip_port_out_of_order() +{ + // Setup + fixture fix; + auto nodes = nodes_from_resp3({ + // clang-format off + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*1\r\n" + "%2\r\n" + "$4\r\nport\r\n$5\r\n26380\r\n$2\r\nip\r\n$8\r\nhost.one\r\n" + // clang-format on + }); + + // Call the function + auto ec = parse_sentinel_response(nodes, role::master, fix.resp); + BOOST_TEST_EQ(ec, error_code()); + + // Check + const address expected_sentinels[] = { + {"host.one", "26380"}, + }; + fix.check_response({"localhost", "6380"}, {}, expected_sentinels); +} + +// Usual response when asking for a replica +void test_replica() +{ + // Setup + fixture fix; + auto nodes = nodes_from_resp3({ + // clang-format off + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*2\r\n" + "%21\r\n" + "$4\r\nname\r\n$14\r\nlocalhost:6381\r\n$2\r\nip\r\n$9\r\nsome.host\r\n$4\r\nport\r\n$4\r\n6381\r\n" + "$5\r\nrunid\r\n$40\r\ncdfa33e2d39958c0b10c0391c0c3d4ab096edfeb\r\n$5\r\nflags\r\n$5\r\nslave\r\n" + "$21\r\nlink-pending-commands\r\n$1\r\n0\r\n$13\r\nlink-refcount\r\n$1\r\n1\r\n$14\r\nlast-ping-sent\r\n$1\r\n0\r\n" + "$18\r\nlast-ok-ping-reply\r\n$3\r\n134\r\n$15\r\nlast-ping-reply\r\n$3\r\n134\r\n$23\r\ndown-after-milliseconds\r\n$5\r\n10000\r\n" + "$12\r\ninfo-refresh\r\n$4\r\n5302\r\n$13\r\nrole-reported\r\n$5\r\nslave\r\n$18\r\nrole-reported-time\r\n$6\r\n442121\r\n" + "$21\r\nmaster-link-down-time\r\n$1\r\n0\r\n$18\r\nmaster-link-status\r\n$2\r\nok\r\n$11\r\nmaster-host\r\n$9\r\nlocalhost\r\n" + "$11\r\nmaster-port\r\n$4\r\n6380\r\n$14\r\nslave-priority\r\n$3\r\n100\r\n$17\r\nslave-repl-offset\r\n$5\r\n29110\r\n" + "$17\r\nreplica-announced\r\n$1\r\n1\r\n" + "%21\r\n" + "$4\r\nname\r\n$14\r\nlocalhost:6382\r\n$2\r\nip\r\n$9\r\ntest.host\r\n$4\r\nport\r\n$4\r\n6382\r\n" + "$5\r\nrunid\r\n$40\r\n11bfea62c25316e211fdf0e1ccd2dbd920e90815\r\n$5\r\nflags\r\n$5\r\nslave\r\n" + "$21\r\nlink-pending-commands\r\n$1\r\n0\r\n$13\r\nlink-refcount\r\n$1\r\n1\r\n$14\r\nlast-ping-sent\r\n$1\r\n0\r\n" + "$18\r\nlast-ok-ping-reply\r\n$3\r\n134\r\n$15\r\nlast-ping-reply\r\n$3\r\n134\r\n$23\r\ndown-after-milliseconds\r\n$5\r\n10000\r\n" + "$12\r\ninfo-refresh\r\n$4\r\n5302\r\n$13\r\nrole-reported\r\n$5\r\nslave\r\n$18\r\nrole-reported-time\r\n$6\r\n442132\r\n" + "$21\r\nmaster-link-down-time\r\n$1\r\n0\r\n$18\r\nmaster-link-status\r\n$2\r\nok\r\n$11\r\nmaster-host\r\n$9\r\nlocalhost\r\n" + "$11\r\nmaster-port\r\n$4\r\n6380\r\n$14\r\nslave-priority\r\n$3\r\n100\r\n$17\r\nslave-repl-offset\r\n$5\r\n29110\r\n" + "$17\r\nreplica-announced\r\n$1\r\n1\r\n", + "*2\r\n" + "%14\r\n" + "$4\r\nname\r\n$40\r\nf14ef06a8a478cdd66ded467ec18accd2a24b731\r\n$2\r\nip\r\n$8\r\nhost.one\r\n$4\r\nport\r\n$5\r\n26380\r\n" + "$5\r\nrunid\r\n$40\r\nf14ef06a8a478cdd66ded467ec18accd2a24b731\r\n$5\r\nflags\r\n$8\r\nsentinel\r\n" + "$21\r\nlink-pending-commands\r\n$1\r\n0\r\n$13\r\nlink-refcount\r\n$1\r\n1\r\n$14\r\nlast-ping-sent\r\n$1\r\n0\r\n" + "$18\r\nlast-ok-ping-reply\r\n$3\r\n696\r\n$15\r\nlast-ping-reply\r\n$3\r\n696\r\n$23\r\ndown-after-milliseconds\r\n$5\r\n10000\r\n" + "$18\r\nlast-hello-message\r\n$3\r\n334\r\n$12\r\nvoted-leader\r\n$1\r\n?\r\n$18\r\nvoted-leader-epoch\r\n$1\r\n0\r\n" + "%14\r\n" + "$4\r\nname\r\n$40\r\nf9b54e79e2e7d3f17ad60527504191ec8a861f27\r\n$2\r\nip\r\n$8\r\nhost.two\r\n$4\r\nport\r\n$5\r\n26381\r\n" + "$5\r\nrunid\r\n$40\r\nf9b54e79e2e7d3f17ad60527504191ec8a861f27\r\n$5\r\nflags\r\n$8\r\nsentinel\r\n" + "$21\r\nlink-pending-commands\r\n$1\r\n0\r\n$13\r\nlink-refcount\r\n$1\r\n1\r\n$14\r\nlast-ping-sent\r\n$1\r\n0\r\n" + "$18\r\nlast-ok-ping-reply\r\n$3\r\n696\r\n$15\r\nlast-ping-reply\r\n$3\r\n696\r\n$23\r\ndown-after-milliseconds\r\n$5\r\n10000\r\n" + "$18\r\nlast-hello-message\r\n$3\r\n134\r\n$12\r\nvoted-leader\r\n$1\r\n?\r\n$18\r\nvoted-leader-epoch\r\n$1\r\n0\r\n", + // clang-format on + }); + + // Call the function + auto ec = parse_sentinel_response(nodes, role::replica, fix.resp); + BOOST_TEST_EQ(ec, error_code()); + + // Check + const address expected_replicas[] = { + {"some.host", "6381"}, + {"test.host", "6382"}, + }; + const address expected_sentinels[] = { + {"host.one", "26380"}, + {"host.two", "26381"}, + }; + fix.check_response({"localhost", "6380"}, expected_replicas, expected_sentinels); +} + +// Like the master case +void test_replica_no_sentinels() +{ + // Setup + fixture fix; + auto nodes = nodes_from_resp3({ + // clang-format off + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*2\r\n" + "%3\r\n" + "$4\r\nname\r\n$14\r\nlocalhost:6381\r\n$2\r\nip\r\n$9\r\nsome.host\r\n$4\r\nport\r\n$4\r\n6381\r\n" + "%3\r\n" + "$4\r\nname\r\n$14\r\nlocalhost:6382\r\n$2\r\nip\r\n$9\r\ntest.host\r\n$4\r\nport\r\n$4\r\n6382\r\n", + "*0\r\n" + // clang-format on + }); + + // Call the function + auto ec = parse_sentinel_response(nodes, role::replica, fix.resp); + BOOST_TEST_EQ(ec, error_code()); + + // Check + const address expected_replicas[] = { + {"some.host", "6381"}, + {"test.host", "6382"}, + }; + fix.check_response({"localhost", "6380"}, expected_replicas, {}); +} + +// Asking for replicas, but there is none +void test_replica_no_replicas() +{ + // Setup + fixture fix; + auto nodes = nodes_from_resp3({ + // clang-format off + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*0\r\n", + "*0\r\n", + // clang-format on + }); + + // Call the function + auto ec = parse_sentinel_response(nodes, role::replica, fix.resp); + BOOST_TEST_EQ(ec, error_code()); + + // Check + fix.check_response({"localhost", "6380"}, {}, {}); +} + +// Setup requests work with replicas, too +void test_replica_setup_request() +{ + // Setup + fixture fix; + auto nodes = nodes_from_resp3({ + // clang-format off + "*2\r\n+OK\r\n+OK\r\n", + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*2\r\n" + "%3\r\n" + "$4\r\nname\r\n$14\r\nlocalhost:6381\r\n$2\r\nip\r\n$9\r\nsome.host\r\n$4\r\nport\r\n$4\r\n6381\r\n" + "%3\r\n" + "$4\r\nname\r\n$14\r\nlocalhost:6382\r\n$2\r\nip\r\n$9\r\ntest.host\r\n$4\r\nport\r\n$4\r\n6382\r\n", + "*2\r\n" + "%3\r\n" + "$4\r\nname\r\n$40\r\nf14ef06a8a478cdd66ded467ec18accd2a24b731\r\n$2\r\nip\r\n$8\r\nhost.one\r\n$4\r\nport\r\n$5\r\n26380\r\n" + "%3\r\n" + "$4\r\nname\r\n$40\r\nf9b54e79e2e7d3f17ad60527504191ec8a861f27\r\n$2\r\nip\r\n$8\r\nhost.two\r\n$4\r\nport\r\n$5\r\n26381\r\n" + // clang-format on + }); + + // Call the function + auto ec = parse_sentinel_response(nodes, role::replica, fix.resp); + BOOST_TEST_EQ(ec, error_code()); + + // Check + const address expected_replicas[] = { + {"some.host", "6381"}, + {"test.host", "6382"}, + }; + const address expected_sentinels[] = { + {"host.one", "26380"}, + {"host.two", "26381"}, + }; + fix.check_response({"localhost", "6380"}, expected_replicas, expected_sentinels); +} + +// IP and port can be out of order +void test_replica_ip_port_out_of_order() +{ + // Setup + fixture fix; + auto nodes = nodes_from_resp3({ + // clang-format off + "*2\r\n$9\r\ntest.host\r\n$4\r\n6389\r\n", + "*1\r\n" + "%2\r\n" + "$4\r\nport\r\n$4\r\n6381\r\n$2\r\nip\r\n$9\r\nsome.host\r\n", + "*0\r\n" + // clang-format on + }); + + // Call the function + auto ec = parse_sentinel_response(nodes, role::replica, fix.resp); + BOOST_TEST_EQ(ec, error_code()); + + // Check + const address expected_replicas[] = { + {"some.host", "6381"}, + }; + fix.check_response({"test.host", "6389"}, expected_replicas, {}); +} + +void test_errors() +{ + const struct { + std::string_view name; + role server_role; + std::vector responses; + std::string_view expected_diagnostic; + error_code expected_ec; + } test_cases[]{ + // clang-format off + { + // A RESP3 simple error + "setup_error_simple", + role::master, + { + "-WRONGPASS invalid username-password pair or user is disabled.\r\n", + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*0\r\n", + }, + "WRONGPASS invalid username-password pair or user is disabled.", + error::resp3_simple_error + }, + { + // A RESP3 blob error + "setup_error_blob", + role::master, + { + "!3\r\nBad\r\n", + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*0\r\n", + }, + "Bad", + error::resp3_blob_error + }, + { + // Errors in intermediate nodes of the user-supplied request + "setup_error_intermediate", + role::master, + { + "+OK\r\n", + "-Something happened!\r\n", + "+OK\r\n", + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*0\r\n", + }, + "Something happened!", + error::resp3_simple_error + }, + { + // Only the first error is processed (e.g. auth failure may cause subsequent cmds to fail) + "setup_error_intermediate", + role::master, + { + "-Something happened!\r\n", + "-Something worse happened!\r\n", + "-Bad\r\n", + "-Worse\r\n", + }, + "Something happened!", + error::resp3_simple_error + }, + { + // This works for replicas, too + "setup_error_replicas", + role::replica, + { + "-Something happened!\r\n", + "-Something worse happened!\r\n", + "-Bad\r\n", + "-Worse\r\n", + }, + "Something happened!", + error::resp3_simple_error + }, + + // SENTINEL GET-MASTER-ADDR-BY-NAME + { + // Unknown master. This returns NULL and causes SENTINEL SENTINELS to fail + "getmasteraddr_unknown_master", + role::master, + { + "_\r\n", + "-ERR Unknown master\r\n", + }, + "", + error::resp3_null + }, + { + // The request errors for any other reason + "getmasteraddr_error", + role::master, + { + "-ERR something happened\r\n", + "*0\r\n", + }, + "ERR something happened", + error::resp3_simple_error + }, + { + // Same, for replicas + "getmasteraddr_unknown_master_replica", + role::replica, + { + "_\r\n", + "-ERR Unknown master\r\n", + "-ERR Unknown master\r\n", + }, + "", + error::resp3_null + }, + { + // Root node should be a list + "getmasteraddr_not_array", + role::master, + { + "+OK\r\n", + "*0\r\n", + }, + "", + error::expects_resp3_array + }, + { + // Root node should have exactly 2 elements + "getmasteraddr_array_size_1", + role::master, + { + "*1\r\n$5\r\nhello\r\n", + "*0\r\n", + }, + "", + error::incompatible_size + }, + { + // Root node should have exactly 2 elements + "getmasteraddr_array_size_3", + role::master, + { + "*3\r\n$5\r\nhello\r\n$3\r\nbye\r\n$3\r\nabc\r\n", + "*0\r\n", + }, + "", + error::incompatible_size + }, + { + // IP should be a string + "getmasteraddr_ip_not_string", + role::master, + { + "*2\r\n+OK\r\n$5\r\nhello\r\n", + "*0\r\n", + }, + "", + error::expects_resp3_string + }, + { + // Port should be a string + "getmasteraddr_port_not_string", + role::master, + { + "*2\r\n$5\r\nhello\r\n+OK\r\n", + "*0\r\n", + }, + "", + error::expects_resp3_string + }, + + // SENTINEL SENTINELS + { + // The request errors + "sentinels_error", + role::master, + { + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "-ERR something went wrong\r\n", + }, + "ERR something went wrong", + error::resp3_simple_error + }, + { + // The root node should be an array + "sentinels_not_array", + role::master, + { + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "+OK\r\n", + }, + "", + error::expects_resp3_array + }, + { + // Each Sentinel object should be a map + "sentinels_subobject_not_map", + role::master, + { + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*1\r\n*1\r\n$9\r\nlocalhost\r\n", + }, + "", + error::expects_resp3_map + }, + { + // Keys in the Sentinel object should be strings + "sentinels_keys_not_strings", + role::master, + { + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*1\r\n%1\r\n*0\r\n$5\r\nhello\r\n", + }, + "", + error::expects_resp3_string + }, + { + // Values in the Sentinel object should be strings + "sentinels_keys_not_strings", + role::master, + { + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*1\r\n%1\r\n$5\r\nhello\r\n*1\r\n+OK\r\n", + }, + "", + error::expects_resp3_string + }, + { + "sentinels_ip_not_found", + role::master, + { + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*1\r\n%1\r\n$4\r\nport\r\n$4\r\n6380\r\n", + }, + "", + error::empty_field + }, + { + "sentinels_port_not_found", + role::master, + { + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*1\r\n%1\r\n$2\r\nip\r\n$9\r\nlocalhost\r\n", + }, + "", + error::empty_field + }, + + // SENTINEL REPLICAS + { + // The request errors + "replicas_error", + role::replica, + { + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "-ERR something went wrong\r\n", + "*0\r\n", + }, + "ERR something went wrong", + error::resp3_simple_error + }, + { + // The root node should be an array + "replicas_not_array", + role::replica, + { + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "+OK\r\n", + "*0\r\n", + }, + "", + error::expects_resp3_array + }, + { + // Each replica object should be a map + "replicas_subobject_not_map", + role::replica, + { + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*1\r\n*1\r\n$9\r\nlocalhost\r\n", + "*0\r\n", + }, + "", + error::expects_resp3_map + }, + { + // Keys in the replica object should be strings + "replicas_keys_not_strings", + role::replica, + { + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*1\r\n%1\r\n*0\r\n$5\r\nhello\r\n", + "*0\r\n", + }, + "", + error::expects_resp3_string + }, + { + // Values in the replica object should be strings + "replicas_keys_not_strings", + role::replica, + { + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*1\r\n%1\r\n$5\r\nhello\r\n*1\r\n+OK\r\n", + "*0\r\n", + }, + "", + error::expects_resp3_string + }, + { + "replicas_ip_not_found", + role::replica, + { + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*1\r\n%1\r\n$4\r\nport\r\n$4\r\n6380\r\n", + "*0\r\n", + }, + "", + error::empty_field + }, + { + "replicas_port_not_found", + role::replica, + { + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*1\r\n%1\r\n$2\r\nip\r\n$9\r\nlocalhost\r\n", + "*0\r\n", + }, + "", + error::empty_field + } + + // clang-format on + }; + + for (const auto& tc : test_cases) { + // Setup + std::cerr << "Running error test case: " << tc.name << std::endl; + fixture fix; + auto nodes = nodes_from_resp3(tc.responses); + + // Call the function + auto ec = parse_sentinel_response(nodes, tc.server_role, fix.resp); + BOOST_TEST_EQ(ec, tc.expected_ec); + BOOST_TEST_EQ(fix.resp.diagnostic, tc.expected_diagnostic); + } +} + +} // namespace + +int main() +{ + test_master(); + test_master_no_sentinels(); + test_master_setup_request(); + test_master_ip_port_out_of_order(); + + test_replica(); + test_replica_no_sentinels(); + test_replica_no_replicas(); + test_replica_setup_request(); + test_replica_ip_port_out_of_order(); + + test_errors(); + + return boost::report_errors(); +} diff --git a/test/test_run_fsm.cpp b/test/test_run_fsm.cpp index b67dc929d..ef5706610 100644 --- a/test/test_run_fsm.cpp +++ b/test/test_run_fsm.cpp @@ -39,6 +39,7 @@ static const char* to_string(run_action_type value) switch (value) { case run_action_type::done: return "run_action_type::done"; case run_action_type::immediate: return "run_action_type::immediate"; + case run_action_type::sentinel_resolve: return "run_action_type::sentinel_resolve"; case run_action_type::connect: return "run_action_type::connect"; case run_action_type::parallel_group: return "run_action_type::parallel_group"; case run_action_type::cancel_receive: return "run_action_type::cancel_receive"; @@ -142,6 +143,30 @@ void test_config_error_unix_ssl() }); } +void test_config_error_unix_sentinel() +{ + // Setup + config cfg; + cfg.sentinel.addresses = { + {"localhost", "26379"} + }; + cfg.unix_socket = "/var/sock"; + fixture fix{std::move(cfg)}; + + // Launching the operation fails immediately + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::immediate); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code(error::sentinel_unix_sockets_unsupported)); + + // Log + fix.check_log({ + {logger::level::err, + "Invalid configuration: The configuration specified UNIX sockets with Sentinel, which is " + "not supported. [boost.redis:28]"}, + }); +} + // An error in connect with reconnection enabled triggers a reconnection void test_connect_error() { @@ -162,10 +187,83 @@ void test_connect_error() act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); BOOST_TEST_EQ(act, run_action_type::parallel_group); - // Run doesn't log, it's the subordinate tasks that do - fix.check_log({}); + // Log + fix.check_log({ + // clang-format off + {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + {logger::level::info, "Failed to connect to Redis server at 127.0.0.1:6379 (TLS disabled): Connect timeout. [boost.redis:18]"}, + {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + {logger::level::info, "Connected to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + // clang-format on + }); +} + +// Check logs for other transport types +void test_connect_error_ssl() +{ + // Setup + fixture fix; + fix.st.cfg.addr = {"my_hostname", "10000"}; + fix.st.cfg.use_ssl = true; + + // Launch the operation + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::connect); + + // Connect errors. We sleep and try to connect again + act = fix.fsm.resume(fix.st, error::connect_timeout, cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::wait_for_reconnection); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::connect); + + // This time we succeed and we launch the parallel group + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::parallel_group); + + // Log + fix.check_log({ + // clang-format off + {logger::level::info, "Trying to connect to Redis server at my_hostname:10000 (TLS enabled)" }, + {logger::level::info, "Failed to connect to Redis server at my_hostname:10000 (TLS enabled): Connect timeout. [boost.redis:18]"}, + {logger::level::info, "Trying to connect to Redis server at my_hostname:10000 (TLS enabled)" }, + {logger::level::info, "Connected to Redis server at my_hostname:10000 (TLS enabled)" }, + // clang-format on + }); } +#ifdef BOOST_ASIO_HAS_LOCAL_SOCKETS +void test_connect_error_unix() +{ + // Setup + fixture fix; + fix.st.cfg.unix_socket = "/tmp/sock"; + + // Launch the operation + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::connect); + + // Connect errors. We sleep and try to connect again + act = fix.fsm.resume(fix.st, error::connect_timeout, cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::wait_for_reconnection); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::connect); + + // This time we succeed and we launch the parallel group + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::parallel_group); + + // Log + fix.check_log({ + // clang-format off + {logger::level::info, "Trying to connect to Redis server at '/tmp/sock'" }, + {logger::level::info, "Failed to connect to Redis server at '/tmp/sock': Connect timeout. [boost.redis:18]"}, + {logger::level::info, "Trying to connect to Redis server at '/tmp/sock'" }, + {logger::level::info, "Connected to Redis server at '/tmp/sock'" }, + // clang-format on + }); +} +#endif + // An error in connect without reconnection enabled makes the operation finish void test_connect_error_no_reconnect() { @@ -180,8 +278,13 @@ void test_connect_error_no_reconnect() act = fix.fsm.resume(fix.st, error::connect_timeout, cancellation_type_t::none); BOOST_TEST_EQ(act, error_code(error::connect_timeout)); - // Run doesn't log, it's the subordinate tasks that do - fix.check_log({}); + // Log + fix.check_log({ + // clang-format off + {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + {logger::level::info, "Failed to connect to Redis server at 127.0.0.1:6379 (TLS disabled): Connect timeout. [boost.redis:18]"}, + // clang-format on + }); } // A cancellation in connect makes the operation finish even with reconnection enabled @@ -198,9 +301,10 @@ void test_connect_cancel() act = fix.fsm.resume(fix.st, asio::error::operation_aborted, cancellation_type_t::terminal); BOOST_TEST_EQ(act, error_code(asio::error::operation_aborted)); - // We log on cancellation only + // Log fix.check_log({ - {logger::level::debug, "Run: cancelled (1)"} + {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)"}, + {logger::level::debug, "Run: cancelled (1)" } }); } @@ -218,9 +322,10 @@ void test_connect_cancel_edge() act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::terminal); BOOST_TEST_EQ(act, error_code(asio::error::operation_aborted)); - // We log on cancellation only + // Log fix.check_log({ - {logger::level::debug, "Run: cancelled (1)"} + {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)"}, + {logger::level::debug, "Run: cancelled (1)" } }); } @@ -247,8 +352,13 @@ void test_parallel_group_error() act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); BOOST_TEST_EQ(act, run_action_type::parallel_group); - // Run doesn't log, it's the subordinate tasks that do - fix.check_log({}); + // Log + fix.check_log({ + {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)"}, + {logger::level::info, "Connected to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)"}, + {logger::level::info, "Connected to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + }); } // An error in the parallel group makes the operation exit if reconnection is disabled @@ -269,8 +379,11 @@ void test_parallel_group_error_no_reconnect() act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); BOOST_TEST_EQ(act, error_code(error::empty_field)); - // Run doesn't log, it's the subordinate tasks that do - fix.check_log({}); + // Log + fix.check_log({ + {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)"}, + {logger::level::info, "Connected to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + }); } // A cancellation in the parallel group makes it exit, even if reconnection is enabled. @@ -292,9 +405,11 @@ void test_parallel_group_cancel() act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::terminal); BOOST_TEST_EQ(act, error_code(asio::error::operation_aborted)); - // We log on cancellation only + // Log fix.check_log({ - {logger::level::debug, "Run: cancelled (2)"} + {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)"}, + {logger::level::info, "Connected to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + {logger::level::debug, "Run: cancelled (2)" } }); } @@ -315,9 +430,11 @@ void test_parallel_group_cancel_no_reconnect() act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::terminal); BOOST_TEST_EQ(act, error_code(asio::error::operation_aborted)); - // We log on cancellation only + // Log fix.check_log({ - {logger::level::debug, "Run: cancelled (2)"} + {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)"}, + {logger::level::info, "Connected to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + {logger::level::debug, "Run: cancelled (2)" } }); } @@ -343,9 +460,11 @@ void test_wait_cancel() act = fix.fsm.resume(fix.st, asio::error::operation_aborted, cancellation_type_t::terminal); BOOST_TEST_EQ(act, error_code(asio::error::operation_aborted)); - // We log on cancellation only + // Log fix.check_log({ - {logger::level::debug, "Run: cancelled (3)"} + {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)"}, + {logger::level::info, "Connected to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + {logger::level::debug, "Run: cancelled (3)" } }); } @@ -370,9 +489,11 @@ void test_wait_cancel_edge() act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::terminal); BOOST_TEST_EQ(act, error_code(asio::error::operation_aborted)); - // We log on cancellation only + // Log fix.check_log({ - {logger::level::debug, "Run: cancelled (3)"} + {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)"}, + {logger::level::info, "Connected to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + {logger::level::debug, "Run: cancelled (3)" } }); } @@ -409,9 +530,16 @@ void test_several_reconnections() act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::terminal); BOOST_TEST_EQ(act, error_code(asio::error::operation_aborted)); - // The cancellation was logged + // Log fix.check_log({ - {logger::level::debug, "Run: cancelled (2)"} + // clang-format off + {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + {logger::level::info, "Failed to connect to Redis server at 127.0.0.1:6379 (TLS disabled): Connect timeout. [boost.redis:18]"}, + {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + {logger::level::info, "Connected to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)"}, + {logger::level::info, "Connected to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + {logger::level::debug, "Run: cancelled (2)" } // clang-format on }); } @@ -481,7 +609,11 @@ void test_setup_request_success() // Check log fix.check_log({ - {logger::level::info, "Setup request execution: success"} + // clang-format off + {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + {logger::level::info, "Connected to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + {logger::level::info, "Setup request execution: success"}, + // clang-format on }); } @@ -501,8 +633,13 @@ void test_setup_request_empty() // Nothing was added to the multiplexer BOOST_TEST_EQ(fix.st.mpx.prepare_write(), 0u); - // Check log - fix.check_log({}); + // Log + fix.check_log({ + // clang-format off + {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + {logger::level::info, "Connected to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + // clang-format on + }); } // A server error would cause the reader to exit @@ -510,7 +647,7 @@ void test_setup_request_server_error() { // Setup fixture fix; - fix.st.setup_diagnostic = "leftover"; // simulate a leftover from previous runs + fix.st.diagnostic = "leftover"; // simulate a leftover from previous runs fix.st.cfg.setup.clear(); fix.st.cfg.setup.push("HELLO", 3); @@ -533,9 +670,147 @@ void test_setup_request_server_error() // Check log fix.check_log({ + {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)"}, + {logger::level::info, "Connected to Redis server at 127.0.0.1:6379 (TLS disabled)" }, {logger::level::info, "Setup request execution: The server response to the setup request sent during connection " - "establishment contains an error. [boost.redis:23] (ERR: wrong command)"} + "establishment contains an error. [boost.redis:23] (ERR: wrong command)" } + }); +} + +// When using Sentinel, reconnection works normally +void test_sentinel_reconnection() +{ + // Setup + fixture fix; + fix.st.cfg.sentinel.addresses = { + {"localhost", "26379"} + }; + + // Resolve succeeds, and a connection is attempted + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::sentinel_resolve); + fix.st.cfg.addr = {"host1", "1000"}; + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::connect); + + // This errors, so we sleep and resolve again + act = fix.fsm.resume(fix.st, error::connect_timeout, cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::wait_for_reconnection); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::sentinel_resolve); + fix.st.cfg.addr = {"host2", "2000"}; + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::connect); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::parallel_group); + + // Sentinel involves always a setup request containing the role check. Run it. + BOOST_TEST_EQ(fix.st.mpx.prepare_write(), 1u); + BOOST_TEST(fix.st.mpx.commit_write(fix.st.mpx.get_write_buffer().size())); + read(fix.st.mpx, "*1\r\n$6\r\nmaster\r\n"); + error_code ec; + auto res = fix.st.mpx.consume(ec); + BOOST_TEST_EQ(ec, error_code()); + BOOST_TEST(res.first == detail::consume_result::got_response); + + // The parallel group errors, so we sleep and resolve again + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::cancel_receive); + act = fix.fsm.resume(fix.st, error::write_timeout, cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::wait_for_reconnection); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::sentinel_resolve); + fix.st.cfg.addr = {"host3", "3000"}; + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::connect); + + // Cancel + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::terminal); + BOOST_TEST_EQ(act, error_code(asio::error::operation_aborted)); + + // Log + fix.check_log({ + // clang-format off + {logger::level::info, "Trying to connect to Redis server at host1:1000 (TLS disabled)"}, + {logger::level::info, "Failed to connect to Redis server at host1:1000 (TLS disabled): Connect timeout. [boost.redis:18]"}, + {logger::level::info, "Trying to connect to Redis server at host2:2000 (TLS disabled)"}, + {logger::level::info, "Connected to Redis server at host2:2000 (TLS disabled)"}, + {logger::level::info, "Setup request execution: success"}, + {logger::level::info, "Trying to connect to Redis server at host3:3000 (TLS disabled)"}, + {logger::level::debug, "Run: cancelled (1)"}, + // clang-format on + }); +} + +// If the Sentinel resolve operation errors, we try again +void test_sentinel_resolve_error() +{ + // Setup + fixture fix; + fix.st.cfg.sentinel.addresses = { + {"localhost", "26379"} + }; + + // Start the Sentinel resolve operation + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::sentinel_resolve); + + // It fails with an error, so we go to sleep + act = fix.fsm.resume(fix.st, error::sentinel_resolve_failed, cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::wait_for_reconnection); + + // Retrying it succeeds + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::sentinel_resolve); + fix.st.cfg.addr = {"myhost", "10000"}; + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::connect); + + // Log + fix.check_log({ + {logger::level::info, "Trying to connect to Redis server at myhost:10000 (TLS disabled)"}, + }); +} + +// The reconnection setting affects Sentinel reconnection, too +void test_sentinel_resolve_error_no_reconnect() +{ + // Setup + fixture fix{config_no_reconnect()}; + fix.st.cfg.sentinel.addresses = { + {"localhost", "26379"} + }; + + // Start the Sentinel resolve operation + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::sentinel_resolve); + + // It fails with an error, so we exit + act = fix.fsm.resume(fix.st, error::sentinel_resolve_failed, cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code(error::sentinel_resolve_failed)); + + // Log + fix.check_log({}); +} + +void test_sentinel_resolve_cancel() +{ + // Setup + fixture fix; + fix.st.cfg.sentinel.addresses = { + {"localhost", "26379"} + }; + + // Start the Sentinel resolve operation + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::sentinel_resolve); + act = fix.fsm.resume(fix.st, asio::error::operation_aborted, cancellation_type_t::terminal); + BOOST_TEST_EQ(act, error_code(asio::error::operation_aborted)); + + // Log + fix.check_log({ + {logger::level::debug, "Run: cancelled (4)"}, }); } @@ -547,8 +822,13 @@ int main() test_config_error_unix(); #endif test_config_error_unix_ssl(); + test_config_error_unix_sentinel(); test_connect_error(); + test_connect_error_ssl(); +#ifdef BOOST_ASIO_HAS_LOCAL_SOCKETS + test_connect_error_unix(); +#endif test_connect_error_no_reconnect(); test_connect_cancel(); test_connect_cancel_edge(); @@ -568,5 +848,10 @@ int main() test_setup_request_empty(); test_setup_request_server_error(); + test_sentinel_reconnection(); + test_sentinel_resolve_error(); + test_sentinel_resolve_error_no_reconnect(); + test_sentinel_resolve_cancel(); + return boost::report_errors(); } diff --git a/test/test_sentinel_resolve_fsm.cpp b/test/test_sentinel_resolve_fsm.cpp new file mode 100644 index 000000000..4eb847d93 --- /dev/null +++ b/test/test_sentinel_resolve_fsm.cpp @@ -0,0 +1,682 @@ +// +// Copyright (c) 2025 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "sansio_utils.hpp" + +#include + +using namespace boost::redis; +namespace asio = boost::asio; +using detail::sentinel_resolve_fsm; +using detail::sentinel_action; +using detail::connection_state; +using detail::nodes_from_resp3; +using boost::system::error_code; +using boost::asio::cancellation_type_t; + +static char const* to_string(sentinel_action::type t) +{ + switch (t) { + case sentinel_action::type::done: return "sentinel_action::type::done"; + case sentinel_action::type::connect: return "sentinel_action::type::connect"; + case sentinel_action::type::request: return "sentinel_action::type::request"; + default: return "sentinel_action::type::"; + } +} + +// Operators +namespace boost::redis::detail { + +std::ostream& operator<<(std::ostream& os, sentinel_action::type type) +{ + os << to_string(type); + return os; +} + +bool operator==(sentinel_action lhs, sentinel_action rhs) noexcept +{ + if (lhs.get_type() != rhs.get_type()) + return false; + else if (lhs.get_type() == sentinel_action::type::done) + return lhs.error() == rhs.error(); + else if (lhs.get_type() == sentinel_action::type::connect) + return lhs.connect_addr() == rhs.connect_addr(); + else + return true; +} + +std::ostream& operator<<(std::ostream& os, sentinel_action act) +{ + os << "exec_action{ .type=" << act.get_type(); + if (act.get_type() == sentinel_action::type::done) + os << ", .error=" << act.error(); + else if (act.get_type() == sentinel_action::type::connect) + os << ", .addr=" << act.connect_addr().host << ":" << act.connect_addr().port; + return os << " }"; +} + +} // namespace boost::redis::detail + +namespace boost::redis { + +std::ostream& operator<<(std::ostream& os, const address& addr) +{ + return os << "address{ .host=" << addr.host << ", .port=" << addr.port << " }"; +} + +} // namespace boost::redis + +namespace { + +struct fixture : detail::log_fixture { + connection_state st{{make_logger()}}; + sentinel_resolve_fsm fsm; + + fixture() + { + st.sentinels = { + {"host1", "1000"}, + {"host2", "2000"}, + {"host3", "3000"}, + }; + st.cfg.sentinel.addresses = { + {"host1", "1000"}, + {"host4", "4000"}, + }; + st.cfg.sentinel.master_name = "mymaster"; + } +}; + +void test_success() +{ + // Setup + fixture fix; + + // Initiate. We should connect to the 1st sentinel + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host1", "1000"})); + + // Now send the request + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, sentinel_action::request()); + fix.st.sentinel_resp_nodes = nodes_from_resp3({ + // clang-format off + "*2\r\n$9\r\ntest.host\r\n$4\r\n6380\r\n", + "*1\r\n" + "%2\r\n" + "$2\r\nip\r\n$8\r\nhost.one\r\n$4\r\nport\r\n$5\r\n26380\r\n", + // clang-format on + }); + + // We received a valid request, so we're done + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code()); + + // The master's address is stored + BOOST_TEST_EQ(fix.st.cfg.addr, (address{"test.host", "6380"})); + + // The Sentinel list is updated + const address expected_sentinels[] = { + {"host1", "1000" }, + {"host.one", "26380"}, + {"host4", "4000" }, + }; + BOOST_TEST_ALL_EQ( + fix.st.sentinels.begin(), + fix.st.sentinels.end(), + std::begin(expected_sentinels), + std::end(expected_sentinels)); + + // Logs + fix.check_log({ + {logger::level::info, "Trying to resolve the address of master 'mymaster' using Sentinel" }, + {logger::level::debug, "Trying to contact Sentinel at host1:1000" }, + {logger::level::debug, "Executing Sentinel request at host1:1000" }, + {logger::level::info, "Sentinel at host1:1000 resolved the server address to test.host:6380"}, + }); +} + +void test_success_replica() +{ + // Setup. Seed the engine so that it returns index 1 + fixture fix; + fix.st.cfg.sentinel.server_role = role::replica; + fix.st.eng.get().seed(static_cast(183984887232u)); + + // Initiate. We should connect to the 1st sentinel + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host1", "1000"})); + + // Now send the request + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, sentinel_action::request()); + fix.st.sentinel_resp_nodes = nodes_from_resp3({ + // clang-format off + "*2\r\n$9\r\ntest.host\r\n$4\r\n6380\r\n", + "*3\r\n" + "%2\r\n" + "$2\r\nip\r\n$11\r\nreplica.one\r\n$4\r\nport\r\n$4\r\n6379\r\n" + "%2\r\n" + "$2\r\nip\r\n$11\r\nreplica.two\r\n$4\r\nport\r\n$4\r\n6379\r\n" + "%2\r\n" + "$2\r\nip\r\n$11\r\nreplica.thr\r\n$4\r\nport\r\n$4\r\n6379\r\n", + "*0\r\n" + // clang-format on + }); + + // We received a valid request, so we're done + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code()); + + // The address of one of the replicas is stored + BOOST_TEST_EQ(fix.st.cfg.addr, (address{"replica.two", "6379"})); + + // Logs + fix.check_log({ + // clang-format off + {logger::level::info, "Trying to resolve the address of a replica of master 'mymaster' using Sentinel" }, + {logger::level::debug, "Trying to contact Sentinel at host1:1000" }, + {logger::level::debug, "Executing Sentinel request at host1:1000" }, + {logger::level::info, "Sentinel at host1:1000 resolved the server address to replica.two:6379" }, + // clang-format on + }); +} + +// The first Sentinel fails connection, but subsequent ones succeed +void test_one_connect_error() +{ + // Setup + fixture fix; + + // Initiate. We should connect to the 1st sentinel + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host1", "1000"})); + + // This errors, so we connect to the 2nd sentinel + act = fix.fsm.resume(fix.st, error::connect_timeout, cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host2", "2000"})); + + // Now send the request + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, sentinel_action::request()); + fix.st.sentinel_resp_nodes = nodes_from_resp3({ + "*2\r\n$9\r\ntest.host\r\n$4\r\n6380\r\n", + "*0\r\n", + }); + + // We received a valid request, so we're done + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code()); + + // The master's address is stored + BOOST_TEST_EQ(fix.st.cfg.addr, (address{"test.host", "6380"})); + + // Logs + fix.check_log({ + // clang-format off + {logger::level::info, "Trying to resolve the address of master 'mymaster' using Sentinel" }, + {logger::level::debug, "Trying to contact Sentinel at host1:1000" }, + {logger::level::info, "Sentinel at host1:1000: connection establishment error: Connect timeout. [boost.redis:18]" }, + {logger::level::debug, "Trying to contact Sentinel at host2:2000" }, + {logger::level::debug, "Executing Sentinel request at host2:2000" }, + {logger::level::info, "Sentinel at host2:2000 resolved the server address to test.host:6380"}, + // clang-format on + }); +} + +// The first Sentinel fails while executing the request, but subsequent ones succeed +void test_one_request_network_error() +{ + // Setup + fixture fix; + + // Initiate, connect to the 1st Sentinel, and send the request + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host1", "1000"})); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, sentinel_action::request()); + + // It fails, so we connect to the 2nd sentinel. This one succeeds + act = fix.fsm.resume(fix.st, error::write_timeout, cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host2", "2000"})); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, sentinel_action::request()); + fix.st.sentinel_resp_nodes = nodes_from_resp3({ + "*2\r\n$9\r\ntest.host\r\n$4\r\n6380\r\n", + "*0\r\n", + }); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code()); + + // The master's address is stored + BOOST_TEST_EQ(fix.st.cfg.addr, (address{"test.host", "6380"})); + + // Logs + fix.check_log({ + // clang-format off + {logger::level::info, "Trying to resolve the address of master 'mymaster' using Sentinel" }, + {logger::level::debug, "Trying to contact Sentinel at host1:1000" }, + {logger::level::debug, "Executing Sentinel request at host1:1000" }, + {logger::level::info, "Sentinel at host1:1000: error while executing request: Timeout while writing data to the server. [boost.redis:27]"}, + {logger::level::debug, "Trying to contact Sentinel at host2:2000" }, + {logger::level::debug, "Executing Sentinel request at host2:2000" }, + {logger::level::info, "Sentinel at host2:2000 resolved the server address to test.host:6380"}, + // clang-format on + }); +} + +// The first Sentinel responds with an invalid message, but subsequent ones succeed +void test_one_request_parse_error() +{ + // Setup + fixture fix; + + // Initiate, connect to the 1st Sentinel, and send the request + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host1", "1000"})); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, sentinel_action::request()); + fix.st.sentinel_resp_nodes = nodes_from_resp3({ + "+OK\r\n", + "+OK\r\n", + }); + + // This fails parsing, so we connect to the 2nd sentinel. This one succeeds + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host2", "2000"})); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, sentinel_action::request()); + fix.st.sentinel_resp_nodes = nodes_from_resp3({ + "*2\r\n$9\r\ntest.host\r\n$4\r\n6380\r\n", + "*0\r\n", + }); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code()); + + // The master's address is stored + BOOST_TEST_EQ(fix.st.cfg.addr, (address{"test.host", "6380"})); + + // Logs + fix.check_log({ + // clang-format off + {logger::level::info, "Trying to resolve the address of master 'mymaster' using Sentinel" }, + {logger::level::debug, "Trying to contact Sentinel at host1:1000" }, + {logger::level::debug, "Executing Sentinel request at host1:1000" }, + {logger::level::info, "Sentinel at host1:1000: error parsing response (maybe forgot to upgrade to RESP3?): " + "Expects a RESP3 array, but got a different data type. [boost.redis:32]"}, + {logger::level::debug, "Trying to contact Sentinel at host2:2000" }, + {logger::level::debug, "Executing Sentinel request at host2:2000" }, + {logger::level::info, "Sentinel at host2:2000 resolved the server address to test.host:6380"}, + // clang-format on + }); +} + +// The first Sentinel responds with an error (e.g. failed auth), but subsequent ones succeed +void test_one_request_error_node() +{ + // Setup + fixture fix; + + // Initiate, connect to the 1st Sentinel, and send the request + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host1", "1000"})); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, sentinel_action::request()); + fix.st.sentinel_resp_nodes = nodes_from_resp3({ + "-ERR needs authentication\r\n", + "-ERR needs authentication\r\n", + }); + + // This fails, so we connect to the 2nd sentinel. This one succeeds + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host2", "2000"})); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, sentinel_action::request()); + fix.st.sentinel_resp_nodes = nodes_from_resp3({ + "*2\r\n$9\r\ntest.host\r\n$4\r\n6380\r\n", + "*0\r\n", + }); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code()); + + // The master's address is stored + BOOST_TEST_EQ(fix.st.cfg.addr, (address{"test.host", "6380"})); + + // Logs + fix.check_log({ + // clang-format off + {logger::level::info, "Trying to resolve the address of master 'mymaster' using Sentinel" }, + {logger::level::debug, "Trying to contact Sentinel at host1:1000" }, + {logger::level::debug, "Executing Sentinel request at host1:1000" }, + {logger::level::info, "Sentinel at host1:1000: responded with an error: ERR needs authentication"}, + {logger::level::debug, "Trying to contact Sentinel at host2:2000" }, + {logger::level::debug, "Executing Sentinel request at host2:2000" }, + {logger::level::info, "Sentinel at host2:2000 resolved the server address to test.host:6380"}, + // clang-format on + }); +} + +// The first Sentinel doesn't know about the master, but others do +void test_one_master_unknown() +{ + // Setup + fixture fix; + + // Initiate, connect to the 1st Sentinel, and send the request + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host1", "1000"})); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, sentinel_action::request()); + fix.st.sentinel_resp_nodes = nodes_from_resp3({ + "_\r\n", + "-ERR unknown master\r\n", + }); + + // It doesn't know about our master, so we connect to the 2nd sentinel. + // This one succeeds + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host2", "2000"})); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, sentinel_action::request()); + fix.st.sentinel_resp_nodes = nodes_from_resp3({ + "*2\r\n$9\r\ntest.host\r\n$4\r\n6380\r\n", + "*0\r\n", + }); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code()); + + // The master's address is stored + BOOST_TEST_EQ(fix.st.cfg.addr, (address{"test.host", "6380"})); + + // Logs + fix.check_log({ + // clang-format off + {logger::level::info, "Trying to resolve the address of master 'mymaster' using Sentinel" }, + {logger::level::debug, "Trying to contact Sentinel at host1:1000" }, + {logger::level::debug, "Executing Sentinel request at host1:1000" }, + {logger::level::info, "Sentinel at host1:1000: doesn't know about the configured master" }, + {logger::level::debug, "Trying to contact Sentinel at host2:2000" }, + {logger::level::debug, "Executing Sentinel request at host2:2000" }, + {logger::level::info, "Sentinel at host2:2000 resolved the server address to test.host:6380"}, + // clang-format on + }); +} + +// The first Sentinel thinks there are no replicas (stale data?), but others do +void test_one_no_replicas() +{ + // Setup + fixture fix; + fix.st.cfg.sentinel.server_role = role::replica; + + // Initiate, connect to the 1st Sentinel, and send the request + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host1", "1000"})); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, sentinel_action::request()); + fix.st.sentinel_resp_nodes = nodes_from_resp3({ + "*2\r\n$9\r\ntest.host\r\n$4\r\n6380\r\n", + "*0\r\n", + "*0\r\n", + }); + + // This errors, so we connect to the 2nd sentinel. This one succeeds + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host2", "2000"})); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, sentinel_action::request()); + fix.st.sentinel_resp_nodes = nodes_from_resp3({ + // clang-format off + "*2\r\n$9\r\ntest.host\r\n$4\r\n6380\r\n", + "*1\r\n" + "%2\r\n" + "$2\r\nip\r\n$11\r\nreplica.one\r\n$4\r\nport\r\n$4\r\n6379\r\n", + "*0\r\n", + // clang-format on + }); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code()); + + // The replica's address is stored + BOOST_TEST_EQ(fix.st.cfg.addr, (address{"replica.one", "6379"})); + + // Logs + fix.check_log({ + // clang-format off + {logger::level::info, "Trying to resolve the address of a replica of master 'mymaster' using Sentinel" }, + {logger::level::debug, "Trying to contact Sentinel at host1:1000" }, + {logger::level::debug, "Executing Sentinel request at host1:1000" }, + {logger::level::info, "Sentinel at host1:1000: the configured master has no replicas" }, + {logger::level::debug, "Trying to contact Sentinel at host2:2000" }, + {logger::level::debug, "Executing Sentinel request at host2:2000" }, + {logger::level::info, "Sentinel at host2:2000 resolved the server address to replica.one:6379"}, + // clang-format on + }); +} + +// If no Sentinel is available, the operation fails. A comprehensive error is logged. +void test_error() +{ + // Setup + fixture fix; + + // 1st Sentinel doesn't know about the master + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host1", "1000"})); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, sentinel_action::request()); + fix.st.sentinel_resp_nodes = nodes_from_resp3({ + "_\r\n", + "-ERR unknown master\r\n", + }); + + // Move to the 2nd Sentinel, which fails to connect + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host2", "2000"})); + + // Move to the 3rd Sentinel, which has authentication misconfigured + act = fix.fsm.resume(fix.st, error::connect_timeout, cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host3", "3000"})); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, sentinel_action::request()); + fix.st.sentinel_resp_nodes = nodes_from_resp3({ + "-ERR unauthorized\r\n", + "-ERR unauthorized\r\n", + }); + + // Sentinel list exhausted + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code(error::sentinel_resolve_failed)); + + // The Sentinel list is not updated + BOOST_TEST_EQ(fix.st.sentinels.size(), 3u); + + // Logs + fix.check_log({ + // clang-format off + {logger::level::info, "Trying to resolve the address of master 'mymaster' using Sentinel" }, + + {logger::level::debug, "Trying to contact Sentinel at host1:1000" }, + {logger::level::debug, "Executing Sentinel request at host1:1000" }, + {logger::level::info, "Sentinel at host1:1000: doesn't know about the configured master" }, + + {logger::level::debug, "Trying to contact Sentinel at host2:2000" }, + {logger::level::info, "Sentinel at host2:2000: connection establishment error: Connect timeout. [boost.redis:18]" }, + + {logger::level::debug, "Trying to contact Sentinel at host3:3000" }, + {logger::level::debug, "Executing Sentinel request at host3:3000" }, + {logger::level::info, "Sentinel at host3:3000: responded with an error: ERR unauthorized"}, + + {logger::level::err, "Failed to resolve the address of master 'mymaster'. Tried the following Sentinels:" + "\n Sentinel at host1:1000: doesn't know about the configured master" + "\n Sentinel at host2:2000: connection establishment error: Connect timeout. [boost.redis:18]" + "\n Sentinel at host3:3000: responded with an error: ERR unauthorized"}, + // clang-format on + }); +} + +// The replica error text is slightly different +void test_error_replica() +{ + // Setup + fixture fix; + fix.st.sentinels = { + {"host1", "1000"} + }; + fix.st.cfg.sentinel.server_role = role::replica; + + // Initiate, connect to the only Sentinel, and send the request + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host1", "1000"})); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, sentinel_action::request()); + fix.st.sentinel_resp_nodes = nodes_from_resp3({ + "*2\r\n$9\r\ntest.host\r\n$4\r\n6380\r\n", + "*0\r\n", + "*0\r\n", + }); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code(error::sentinel_resolve_failed)); + + // Logs + fix.check_log({ + // clang-format off + {logger::level::info, "Trying to resolve the address of a replica of master 'mymaster' using Sentinel" }, + + {logger::level::debug, "Trying to contact Sentinel at host1:1000" }, + {logger::level::debug, "Executing Sentinel request at host1:1000" }, + {logger::level::info, "Sentinel at host1:1000: the configured master has no replicas" }, + + {logger::level::err, "Failed to resolve the address of a replica of master 'mymaster'. Tried the following Sentinels:" + "\n Sentinel at host1:1000: the configured master has no replicas"}, + // clang-format on + }); +} + +// Cancellations +void test_cancel_connect() +{ + // Setup + fixture fix; + + // Initiate. We should connect to the 1st sentinel + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host1", "1000"})); + + // Cancellation + act = fix.fsm.resume(fix.st, asio::error::operation_aborted, cancellation_type_t::terminal); + BOOST_TEST_EQ(act, error_code(asio::error::operation_aborted)); + + // Logs + fix.check_log({ + {logger::level::info, "Trying to resolve the address of master 'mymaster' using Sentinel"}, + {logger::level::debug, "Trying to contact Sentinel at host1:1000" }, + {logger::level::debug, "Sentinel resolve: cancelled (1)" }, + }); +} + +void test_cancel_connect_edge() +{ + // Setup + fixture fix; + + // Initiate. We should connect to the 1st sentinel + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host1", "1000"})); + + // Cancellation (without error code) + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::terminal); + BOOST_TEST_EQ(act, error_code(asio::error::operation_aborted)); + + // Logs + fix.check_log({ + {logger::level::info, "Trying to resolve the address of master 'mymaster' using Sentinel"}, + {logger::level::debug, "Trying to contact Sentinel at host1:1000" }, + {logger::level::debug, "Sentinel resolve: cancelled (1)" }, + }); +} + +void test_cancel_request() +{ + // Setup + fixture fix; + + // Initiate. We should connect to the 1st sentinel + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host1", "1000"})); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, sentinel_action::request()); + act = fix.fsm.resume(fix.st, asio::error::operation_aborted, cancellation_type_t::terminal); + BOOST_TEST_EQ(act, error_code(asio::error::operation_aborted)); + + // Logs + fix.check_log({ + {logger::level::info, "Trying to resolve the address of master 'mymaster' using Sentinel"}, + {logger::level::debug, "Trying to contact Sentinel at host1:1000" }, + {logger::level::debug, "Executing Sentinel request at host1:1000" }, + {logger::level::debug, "Sentinel resolve: cancelled (2)" }, + }); +} + +void test_cancel_request_edge() +{ + // Setup + fixture fix; + + // Initiate. We should connect to the 1st sentinel + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, (address{"host1", "1000"})); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, sentinel_action::request()); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::terminal); + BOOST_TEST_EQ(act, error_code(asio::error::operation_aborted)); + + // Logs + fix.check_log({ + {logger::level::info, "Trying to resolve the address of master 'mymaster' using Sentinel"}, + {logger::level::debug, "Trying to contact Sentinel at host1:1000" }, + {logger::level::debug, "Executing Sentinel request at host1:1000" }, + {logger::level::debug, "Sentinel resolve: cancelled (2)" }, + }); +} + +} // namespace + +int main() +{ + test_success(); + test_success_replica(); + + test_one_connect_error(); + test_one_request_network_error(); + test_one_request_parse_error(); + test_one_request_error_node(); + test_one_master_unknown(); + test_one_no_replicas(); + + test_error(); + test_error_replica(); + + test_cancel_connect(); + test_cancel_connect_edge(); + test_cancel_request(); + test_cancel_request_edge(); + + return boost::report_errors(); +} diff --git a/test/test_setup_adapter.cpp b/test/test_setup_adapter.cpp new file mode 100644 index 000000000..694700eb2 --- /dev/null +++ b/test/test_setup_adapter.cpp @@ -0,0 +1,349 @@ +// +// Copyright (c) 2025 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#include +#include +#include + +#include +#include +#include + +#include + +using namespace boost::redis; +using detail::setup_adapter; +using detail::connection_state; +using detail::compose_setup_request; +using boost::system::error_code; + +namespace { + +void test_success() +{ + // Setup + connection_state st; + st.cfg.use_setup = true; + st.cfg.setup.push("SELECT", 2); + compose_setup_request(st.cfg); + setup_adapter adapter{st}; + + // Response to HELLO + resp3::parser p; + error_code ec; + bool done = resp3::parse(p, "%1\r\n$6\r\nserver\r\n$5\r\nredis\r\n", adapter, ec); + BOOST_TEST(done); + BOOST_TEST_EQ(ec, error_code()); + + // Response to the SELECT command + p.reset(); + done = resp3::parse(p, "+OK\r\n", adapter, ec); + BOOST_TEST(done); + BOOST_TEST_EQ(ec, error_code()); + + // No diagnostic + BOOST_TEST_EQ(st.diagnostic, ""); +} + +void test_simple_error() +{ + // Setup + connection_state st; + st.cfg.use_setup = true; + compose_setup_request(st.cfg); + setup_adapter adapter{st}; + + // Response to HELLO contains an error + resp3::parser p; + error_code ec; + bool done = resp3::parse(p, "-ERR unauthorized\r\n", adapter, ec); + BOOST_TEST(done); + BOOST_TEST_EQ(ec, error::resp3_hello); + BOOST_TEST_EQ(st.diagnostic, "ERR unauthorized"); +} + +void test_blob_error() +{ + // Setup + connection_state st; + st.cfg.use_setup = true; + st.cfg.setup.push("SELECT", 1); + compose_setup_request(st.cfg); + setup_adapter adapter{st}; + + // Response to HELLO + resp3::parser p; + error_code ec; + bool done = resp3::parse(p, "%1\r\n$6\r\nserver\r\n$5\r\nredis\r\n", adapter, ec); + BOOST_TEST(done); + BOOST_TEST_EQ(ec, error_code()); + + // Response to select contains an error + p.reset(); + done = resp3::parse(p, "!3\r\nBad\r\n", adapter, ec); + BOOST_TEST(done); + BOOST_TEST_EQ(ec, error::resp3_hello); + BOOST_TEST_EQ(st.diagnostic, "Bad"); +} + +// A NULL is not an error +void test_null() +{ + // Setup + connection_state st; + st.cfg.use_setup = true; + compose_setup_request(st.cfg); + setup_adapter adapter{st}; + + // Response to HELLO + resp3::parser p; + error_code ec; + bool done = resp3::parse(p, "_\r\n", adapter, ec); + BOOST_TEST(done); + BOOST_TEST_EQ(ec, error_code()); + + // No diagnostic + BOOST_TEST_EQ(st.diagnostic, ""); +} + +// Sentinel adds a ROLE command and checks its output. +// These are real wire values. +constexpr std::string_view role_master_response = + "*3\r\n$6\r\nmaster\r\n:567942\r\n*2\r\n" + "*3\r\n$9\r\nlocalhost\r\n$4\r\n6381\r\n$6\r\n567809\r\n*3\r\n$9\r\nlocalhost\r\n" + "$4\r\n6382\r\n$6\r\n567809\r\n"; +constexpr std::string_view role_replica_response = + "*5\r\n$5\r\nslave\r\n$9\r\nlocalhost\r\n:6380\r\n$9\r\nconnected\r\n:617355\r\n"; + +void test_sentinel_master() +{ + // Setup + connection_state st; + st.cfg.use_setup = true; + st.cfg.setup.push("SELECT", 2); + st.cfg.sentinel.addresses = { + {"localhost", "26379"} + }; + compose_setup_request(st.cfg); + setup_adapter adapter{st}; + + // Response to HELLO + resp3::parser p; + error_code ec; + bool done = resp3::parse(p, "%1\r\n$6\r\nserver\r\n$5\r\nredis\r\n", adapter, ec); + BOOST_TEST(done); + BOOST_TEST_EQ(ec, error_code()); + + // Response to the SELECT command + p.reset(); + done = resp3::parse(p, "+OK\r\n", adapter, ec); + BOOST_TEST(done); + BOOST_TEST_EQ(ec, error_code()); + + // Response to the ROLE command + p.reset(); + done = resp3::parse(p, role_master_response, adapter, ec); + BOOST_TEST(done); + BOOST_TEST_EQ(ec, error_code()); + + // No diagnostic + BOOST_TEST_EQ(st.diagnostic, ""); +} + +void test_sentinel_replica() +{ + // Setup + connection_state st; + st.cfg.use_setup = true; + st.cfg.sentinel.addresses = { + {"localhost", "26379"} + }; + st.cfg.sentinel.server_role = role::replica; + compose_setup_request(st.cfg); + setup_adapter adapter{st}; + + // Response to HELLO + resp3::parser p; + error_code ec; + bool done = resp3::parse(p, "%1\r\n$6\r\nserver\r\n$5\r\nredis\r\n", adapter, ec); + BOOST_TEST(done); + BOOST_TEST_EQ(ec, error_code()); + + // Response to the ROLE command + p.reset(); + done = resp3::parse(p, role_replica_response, adapter, ec); + BOOST_TEST(done); + BOOST_TEST_EQ(ec, error_code()); + + // No diagnostic + BOOST_TEST_EQ(st.diagnostic, ""); +} + +// If the role is not the one expected, a role failed error is issued +void test_sentinel_role_check_failed_master() +{ + // Setup + connection_state st; + st.cfg.use_setup = true; + st.cfg.sentinel.addresses = { + {"localhost", "26379"} + }; + compose_setup_request(st.cfg); + setup_adapter adapter{st}; + + // Response to HELLO + resp3::parser p; + error_code ec; + bool done = resp3::parse(p, "%1\r\n$6\r\nserver\r\n$5\r\nredis\r\n", adapter, ec); + BOOST_TEST(done); + BOOST_TEST_EQ(ec, error_code()); + + // Response to the ROLE command + p.reset(); + done = resp3::parse(p, role_replica_response, adapter, ec); + BOOST_TEST(done); + BOOST_TEST_EQ(ec, error::role_check_failed); + + // No diagnostic + BOOST_TEST_EQ(st.diagnostic, ""); +} + +void test_sentinel_role_check_failed_replica() +{ + // Setup + connection_state st; + st.cfg.use_setup = true; + st.cfg.sentinel.addresses = { + {"localhost", "26379"} + }; + st.cfg.sentinel.server_role = role::replica; + compose_setup_request(st.cfg); + setup_adapter adapter{st}; + + // Response to HELLO + resp3::parser p; + error_code ec; + bool done = resp3::parse(p, "%1\r\n$6\r\nserver\r\n$5\r\nredis\r\n", adapter, ec); + BOOST_TEST(done); + BOOST_TEST_EQ(ec, error_code()); + + // Response to the ROLE command + p.reset(); + done = resp3::parse(p, role_master_response, adapter, ec); + BOOST_TEST(done); + BOOST_TEST_EQ(ec, error::role_check_failed); + + // No diagnostic + BOOST_TEST_EQ(st.diagnostic, ""); +} + +// If the role command errors or has an unexpected format, we fail +void test_sentinel_role_error_node() +{ + // Setup + connection_state st; + st.cfg.use_setup = true; + st.cfg.setup.clear(); + st.cfg.sentinel.addresses = { + {"localhost", "26379"} + }; + compose_setup_request(st.cfg); + setup_adapter adapter{st}; + + // Response to ROLE + resp3::parser p; + error_code ec; + bool done = resp3::parse(p, "-ERR unauthorized\r\n", adapter, ec); + BOOST_TEST(done); + BOOST_TEST_EQ(ec, error::resp3_hello); + BOOST_TEST_EQ(st.diagnostic, "ERR unauthorized"); +} + +void test_sentinel_role_not_array() +{ + // Setup + connection_state st; + st.cfg.use_setup = true; + st.cfg.setup.clear(); + st.cfg.sentinel.addresses = { + {"localhost", "26379"} + }; + compose_setup_request(st.cfg); + setup_adapter adapter{st}; + + // Response to ROLE + resp3::parser p; + error_code ec; + bool done = resp3::parse(p, "+OK\r\n", adapter, ec); + BOOST_TEST(done); + BOOST_TEST_EQ(ec, error::invalid_data_type); + BOOST_TEST_EQ(st.diagnostic, ""); +} + +void test_sentinel_role_empty_array() +{ + // Setup + connection_state st; + st.cfg.use_setup = true; + st.cfg.setup.clear(); + st.cfg.sentinel.addresses = { + {"localhost", "26379"} + }; + compose_setup_request(st.cfg); + setup_adapter adapter{st}; + + // Response to ROLE + resp3::parser p; + error_code ec; + bool done = resp3::parse(p, "*0\r\n", adapter, ec); + BOOST_TEST(done); + BOOST_TEST_EQ(ec, error::incompatible_size); + BOOST_TEST_EQ(st.diagnostic, ""); +} + +void test_sentinel_role_first_element_not_string() +{ + // Setup + connection_state st; + st.cfg.use_setup = true; + st.cfg.setup.clear(); + st.cfg.sentinel.addresses = { + {"localhost", "26379"} + }; + compose_setup_request(st.cfg); + setup_adapter adapter{st}; + + // Response to ROLE + resp3::parser p; + error_code ec; + bool done = resp3::parse(p, "*1\r\n:2000\r\n", adapter, ec); + BOOST_TEST(done); + BOOST_TEST_EQ(ec, error::invalid_data_type); + BOOST_TEST_EQ(st.diagnostic, ""); +} + +} // namespace + +int main() +{ + test_success(); + test_simple_error(); + test_blob_error(); + test_null(); + + test_sentinel_master(); + test_sentinel_replica(); + test_sentinel_role_check_failed_master(); + test_sentinel_role_check_failed_replica(); + test_sentinel_role_error_node(); + test_sentinel_role_not_array(); + test_sentinel_role_empty_array(); + test_sentinel_role_first_element_not_string(); + + return boost::report_errors(); +} \ No newline at end of file diff --git a/test/test_setup_request_utils.cpp b/test/test_setup_request_utils.cpp index c6fad370d..b2bab2ef8 100644 --- a/test/test_setup_request_utils.cpp +++ b/test/test_setup_request_utils.cpp @@ -23,6 +23,8 @@ namespace redis = boost::redis; using redis::detail::compose_setup_request; using boost::system::error_code; +// TODO: rename the file + namespace { void test_compose_setup() @@ -178,6 +180,50 @@ void test_compose_setup_use_setup_flags() BOOST_TEST(cfg.setup.get_config().cancel_on_connection_lost); } +// When using Sentinel, a ROLE command is added. This works +// both with the old HELLO and new setup strategies. +void test_compose_setup_sentinel_auth() +{ + redis::config cfg; + cfg.sentinel.addresses = { + {"localhost", "26379"} + }; + cfg.clientname = ""; + cfg.username = "foo"; + cfg.password = "bar"; + + compose_setup_request(cfg); + + std::string_view const expected = + "*5\r\n$5\r\nHELLO\r\n$1\r\n3\r\n$4\r\nAUTH\r\n$3\r\nfoo\r\n$3\r\nbar\r\n" + "*1\r\n$4\r\nROLE\r\n"; + BOOST_TEST_EQ(cfg.setup.payload(), expected); + BOOST_TEST(cfg.setup.has_hello_priority()); + BOOST_TEST(cfg.setup.get_config().cancel_if_unresponded); + BOOST_TEST(cfg.setup.get_config().cancel_on_connection_lost); +} + +void test_compose_setup_sentinel_use_setup() +{ + redis::config cfg; + cfg.sentinel.addresses = { + {"localhost", "26379"} + }; + cfg.use_setup = true; + cfg.setup.push("SELECT", 42); + + compose_setup_request(cfg); + + std::string_view const expected = + "*2\r\n$5\r\nHELLO\r\n$1\r\n3\r\n" + "*2\r\n$6\r\nSELECT\r\n$2\r\n42\r\n" + "*1\r\n$4\r\nROLE\r\n"; + BOOST_TEST_EQ(cfg.setup.payload(), expected); + BOOST_TEST(cfg.setup.has_hello_priority()); + BOOST_TEST(cfg.setup.get_config().cancel_if_unresponded); + BOOST_TEST(cfg.setup.get_config().cancel_on_connection_lost); +} + } // namespace int main() @@ -191,6 +237,8 @@ int main() test_compose_setup_use_setup(); test_compose_setup_use_setup_no_hello(); test_compose_setup_use_setup_flags(); + test_compose_setup_sentinel_auth(); + test_compose_setup_sentinel_use_setup(); return boost::report_errors(); } \ No newline at end of file diff --git a/test/test_unix_sockets.cpp b/test/test_unix_sockets.cpp index 751981575..be4fa1394 100644 --- a/test/test_unix_sockets.cpp +++ b/test/test_unix_sockets.cpp @@ -131,7 +131,7 @@ void test_switch_between_transports() // Create configurations for TLS and UNIX connections auto tcp_tls_cfg = make_test_config(); tcp_tls_cfg.use_ssl = true; - tcp_tls_cfg.addr.port = "6380"; + tcp_tls_cfg.addr.port = "16380"; auto unix_cfg = make_test_config(); unix_cfg.unix_socket = unix_socket_path; @@ -194,7 +194,7 @@ void test_error_unix_tls() connection conn{ioc}; auto cfg = make_test_config(); cfg.use_ssl = true; - cfg.addr.port = "6380"; + cfg.addr.port = "16380"; cfg.unix_socket = unix_socket_path; bool finished = false; diff --git a/test/test_update_sentinel_list.cpp b/test/test_update_sentinel_list.cpp new file mode 100644 index 000000000..f66de0497 --- /dev/null +++ b/test/test_update_sentinel_list.cpp @@ -0,0 +1,212 @@ +// +// Copyright (c) 2025 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#include +#include + +#include +#include + +#include + +using namespace boost::redis; +using detail::update_sentinel_list; +using boost::system::error_code; + +// Operators +namespace boost::redis { + +std::ostream& operator<<(std::ostream& os, const address& addr) +{ + return os << "address{ .host=" << addr.host << ", .port=" << addr.port << " }"; +} + +} // namespace boost::redis + +namespace { + +// The only Sentinel resolved the address successfully, and there's no newly discovered Sentinels +void test_single_sentinel() +{ + const std::vector
initial_sentinels{ + {"host1", "1000"} + }; + std::vector
sentinels{initial_sentinels}; + + update_sentinel_list(sentinels, 0u, {}, initial_sentinels); + + BOOST_TEST_ALL_EQ( + sentinels.begin(), + sentinels.end(), + initial_sentinels.begin(), + initial_sentinels.end()); +} + +// Some new Sentinels were discovered using SENTINEL SENTINELS +void test_new_sentinels() +{ + const std::vector
initial_sentinels{ + {"host1", "1000"} + }; + std::vector
sentinels{initial_sentinels}; + const address new_sentinels[]{ + {"host2", "2000"}, + {"host3", "3000"}, + }; + + update_sentinel_list(sentinels, 0u, new_sentinels, initial_sentinels); + + const address expected_sentinels[]{ + {"host1", "1000"}, + {"host2", "2000"}, + {"host3", "3000"}, + }; + + BOOST_TEST_ALL_EQ( + sentinels.begin(), + sentinels.end(), + std::begin(expected_sentinels), + std::end(expected_sentinels)); +} + +// Some of the new Sentinels are already in the list +void test_new_sentinels_known() +{ + const std::vector
initial_sentinels{ + {"host1", "1000"}, + {"host2", "2000"}, + }; + std::vector
sentinels{initial_sentinels}; + const address new_sentinels[]{ + {"host2", "2000"}, + {"host3", "3000"}, + }; + + update_sentinel_list(sentinels, 0u, new_sentinels, initial_sentinels); + + const address expected_sentinels[]{ + {"host1", "1000"}, + {"host2", "2000"}, + {"host3", "3000"}, + }; + + BOOST_TEST_ALL_EQ( + sentinels.begin(), + sentinels.end(), + std::begin(expected_sentinels), + std::end(expected_sentinels)); +} + +// The Sentinel that succeeded should be placed first +void test_success_sentinel_not_first() +{ + const std::vector
initial_sentinels{ + {"host1", "1000"}, + {"host2", "2000"}, + {"host3", "3000"}, + }; + std::vector
sentinels{initial_sentinels}; + const address new_sentinels[]{ + {"host1", "1000"}, + {"host2", "2000"}, + }; + + update_sentinel_list(sentinels, 2u, new_sentinels, initial_sentinels); + + const address expected_sentinels[]{ + {"host3", "3000"}, + {"host1", "1000"}, + {"host2", "2000"}, + }; + + BOOST_TEST_ALL_EQ( + sentinels.begin(), + sentinels.end(), + std::begin(expected_sentinels), + std::end(expected_sentinels)); +} + +// If a discovered Sentinel is not returned in subsequent iterations, it's removed from the list +void test_new_sentinel_removed() +{ + const std::vector
initial_sentinels{ + {"host1", "1000"}, + }; + std::vector
sentinels{ + {"host1", "1000"}, + {"host4", "4000"}, + }; + const address new_sentinels[]{ + {"host2", "2000"}, + {"host3", "3000"}, + }; + + update_sentinel_list(sentinels, 0u, new_sentinels, initial_sentinels); + + const address expected_sentinels[]{ + {"host1", "1000"}, + {"host2", "2000"}, + {"host3", "3000"}, + }; + + BOOST_TEST_ALL_EQ( + sentinels.begin(), + sentinels.end(), + std::begin(expected_sentinels), + std::end(expected_sentinels)); +} + +// Bootstrap Sentinels are never removed +void test_bootstrap_sentinel_removed() +{ + const std::vector
initial_sentinels{ + {"host1", "1000"}, + {"host2", "2000"}, + {"host3", "3000"}, + }; + std::vector
sentinels{ + {"host1", "1000"}, + {"host2", "2000"}, + {"host3", "3000"}, + {"host4", "4000"}, + {"host5", "5000"}, + }; + const address new_sentinels[]{ + {"host2", "2000"}, + {"host4", "4000"}, + }; + + update_sentinel_list(sentinels, 0u, new_sentinels, initial_sentinels); + + const address expected_sentinels[]{ + {"host1", "1000"}, + {"host2", "2000"}, + {"host4", "4000"}, + {"host3", "3000"}, // bootstrap Sentinels placed last + }; + + BOOST_TEST_ALL_EQ( + sentinels.begin(), + sentinels.end(), + std::begin(expected_sentinels), + std::end(expected_sentinels)); +} + +} // namespace + +int main() +{ + test_single_sentinel(); + test_new_sentinels(); + test_new_sentinels_known(); + test_success_sentinel_not_first(); + test_new_sentinel_removed(); + test_bootstrap_sentinel_removed(); + + return boost::report_errors(); +} diff --git a/tools/docker-compose.yml b/tools/docker-compose.yml index 8c139362a..c524dcf18 100644 --- a/tools/docker-compose.yml +++ b/tools/docker-compose.yml @@ -1,19 +1,140 @@ services: - redis: + redis-master: + container_name: redis-master image: ${SERVER_IMAGE} - entrypoint: "/docker/entrypoint.sh" + network_mode: host + command: > + sh -c 'chmod 777 /tmp/redis-socks && + redis-server \ + --replica-announce-ip localhost \ + --port 6379 \ + --tls-port 16379 \ + --tls-cert-file /docker/tls/server.crt \ + --tls-key-file /docker/tls/server.key \ + --tls-ca-cert-file /docker/tls/ca.crt \ + --tls-auth-clients no \ + --unixsocket /tmp/redis-socks/redis.sock \ + --unixsocketperm 777' volumes: - ./docker:/docker - /tmp/redis-socks:/tmp/redis-socks - ports: - - 6379:6379 - - 6380:6380 + + redis-replica-1: + container_name: redis-replica-1 + image: ${SERVER_IMAGE} + network_mode: host + command: + [ + "redis-server", + "--replica-announce-ip", "localhost", + "--replicaof", "localhost", "6379", + "--port", "6380", + "--tls-port", "16380", + "--tls-cert-file", "/docker/tls/server.crt", + "--tls-key-file", "/docker/tls/server.key", + "--tls-ca-cert-file", "/docker/tls/ca.crt", + "--tls-auth-clients", "no", + ] + volumes: + - ./docker:/docker + + + redis-replica-2: + container_name: redis-replica-2 + image: ${SERVER_IMAGE} + network_mode: host + command: + [ + "redis-server", + "--replica-announce-ip", "localhost", + "--replicaof", "localhost", "6379", + "--port", "6381", + "--tls-port", "16381", + "--tls-cert-file", "/docker/tls/server.crt", + "--tls-key-file", "/docker/tls/server.key", + "--tls-ca-cert-file", "/docker/tls/ca.crt", + "--tls-auth-clients", "no", + ] + volumes: + - ./docker:/docker + + + sentinel-1: + container_name: sentinel-1 + image: ${SERVER_IMAGE} + network_mode: host + command: > + sh -c 'cat << EOF > /etc/sentinel.conf && redis-sentinel /etc/sentinel.conf + port 26379 + tls-port 36379 + tls-cert-file /docker/tls/server.crt + tls-key-file /docker/tls/server.key + tls-ca-cert-file /docker/tls/ca.crt + tls-auth-clients no + sentinel resolve-hostnames yes + sentinel announce-hostnames yes + sentinel announce-ip localhost + sentinel monitor mymaster localhost 6379 2 + sentinel down-after-milliseconds mymaster 10000 + sentinel failover-timeout mymaster 10000 + sentinel parallel-syncs mymaster 1 + EOF' + volumes: + - ./docker:/docker + + + sentinel-2: + container_name: sentinel-2 + image: ${SERVER_IMAGE} + network_mode: host + command: > + sh -c 'cat << EOF > /etc/sentinel.conf && redis-sentinel /etc/sentinel.conf + port 26380 + tls-port 36380 + tls-cert-file /docker/tls/server.crt + tls-key-file /docker/tls/server.key + tls-ca-cert-file /docker/tls/ca.crt + tls-auth-clients no + sentinel resolve-hostnames yes + sentinel announce-hostnames yes + sentinel announce-ip localhost + sentinel monitor mymaster localhost 6379 2 + sentinel down-after-milliseconds mymaster 10000 + sentinel failover-timeout mymaster 10000 + sentinel parallel-syncs mymaster 1 + EOF' + volumes: + - ./docker:/docker + + sentinel-3: + container_name: sentinel-3 + image: ${SERVER_IMAGE} + network_mode: host + command: > + sh -c 'cat << EOF > /etc/sentinel.conf && redis-sentinel /etc/sentinel.conf + port 26381 + tls-port 36381 + tls-cert-file /docker/tls/server.crt + tls-key-file /docker/tls/server.key + tls-ca-cert-file /docker/tls/ca.crt + tls-auth-clients no + sentinel resolve-hostnames yes + sentinel announce-hostnames yes + sentinel announce-ip localhost + sentinel monitor mymaster localhost 6379 2 + sentinel down-after-milliseconds mymaster 10000 + sentinel failover-timeout mymaster 10000 + sentinel parallel-syncs mymaster 1 + EOF' + volumes: + - ./docker:/docker + + builder: - image: ${BUILDER_IMAGE} container_name: builder + image: ${BUILDER_IMAGE} + network_mode: host tty: true - environment: - - BOOST_REDIS_TEST_SERVER=redis volumes: - ../:/boost-redis - /tmp/redis-socks:/tmp/redis-socks diff --git a/tools/docker/entrypoint.sh b/tools/docker/entrypoint.sh deleted file mode 100755 index 318ea93c7..000000000 --- a/tools/docker/entrypoint.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -# The Redis container entrypoint. Runs the server with the required -# flags and makes the socket accessible - -set -e - -chmod 777 /tmp/redis-socks - -redis-server \ - --tls-port 6380 \ - --tls-cert-file /docker/tls/server.crt \ - --tls-key-file /docker/tls/server.key \ - --tls-ca-cert-file /docker/tls/ca.crt \ - --tls-auth-clients no \ - --unixsocket /tmp/redis-socks/redis.sock \ - --unixsocketperm 777 From 91afb4a27910ee87449c0e2c84c353eaf6484bea Mon Sep 17 00:00:00 2001 From: Marcelo Date: Thu, 20 Nov 2025 14:38:49 +0100 Subject: [PATCH 18/37] Fixes SBO problem and adds some tests (#356) --- include/boost/redis/impl/flat_tree.ipp | 2 +- include/boost/redis/resp3/flat_tree.hpp | 3 +- test/test_low_level_sync_sans_io.cpp | 44 ++++++++++++++++++------- 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/include/boost/redis/impl/flat_tree.ipp b/include/boost/redis/impl/flat_tree.ipp index 48be5040c..095e48ae6 100644 --- a/include/boost/redis/impl/flat_tree.ipp +++ b/include/boost/redis/impl/flat_tree.ipp @@ -81,7 +81,7 @@ void flat_tree::add_node_impl(node_view const& node) ranges_.push_back({data_.size(), node.value.size()}); // This must come after setting the offset above. - data_.append(node.value.data(), node.value.size()); + data_.insert(data_.end(), node.value.begin(), node.value.end()); view_tree_.push_back(node); } diff --git a/include/boost/redis/resp3/flat_tree.hpp b/include/boost/redis/resp3/flat_tree.hpp index 90b232143..4635b95b9 100644 --- a/include/boost/redis/resp3/flat_tree.hpp +++ b/include/boost/redis/resp3/flat_tree.hpp @@ -12,7 +12,6 @@ #include #include -#include #include namespace boost::redis { @@ -108,7 +107,7 @@ struct flat_tree { friend bool operator==(range const&, range const&); - std::string data_; + std::vector data_; view_tree view_tree_; std::vector ranges_; std::size_t pos_ = 0u; diff --git a/test/test_low_level_sync_sans_io.cpp b/test/test_low_level_sync_sans_io.cpp index f60b17113..687263d5b 100644 --- a/test/test_low_level_sync_sans_io.cpp +++ b/test/test_low_level_sync_sans_io.cpp @@ -395,10 +395,10 @@ BOOST_AUTO_TEST_CASE(flat_tree_views_are_set) deserialize(resp3_set, adapt2(resp3), ec); BOOST_CHECK_EQUAL(ec, error_code{}); - BOOST_CHECK_EQUAL(resp2.get_reallocs(), 1u); + BOOST_CHECK_EQUAL(resp2.get_reallocs(), 4u); BOOST_CHECK_EQUAL(resp2.get_total_msgs(), 1u); - BOOST_CHECK_EQUAL(resp3.value().get_reallocs(), 1u); + BOOST_CHECK_EQUAL(resp3.value().get_reallocs(), 4u); BOOST_CHECK_EQUAL(resp3.value().get_total_msgs(), 1u); auto const tmp2 = from_flat(resp2); @@ -418,7 +418,7 @@ BOOST_AUTO_TEST_CASE(flat_tree_reuse) deserialize(resp3_set, adapt2(tmp), ec); BOOST_CHECK_EQUAL(ec, error_code{}); - BOOST_CHECK_EQUAL(tmp.get_reallocs(), 1u); + BOOST_CHECK_EQUAL(tmp.get_reallocs(), 4u); BOOST_CHECK_EQUAL(tmp.get_total_msgs(), 1u); // Copy to compare after the reuse. @@ -438,25 +438,45 @@ BOOST_AUTO_TEST_CASE(flat_tree_reuse) BOOST_AUTO_TEST_CASE(flat_tree_copy_assign) { - flat_tree resp; + flat_tree ref1; + flat_tree ref2; + flat_tree ref3; + flat_tree ref4; error_code ec; - deserialize(resp3_set, adapt2(resp), ec); + deserialize(resp3_set, adapt2(ref1), ec); + deserialize(resp3_set, adapt2(ref2), ec); + deserialize(resp3_set, adapt2(ref3), ec); + deserialize(resp3_set, adapt2(ref4), ec); BOOST_CHECK_EQUAL(ec, error_code{}); - // Copy - resp3::flat_tree copy1{resp}; + // Copy ctor + resp3::flat_tree copy1{ref1}; + + // Move ctor + resp3::flat_tree move1{std::move(ref2)}; // Copy assignment - resp3::flat_tree copy2 = resp; + resp3::flat_tree copy2 = ref1; + + // Move assignment + resp3::flat_tree move2 = std::move(ref3); // Assignment resp3::flat_tree copy3; - copy3 = resp; + copy3 = ref1; + + // Move assignment + resp3::flat_tree move3; + move3 = std::move(ref4); + + BOOST_TEST((copy1 == ref1)); + BOOST_TEST((copy2 == ref1)); + BOOST_TEST((copy3 == ref1)); - BOOST_TEST((copy1 == resp)); - BOOST_TEST((copy2 == resp)); - BOOST_TEST((copy3 == resp)); + BOOST_TEST((move1 == ref1)); + BOOST_TEST((move2 == ref1)); + BOOST_TEST((move3 == ref1)); } BOOST_AUTO_TEST_CASE(generic_flat_response_simple_error) From d9e4b2c720b39aef599f4a25a93bb9a17bb408f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anarthal=20=28Rub=C3=A9n=20P=C3=A9rez=29?= <34971811+anarthal@users.noreply.github.com> Date: Sat, 29 Nov 2025 21:35:53 +0100 Subject: [PATCH 19/37] Improves flat_tree implementation (#358) * Makes flat_tree implementation use a custom buffer. This allows: * Never dangling nodes (previously, node values could dangle after calling reserve() or if notify_done() wasn't called). * Reduced memory consumption * Increased runtime speed * Changes flat_tree assignment to the usual signature and semantics * Fixes a bug causing an assertion to trigger when copy-constructing an empty flat_tree. * Changes basic_node operator== and operator!= return type * Adds generic_flat_response, basic_tree, tree, view_tree, flat_tree to the reference page. * Adds a missing resp3:: qualifier to all names in the reference page that belong to the resp3 namespace. * Adds reference documentation to flat_tree. * Mentions generic_flat_response in the discussion. * Adds operator!= for basic_node to basic_node's reference page. * Adds test_flat_tree. close #357 close #354 close #352 --- doc/modules/ROOT/pages/reference.adoc | 24 +- .../ROOT/pages/requests_responses.adoc | 5 +- include/boost/redis/impl/flat_tree.ipp | 210 +++-- include/boost/redis/resp3/flat_tree.hpp | 258 +++-- include/boost/redis/resp3/node.hpp | 11 +- test/CMakeLists.txt | 1 + test/Jamfile | 1 + test/test_flat_tree.cpp | 881 ++++++++++++++++++ test/test_low_level_sync_sans_io.cpp | 144 --- 9 files changed, 1238 insertions(+), 297 deletions(-) create mode 100644 test/test_flat_tree.cpp diff --git a/doc/modules/ROOT/pages/reference.adoc b/doc/modules/ROOT/pages/reference.adoc index e87ee7cd8..6281911ee 100644 --- a/doc/modules/ROOT/pages/reference.adoc +++ b/doc/modules/ROOT/pages/reference.adoc @@ -55,6 +55,8 @@ xref:reference:boost/redis/response.adoc[`response`] xref:reference:boost/redis/generic_response.adoc[`generic_response`] +xref:reference:boost/redis/generic_flat_response.adoc[`generic_flat_response`] + xref:reference:boost/redis/consume_one-08.adoc[`consume_one`] @@ -70,25 +72,33 @@ xref:reference:boost/redis/adapter/result.adoc[`adapter::result`] xref:reference:boost/redis/any_adapter.adoc[`any_adapter`] | -xref:reference:boost/redis/resp3/basic_node.adoc[`basic_node`] +xref:reference:boost/redis/resp3/basic_node.adoc[`resp3::basic_node`] + +xref:reference:boost/redis/resp3/node.adoc[`resp3::node`] + +xref:reference:boost/redis/resp3/node_view.adoc[`resp3::node_view`] + +xref:reference:boost/redis/resp3/basic_tree.adoc[`resp3::basic_tree`] + +xref:reference:boost/redis/resp3/tree.adoc[`resp3::tree`] -xref:reference:boost/redis/resp3/node.adoc[`node`] +xref:reference:boost/redis/resp3/view_tree.adoc[`resp3::view_tree`] -xref:reference:boost/redis/resp3/node_view.adoc[`node_view`] +xref:reference:boost/redis/resp3/flat_tree.adoc[`resp3::flat_tree`] xref:reference:boost/redis/resp3/boost_redis_to_bulk-08.adoc[`boost_redis_to_bulk`] -xref:reference:boost/redis/resp3/type.adoc[`type`] +xref:reference:boost/redis/resp3/type.adoc[`resp3::type`] -xref:reference:boost/redis/resp3/is_aggregate.adoc[`is_aggregate`] +xref:reference:boost/redis/resp3/is_aggregate.adoc[`resp3::is_aggregate`] | xref:reference:boost/redis/adapter/adapt2.adoc[`adapter::adapt2`] -xref:reference:boost/redis/resp3/parser.adoc[`parser`] +xref:reference:boost/redis/resp3/parser.adoc[`resp3::parser`] -xref:reference:boost/redis/resp3/parse.adoc[`parse`] +xref:reference:boost/redis/resp3/parse.adoc[`resp3::parse`] |=== \ No newline at end of file diff --git a/doc/modules/ROOT/pages/requests_responses.adoc b/doc/modules/ROOT/pages/requests_responses.adoc index 59938dd5c..7f40fed11 100644 --- a/doc/modules/ROOT/pages/requests_responses.adoc +++ b/doc/modules/ROOT/pages/requests_responses.adoc @@ -278,7 +278,8 @@ struct basic_node { ---- Any response to a Redis command can be parsed into a -xref:reference:boost/redis/generic_response.adoc[boost::redis::generic_response]. +xref:reference:boost/redis/generic_response.adoc[boost::redis::generic_response] +and its counterpart xref:reference:boost/redis/generic_flat_response.adoc[boost::redis::generic_flat_response]. The vector can be seen as a pre-order view of the response tree. Using it is not different than using other types: @@ -292,7 +293,7 @@ co_await conn->async_exec(req, resp); For example, suppose we want to retrieve a hash data structure from Redis with `HGETALL`, some of the options are -* `boost::redis::generic_response`: always works. +* `boost::redis::generic_response` and `boost::redis::generic_flat_response`: always works. * `std::vector`: efficient and flat, all elements as string. * `std::map`: efficient if you need the data as a `std::map`. * `std::map`: efficient if you are storing serialized data. Avoids temporaries and requires `boost_redis_from_bulk` for `U` and `V`. diff --git a/include/boost/redis/impl/flat_tree.ipp b/include/boost/redis/impl/flat_tree.ipp index 095e48ae6..73aeabe30 100644 --- a/include/boost/redis/impl/flat_tree.ipp +++ b/include/boost/redis/impl/flat_tree.ipp @@ -7,119 +7,183 @@ // #include +#include +#include + #include +#include +#include +#include +#include + namespace boost::redis::resp3 { -flat_tree::flat_tree(flat_tree const& other) -: data_{other.data_} -, view_tree_{other.view_tree_} -, ranges_{other.ranges_} -, pos_{0u} -, reallocs_{0u} -, total_msgs_{other.total_msgs_} +namespace detail { + +// Updates string views by performing pointer arithmetic +inline void rebase_strings(view_tree& nodes, const char* old_base, const char* new_base) { - view_tree_.resize(ranges_.size()); - set_views(); + for (auto& nd : nodes) { + if (!nd.value.empty()) { + const auto offset = nd.value.data() - old_base; + BOOST_ASSERT(offset >= 0); + nd.value = {new_base + offset, nd.value.size()}; + } + } } -flat_tree& -flat_tree::operator=(flat_tree other) +// --- Operations in flat_buffer --- + +// Compute the new capacity upon reallocation. We always use powers of 2, +// starting in 512, to prevent many small allocations +inline std::size_t compute_capacity(std::size_t current, std::size_t requested) { - swap(*this, other); - return *this; + std::size_t res = (std::max)(current, static_cast(512u)); + while (res < requested) + res *= 2u; + return res; } -void flat_tree::reserve(std::size_t bytes, std::size_t nodes) +// Copy construction +inline flat_buffer copy_construct(const flat_buffer& other) { - data_.reserve(bytes); - view_tree_.reserve(nodes); - ranges_.reserve(nodes); + flat_buffer res{{}, other.size, 0u, 0u}; + + if (other.size > 0u) { + const std::size_t capacity = compute_capacity(0u, other.size); + res.data.reset(new char[capacity]); + res.capacity = capacity; + res.reallocs = 1u; + std::copy(other.data.get(), other.data.get() + other.size, res.data.get()); + } + + return res; } -void flat_tree::clear() +// Copy assignment +inline void copy_assign(flat_buffer& buff, const flat_buffer& other) { - pos_ = 0u; - total_msgs_ = 0u; - reallocs_ = 0u; - data_.clear(); - view_tree_.clear(); - ranges_.clear(); + // Make space if required + if (buff.capacity < other.size) { + const std::size_t capacity = compute_capacity(buff.capacity, other.size); + buff.data.reset(new char[capacity]); + buff.capacity = capacity; + ++buff.reallocs; + } + + // Copy the contents + std::copy(other.data.get(), other.data.get() + other.size, buff.data.get()); + buff.size = other.size; } -void flat_tree::set_views() +// Grows the buffer until reaching a target size. +// Might rebase the strings in nodes +inline void grow(flat_buffer& buff, std::size_t new_capacity, view_tree& nodes) { - BOOST_ASSERT_MSG(pos_ < view_tree_.size(), "notify_done called but no nodes added."); - BOOST_ASSERT_MSG(view_tree_.size() == ranges_.size(), "Incompatible sizes."); + if (new_capacity <= buff.capacity) + return; - for (; pos_ < view_tree_.size(); ++pos_) { - auto const& r = ranges_.at(pos_); - view_tree_.at(pos_).value = std::string_view{data_.data() + r.offset, r.size}; - } + // Compute the actual capacity that we will be using + new_capacity = compute_capacity(buff.capacity, new_capacity); + + // Allocate space + std::unique_ptr new_buffer{new char[new_capacity]}; + + // Copy any data into the newly allocated space + const char* data_before = buff.data.get(); + char* data_after = new_buffer.get(); + std::copy(data_before, data_before + buff.size, data_after); + + // Update the string views so they don't dangle + rebase_strings(nodes, data_before, data_after); + + // Replace the buffer. Note that size hasn't changed here + buff.data = std::move(new_buffer); + buff.capacity = new_capacity; + ++buff.reallocs; } -void flat_tree::notify_done() +// Appends a string to the buffer. +// Might rebase the string in nodes, but doesn't append any new node. +inline std::string_view append(flat_buffer& buff, std::string_view value, view_tree& nodes) { - total_msgs_ += 1; - set_views(); + // If there is nothing to copy, do nothing + if (value.empty()) + return value; + + // Make space for the new string + const std::size_t new_size = buff.size + value.size(); + grow(buff, new_size, nodes); + + // Copy the new value + const std::size_t offset = buff.size; + std::copy(value.data(), value.data() + value.size(), buff.data.get() + offset); + buff.size = new_size; + return {buff.data.get() + offset, value.size()}; } -void flat_tree::push(node_view const& node) -{ - auto data_before = data_.data(); - add_node_impl(node); - auto data_after = data_.data(); +} // namespace detail - if (data_after != data_before) { - pos_ = 0; - reallocs_ += 1; - } +flat_tree::flat_tree(flat_tree const& other) +: data_{detail::copy_construct(other.data_)} +, view_tree_{other.view_tree_} +, total_msgs_{other.total_msgs_} +{ + detail::rebase_strings(view_tree_, other.data_.data.get(), data_.data.get()); } -void flat_tree::add_node_impl(node_view const& node) +flat_tree& flat_tree::operator=(const flat_tree& other) { - ranges_.push_back({data_.size(), node.value.size()}); + if (this != &other) { + // Copy the data + detail::copy_assign(data_, other.data_); - // This must come after setting the offset above. - data_.insert(data_.end(), node.value.begin(), node.value.end()); + // Copy the nodes + view_tree_ = other.view_tree_; + detail::rebase_strings(view_tree_, other.data_.data.get(), data_.data.get()); - view_tree_.push_back(node); + // Copy the other fields + total_msgs_ = other.total_msgs_; + } + + return *this; } -void swap(flat_tree& a, flat_tree& b) +void flat_tree::reserve(std::size_t bytes, std::size_t nodes) { - using std::swap; - - swap(a.data_, b.data_); - swap(a.view_tree_, b.view_tree_); - swap(a.ranges_, b.ranges_); - swap(a.pos_, b.pos_); - swap(a.reallocs_, b.reallocs_); - swap(a.total_msgs_, b.total_msgs_); + // Space for the strings + detail::grow(data_, bytes, view_tree_); + + // Space for the nodes + view_tree_.reserve(nodes); } -bool -operator==( - flat_tree::range const& a, - flat_tree::range const& b) +void flat_tree::clear() noexcept { - return a.offset == b.offset && a.size == b.size; + data_.size = 0u; + view_tree_.clear(); + total_msgs_ = 0u; } -bool operator==(flat_tree const& a, flat_tree const& b) +void flat_tree::push(node_view const& nd) { - return - a.data_ == b.data_ && - a.view_tree_ == b.view_tree_ && - a.ranges_ == b.ranges_ && - a.pos_ == b.pos_ && - //a.reallocs_ == b.reallocs_ && - a.total_msgs_ == b.total_msgs_; + // Add the string + const std::string_view str = detail::append(data_, nd.value, view_tree_); + + // Add the node + view_tree_.push_back({ + nd.data_type, + nd.aggregate_size, + nd.depth, + str, + }); } -bool operator!=(flat_tree const& a, flat_tree const& b) +bool operator==(flat_tree const& a, flat_tree const& b) { - return !(a == b); + // data is already taken into account by comparing the nodes. + return a.view_tree_ == b.view_tree_ && a.total_msgs_ == b.total_msgs_; } -} // namespace boost::redis +} // namespace boost::redis::resp3 diff --git a/include/boost/redis/resp3/flat_tree.hpp b/include/boost/redis/resp3/flat_tree.hpp index 4635b95b9..b0bff1014 100644 --- a/include/boost/redis/resp3/flat_tree.hpp +++ b/include/boost/redis/resp3/flat_tree.hpp @@ -12,119 +12,241 @@ #include #include -#include +#include +#include namespace boost::redis { namespace adapter::detail { - template class general_aggregate; -} +template class general_aggregate; +} // namespace adapter::detail namespace resp3 { -/** @brief A generic-response that stores data contiguously +namespace detail { + +struct flat_buffer { + std::unique_ptr data; + std::size_t size = 0u; + std::size_t capacity = 0u; + std::size_t reallocs = 0u; +}; + +} // namespace detail + +/** @brief A generic response that stores data contiguously. * - * Similar to the @ref boost::redis::resp3::tree but data is - * stored contiguously. + * Implements a container of RESP3 nodes. It's similar to @ref boost::redis::resp3::tree, + * but node data is stored contiguously. This allows for amortized no allocations + * when re-using `flat_tree` objects. Like `tree`, it can contain the response + * to several Redis commands or several server pushes. Use @ref get_total_msgs + * to obtain how many responses this object contains. + * + * Objects are typically created by the user and passed to @ref connection::async_exec + * to be used as response containers. Call @ref get_view to access the actual RESP3 nodes. + * Once populated, `flat_tree` can't be modified, except for @ref clear and assignment. + * + * A `flat_tree` is conceptually similar to a pair of `std::vector` objects, one holding + * @ref resp3::node_view objects, and another owning the the string data that these views + * point to. The node capacity and the data capacity are the capacities of these two vectors. */ -struct flat_tree { +class flat_tree { public: - /// Default constructor + /** + * @brief Default constructor. + * + * Constructs an empty tree, with no nodes, zero node capacity and zero data capacity. + * + * @par Exception safety + * No-throw guarantee. + */ flat_tree() = default; - /// Move constructor - flat_tree(flat_tree&&) noexcept = default; + /** + * @brief Move constructor. + * + * Constructs a tree by taking ownership of the nodes in `other`. + * + * @par Object lifetimes + * References to the nodes and strings in `other` remain valid. + * + * @par Exception safety + * No-throw guarantee. + */ + flat_tree(flat_tree&& other) noexcept = default; - /// Copy constructor + /** + * @brief Copy constructor. + * + * Constructs a tree by copying the nodes in `other`. After the copy, + * `*this` and `other` have independent lifetimes (usual copy semantics). + * + * @par Exception safety + * Strong guarantee. Memory allocations might throw. + */ flat_tree(flat_tree const& other); - /// Copy assignment - flat_tree& operator=(flat_tree other); + /** + * @brief Move assignment. + * + * Replaces the nodes in `*this` by taking ownership of the nodes in `other`. + * `other` is left in a valid but unspecified state. + * + * @par Object lifetimes + * References to the nodes and strings in `other` remain valid. + * References to the nodes and strings in `*this` are invalidated. + * + * @par Exception safety + * No-throw guarantee. + */ + flat_tree& operator=(flat_tree&& other) = default; - friend void swap(flat_tree&, flat_tree&); + /** + * @brief Copy assignment. + * + * Replaces the nodes in `*this` by copying the nodes in `other`. + * After the copy, `*this` and `other` have independent lifetimes (usual copy semantics). + * + * @par Object lifetimes + * References to the nodes and strings in `*this` are invalidated. + * + * @par Exception safety + * Basic guarantee. Memory allocations might throw. + */ + flat_tree& operator=(const flat_tree& other); - friend - bool operator==(flat_tree const&, flat_tree const&); + friend bool operator==(flat_tree const&, flat_tree const&); - friend - bool operator!=(flat_tree const&, flat_tree const&); + friend bool operator!=(flat_tree const&, flat_tree const&); - /** @brief Reserve capacity + /** @brief Reserves capacity for incoming data. * - * Reserve memory for incoming data. + * Adding nodes (e.g. by passing the tree to `async_exec`) + * won't cause reallocations until the data or node capacities + * are exceeded, following the usual vector semantics. + * The implementation might reserve more capacity than the one requested. + * + * @par Object lifetimes + * References to the nodes and strings in `*this` are invalidated. * - * @param bytes Number of bytes to reserve for data. - * @param nodes Number of nodes to reserve. + * @par Exception safety + * Basic guarantee. Memory allocations might throw. + * + * @param bytes Number of bytes to reserve for data. + * @param nodes Number of nodes to reserve. */ void reserve(std::size_t bytes, std::size_t nodes); - /** @brief Clear both the data and the node buffers - * - * @Note: A `boost::redis:.flat_tree` can contain the - * response to multiple Redis commands and server pushes. Calling - * this function will erase everything contained in it. + /** @brief Clears the tree so it contains no nodes. + * + * Calling this function removes every node, making + * @ref get_view return empty and @ref get_total_msgs + * return zero. It does not modify the object's capacity. + * + * To re-use a `flat_tree` for several requests, + * use `clear()` before each `async_exec` call. + * + * @par Object lifetimes + * References to the nodes and strings in `*this` are invalidated. + * + * @par Exception safety + * No-throw guarantee. + */ + void clear() noexcept; + + /** @brief Returns the size of the data buffer, in bytes. + * + * You may use this function to calculate how much capacity + * should be reserved for data when calling @ref reserve. + * + * @par Exception safety + * No-throw guarantee. + * + * @returns The number of bytes in use in the data buffer. */ - void clear(); + auto data_size() const noexcept -> std::size_t { return data_.size; } - /// Returns the size of the data buffer - auto data_size() const noexcept -> std::size_t - { return data_.size(); } + /** @brief Returns the capacity of the data buffer, in bytes. + * + * Note that the actual capacity of the data buffer may be bigger + * than the one requested by @ref reserve. + * + * @par Exception safety + * No-throw guarantee. + * + * @returns The capacity of the data buffer, in bytes. + */ + auto data_capacity() const noexcept -> std::size_t { return data_.capacity; } - /// Returns the RESP3 response - auto get_view() const -> view_tree const& - { return view_tree_; } + /** @brief Returns a vector with the nodes in the tree. + * + * This is the main way to access the contents of the tree. + * + * @par Exception safety + * No-throw guarantee. + * + * @returns The nodes in the tree. + */ + auto get_view() const noexcept -> view_tree const& { return view_tree_; } - /** @brief Returns the number of times reallocation took place + /** @brief Returns the number of memory reallocations that took place in the data buffer. + * + * This function returns how many reallocations in the data buffer were performed and + * can be useful to determine how much memory to reserve upfront. + * + * @par Exception safety + * No-throw guarantee. * - * This function returns how many reallocations were performed and - * can be useful to determine how much memory to reserve upfront. + * @returns The number of times that the data buffer reallocated its memory. */ - auto get_reallocs() const noexcept -> std::size_t - { return reallocs_; } + auto get_reallocs() const noexcept -> std::size_t { return data_.reallocs; } - /// Returns the number of complete RESP3 messages contained in this object. - std::size_t get_total_msgs() const noexcept - { return total_msgs_; } + /** @brief Returns the number of complete RESP3 messages contained in this object. + * + * This value is equal to the number of nodes in the tree with a depth of zero. + * + * @par Exception safety + * No-throw guarantee. + * + * @returns The number of complete RESP3 messages contained in this object. + */ + std::size_t get_total_msgs() const noexcept { return total_msgs_; } private: template friend class adapter::detail::general_aggregate; - // Notify the object that all nodes were pushed. - void notify_done(); + void notify_done() { ++total_msgs_; } // Push a new node to the response void push(node_view const& node); - void add_node_impl(node_view const& node); - - void set_views(); - - // Range into the data buffer. - struct range { - std::size_t offset; - std::size_t size; - }; - - friend bool operator==(range const&, range const&); - - std::vector data_; + detail::flat_buffer data_; view_tree view_tree_; - std::vector ranges_; - std::size_t pos_ = 0u; - std::size_t reallocs_ = 0u; std::size_t total_msgs_ = 0u; }; -/// Swaps two responses -void swap(flat_tree&, flat_tree&); - -/// Equality operator +/** + * @brief Equality operator. + * @relates flat_tree + * + * Two trees are equal if they contain the same nodes in the same order. + * Capacities are not taken into account. + * + * @par Exception safety + * No-throw guarantee. + */ bool operator==(flat_tree const&, flat_tree const&); -/// Inequality operator -bool operator!=(flat_tree const&, flat_tree const&); +/** + * @brief Inequality operator. + * @relates flat_tree + * + * @par Exception safety + * No-throw guarantee. + */ +inline bool operator!=(flat_tree const& lhs, flat_tree const& rhs) { return !(lhs == rhs); } -} // resp3 -} // namespace boost::redis +} // namespace resp3 +} // namespace boost::redis #endif // BOOST_REDIS_RESP3_FLAT_TREE_HPP diff --git a/include/boost/redis/resp3/node.hpp b/include/boost/redis/resp3/node.hpp index 1e284dab3..e7a768222 100644 --- a/include/boost/redis/resp3/node.hpp +++ b/include/boost/redis/resp3/node.hpp @@ -43,7 +43,7 @@ struct basic_node { * @param b Right hand side node object. */ template -auto operator==(basic_node const& a, basic_node const& b) +bool operator==(basic_node const& a, basic_node const& b) { // clang-format off return a.aggregate_size == b.aggregate_size @@ -53,9 +53,14 @@ auto operator==(basic_node const& a, basic_node const& b) // clang-format on }; -/// Inequality operator for RESP3 nodes +/** @brief Inequality operator for RESP3 nodes. + * @relates basic_node + * + * @param a Left hand side node object. + * @param b Right hand side node object. + */ template -auto operator!=(basic_node const& a, basic_node const& b) +bool operator!=(basic_node const& a, basic_node const& b) { return !(a == b); }; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 58ad78a4f..72a28a29d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -51,6 +51,7 @@ make_test(test_setup_adapter) make_test(test_multiplexer) make_test(test_parse_sentinel_response) make_test(test_update_sentinel_list) +make_test(test_flat_tree) # Tests that require a real Redis server make_test(test_conn_quit) diff --git a/test/Jamfile b/test/Jamfile index 4ac93900a..5db6c0ae4 100644 --- a/test/Jamfile +++ b/test/Jamfile @@ -68,6 +68,7 @@ local tests = test_multiplexer test_parse_sentinel_response test_update_sentinel_list + test_flat_tree ; # Build and run the tests diff --git a/test/test_flat_tree.cpp b/test/test_flat_tree.cpp new file mode 100644 index 000000000..095af2c7f --- /dev/null +++ b/test/test_flat_tree.cpp @@ -0,0 +1,881 @@ +/* Copyright (c) 2018-2022 Marcelo Zimbres Silva (mzimbres@gmail.com) + * + * Distributed under the Boost Software License, Version 1.0. (See + * accompanying file LICENSE.txt) + */ + +#include +#include +#include +#include + +#include +#include +#include + +#include "print_node.hpp" + +#include +#include +#include +#include +#include +#include +#include + +using boost::redis::adapter::adapt2; +using boost::redis::adapter::result; +using boost::redis::resp3::tree; +using boost::redis::resp3::flat_tree; +using boost::redis::generic_flat_response; +using boost::redis::resp3::type; +using boost::redis::resp3::detail::deserialize; +using boost::redis::resp3::node; +using boost::redis::resp3::node_view; +using boost::redis::resp3::to_string; +using boost::redis::response; +using boost::system::error_code; + +namespace { + +void add_nodes( + flat_tree& to, + std::string_view data, + boost::source_location loc = BOOST_CURRENT_LOCATION) +{ + error_code ec; + deserialize(data, adapt2(to), ec); + if (!BOOST_TEST_EQ(ec, error_code{})) + std::cerr << "Called from " << loc << std::endl; +} + +void check_nodes( + const flat_tree& tree, + boost::span expected, + boost::source_location loc = BOOST_CURRENT_LOCATION) +{ + if (!BOOST_TEST_ALL_EQ( + tree.get_view().begin(), + tree.get_view().end(), + expected.begin(), + expected.end())) + std::cerr << "Called from " << loc << std::endl; +} + +// --- Adding nodes --- +// Adding nodes works, even when reallocations happen. +// Empty nodes don't cause trouble +void test_add_nodes() +{ + flat_tree t; + + // Add a bunch of nodes. Single allocation. Some nodes are empty. + add_nodes(t, "*2\r\n+hello\r\n+world\r\n"); + std::vector expected_nodes{ + {type::array, 2u, 0u, "" }, + {type::simple_string, 1u, 1u, "hello"}, + {type::simple_string, 1u, 1u, "world"}, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 10u); + BOOST_TEST_EQ(t.data_capacity(), 512u); + BOOST_TEST_EQ(t.get_reallocs(), 1u); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); + + // Capacity will have raised to 512 bytes, at least. Add some more without reallocations + add_nodes(t, "$3\r\nbye\r\n"); + expected_nodes.push_back({type::blob_string, 1u, 0u, "bye"}); + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 13u); + BOOST_TEST_EQ(t.data_capacity(), 512u); + BOOST_TEST_EQ(t.get_reallocs(), 1u); + BOOST_TEST_EQ(t.get_total_msgs(), 2u); + + // Add nodes above the first reallocation threshold. Node strings are still valid + const std::string long_value(600u, 'a'); + add_nodes(t, "+" + long_value + "\r\n"); + expected_nodes.push_back({type::simple_string, 1u, 0u, long_value}); + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 613u); + BOOST_TEST_EQ(t.data_capacity(), 1024u); + BOOST_TEST_EQ(t.get_reallocs(), 2u); + BOOST_TEST_EQ(t.get_total_msgs(), 3u); + + // Add some more nodes, still within the reallocation threshold + add_nodes(t, "+some_other_value\r\n"); + expected_nodes.push_back({type::simple_string, 1u, 0u, "some_other_value"}); + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 629u); + BOOST_TEST_EQ(t.data_capacity(), 1024u); + BOOST_TEST_EQ(t.get_reallocs(), 2u); + BOOST_TEST_EQ(t.get_total_msgs(), 4u); + + // Add some more, causing another reallocation + add_nodes(t, "+" + long_value + "\r\n"); + expected_nodes.push_back({type::simple_string, 1u, 0u, long_value}); + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 1229u); + BOOST_TEST_EQ(t.data_capacity(), 2048u); + BOOST_TEST_EQ(t.get_reallocs(), 3u); + BOOST_TEST_EQ(t.get_total_msgs(), 5u); +} + +// Strings are really copied into the object +void test_add_nodes_copies() +{ + flat_tree t; + + // Place the message in dynamic memory + constexpr std::string_view const_msg = "+some_long_value_for_a_node\r\n"; + std::unique_ptr data{new char[100]{}}; + std::copy(const_msg.begin(), const_msg.end(), data.get()); + + // Add nodes pointing into this message + add_nodes(t, data.get()); + + // Invalidate the original message + data.reset(); + + // Check + std::vector expected_nodes{ + {type::simple_string, 1u, 0u, "some_long_value_for_a_node"}, + }; + check_nodes(t, expected_nodes); +} + +// Reallocations happen only when we would exceed capacity +void test_add_nodes_capacity_limit() +{ + flat_tree t; + + // Add a node to reach capacity 512 + add_nodes(t, "+hello\r\n"); + BOOST_TEST_EQ(t.data_size(), 5u); + BOOST_TEST_EQ(t.data_capacity(), 512u); + + // Fill the rest of the capacity + add_nodes(t, "+" + std::string(507u, 'b') + "\r\n"); + BOOST_TEST_EQ(t.data_size(), 512u); + BOOST_TEST_EQ(t.data_capacity(), 512u); + + // Adding an empty node here doesn't change capacity + add_nodes(t, "_\r\n"); + BOOST_TEST_EQ(t.data_size(), 512u); + BOOST_TEST_EQ(t.data_capacity(), 512u); + + // Adding more data causes a reallocation + add_nodes(t, "+a\r\n"); + BOOST_TEST_EQ(t.data_size(), 513u); + BOOST_TEST_EQ(t.data_capacity(), 1024); + + // Same goes for the next capacity limit + add_nodes(t, "+" + std::string(511u, 'c') + "\r\n"); + BOOST_TEST_EQ(t.data_size(), 1024); + BOOST_TEST_EQ(t.data_capacity(), 1024); + + // Reallocation + add_nodes(t, "+u\r\n"); + BOOST_TEST_EQ(t.data_size(), 1025u); + BOOST_TEST_EQ(t.data_capacity(), 2048u); + + // This would continue + add_nodes(t, "+" + std::string(1024u, 'd') + "\r\n"); + BOOST_TEST_EQ(t.data_size(), 2049u); + BOOST_TEST_EQ(t.data_capacity(), 4096u); +} + +// It's no problem if a node is big enough to surpass several reallocation limits +void test_add_nodes_big_node() +{ + flat_tree t; + + // Add a bunch of nodes. Single allocation. Some nodes are empty. + const std::string long_value(1500u, 'h'); + add_nodes(t, "+" + long_value + "\r\n"); + std::vector expected_nodes{ + {type::simple_string, 1u, 0u, long_value}, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 1500u); + BOOST_TEST_EQ(t.data_capacity(), 2048u); + BOOST_TEST_EQ(t.get_reallocs(), 1u); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); +} + +// --- Reserving space --- +// The usual case, calling it before using it +void test_reserve() +{ + flat_tree t; + + t.reserve(1024u, 5u); + check_nodes(t, {}); + BOOST_TEST_EQ(t.get_view().capacity(), 5u); + BOOST_TEST_EQ(t.data_size(), 0u); + BOOST_TEST_EQ(t.data_capacity(), 1024); + BOOST_TEST_EQ(t.get_reallocs(), 1u); + BOOST_TEST_EQ(t.get_total_msgs(), 0u); + + // Adding some nodes now works + add_nodes(t, "+hello\r\n"); + std::vector expected_nodes{ + {type::simple_string, 1u, 0u, "hello"}, + }; + check_nodes(t, expected_nodes); +} + +// Reserving space uses the same allocation thresholds +void test_reserve_not_power_of_2() +{ + flat_tree t; + + // First threshold at 512 + t.reserve(200u, 5u); + BOOST_TEST_EQ(t.data_capacity(), 512u); + BOOST_TEST_EQ(t.get_reallocs(), 1u); + + // Second threshold at 1024 + t.reserve(600u, 5u); + BOOST_TEST_EQ(t.data_capacity(), 1024u); + BOOST_TEST_EQ(t.get_reallocs(), 2u); +} + +// Requesting a capacity below the current one does nothing +void test_reserve_below_current_capacity() +{ + flat_tree t; + + // Reserving with a zero capacity does nothing + t.reserve(0u, 0u); + BOOST_TEST_EQ(t.data_capacity(), 0u); + BOOST_TEST_EQ(t.get_reallocs(), 0u); + + // Increase capacity + t.reserve(400u, 5u); + BOOST_TEST_EQ(t.data_capacity(), 512u); + BOOST_TEST_EQ(t.get_reallocs(), 1u); + + // Reserving again does nothing + t.reserve(400u, 5u); + t.reserve(512u, 5u); + t.reserve(0u, 5u); + BOOST_TEST_EQ(t.data_capacity(), 512u); + BOOST_TEST_EQ(t.get_reallocs(), 1u); +} + +// Reserving might reallocate. If there are nodes, strings remain valid +void test_reserve_with_data() +{ + flat_tree t; + + // Add a bunch of nodes, and then reserve + add_nodes(t, "*2\r\n+hello\r\n+world\r\n"); + t.reserve(1000u, 10u); + + // Check + std::vector expected_nodes{ + {type::array, 2u, 0u, "" }, + {type::simple_string, 1u, 1u, "hello"}, + {type::simple_string, 1u, 1u, "world"}, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 10u); + BOOST_TEST_EQ(t.data_capacity(), 1024u); + BOOST_TEST_EQ(t.get_reallocs(), 2u); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); +} + +// --- Clear --- +void test_clear() +{ + flat_tree t; + + // Add a bunch of nodes, then clear + add_nodes(t, "*2\r\n+hello\r\n+world\r\n"); + t.clear(); + + // Nodes are no longer there, but memory hasn't been fred + check_nodes(t, {}); + BOOST_TEST_EQ(t.data_size(), 0u); + BOOST_TEST_EQ(t.data_capacity(), 512u); + BOOST_TEST_EQ(t.get_reallocs(), 1u); + BOOST_TEST_EQ(t.get_total_msgs(), 0u); +} + +// Clearing an empty tree doesn't cause trouble +void test_clear_empty() +{ + flat_tree t; + t.clear(); + + check_nodes(t, {}); + BOOST_TEST_EQ(t.data_size(), 0u); + BOOST_TEST_EQ(t.data_capacity(), 0u); + BOOST_TEST_EQ(t.get_reallocs(), 0u); + BOOST_TEST_EQ(t.get_total_msgs(), 0u); +} + +// With clear, memory can be reused +// The response should be reusable. +void test_clear_reuse() +{ + flat_tree t; + + // First use + add_nodes(t, "~6\r\n+orange\r\n+apple\r\n+one\r\n+two\r\n+three\r\n+orange\r\n"); + std::vector expected_nodes{ + {type::set, 6u, 0u, "" }, + {type::simple_string, 1u, 1u, "orange"}, + {type::simple_string, 1u, 1u, "apple" }, + {type::simple_string, 1u, 1u, "one" }, + {type::simple_string, 1u, 1u, "two" }, + {type::simple_string, 1u, 1u, "three" }, + {type::simple_string, 1u, 1u, "orange"}, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.get_reallocs(), 1u); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); + + // Second use + t.clear(); + add_nodes(t, "*2\r\n+hello\r\n+world\r\n"); + expected_nodes = { + {type::array, 2u, 0u, "" }, + {type::simple_string, 1u, 1u, "hello"}, + {type::simple_string, 1u, 1u, "world"}, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.get_reallocs(), 1u); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); +} + +// --- Default ctor --- +void test_default_constructor() +{ + flat_tree t; + + check_nodes(t, {}); + BOOST_TEST_EQ(t.data_size(), 0u); + BOOST_TEST_EQ(t.get_reallocs(), 0u); + BOOST_TEST_EQ(t.get_total_msgs(), 0u); +} + +// --- Copy ctor --- +void test_copy_ctor() +{ + // Setup + auto t = std::make_unique(); + add_nodes(*t, "*2\r\n+hello\r\n+world\r\n"); + std::vector expected_nodes{ + {type::array, 2u, 0u, "" }, + {type::simple_string, 1u, 1u, "hello"}, + {type::simple_string, 1u, 1u, "world"}, + }; + + // Construct, then destroy the original copy + flat_tree t2{*t}; + t.reset(); + + // Check + check_nodes(t2, expected_nodes); + BOOST_TEST_EQ(t2.data_size(), 10u); + BOOST_TEST_EQ(t2.data_capacity(), 512u); + BOOST_TEST_EQ(t2.get_reallocs(), 1u); + BOOST_TEST_EQ(t2.get_total_msgs(), 1u); +} + +// Copying an empty tree doesn't cause problems +void test_copy_ctor_empty() +{ + flat_tree t; + flat_tree t2{t}; + check_nodes(t2, {}); + BOOST_TEST_EQ(t2.data_size(), 0u); + BOOST_TEST_EQ(t2.data_capacity(), 0u); + BOOST_TEST_EQ(t2.get_reallocs(), 0u); + BOOST_TEST_EQ(t2.get_total_msgs(), 0u); +} + +// Copying an object that has no elements but some capacity doesn't cause trouble +void test_copy_ctor_empty_with_capacity() +{ + flat_tree t; + t.reserve(300u, 8u); + + flat_tree t2{t}; + check_nodes(t2, {}); + BOOST_TEST_EQ(t2.data_size(), 0u); + BOOST_TEST_EQ(t2.data_capacity(), 0u); + BOOST_TEST_EQ(t2.get_reallocs(), 0u); + BOOST_TEST_EQ(t2.get_total_msgs(), 0u); +} + +// Copying an object with more capacity than required adjusts its capacity +void test_copy_ctor_adjust_capacity() +{ + // Setup + flat_tree t; + add_nodes(t, "+hello\r\n"); + std::vector expected_nodes{ + {type::simple_string, 1u, 0u, "hello"}, + }; + + // Cause reallocations + t.reserve(1000u, 10u); + t.reserve(2000u, 10u); + t.reserve(4000u, 10u); + + // Copy + flat_tree t2{t}; + + // The target object has the minimum required capacity, + // and the number of reallocs has been reset + check_nodes(t2, expected_nodes); + BOOST_TEST_EQ(t2.data_size(), 5u); + BOOST_TEST_EQ(t2.data_capacity(), 512u); + BOOST_TEST_EQ(t2.get_reallocs(), 1u); + BOOST_TEST_EQ(t2.get_total_msgs(), 1u); +} + +// --- Move ctor --- +void test_move_ctor() +{ + flat_tree t; + add_nodes(t, "*2\r\n+hello\r\n+world\r\n"); + + flat_tree t2{std::move(t)}; + + std::vector expected_nodes{ + {type::array, 2u, 0u, "" }, + {type::simple_string, 1u, 1u, "hello"}, + {type::simple_string, 1u, 1u, "world"}, + }; + check_nodes(t2, expected_nodes); + BOOST_TEST_EQ(t2.data_size(), 10u); + BOOST_TEST_EQ(t2.data_capacity(), 512u); + BOOST_TEST_EQ(t2.get_reallocs(), 1u); + BOOST_TEST_EQ(t2.get_total_msgs(), 1u); +} + +// Moving an empty object doesn't cause trouble +void test_move_ctor_empty() +{ + flat_tree t; + + flat_tree t2{std::move(t)}; + + check_nodes(t2, {}); + BOOST_TEST_EQ(t2.data_size(), 0u); + BOOST_TEST_EQ(t2.data_capacity(), 0u); + BOOST_TEST_EQ(t2.get_reallocs(), 0u); + BOOST_TEST_EQ(t2.get_total_msgs(), 0u); +} + +// Moving an object with capacity but no data doesn't cause trouble +void test_move_ctor_with_capacity() +{ + flat_tree t; + t.reserve(1000u, 10u); + + flat_tree t2{std::move(t)}; + + check_nodes(t2, {}); + BOOST_TEST_EQ(t2.data_size(), 0u); + BOOST_TEST_EQ(t2.data_capacity(), 1024u); + BOOST_TEST_EQ(t2.get_reallocs(), 1u); + BOOST_TEST_EQ(t2.get_total_msgs(), 0u); +} + +// --- Copy assignment --- +void test_copy_assign() +{ + flat_tree t; + add_nodes(t, "+some_data\r\n"); + + auto t2 = std::make_unique(); + add_nodes(*t2, "*2\r\n+hello\r\n+world\r\n"); + + t = *t2; + + // Delete the source object, to check that we copied the contents + t2.reset(); + + std::vector expected_nodes{ + {type::array, 2u, 0u, "" }, + {type::simple_string, 1u, 1u, "hello"}, + {type::simple_string, 1u, 1u, "world"}, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 10u); + BOOST_TEST_EQ(t.data_capacity(), 512u); + BOOST_TEST_EQ(t.get_reallocs(), 1u); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); +} + +// The lhs is empty and doesn't have any capacity +void test_copy_assign_target_empty() +{ + flat_tree t; + + flat_tree t2; + add_nodes(t2, "+hello\r\n"); + + t = t2; + + std::vector expected_nodes{ + {type::simple_string, 1u, 0u, "hello"}, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 5u); + BOOST_TEST_EQ(t.data_capacity(), 512u); + BOOST_TEST_EQ(t.get_reallocs(), 1u); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); +} + +// If the target doesn't have enough capacity, a reallocation happens +void test_copy_assign_target_not_enough_capacity() +{ + flat_tree t; + add_nodes(t, "+hello\r\n"); + + const std::string big_node(2000u, 'a'); + flat_tree t2; + add_nodes(t2, "+" + big_node + "\r\n"); + + t = t2; + + std::vector expected_nodes{ + {type::simple_string, 1u, 0u, big_node}, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 2000u); + BOOST_TEST_EQ(t.data_capacity(), 2048u); + BOOST_TEST_EQ(t.get_reallocs(), 2u); // initial + assignment + BOOST_TEST_EQ(t.get_total_msgs(), 1u); +} + +// If the source of the assignment is empty, nothing bad happens +void test_copy_assign_source_empty() +{ + flat_tree t; + add_nodes(t, "+hello\r\n"); + + flat_tree t2; + + t = t2; + + check_nodes(t, {}); + BOOST_TEST_EQ(t.data_size(), 0u); + BOOST_TEST_EQ(t.data_capacity(), 512u); // capacity is kept + BOOST_TEST_EQ(t.get_reallocs(), 1u); + BOOST_TEST_EQ(t.get_total_msgs(), 0u); +} + +// If the source of the assignment has capacity but no data, we're OK +void test_copy_assign_source_with_capacity() +{ + flat_tree t; + add_nodes(t, "+hello\r\n"); + + flat_tree t2; + t2.reserve(1000u, 4u); + t2.reserve(4000u, 8u); + + t = t2; + + check_nodes(t, {}); + BOOST_TEST_EQ(t.data_size(), 0u); + BOOST_TEST_EQ(t.data_capacity(), 512u); // capacity is kept + BOOST_TEST_EQ(t.get_reallocs(), 1u); // not propagated + BOOST_TEST_EQ(t.get_total_msgs(), 0u); +} + +// If the source of the assignment has data with extra capacity +// and a reallocation is needed, the minimum amount of space is allocated +void test_copy_assign_source_with_extra_capacity() +{ + flat_tree t; + + flat_tree t2; + add_nodes(t2, "+hello\r\n"); + t2.reserve(4000u, 8u); + + t = t2; + + std::vector expected_nodes{ + {type::simple_string, 1u, 0u, "hello"}, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 5u); + BOOST_TEST_EQ(t.data_capacity(), 512u); + BOOST_TEST_EQ(t.get_reallocs(), 1u); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); +} + +void test_copy_assign_both_empty() +{ + flat_tree t; + flat_tree t2; + + t = t2; + + check_nodes(t, {}); + BOOST_TEST_EQ(t.data_size(), 0u); + BOOST_TEST_EQ(t.data_capacity(), 0u); + BOOST_TEST_EQ(t.get_reallocs(), 0u); + BOOST_TEST_EQ(t.get_total_msgs(), 0u); +} + +// Self-assignment doesn't cause trouble +void test_copy_assign_self() +{ + flat_tree t; + add_nodes(t, "+hello\r\n"); + + const auto& tref = t; + t = tref; + + std::vector expected_nodes{ + {type::simple_string, 1u, 0u, "hello"}, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 5u); + BOOST_TEST_EQ(t.data_capacity(), 512u); + BOOST_TEST_EQ(t.get_reallocs(), 1u); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); +} + +// --- Move assignment --- +void test_move_assign() +{ + flat_tree t; + add_nodes(t, "+some_data\r\n"); + + flat_tree t2; + add_nodes(t2, "*2\r\n+hello\r\n+world\r\n"); + + t = std::move(t2); + + std::vector expected_nodes{ + {type::array, 2u, 0u, "" }, + {type::simple_string, 1u, 1u, "hello"}, + {type::simple_string, 1u, 1u, "world"}, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 10u); + BOOST_TEST_EQ(t.data_capacity(), 512u); + BOOST_TEST_EQ(t.get_reallocs(), 1u); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); +} + +// The lhs is empty and doesn't have any capacity +void test_move_assign_target_empty() +{ + flat_tree t; + + flat_tree t2; + add_nodes(t2, "+hello\r\n"); + + t = std::move(t2); + + std::vector expected_nodes{ + {type::simple_string, 1u, 0u, "hello"}, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 5u); + BOOST_TEST_EQ(t.data_capacity(), 512u); + BOOST_TEST_EQ(t.get_reallocs(), 1u); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); +} + +// If the source of the assignment is empty, nothing bad happens +void test_move_assign_source_empty() +{ + flat_tree t; + add_nodes(t, "+hello\r\n"); + + flat_tree t2; + + t = std::move(t2); + + check_nodes(t, {}); + BOOST_TEST_EQ(t.data_size(), 0u); + BOOST_TEST_EQ(t.data_capacity(), 0u); + BOOST_TEST_EQ(t.get_reallocs(), 0u); + BOOST_TEST_EQ(t.get_total_msgs(), 0u); +} + +// If both source and target are empty, nothing bad happens +void test_move_assign_both_empty() +{ + flat_tree t; + + flat_tree t2; + + t = std::move(t2); + + check_nodes(t, {}); + BOOST_TEST_EQ(t.data_size(), 0u); + BOOST_TEST_EQ(t.data_capacity(), 0u); + BOOST_TEST_EQ(t.get_reallocs(), 0u); + BOOST_TEST_EQ(t.get_total_msgs(), 0u); +} + +// --- Comparison --- +void test_comparison_different() +{ + flat_tree t; + add_nodes(t, "+some_data\r\n"); + + flat_tree t2; + add_nodes(t2, "*2\r\n+hello\r\n+world\r\n"); + + BOOST_TEST_NOT(t == t2); + BOOST_TEST(t != t2); + BOOST_TEST_NOT(t2 == t); + BOOST_TEST(t2 != t); +} + +// The only difference is node types +void test_comparison_different_node_types() +{ + flat_tree t; + add_nodes(t, "+hello\r\n"); + + flat_tree t2; + add_nodes(t2, "$5\r\nhello\r\n"); + + BOOST_TEST_NOT(t == t2); + BOOST_TEST(t != t2); +} + +void test_comparison_equal() +{ + flat_tree t; + add_nodes(t, "+some_data\r\n"); + + flat_tree t2; + add_nodes(t2, "+some_data\r\n"); + + BOOST_TEST(t == t2); + BOOST_TEST_NOT(t != t2); +} + +// Allocations are not taken into account when comparing +void test_comparison_equal_reallocations() +{ + const std::string big_node(2000u, 'a'); + flat_tree t; + t.reserve(100u, 5u); + add_nodes(t, "+" + big_node + "\r\n"); + BOOST_TEST_EQ(t.get_reallocs(), 2u); + + flat_tree t2; + t2.reserve(2048u, 5u); + add_nodes(t2, "+" + big_node + "\r\n"); + BOOST_TEST_EQ(t2.get_reallocs(), 1u); + + BOOST_TEST(t == t2); + BOOST_TEST_NOT(t != t2); +} + +// Capacity is not taken into account when comparing +void test_comparison_equal_capacity() +{ + flat_tree t; + add_nodes(t, "+hello\r\n"); + + flat_tree t2; + t2.reserve(2048u, 5u); + add_nodes(t2, "+hello\r\n"); + + BOOST_TEST(t == t2); + BOOST_TEST_NOT(t != t2); +} + +// Empty containers don't cause trouble +void test_comparison_empty() +{ + flat_tree t; + add_nodes(t, "$5\r\nhello\r\n"); + + flat_tree tempty, tempty2; + + BOOST_TEST_NOT(t == tempty); + BOOST_TEST(t != tempty); + + BOOST_TEST_NOT(tempty == t); + BOOST_TEST(tempty != t); + + BOOST_TEST(tempty == tempty2); + BOOST_TEST_NOT(tempty != tempty2); +} + +// Self comparisons don't cause trouble +void test_comparison_self() +{ + flat_tree t; + add_nodes(t, "$5\r\nhello\r\n"); + + flat_tree tempty; + + BOOST_TEST(t == t); + BOOST_TEST_NOT(t != t); + + BOOST_TEST(tempty == tempty); + BOOST_TEST_NOT(tempty != tempty); +} + +} // namespace + +int main() +{ + test_add_nodes(); + test_add_nodes_copies(); + test_add_nodes_capacity_limit(); + test_add_nodes_big_node(); + + test_reserve(); + test_reserve_not_power_of_2(); + test_reserve_below_current_capacity(); + test_reserve_with_data(); + + test_clear(); + test_clear_empty(); + test_clear_reuse(); + + test_default_constructor(); + + test_copy_ctor(); + test_copy_ctor_empty(); + test_copy_ctor_empty_with_capacity(); + test_copy_ctor_adjust_capacity(); + + test_move_ctor(); + test_move_ctor_empty(); + test_move_ctor_with_capacity(); + + test_move_assign(); + test_move_assign_target_empty(); + test_move_assign_source_empty(); + test_move_assign_both_empty(); + + test_copy_assign(); + test_copy_assign_target_empty(); + test_copy_assign_target_not_enough_capacity(); + test_copy_assign_source_empty(); + test_copy_assign_source_with_capacity(); + test_copy_assign_source_with_extra_capacity(); + test_copy_assign_both_empty(); + test_copy_assign_self(); + + test_comparison_different(); + test_comparison_different_node_types(); + test_comparison_equal(); + test_comparison_equal_reallocations(); + test_comparison_equal_capacity(); + test_comparison_empty(); + test_comparison_self(); + + return boost::report_errors(); +} diff --git a/test/test_low_level_sync_sans_io.cpp b/test/test_low_level_sync_sans_io.cpp index 687263d5b..f840fa555 100644 --- a/test/test_low_level_sync_sans_io.cpp +++ b/test/test_low_level_sync_sans_io.cpp @@ -13,8 +13,6 @@ #include #include -#include "print_node.hpp" - #define BOOST_TEST_MODULE low_level_sync_sans_io #include @@ -337,148 +335,6 @@ BOOST_AUTO_TEST_CASE(check_counter_adapter) BOOST_CHECK_EQUAL(done, 1); } -namespace boost::redis::resp3 { - -template -std::ostream& operator<<(std::ostream& os, basic_tree const& resp) -{ - for (auto const& e : resp) - os << e << ","; - return os; -} - -} // namespace boost::redis::resp3 - -node from_node_view(node_view const& v) -{ - node ret; - ret.data_type = v.data_type; - ret.aggregate_size = v.aggregate_size; - ret.depth = v.depth; - ret.value = v.value; - return ret; -} - -tree from_flat(flat_tree const& resp) -{ - tree ret; - for (auto const& e : resp.get_view()) - ret.push_back(from_node_view(e)); - - return ret; -} - -tree from_flat(generic_flat_response const& resp) -{ - tree ret; - for (auto const& e : resp.value().get_view()) - ret.push_back(from_node_view(e)); - - return ret; -} - -// Parses the same data into a tree and a -// flat_tree, they should be equal to each other. -BOOST_AUTO_TEST_CASE(flat_tree_views_are_set) -{ - tree resp1; - flat_tree resp2; - generic_flat_response resp3; - - error_code ec; - deserialize(resp3_set, adapt2(resp1), ec); - BOOST_CHECK_EQUAL(ec, error_code{}); - - deserialize(resp3_set, adapt2(resp2), ec); - BOOST_CHECK_EQUAL(ec, error_code{}); - - deserialize(resp3_set, adapt2(resp3), ec); - BOOST_CHECK_EQUAL(ec, error_code{}); - - BOOST_CHECK_EQUAL(resp2.get_reallocs(), 4u); - BOOST_CHECK_EQUAL(resp2.get_total_msgs(), 1u); - - BOOST_CHECK_EQUAL(resp3.value().get_reallocs(), 4u); - BOOST_CHECK_EQUAL(resp3.value().get_total_msgs(), 1u); - - auto const tmp2 = from_flat(resp2); - BOOST_CHECK_EQUAL(resp1, tmp2); - - auto const tmp3 = from_flat(resp3); - BOOST_CHECK_EQUAL(resp1, tmp3); -} - -// The response should be reusable. -BOOST_AUTO_TEST_CASE(flat_tree_reuse) -{ - flat_tree tmp; - - // First use - error_code ec; - deserialize(resp3_set, adapt2(tmp), ec); - BOOST_CHECK_EQUAL(ec, error_code{}); - - BOOST_CHECK_EQUAL(tmp.get_reallocs(), 4u); - BOOST_CHECK_EQUAL(tmp.get_total_msgs(), 1u); - - // Copy to compare after the reuse. - auto const resp1 = tmp.get_view(); - tmp.clear(); - - // Second use - deserialize(resp3_set, adapt2(tmp), ec); - BOOST_CHECK_EQUAL(ec, error_code{}); - - // No reallocation this time - BOOST_CHECK_EQUAL(tmp.get_reallocs(), 0u); - BOOST_CHECK_EQUAL(tmp.get_total_msgs(), 1u); - - BOOST_CHECK_EQUAL(resp1, tmp.get_view()); -} - -BOOST_AUTO_TEST_CASE(flat_tree_copy_assign) -{ - flat_tree ref1; - flat_tree ref2; - flat_tree ref3; - flat_tree ref4; - - error_code ec; - deserialize(resp3_set, adapt2(ref1), ec); - deserialize(resp3_set, adapt2(ref2), ec); - deserialize(resp3_set, adapt2(ref3), ec); - deserialize(resp3_set, adapt2(ref4), ec); - BOOST_CHECK_EQUAL(ec, error_code{}); - - // Copy ctor - resp3::flat_tree copy1{ref1}; - - // Move ctor - resp3::flat_tree move1{std::move(ref2)}; - - // Copy assignment - resp3::flat_tree copy2 = ref1; - - // Move assignment - resp3::flat_tree move2 = std::move(ref3); - - // Assignment - resp3::flat_tree copy3; - copy3 = ref1; - - // Move assignment - resp3::flat_tree move3; - move3 = std::move(ref4); - - BOOST_TEST((copy1 == ref1)); - BOOST_TEST((copy2 == ref1)); - BOOST_TEST((copy3 == ref1)); - - BOOST_TEST((move1 == ref1)); - BOOST_TEST((move2 == ref1)); - BOOST_TEST((move3 == ref1)); -} - BOOST_AUTO_TEST_CASE(generic_flat_response_simple_error) { generic_flat_response resp; From 755d14a10dd8460a7a6ce947cf33d83428fa797c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anarthal=20=28Rub=C3=A9n=20P=C3=A9rez=29?= <34971811+anarthal@users.noreply.github.com> Date: Sat, 29 Nov 2025 22:26:37 +0100 Subject: [PATCH 20/37] Renames test_setup_request_utils to test_compose_setup_request (#359) --- test/CMakeLists.txt | 2 +- test/Jamfile | 2 +- ...ils.cpp => test_compose_setup_request.cpp} | 46 +++++++++---------- 3 files changed, 24 insertions(+), 26 deletions(-) rename test/{test_setup_request_utils.cpp => test_compose_setup_request.cpp} (88%) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 72a28a29d..a7ab35180 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -46,7 +46,7 @@ make_test(test_reader_fsm) make_test(test_connect_fsm) make_test(test_sentinel_resolve_fsm) make_test(test_run_fsm) -make_test(test_setup_request_utils) +make_test(test_compose_setup_request) make_test(test_setup_adapter) make_test(test_multiplexer) make_test(test_parse_sentinel_response) diff --git a/test/Jamfile b/test/Jamfile index 5db6c0ae4..b6676bf67 100644 --- a/test/Jamfile +++ b/test/Jamfile @@ -63,7 +63,7 @@ local tests = test_sentinel_resolve_fsm test_run_fsm test_connect_fsm - test_setup_request_utils + test_compose_setup_request test_setup_adapter test_multiplexer test_parse_sentinel_response diff --git a/test/test_setup_request_utils.cpp b/test/test_compose_setup_request.cpp similarity index 88% rename from test/test_setup_request_utils.cpp rename to test/test_compose_setup_request.cpp index b2bab2ef8..2f91a9470 100644 --- a/test/test_setup_request_utils.cpp +++ b/test/test_compose_setup_request.cpp @@ -23,11 +23,9 @@ namespace redis = boost::redis; using redis::detail::compose_setup_request; using boost::system::error_code; -// TODO: rename the file - namespace { -void test_compose_setup() +void test_hello() { redis::config cfg; cfg.clientname = ""; @@ -41,7 +39,7 @@ void test_compose_setup() BOOST_TEST(cfg.setup.get_config().cancel_on_connection_lost); } -void test_compose_setup_select() +void test_select() { redis::config cfg; cfg.clientname = ""; @@ -58,7 +56,7 @@ void test_compose_setup_select() BOOST_TEST(cfg.setup.get_config().cancel_on_connection_lost); } -void test_compose_setup_clientname() +void test_clientname() { redis::config cfg; @@ -72,7 +70,7 @@ void test_compose_setup_clientname() BOOST_TEST(cfg.setup.get_config().cancel_on_connection_lost); } -void test_compose_setup_auth() +void test_auth() { redis::config cfg; cfg.clientname = ""; @@ -89,7 +87,7 @@ void test_compose_setup_auth() BOOST_TEST(cfg.setup.get_config().cancel_on_connection_lost); } -void test_compose_setup_auth_empty_password() +void test_auth_empty_password() { redis::config cfg; cfg.clientname = ""; @@ -105,7 +103,7 @@ void test_compose_setup_auth_empty_password() BOOST_TEST(cfg.setup.get_config().cancel_on_connection_lost); } -void test_compose_setup_auth_setname() +void test_auth_setname() { redis::config cfg; cfg.clientname = "mytest"; @@ -123,7 +121,7 @@ void test_compose_setup_auth_setname() BOOST_TEST(cfg.setup.get_config().cancel_on_connection_lost); } -void test_compose_setup_use_setup() +void test_use_setup() { redis::config cfg; cfg.clientname = "mytest"; @@ -145,7 +143,7 @@ void test_compose_setup_use_setup() } // Regression check: we set the priority flag -void test_compose_setup_use_setup_no_hello() +void test_use_setup_no_hello() { redis::config cfg; cfg.use_setup = true; @@ -162,7 +160,7 @@ void test_compose_setup_use_setup_no_hello() } // Regression check: we set the relevant cancellation flags in the request -void test_compose_setup_use_setup_flags() +void test_use_setup_flags() { redis::config cfg; cfg.use_setup = true; @@ -182,7 +180,7 @@ void test_compose_setup_use_setup_flags() // When using Sentinel, a ROLE command is added. This works // both with the old HELLO and new setup strategies. -void test_compose_setup_sentinel_auth() +void test_sentinel_auth() { redis::config cfg; cfg.sentinel.addresses = { @@ -203,7 +201,7 @@ void test_compose_setup_sentinel_auth() BOOST_TEST(cfg.setup.get_config().cancel_on_connection_lost); } -void test_compose_setup_sentinel_use_setup() +void test_sentinel_use_setup() { redis::config cfg; cfg.sentinel.addresses = { @@ -228,17 +226,17 @@ void test_compose_setup_sentinel_use_setup() int main() { - test_compose_setup(); - test_compose_setup_select(); - test_compose_setup_clientname(); - test_compose_setup_auth(); - test_compose_setup_auth_empty_password(); - test_compose_setup_auth_setname(); - test_compose_setup_use_setup(); - test_compose_setup_use_setup_no_hello(); - test_compose_setup_use_setup_flags(); - test_compose_setup_sentinel_auth(); - test_compose_setup_sentinel_use_setup(); + test_hello(); + test_select(); + test_clientname(); + test_auth(); + test_auth_empty_password(); + test_auth_setname(); + test_use_setup(); + test_use_setup_no_hello(); + test_use_setup_flags(); + test_sentinel_auth(); + test_sentinel_use_setup(); return boost::report_errors(); } \ No newline at end of file From 6005ebd04a42c01edc95db04c1a54194d6bec75b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anarthal=20=28Rub=C3=A9n=20P=C3=A9rez=29?= <34971811+anarthal@users.noreply.github.com> Date: Tue, 2 Dec 2025 11:20:01 +0100 Subject: [PATCH 21/37] Fixes std::tuple serialization and adds tests (#363) * Fixes a problem that caused passing ranges containing tuples into `request::push_range` to generate invalid commands. * Adds test_serialization * Updates request reference docs to reflect the requirements of the types passed to push and push_range close #360 --- include/boost/redis/request.hpp | 123 ++++++++++--- include/boost/redis/resp3/serialization.hpp | 12 +- test/CMakeLists.txt | 1 + test/Jamfile | 1 + test/test_request.cpp | 2 - test/test_serialization.cpp | 193 ++++++++++++++++++++ 6 files changed, 297 insertions(+), 35 deletions(-) create mode 100644 test/test_serialization.cpp diff --git a/include/boost/redis/request.hpp b/include/boost/redis/request.hpp index ca6f99521..fc3160c06 100644 --- a/include/boost/redis/request.hpp +++ b/include/boost/redis/request.hpp @@ -10,7 +10,6 @@ #include #include -#include #include #include @@ -34,11 +33,9 @@ struct request_access; * * @code * request r; - * r.push("HELLO", 3); - * r.push("FLUSHALL"); - * r.push("PING"); - * r.push("PING", "key"); - * r.push("QUIT"); + * r.push("SET", "k1", "some_value"); + * r.push("SET", "k2", "other_value"); + * r.push("GET", "k3"); * @endcode * * Uses a `std::string` for internal storage. @@ -146,14 +143,14 @@ class request { * * @code * request req; - * req.push("SET", "key", "some string", "EX", "2"); + * req.push("SET", "key", "some string", "EX", 2); * @endcode * * This will add a `SET` command with value `"some string"` and an * expiration of 2 seconds. * - * Command arguments should either be convertible to `std::string_view` - * or support the `boost_redis_to_bulk` function. + * Command arguments should either be convertible to `std::string_view`, + * integral types, or support the `boost_redis_to_bulk` function. * This function is a customization point that must be made available * using ADL and must have the following signature: * @@ -165,7 +162,7 @@ class request { * See cpp20_serialization.cpp * * @param cmd The command to execute. It should be a redis or sentinel command, like `"SET"`. - * @param args Command arguments. Non-string types will be converted to string by calling `boost_redis_to_bulk` on each argument. + * @param args Command arguments. `args` is allowed to be empty. * @tparam Ts Types of the command arguments. * */ @@ -196,21 +193,36 @@ class request { * req.push_range("HSET", "key", map.cbegin(), map.cend()); * @endcode * - * Command arguments should either be convertible to `std::string_view` - * or support the `boost_redis_to_bulk` function. - * This function is a customization point that must be made available - * using ADL and must have the following signature: + * This will generate the following command: * * @code - * void boost_redis_to_bulk(std::string& to, T const& t); + * HSET key key1 value1 key2 value2 key3 value3 * @endcode - * + * + * *If the passed range is empty, no command is added* and this + * function becomes a no-op. + * + * The value type of the passed range should satisfy one of the following: + * + * @li The type is convertible to `std::string_view`. One argument is added + * per element in the range. + * @li The type is an integral type. One argument is added + * per element in the range. + * @li The type supports the `boost_redis_to_bulk` function. One argument is added + * per element in the range. This function is a customization point that must be made available + * using ADL and must have the signature `void boost_redis_to_bulk(std::string& to, T const& t);`. + * @li The type is a `std::pair` instantiation, with both arguments supporting one of + * the points above. Two arguments are added per element in the range. + * Nested pairs are not allowed. + * @li The type is a `std::tuple` instantiation, with every argument supporting + * one of the points above. N arguments are added per element in the range, + * with N being the tuple size. Nested tuples are not allowed. + * * @param cmd The command to execute. It should be a redis or sentinel command, like `"SET"`. * @param key The command key. It will be added as the first argument to the command. * @param begin Iterator to the begin of the range. * @param end Iterator to the end of the range. - * @tparam ForwardIterator A forward iterator with an element type that is convertible to `std::string_view` - * or supports `boost_redis_to_bulk`. + * @tparam ForwardIterator A forward iterator with an element type that supports one of the points above. * * See cpp20_serialization.cpp */ @@ -249,23 +261,38 @@ class request { * { "channel1" , "channel2" , "channel3" }; * * request req; - * req.push("SUBSCRIBE", std::cbegin(channels), std::cend(channels)); + * req.push("SUBSCRIBE", channels.cbegin(), channels.cend()); * @endcode * - * Command arguments should either be convertible to `std::string_view` - * or support the `boost_redis_to_bulk` function. - * This function is a customization point that must be made available - * using ADL and must have the following signature: + * This will generate the following command: * * @code - * void boost_redis_to_bulk(std::string& to, T const& t); + * SUBSCRIBE channel1 channel2 channel3 * @endcode + * + * *If the passed range is empty, no command is added* and this + * function becomes a no-op. + * + * The value type of the passed range should satisfy one of the following: + * + * @li The type is convertible to `std::string_view`. One argument is added + * per element in the range. + * @li The type is an integral type. One argument is added + * per element in the range. + * @li The type supports the `boost_redis_to_bulk` function. One argument is added + * per element in the range. This function is a customization point that must be made available + * using ADL and must have the signature `void boost_redis_to_bulk(std::string& to, T const& t);`. + * @li The type is a `std::pair` instantiation, with both arguments supporting one of + * the points above. Two arguments are added per element in the range. + * Nested pairs are not allowed. + * @li The type is a `std::tuple` instantiation, with every argument supporting + * one of the points above. N arguments are added per element in the range, + * with N being the tuple size. Nested tuples are not allowed. * * @param cmd The command to execute. It should be a redis or sentinel command, like `"SET"`. * @param begin Iterator to the begin of the range. * @param end Iterator to the end of the range. - * @tparam ForwardIterator A forward iterator with an element type that is convertible to `std::string_view` - * or supports `boost_redis_to_bulk`. + * @tparam ForwardIterator A forward iterator with an element type that supports one of the points above. * * See cpp20_serialization.cpp */ @@ -296,13 +323,31 @@ class request { * * Equivalent to the overload taking a range of begin and end * iterators. + * + * *If the passed range is empty, no command is added* and this + * function becomes a no-op. + * + * The value type of the passed range should satisfy one of the following: + * + * @li The type is convertible to `std::string_view`. One argument is added + * per element in the range. + * @li The type is an integral type. One argument is added + * per element in the range. + * @li The type supports the `boost_redis_to_bulk` function. One argument is added + * per element in the range. This function is a customization point that must be made available + * using ADL and must have the signature `void boost_redis_to_bulk(std::string& to, T const& t);`. + * @li The type is a `std::pair` instantiation, with both arguments supporting one of + * the points above. Two arguments are added per element in the range. + * Nested pairs are not allowed. + * @li The type is a `std::tuple` instantiation, with every argument supporting + * one of the points above. N arguments are added per element in the range, + * with N being the tuple size. Nested tuples are not allowed. * * @param cmd The command to execute. It should be a redis or sentinel command, like `"SET"`. * @param key The command key. It will be added as the first argument to the command. * @param range Range containing the command arguments. * @tparam Range A type that can be passed to `std::begin()` and `std::end()` to obtain - * iterators. The range elements should be convertible to `std::string_view` - * or support `boost_redis_to_bulk`. + * iterators. */ template void push_range( @@ -320,12 +365,30 @@ class request { * * Equivalent to the overload taking a range of begin and end * iterators. + * + * *If the passed range is empty, no command is added* and this + * function becomes a no-op. + * + * The value type of the passed range should satisfy one of the following: + * + * @li The type is convertible to `std::string_view`. One argument is added + * per element in the range. + * @li The type is an integral type. One argument is added + * per element in the range. + * @li The type supports the `boost_redis_to_bulk` function. One argument is added + * per element in the range. This function is a customization point that must be made available + * using ADL and must have the signature `void boost_redis_to_bulk(std::string& to, T const& t);`. + * @li The type is a `std::pair` instantiation, with both arguments supporting one of + * the points above. Two arguments are added per element in the range. + * Nested pairs are not allowed. + * @li The type is a `std::tuple` instantiation, with every argument supporting + * one of the points above. N arguments are added per element in the range, + * with N being the tuple size. Nested tuples are not allowed. * * @param cmd The command to execute. It should be a redis or sentinel command, like `"SET"`. * @param range Range containing the command arguments. * @tparam Range A type that can be passed to `std::begin()` and `std::end()` to obtain - * iterators. The range elements should be convertible to `std::string_view` - * or support `boost_redis_to_bulk`. + * iterators. */ template void push_range( diff --git a/include/boost/redis/resp3/serialization.hpp b/include/boost/redis/resp3/serialization.hpp index 72de1b28f..28835925f 100644 --- a/include/boost/redis/resp3/serialization.hpp +++ b/include/boost/redis/resp3/serialization.hpp @@ -16,9 +16,6 @@ #include #include -// NOTE: Consider detecting tuples in the type in the parameter pack -// to calculate the header size correctly. - namespace boost::redis::resp3 { /** @brief Adds a bulk to the request. @@ -35,6 +32,10 @@ namespace boost::redis::resp3 { * } * @endcode * + * The function must add exactly one bulk string RESP3 node. + * If you're using `boost_redis_to_bulk` with a string argument, + * you're safe. + * * @param payload Storage on which data will be copied into. * @param data Data that will be serialized and stored in `payload`. */ @@ -100,6 +101,11 @@ struct bulk_counter> { static constexpr auto size = 2U; }; +template +struct bulk_counter> { + static constexpr auto size = sizeof...(T); +}; + void add_blob(std::string& payload, std::string_view blob); void add_separator(std::string& payload); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a7ab35180..d73055a32 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -35,6 +35,7 @@ endmacro() # Unit tests make_test(test_low_level) make_test(test_request) +make_test(test_serialization) make_test(test_low_level_sync_sans_io) make_test(test_any_adapter) make_test(test_log_to_file) diff --git a/test/Jamfile b/test/Jamfile index b6676bf67..5ed7d7980 100644 --- a/test/Jamfile +++ b/test/Jamfile @@ -52,6 +52,7 @@ lib redis_test_common local tests = test_low_level test_request + test_serialization test_low_level_sync_sans_io test_any_adapter test_log_to_file diff --git a/test/test_request.cpp b/test/test_request.cpp index 4be2425ef..63b59ca3c 100644 --- a/test/test_request.cpp +++ b/test/test_request.cpp @@ -14,8 +14,6 @@ using boost::redis::request; -// TODO: Serialization. - namespace { void test_push_no_args() diff --git a/test/test_serialization.cpp b/test/test_serialization.cpp new file mode 100644 index 000000000..18185ac41 --- /dev/null +++ b/test/test_serialization.cpp @@ -0,0 +1,193 @@ +// +// Copyright (c) 2025 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +using boost::redis::request; + +namespace other { + +struct my_struct { + int value; +}; + +void boost_redis_to_bulk(std::string& to, my_struct value) +{ + boost::redis::resp3::boost_redis_to_bulk(to, value.value); +} + +} // namespace other + +namespace { + +// --- Strings --- +void test_string_view() +{ + request req; + req.push("GET", std::string_view("key")); + BOOST_TEST_EQ(req.payload(), "*2\r\n$3\r\nGET\r\n$3\r\nkey\r\n"); +} + +void test_string() +{ + std::string s{"k1"}; + const std::string s2{"k2"}; + request req; + req.push("GET", s, s2, std::string("k3")); + BOOST_TEST_EQ(req.payload(), "*4\r\n$3\r\nGET\r\n$2\r\nk1\r\n$2\r\nk2\r\n$2\r\nk3\r\n"); +} + +void test_c_string() +{ + request req; + req.push("GET", "k1", static_cast("k2")); + BOOST_TEST_EQ(req.payload(), "*3\r\n$3\r\nGET\r\n$2\r\nk1\r\n$2\r\nk2\r\n"); +} + +void test_string_empty() +{ + request req; + req.push("GET", std::string_view{}); + BOOST_TEST_EQ(req.payload(), "*2\r\n$3\r\nGET\r\n$0\r\n\r\n"); +} + +// --- Integers --- +void test_signed_ints() +{ + request req; + req.push("GET", static_cast(20), static_cast(-42), -1, 80l, 200ll); + BOOST_TEST_EQ( + req.payload(), + "*6\r\n$3\r\nGET\r\n$2\r\n20\r\n$3\r\n-42\r\n$2\r\n-1\r\n$2\r\n80\r\n$3\r\n200\r\n"); +} + +void test_unsigned_ints() +{ + request req; + req.push( + "GET", + static_cast(20), + static_cast(42), + 50u, + 80ul, + 200ull); + BOOST_TEST_EQ( + req.payload(), + "*6\r\n$3\r\nGET\r\n$2\r\n20\r\n$2\r\n42\r\n$2\r\n50\r\n$2\r\n80\r\n$3\r\n200\r\n"); +} + +// We don't overflow for big ints +void test_signed_ints_minmax() +{ + using lims = std::numeric_limits; + request req; + req.push("GET", (lims::min)(), (lims::max)()); + BOOST_TEST_EQ( + req.payload(), + "*3\r\n$3\r\nGET\r\n$20\r\n-9223372036854775808\r\n$19\r\n9223372036854775807\r\n"); +} + +void test_unsigned_ints_max() +{ + request req; + req.push("GET", (std::numeric_limits::max)()); + BOOST_TEST_EQ(req.payload(), "*2\r\n$3\r\nGET\r\n$20\r\n18446744073709551615\r\n"); +} + +// Custom type +void test_custom() +{ + request req; + req.push("GET", other::my_struct{42}); + BOOST_TEST_EQ(req.payload(), "*2\r\n$3\r\nGET\r\n$2\r\n42\r\n"); +} + +// --- Pairs and tuples (only supported in the range versions) --- +// Nested structures are not supported (compile time error) +void test_pair() +{ + std::vector> vec{ + {"k1", 42} + }; + request req; + req.push_range("GET", vec); + BOOST_TEST_EQ(req.payload(), "*3\r\n$3\r\nGET\r\n$2\r\nk1\r\n$2\r\n42\r\n"); +} + +void test_pair_custom() +{ + std::vector> vec{ + {"k1", {42}} + }; + request req; + req.push_range("GET", vec); + BOOST_TEST_EQ(req.payload(), "*3\r\n$3\r\nGET\r\n$2\r\nk1\r\n$2\r\n42\r\n"); +} + +void test_tuple() +{ + std::vector> vec{ + {"k1", 42, 1} + }; + request req; + req.push_range("GET", vec); + BOOST_TEST_EQ(req.payload(), "*4\r\n$3\r\nGET\r\n$2\r\nk1\r\n$2\r\n42\r\n$1\r\n1\r\n"); +} + +void test_tuple_custom() +{ + std::vector> vec{ + {"k1", {42}} + }; + request req; + req.push_range("GET", vec); + BOOST_TEST_EQ(req.payload(), "*3\r\n$3\r\nGET\r\n$2\r\nk1\r\n$2\r\n42\r\n"); +} + +void test_tuple_empty() +{ + std::vector> vec{{}}; + request req; + req.push_range("GET", vec); + BOOST_TEST_EQ(req.payload(), "*1\r\n$3\r\nGET\r\n"); +} + +} // namespace + +int main() +{ + test_string_view(); + test_string(); + test_c_string(); + test_string_empty(); + + test_signed_ints(); + test_unsigned_ints(); + test_signed_ints_minmax(); + test_unsigned_ints_max(); + + test_custom(); + + test_pair(); + test_pair_custom(); + test_tuple(); + test_tuple_custom(); + test_tuple_empty(); + + return boost::report_errors(); +} From 02632b31c6285a179a24e1eab9e10d797be873b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anarthal=20=28Rub=C3=A9n=20P=C3=A9rez=29?= <34971811+anarthal@users.noreply.github.com> Date: Tue, 2 Dec 2025 11:20:32 +0100 Subject: [PATCH 22/37] Removes ostream inclusion from public headers (#364) close #361 --- include/boost/redis/resp3/impl/type.ipp | 2 ++ include/boost/redis/resp3/node.hpp | 4 ++++ include/boost/redis/resp3/parser.hpp | 1 - include/boost/redis/resp3/type.hpp | 5 ++--- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/include/boost/redis/resp3/impl/type.ipp b/include/boost/redis/resp3/impl/type.ipp index 983f69feb..3ad9a3e8a 100644 --- a/include/boost/redis/resp3/impl/type.ipp +++ b/include/boost/redis/resp3/impl/type.ipp @@ -8,6 +8,8 @@ #include +#include + namespace boost::redis::resp3 { auto to_string(type t) noexcept -> char const* diff --git a/include/boost/redis/resp3/node.hpp b/include/boost/redis/resp3/node.hpp index e7a768222..5486b74ae 100644 --- a/include/boost/redis/resp3/node.hpp +++ b/include/boost/redis/resp3/node.hpp @@ -9,6 +9,10 @@ #include +#include +#include +#include + namespace boost::redis::resp3 { /** @brief A node in the response tree. diff --git a/include/boost/redis/resp3/parser.hpp b/include/boost/redis/resp3/parser.hpp index 33436d79d..56abd4f59 100644 --- a/include/boost/redis/resp3/parser.hpp +++ b/include/boost/redis/resp3/parser.hpp @@ -12,7 +12,6 @@ #include #include -#include #include #include diff --git a/include/boost/redis/resp3/type.hpp b/include/boost/redis/resp3/type.hpp index be2c24a8c..0a2a6ed28 100644 --- a/include/boost/redis/resp3/type.hpp +++ b/include/boost/redis/resp3/type.hpp @@ -9,9 +9,8 @@ #include -#include -#include -#include +#include +#include namespace boost::redis::resp3 { From 2bbf0090b55a4d34e83d494a3fdb7d355b20751c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anarthal=20=28Rub=C3=A9n=20P=C3=A9rez=29?= <34971811+anarthal@users.noreply.github.com> Date: Tue, 2 Dec 2025 11:20:56 +0100 Subject: [PATCH 23/37] Mark consume_one as deprecated (#365) close #353 --- include/boost/redis/impl/response.ipp | 12 ++++++++++-- include/boost/redis/response.hpp | 8 +++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/include/boost/redis/impl/response.ipp b/include/boost/redis/impl/response.ipp index a4b09a6e7..c805be703 100644 --- a/include/boost/redis/impl/response.ipp +++ b/include/boost/redis/impl/response.ipp @@ -9,9 +9,13 @@ #include +#include + namespace boost::redis { -void consume_one(generic_response& r, system::error_code& ec) +namespace detail { + +inline void consume_one_impl(generic_response& r, system::error_code& ec) { if (r.has_error()) return; // Nothing to consume. @@ -38,10 +42,14 @@ void consume_one(generic_response& r, system::error_code& ec) r.value().erase(std::cbegin(r.value()), match); } +} // namespace detail + +void consume_one(generic_response& r, system::error_code& ec) { detail::consume_one_impl(r, ec); } + void consume_one(generic_response& r) { system::error_code ec; - consume_one(r, ec); + detail::consume_one_impl(r, ec); if (ec) throw system::system_error(ec); } diff --git a/include/boost/redis/response.hpp b/include/boost/redis/response.hpp index c39dc632d..4c83cf9d9 100644 --- a/include/boost/redis/response.hpp +++ b/include/boost/redis/response.hpp @@ -8,15 +8,13 @@ #define BOOST_REDIS_RESPONSE_HPP #include +#include #include #include -#include #include -#include #include -#include namespace boost::redis { @@ -75,7 +73,7 @@ using generic_flat_response = adapter::result; * @param r The response to modify. * @param ec Will be populated in case of error. */ -//BOOST_DEPRECATED("This function is not needed anymore to consume server pushes.") +BOOST_DEPRECATED("This function is not needed anymore to consume server pushes.") void consume_one(generic_response& r, system::error_code& ec); /** @@ -83,7 +81,7 @@ void consume_one(generic_response& r, system::error_code& ec); * * @param r The response to modify. */ -//BOOST_DEPRECATED("This function is not needed anymore to consume server pushes.") +BOOST_DEPRECATED("This function is not needed anymore to consume server pushes.") void consume_one(generic_response& r); } // namespace boost::redis From 7750a6b126e0728dfcd4ade6d3becf12e1b04f4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anarthal=20=28Rub=C3=A9n=20P=C3=A9rez=29?= <34971811+anarthal@users.noreply.github.com> Date: Wed, 3 Dec 2025 08:53:03 +0100 Subject: [PATCH 24/37] Splits read_buffer tests to a separate file (#368) --- test/CMakeLists.txt | 1 + test/Jamfile | 1 + test/test_low_level_sync_sans_io.cpp | 81 --------------------- test/test_read_buffer.cpp | 102 +++++++++++++++++++++++++++ 4 files changed, 104 insertions(+), 81 deletions(-) create mode 100644 test/test_read_buffer.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d73055a32..ea1e4b165 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -53,6 +53,7 @@ make_test(test_multiplexer) make_test(test_parse_sentinel_response) make_test(test_update_sentinel_list) make_test(test_flat_tree) +make_test(test_read_buffer) # Tests that require a real Redis server make_test(test_conn_quit) diff --git a/test/Jamfile b/test/Jamfile index 5ed7d7980..3bf9b443c 100644 --- a/test/Jamfile +++ b/test/Jamfile @@ -70,6 +70,7 @@ local tests = test_parse_sentinel_response test_update_sentinel_list test_flat_tree + test_read_buffer ; # Build and run the tests diff --git a/test/test_low_level_sync_sans_io.cpp b/test/test_low_level_sync_sans_io.cpp index f840fa555..69c3c221f 100644 --- a/test/test_low_level_sync_sans_io.cpp +++ b/test/test_low_level_sync_sans_io.cpp @@ -210,87 +210,6 @@ BOOST_AUTO_TEST_CASE(issue_233_optional_array_with_null) } } -BOOST_AUTO_TEST_CASE(read_buffer_prepare_error) -{ - using boost::redis::detail::read_buffer; - - read_buffer buf; - - // Usual case, max size is bigger then requested size. - buf.set_config({10, 10}); - auto ec = buf.prepare(); - BOOST_TEST(!ec); - buf.commit(10); - - // Corner case, max size is equal to the requested size. - buf.set_config({10, 20}); - ec = buf.prepare(); - BOOST_TEST(!ec); - buf.commit(10); - buf.consume(20); - - auto const tmp = buf; - - // Error case, max size is smaller to the requested size. - buf.set_config({10, 9}); - ec = buf.prepare(); - BOOST_TEST(ec == error_code{boost::redis::error::exceeds_maximum_read_buffer_size}); - - // Check that an error call has no side effects. - auto const res = buf == tmp; - BOOST_TEST(res); -} - -BOOST_AUTO_TEST_CASE(read_buffer_prepare_consume_only_committed_data) -{ - using boost::redis::detail::read_buffer; - - read_buffer buf; - - buf.set_config({10, 10}); - auto ec = buf.prepare(); - BOOST_TEST(!ec); - - auto res = buf.consume(5); - - // No data has been committed yet so nothing can be consummed. - BOOST_CHECK_EQUAL(res.consumed, 0u); - - // If nothing was consumed, nothing got rotated. - BOOST_CHECK_EQUAL(res.rotated, 0u); - - buf.commit(10); - res = buf.consume(5); - - // All five bytes should have been consumed. - BOOST_CHECK_EQUAL(res.consumed, 5u); - - // We added a total of 10 bytes and consumed 5, that means, 5 were - // rotated. - BOOST_CHECK_EQUAL(res.rotated, 5u); - - res = buf.consume(7); - - // Only the remaining five bytes can be consumed - BOOST_CHECK_EQUAL(res.consumed, 5u); - - // No bytes to rotated. - BOOST_CHECK_EQUAL(res.rotated, 0u); -} - -BOOST_AUTO_TEST_CASE(read_buffer_check_buffer_size) -{ - using boost::redis::detail::read_buffer; - - read_buffer buf; - - buf.set_config({10, 10}); - auto ec = buf.prepare(); - BOOST_TEST(!ec); - - BOOST_CHECK_EQUAL(buf.get_prepared().size(), 10u); -} - BOOST_AUTO_TEST_CASE(check_counter_adapter) { using boost::redis::any_adapter; diff --git a/test/test_read_buffer.cpp b/test/test_read_buffer.cpp new file mode 100644 index 000000000..85c0b07cb --- /dev/null +++ b/test/test_read_buffer.cpp @@ -0,0 +1,102 @@ +/* Copyright (c) 2018-2022 Marcelo Zimbres Silva (mzimbres@gmail.com) + * + * Distributed under the Boost Software License, Version 1.0. (See + * accompanying file LICENSE.txt) + */ + +#include +#include + +#include +#include + +using namespace boost::redis; +using detail::read_buffer; +using boost::system::error_code; + +namespace { + +void test_prepare_error() +{ + read_buffer buf; + + // Usual case, max size is bigger then requested size. + buf.set_config({10, 10}); + auto ec = buf.prepare(); + BOOST_TEST_EQ(ec, error_code()); + buf.commit(10); + + // Corner case, max size is equal to the requested size. + buf.set_config({10, 20}); + ec = buf.prepare(); + BOOST_TEST_EQ(ec, error_code()); + buf.commit(10); + buf.consume(20); + + auto const tmp = buf; + + // Error case, max size is smaller to the requested size. + buf.set_config({10, 9}); + ec = buf.prepare(); + BOOST_TEST_EQ(ec, error_code{error::exceeds_maximum_read_buffer_size}); + + // Check that an error call has no side effects. + BOOST_TEST(buf == tmp); +} + +void test_prepare_consume_only_committed_data() +{ + read_buffer buf; + + buf.set_config({10, 10}); + auto ec = buf.prepare(); + BOOST_TEST(!ec); + + auto res = buf.consume(5); + + // No data has been committed yet so nothing can be consummed. + BOOST_TEST_EQ(res.consumed, 0u); + + // If nothing was consumed, nothing got rotated. + BOOST_TEST_EQ(res.rotated, 0u); + + buf.commit(10); + res = buf.consume(5); + + // All five bytes should have been consumed. + BOOST_TEST_EQ(res.consumed, 5u); + + // We added a total of 10 bytes and consumed 5, that means, 5 were + // rotated. + BOOST_TEST_EQ(res.rotated, 5u); + + res = buf.consume(7); + + // Only the remaining five bytes can be consumed + BOOST_TEST_EQ(res.consumed, 5u); + + // No bytes to rotated. + BOOST_TEST_EQ(res.rotated, 0u); +} + +void test_check_buffer_size() +{ + read_buffer buf; + + buf.set_config({10, 10}); + auto ec = buf.prepare(); + BOOST_TEST_EQ(ec, error_code()); + + BOOST_TEST_EQ(buf.get_prepared().size(), 10u); +} + +} // namespace + +int main() +{ + test_prepare_error(); + test_prepare_consume_only_committed_data(); + test_check_buffer_size(); + + return boost::report_errors(); +} From 3b07119e541ea80425160fcf04905471c8d9fc4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anarthal=20=28Rub=C3=A9n=20P=C3=A9rez=29?= <34971811+anarthal@users.noreply.github.com> Date: Wed, 7 Jan 2026 09:55:23 +0100 Subject: [PATCH 25/37] Makes flat_tree aware of incremental parsing to avoid race conditions with pushes (#378) Adds the concept of a "temporary working area" to flat_tree. Nodes in this area belong to a partially parsed message, and are hidden from the user. Now flat_tree can be used as the receive response without explicitly handling partial messages. Changes flat_tree::get_view return type from const vector& to span. Adds flat_tree::capacity. Splits generic_flat_response tests to a separate file and adds extra cases. close #369 --- CMakeLists.txt | 1 + .../boost/redis/adapter/detail/adapters.hpp | 16 +- include/boost/redis/impl/flat_tree.ipp | 59 ++- include/boost/redis/resp3/flat_tree.hpp | 25 +- test/CMakeLists.txt | 1 + test/Jamfile | 1 + test/test_conn_exec.cpp | 28 ++ test/test_flat_tree.cpp | 458 +++++++++++++++++- test/test_generic_flat_response.cpp | 119 +++++ test/test_low_level_sync_sans_io.cpp | 37 -- 10 files changed, 689 insertions(+), 56 deletions(-) create mode 100644 test/test_generic_flat_response.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 35df41fcd..967f1ccd7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,6 +77,7 @@ if (BOOST_REDIS_MAIN_PROJECT) test json endian + compat ) foreach(dep IN LISTS deps) diff --git a/include/boost/redis/adapter/detail/adapters.hpp b/include/boost/redis/adapter/detail/adapters.hpp index 7830f0ed3..8419e2200 100644 --- a/include/boost/redis/adapter/detail/adapters.hpp +++ b/include/boost/redis/adapter/detail/adapters.hpp @@ -9,10 +9,10 @@ #include #include +#include #include #include #include -#include #include #include @@ -216,7 +216,12 @@ class general_aggregate { : tree_(c) { } - void on_init() { } + void on_init() + { + if (tree_->has_value()) { + tree_->value().notify_init(); + } + } void on_done() { BOOST_ASSERT_MSG(!!tree_, "Unexpected null pointer"); @@ -255,11 +260,8 @@ class general_aggregate { : tree_(c) { } - void on_init() { } - void on_done() - { - tree_->notify_done(); - } + void on_init() { tree_->notify_init(); } + void on_done() { tree_->notify_done(); } template void on_node(resp3::basic_node const& nd, system::error_code&) diff --git a/include/boost/redis/impl/flat_tree.ipp b/include/boost/redis/impl/flat_tree.ipp index 73aeabe30..7ee1aac4f 100644 --- a/include/boost/redis/impl/flat_tree.ipp +++ b/include/boost/redis/impl/flat_tree.ipp @@ -104,6 +104,27 @@ inline void grow(flat_buffer& buff, std::size_t new_capacity, view_tree& nodes) ++buff.reallocs; } +// Erases the first num_bytes bytes from the buffer by moving +// the remaining bytes forward. Rebases the strings in nodes as required. +inline void erase_first(flat_buffer& buff, std::size_t num_bytes, view_tree& nodes) +{ + BOOST_ASSERT(num_bytes <= buff.size); + if (num_bytes > 0u) { + // If we have any data to move, we should always have a buffer + BOOST_ASSERT(buff.data.get() != nullptr); + + // Record the old base + const char* old_base = buff.data.get() + num_bytes; + + // Move all that we're gonna keep to the start of the buffer + auto bytes_left = buff.size - num_bytes; + std::memmove(buff.data.get(), old_base, bytes_left); + + // Rebase strings + rebase_strings(nodes, old_base, buff.data.get()); + } +} + // Appends a string to the buffer. // Might rebase the string in nodes, but doesn't append any new node. inline std::string_view append(flat_buffer& buff, std::string_view value, view_tree& nodes) @@ -129,6 +150,8 @@ flat_tree::flat_tree(flat_tree const& other) : data_{detail::copy_construct(other.data_)} , view_tree_{other.view_tree_} , total_msgs_{other.total_msgs_} +, node_tmp_offset_{other.node_tmp_offset_} +, data_tmp_offset_{other.data_tmp_offset_} { detail::rebase_strings(view_tree_, other.data_.data.get(), data_.data.get()); } @@ -145,6 +168,8 @@ flat_tree& flat_tree::operator=(const flat_tree& other) // Copy the other fields total_msgs_ = other.total_msgs_; + node_tmp_offset_ = other.node_tmp_offset_; + data_tmp_offset_ = other.data_tmp_offset_; } return *this; @@ -161,8 +186,15 @@ void flat_tree::reserve(std::size_t bytes, std::size_t nodes) void flat_tree::clear() noexcept { - data_.size = 0u; - view_tree_.clear(); + // Discard everything except for the tmp area + view_tree_.erase(view_tree_.begin(), view_tree_.begin() + node_tmp_offset_); + node_tmp_offset_ = 0u; + + // Do the same for the data area + detail::erase_first(data_, data_tmp_offset_, view_tree_); + data_tmp_offset_ = 0u; + + // We now have no messages total_msgs_ = 0u; } @@ -180,10 +212,31 @@ void flat_tree::push(node_view const& nd) }); } +void flat_tree::notify_init() +{ + // Discard any data in the tmp area, as it belongs to an operation that never finished + BOOST_ASSERT(node_tmp_offset_ <= view_tree_.size()); + BOOST_ASSERT(data_tmp_offset_ <= data_.size); + view_tree_.resize(node_tmp_offset_); + data_.size = data_tmp_offset_; +} + +void flat_tree::notify_done() +{ + ++total_msgs_; + node_tmp_offset_ = view_tree_.size(); + data_tmp_offset_ = data_.size; +} + bool operator==(flat_tree const& a, flat_tree const& b) { // data is already taken into account by comparing the nodes. - return a.view_tree_ == b.view_tree_ && a.total_msgs_ == b.total_msgs_; + // Only committed nodes should be taken into account. + auto a_nodes = a.get_view(); + auto b_nodes = b.get_view(); + return a_nodes.size() == b_nodes.size() && + std::equal(a_nodes.begin(), a_nodes.end(), b_nodes.begin()) && + a.total_msgs_ == b.total_msgs_; } } // namespace boost::redis::resp3 diff --git a/include/boost/redis/resp3/flat_tree.hpp b/include/boost/redis/resp3/flat_tree.hpp index b0bff1014..4c6158c53 100644 --- a/include/boost/redis/resp3/flat_tree.hpp +++ b/include/boost/redis/resp3/flat_tree.hpp @@ -12,6 +12,8 @@ #include #include +#include + #include #include @@ -164,7 +166,16 @@ class flat_tree { * * @returns The number of bytes in use in the data buffer. */ - auto data_size() const noexcept -> std::size_t { return data_.size; } + auto data_size() const noexcept -> std::size_t { return data_tmp_offset_; } + + /** @brief Returns the capacity of the node container. + * + * @par Exception safety + * No-throw guarantee. + * + * @returns The capacity of the object, in number of nodes. + */ + auto capacity() const noexcept -> std::size_t { return view_tree_.capacity(); } /** @brief Returns the capacity of the data buffer, in bytes. * @@ -187,7 +198,7 @@ class flat_tree { * * @returns The nodes in the tree. */ - auto get_view() const noexcept -> view_tree const& { return view_tree_; } + span get_view() const noexcept { return {view_tree_.data(), node_tmp_offset_}; } /** @brief Returns the number of memory reallocations that took place in the data buffer. * @@ -215,7 +226,8 @@ class flat_tree { private: template friend class adapter::detail::general_aggregate; - void notify_done() { ++total_msgs_; } + void notify_init(); + void notify_done(); // Push a new node to the response void push(node_view const& node); @@ -223,6 +235,13 @@ class flat_tree { detail::flat_buffer data_; view_tree view_tree_; std::size_t total_msgs_ = 0u; + + // flat_tree supports a "temporary working area" for incrementally reading messages. + // Nodes in the tmp area are not part of the object representation until they + // are committed with notify_done(). + // These offsets delimit this area. + std::size_t node_tmp_offset_ = 0u; + std::size_t data_tmp_offset_ = 0u; }; /** diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ea1e4b165..40a1ea05d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -53,6 +53,7 @@ make_test(test_multiplexer) make_test(test_parse_sentinel_response) make_test(test_update_sentinel_list) make_test(test_flat_tree) +make_test(test_generic_flat_response) make_test(test_read_buffer) # Tests that require a real Redis server diff --git a/test/Jamfile b/test/Jamfile index 3bf9b443c..3aa72460f 100644 --- a/test/Jamfile +++ b/test/Jamfile @@ -70,6 +70,7 @@ local tests = test_parse_sentinel_response test_update_sentinel_list test_flat_tree + test_generic_flat_response test_read_buffer ; diff --git a/test/test_conn_exec.cpp b/test/test_conn_exec.cpp index 28a23497b..22216f9b8 100644 --- a/test/test_conn_exec.cpp +++ b/test/test_conn_exec.cpp @@ -6,6 +6,7 @@ #include #include +#include #include @@ -180,4 +181,31 @@ BOOST_AUTO_TEST_CASE(exec_any_adapter) BOOST_TEST(std::get<0>(res).value() == "PONG"); } +BOOST_AUTO_TEST_CASE(exec_generic_flat_response) +{ + // Executing with a generic_flat_response works + request req; + req.push("PING", "PONG"); + boost::redis::generic_flat_response resp; + + net::io_context ioc; + + auto conn = std::make_shared(ioc); + + bool finished = false; + + conn->async_exec(req, resp, [&](error_code ec, std::size_t) { + BOOST_TEST(ec == error_code()); + conn->cancel(); + finished = true; + }); + + run(conn); + ioc.run_for(test_timeout); + BOOST_TEST_REQUIRE(finished); + + BOOST_TEST(resp.has_value()); + BOOST_TEST(resp->get_view().front().value == "PONG"); +} + } // namespace diff --git a/test/test_flat_tree.cpp b/test/test_flat_tree.cpp index 095af2c7f..87b7f804a 100644 --- a/test/test_flat_tree.cpp +++ b/test/test_flat_tree.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -32,6 +33,7 @@ using boost::redis::resp3::type; using boost::redis::resp3::detail::deserialize; using boost::redis::resp3::node; using boost::redis::resp3::node_view; +using boost::redis::resp3::parser; using boost::redis::resp3::to_string; using boost::redis::response; using boost::system::error_code; @@ -49,6 +51,20 @@ void add_nodes( std::cerr << "Called from " << loc << std::endl; } +bool parse_checked( + flat_tree& to, + parser& p, + std::string_view data, + boost::source_location loc = BOOST_CURRENT_LOCATION) +{ + error_code ec; + auto adapter = adapt2(to); + bool done = boost::redis::resp3::parse(p, data, adapter, ec); + if (!BOOST_TEST_EQ(ec, error_code{})) + std::cerr << "Called from " << loc << std::endl; + return done; +} + void check_nodes( const flat_tree& tree, boost::span expected, @@ -202,6 +218,116 @@ void test_add_nodes_big_node() BOOST_TEST_EQ(t.get_total_msgs(), 1u); } +// Flat trees have a temporary area (tmp) where nodes are stored while +// messages are being parsed. Nodes in the tmp area are not part of the representation +// until they are committed when the message has been fully parsed +void test_add_nodes_tmp() +{ + flat_tree t; + parser p; + + // Add part of a message, but not all of it. + // These nodes are stored but are not part of the user-facing representation + BOOST_TEST_NOT(parse_checked(t, p, "*2\r\n+hello\r\n")); + check_nodes(t, {}); + BOOST_TEST_EQ(t.data_size(), 0u); + BOOST_TEST_EQ(t.data_capacity(), 512u); + BOOST_TEST_EQ(t.get_total_msgs(), 0u); + + // Finish the message. Nodes will now show up + BOOST_TEST(parse_checked(t, p, "*2\r\n+hello\r\n+world\r\n")); + std::vector expected_nodes{ + {type::array, 2u, 0u, "" }, + {type::simple_string, 1u, 1u, "hello"}, + {type::simple_string, 1u, 1u, "world"}, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 10u); + BOOST_TEST_EQ(t.data_capacity(), 512u); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); + + // We can repeat this cycle again + p.reset(); + BOOST_TEST_NOT(parse_checked(t, p, ">2\r\n+good\r\n")); + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 10u); + BOOST_TEST_EQ(t.data_capacity(), 512u); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); + + BOOST_TEST(parse_checked(t, p, ">2\r\n+good\r\n+bye\r\n")); + expected_nodes.push_back({type::push, 2u, 0u, ""}); + expected_nodes.push_back({type::simple_string, 1u, 1u, "good"}); + expected_nodes.push_back({type::simple_string, 1u, 1u, "bye"}); + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 17u); + BOOST_TEST_EQ(t.data_capacity(), 512u); + BOOST_TEST_EQ(t.get_total_msgs(), 2u); +} + +// If there was an unfinished message when another message is started, +// the former is discarded +void test_add_nodes_existing_tmp() +{ + flat_tree t; + parser p; + + // Add part of a message + BOOST_TEST_NOT(parse_checked(t, p, ">3\r\n+some message\r\n")); + check_nodes(t, {}); + BOOST_TEST_EQ(t.data_size(), 0u); + BOOST_TEST_EQ(t.get_total_msgs(), 0u); + + // This message is abandoned, and another one is started + p.reset(); + BOOST_TEST_NOT(parse_checked(t, p, "%66\r\n+abandoned\r\n")); + check_nodes(t, {}); + BOOST_TEST_EQ(t.data_size(), 0u); + BOOST_TEST_EQ(t.get_total_msgs(), 0u); + + // This happens again, but this time a complete message is added + add_nodes(t, "*2\r\n+hello\r\n+world\r\n"); + std::vector expected_nodes{ + {type::array, 2u, 0u, "" }, + {type::simple_string, 1u, 1u, "hello"}, + {type::simple_string, 1u, 1u, "world"}, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 10u); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); +} + +// The same works even if there is existing committed data +void test_add_nodes_existing_data_and_tmp() +{ + flat_tree t; + parser p; + + // Add a full message + add_nodes(t, "*2\r\n+hello\r\n+world\r\n"); + std::vector expected_nodes{ + {type::array, 2u, 0u, "" }, + {type::simple_string, 1u, 1u, "hello"}, + {type::simple_string, 1u, 1u, "world"}, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 10u); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); + + // Add part of a message + p.reset(); + BOOST_TEST_NOT(parse_checked(t, p, "%66\r\n+abandoned\r\n")); + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 10u); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); + + // This message is abandoned, and replaced by a full one + add_nodes(t, "+complete message\r\n"); + expected_nodes.push_back({type::simple_string, 1u, 0u, "complete message"}); + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 26u); + BOOST_TEST_EQ(t.get_total_msgs(), 2u); +} + // --- Reserving space --- // The usual case, calling it before using it void test_reserve() @@ -210,7 +336,7 @@ void test_reserve() t.reserve(1024u, 5u); check_nodes(t, {}); - BOOST_TEST_EQ(t.get_view().capacity(), 5u); + BOOST_TEST_GE(t.capacity(), 5u); BOOST_TEST_EQ(t.data_size(), 0u); BOOST_TEST_EQ(t.data_capacity(), 1024); BOOST_TEST_EQ(t.get_reallocs(), 1u); @@ -285,6 +411,32 @@ void test_reserve_with_data() BOOST_TEST_EQ(t.get_total_msgs(), 1u); } +// Reserve also handles the tmp area +void test_reserve_with_tmp() +{ + flat_tree t; + parser p; + + // Add a partial message, and then reserve + BOOST_TEST_NOT(parse_checked(t, p, "*2\r\n+hello\r\n")); + t.reserve(1000u, 10u); + + // Finish the current message so nodes in the tmp area show up + BOOST_TEST(parse_checked(t, p, "*2\r\n+hello\r\n+world\r\n")); + + // Check + std::vector expected_nodes{ + {type::array, 2u, 0u, "" }, + {type::simple_string, 1u, 1u, "hello"}, + {type::simple_string, 1u, 1u, "world"}, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 10u); + BOOST_TEST_EQ(t.data_capacity(), 1024u); + BOOST_TEST_EQ(t.get_reallocs(), 2u); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); +} + // --- Clear --- void test_clear() { @@ -349,6 +501,93 @@ void test_clear_reuse() BOOST_TEST_EQ(t.get_total_msgs(), 1u); } +// Clear doesn't remove the tmp area +void test_clear_tmp() +{ + flat_tree t; + parser p; + + // Add a full message and part of another + add_nodes(t, ">2\r\n+orange\r\n+apple\r\n"); + BOOST_TEST_NOT(parse_checked(t, p, "*2\r\n+hello\r\n")); + std::vector expected_nodes{ + {type::push, 2u, 0u, "" }, + {type::simple_string, 1u, 1u, "orange"}, + {type::simple_string, 1u, 1u, "apple" }, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); + + // Clearing removes the user-facing representation + t.clear(); + check_nodes(t, {}); + BOOST_TEST_EQ(t.get_total_msgs(), 0u); + + // The nodes in the tmp area are still alive. Adding the remaining yields the full message + BOOST_TEST(parse_checked(t, p, "*2\r\n+hello\r\n+world\r\n")); + expected_nodes = { + {type::array, 2u, 0u, "" }, + {type::simple_string, 1u, 1u, "hello"}, + {type::simple_string, 1u, 1u, "world"}, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); +} + +// Clearing having only tmp area is safe +void test_clear_only_tmp() +{ + flat_tree t; + parser p; + + // Add part of a message + BOOST_TEST_NOT(parse_checked(t, p, "*2\r\n+hello\r\n")); + check_nodes(t, {}); + BOOST_TEST_EQ(t.get_total_msgs(), 0u); + + // Clearing here does nothing + t.clear(); + check_nodes(t, {}); + BOOST_TEST_EQ(t.get_total_msgs(), 0u); + + // The nodes in the tmp area are still alive. Adding the remaining yields the full message + BOOST_TEST(parse_checked(t, p, "*2\r\n+hello\r\n+world\r\n")); + std::vector expected_nodes = { + {type::array, 2u, 0u, "" }, + {type::simple_string, 1u, 1u, "hello"}, + {type::simple_string, 1u, 1u, "world"}, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); +} + +// Clearing having tmp nodes but no data is also safe +void test_clear_only_tmp_nodes() +{ + flat_tree t; + parser p; + + // Add part of a message + BOOST_TEST_NOT(parse_checked(t, p, "*2\r\n")); + check_nodes(t, {}); + BOOST_TEST_EQ(t.get_total_msgs(), 0u); + + // Clearing here does nothing + t.clear(); + check_nodes(t, {}); + BOOST_TEST_EQ(t.get_total_msgs(), 0u); + + // The nodes in the tmp area are still alive. Adding the remaining yields the full message + BOOST_TEST(parse_checked(t, p, "*2\r\n+hello\r\n+world\r\n")); + std::vector expected_nodes = { + {type::array, 2u, 0u, "" }, + {type::simple_string, 1u, 1u, "hello"}, + {type::simple_string, 1u, 1u, "world"}, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); +} + // --- Default ctor --- void test_default_constructor() { @@ -437,6 +676,37 @@ void test_copy_ctor_adjust_capacity() BOOST_TEST_EQ(t2.get_total_msgs(), 1u); } +// Copying an object also copies its tmp area +void test_copy_ctor_tmp() +{ + // Setup + flat_tree t; + parser p; + add_nodes(t, "+message\r\n"); + BOOST_TEST_NOT(parse_checked(t, p, "*2\r\n+hello\r\n")); + std::vector expected_nodes{ + {type::simple_string, 1u, 0u, "message"}, + }; + + // Copy. The copy has the tmp nodes but they're hidden in its tmp area + flat_tree t2{t}; + check_nodes(t2, expected_nodes); + BOOST_TEST_EQ(t2.data_size(), 7u); + BOOST_TEST_EQ(t2.get_total_msgs(), 1u); + + // Finishing the message in the copy works + BOOST_TEST(parse_checked(t2, p, "*2\r\n+hello\r\n+world\r\n")); + expected_nodes = { + {type::simple_string, 1u, 0u, "message"}, + {type::array, 2u, 0u, "" }, + {type::simple_string, 1u, 1u, "hello" }, + {type::simple_string, 1u, 1u, "world" }, + }; + check_nodes(t2, expected_nodes); + BOOST_TEST_EQ(t2.data_size(), 17u); + BOOST_TEST_EQ(t2.get_total_msgs(), 2u); +} + // --- Move ctor --- void test_move_ctor() { @@ -486,6 +756,37 @@ void test_move_ctor_with_capacity() BOOST_TEST_EQ(t2.get_total_msgs(), 0u); } +// Moving an object also moves its tmp area +void test_move_ctor_tmp() +{ + // Setup + flat_tree t; + parser p; + add_nodes(t, "+message\r\n"); + BOOST_TEST_NOT(parse_checked(t, p, "*2\r\n+hello\r\n")); + std::vector expected_nodes{ + {type::simple_string, 1u, 0u, "message"}, + }; + + // Move. The new object has the same tmp area + flat_tree t2{std::move(t)}; + check_nodes(t2, expected_nodes); + BOOST_TEST_EQ(t2.data_size(), 7u); + BOOST_TEST_EQ(t2.get_total_msgs(), 1u); + + // Finishing the message in the copy works + BOOST_TEST(parse_checked(t2, p, "*2\r\n+hello\r\n+world\r\n")); + expected_nodes = { + {type::simple_string, 1u, 0u, "message"}, + {type::array, 2u, 0u, "" }, + {type::simple_string, 1u, 1u, "hello" }, + {type::simple_string, 1u, 1u, "world" }, + }; + check_nodes(t2, expected_nodes); + BOOST_TEST_EQ(t2.data_size(), 17u); + BOOST_TEST_EQ(t2.get_total_msgs(), 2u); +} + // --- Copy assignment --- void test_copy_assign() { @@ -645,6 +946,40 @@ void test_copy_assign_self() BOOST_TEST_EQ(t.get_total_msgs(), 1u); } +// Copy assignment also assigns the tmp area +void test_copy_assign_tmp() +{ + parser p; + + flat_tree t; + add_nodes(t, "+some_data\r\n"); + + flat_tree t2; + add_nodes(t2, "+message\r\n"); + BOOST_TEST_NOT(parse_checked(t2, p, "*2\r\n+hello\r\n")); + + // Assigning also copies where the tmp area starts + t = t2; + std::vector expected_nodes{ + {type::simple_string, 1u, 0u, "message"}, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 7u); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); + + // The tmp area was also copied + BOOST_TEST(parse_checked(t, p, "*2\r\n+hello\r\n+world\r\n")); + expected_nodes = { + {type::simple_string, 1u, 0u, "message"}, + {type::array, 2u, 0u, "" }, + {type::simple_string, 1u, 1u, "hello" }, + {type::simple_string, 1u, 1u, "world" }, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 17u); + BOOST_TEST_EQ(t.get_total_msgs(), 2u); +} + // --- Move assignment --- void test_move_assign() { @@ -721,6 +1056,40 @@ void test_move_assign_both_empty() BOOST_TEST_EQ(t.get_total_msgs(), 0u); } +// Move assignment also propagates the tmp area +void test_move_assign_tmp() +{ + parser p; + + flat_tree t; + add_nodes(t, "+some_data\r\n"); + + flat_tree t2; + add_nodes(t2, "+message\r\n"); + BOOST_TEST_NOT(parse_checked(t2, p, "*2\r\n+hello\r\n")); + + // When moving, the tmp area is moved, too + t = std::move(t2); + std::vector expected_nodes{ + {type::simple_string, 1u, 0u, "message"}, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 7u); + BOOST_TEST_EQ(t.get_total_msgs(), 1u); + + // Finish the message + BOOST_TEST(parse_checked(t, p, "*2\r\n+hello\r\n+world\r\n")); + expected_nodes = { + {type::simple_string, 1u, 0u, "message"}, + {type::array, 2u, 0u, "" }, + {type::simple_string, 1u, 1u, "hello" }, + {type::simple_string, 1u, 1u, "world" }, + }; + check_nodes(t, expected_nodes); + BOOST_TEST_EQ(t.data_size(), 17u); + BOOST_TEST_EQ(t.get_total_msgs(), 2u); +} + // --- Comparison --- void test_comparison_different() { @@ -826,6 +1195,67 @@ void test_comparison_self() BOOST_TEST_NOT(tempty != tempty); } +// The tmp area is not taken into account when comparing +void test_comparison_tmp() +{ + flat_tree t; + add_nodes(t, "+hello\r\n"); + + flat_tree t2; + add_nodes(t2, "+hello\r\n"); + parser p; + BOOST_TEST_NOT(parse_checked(t2, p, "*2\r\n+more data\r\n")); + + BOOST_TEST(t == t2); + BOOST_TEST_NOT(t != t2); +} + +void test_comparison_tmp_different() +{ + flat_tree t; + add_nodes(t, "+hello\r\n"); + + flat_tree t2; + add_nodes(t2, "+world\r\n"); + parser p; + BOOST_TEST_NOT(parse_checked(t2, p, "*2\r\n+more data\r\n")); + + BOOST_TEST_NOT(t == t2); + BOOST_TEST(t != t2); +} + +// Comparing object with only tmp area doesn't cause trouble +void test_comparison_only_tmp() +{ + flat_tree t; + parser p; + BOOST_TEST_NOT(parse_checked(t, p, "*2\r\n+more data\r\n")); + + flat_tree t2; + parser p2; + BOOST_TEST_NOT(parse_checked(t2, p2, "*2\r\n+random\r\n")); + + BOOST_TEST(t == t2); + BOOST_TEST_NOT(t != t2); +} + +// --- Capacity --- +// Delegates to the underlying vector function +void test_capacity() +{ + flat_tree t; + BOOST_TEST_EQ(t.capacity(), 0u); + + // Inserting a node increases capacity. + // It is not specified how capacity grows, though. + add_nodes(t, "+hello\r\n"); + BOOST_TEST_GE(t.capacity(), 1u); + + // Reserve also affects capacity + t.reserve(1000u, 8u); + BOOST_TEST_GE(t.capacity(), 8u); +} + } // namespace int main() @@ -834,15 +1264,22 @@ int main() test_add_nodes_copies(); test_add_nodes_capacity_limit(); test_add_nodes_big_node(); + test_add_nodes_tmp(); + test_add_nodes_existing_tmp(); + test_add_nodes_existing_data_and_tmp(); test_reserve(); test_reserve_not_power_of_2(); test_reserve_below_current_capacity(); test_reserve_with_data(); + test_reserve_with_tmp(); test_clear(); test_clear_empty(); test_clear_reuse(); + test_clear_tmp(); + test_clear_only_tmp(); + test_clear_only_tmp_nodes(); test_default_constructor(); @@ -850,15 +1287,12 @@ int main() test_copy_ctor_empty(); test_copy_ctor_empty_with_capacity(); test_copy_ctor_adjust_capacity(); + test_copy_ctor_tmp(); test_move_ctor(); test_move_ctor_empty(); test_move_ctor_with_capacity(); - - test_move_assign(); - test_move_assign_target_empty(); - test_move_assign_source_empty(); - test_move_assign_both_empty(); + test_move_ctor_tmp(); test_copy_assign(); test_copy_assign_target_empty(); @@ -868,6 +1302,13 @@ int main() test_copy_assign_source_with_extra_capacity(); test_copy_assign_both_empty(); test_copy_assign_self(); + test_copy_assign_tmp(); + + test_move_assign(); + test_move_assign_target_empty(); + test_move_assign_source_empty(); + test_move_assign_both_empty(); + test_move_assign_tmp(); test_comparison_different(); test_comparison_different_node_types(); @@ -876,6 +1317,11 @@ int main() test_comparison_equal_capacity(); test_comparison_empty(); test_comparison_self(); + test_comparison_tmp(); + test_comparison_tmp_different(); + test_comparison_only_tmp(); + + test_capacity(); return boost::report_errors(); } diff --git a/test/test_generic_flat_response.cpp b/test/test_generic_flat_response.cpp new file mode 100644 index 000000000..ff9e2b863 --- /dev/null +++ b/test/test_generic_flat_response.cpp @@ -0,0 +1,119 @@ +/* Copyright (c) 2018-2026 Marcelo Zimbres Silva (mzimbres@gmail.com), + * Ruben Perez Hidalgo (rubenperez038@gmail.com) + * + * Distributed under the Boost Software License, Version 1.0. (See + * accompanying file LICENSE.txt) + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include "print_node.hpp" + +using namespace boost::redis; +using boost::system::error_code; +using boost::redis::resp3::detail::deserialize; +using resp3::node_view; +using resp3::type; +using adapter::adapt2; + +// Regular nodes are just stored +void test_success() +{ + generic_flat_response resp; + + error_code ec; + deserialize("+hello\r\n", adapt2(resp), ec); + BOOST_TEST_EQ(ec, error_code{}); + + BOOST_TEST(resp.has_value()); + std::vector expected_nodes{ + {type::simple_string, 1u, 0u, "hello"}, + }; + BOOST_TEST_ALL_EQ( + resp->get_view().begin(), + resp->get_view().end(), + expected_nodes.begin(), + expected_nodes.end()); +} + +// If an error of any kind appears, we set the overall result to error +void test_simple_error() +{ + generic_flat_response resp; + + error_code ec; + deserialize("-Error\r\n", adapt2(resp), ec); + BOOST_TEST_EQ(ec, error_code{}); + + BOOST_TEST(resp.has_error()); + auto const error = resp.error(); + + BOOST_TEST_EQ(error.data_type, resp3::type::simple_error); + BOOST_TEST_EQ(error.diagnostic, "Error"); +} + +void test_blob_error() +{ + generic_flat_response resp; + + error_code ec; + deserialize("!5\r\nError\r\n", adapt2(resp), ec); + BOOST_TEST_EQ(ec, error_code{}); + + BOOST_TEST(resp.has_error()); + auto const error = resp.error(); + + BOOST_TEST_EQ(error.data_type, resp3::type::blob_error); + BOOST_TEST_EQ(error.diagnostic, "Error"); +} + +// Mixing success and error nodes is safe. Only the last error is stored +void test_mix_success_error() +{ + generic_flat_response resp; + error_code ec; + + // Success message + deserialize("+message\r\n", adapt2(resp), ec); + BOOST_TEST_EQ(ec, error_code{}); + + // An error + deserialize("-Error\r\n", adapt2(resp), ec); + BOOST_TEST_EQ(ec, error_code{}); + + // Another success message + deserialize("+other data\r\n", adapt2(resp), ec); + BOOST_TEST_EQ(ec, error_code{}); + + // Another error + deserialize("-Different err\r\n", adapt2(resp), ec); + BOOST_TEST_EQ(ec, error_code{}); + + // Final success message + deserialize("*1\r\n+last message\r\n", adapt2(resp), ec); + BOOST_TEST_EQ(ec, error_code{}); + + // Check + BOOST_TEST(resp.has_error()); + auto const error = resp.error(); + + BOOST_TEST_EQ(error.data_type, resp3::type::simple_error); + BOOST_TEST_EQ(error.diagnostic, "Different err"); +} + +int main() +{ + test_success(); + test_simple_error(); + test_blob_error(); + test_mix_success_error(); + + return boost::report_errors(); +} diff --git a/test/test_low_level_sync_sans_io.cpp b/test/test_low_level_sync_sans_io.cpp index 69c3c221f..9628dfdb5 100644 --- a/test/test_low_level_sync_sans_io.cpp +++ b/test/test_low_level_sync_sans_io.cpp @@ -23,7 +23,6 @@ using boost::redis::request; using boost::redis::adapter::adapt2; using boost::redis::adapter::result; using boost::redis::resp3::tree; -using boost::redis::resp3::flat_tree; using boost::redis::generic_flat_response; using boost::redis::ignore_t; using boost::redis::resp3::detail::deserialize; @@ -253,39 +252,3 @@ BOOST_AUTO_TEST_CASE(check_counter_adapter) BOOST_CHECK_EQUAL(node, 7); BOOST_CHECK_EQUAL(done, 1); } - -BOOST_AUTO_TEST_CASE(generic_flat_response_simple_error) -{ - generic_flat_response resp; - - char const* wire = "-Error\r\n"; - - error_code ec; - deserialize(wire, adapt2(resp), ec); - BOOST_CHECK_EQUAL(ec, error_code{}); - - BOOST_TEST(!resp.has_value()); - BOOST_TEST(resp.has_error()); - auto const error = resp.error(); - - BOOST_CHECK_EQUAL(error.data_type, boost::redis::resp3::type::simple_error); - BOOST_CHECK_EQUAL(error.diagnostic, std::string{"Error"}); -} - -BOOST_AUTO_TEST_CASE(generic_flat_response_blob_error) -{ - generic_flat_response resp; - - char const* wire = "!5\r\nError\r\n"; - - error_code ec; - deserialize(wire, adapt2(resp), ec); - BOOST_CHECK_EQUAL(ec, error_code{}); - - BOOST_TEST(!resp.has_value()); - BOOST_TEST(resp.has_error()); - auto const error = resp.error(); - - BOOST_CHECK_EQUAL(error.data_type, boost::redis::resp3::type::blob_error); - BOOST_CHECK_EQUAL(error.diagnostic, std::string{"Error"}); -} From bea547481aadc5a064c409c87ab5468316acfc62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anarthal=20=28Rub=C3=A9n=20P=C3=A9rez=29?= <34971811+anarthal@users.noreply.github.com> Date: Fri, 9 Jan 2026 21:08:54 +0100 Subject: [PATCH 26/37] Adds support for PubSub state restoration (#375) Adds request::{subscribe, unsubscribe, psubscribe, punsubscribe}. When requests created with these functions are executed successfully, the created subscriptions are tracked and restore on re-connection. close #367 --- README.md | 47 +- doc/modules/ROOT/pages/index.adoc | 47 +- .../ROOT/pages/requests_responses.adoc | 2 +- example/cpp20_chat_room.cpp | 52 +- example/cpp20_subscriber.cpp | 51 +- include/boost/redis/connection.hpp | 7 +- .../boost/redis/detail/connection_state.hpp | 3 + include/boost/redis/detail/exec_fsm.hpp | 13 +- .../redis/detail/subscription_tracker.hpp | 35 ++ include/boost/redis/impl/exec_fsm.ipp | 18 +- include/boost/redis/impl/request.ipp | 27 + include/boost/redis/impl/run_fsm.ipp | 11 +- .../boost/redis/impl/setup_request_utils.hpp | 31 +- .../boost/redis/impl/subscription_tracker.ipp | 44 ++ include/boost/redis/request.hpp | 353 +++++++++++- include/boost/redis/src.hpp | 3 +- test/CMakeLists.txt | 1 + test/Jamfile | 1 + test/test_compose_setup_request.cpp | 292 +++++----- test/test_conn_echo_stress.cpp | 20 +- test/test_conn_push2.cpp | 158 ++++++ test/test_conn_sentinel.cpp | 2 +- test/test_exec_fsm.cpp | 224 +++++--- test/test_request.cpp | 534 +++++++++++++++++- test/test_run_fsm.cpp | 4 +- test/test_setup_adapter.cpp | 24 +- test/test_subscription_tracker.cpp | 276 +++++++++ 27 files changed, 1933 insertions(+), 347 deletions(-) create mode 100644 include/boost/redis/detail/subscription_tracker.hpp create mode 100644 include/boost/redis/impl/subscription_tracker.ipp create mode 100644 test/test_subscription_tracker.cpp diff --git a/README.md b/README.md index af0b575b7..62a038963 100644 --- a/README.md +++ b/README.md @@ -93,32 +93,39 @@ The coroutine below shows how to use it ```cpp -auto -receiver(std::shared_ptr conn) -> net::awaitable +auto receiver(std::shared_ptr conn) -> asio::awaitable { - request req; - req.push("SUBSCRIBE", "channel"); - - flat_tree resp; + generic_flat_response resp; conn->set_receive_response(resp); - // Loop while reconnection is enabled - while (conn->will_reconnect()) { - - // Reconnect to channels. - co_await conn->async_exec(req); + // Subscribe to the channel 'mychannel'. You can add any number of channels here. + request req; + req.subscribe({"mychannel"}); + co_await conn->async_exec(req); + + // You're now subscribed to 'mychannel'. Pushes sent over this channel will be stored + // in resp. If the connection encounters a network error and reconnects to the server, + // it will automatically subscribe to 'mychannel' again. This is transparent to the user. + + // Loop to read Redis push messages. + for (error_code ec;;) { + // Wait for pushes + co_await conn->async_receive2(asio::redirect_error(ec)); + + // Check for errors and cancellations + if (ec && (ec != asio::experimental::error::channel_cancelled || !conn->will_reconnect())) { + std::cerr << "Error during receive2: " << ec << std::endl; + break; + } - // Loop reading Redis pushes. - for (error_code ec;;) { - co_await conn->async_receive2(resp, redirect_error(ec)); - if (ec) - break; // Connection lost, break so we can reconnect to channels. + // The response must be consumed without suspending the + // coroutine i.e. without the use of async operations. + for (auto const& elem : resp.value().get_view()) + std::cout << elem.value << "\n"; - // Use the response resp in some way and then clear it. - ... + std::cout << std::endl; - resp.clear(); - } + resp.value().clear(); } } ``` diff --git a/doc/modules/ROOT/pages/index.adoc b/doc/modules/ROOT/pages/index.adoc index 36b83d493..b7194d1b9 100644 --- a/doc/modules/ROOT/pages/index.adoc +++ b/doc/modules/ROOT/pages/index.adoc @@ -104,32 +104,39 @@ The coroutine below shows how to use it [source,cpp] ---- -auto -receiver(std::shared_ptr conn) -> net::awaitable +auto receiver(std::shared_ptr conn) -> asio::awaitable { - request req; - req.push("SUBSCRIBE", "channel"); - - flat_tree resp; + generic_flat_response resp; conn->set_receive_response(resp); - // Loop while reconnection is enabled - while (conn->will_reconnect()) { - - // Reconnect to channels. - co_await conn->async_exec(req); + // Subscribe to the channel 'mychannel'. You can add any number of channels here. + request req; + req.subscribe({"mychannel"}); + co_await conn->async_exec(req); + + // You're now subscribed to 'mychannel'. Pushes sent over this channel will be stored + // in resp. If the connection encounters a network error and reconnects to the server, + // it will automatically subscribe to 'mychannel' again. This is transparent to the user. + + // Loop to read Redis push messages. + for (error_code ec;;) { + // Wait for pushes + co_await conn->async_receive2(asio::redirect_error(ec)); + + // Check for errors and cancellations + if (ec && (ec != asio::experimental::error::channel_cancelled || !conn->will_reconnect())) { + std::cerr << "Error during receive2: " << ec << std::endl; + break; + } - // Loop reading Redis pushes. - for (error_code ec;;) { - co_await conn->async_receive2(resp, redirect_error(ec)); - if (ec) - break; // Connection lost, break so we can reconnect to channels. + // The response must be consumed without suspending the + // coroutine i.e. without the use of async operations. + for (auto const& elem : resp.value().get_view()) + std::cout << elem.value << "\n"; - // Use the response here and then clear it. - ... + std::cout << std::endl; - resp.clear(); - } + resp.value().clear(); } } ---- diff --git a/doc/modules/ROOT/pages/requests_responses.adoc b/doc/modules/ROOT/pages/requests_responses.adoc index 7f40fed11..69728b63c 100644 --- a/doc/modules/ROOT/pages/requests_responses.adoc +++ b/doc/modules/ROOT/pages/requests_responses.adoc @@ -184,7 +184,7 @@ must **NOT** be included in the response tuple. For example, the following reque ---- request req; req.push("PING"); -req.push("SUBSCRIBE", "channel"); +req.subscribe({"channel"}); req.push("QUIT"); ---- diff --git a/example/cpp20_chat_room.cpp b/example/cpp20_chat_room.cpp index 562225f56..4dd8180d7 100644 --- a/example/cpp20_chat_room.cpp +++ b/example/cpp20_chat_room.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -40,31 +41,44 @@ using namespace std::chrono_literals; // Chat over Redis pubsub. To test, run this program from multiple // terminals and type messages to stdin. +namespace { + +auto rethrow_on_error = [](std::exception_ptr exc) { + if (exc) + std::rethrow_exception(exc); +}; + auto receiver(std::shared_ptr conn) -> awaitable { - request req; - req.push("SUBSCRIBE", "channel"); - + // Set the receive response, so pushes are stored in resp generic_flat_response resp; conn->set_receive_response(resp); - while (conn->will_reconnect()) { - // Subscribe to channels. - co_await conn->async_exec(req); + // Subscribe to the channel 'channel'. Using request::subscribe() + // (instead of request::push()) makes the connection re-subscribe + // to 'channel' whenever it re-connects to the server. + request req; + req.subscribe({"channel"}); + co_await conn->async_exec(req); - // Loop reading Redis push messages. - for (error_code ec;;) { - co_await conn->async_receive2(redirect_error(ec)); - if (ec) - break; // Connection lost, break so we can reconnect to channels. + for (error_code ec;;) { + // Wait for pushes + co_await conn->async_receive2(asio::redirect_error(ec)); - for (auto const& elem: resp.value().get_view()) - std::cout << elem.value << "\n"; + // Check for errors and cancellations + if (ec && (ec != asio::experimental::error::channel_cancelled || !conn->will_reconnect())) { + std::cerr << "Error during receive2: " << ec << std::endl; + break; + } - std::cout << std::endl; + // The response must be consumed without suspending the + // coroutine i.e. without the use of async operations. + for (auto const& elem : resp.value().get_view()) + std::cout << elem.value << "\n"; - resp.value().clear(); - } + std::cout << std::endl; + + resp.value().clear(); } } @@ -81,6 +95,8 @@ auto publisher(std::shared_ptr in, std::shared_ptr awaitable { @@ -88,8 +104,8 @@ auto co_main(config cfg) -> awaitable auto conn = std::make_shared(ex); auto stream = std::make_shared(ex, ::dup(STDIN_FILENO)); - co_spawn(ex, receiver(conn), detached); - co_spawn(ex, publisher(stream, conn), detached); + co_spawn(ex, receiver(conn), rethrow_on_error); + co_spawn(ex, publisher(stream, conn), rethrow_on_error); conn->async_run(cfg, consign(detached, conn)); signal_set sig_set{ex, SIGINT, SIGTERM}; diff --git a/example/cpp20_subscriber.cpp b/example/cpp20_subscriber.cpp index 1f0dd86fe..b3eb9cdd4 100644 --- a/example/cpp20_subscriber.cpp +++ b/example/cpp20_subscriber.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -32,7 +33,7 @@ using asio::signal_set; * To test send messages with redis-cli * * $ redis-cli -3 - * 127.0.0.1:6379> PUBLISH channel some-message + * 127.0.0.1:6379> PUBLISH mychannel some-message * (integer) 3 * 127.0.0.1:6379> * @@ -46,33 +47,39 @@ using asio::signal_set; // Receives server pushes. auto receiver(std::shared_ptr conn) -> asio::awaitable { - request req; - req.push("SUBSCRIBE", "channel"); - generic_flat_response resp; conn->set_receive_response(resp); - // Loop while reconnection is enabled - while (conn->will_reconnect()) { - // Reconnect to the channels. - co_await conn->async_exec(req); - - // Loop to read Redis push messages. - for (error_code ec;;) { - // Wait for pushes - co_await conn->async_receive2(asio::redirect_error(ec)); - if (ec) - break; // Connection lost, break so we can reconnect to channels. + // Subscribe to the channel 'mychannel'. You can add any number of channels here. + request req; + req.subscribe({"mychannel"}); + co_await conn->async_exec(req); + + // You're now subscribed to 'mychannel'. Pushes sent over this channel will be stored + // in resp. If the connection encounters a network error and reconnects to the server, + // it will automatically subscribe to 'mychannel' again. This is transparent to the user. + // You need to use specialized request::subscribe() function (instead of request::push) + // to enable this behavior. + + // Loop to read Redis push messages. + for (error_code ec;;) { + // Wait for pushes + co_await conn->async_receive2(asio::redirect_error(ec)); + + // Check for errors and cancellations + if (ec && (ec != asio::experimental::error::channel_cancelled || !conn->will_reconnect())) { + std::cerr << "Error during receive2: " << ec << std::endl; + break; + } - // The response must be consumed without suspending the - // coroutine i.e. without the use of async operations. - for (auto const& elem: resp.value().get_view()) - std::cout << elem.value << "\n"; + // The response must be consumed without suspending the + // coroutine i.e. without the use of async operations. + for (auto const& elem : resp.value().get_view()) + std::cout << elem.value << "\n"; - std::cout << std::endl; + std::cout << std::endl; - resp.value().clear(); - } + resp.value().clear(); } } diff --git a/include/boost/redis/connection.hpp b/include/boost/redis/connection.hpp index 3e8298f39..1fcf2d6b1 100644 --- a/include/boost/redis/connection.hpp +++ b/include/boost/redis/connection.hpp @@ -108,7 +108,10 @@ struct connection_impl { { while (true) { // Invoke the state machine - auto act = fsm_.resume(obj_->is_open(), self.get_cancellation_state().cancelled()); + auto act = fsm_.resume( + obj_->is_open(), + obj_->st_, + self.get_cancellation_state().cancelled()); // Do what the FSM said switch (act.type()) { @@ -203,7 +206,7 @@ struct connection_impl { }); return asio::async_compose( - exec_op{this, notifier, exec_fsm(st_.mpx, std::move(info))}, + exec_op{this, notifier, exec_fsm(std::move(info))}, token, writer_cv_); } diff --git a/include/boost/redis/detail/connection_state.hpp b/include/boost/redis/detail/connection_state.hpp index b4d2e1a01..75ddf1772 100644 --- a/include/boost/redis/detail/connection_state.hpp +++ b/include/boost/redis/detail/connection_state.hpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -47,7 +48,9 @@ struct connection_state { config cfg{}; multiplexer mpx{}; std::string diagnostic{}; // Used by the setup request and Sentinel + request setup_req{}; request ping_req{}; + subscription_tracker tracker{}; // Sentinel stuff lazy_random_engine eng{}; diff --git a/include/boost/redis/detail/exec_fsm.hpp b/include/boost/redis/detail/exec_fsm.hpp index 3727d07c3..7dc2f8c12 100644 --- a/include/boost/redis/detail/exec_fsm.hpp +++ b/include/boost/redis/detail/exec_fsm.hpp @@ -21,6 +21,8 @@ namespace boost::redis::detail { +struct connection_state; + // What should we do next? enum class exec_action_type { @@ -54,16 +56,17 @@ class exec_action { class exec_fsm { int resume_point_{0}; - multiplexer* mpx_{nullptr}; std::shared_ptr elem_; public: - exec_fsm(multiplexer& mpx, std::shared_ptr elem) noexcept - : mpx_(&mpx) - , elem_(std::move(elem)) + exec_fsm(std::shared_ptr elem) noexcept + : elem_(std::move(elem)) { } - exec_action resume(bool connection_is_open, asio::cancellation_type_t cancel_state); + exec_action resume( + bool connection_is_open, + connection_state& st, + asio::cancellation_type_t cancel_state); }; } // namespace boost::redis::detail diff --git a/include/boost/redis/detail/subscription_tracker.hpp b/include/boost/redis/detail/subscription_tracker.hpp new file mode 100644 index 000000000..865d4113e --- /dev/null +++ b/include/boost/redis/detail/subscription_tracker.hpp @@ -0,0 +1,35 @@ +// +// Copyright (c) 2025 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#ifndef BOOST_REDIS_SUBSCRIPTION_TRACKER_HPP +#define BOOST_REDIS_SUBSCRIPTION_TRACKER_HPP + +#include +#include + +namespace boost::redis { + +class request; + +namespace detail { + +class subscription_tracker { + std::set channels_; + std::set pchannels_; + +public: + subscription_tracker() = default; + void clear(); + void commit_changes(const request& req); + void compose_subscribe_request(request& to) const; +}; + +} // namespace detail +} // namespace boost::redis + +#endif diff --git a/include/boost/redis/impl/exec_fsm.ipp b/include/boost/redis/impl/exec_fsm.ipp index b7be8c608..3898d1e18 100644 --- a/include/boost/redis/impl/exec_fsm.ipp +++ b/include/boost/redis/impl/exec_fsm.ipp @@ -9,6 +9,7 @@ #ifndef BOOST_REDIS_EXEC_FSM_IPP #define BOOST_REDIS_EXEC_FSM_IPP +#include #include #include #include @@ -28,7 +29,10 @@ inline bool is_total_cancel(asio::cancellation_type_t type) return !!(type & asio::cancellation_type_t::total); } -exec_action exec_fsm::resume(bool connection_is_open, asio::cancellation_type_t cancel_state) +exec_action exec_fsm::resume( + bool connection_is_open, + connection_state& st, + asio::cancellation_type_t cancel_state) { switch (resume_point_) { BOOST_REDIS_CORO_INITIAL @@ -47,7 +51,7 @@ exec_action exec_fsm::resume(bool connection_is_open, asio::cancellation_type_t BOOST_REDIS_YIELD(resume_point_, 2, exec_action_type::setup_cancellation) // Add the request to the multiplexer - mpx_->add(elem_); + st.mpx.add(elem_); // Notify the writer task that there is work to do. If the task is not // listening (e.g. it's already writing or the connection is not healthy), @@ -61,8 +65,14 @@ exec_action exec_fsm::resume(bool connection_is_open, asio::cancellation_type_t // If the request has completed (with error or not), we're done if (elem_->is_done()) { + // If the request completed successfully and we were configured to do so, + // record the changes applied to the pubsub state + if (!elem_->get_error()) + st.tracker.commit_changes(elem_->get_request()); + + // Deallocate memory before finalizing exec_action act{elem_->get_error(), elem_->get_read_size()}; - elem_.reset(); // Deallocate memory before finalizing + elem_.reset(); return act; } @@ -71,7 +81,7 @@ exec_action exec_fsm::resume(bool connection_is_open, asio::cancellation_type_t if ( (is_total_cancel(cancel_state) && elem_->is_waiting()) || is_partial_or_terminal_cancel(cancel_state)) { - mpx_->cancel(elem_); + st.mpx.cancel(elem_); elem_.reset(); // Deallocate memory before finalizing return exec_action{asio::error::operation_aborted}; } diff --git a/include/boost/redis/impl/request.ipp b/include/boost/redis/impl/request.ipp index 741c3b5d3..72653897b 100644 --- a/include/boost/redis/impl/request.ipp +++ b/include/boost/redis/impl/request.ipp @@ -5,7 +5,9 @@ */ #include +#include +#include #include namespace boost::redis::detail { @@ -34,7 +36,32 @@ request make_hello_request() void boost::redis::request::append(const request& other) { + // Remember the old payload size, to update offsets + std::size_t old_offset = payload_.size(); + + // Add the payload payload_ += other.payload_; commands_ += other.commands_; expected_responses_ += other.expected_responses_; + + // Add the pubsub changes. Offsets need to be updated + pubsub_changes_.reserve(pubsub_changes_.size() + other.pubsub_changes_.size()); + for (const auto& change : other.pubsub_changes_) { + pubsub_changes_.push_back({ + change.type, + change.channel_offset + old_offset, + change.channel_size, + }); + } +} + +void boost::redis::request::add_pubsub_arg(detail::pubsub_change_type type, std::string_view value) +{ + // Add the argument + resp3::add_bulk(payload_, value); + + // Track the change. + // The final \r\n adds 2 bytes + std::size_t offset = payload_.size() - value.size() - 2u; + pubsub_changes_.push_back({type, offset, value.size()}); } diff --git a/include/boost/redis/impl/run_fsm.ipp b/include/boost/redis/impl/run_fsm.ipp index 845f9d202..d477d549d 100644 --- a/include/boost/redis/impl/run_fsm.ipp +++ b/include/boost/redis/impl/run_fsm.ipp @@ -101,10 +101,10 @@ run_action run_fsm::resume( return stored_ec_; } - // Compose the setup request. This only depends on the config, so it can be done just once - compose_setup_request(st.cfg); + // Clear any remainder from previous runs + st.tracker.clear(); - // Compose the PING request. Same as above + // Compose the PING request. This only depends on the config, so it can be done just once compose_ping_request(st.cfg, st.ping_req); if (use_sentinel(st.cfg)) { @@ -159,10 +159,11 @@ run_action run_fsm::resume( // Initialization st.mpx.reset(); st.diagnostic.clear(); + compose_setup_request(st.cfg, st.tracker, st.setup_req); // Add the setup request to the multiplexer - if (st.cfg.setup.get_commands() != 0u) { - auto elm = make_elem(st.cfg.setup, make_any_adapter_impl(setup_adapter{st})); + if (st.setup_req.get_commands() != 0u) { + auto elm = make_elem(st.setup_req, make_any_adapter_impl(setup_adapter{st})); elm->set_done_callback([&elem_ref = *elm, &st] { on_setup_done(elem_ref, st); }); diff --git a/include/boost/redis/impl/setup_request_utils.hpp b/include/boost/redis/impl/setup_request_utils.hpp index c220d98e9..2fb0c6033 100644 --- a/include/boost/redis/impl/setup_request_utils.hpp +++ b/include/boost/redis/impl/setup_request_utils.hpp @@ -9,6 +9,7 @@ #include #include +#include #include #include // use_sentinel #include @@ -22,14 +23,25 @@ namespace boost::redis::detail { // Modifies config::setup to make a request suitable to be sent // to the server using async_exec -inline void compose_setup_request(config& cfg) +inline void compose_setup_request( + const config& cfg, + const subscription_tracker& pubsub_st, + request& req) { - auto& req = cfg.setup; + // Clear any previous contents + req.clear(); - if (!cfg.use_setup) { + // Set the appropriate flags + request_access::set_priority(req, true); + req.get_config().cancel_if_unresponded = true; + req.get_config().cancel_on_connection_lost = true; + + if (cfg.use_setup) { + // We should use the provided request as-is + req.append(cfg.setup); + } else { // We're not using the setup request as-is, but should compose one based on // the values passed by the user - req.clear(); // Which parts of the command should we send? // Don't send AUTH if the user is the default and the password is empty. @@ -59,12 +71,8 @@ inline void compose_setup_request(config& cfg) if (use_sentinel(cfg)) req.push("ROLE"); - // In any case, the setup request should have the priority - // flag set so it's executed before any other request. - // The setup request should never be retried. - request_access::set_priority(req, true); - req.get_config().cancel_if_unresponded = true; - req.get_config().cancel_on_connection_lost = true; + // Add any subscription commands require to restore the PubSub state + pubsub_st.compose_subscribe_request(req); } class setup_adapter { @@ -83,7 +91,8 @@ class setup_adapter { // When using Sentinel, we add a ROLE command at the end. // We need to ensure that this instance is a master. - if (use_sentinel(st_->cfg) && response_idx_ == st_->cfg.setup.get_expected_responses() - 1u) { + // ROLE may be followed by subscribe requests, but these don't expect any response. + if (use_sentinel(st_->cfg) && response_idx_ == st_->setup_req.get_expected_responses() - 1u) { // ROLE's response should be an array of at least 1 element if (nd.depth == 0u) { if (nd.data_type != resp3::type::array) diff --git a/include/boost/redis/impl/subscription_tracker.ipp b/include/boost/redis/impl/subscription_tracker.ipp new file mode 100644 index 000000000..ec23675fb --- /dev/null +++ b/include/boost/redis/impl/subscription_tracker.ipp @@ -0,0 +1,44 @@ +// +// Copyright (c) 2025 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#include +#include + +#include + +#include + +namespace boost::redis::detail { + +void subscription_tracker::clear() +{ + channels_.clear(); + pchannels_.clear(); +} + +void subscription_tracker::commit_changes(const request& req) +{ + for (const auto& ch : request_access::pubsub_changes(req)) { + std::string channel{req.payload().substr(ch.channel_offset, ch.channel_size)}; + switch (ch.type) { + case pubsub_change_type::subscribe: channels_.insert(std::move(channel)); break; + case pubsub_change_type::unsubscribe: channels_.erase(std::move(channel)); break; + case pubsub_change_type::psubscribe: pchannels_.insert(std::move(channel)); break; + case pubsub_change_type::punsubscribe: pchannels_.erase(std::move(channel)); break; + default: BOOST_ASSERT(false); + } + } +} + +void subscription_tracker::compose_subscribe_request(request& to) const +{ + to.push_range("SUBSCRIBE", channels_); + to.push_range("PSUBSCRIBE", pchannels_); +} + +} // namespace boost::redis::detail diff --git a/include/boost/redis/request.hpp b/include/boost/redis/request.hpp index fc3160c06..d3104a785 100644 --- a/include/boost/redis/request.hpp +++ b/include/boost/redis/request.hpp @@ -10,8 +10,12 @@ #include #include +#include #include +#include #include +#include +#include // NOTE: For some commands like hset it would be a good idea to assert // the value type is a pair. @@ -21,6 +25,21 @@ namespace boost::redis { namespace detail { auto has_response(std::string_view cmd) -> bool; struct request_access; + +enum class pubsub_change_type +{ + subscribe, + unsubscribe, + psubscribe, + punsubscribe, +}; + +struct pubsub_change { + pubsub_change_type type; + std::size_t channel_offset; + std::size_t channel_size; +}; + } // namespace detail /** @brief Represents a Redis request. @@ -123,6 +142,7 @@ class request { void clear() { payload_.clear(); + pubsub_changes_.clear(); commands_ = 0; expected_responses_ = 0; has_hello_priority_ = false; @@ -257,17 +277,17 @@ class request { * of arguments and don't have a key. For example: * * @code - * std::set channels - * { "channel1" , "channel2" , "channel3" }; + * std::set keys + * { "key1" , "key2" , "key3" }; * * request req; - * req.push("SUBSCRIBE", channels.cbegin(), channels.cend()); + * req.push("MGET", keys.begin(), keys.end()); * @endcode * * This will generate the following command: * * @code - * SUBSCRIBE channel1 channel2 channel3 + * MGET key1 key2 key3 * @endcode * * *If the passed range is empty, no command is added* and this @@ -412,6 +432,298 @@ class request { */ void append(const request& other); + /** + * @brief Appends a SUBSCRIBE command to the end of the request. + * + * If `channels` contains `{"ch1", "ch2"}`, the resulting command + * is `SUBSCRIBE ch1 ch2`. + * + * Subscriptions created using this function are tracked + * to enable PubSub state restoration. After successfully executing + * the request, the connection will store any newly subscribed channels and patterns. + * Every time a reconnection happens, + * a suitable `SUBSCRIBE`/`PSUBSCRIBE` command is issued automatically, + * to restore the subscriptions that were active before the reconnection. + * + * PubSub store restoration only happens when using @ref subscribe, + * @ref unsubscribe, @ref psubscribe or @ref punsubscribe. + * Subscription commands added by @ref push or @ref push_range are not tracked. + */ + void subscribe(std::initializer_list channels) + { + subscribe(channels.begin(), channels.end()); + } + + /** + * @brief Appends a SUBSCRIBE command to the end of the request. + * + * If `channels` contains `["ch1", "ch2"]`, the resulting command + * is `SUBSCRIBE ch1 ch2`. + * + * Subscriptions created using this function are tracked + * to enable PubSub state restoration. After successfully executing + * the request, the connection will store any newly subscribed channels and patterns. + * Every time a reconnection happens, + * a suitable `SUBSCRIBE`/`PSUBSCRIBE` command is issued automatically, + * to restore the subscriptions that were active before the reconnection. + * + * PubSub store restoration only happens when using @ref subscribe, + * @ref unsubscribe, @ref psubscribe or @ref punsubscribe. + * Subscription commands added by @ref push or @ref push_range are not tracked. + */ + template + void subscribe(Range&& channels, decltype(std::cbegin(channels))* = nullptr) + { + subscribe(std::cbegin(channels), std::cend(channels)); + } + + /** + * @brief Appends a SUBSCRIBE command to the end of the request. + * + * [`channels_begin`, `channels_end`) should point to a valid + * range of elements convertible to `std::string_view`. + * If the range contains `["ch1", "ch2"]`, the resulting command + * is `SUBSCRIBE ch1 ch2`. + * + * Subscriptions created using this function are tracked + * to enable PubSub state restoration. After successfully executing + * the request, the connection will store any newly subscribed channels and patterns. + * Every time a reconnection happens, + * a suitable `SUBSCRIBE`/`PSUBSCRIBE` command is issued automatically, + * to restore the subscriptions that were active before the reconnection. + * + * PubSub store restoration only happens when using @ref subscribe, + * @ref unsubscribe, @ref psubscribe or @ref punsubscribe. + * Subscription commands added by @ref push or @ref push_range are not tracked. + */ + template + void subscribe(ForwardIt channels_begin, ForwardIt channels_end) + { + push_pubsub("SUBSCRIBE", detail::pubsub_change_type::subscribe, channels_begin, channels_end); + } + + /** + * @brief Appends an UNSUBSCRIBE command to the end of the request. + * + * If `channels` contains `{"ch1", "ch2"}`, the resulting command + * is `UNSUBSCRIBE ch1 ch2`. + * + * Subscriptions removed using this function are tracked + * to enable PubSub state restoration. After successfully executing + * the request, the connection will store any newly subscribed channels and patterns. + * Every time a reconnection happens, + * a suitable `SUBSCRIBE`/`PSUBSCRIBE` command is issued automatically, + * to restore the subscriptions that were active before the reconnection. + * + * PubSub store restoration only happens when using @ref subscribe, + * @ref unsubscribe, @ref psubscribe or @ref punsubscribe. + * Subscription commands added by @ref push or @ref push_range are not tracked. + */ + void unsubscribe(std::initializer_list channels) + { + unsubscribe(channels.begin(), channels.end()); + } + + /** + * @brief Appends an UNSUBSCRIBE command to the end of the request. + * + * If `channels` contains `["ch1", "ch2"]`, the resulting command + * is `UNSUBSCRIBE ch1 ch2`. + * + * Subscriptions removed using this function are tracked + * to enable PubSub state restoration. After successfully executing + * the request, the connection will store any newly subscribed channels and patterns. + * Every time a reconnection happens, + * a suitable `SUBSCRIBE`/`PSUBSCRIBE` command is issued automatically, + * to restore the subscriptions that were active before the reconnection. + * + * PubSub store restoration only happens when using @ref subscribe, + * @ref unsubscribe, @ref psubscribe or @ref punsubscribe. + * Subscription commands added by @ref push or @ref push_range are not tracked. + */ + template + void unsubscribe(Range&& channels, decltype(std::cbegin(channels))* = nullptr) + { + unsubscribe(std::cbegin(channels), std::cend(channels)); + } + + /** + * @brief Appends an UNSUBSCRIBE command to the end of the request. + * + * [`channels_begin`, `channels_end`) should point to a valid + * range of elements convertible to `std::string_view`. + * If the range contains `["ch1", "ch2"]`, the resulting command + * is `UNSUBSCRIBE ch1 ch2`. + * + * Subscriptions removed using this function are tracked + * to enable PubSub state restoration. After successfully executing + * the request, the connection will store any newly subscribed channels and patterns. + * Every time a reconnection happens, + * a suitable `SUBSCRIBE`/`PSUBSCRIBE` command is issued automatically, + * to restore the subscriptions that were active before the reconnection. + * + * PubSub store restoration only happens when using @ref subscribe, + * @ref unsubscribe, @ref psubscribe or @ref punsubscribe. + * Subscription commands added by @ref push or @ref push_range are not tracked. + */ + template + void unsubscribe(ForwardIt channels_begin, ForwardIt channels_end) + { + push_pubsub( + "UNSUBSCRIBE", + detail::pubsub_change_type::unsubscribe, + channels_begin, + channels_end); + } + + /** + * @brief Appends a PSUBSCRIBE command to the end of the request. + * + * If `patterns` contains `{"news.*", "events.*"}`, the resulting command + * is `PSUBSCRIBE news.* events.*`. + * + * Subscriptions created using this function are tracked + * to enable PubSub state restoration. After successfully executing + * the request, the connection will store any newly subscribed channels and patterns. + * Every time a reconnection happens, + * a suitable `SUBSCRIBE`/`PSUBSCRIBE` command is issued automatically, + * to restore the subscriptions that were active before the reconnection. + * + * PubSub store restoration only happens when using @ref subscribe, + * @ref unsubscribe, @ref psubscribe or @ref punsubscribe. + * Subscription commands added by @ref push or @ref push_range are not tracked. + */ + void psubscribe(std::initializer_list patterns) + { + psubscribe(patterns.begin(), patterns.end()); + } + + /** + * @brief Appends a PSUBSCRIBE command to the end of the request. + * + * If `patterns` contains `["news.*", "events.*"]`, the resulting command + * is `PSUBSCRIBE news.* events.*`. + * + * Subscriptions created using this function are tracked + * to enable PubSub state restoration. After successfully executing + * the request, the connection will store any newly subscribed channels and patterns. + * Every time a reconnection happens, + * a suitable `SUBSCRIBE`/`PSUBSCRIBE` command is issued automatically, + * to restore the subscriptions that were active before the reconnection. + * + * PubSub store restoration only happens when using @ref subscribe, + * @ref unsubscribe, @ref psubscribe or @ref punsubscribe. + * Subscription commands added by @ref push or @ref push_range are not tracked. + */ + template + void psubscribe(Range&& patterns, decltype(std::cbegin(patterns))* = nullptr) + { + psubscribe(std::cbegin(patterns), std::cend(patterns)); + } + + /** + * @brief Appends a PSUBSCRIBE command to the end of the request. + * + * [`patterns_begin`, `patterns_end`) should point to a valid + * range of elements convertible to `std::string_view`. + * If the range contains `["news.*", "events.*"]`, the resulting command + * is `PSUBSCRIBE news.* events.*`. + * + * Subscriptions created using this function are tracked + * to enable PubSub state restoration. After successfully executing + * the request, the connection will store any newly subscribed channels and patterns. + * Every time a reconnection happens, + * a suitable `SUBSCRIBE`/`PSUBSCRIBE` command is issued automatically, + * to restore the subscriptions that were active before the reconnection. + * + * PubSub store restoration only happens when using @ref subscribe, + * @ref unsubscribe, @ref psubscribe or @ref punsubscribe. + * Subscription commands added by @ref push or @ref push_range are not tracked. + */ + template + void psubscribe(ForwardIt patterns_begin, ForwardIt patterns_end) + { + push_pubsub( + "PSUBSCRIBE", + detail::pubsub_change_type::psubscribe, + patterns_begin, + patterns_end); + } + + /** + * @brief Appends a PUNSUBSCRIBE command to the end of the request. + * + * If `patterns` contains `{"news.*", "events.*"}`, the resulting command + * is `PUNSUBSCRIBE news.* events.*`. + * + * Subscriptions removed using this function are tracked + * to enable PubSub state restoration. After successfully executing + * the request, the connection will store any newly subscribed channels and patterns. + * Every time a reconnection happens, + * a suitable `SUBSCRIBE`/`PSUBSCRIBE` command is issued automatically, + * to restore the subscriptions that were active before the reconnection. + * + * PubSub store restoration only happens when using @ref subscribe, + * @ref unsubscribe, @ref psubscribe or @ref punsubscribe. + * Subscription commands added by @ref push or @ref push_range are not tracked. + */ + void punsubscribe(std::initializer_list patterns) + { + punsubscribe(patterns.begin(), patterns.end()); + } + + /** + * @brief Appends a PUNSUBSCRIBE command to the end of the request. + * + * If `patterns` contains `["news.*", "events.*"]`, the resulting command + * is `PUNSUBSCRIBE news.* events.*`. + * + * Subscriptions removed using this function are tracked + * to enable PubSub state restoration. After successfully executing + * the request, the connection will store any newly subscribed channels and patterns. + * Every time a reconnection happens, + * a suitable `SUBSCRIBE`/`PSUBSCRIBE` command is issued automatically, + * to restore the subscriptions that were active before the reconnection. + * + * PubSub store restoration only happens when using @ref subscribe, + * @ref unsubscribe, @ref psubscribe or @ref punsubscribe. + * Subscription commands added by @ref push or @ref push_range are not tracked. + */ + template + void punsubscribe(Range&& patterns, decltype(std::cbegin(patterns))* = nullptr) + { + punsubscribe(std::cbegin(patterns), std::cend(patterns)); + } + + /** + * @brief Appends a PUNSUBSCRIBE command to the end of the request. + * + * [`patterns_begin`, `patterns_end`) should point to a valid + * range of elements convertible to `std::string_view`. + * If the range contains `["news.*", "events.*"]`, the resulting command + * is `PUNSUBSCRIBE news.* events.*`. + * + * Subscriptions removed using this function are tracked + * to enable PubSub state restoration. After successfully executing + * the request, the connection will store any newly subscribed channels and patterns. + * Every time a reconnection happens, + * a suitable `SUBSCRIBE`/`PSUBSCRIBE` command is issued automatically, + * to restore the subscriptions that were active before the reconnection. + * + * PubSub store restoration only happens when using @ref subscribe, + * @ref unsubscribe, @ref psubscribe or @ref punsubscribe. + * Subscription commands added by @ref push or @ref push_range are not tracked. + */ + template + void punsubscribe(ForwardIt patterns_begin, ForwardIt patterns_end) + { + push_pubsub( + "PUNSUBSCRIBE", + detail::pubsub_change_type::punsubscribe, + patterns_begin, + patterns_end); + } + private: void check_cmd(std::string_view cmd) { @@ -429,6 +741,35 @@ class request { std::size_t commands_ = 0; std::size_t expected_responses_ = 0; bool has_hello_priority_ = false; + std::vector pubsub_changes_{}; + + void add_pubsub_arg(detail::pubsub_change_type type, std::string_view value); + + template + void push_pubsub( + std::string_view cmd, + detail::pubsub_change_type type, + ForwardIt channels_begin, + ForwardIt channels_end) + { + static_assert( + std::is_convertible_v< + typename std::iterator_traits::value_type, + std::string_view>, + "subscribe, psubscribe, unsubscribe and punsubscribe should be passed ranges of elements " + "convertible to std::string_view"); + if (channels_begin == channels_end) + return; + + auto const distance = std::distance(channels_begin, channels_end); + resp3::add_header(payload_, resp3::type::array, 1 + distance); + resp3::add_bulk(payload_, cmd); + + for (; channels_begin != channels_end; ++channels_begin) + add_pubsub_arg(type, *channels_begin); + + ++commands_; // these commands don't have a response + } friend struct detail::request_access; }; @@ -438,6 +779,10 @@ namespace detail { struct request_access { inline static void set_priority(request& r, bool value) { r.has_hello_priority_ = value; } inline static bool has_priority(const request& r) { return r.has_hello_priority_; } + inline static const std::vector& pubsub_changes(const request& r) + { + return r.pubsub_changes_; + } }; // Creates a HELLO 3 request diff --git a/include/boost/redis/src.hpp b/include/boost/redis/src.hpp index eb48981b9..647e643de 100644 --- a/include/boost/redis/src.hpp +++ b/include/boost/redis/src.hpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -18,8 +19,8 @@ #include #include #include +#include #include -#include #include #include #include diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 40a1ea05d..9c6a97e0e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -55,6 +55,7 @@ make_test(test_update_sentinel_list) make_test(test_flat_tree) make_test(test_generic_flat_response) make_test(test_read_buffer) +make_test(test_subscription_tracker) # Tests that require a real Redis server make_test(test_conn_quit) diff --git a/test/Jamfile b/test/Jamfile index 3aa72460f..ccd47060c 100644 --- a/test/Jamfile +++ b/test/Jamfile @@ -72,6 +72,7 @@ local tests = test_flat_tree test_generic_flat_response test_read_buffer + test_subscription_tracker ; # Build and run the tests diff --git a/test/test_compose_setup_request.cpp b/test/test_compose_setup_request.cpp index 2f91a9470..566799502 100644 --- a/test/test_compose_setup_request.cpp +++ b/test/test_compose_setup_request.cpp @@ -6,220 +6,231 @@ // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // -#include #include +#include #include #include #include #include -#include #include +#include #include -#include +#include +#include + +using namespace boost::redis; namespace asio = boost::asio; -namespace redis = boost::redis; -using redis::detail::compose_setup_request; +using detail::compose_setup_request; +using detail::subscription_tracker; using boost::system::error_code; namespace { +struct fixture { + subscription_tracker tracker; + request out; + config cfg; + + void run(std::string_view expected_payload, boost::source_location loc = BOOST_CURRENT_LOCATION) + { + out.push("PING", "leftover"); // verify that we clear the request + + compose_setup_request(cfg, tracker, out); + + if (!BOOST_TEST_EQ(out.payload(), expected_payload)) + std::cerr << "Called from " << loc << std::endl; + + if (!BOOST_TEST(out.has_hello_priority())) + std::cerr << "Called from " << loc << std::endl; + + if (!BOOST_TEST(out.get_config().cancel_if_unresponded)) + std::cerr << "Called from " << loc << std::endl; + + if (!BOOST_TEST(out.get_config().cancel_on_connection_lost)) + std::cerr << "Called from " << loc << std::endl; + } +}; + void test_hello() { - redis::config cfg; - cfg.clientname = ""; - - compose_setup_request(cfg); + fixture fix; + fix.cfg.clientname = ""; - std::string_view const expected = "*2\r\n$5\r\nHELLO\r\n$1\r\n3\r\n"; - BOOST_TEST_EQ(cfg.setup.payload(), expected); - BOOST_TEST(cfg.setup.has_hello_priority()); - BOOST_TEST(cfg.setup.get_config().cancel_if_unresponded); - BOOST_TEST(cfg.setup.get_config().cancel_on_connection_lost); + fix.run("*2\r\n$5\r\nHELLO\r\n$1\r\n3\r\n"); } void test_select() { - redis::config cfg; - cfg.clientname = ""; - cfg.database_index = 10; + fixture fix; + fix.cfg.clientname = ""; + fix.cfg.database_index = 10; - compose_setup_request(cfg); - - std::string_view const expected = + fix.run( "*2\r\n$5\r\nHELLO\r\n$1\r\n3\r\n" - "*2\r\n$6\r\nSELECT\r\n$2\r\n10\r\n"; - BOOST_TEST_EQ(cfg.setup.payload(), expected); - BOOST_TEST(cfg.setup.has_hello_priority()); - BOOST_TEST(cfg.setup.get_config().cancel_if_unresponded); - BOOST_TEST(cfg.setup.get_config().cancel_on_connection_lost); + "*2\r\n$6\r\nSELECT\r\n$2\r\n10\r\n"); } void test_clientname() { - redis::config cfg; - - compose_setup_request(cfg); + fixture fix; - std::string_view const - expected = "*4\r\n$5\r\nHELLO\r\n$1\r\n3\r\n$7\r\nSETNAME\r\n$11\r\nBoost.Redis\r\n"; - BOOST_TEST_EQ(cfg.setup.payload(), expected); - BOOST_TEST(cfg.setup.has_hello_priority()); - BOOST_TEST(cfg.setup.get_config().cancel_if_unresponded); - BOOST_TEST(cfg.setup.get_config().cancel_on_connection_lost); + fix.run("*4\r\n$5\r\nHELLO\r\n$1\r\n3\r\n$7\r\nSETNAME\r\n$11\r\nBoost.Redis\r\n"); } void test_auth() { - redis::config cfg; - cfg.clientname = ""; - cfg.username = "foo"; - cfg.password = "bar"; - - compose_setup_request(cfg); - - std::string_view const - expected = "*5\r\n$5\r\nHELLO\r\n$1\r\n3\r\n$4\r\nAUTH\r\n$3\r\nfoo\r\n$3\r\nbar\r\n"; - BOOST_TEST_EQ(cfg.setup.payload(), expected); - BOOST_TEST(cfg.setup.has_hello_priority()); - BOOST_TEST(cfg.setup.get_config().cancel_if_unresponded); - BOOST_TEST(cfg.setup.get_config().cancel_on_connection_lost); + fixture fix; + fix.cfg.clientname = ""; + fix.cfg.username = "foo"; + fix.cfg.password = "bar"; + + fix.run("*5\r\n$5\r\nHELLO\r\n$1\r\n3\r\n$4\r\nAUTH\r\n$3\r\nfoo\r\n$3\r\nbar\r\n"); } void test_auth_empty_password() { - redis::config cfg; - cfg.clientname = ""; - cfg.username = "foo"; - - compose_setup_request(cfg); - - std::string_view const - expected = "*5\r\n$5\r\nHELLO\r\n$1\r\n3\r\n$4\r\nAUTH\r\n$3\r\nfoo\r\n$0\r\n\r\n"; - BOOST_TEST_EQ(cfg.setup.payload(), expected); - BOOST_TEST(cfg.setup.has_hello_priority()); - BOOST_TEST(cfg.setup.get_config().cancel_if_unresponded); - BOOST_TEST(cfg.setup.get_config().cancel_on_connection_lost); + fixture fix; + fix.cfg.clientname = ""; + fix.cfg.username = "foo"; + + fix.run("*5\r\n$5\r\nHELLO\r\n$1\r\n3\r\n$4\r\nAUTH\r\n$3\r\nfoo\r\n$0\r\n\r\n"); } void test_auth_setname() { - redis::config cfg; - cfg.clientname = "mytest"; - cfg.username = "foo"; - cfg.password = "bar"; - - compose_setup_request(cfg); + fixture fix; + fix.cfg.clientname = "mytest"; + fix.cfg.username = "foo"; + fix.cfg.password = "bar"; - std::string_view const expected = + fix.run( "*7\r\n$5\r\nHELLO\r\n$1\r\n3\r\n$4\r\nAUTH\r\n$3\r\nfoo\r\n$3\r\nbar\r\n$7\r\nSETNAME\r\n$" - "6\r\nmytest\r\n"; - BOOST_TEST_EQ(cfg.setup.payload(), expected); - BOOST_TEST(cfg.setup.has_hello_priority()); - BOOST_TEST(cfg.setup.get_config().cancel_if_unresponded); - BOOST_TEST(cfg.setup.get_config().cancel_on_connection_lost); + "6\r\nmytest\r\n"); } void test_use_setup() { - redis::config cfg; - cfg.clientname = "mytest"; - cfg.username = "foo"; - cfg.password = "bar"; - cfg.database_index = 4; - cfg.use_setup = true; - cfg.setup.push("SELECT", 8); - - compose_setup_request(cfg); - - std::string_view const expected = + fixture fix; + fix.cfg.clientname = "mytest"; + fix.cfg.username = "foo"; + fix.cfg.password = "bar"; + fix.cfg.database_index = 4; + fix.cfg.use_setup = true; + fix.cfg.setup.push("SELECT", 8); + + fix.run( "*2\r\n$5\r\nHELLO\r\n$1\r\n3\r\n" - "*2\r\n$6\r\nSELECT\r\n$1\r\n8\r\n"; - BOOST_TEST_EQ(cfg.setup.payload(), expected); - BOOST_TEST(cfg.setup.has_hello_priority()); - BOOST_TEST(cfg.setup.get_config().cancel_if_unresponded); - BOOST_TEST(cfg.setup.get_config().cancel_on_connection_lost); + "*2\r\n$6\r\nSELECT\r\n$1\r\n8\r\n"); } // Regression check: we set the priority flag void test_use_setup_no_hello() { - redis::config cfg; - cfg.use_setup = true; - cfg.setup.clear(); - cfg.setup.push("SELECT", 8); - - compose_setup_request(cfg); - - std::string_view const expected = "*2\r\n$6\r\nSELECT\r\n$1\r\n8\r\n"; - BOOST_TEST_EQ(cfg.setup.payload(), expected); - BOOST_TEST(cfg.setup.has_hello_priority()); - BOOST_TEST(cfg.setup.get_config().cancel_if_unresponded); - BOOST_TEST(cfg.setup.get_config().cancel_on_connection_lost); + fixture fix; + fix.cfg.use_setup = true; + fix.cfg.setup.clear(); + fix.cfg.setup.push("SELECT", 8); + + fix.run("*2\r\n$6\r\nSELECT\r\n$1\r\n8\r\n"); } // Regression check: we set the relevant cancellation flags in the request void test_use_setup_flags() { - redis::config cfg; - cfg.use_setup = true; - cfg.setup.clear(); - cfg.setup.push("SELECT", 8); - cfg.setup.get_config().cancel_if_unresponded = false; - cfg.setup.get_config().cancel_on_connection_lost = false; - - compose_setup_request(cfg); - - std::string_view const expected = "*2\r\n$6\r\nSELECT\r\n$1\r\n8\r\n"; - BOOST_TEST_EQ(cfg.setup.payload(), expected); - BOOST_TEST(cfg.setup.has_hello_priority()); - BOOST_TEST(cfg.setup.get_config().cancel_if_unresponded); - BOOST_TEST(cfg.setup.get_config().cancel_on_connection_lost); + fixture fix; + fix.cfg.use_setup = true; + fix.cfg.setup.clear(); + fix.cfg.setup.push("SELECT", 8); + fix.cfg.setup.get_config().cancel_if_unresponded = false; + fix.cfg.setup.get_config().cancel_on_connection_lost = false; + + fix.run("*2\r\n$6\r\nSELECT\r\n$1\r\n8\r\n"); +} + +// If we have tracked subscriptions, these are added at the end +void test_tracked_subscriptions() +{ + fixture fix; + fix.cfg.clientname = ""; + + // Populate the tracker + request sub_req; + sub_req.subscribe({"ch1", "ch2"}); + fix.tracker.commit_changes(sub_req); + + fix.run( + "*2\r\n$5\r\nHELLO\r\n$1\r\n3\r\n" + "*3\r\n$9\r\nSUBSCRIBE\r\n$3\r\nch1\r\n$3\r\nch2\r\n"); +} + +void test_tracked_subscriptions_use_setup() +{ + fixture fix; + fix.cfg.use_setup = true; + fix.cfg.setup.clear(); + fix.cfg.setup.push("PING", "value"); + + // Populate the tracker + request sub_req; + sub_req.subscribe({"ch1", "ch2"}); + fix.tracker.commit_changes(sub_req); + + fix.run( + "*2\r\n$4\r\nPING\r\n$5\r\nvalue\r\n" + "*3\r\n$9\r\nSUBSCRIBE\r\n$3\r\nch1\r\n$3\r\nch2\r\n"); } // When using Sentinel, a ROLE command is added. This works -// both with the old HELLO and new setup strategies. +// both with the old HELLO and new setup strategies, and with tracked subscriptions void test_sentinel_auth() { - redis::config cfg; - cfg.sentinel.addresses = { + fixture fix; + fix.cfg.sentinel.addresses = { {"localhost", "26379"} }; - cfg.clientname = ""; - cfg.username = "foo"; - cfg.password = "bar"; - - compose_setup_request(cfg); + fix.cfg.clientname = ""; + fix.cfg.username = "foo"; + fix.cfg.password = "bar"; - std::string_view const expected = + fix.run( "*5\r\n$5\r\nHELLO\r\n$1\r\n3\r\n$4\r\nAUTH\r\n$3\r\nfoo\r\n$3\r\nbar\r\n" - "*1\r\n$4\r\nROLE\r\n"; - BOOST_TEST_EQ(cfg.setup.payload(), expected); - BOOST_TEST(cfg.setup.has_hello_priority()); - BOOST_TEST(cfg.setup.get_config().cancel_if_unresponded); - BOOST_TEST(cfg.setup.get_config().cancel_on_connection_lost); + "*1\r\n$4\r\nROLE\r\n"); } void test_sentinel_use_setup() { - redis::config cfg; - cfg.sentinel.addresses = { + fixture fix; + fix.cfg.sentinel.addresses = { {"localhost", "26379"} }; - cfg.use_setup = true; - cfg.setup.push("SELECT", 42); + fix.cfg.use_setup = true; + fix.cfg.setup.push("SELECT", 42); - compose_setup_request(cfg); - - std::string_view const expected = + fix.run( "*2\r\n$5\r\nHELLO\r\n$1\r\n3\r\n" "*2\r\n$6\r\nSELECT\r\n$2\r\n42\r\n" - "*1\r\n$4\r\nROLE\r\n"; - BOOST_TEST_EQ(cfg.setup.payload(), expected); - BOOST_TEST(cfg.setup.has_hello_priority()); - BOOST_TEST(cfg.setup.get_config().cancel_if_unresponded); - BOOST_TEST(cfg.setup.get_config().cancel_on_connection_lost); + "*1\r\n$4\r\nROLE\r\n"); +} + +void test_sentinel_tracked_subscriptions() +{ + fixture fix; + fix.cfg.clientname = ""; + fix.cfg.sentinel.addresses = { + {"localhost", "26379"} + }; + + // Populate the tracker + request sub_req; + sub_req.subscribe({"ch1", "ch2"}); + fix.tracker.commit_changes(sub_req); + + fix.run( + "*2\r\n$5\r\nHELLO\r\n$1\r\n3\r\n" + "*1\r\n$4\r\nROLE\r\n" + "*3\r\n$9\r\nSUBSCRIBE\r\n$3\r\nch1\r\n$3\r\nch2\r\n"); } } // namespace @@ -235,8 +246,11 @@ int main() test_use_setup(); test_use_setup_no_hello(); test_use_setup_flags(); + test_tracked_subscriptions(); + test_tracked_subscriptions_use_setup(); test_sentinel_auth(); test_sentinel_use_setup(); + test_sentinel_tracked_subscriptions(); return boost::report_errors(); } \ No newline at end of file diff --git a/test/test_conn_echo_stress.cpp b/test/test_conn_echo_stress.cpp index 48157b0be..e1455b2aa 100644 --- a/test/test_conn_echo_stress.cpp +++ b/test/test_conn_echo_stress.cpp @@ -55,11 +55,7 @@ std::ostream& operator<<(std::ostream& os, usage const& u) namespace { -auto -receiver( - connection& conn, - flat_tree& resp, - std::size_t expected) -> net::awaitable +auto receiver(connection& conn, flat_tree& resp, std::size_t expected) -> net::awaitable { std::size_t push_counter = 0; while (push_counter != expected) { @@ -135,7 +131,7 @@ BOOST_AUTO_TEST_CASE(echo_stress) // Subscribe, then launch the coroutines request req; - req.push("SUBSCRIBE", "channel"); + req.subscribe({"channel"}); conn.async_exec(req, ignore, [&](error_code ec, std::size_t) { subscribe_finished = true; BOOST_TEST(ec == error_code()); @@ -150,13 +146,11 @@ BOOST_AUTO_TEST_CASE(echo_stress) BOOST_TEST(subscribe_finished); // Print statistics - std::cout - << "-------------------\n" - << "Usage data: \n" - << conn.get_usage() << "\n" - << "-------------------\n" - << "Reallocations: " << resp.get_reallocs() - << std::endl; + std::cout << "-------------------\n" + << "Usage data: \n" + << conn.get_usage() << "\n" + << "-------------------\n" + << "Reallocations: " << resp.get_reallocs() << std::endl; } } // namespace diff --git a/test/test_conn_push2.cpp b/test/test_conn_push2.cpp index 9e3d7f623..9cf638e92 100644 --- a/test/test_conn_push2.cpp +++ b/test/test_conn_push2.cpp @@ -7,12 +7,15 @@ #include #include #include +#include #include #include #include +#include #include +#include #define BOOST_TEST_MODULE conn_push #include @@ -35,6 +38,8 @@ using boost::redis::ignore; using boost::redis::ignore_t; using boost::system::error_code; using boost::redis::logger; +using boost::redis::resp3::node_view; +using boost::redis::resp3::type; using namespace std::chrono_literals; namespace { @@ -389,4 +394,157 @@ BOOST_AUTO_TEST_CASE(test_unsubscribe) BOOST_TEST(run_finished); } +class test_pubsub_state_restoration_ { + net::io_context ioc; + connection conn{ioc}; + request req; + response resp_str; + flat_tree resp_push; + bool exec_finished = false; + + void check_subscriptions() + { + // Checks for the expected subscriptions and patterns after restoration + std::set seen_channels, seen_patterns; + for (auto it = resp_push.get_view().begin(); it != resp_push.get_view().end();) { + // The root element should be a push + BOOST_TEST_REQUIRE(it->data_type == type::push); + BOOST_TEST_REQUIRE(it->aggregate_size >= 2u); + BOOST_TEST_REQUIRE((++it != resp_push.get_view().end())); + + // The next element should be the message type + std::string_view msg_type = it->value; + BOOST_TEST_REQUIRE((++it != resp_push.get_view().end())); + + // The next element is the channel or pattern + if (msg_type == "subscribe") + seen_channels.insert(it->value); + else if (msg_type == "psubscribe") + seen_patterns.insert(it->value); + + // Skip the rest of the nodes + while (it != resp_push.get_view().end() && it->depth != 0u) + ++it; + } + + const std::string_view expected_channels[] = {"ch1", "ch3", "ch5"}; + const std::string_view expected_patterns[] = {"ch1*", "ch3*", "ch4*", "ch8*"}; + + BOOST_TEST(seen_channels == expected_channels, boost::test_tools::per_element()); + BOOST_TEST(seen_patterns == expected_patterns, boost::test_tools::per_element()); + } + + void sub1() + { + // Subscribe to some channels and patterns + req.clear(); + req.subscribe({"ch1", "ch2", "ch3"}); // active: 1, 2, 3 + req.psubscribe({"ch1*", "ch2*", "ch3*", "ch4*"}); // active: 1, 2, 3, 4 + conn.async_exec(req, ignore, [this](error_code ec, std::size_t) { + BOOST_TEST(ec == error_code()); + unsub(); + }); + } + + void unsub() + { + // Unsubscribe from some channels and patterns. + // Unsubscribing from a channel/pattern that we weren't subscribed to is OK. + req.clear(); + req.unsubscribe({"ch2", "ch1", "ch5"}); // active: 3 + req.punsubscribe({"ch2*", "ch4*", "ch9*"}); // active: 1, 3 + conn.async_exec(req, ignore, [this](error_code ec, std::size_t) { + BOOST_TEST(ec == error_code()); + sub2(); + }); + } + + void sub2() + { + // Subscribe to other channels/patterns. + // Re-subscribing to channels/patterns we unsubscribed from is OK. + // Subscribing to the same channel/pattern twice is OK. + req.clear(); + req.subscribe({"ch1", "ch3", "ch5"}); // active: 1, 3, 5 + req.psubscribe({"ch3*", "ch4*", "ch8*"}); // active: 1, 3, 4, 8 + + // Subscriptions created by push() don't survive reconnection + req.push("SUBSCRIBE", "ch10"); // active: 1, 3, 5, 10 + req.push("PSUBSCRIBE", "ch10*"); // active: 1, 3, 4, 8, 10 + + // Validate that we're subscribed to what we expect + req.push("CLIENT", "INFO"); + + conn.async_exec(req, resp_str, [this](error_code ec, std::size_t) { + BOOST_TEST(ec == error_code()); + + // We are subscribed to 4 channels and 5 patterns + BOOST_TEST(std::get<0>(resp_str).has_value()); + BOOST_TEST(find_client_info(std::get<0>(resp_str).value(), "sub") == "4"); + BOOST_TEST(find_client_info(std::get<0>(resp_str).value(), "psub") == "5"); + + resp_push.clear(); + + quit(); + }); + } + + void quit() + { + req.clear(); + req.push("QUIT"); + + conn.async_exec(req, ignore, [this](error_code, std::size_t) { + // we don't know if this request will complete successfully or not + client_info(); + }); + } + + void client_info() + { + req.clear(); + req.push("CLIENT", "INFO"); + req.get_config().cancel_if_unresponded = false; + + conn.async_exec(req, resp_str, [this](error_code ec, std::size_t) { + BOOST_TEST(ec == error_code()); + + // We are subscribed to 3 channels and 4 patterns (1 of each didn't survive reconnection) + BOOST_TEST(std::get<0>(resp_str).has_value()); + BOOST_TEST(find_client_info(std::get<0>(resp_str).value(), "sub") == "3"); + BOOST_TEST(find_client_info(std::get<0>(resp_str).value(), "psub") == "4"); + + // We have received pushes confirming it + check_subscriptions(); + + exec_finished = true; + conn.cancel(); + }); + } + +public: + void run() + { + conn.set_receive_response(resp_push); + + // Start the request chain + sub1(); + + // Start running + bool run_finished = false; + conn.async_run(make_test_config(), [&run_finished](error_code ec) { + BOOST_TEST(ec == net::error::operation_aborted); + run_finished = true; + }); + + ioc.run_for(test_timeout); + + // Done + BOOST_TEST(exec_finished); + BOOST_TEST(run_finished); + } +}; + +BOOST_AUTO_TEST_CASE(test_pubsub_state_restoration) { test_pubsub_state_restoration_().run(); } + } // namespace diff --git a/test/test_conn_sentinel.cpp b/test/test_conn_sentinel.cpp index 7caec74ec..f18c1de08 100644 --- a/test/test_conn_sentinel.cpp +++ b/test/test_conn_sentinel.cpp @@ -96,7 +96,7 @@ void test_receive() // Subscribe to a channel. This produces a push message on itself request req; - req.push("SUBSCRIBE", "sentinel_channel"); + req.subscribe({"sentinel_channel"}); bool exec_finished = false, receive_finished = false, run_finished = false; diff --git a/test/test_exec_fsm.cpp b/test/test_exec_fsm.cpp index f23853556..16301554d 100644 --- a/test/test_exec_fsm.cpp +++ b/test/test_exec_fsm.cpp @@ -6,6 +6,7 @@ // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // +#include #include #include #include @@ -30,6 +31,7 @@ using detail::multiplexer; using detail::exec_action_type; using detail::consume_result; using detail::exec_action; +using detail::connection_state; using boost::system::error_code; using boost::asio::cancellation_type_t; @@ -102,11 +104,23 @@ struct elem_and_request { std::shared_ptr elm; std::weak_ptr weak_elm; // check that we free memory - elem_and_request(request::config cfg = {}) - : req(cfg) + static request make_request(request::config cfg) { + request req{cfg}; + // Empty requests are not valid. The request needs to be populated before creating the element req.push("get", "mykey"); + + return req; + } + + elem_and_request(request::config cfg = {}) + : elem_and_request(make_request(cfg)) + { } + + elem_and_request(request input_req) + : req(std::move(input_req)) + { elm = std::make_shared(req, any_adapter{}); elm->set_done_callback([this] { @@ -121,35 +135,35 @@ struct elem_and_request { void test_success() { // Setup - multiplexer mpx; + connection_state st; elem_and_request input; - exec_fsm fsm(mpx, std::move(input.elm)); + exec_fsm fsm(std::move(input.elm)); error_code ec; // Initiate - auto act = fsm.resume(true, cancellation_type_t::none); + auto act = fsm.resume(true, st, cancellation_type_t::none); BOOST_TEST_EQ(act, exec_action_type::setup_cancellation); - act = fsm.resume(true, cancellation_type_t::none); + act = fsm.resume(true, st, cancellation_type_t::none); BOOST_TEST_EQ(act, exec_action_type::notify_writer); // We should now wait for a response - act = fsm.resume(true, cancellation_type_t::none); + act = fsm.resume(true, st, cancellation_type_t::none); BOOST_TEST_EQ(act, exec_action_type::wait_for_response); // Simulate a successful write - BOOST_TEST_EQ(mpx.prepare_write(), 1u); // one request was placed in the packet to write - BOOST_TEST(mpx.commit_write(mpx.get_write_buffer().size())); + BOOST_TEST_EQ(st.mpx.prepare_write(), 1u); // one request was placed in the packet to write + BOOST_TEST(st.mpx.commit_write(st.mpx.get_write_buffer().size())); // Simulate a successful read - read(mpx, "$5\r\nhello\r\n"); - auto req_status = mpx.consume(ec); + read(st.mpx, "$5\r\nhello\r\n"); + auto req_status = st.mpx.consume(ec); BOOST_TEST_EQ(ec, error_code()); BOOST_TEST_EQ(req_status.first, consume_result::got_response); BOOST_TEST_EQ(req_status.second, 11u); // the entire buffer was consumed BOOST_TEST_EQ(input.done_calls, 1u); // This will awaken the exec operation, and should complete the operation - act = fsm.resume(true, cancellation_type_t::none); + act = fsm.resume(true, st, cancellation_type_t::none); BOOST_TEST_EQ(act, exec_action(error_code(), 11u)); // All memory should have been freed by now @@ -160,37 +174,37 @@ void test_success() void test_parse_error() { // Setup - multiplexer mpx; + connection_state st; elem_and_request input; - exec_fsm fsm(mpx, std::move(input.elm)); + exec_fsm fsm(std::move(input.elm)); error_code ec; // Initiate - auto act = fsm.resume(true, cancellation_type_t::none); + auto act = fsm.resume(true, st, cancellation_type_t::none); BOOST_TEST_EQ(act, exec_action_type::setup_cancellation); - act = fsm.resume(true, cancellation_type_t::none); + act = fsm.resume(true, st, cancellation_type_t::none); BOOST_TEST_EQ(act, exec_action_type::notify_writer); // We should now wait for a response - act = fsm.resume(true, cancellation_type_t::none); + act = fsm.resume(true, st, cancellation_type_t::none); BOOST_TEST_EQ(act, exec_action_type::wait_for_response); // Simulate a successful write - BOOST_TEST_EQ(mpx.prepare_write(), 1u); // one request was placed in the packet to write - BOOST_TEST(mpx.commit_write(mpx.get_write_buffer().size())); + BOOST_TEST_EQ(st.mpx.prepare_write(), 1u); // one request was placed in the packet to write + BOOST_TEST(st.mpx.commit_write(st.mpx.get_write_buffer().size())); // Simulate a read that will trigger an error. // The second field should be a number (rather than the empty string). // Note that although part of the buffer was consumed, the multiplexer // currently throws this information away. - read(mpx, "*2\r\n$5\r\nhello\r\n:\r\n"); - auto req_status = mpx.consume(ec); + read(st.mpx, "*2\r\n$5\r\nhello\r\n:\r\n"); + auto req_status = st.mpx.consume(ec); BOOST_TEST_EQ(ec, error::empty_field); BOOST_TEST_EQ(req_status.second, 15u); BOOST_TEST_EQ(input.done_calls, 1u); // This will awaken the exec operation, and should complete the operation - act = fsm.resume(true, cancellation_type_t::none); + act = fsm.resume(true, st, cancellation_type_t::none); BOOST_TEST_EQ(act, exec_action(error::empty_field, 0u)); // All memory should have been freed by now @@ -201,17 +215,17 @@ void test_parse_error() void test_cancel_if_not_connected() { // Setup - multiplexer mpx; + connection_state st; request::config cfg; cfg.cancel_if_not_connected = true; elem_and_request input(cfg); - exec_fsm fsm(mpx, std::move(input.elm)); + exec_fsm fsm(std::move(input.elm)); // Initiate. We're not connected, so the request gets cancelled - auto act = fsm.resume(false, cancellation_type_t::none); + auto act = fsm.resume(false, st, cancellation_type_t::none); BOOST_TEST_EQ(act, exec_action_type::immediate); - act = fsm.resume(false, cancellation_type_t::none); + act = fsm.resume(false, st, cancellation_type_t::none); BOOST_TEST_EQ(act, exec_action(error::not_connected)); // We didn't leave memory behind @@ -222,35 +236,35 @@ void test_cancel_if_not_connected() void test_not_connected() { // Setup - multiplexer mpx; + connection_state st; elem_and_request input; - exec_fsm fsm(mpx, std::move(input.elm)); + exec_fsm fsm(std::move(input.elm)); error_code ec; // Initiate - auto act = fsm.resume(false, cancellation_type_t::none); + auto act = fsm.resume(false, st, cancellation_type_t::none); BOOST_TEST_EQ(act, exec_action_type::setup_cancellation); - act = fsm.resume(true, cancellation_type_t::none); + act = fsm.resume(true, st, cancellation_type_t::none); BOOST_TEST_EQ(act, exec_action_type::notify_writer); // We should now wait for a response - act = fsm.resume(true, cancellation_type_t::none); + act = fsm.resume(true, st, cancellation_type_t::none); BOOST_TEST_EQ(act, exec_action_type::wait_for_response); // Simulate a successful write - BOOST_TEST_EQ(mpx.prepare_write(), 1u); // one request was placed in the packet to write - BOOST_TEST(mpx.commit_write(mpx.get_write_buffer().size())); + BOOST_TEST_EQ(st.mpx.prepare_write(), 1u); // one request was placed in the packet to write + BOOST_TEST(st.mpx.commit_write(st.mpx.get_write_buffer().size())); // Simulate a successful read - read(mpx, "$5\r\nhello\r\n"); - auto req_status = mpx.consume(ec); + read(st.mpx, "$5\r\nhello\r\n"); + auto req_status = st.mpx.consume(ec); BOOST_TEST_EQ(ec, error_code()); BOOST_TEST_EQ(req_status.first, consume_result::got_response); BOOST_TEST_EQ(req_status.second, 11u); // the entire buffer was consumed BOOST_TEST_EQ(input.done_calls, 1u); // This will awaken the exec operation, and should complete the operation - act = fsm.resume(true, cancellation_type_t::none); + act = fsm.resume(true, st, cancellation_type_t::none); BOOST_TEST_EQ(act, exec_action(error_code(), 11u)); // All memory should have been freed by now @@ -277,24 +291,24 @@ void test_cancel_waiting() for (const auto& tc : test_cases) { // Setup - multiplexer mpx; + connection_state st; elem_and_request input, input2; - exec_fsm fsm(mpx, std::move(input.elm)); + exec_fsm fsm(std::move(input.elm)); // Another request enters the multiplexer, so it's busy when we start - mpx.add(input2.elm); - BOOST_TEST_EQ_MSG(mpx.prepare_write(), 1u, tc.name); + st.mpx.add(input2.elm); + BOOST_TEST_EQ_MSG(st.mpx.prepare_write(), 1u, tc.name); // Initiate and wait - auto act = fsm.resume(true, cancellation_type_t::none); + auto act = fsm.resume(true, st, cancellation_type_t::none); BOOST_TEST_EQ_MSG(act, exec_action_type::setup_cancellation, tc.name); - act = fsm.resume(true, cancellation_type_t::none); + act = fsm.resume(true, st, cancellation_type_t::none); BOOST_TEST_EQ_MSG(act, exec_action_type::notify_writer, tc.name); - act = fsm.resume(true, cancellation_type_t::none); + act = fsm.resume(true, st, cancellation_type_t::none); BOOST_TEST_EQ_MSG(act, exec_action_type::wait_for_response, tc.name); // We get notified because the request got cancelled - act = fsm.resume(true, tc.type); + act = fsm.resume(true, st, tc.type); BOOST_TEST_EQ_MSG(act, exec_action(asio::error::operation_aborted), tc.name); BOOST_TEST_EQ_MSG(input.weak_elm.expired(), true, tc.name); // we didn't leave memory behind } @@ -314,32 +328,32 @@ void test_cancel_notwaiting_terminal_partial() for (const auto& tc : test_cases) { // Setup - multiplexer mpx; + connection_state st; auto input = std::make_unique(); - exec_fsm fsm(mpx, std::move(input->elm)); + exec_fsm fsm(std::move(input->elm)); // Initiate - auto act = fsm.resume(false, cancellation_type_t::none); + auto act = fsm.resume(false, st, cancellation_type_t::none); BOOST_TEST_EQ_MSG(act, exec_action_type::setup_cancellation, tc.name); - act = fsm.resume(true, cancellation_type_t::none); + act = fsm.resume(true, st, cancellation_type_t::none); BOOST_TEST_EQ_MSG(act, exec_action_type::notify_writer, tc.name); - act = fsm.resume(true, cancellation_type_t::none); + act = fsm.resume(true, st, cancellation_type_t::none); BOOST_TEST_EQ_MSG(act, exec_action_type::wait_for_response, tc.name); // The multiplexer starts writing the request - BOOST_TEST_EQ_MSG(mpx.prepare_write(), 1u, tc.name); - BOOST_TEST_EQ_MSG(mpx.commit_write(mpx.get_write_buffer().size()), true, tc.name); + BOOST_TEST_EQ_MSG(st.mpx.prepare_write(), 1u, tc.name); + BOOST_TEST_EQ_MSG(st.mpx.commit_write(st.mpx.get_write_buffer().size()), true, tc.name); // A cancellation arrives - act = fsm.resume(true, tc.type); + act = fsm.resume(true, st, tc.type); BOOST_TEST_EQ(act, exec_action(asio::error::operation_aborted)); input.reset(); // Verify we don't access the request or response after completion error_code ec; // When the response to this request arrives, it gets ignored - read(mpx, "-ERR wrong command\r\n"); - auto res = mpx.consume(ec); + read(st.mpx, "-ERR wrong command\r\n"); + auto res = st.mpx.consume(ec); BOOST_TEST_EQ_MSG(ec, error_code(), tc.name); BOOST_TEST_EQ_MSG(res.first, consume_result::got_response, tc.name); @@ -352,44 +366,122 @@ void test_cancel_notwaiting_terminal_partial() void test_cancel_notwaiting_total() { // Setup - multiplexer mpx; + connection_state st; elem_and_request input; - exec_fsm fsm(mpx, std::move(input.elm)); + exec_fsm fsm(std::move(input.elm)); error_code ec; // Initiate - auto act = fsm.resume(true, cancellation_type_t::none); + auto act = fsm.resume(true, st, cancellation_type_t::none); BOOST_TEST_EQ(act, exec_action_type::setup_cancellation); - act = fsm.resume(true, cancellation_type_t::none); + act = fsm.resume(true, st, cancellation_type_t::none); BOOST_TEST_EQ(act, exec_action_type::notify_writer); - act = fsm.resume(true, cancellation_type_t::none); + act = fsm.resume(true, st, cancellation_type_t::none); BOOST_TEST_EQ(act, exec_action_type::wait_for_response); // Simulate a successful write - BOOST_TEST_EQ(mpx.prepare_write(), 1u); - BOOST_TEST(mpx.commit_write(mpx.get_write_buffer().size())); + BOOST_TEST_EQ(st.mpx.prepare_write(), 1u); + BOOST_TEST(st.mpx.commit_write(st.mpx.get_write_buffer().size())); // We got requested a cancellation here, but we can't honor it - act = fsm.resume(true, asio::cancellation_type_t::total); + act = fsm.resume(true, st, asio::cancellation_type_t::total); BOOST_TEST_EQ(act, exec_action_type::wait_for_response); // Simulate a successful read - read(mpx, "$5\r\nhello\r\n"); - auto req_status = mpx.consume(ec); + read(st.mpx, "$5\r\nhello\r\n"); + auto req_status = st.mpx.consume(ec); BOOST_TEST_EQ(ec, error_code()); BOOST_TEST_EQ(req_status.first, consume_result::got_response); BOOST_TEST_EQ(req_status.second, 11u); // the entire buffer was consumed BOOST_TEST_EQ(input.done_calls, 1u); // This will awaken the exec operation, and should complete the operation - act = fsm.resume(true, cancellation_type_t::none); + act = fsm.resume(true, st, cancellation_type_t::none); BOOST_TEST_EQ(act, exec_action(error_code(), 11u)); // All memory should have been freed by now BOOST_TEST_EQ(input.weak_elm.expired(), true); } +// If a request completes successfully and contained pubsub changes, these are committed +void test_subscription_tracking_success() +{ + // Setup + request req; + req.subscribe({"ch1", "ch2"}); + connection_state st; + elem_and_request input{std::move(req)}; + exec_fsm fsm(std::move(input.elm)); + + // Initiate + auto act = fsm.resume(true, st, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_action_type::setup_cancellation); + act = fsm.resume(true, st, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_action_type::notify_writer); + + // We should now wait for a response + act = fsm.resume(true, st, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_action_type::wait_for_response); + + // Simulate a successful write + BOOST_TEST_EQ(st.mpx.prepare_write(), 1u); // one request was placed in the packet to write + BOOST_TEST(st.mpx.commit_write(st.mpx.get_write_buffer().size())); + + // The request doesn't have a response, so this will + // awaken the exec operation, and should complete the operation + act = fsm.resume(true, st, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_action(error_code(), 0u)); + + // All memory should have been freed by now + BOOST_TEST(input.weak_elm.expired()); + + // The subscription has been added to the tracker + request tracker_req; + st.tracker.compose_subscribe_request(tracker_req); + + request expected_req; + expected_req.push("SUBSCRIBE", "ch1", "ch2"); + BOOST_TEST_EQ(tracker_req.payload(), expected_req.payload()); +} + +// If the request errors, tracked subscriptions are not committed +void test_subscription_tracking_error() +{ + // Setup + request req; + req.subscribe({"ch1", "ch2"}); + connection_state st; + elem_and_request input{std::move(req)}; + exec_fsm fsm(std::move(input.elm)); + + // Initiate + auto act = fsm.resume(true, st, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_action_type::setup_cancellation); + act = fsm.resume(true, st, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_action_type::notify_writer); + + // We should now wait for a response + act = fsm.resume(true, st, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_action_type::wait_for_response); + + // Simulate a write error, which would trigger a reconnection + BOOST_TEST_EQ(st.mpx.prepare_write(), 1u); // one request was placed in the packet to write + st.mpx.cancel_on_conn_lost(); + + // This awakens the request + act = fsm.resume(true, st, cancellation_type_t::none); + BOOST_TEST_EQ(act, exec_action(asio::error::operation_aborted, 0u)); + + // All memory should have been freed by now + BOOST_TEST(input.weak_elm.expired()); + + // The subscription has not been added to the tracker + request tracker_req; + st.tracker.compose_subscribe_request(tracker_req); + BOOST_TEST_EQ(tracker_req.payload(), ""); +} + } // namespace int main() @@ -401,6 +493,8 @@ int main() test_cancel_waiting(); test_cancel_notwaiting_terminal_partial(); test_cancel_notwaiting_total(); + test_subscription_tracking_success(); + test_subscription_tracking_error(); return boost::report_errors(); } diff --git a/test/test_request.cpp b/test/test_request.cpp index 63b59ca3c..9777e790c 100644 --- a/test/test_request.cpp +++ b/test/test_request.cpp @@ -6,16 +6,72 @@ #include +#include #include +#include +#include +#include +#include #include +#include +#include #include #include +#include -using boost::redis::request; +using namespace boost::redis; +using detail::pubsub_change; +using detail::pubsub_change_type; namespace { +// --- Utilities to check subscription tracking --- +const char* to_string(pubsub_change_type type) +{ + switch (type) { + case pubsub_change_type::subscribe: return "subscribe"; + case pubsub_change_type::unsubscribe: return "unsubscribe"; + case pubsub_change_type::psubscribe: return "psubscribe"; + case pubsub_change_type::punsubscribe: return "punsubscribe"; + default: return ""; + } +} + +// Like pubsub_change, but using a string instead of an offset +struct pubsub_change_str { + pubsub_change_type type; + std::string_view value; + + friend bool operator==(const pubsub_change_str& lhs, const pubsub_change_str& rhs) + { + return lhs.type == rhs.type && lhs.value == rhs.value; + } + + friend std::ostream& operator<<(std::ostream& os, const pubsub_change_str& value) + { + return os << "{ " << to_string(value.type) << ", " << value.value << " }"; + } +}; + +void check_pubsub_changes( + const request& req, + boost::span expected, + boost::source_location loc = BOOST_CURRENT_LOCATION) +{ + // Convert from offsets to strings + std::vector actual; + for (const auto& change : detail::request_access::pubsub_changes(req)) { + actual.push_back( + {change.type, req.payload().substr(change.channel_offset, change.channel_size)}); + } + + // Check + if (!BOOST_TEST_ALL_EQ(actual.begin(), actual.end(), expected.begin(), expected.end())) + std::cerr << "Called from " << loc << std::endl; +} + +// --- Generic functions to add commands --- void test_push_no_args() { request req1; @@ -38,6 +94,26 @@ void test_push_multiple_args() BOOST_TEST_EQ(req.payload(), res); } +// Subscription commands added with push are not tracked +void test_push_pubsub() +{ + request req; + req.push("SUBSCRIBE", "ch1"); + req.push("UNSUBSCRIBE", "ch2"); + req.push("PSUBSCRIBE", "ch3*"); + req.push("PUNSUBSCRIBE", "ch4*"); + + char const* res = + "*2\r\n$9\r\nSUBSCRIBE\r\n$3\r\nch1\r\n" + "*2\r\n$11\r\nUNSUBSCRIBE\r\n$3\r\nch2\r\n" + "*2\r\n$10\r\nPSUBSCRIBE\r\n$4\r\nch3*\r\n" + "*2\r\n$12\r\nPUNSUBSCRIBE\r\n$4\r\nch4*\r\n"; + BOOST_TEST_EQ(req.payload(), res); + BOOST_TEST_EQ(req.get_expected_responses(), 0u); + check_pubsub_changes(req, {}); +} + +// --- push_range --- void test_push_range() { std::map in{ @@ -58,7 +134,340 @@ void test_push_range() BOOST_TEST_EQ(req2.payload(), expected); } -// Append +// Subscription commands added with push_range are not tracked +void test_push_range_pubsub() +{ + const std::vector channels1{"ch1", "ch2"}, channels2{"ch3"}, patterns1{"ch3*"}, + patterns2{"ch4*"}; + request req; + req.push_range("SUBSCRIBE", channels1); + req.push_range("UNSUBSCRIBE", channels2); + req.push_range("PSUBSCRIBE", patterns1); + req.push_range("PUNSUBSCRIBE", patterns2); + + char const* res = + "*3\r\n$9\r\nSUBSCRIBE\r\n$3\r\nch1\r\n$3\r\nch2\r\n" + "*2\r\n$11\r\nUNSUBSCRIBE\r\n$3\r\nch3\r\n" + "*2\r\n$10\r\nPSUBSCRIBE\r\n$4\r\nch3*\r\n" + "*2\r\n$12\r\nPUNSUBSCRIBE\r\n$4\r\nch4*\r\n"; + BOOST_TEST_EQ(req.payload(), res); + BOOST_TEST_EQ(req.get_expected_responses(), 0u); + check_pubsub_changes(req, {}); +} + +// --- subscribe --- +// Most of the tests build the same request using different overloads. +// This fixture makes checking easier +struct subscribe_fixture { + request req; + + void check_impl( + std::string_view expected_payload, + pubsub_change_type expected_type, + boost::source_location loc = BOOST_CURRENT_LOCATION) + { + if (!BOOST_TEST_EQ(req.payload(), expected_payload)) + std::cerr << "Called from " << loc << std::endl; + + if (!BOOST_TEST_EQ(req.get_commands(), 1u)) + std::cerr << "Called from " << loc << std::endl; + + if (!BOOST_TEST_EQ(req.get_expected_responses(), 0u)) + std::cerr << "Called from " << loc << std::endl; + + const pubsub_change_str expected_changes[] = { + {expected_type, "ch1"}, + {expected_type, "ch2"}, + }; + check_pubsub_changes(req, expected_changes, loc); + } + + void check_subscribe(boost::source_location loc = BOOST_CURRENT_LOCATION) + { + check_impl( + "*3\r\n$9\r\nSUBSCRIBE\r\n$3\r\nch1\r\n$3\r\nch2\r\n", + pubsub_change_type::subscribe, + loc); + } + + void check_unsubscribe(boost::source_location loc = BOOST_CURRENT_LOCATION) + { + check_impl( + "*3\r\n$11\r\nUNSUBSCRIBE\r\n$3\r\nch1\r\n$3\r\nch2\r\n", + pubsub_change_type::unsubscribe, + loc); + } + + void check_psubscribe(boost::source_location loc = BOOST_CURRENT_LOCATION) + { + check_impl( + "*3\r\n$10\r\nPSUBSCRIBE\r\n$3\r\nch1\r\n$3\r\nch2\r\n", + pubsub_change_type::psubscribe, + loc); + } + + void check_punsubscribe(boost::source_location loc = BOOST_CURRENT_LOCATION) + { + check_impl( + "*3\r\n$12\r\nPUNSUBSCRIBE\r\n$3\r\nch1\r\n$3\r\nch2\r\n", + pubsub_change_type::punsubscribe, + loc); + } +}; + +void test_subscribe_iterators() +{ + subscribe_fixture fix; + const std::forward_list channels{"ch1", "ch2"}; + + fix.req.subscribe(channels.begin(), channels.end()); + + fix.check_subscribe(); +} + +// Like push_range, if the range is empty, this is a no-op +void test_subscribe_iterators_empty() +{ + const std::forward_list channels; + request req; + + req.subscribe(channels.begin(), channels.end()); + + BOOST_TEST_EQ(req.payload(), ""); + BOOST_TEST_EQ(req.get_commands(), 0u); + BOOST_TEST_EQ(req.get_expected_responses(), 0u); + check_pubsub_changes(req, {}); +} + +// Iterators whose value_type is convertible to std::string_view work +void test_subscribe_iterators_convertible_string_view() +{ + subscribe_fixture fix; + const std::vector channels{"ch1", "ch2"}; + + fix.req.subscribe(channels.begin(), channels.end()); + + fix.check_subscribe(); +} + +// The range overload just dispatches to the iterator one +void test_subscribe_range() +{ + subscribe_fixture fix; + const std::vector channels{"ch1", "ch2"}; + + fix.req.subscribe(channels); + + fix.check_subscribe(); +} + +// The initializer_list overload just dispatches to the iterator one +void test_subscribe_initializer_list() +{ + subscribe_fixture fix; + + fix.req.subscribe({"ch1", "ch2"}); + + fix.check_subscribe(); +} + +// --- unsubscribe --- +void test_unsubscribe_iterators() +{ + subscribe_fixture fix; + const std::forward_list channels{"ch1", "ch2"}; + + fix.req.unsubscribe(channels.begin(), channels.end()); + + fix.check_unsubscribe(); +} + +// Like push_range, if the range is empty, this is a no-op +void test_unsubscribe_iterators_empty() +{ + const std::forward_list channels; + request req; + + req.unsubscribe(channels.begin(), channels.end()); + + BOOST_TEST_EQ(req.payload(), ""); + BOOST_TEST_EQ(req.get_commands(), 0u); + BOOST_TEST_EQ(req.get_expected_responses(), 0u); + check_pubsub_changes(req, {}); +} + +// Iterators whose value_type is convertible to std::string_view work +void test_unsubscribe_iterators_convertible_string_view() +{ + subscribe_fixture fix; + const std::vector channels{"ch1", "ch2"}; + + fix.req.unsubscribe(channels.begin(), channels.end()); + + fix.check_unsubscribe(); +} + +// The range overload just dispatches to the iterator one +void test_unsubscribe_range() +{ + subscribe_fixture fix; + const std::vector channels{"ch1", "ch2"}; + + fix.req.unsubscribe(channels); + + fix.check_unsubscribe(); +} + +// The initializer_list overload just dispatches to the iterator one +void test_unsubscribe_initializer_list() +{ + subscribe_fixture fix; + + fix.req.unsubscribe({"ch1", "ch2"}); + + fix.check_unsubscribe(); +} + +// --- psubscribe --- +void test_psubscribe_iterators() +{ + subscribe_fixture fix; + const std::forward_list channels{"ch1", "ch2"}; + + fix.req.psubscribe(channels.begin(), channels.end()); + + fix.check_psubscribe(); +} + +// Like push_range, if the range is empty, this is a no-op +void test_psubscribe_iterators_empty() +{ + const std::forward_list channels; + request req; + + req.psubscribe(channels.begin(), channels.end()); + + BOOST_TEST_EQ(req.payload(), ""); + BOOST_TEST_EQ(req.get_commands(), 0u); + BOOST_TEST_EQ(req.get_expected_responses(), 0u); + check_pubsub_changes(req, {}); +} + +// Iterators whose value_type is convertible to std::string_view work +void test_psubscribe_iterators_convertible_string_view() +{ + subscribe_fixture fix; + const std::vector channels{"ch1", "ch2"}; + + fix.req.psubscribe(channels.begin(), channels.end()); + + fix.check_psubscribe(); +} + +// The range overload just dispatches to the iterator one +void test_psubscribe_range() +{ + subscribe_fixture fix; + const std::vector channels{"ch1", "ch2"}; + + fix.req.psubscribe(channels); + + fix.check_psubscribe(); +} + +// The initializer_list overload just dispatches to the iterator one +void test_psubscribe_initializer_list() +{ + subscribe_fixture fix; + + fix.req.psubscribe({"ch1", "ch2"}); + + fix.check_psubscribe(); +} + +// --- punsubscribe --- +void test_punsubscribe_iterators() +{ + subscribe_fixture fix; + const std::forward_list channels{"ch1", "ch2"}; + + fix.req.punsubscribe(channels.begin(), channels.end()); + + fix.check_punsubscribe(); +} + +// Like push_range, if the range is empty, this is a no-op +void test_punsubscribe_iterators_empty() +{ + const std::forward_list channels; + request req; + + req.punsubscribe(channels.begin(), channels.end()); + + BOOST_TEST_EQ(req.payload(), ""); + BOOST_TEST_EQ(req.get_commands(), 0u); + BOOST_TEST_EQ(req.get_expected_responses(), 0u); + check_pubsub_changes(req, {}); +} + +// Iterators whose value_type is convertible to std::string_view work +void test_punsubscribe_iterators_convertible_string_view() +{ + subscribe_fixture fix; + const std::vector channels{"ch1", "ch2"}; + + fix.req.punsubscribe(channels.begin(), channels.end()); + + fix.check_punsubscribe(); +} + +// The range overload just dispatches to the iterator one +void test_punsubscribe_range() +{ + subscribe_fixture fix; + const std::vector channels{"ch1", "ch2"}; + + fix.req.punsubscribe(channels); + + fix.check_punsubscribe(); +} + +// The initializer_list overload just dispatches to the iterator one +void test_punsubscribe_initializer_list() +{ + subscribe_fixture fix; + + fix.req.punsubscribe({"ch1", "ch2"}); + + fix.check_punsubscribe(); +} + +// Mixing regular commands and pubsub commands is OK +void test_mix_pubsub_regular() +{ + request req; + req.push("PING"); + req.subscribe({"ch1", "ch2"}); + req.push("GET", "key"); + req.punsubscribe({"ch4*"}); + + constexpr std::string_view expected = + "*1\r\n$4\r\nPING\r\n" + "*3\r\n$9\r\nSUBSCRIBE\r\n$3\r\nch1\r\n$3\r\nch2\r\n" + "*2\r\n$3\r\nGET\r\n$3\r\nkey\r\n" + "*2\r\n$12\r\nPUNSUBSCRIBE\r\n$4\r\nch4*\r\n"; + BOOST_TEST_EQ(req.payload(), expected); + BOOST_TEST_EQ(req.get_commands(), 4u); + BOOST_TEST_EQ(req.get_expected_responses(), 2u); + constexpr pubsub_change_str expected_changes[] = { + {pubsub_change_type::subscribe, "ch1" }, + {pubsub_change_type::subscribe, "ch2" }, + {pubsub_change_type::punsubscribe, "ch4*"}, + }; + check_pubsub_changes(req, expected_changes); +} + +// --- append --- void test_append() { request req1; @@ -77,6 +486,7 @@ void test_append() BOOST_TEST_EQ(req1.payload(), expected); BOOST_TEST_EQ(req1.get_commands(), 3u); BOOST_TEST_EQ(req1.get_expected_responses(), 3u); + check_pubsub_changes(req1, {}); } // Commands without responses are handled correctly @@ -98,6 +508,7 @@ void test_append_no_response() BOOST_TEST_EQ(req1.payload(), expected); BOOST_TEST_EQ(req1.get_commands(), 3u); BOOST_TEST_EQ(req1.get_expected_responses(), 2u); + check_pubsub_changes(req1, {}); } // Flags are not modified by append @@ -140,6 +551,7 @@ void test_append_target_empty() BOOST_TEST_EQ(req1.payload(), expected); BOOST_TEST_EQ(req1.get_commands(), 1u); BOOST_TEST_EQ(req1.get_expected_responses(), 1u); + check_pubsub_changes(req1, {}); } void test_append_source_empty() @@ -155,6 +567,7 @@ void test_append_source_empty() BOOST_TEST_EQ(req1.payload(), expected); BOOST_TEST_EQ(req1.get_commands(), 1u); BOOST_TEST_EQ(req1.get_expected_responses(), 1u); + check_pubsub_changes(req1, {}); } void test_append_both_empty() @@ -167,6 +580,89 @@ void test_append_both_empty() BOOST_TEST_EQ(req1.payload(), ""); BOOST_TEST_EQ(req1.get_commands(), 0u); BOOST_TEST_EQ(req1.get_expected_responses(), 0u); + check_pubsub_changes(req1, {}); +} + +// Append correctly handles requests with pubsub changes +void test_append_pubsub() +{ + request req1; + req1.subscribe({"ch1"}); + + auto req2 = std::make_unique(); + req2->unsubscribe({"ch2"}); + req2->psubscribe({"really_very_long_pattern_name*"}); + + req1.append(*req2); + req2.reset(); // make sure we don't leave dangling pointers + + constexpr std::string_view expected = + "*2\r\n$9\r\nSUBSCRIBE\r\n$3\r\nch1\r\n" + "*2\r\n$11\r\nUNSUBSCRIBE\r\n$3\r\nch2\r\n" + "*2\r\n$10\r\nPSUBSCRIBE\r\n$30\r\nreally_very_long_pattern_name*\r\n"; + BOOST_TEST_EQ(req1.payload(), expected); + const pubsub_change_str expected_changes[] = { + {pubsub_change_type::subscribe, "ch1" }, + {pubsub_change_type::unsubscribe, "ch2" }, + {pubsub_change_type::psubscribe, "really_very_long_pattern_name*"}, + }; + check_pubsub_changes(req1, expected_changes); +} + +// If the target is empty and the source has pubsub changes, that's OK +void test_append_pubsub_target_empty() +{ + request req1; + + request req2; + req2.punsubscribe({"ch2"}); + + req1.append(req2); + + constexpr std::string_view expected = "*2\r\n$12\r\nPUNSUBSCRIBE\r\n$3\r\nch2\r\n"; + BOOST_TEST_EQ(req1.payload(), expected); + const pubsub_change_str expected_changes[] = { + {pubsub_change_type::punsubscribe, "ch2"}, + }; + check_pubsub_changes(req1, expected_changes); +} + +// --- clear --- +void test_clear() +{ + // Create request with some commands and some pubsub changes + request req; + req.push("PING", "value"); + req.push("GET", "key"); + req.subscribe({"ch1", "ch2"}); + req.punsubscribe({"ch3*"}); + + // Clear removes the payload, the commands and the pubsub changes + req.clear(); + BOOST_TEST_EQ(req.payload(), ""); + BOOST_TEST_EQ(req.get_commands(), 0u); + BOOST_TEST_EQ(req.get_expected_responses(), 0u); + check_pubsub_changes(req, {}); + + // Clearing again does nothing + req.clear(); + BOOST_TEST_EQ(req.payload(), ""); + BOOST_TEST_EQ(req.get_commands(), 0u); + BOOST_TEST_EQ(req.get_expected_responses(), 0u); + check_pubsub_changes(req, {}); +} + +// Clearing an empty request doesn't cause trouble +void test_clear_empty() +{ + request req; + + req.clear(); + + BOOST_TEST_EQ(req.payload(), ""); + BOOST_TEST_EQ(req.get_commands(), 0u); + BOOST_TEST_EQ(req.get_expected_responses(), 0u); + check_pubsub_changes(req, {}); } } // namespace @@ -176,7 +672,36 @@ int main() test_push_no_args(); test_push_int(); test_push_multiple_args(); + test_push_pubsub(); + test_push_range(); + test_push_range_pubsub(); + + test_subscribe_iterators(); + test_subscribe_iterators_empty(); + test_subscribe_iterators_convertible_string_view(); + test_subscribe_range(); + test_subscribe_initializer_list(); + + test_unsubscribe_iterators(); + test_unsubscribe_iterators_empty(); + test_unsubscribe_iterators_convertible_string_view(); + test_unsubscribe_range(); + test_unsubscribe_initializer_list(); + + test_psubscribe_iterators(); + test_psubscribe_iterators_empty(); + test_psubscribe_iterators_convertible_string_view(); + test_psubscribe_range(); + test_psubscribe_initializer_list(); + + test_punsubscribe_iterators(); + test_punsubscribe_iterators_empty(); + test_punsubscribe_iterators_convertible_string_view(); + test_punsubscribe_range(); + test_punsubscribe_initializer_list(); + + test_mix_pubsub_regular(); test_append(); test_append_no_response(); @@ -184,6 +709,11 @@ int main() test_append_target_empty(); test_append_source_empty(); test_append_both_empty(); + test_append_pubsub(); + test_append_pubsub_target_empty(); + + test_clear(); + test_clear_empty(); return boost::report_errors(); } diff --git a/test/test_run_fsm.cpp b/test/test_run_fsm.cpp index ef5706610..53f413f6f 100644 --- a/test/test_run_fsm.cpp +++ b/test/test_run_fsm.cpp @@ -565,7 +565,7 @@ void test_setup_ping_requests() const std::string_view expected_setup = "*5\r\n$5\r\nHELLO\r\n$1\r\n3\r\n$4\r\nAUTH\r\n$3\r\nfoo\r\n$3\r\nbar\r\n"; BOOST_TEST_EQ(fix.st.ping_req.payload(), expected_ping); - BOOST_TEST_EQ(fix.st.cfg.setup.payload(), expected_setup); + BOOST_TEST_EQ(fix.st.setup_req.payload(), expected_setup); // Reconnect act = fix.fsm.resume(fix.st, error::empty_field, cancellation_type_t::none); @@ -579,7 +579,7 @@ void test_setup_ping_requests() // The requests haven't been modified BOOST_TEST_EQ(fix.st.ping_req.payload(), expected_ping); - BOOST_TEST_EQ(fix.st.cfg.setup.payload(), expected_setup); + BOOST_TEST_EQ(fix.st.setup_req.payload(), expected_setup); } // We correctly send and log the setup request diff --git a/test/test_setup_adapter.cpp b/test/test_setup_adapter.cpp index 694700eb2..fab4ce95e 100644 --- a/test/test_setup_adapter.cpp +++ b/test/test_setup_adapter.cpp @@ -30,7 +30,7 @@ void test_success() connection_state st; st.cfg.use_setup = true; st.cfg.setup.push("SELECT", 2); - compose_setup_request(st.cfg); + compose_setup_request(st.cfg, st.tracker, st.setup_req); setup_adapter adapter{st}; // Response to HELLO @@ -55,7 +55,7 @@ void test_simple_error() // Setup connection_state st; st.cfg.use_setup = true; - compose_setup_request(st.cfg); + compose_setup_request(st.cfg, st.tracker, st.setup_req); setup_adapter adapter{st}; // Response to HELLO contains an error @@ -73,7 +73,7 @@ void test_blob_error() connection_state st; st.cfg.use_setup = true; st.cfg.setup.push("SELECT", 1); - compose_setup_request(st.cfg); + compose_setup_request(st.cfg, st.tracker, st.setup_req); setup_adapter adapter{st}; // Response to HELLO @@ -97,7 +97,7 @@ void test_null() // Setup connection_state st; st.cfg.use_setup = true; - compose_setup_request(st.cfg); + compose_setup_request(st.cfg, st.tracker, st.setup_req); setup_adapter adapter{st}; // Response to HELLO @@ -129,7 +129,7 @@ void test_sentinel_master() st.cfg.sentinel.addresses = { {"localhost", "26379"} }; - compose_setup_request(st.cfg); + compose_setup_request(st.cfg, st.tracker, st.setup_req); setup_adapter adapter{st}; // Response to HELLO @@ -164,7 +164,7 @@ void test_sentinel_replica() {"localhost", "26379"} }; st.cfg.sentinel.server_role = role::replica; - compose_setup_request(st.cfg); + compose_setup_request(st.cfg, st.tracker, st.setup_req); setup_adapter adapter{st}; // Response to HELLO @@ -193,7 +193,7 @@ void test_sentinel_role_check_failed_master() st.cfg.sentinel.addresses = { {"localhost", "26379"} }; - compose_setup_request(st.cfg); + compose_setup_request(st.cfg, st.tracker, st.setup_req); setup_adapter adapter{st}; // Response to HELLO @@ -222,7 +222,7 @@ void test_sentinel_role_check_failed_replica() {"localhost", "26379"} }; st.cfg.sentinel.server_role = role::replica; - compose_setup_request(st.cfg); + compose_setup_request(st.cfg, st.tracker, st.setup_req); setup_adapter adapter{st}; // Response to HELLO @@ -252,7 +252,7 @@ void test_sentinel_role_error_node() st.cfg.sentinel.addresses = { {"localhost", "26379"} }; - compose_setup_request(st.cfg); + compose_setup_request(st.cfg, st.tracker, st.setup_req); setup_adapter adapter{st}; // Response to ROLE @@ -273,7 +273,7 @@ void test_sentinel_role_not_array() st.cfg.sentinel.addresses = { {"localhost", "26379"} }; - compose_setup_request(st.cfg); + compose_setup_request(st.cfg, st.tracker, st.setup_req); setup_adapter adapter{st}; // Response to ROLE @@ -294,7 +294,7 @@ void test_sentinel_role_empty_array() st.cfg.sentinel.addresses = { {"localhost", "26379"} }; - compose_setup_request(st.cfg); + compose_setup_request(st.cfg, st.tracker, st.setup_req); setup_adapter adapter{st}; // Response to ROLE @@ -315,7 +315,7 @@ void test_sentinel_role_first_element_not_string() st.cfg.sentinel.addresses = { {"localhost", "26379"} }; - compose_setup_request(st.cfg); + compose_setup_request(st.cfg, st.tracker, st.setup_req); setup_adapter adapter{st}; // Response to ROLE diff --git a/test/test_subscription_tracker.cpp b/test/test_subscription_tracker.cpp new file mode 100644 index 000000000..7f5d5aefe --- /dev/null +++ b/test/test_subscription_tracker.cpp @@ -0,0 +1,276 @@ +// +// Copyright (c) 2025-2026 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#include +#include + +#include + +using namespace boost::redis; +using detail::subscription_tracker; + +namespace { + +// State originated by SUBSCRIBE commands, only +void test_subscribe() +{ + subscription_tracker tracker; + request req1, req2, req_output, req_expected; + + // Add some changes to the tracker + req1.subscribe({"channel_a", "channel_b"}); + tracker.commit_changes(req1); + + req2.subscribe({"channel_c"}); + tracker.commit_changes(req2); + + // Check that we generate the correct response + tracker.compose_subscribe_request(req_output); + req_expected.push("SUBSCRIBE", "channel_a", "channel_b", "channel_c"); + BOOST_TEST_EQ(req_output.payload(), req_expected.payload()); +} + +// State originated by PSUBSCRIBE commands, only +void test_psubscribe() +{ + subscription_tracker tracker; + request req1, req2, req_output, req_expected; + + // Add some changes to the tracker + req1.psubscribe({"channel_b*", "channel_c*"}); + tracker.commit_changes(req1); + + req2.psubscribe({"channel_a*"}); + tracker.commit_changes(req2); + + // Check that we generate the correct response + tracker.compose_subscribe_request(req_output); + req_expected.push("PSUBSCRIBE", "channel_a*", "channel_b*", "channel_c*"); // we sort them + BOOST_TEST_EQ(req_output.payload(), req_expected.payload()); +} + +// We can mix SUBSCRIBE and PSUBSCRIBE operations +void test_subscribe_psubscribe() +{ + subscription_tracker tracker; + request req1, req2, req_output, req_expected; + + // Add some changes to the tracker + req1.psubscribe({"channel_a*", "channel_b*"}); + req1.subscribe({"ch1"}); + tracker.commit_changes(req1); + + req2.subscribe({"ch2"}); + req2.psubscribe({"channel_c*"}); + tracker.commit_changes(req2); + + // Check that we generate the correct response + tracker.compose_subscribe_request(req_output); + req_expected.push("SUBSCRIBE", "ch1", "ch2"); + req_expected.push("PSUBSCRIBE", "channel_a*", "channel_b*", "channel_c*"); + BOOST_TEST_EQ(req_output.payload(), req_expected.payload()); +} + +// We can have subscribe and psubscribe commands with the same argument +void test_subscribe_psubscribe_same_arg() +{ + subscription_tracker tracker; + request req, req_output, req_expected; + + req.subscribe({"ch1"}); + req.psubscribe({"ch1"}); + tracker.commit_changes(req); + + tracker.compose_subscribe_request(req_output); + req_expected.push("SUBSCRIBE", "ch1"); + req_expected.push("PSUBSCRIBE", "ch1"); + BOOST_TEST_EQ(req_output.payload(), req_expected.payload()); +} + +// An unsubscribe/punsubscribe balances a matching subscribe +void test_unsubscribe() +{ + subscription_tracker tracker; + request req1, req2, req_output, req_expected; + + // Add some changes to the tracker + req1.subscribe({"ch1", "ch2"}); + req1.psubscribe({"ch1*", "ch2*"}); + tracker.commit_changes(req1); + + // Unsubscribe from some channels + req2.punsubscribe({"ch2*"}); + req2.unsubscribe({"ch1"}); + tracker.commit_changes(req2); + + // Check that we generate the correct response + tracker.compose_subscribe_request(req_output); + req_expected.push("SUBSCRIBE", "ch2"); + req_expected.push("PSUBSCRIBE", "ch1*"); + BOOST_TEST_EQ(req_output.payload(), req_expected.payload()); +} + +// After an unsubscribe, we can subscribe again +void test_resubscribe() +{ + subscription_tracker tracker; + request req, req_output, req_expected; + + // Subscribe to some channels + req.subscribe({"ch1", "ch2"}); + req.psubscribe({"ch1*", "ch2*"}); + tracker.commit_changes(req); + + // Unsubscribe from some channels + req.clear(); + req.punsubscribe({"ch2*"}); + req.unsubscribe({"ch1"}); + tracker.commit_changes(req); + + // Subscribe again + req.clear(); + req.subscribe({"ch1"}); + req.psubscribe({"ch2*"}); + tracker.commit_changes(req); + + // Check that we generate the correct response + tracker.compose_subscribe_request(req_output); + req_expected.push("SUBSCRIBE", "ch1", "ch2"); + req_expected.push("PSUBSCRIBE", "ch1*", "ch2*"); + BOOST_TEST_EQ(req_output.payload(), req_expected.payload()); +} + +// Subscribing twice is not a problem +void test_subscribe_twice() +{ + subscription_tracker tracker; + request req, req_output, req_expected; + + // Subscribe to some channels + req.subscribe({"ch1", "ch2"}); + req.psubscribe({"ch1*", "ch2*"}); + tracker.commit_changes(req); + + // Subscribe to the same channels again + req.clear(); + req.subscribe({"ch2"}); + req.psubscribe({"ch1*"}); + tracker.commit_changes(req); + + // Check that we generate the correct response + tracker.compose_subscribe_request(req_output); + req_expected.push("SUBSCRIBE", "ch1", "ch2"); + req_expected.push("PSUBSCRIBE", "ch1*", "ch2*"); + BOOST_TEST_EQ(req_output.payload(), req_expected.payload()); +} + +// Unsubscribing from channels we haven't subscribed to is not a problem +void test_lone_unsubscribe() +{ + subscription_tracker tracker; + request req, req_output, req_expected; + + // Subscribe to some channels + req.subscribe({"ch1", "ch2"}); + req.psubscribe({"ch1*", "ch2*"}); + tracker.commit_changes(req); + + // Unsubscribe from channels we haven't subscribed to + req.clear(); + req.unsubscribe({"other"}); + req.punsubscribe({"other*"}); + tracker.commit_changes(req); + + // Check that we generate the correct response + tracker.compose_subscribe_request(req_output); + req_expected.push("SUBSCRIBE", "ch1", "ch2"); + req_expected.push("PSUBSCRIBE", "ch1*", "ch2*"); + BOOST_TEST_EQ(req_output.payload(), req_expected.payload()); +} + +// A state with no changes is not a problem +void test_empty() +{ + subscription_tracker tracker; + request req_output; + + tracker.compose_subscribe_request(req_output); + BOOST_TEST_EQ(req_output.payload(), ""); +} + +// If the output request is not empty, the commands are added to it, rather than replaced +void test_output_request_not_empty() +{ + subscription_tracker tracker; + request req, req_output, req_expected; + + // Subscribe to some channels + req.subscribe({"ch1", "ch2"}); + req.psubscribe({"ch1*", "ch2*"}); + tracker.commit_changes(req); + + // Compose the output request + req_output.push("PING", "hello"); + tracker.compose_subscribe_request(req_output); + + // Check that we generate the correct response + req_expected.push("PING", "hello"); + req_expected.push("SUBSCRIBE", "ch1", "ch2"); + req_expected.push("PSUBSCRIBE", "ch1*", "ch2*"); + BOOST_TEST_EQ(req_output.payload(), req_expected.payload()); +} + +// Clear removes everything from the state +void test_clear() +{ + subscription_tracker tracker; + request req, req_output, req_expected; + + // Subscribe to some channels + req.subscribe({"ch1", "ch2"}); + req.psubscribe({"ch1*", "ch2*"}); + tracker.commit_changes(req); + + // Clear + tracker.clear(); + + // Nothing should be generated + tracker.compose_subscribe_request(req_output); + BOOST_TEST_EQ(req_output.payload(), ""); + + // We can reuse the tracker by now committing some more changes + req.clear(); + req.subscribe({"ch5"}); + req.psubscribe({"ch6*"}); + tracker.commit_changes(req); + + // Check that we generate the correct response + tracker.compose_subscribe_request(req_output); + req_expected.push("SUBSCRIBE", "ch5"); + req_expected.push("PSUBSCRIBE", "ch6*"); + BOOST_TEST_EQ(req_output.payload(), req_expected.payload()); +} + +} // namespace + +int main() +{ + test_subscribe(); + test_psubscribe(); + test_subscribe_psubscribe(); + test_subscribe_psubscribe_same_arg(); + test_unsubscribe(); + test_resubscribe(); + test_subscribe_twice(); + test_lone_unsubscribe(); + test_empty(); + test_output_request_not_empty(); + test_clear(); + + return boost::report_errors(); +} From c11a5194d81e16547a5c390ba2e539e08173bfa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anarthal=20=28Rub=C3=A9n=20P=C3=A9rez=29?= <34971811+anarthal@users.noreply.github.com> Date: Wed, 14 Jan 2026 21:54:34 +0100 Subject: [PATCH 27/37] Reworks test_conn_push and test_conn_push2 (#380) Adds a test to verify that async_receive (v1) is cancelled on reconnection Adds a test to verify that a reconnection is triggered when the receive adapter generates an error Makes the unsubscribe and push adapter tests live only in test_conn_push2, since they test functionality common to async_receive and async_receive2 Migrates all tests to lightweight_test Entails no functional change --- test/test_conn_push.cpp | 524 +++++++++++++++++++-------------------- test/test_conn_push2.cpp | 473 +++++++++++++++++++---------------- 2 files changed, 521 insertions(+), 476 deletions(-) diff --git a/test/test_conn_push.cpp b/test/test_conn_push.cpp index c3f1c339f..3b2645522 100644 --- a/test/test_conn_push.cpp +++ b/test/test_conn_push.cpp @@ -5,255 +5,306 @@ */ #include +#include #include #include +#include +#include #include +#include #include -#include - -#include - -#define BOOST_TEST_MODULE conn_push -#include +#include +#include #include "common.hpp" #include -#include +#include namespace net = boost::asio; -namespace redis = boost::redis; - -using boost::redis::operation; -using boost::redis::connection; -using boost::system::error_code; -using boost::redis::request; -using boost::redis::response; -using boost::redis::ignore; -using boost::redis::ignore_t; -using boost::system::error_code; -using boost::redis::logger; +using namespace boost::redis; using namespace std::chrono_literals; +using boost::system::error_code; + +// Focuses on the deprecated async_receive and receive +// functions. test_conn_push2 covers the newer receive functionality. namespace { -BOOST_AUTO_TEST_CASE(receives_push_waiting_resps) +// async_receive is outstanding when a push is received +void test_async_receive_waiting_for_push() { + resp3::flat_tree resp; + net::io_context ioc; + connection conn{ioc}; + conn.set_receive_response(resp); + request req1; - req1.push("HELLO", 3); req1.push("PING", "Message1"); + req1.push("SUBSCRIBE", "test_async_receive_waiting_for_push"); request req2; - req2.push("SUBSCRIBE", "channel"); - - request req3; - req3.push("PING", "Message2"); - req3.push("QUIT"); - - net::io_context ioc; - - auto conn = std::make_shared(ioc); - - bool push_received = false, c1_called = false, c2_called = false, c3_called = false; + req2.push("PING", "Message2"); - auto c3 = [&](error_code ec, std::size_t) { - c3_called = true; - std::cout << "c3: " << ec.message() << std::endl; - }; - - auto c2 = [&, conn](error_code ec, std::size_t) { - c2_called = true; - BOOST_TEST(ec == error_code()); - conn->async_exec(req3, ignore, c3); - }; + bool run_finished = false, push_received = false, exec1_finished = false, exec2_finished = false; - auto c1 = [&, conn](error_code ec, std::size_t) { - c1_called = true; - BOOST_TEST(ec == error_code()); - conn->async_exec(req2, ignore, c2); + auto on_exec2 = [&](error_code ec2, std::size_t) { + BOOST_TEST_EQ(ec2, error_code()); + exec2_finished = true; + conn.cancel(); }; - conn->async_exec(req1, ignore, c1); - - run(conn, make_test_config(), {}); + conn.async_exec(req1, ignore, [&](error_code ec, std::size_t) { + BOOST_TEST_EQ(ec, error_code()); + exec1_finished = true; + }); - conn->async_receive([&, conn](error_code ec, std::size_t) { - std::cout << "async_receive" << std::endl; - BOOST_TEST(ec == error_code()); + conn.async_receive([&](error_code ec, std::size_t) { + BOOST_TEST_EQ(ec, error_code()); + BOOST_TEST_EQ(resp.get_total_msgs(), 1u); push_received = true; - conn->cancel(); + conn.async_exec(req2, ignore, on_exec2); + }); + + conn.async_run(make_test_config(), [&](error_code ec) { + BOOST_TEST_EQ(ec, net::error::operation_aborted); + run_finished = true; }); ioc.run_for(test_timeout); BOOST_TEST(push_received); - BOOST_TEST(c1_called); - BOOST_TEST(c2_called); - BOOST_TEST(c3_called); + BOOST_TEST(exec1_finished); + BOOST_TEST(exec2_finished); + BOOST_TEST(run_finished); } -BOOST_AUTO_TEST_CASE(push_received1) +// A push is already available when async_receive is called +void test_async_receive_push_available() { net::io_context ioc; - auto conn = std::make_shared(ioc); + connection conn{ioc}; + resp3::flat_tree resp; + conn.set_receive_response(resp); - // Trick: Uses SUBSCRIBE because this command has no response or - // better said, its response is a server push, which is what we - // want to test. We send two because we want to test both - // async_receive and receive. + // SUBSCRIBE doesn't have a response, but causes a push to be delivered. + // Add a PING so the overall request has a response. + // This ensures that when async_exec completes, the push has been delivered request req; - req.push("SUBSCRIBE", "channel1"); - req.push("SUBSCRIBE", "channel2"); + req.push("SUBSCRIBE", "test_async_receive_push_available"); + req.push("PING", "message"); - bool push_received = false, exec_finished = false; - - conn->async_exec(req, ignore, [&, conn](error_code ec, std::size_t) { - exec_finished = true; - std::cout << "async_exec" << std::endl; - BOOST_TEST(ec == error_code()); - }); + bool push_received = false, exec_finished = false, run_finished = false; - conn->async_receive([&, conn](error_code ec, std::size_t) { + auto on_receive = [&](error_code ec, std::size_t) { push_received = true; - std::cout << "(1) async_receive" << std::endl; - - BOOST_TEST(ec == error_code()); - - // Receives the second push synchronously. - error_code ec2; - std::size_t res = 0; - res = conn->receive(ec2); - BOOST_TEST(!ec2); - BOOST_TEST(res != std::size_t(0)); + BOOST_TEST_EQ(ec, error_code()); + BOOST_TEST_EQ(resp.get_total_msgs(), 1u); + conn.cancel(); + }; - // Tries to receive a third push synchronously. - ec2 = {}; - res = conn->receive(ec2); - BOOST_CHECK_EQUAL( - ec2, - boost::redis::make_error_code(boost::redis::error::sync_receive_push_failed)); + conn.async_exec(req, ignore, [&](error_code ec, std::size_t) { + exec_finished = true; + BOOST_TEST_EQ(ec, error_code()); + conn.async_receive(on_receive); + }); - conn->cancel(); + conn.async_run(make_test_config(), [&](error_code ec) { + run_finished = true; + BOOST_TEST_EQ(ec, net::error::operation_aborted); }); - run(conn); ioc.run_for(test_timeout); BOOST_TEST(exec_finished); BOOST_TEST(push_received); + BOOST_TEST(run_finished); } -BOOST_AUTO_TEST_CASE(push_filtered_out) +// Synchronous receive can be used to try to read a message +void test_sync_receive() { net::io_context ioc; - auto conn = std::make_shared(ioc); + connection conn{ioc}; + resp3::flat_tree resp; + conn.set_receive_response(resp); + // Subscribing to 2 channels causes 2 pushes to be delivered. + // Adding a PING guarantees that after exec finishes, the push has been read request req; - req.push("HELLO", 3); - req.push("PING"); - req.push("SUBSCRIBE", "channel"); - req.push("QUIT"); - - response resp; + req.push("SUBSCRIBE", "test_sync_receive_channel1"); + req.push("SUBSCRIBE", "test_sync_receive_channel2"); + req.push("PING", "message"); - bool exec_finished = false, push_received = false; + bool exec_finished = false, run_finished = false; - conn->async_exec(req, resp, [conn, &exec_finished](error_code ec, std::size_t) { + conn.async_exec(req, ignore, [&](error_code ec, std::size_t) { exec_finished = true; - BOOST_TEST(ec == error_code()); + BOOST_TEST_EQ(ec, error_code()); + + // At this point, the receive response contains all the pushes + BOOST_TEST_EQ(resp.get_total_msgs(), 2u); + + // Receive the 1st push synchronously + std::size_t push_bytes = conn.receive(ec); + BOOST_TEST_EQ(ec, error_code()); + BOOST_TEST_GT(push_bytes, 0u); + + // Receive the 2nd push synchronously + push_bytes = conn.receive(ec); + BOOST_TEST_EQ(ec, error_code()); + BOOST_TEST_GT(push_bytes, 0u); + + // There are no more pushes. Trying to receive one more fails + push_bytes = conn.receive(ec); + BOOST_TEST_EQ(ec, error::sync_receive_push_failed); + BOOST_TEST_EQ(push_bytes, 0u); + + conn.cancel(); }); - conn->async_receive([&, conn](error_code ec, std::size_t) { - push_received = true; - BOOST_TEST(ec == error_code()); - conn->cancel(operation::reconnection); + conn.async_run(make_test_config(), [&](error_code ec) { + BOOST_TEST_EQ(ec, net::error::operation_aborted); + run_finished = true; }); - run(conn); + // Trying to receive a push before one is received fails + error_code ec; + std::size_t push_bytes = conn.receive(ec); + BOOST_TEST_EQ(ec, error::sync_receive_push_failed); + BOOST_TEST_EQ(push_bytes, 0u); ioc.run_for(test_timeout); - BOOST_TEST(exec_finished); - BOOST_TEST(push_received); - BOOST_CHECK_EQUAL(std::get<1>(resp).value(), "PONG"); - BOOST_CHECK_EQUAL(std::get<2>(resp).value(), "OK"); + BOOST_TEST(exec_finished); + BOOST_TEST(run_finished); } -struct response_error_tag { }; -response_error_tag error_tag_obj; +// async_receive is cancelled every time a reconnection happens, +// so we can re-establish subscriptions +struct test_async_receive_cancelled_on_reconnection_impl { + net::io_context ioc; + connection conn{ioc}; + resp3::flat_tree resp{}; + request req_subscribe{}, req_quit{}; + bool receive_finished = false, quit_finished = false; -struct response_error_adapter { - void on_init() { } - void on_done() { } + // Subscribe to a channel. This will cause a push to be received + void start_subscribe1() + { + conn.async_exec(req_subscribe, ignore, [this](error_code ec, std::size_t) { + BOOST_TEST_EQ(ec, error_code()); + start_receive1(); + }); + } - void on_node( - boost::redis::resp3::basic_node const&, - boost::system::error_code& ec) + // Receive the push triggered by the subscribe + void start_receive1() { - ec = boost::redis::error::incompatible_size; + conn.async_receive([this](error_code ec, std::size_t) { + BOOST_TEST_EQ(ec, error_code()); + BOOST_TEST_EQ(resp.get_total_msgs(), 1u); + resp.clear(); + + // In parallel, trigger a reconnection and start a receive operation + start_receive_reconnection(); + start_quit(); + }); } -}; -auto boost_redis_adapt(response_error_tag&) { return response_error_adapter{}; } + // The next receive operation will be cancelled by the reconnection + void start_receive_reconnection() + { + conn.async_receive([this](error_code ec, std::size_t) { + BOOST_TEST_EQ(ec, net::experimental::channel_errc::channel_cancelled); + BOOST_TEST_EQ(resp.get_total_msgs(), 0u); + start_subscribe2(); + }); + } -BOOST_AUTO_TEST_CASE(test_push_adapter) -{ - net::io_context ioc; - auto conn = std::make_shared(ioc); + // Trigger a reconnection. This is a "leaf" operation + void start_quit() + { + conn.async_exec(req_quit, ignore, [this](error_code, std::size_t) { + quit_finished = true; + }); + } - request req; - req.push("HELLO", 3); - req.push("PING"); - req.push("SUBSCRIBE", "channel"); - req.push("PING"); + // Resubscribe after the reconnection + void start_subscribe2() + { + conn.async_exec(req_subscribe, ignore, [this](error_code ec, std::size_t) { + BOOST_TEST_EQ(ec, error_code()); + start_receive2(); + }); + } - conn->set_receive_response(error_tag_obj); + // Receive the push triggered by the 2nd subscribe + void start_receive2() + { + conn.async_receive([this](error_code ec, std::size_t) { + BOOST_TEST_EQ(ec, error_code()); + BOOST_TEST_EQ(resp.get_total_msgs(), 1u); + receive_finished = true; + conn.cancel(); + }); + } - bool push_received = false, exec_finished = false, run_finished = false; + void run() + { + req_subscribe.push("SUBSCRIBE", "test_async_receive_cancelled_on_reconnection"); + req_subscribe.push("PING"); - conn->async_receive([&, conn](error_code ec, std::size_t) { - BOOST_CHECK_EQUAL(ec, boost::asio::experimental::error::channel_cancelled); - push_received = true; - }); + req_quit.push("QUIT"); - conn->async_exec(req, ignore, [&exec_finished](error_code ec, std::size_t) { - BOOST_CHECK_EQUAL(ec, boost::system::errc::errc_t::operation_canceled); - exec_finished = true; - }); + conn.set_receive_response(resp); - auto cfg = make_test_config(); - cfg.reconnect_wait_interval = 0s; - conn->async_run(cfg, [&run_finished](error_code ec) { - BOOST_CHECK_EQUAL(ec, redis::error::incompatible_size); - run_finished = true; - }); + bool run_finished = false; - ioc.run_for(test_timeout); - BOOST_TEST(push_received); - BOOST_TEST(exec_finished); - BOOST_TEST(run_finished); + start_subscribe1(); - // TODO: Reset the ioc reconnect and send a quit to ensure - // reconnection is possible after an error. -} + auto cfg = make_test_config(); + cfg.reconnect_wait_interval = 50ms; // make the test run faster + conn.async_run(cfg, [&](error_code ec) { + run_finished = true; + BOOST_TEST_EQ(ec, net::error::operation_aborted); + }); + + ioc.run_for(test_timeout); -void launch_push_consumer(std::shared_ptr conn) + BOOST_TEST(run_finished); + BOOST_TEST(receive_finished); + BOOST_TEST(quit_finished); + } +}; + +void test_async_receive_cancelled_on_reconnection() { - conn->async_receive([conn](error_code ec, std::size_t) { - if (ec) { - BOOST_TEST(ec == net::experimental::error::channel_cancelled); - return; - } - launch_push_consumer(conn); - }); + test_async_receive_cancelled_on_reconnection_impl{}.run(); } -BOOST_AUTO_TEST_CASE(many_subscribers) +// After an async_receive operation finishes, another one can be issued +void test_consecutive_receives() { - request req0; - req0.get_config().cancel_on_connection_lost = false; - req0.push("HELLO", 3); + net::io_context ioc; + connection conn{ioc}; + resp3::flat_tree resp; + bool push_consumer_finished{false}; + + std::function launch_push_consumer = [&]() { + conn.async_receive([&](error_code ec, std::size_t) { + if (ec) { + BOOST_TEST_EQ(ec, net::experimental::error::channel_cancelled); + push_consumer_finished = true; + resp.clear(); + return; + } + launch_push_consumer(); + }); + }; + + conn.set_receive_response(resp); request req1; req1.get_config().cancel_on_connection_lost = false; @@ -263,139 +314,74 @@ BOOST_AUTO_TEST_CASE(many_subscribers) req2.get_config().cancel_on_connection_lost = false; req2.push("SUBSCRIBE", "channel"); - request req3; - req3.get_config().cancel_on_connection_lost = false; - req3.push("QUIT"); + bool exec_finished = false, run_finished = false; - net::io_context ioc; - auto conn = std::make_shared(ioc); - - bool finished = false; - - auto c11 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->cancel(operation::reconnection); - finished = true; - }; auto c10 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req3, ignore, c11); + BOOST_TEST_EQ(ec, error_code()); + exec_finished = true; + conn.cancel(); }; auto c9 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req2, ignore, c10); + BOOST_TEST_EQ(ec, error_code()); + conn.async_exec(req2, ignore, c10); }; auto c8 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req1, ignore, c9); + BOOST_TEST_EQ(ec, error_code()); + conn.async_exec(req1, ignore, c9); }; auto c7 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req2, ignore, c8); + BOOST_TEST_EQ(ec, error_code()); + conn.async_exec(req2, ignore, c8); }; auto c6 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req2, ignore, c7); + BOOST_TEST_EQ(ec, error_code()); + conn.async_exec(req2, ignore, c7); }; auto c5 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req1, ignore, c6); + BOOST_TEST_EQ(ec, error_code()); + conn.async_exec(req1, ignore, c6); }; auto c4 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req2, ignore, c5); + BOOST_TEST_EQ(ec, error_code()); + conn.async_exec(req2, ignore, c5); }; auto c3 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req1, ignore, c4); + BOOST_TEST_EQ(ec, error_code()); + conn.async_exec(req1, ignore, c4); }; auto c2 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req2, ignore, c3); + BOOST_TEST_EQ(ec, error_code()); + conn.async_exec(req2, ignore, c3); }; auto c1 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req2, ignore, c2); + BOOST_TEST_EQ(ec, error_code()); + conn.async_exec(req2, ignore, c2); }; - auto c0 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req1, ignore, c1); - }; - - conn->async_exec(req0, ignore, c0); - launch_push_consumer(conn); - - run(conn, make_test_config(), {}); - - ioc.run_for(test_timeout); - BOOST_TEST(finished); -} - -BOOST_AUTO_TEST_CASE(test_unsubscribe) -{ - net::io_context ioc; - connection conn{ioc}; - - // Subscribe to 3 channels and 2 patterns. Use CLIENT INFO to verify this took effect - request req_subscribe; - req_subscribe.push("SUBSCRIBE", "ch1", "ch2", "ch3"); - req_subscribe.push("PSUBSCRIBE", "ch1*", "ch2*"); - req_subscribe.push("CLIENT", "INFO"); - - // Then, unsubscribe from some of them, and verify again - request req_unsubscribe; - req_unsubscribe.push("UNSUBSCRIBE", "ch1"); - req_unsubscribe.push("PUNSUBSCRIBE", "ch2*"); - req_unsubscribe.push("CLIENT", "INFO"); - - // Finally, ping to verify that the connection is still usable - request req_ping; - req_ping.push("PING", "test_unsubscribe"); - - response resp_subscribe, resp_unsubscribe, resp_ping; - bool subscribe_finished = false, unsubscribe_finished = false, ping_finished = false, - run_finished = false; + conn.async_exec(req1, ignore, c1); + launch_push_consumer(); - auto on_ping = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - ping_finished = true; - BOOST_TEST(std::get<0>(resp_ping).has_value()); - BOOST_TEST(std::get<0>(resp_ping).value() == "test_unsubscribe"); - conn.cancel(); - }; - - auto on_unsubscribe = [&](error_code ec, std::size_t) { - unsubscribe_finished = true; - BOOST_TEST(ec == error_code()); - BOOST_TEST(std::get<0>(resp_unsubscribe).has_value()); - BOOST_TEST(find_client_info(std::get<0>(resp_unsubscribe).value(), "sub") == "2"); - BOOST_TEST(find_client_info(std::get<0>(resp_unsubscribe).value(), "psub") == "1"); - conn.async_exec(req_ping, resp_ping, on_ping); - }; - - auto on_subscribe = [&](error_code ec, std::size_t) { - subscribe_finished = true; - BOOST_TEST(ec == error_code()); - BOOST_TEST(std::get<0>(resp_subscribe).has_value()); - BOOST_TEST(find_client_info(std::get<0>(resp_subscribe).value(), "sub") == "3"); - BOOST_TEST(find_client_info(std::get<0>(resp_subscribe).value(), "psub") == "2"); - conn.async_exec(req_unsubscribe, resp_unsubscribe, on_unsubscribe); - }; - - conn.async_exec(req_subscribe, resp_subscribe, on_subscribe); - - conn.async_run(make_test_config(), [&run_finished](error_code ec) { - BOOST_TEST(ec == net::error::operation_aborted); + conn.async_run(make_test_config(), [&](error_code ec) { run_finished = true; + BOOST_TEST_EQ(ec, net::error::operation_aborted); }); ioc.run_for(test_timeout); - BOOST_TEST(subscribe_finished); - BOOST_TEST(unsubscribe_finished); - BOOST_TEST(ping_finished); + BOOST_TEST(exec_finished); BOOST_TEST(run_finished); -} + BOOST_TEST(push_consumer_finished); +}; } // namespace + +int main() +{ + test_async_receive_waiting_for_push(); + test_async_receive_push_available(); + test_sync_receive(); + test_async_receive_cancelled_on_reconnection(); + test_consecutive_receives(); + + return boost::report_errors(); +} diff --git a/test/test_conn_push2.cpp b/test/test_conn_push2.cpp index 9cf638e92..5c260f8c3 100644 --- a/test/test_conn_push2.cpp +++ b/test/test_conn_push2.cpp @@ -10,218 +10,209 @@ #include #include +#include #include -#include - -#include -#include -#include - -#define BOOST_TEST_MODULE conn_push -#include +#include #include "common.hpp" #include -#include +#include +#include +#include +#include +#include namespace net = boost::asio; -namespace redis = boost::redis; - -using boost::redis::operation; -using boost::redis::connection; -using boost::system::error_code; -using boost::redis::request; -using boost::redis::response; -using boost::redis::resp3::flat_tree; -using boost::redis::ignore; -using boost::redis::ignore_t; -using boost::system::error_code; -using boost::redis::logger; -using boost::redis::resp3::node_view; -using boost::redis::resp3::type; +using namespace boost::redis; using namespace std::chrono_literals; +using boost::system::error_code; +using resp3::flat_tree; +using resp3::node_view; +using resp3::type; + +// Covers all receive functionality except for the deprecated +// async_receive and receive functions. namespace { -BOOST_AUTO_TEST_CASE(receives_push_waiting_resps) +// async_receive2 is outstanding when a push is received +void test_async_receive2_waiting_for_push() { + resp3::flat_tree resp; + net::io_context ioc; + connection conn{ioc}; + conn.set_receive_response(resp); + request req1; - req1.push("HELLO", 3); req1.push("PING", "Message1"); + req1.push("SUBSCRIBE", "test_async_receive_waiting_for_push"); request req2; - req2.push("SUBSCRIBE", "channel"); - - request req3; - req3.push("PING", "Message2"); - req3.push("QUIT"); - - net::io_context ioc; - - auto conn = std::make_shared(ioc); - - bool push_received = false, c1_called = false, c2_called = false, c3_called = false; + req2.push("PING", "Message2"); - auto c3 = [&](error_code ec, std::size_t) { - c3_called = true; - std::cout << "c3: " << ec.message() << std::endl; - }; - - auto c2 = [&, conn](error_code ec, std::size_t) { - c2_called = true; - BOOST_TEST(ec == error_code()); - conn->async_exec(req3, ignore, c3); - }; + bool run_finished = false, push_received = false, exec1_finished = false, exec2_finished = false; - auto c1 = [&, conn](error_code ec, std::size_t) { - c1_called = true; - BOOST_TEST(ec == error_code()); - conn->async_exec(req2, ignore, c2); + auto on_exec2 = [&](error_code ec2, std::size_t) { + BOOST_TEST_EQ(ec2, error_code()); + exec2_finished = true; + conn.cancel(); }; - conn->async_exec(req1, ignore, c1); - - run(conn, make_test_config(), {}); + conn.async_exec(req1, ignore, [&](error_code ec, std::size_t) { + BOOST_TEST_EQ(ec, error_code()); + exec1_finished = true; + }); - conn->async_receive2([&, conn](error_code ec) { - std::cout << "async_receive2" << std::endl; - BOOST_TEST(ec == error_code()); + conn.async_receive2([&](error_code ec) { + BOOST_TEST_EQ(ec, error_code()); + BOOST_TEST_EQ(resp.get_total_msgs(), 1u); push_received = true; - conn->cancel(); + conn.async_exec(req2, ignore, on_exec2); + }); + + conn.async_run(make_test_config(), [&](error_code ec) { + BOOST_TEST_EQ(ec, net::error::operation_aborted); + run_finished = true; }); ioc.run_for(test_timeout); BOOST_TEST(push_received); - BOOST_TEST(c1_called); - BOOST_TEST(c2_called); - BOOST_TEST(c3_called); + BOOST_TEST(exec1_finished); + BOOST_TEST(exec2_finished); + BOOST_TEST(run_finished); } -BOOST_AUTO_TEST_CASE(push_received1) +// A push is already available when async_receive2 is called +void test_async_receive2_push_available() { net::io_context ioc; - auto conn = std::make_shared(ioc); - - flat_tree resp; - conn->set_receive_response(resp); + connection conn{ioc}; + resp3::flat_tree resp; + conn.set_receive_response(resp); - // Trick: Uses SUBSCRIBE because this command has no response or - // better said, its response is a server push, which is what we - // want to test. + // SUBSCRIBE doesn't have a response, but causes a push to be delivered. + // Add a PING so the overall request has a response. + // This ensures that when async_exec completes, the push has been delivered request req; - req.push("SUBSCRIBE", "channel1"); - req.push("SUBSCRIBE", "channel2"); - - bool push_received = false, exec_finished = false; + req.push("SUBSCRIBE", "test_async_receive_push_available"); + req.push("PING", "message"); - conn->async_exec(req, ignore, [&, conn](error_code ec, std::size_t) { - exec_finished = true; - std::cout << "async_exec" << std::endl; - BOOST_TEST(ec == error_code()); - }); + bool push_received = false, exec_finished = false, run_finished = false; - conn->async_receive2([&, conn](error_code ec) { + auto on_receive = [&](error_code ec, std::size_t) { push_received = true; - std::cout << "async_receive2" << std::endl; - - BOOST_TEST(ec == error_code()); + BOOST_TEST_EQ(ec, error_code()); + BOOST_TEST_EQ(resp.get_total_msgs(), 1u); + conn.cancel(); + }; - BOOST_CHECK_EQUAL(resp.get_total_msgs(), 2u); + conn.async_exec(req, ignore, [&](error_code ec, std::size_t) { + exec_finished = true; + BOOST_TEST_EQ(ec, error_code()); + conn.async_receive(on_receive); + }); - conn->cancel(); + conn.async_run(make_test_config(), [&](error_code ec) { + run_finished = true; + BOOST_TEST_EQ(ec, net::error::operation_aborted); }); - run(conn); ioc.run_for(test_timeout); BOOST_TEST(exec_finished); BOOST_TEST(push_received); + BOOST_TEST(run_finished); } -BOOST_AUTO_TEST_CASE(push_filtered_out) +// A push may be interleaved between regular responses. +// It is handed to the receive adapter (filtered out). +void test_exec_push_interleaved() { net::io_context ioc; - auto conn = std::make_shared(ioc); + connection conn{ioc}; + resp3::flat_tree receive_resp; + conn.set_receive_response(receive_resp); request req; - req.push("HELLO", 3); - req.push("PING"); - req.push("SUBSCRIBE", "channel"); - req.push("QUIT"); + req.push("PING", "msg1"); + req.push("SUBSCRIBE", "test_exec_push_interleaved"); + req.push("PING", "msg2"); - response resp; + response resp; - bool exec_finished = false, push_received = false; + bool exec_finished = false, push_received = false, run_finished = false; - conn->async_exec(req, resp, [conn, &exec_finished](error_code ec, std::size_t) { + conn.async_exec(req, resp, [&](error_code ec, std::size_t) { exec_finished = true; - BOOST_TEST(ec == error_code()); + BOOST_TEST_EQ(ec, error_code()); + BOOST_TEST_EQ(std::get<0>(resp).value(), "msg1"); + BOOST_TEST_EQ(std::get<1>(resp).value(), "msg2"); + conn.cancel(); }); - conn->async_receive2([&, conn](error_code ec) { + conn.async_receive2([&](error_code ec) { push_received = true; - BOOST_TEST(ec == error_code()); - conn->cancel(operation::reconnection); + BOOST_TEST_EQ(ec, error_code()); + BOOST_TEST_EQ(receive_resp.get_total_msgs(), 1u); }); - run(conn); + conn.async_run(make_test_config(), [&](error_code ec) { + run_finished = true; + BOOST_TEST_EQ(ec, net::error::operation_aborted); + }); ioc.run_for(test_timeout); + BOOST_TEST(exec_finished); BOOST_TEST(push_received); - - BOOST_CHECK_EQUAL(std::get<1>(resp).value(), "PONG"); - BOOST_CHECK_EQUAL(std::get<2>(resp).value(), "OK"); + BOOST_TEST(run_finished); } +// An adapter that always errors struct response_error_tag { }; response_error_tag error_tag_obj; struct response_error_adapter { void on_init() { } void on_done() { } - - void on_node( - boost::redis::resp3::basic_node const&, - boost::system::error_code& ec) - { - ec = boost::redis::error::incompatible_size; - } + void on_node(node_view const&, error_code& ec) { ec = error::incompatible_size; } }; auto boost_redis_adapt(response_error_tag&) { return response_error_adapter{}; } -BOOST_AUTO_TEST_CASE(test_push_adapter) +// If the push adapter returns an error, the connection is torn down +void test_push_adapter_error() { net::io_context ioc; - auto conn = std::make_shared(ioc); + connection conn{ioc}; + conn.set_receive_response(error_tag_obj); request req; - req.push("HELLO", 3); req.push("PING"); req.push("SUBSCRIBE", "channel"); req.push("PING"); - conn->set_receive_response(error_tag_obj); - bool push_received = false, exec_finished = false, run_finished = false; - conn->async_receive2([&, conn](error_code ec) { - BOOST_CHECK_EQUAL(ec, boost::asio::experimental::error::channel_cancelled); + // async_receive2 is cancelled every reconnection cycle + conn.async_receive2([&](error_code ec) { + BOOST_TEST_EQ(ec, net::experimental::error::channel_cancelled); push_received = true; }); - conn->async_exec(req, ignore, [&exec_finished](error_code ec, std::size_t) { - BOOST_CHECK_EQUAL(ec, boost::system::errc::errc_t::operation_canceled); + // The request is cancelled because the PING response isn't processed + // by the time the error is generated + conn.async_exec(req, ignore, [&exec_finished](error_code ec, std::size_t) { + BOOST_TEST_EQ(ec, net::error::operation_aborted); exec_finished = true; }); auto cfg = make_test_config(); - cfg.reconnect_wait_interval = 0s; - conn->async_run(cfg, [&run_finished](error_code ec) { - BOOST_CHECK_EQUAL(ec, redis::error::incompatible_size); + cfg.reconnect_wait_interval = 0s; // so we can validate the generated error + conn.async_run(cfg, [&run_finished](error_code ec) { + BOOST_TEST_EQ(ec, error::incompatible_size); run_finished = true; }); @@ -229,27 +220,83 @@ BOOST_AUTO_TEST_CASE(test_push_adapter) BOOST_TEST(push_received); BOOST_TEST(exec_finished); BOOST_TEST(run_finished); - - // TODO: Reset the ioc reconnect and send a quit to ensure - // reconnection is possible after an error. } -void launch_push_consumer(std::shared_ptr conn) +// A push response error triggers a reconnection +void test_push_adapter_error_reconnection() { - conn->async_receive2([conn](error_code ec) { - if (ec) { - BOOST_TEST(ec == net::experimental::error::channel_cancelled); - return; - } - launch_push_consumer(conn); + net::io_context ioc; + connection conn{ioc}; + conn.set_receive_response(error_tag_obj); + + request req; + req.push("PING"); + req.push("SUBSCRIBE", "channel"); + req.push("PING"); + + request req2; + req2.push("PING", "msg2"); + req2.get_config().cancel_if_unresponded = false; + + response resp; + + bool push_received = false, exec_finished = false, run_finished = false; + + // async_receive2 is cancelled every reconnection cycle + conn.async_receive2([&](error_code ec) { + BOOST_TEST_EQ(ec, net::experimental::error::channel_cancelled); + push_received = true; + }); + + auto on_exec2 = [&](error_code ec, std::size_t) { + BOOST_TEST_EQ(ec, error_code()); + BOOST_TEST_EQ(std::get<0>(resp).value(), "msg2"); + exec_finished = true; + conn.cancel(); + }; + + // The request is cancelled because the PING response isn't processed + // by the time the error is generated + conn.async_exec(req, ignore, [&](error_code ec, std::size_t) { + BOOST_TEST_EQ(ec, net::error::operation_aborted); + conn.async_exec(req2, resp, on_exec2); + }); + + auto cfg = make_test_config(); + cfg.reconnect_wait_interval = 50ms; // make the test run faster + conn.async_run(cfg, [&run_finished](error_code ec) { + BOOST_TEST_EQ(ec, net::error::operation_aborted); + run_finished = true; }); + + ioc.run_for(test_timeout); + + BOOST_TEST(push_received); + BOOST_TEST(exec_finished); + BOOST_TEST(run_finished); } -BOOST_AUTO_TEST_CASE(many_subscribers) +// After an async_receive2 operation finishes, another one can be issued +void test_consecutive_receives() { - request req0; - req0.get_config().cancel_on_connection_lost = false; - req0.push("HELLO", 3); + net::io_context ioc; + connection conn{ioc}; + resp3::flat_tree resp; + bool push_consumer_finished{false}; + + std::function launch_push_consumer = [&]() { + conn.async_receive2([&](error_code ec) { + if (ec) { + BOOST_TEST_EQ(ec, net::experimental::error::channel_cancelled); + push_consumer_finished = true; + resp.clear(); + return; + } + launch_push_consumer(); + }); + }; + + conn.set_receive_response(resp); request req1; req1.get_config().cancel_on_connection_lost = false; @@ -259,75 +306,67 @@ BOOST_AUTO_TEST_CASE(many_subscribers) req2.get_config().cancel_on_connection_lost = false; req2.push("SUBSCRIBE", "channel"); - request req3; - req3.get_config().cancel_on_connection_lost = false; - req3.push("QUIT"); - - net::io_context ioc; - auto conn = std::make_shared(ioc); - - bool finished = false; + bool exec_finished = false, run_finished = false; - auto c11 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->cancel(operation::reconnection); - finished = true; - }; auto c10 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req3, ignore, c11); + BOOST_TEST_EQ(ec, error_code()); + exec_finished = true; + conn.cancel(); }; auto c9 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req2, ignore, c10); + BOOST_TEST_EQ(ec, error_code()); + conn.async_exec(req2, ignore, c10); }; auto c8 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req1, ignore, c9); + BOOST_TEST_EQ(ec, error_code()); + conn.async_exec(req1, ignore, c9); }; auto c7 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req2, ignore, c8); + BOOST_TEST_EQ(ec, error_code()); + conn.async_exec(req2, ignore, c8); }; auto c6 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req2, ignore, c7); + BOOST_TEST_EQ(ec, error_code()); + conn.async_exec(req2, ignore, c7); }; auto c5 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req1, ignore, c6); + BOOST_TEST_EQ(ec, error_code()); + conn.async_exec(req1, ignore, c6); }; auto c4 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req2, ignore, c5); + BOOST_TEST_EQ(ec, error_code()); + conn.async_exec(req2, ignore, c5); }; auto c3 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req1, ignore, c4); + BOOST_TEST_EQ(ec, error_code()); + conn.async_exec(req1, ignore, c4); }; auto c2 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req2, ignore, c3); + BOOST_TEST_EQ(ec, error_code()); + conn.async_exec(req2, ignore, c3); }; auto c1 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req2, ignore, c2); - }; - auto c0 = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); - conn->async_exec(req1, ignore, c1); + BOOST_TEST_EQ(ec, error_code()); + conn.async_exec(req2, ignore, c2); }; - conn->async_exec(req0, ignore, c0); - launch_push_consumer(conn); + conn.async_exec(req1, ignore, c1); + launch_push_consumer(); - run(conn, make_test_config(), {}); + conn.async_run(make_test_config(), [&](error_code ec) { + run_finished = true; + BOOST_TEST_EQ(ec, net::error::operation_aborted); + }); ioc.run_for(test_timeout); - BOOST_TEST(finished); + + BOOST_TEST(exec_finished); + BOOST_TEST(run_finished); + BOOST_TEST(push_consumer_finished); } -BOOST_AUTO_TEST_CASE(test_unsubscribe) +// UNSUBSCRIBE and PUNSUBSCRIBE work +void test_unsubscribe() { net::io_context ioc; connection conn{ioc}; @@ -354,35 +393,35 @@ BOOST_AUTO_TEST_CASE(test_unsubscribe) run_finished = false; auto on_ping = [&](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); + BOOST_TEST_EQ(ec, error_code()); ping_finished = true; BOOST_TEST(std::get<0>(resp_ping).has_value()); - BOOST_TEST(std::get<0>(resp_ping).value() == "test_unsubscribe"); + BOOST_TEST_EQ(std::get<0>(resp_ping).value(), "test_unsubscribe"); conn.cancel(); }; auto on_unsubscribe = [&](error_code ec, std::size_t) { unsubscribe_finished = true; - BOOST_TEST(ec == error_code()); + BOOST_TEST_EQ(ec, error_code()); BOOST_TEST(std::get<0>(resp_unsubscribe).has_value()); - BOOST_TEST(find_client_info(std::get<0>(resp_unsubscribe).value(), "sub") == "2"); - BOOST_TEST(find_client_info(std::get<0>(resp_unsubscribe).value(), "psub") == "1"); + BOOST_TEST_EQ(find_client_info(std::get<0>(resp_unsubscribe).value(), "sub"), "2"); + BOOST_TEST_EQ(find_client_info(std::get<0>(resp_unsubscribe).value(), "psub"), "1"); conn.async_exec(req_ping, resp_ping, on_ping); }; auto on_subscribe = [&](error_code ec, std::size_t) { subscribe_finished = true; - BOOST_TEST(ec == error_code()); + BOOST_TEST_EQ(ec, error_code()); BOOST_TEST(std::get<0>(resp_subscribe).has_value()); - BOOST_TEST(find_client_info(std::get<0>(resp_subscribe).value(), "sub") == "3"); - BOOST_TEST(find_client_info(std::get<0>(resp_subscribe).value(), "psub") == "2"); + BOOST_TEST_EQ(find_client_info(std::get<0>(resp_subscribe).value(), "sub"), "3"); + BOOST_TEST_EQ(find_client_info(std::get<0>(resp_subscribe).value(), "psub"), "2"); conn.async_exec(req_unsubscribe, resp_unsubscribe, on_unsubscribe); }; conn.async_exec(req_subscribe, resp_subscribe, on_subscribe); conn.async_run(make_test_config(), [&run_finished](error_code ec) { - BOOST_TEST(ec == net::error::operation_aborted); + BOOST_TEST_EQ(ec, net::error::operation_aborted); run_finished = true; }); @@ -394,12 +433,12 @@ BOOST_AUTO_TEST_CASE(test_unsubscribe) BOOST_TEST(run_finished); } -class test_pubsub_state_restoration_ { +struct test_pubsub_state_restoration_impl { net::io_context ioc; connection conn{ioc}; - request req; - response resp_str; - flat_tree resp_push; + request req{}; + response resp_str{}; + flat_tree resp_push{}; bool exec_finished = false; void check_subscriptions() @@ -408,13 +447,13 @@ class test_pubsub_state_restoration_ { std::set seen_channels, seen_patterns; for (auto it = resp_push.get_view().begin(); it != resp_push.get_view().end();) { // The root element should be a push - BOOST_TEST_REQUIRE(it->data_type == type::push); - BOOST_TEST_REQUIRE(it->aggregate_size >= 2u); - BOOST_TEST_REQUIRE((++it != resp_push.get_view().end())); + BOOST_TEST_EQ(it->data_type, type::push); + BOOST_TEST_GE(it->aggregate_size, 2u); + BOOST_TEST(++it != resp_push.get_view().end()); // The next element should be the message type std::string_view msg_type = it->value; - BOOST_TEST_REQUIRE((++it != resp_push.get_view().end())); + BOOST_TEST(++it != resp_push.get_view().end()); // The next element is the channel or pattern if (msg_type == "subscribe") @@ -430,8 +469,16 @@ class test_pubsub_state_restoration_ { const std::string_view expected_channels[] = {"ch1", "ch3", "ch5"}; const std::string_view expected_patterns[] = {"ch1*", "ch3*", "ch4*", "ch8*"}; - BOOST_TEST(seen_channels == expected_channels, boost::test_tools::per_element()); - BOOST_TEST(seen_patterns == expected_patterns, boost::test_tools::per_element()); + BOOST_TEST_ALL_EQ( + seen_channels.begin(), + seen_channels.end(), + std::begin(expected_channels), + std::end(expected_channels)); + BOOST_TEST_ALL_EQ( + seen_patterns.begin(), + seen_patterns.end(), + std::begin(expected_patterns), + std::end(expected_patterns)); } void sub1() @@ -441,7 +488,7 @@ class test_pubsub_state_restoration_ { req.subscribe({"ch1", "ch2", "ch3"}); // active: 1, 2, 3 req.psubscribe({"ch1*", "ch2*", "ch3*", "ch4*"}); // active: 1, 2, 3, 4 conn.async_exec(req, ignore, [this](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); + BOOST_TEST_EQ(ec, error_code()); unsub(); }); } @@ -454,7 +501,7 @@ class test_pubsub_state_restoration_ { req.unsubscribe({"ch2", "ch1", "ch5"}); // active: 3 req.punsubscribe({"ch2*", "ch4*", "ch9*"}); // active: 1, 3 conn.async_exec(req, ignore, [this](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); + BOOST_TEST_EQ(ec, error_code()); sub2(); }); } @@ -476,12 +523,12 @@ class test_pubsub_state_restoration_ { req.push("CLIENT", "INFO"); conn.async_exec(req, resp_str, [this](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); + BOOST_TEST_EQ(ec, error_code()); // We are subscribed to 4 channels and 5 patterns BOOST_TEST(std::get<0>(resp_str).has_value()); - BOOST_TEST(find_client_info(std::get<0>(resp_str).value(), "sub") == "4"); - BOOST_TEST(find_client_info(std::get<0>(resp_str).value(), "psub") == "5"); + BOOST_TEST_EQ(find_client_info(std::get<0>(resp_str).value(), "sub"), "4"); + BOOST_TEST_EQ(find_client_info(std::get<0>(resp_str).value(), "psub"), "5"); resp_push.clear(); @@ -507,12 +554,12 @@ class test_pubsub_state_restoration_ { req.get_config().cancel_if_unresponded = false; conn.async_exec(req, resp_str, [this](error_code ec, std::size_t) { - BOOST_TEST(ec == error_code()); + BOOST_TEST_EQ(ec, error_code()); // We are subscribed to 3 channels and 4 patterns (1 of each didn't survive reconnection) BOOST_TEST(std::get<0>(resp_str).has_value()); - BOOST_TEST(find_client_info(std::get<0>(resp_str).value(), "sub") == "3"); - BOOST_TEST(find_client_info(std::get<0>(resp_str).value(), "psub") == "4"); + BOOST_TEST_EQ(find_client_info(std::get<0>(resp_str).value(), "sub"), "3"); + BOOST_TEST_EQ(find_client_info(std::get<0>(resp_str).value(), "psub"), "4"); // We have received pushes confirming it check_subscriptions(); @@ -522,7 +569,6 @@ class test_pubsub_state_restoration_ { }); } -public: void run() { conn.set_receive_response(resp_push); @@ -533,7 +579,7 @@ class test_pubsub_state_restoration_ { // Start running bool run_finished = false; conn.async_run(make_test_config(), [&run_finished](error_code ec) { - BOOST_TEST(ec == net::error::operation_aborted); + BOOST_TEST_EQ(ec, net::error::operation_aborted); run_finished = true; }); @@ -544,7 +590,20 @@ class test_pubsub_state_restoration_ { BOOST_TEST(run_finished); } }; - -BOOST_AUTO_TEST_CASE(test_pubsub_state_restoration) { test_pubsub_state_restoration_().run(); } +void test_pubsub_state_restoration() { test_pubsub_state_restoration_impl{}.run(); } } // namespace + +int main() +{ + test_async_receive2_waiting_for_push(); + test_async_receive2_push_available(); + test_exec_push_interleaved(); + test_push_adapter_error(); + test_push_adapter_error_reconnection(); + test_consecutive_receives(); + test_unsubscribe(); + test_pubsub_state_restoration(); + + return boost::report_errors(); +} From 002b616dd98cc3edef5f2bc28ae36458f6583491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anarthal=20=28Rub=C3=A9n=20P=C3=A9rez=29?= <34971811+anarthal@users.noreply.github.com> Date: Sun, 18 Jan 2026 15:44:55 +0100 Subject: [PATCH 28/37] Replaces --with-system with --with-headers when building Boost in CI (#382) --- tools/ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci.py b/tools/ci.py index 68f6a8514..6e1ba13e2 100755 --- a/tools/ci.py +++ b/tools/ci.py @@ -134,7 +134,7 @@ def _build_b2_distro( _run([ _b2_command, '--prefix={}'.format(_b2_distro), - '--with-system', + '--with-headers', 'toolset={}'.format(toolset), '-d0', 'install' From 89e44dc0170121a4289c4067d614e338ce7285b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anarthal=20=28Rub=C3=A9n=20P=C3=A9rez=29?= <34971811+anarthal@users.noreply.github.com> Date: Sun, 18 Jan 2026 16:14:53 +0100 Subject: [PATCH 29/37] Makes async_receive2 not cancel on reconnection (#381) async_receive2 is now only cancelled after calling connection::cancel() or using per-operation cancellation Adds a restriction to only have one outstanding async_receive2 operation per connection Adds error::already_running Adds support for asio::cancel_after for async_receive2 Deprecates cancel(operation::receive) Adds more documentation to async_receive2 close #331 --- README.md | 10 +- doc/modules/ROOT/pages/index.adoc | 10 +- example/cpp20_chat_room.cpp | 11 +- example/cpp20_subscriber.cpp | 12 +- include/boost/redis/connection.hpp | 159 +++++++--- .../boost/redis/detail/connection_state.hpp | 1 + include/boost/redis/detail/receive_fsm.hpp | 58 ++++ include/boost/redis/error.hpp | 4 + include/boost/redis/impl/connection.ipp | 6 + include/boost/redis/impl/error.ipp | 3 + include/boost/redis/impl/receive_fsm.ipp | 85 ++++++ include/boost/redis/operation.hpp | 7 +- include/boost/redis/src.hpp | 1 + test/CMakeLists.txt | 1 + test/Jamfile | 1 + test/common.cpp | 3 + test/test_conn_cancel_after.cpp | 24 ++ test/test_conn_check_health.cpp | 1 - test/test_conn_push.cpp | 4 +- test/test_conn_push2.cpp | 272 +++++++++++++++++- test/test_conn_reconnect.cpp | 4 +- test/test_conn_tls.cpp | 4 +- test/test_receive_fsm.cpp | 246 ++++++++++++++++ test/test_unix_sockets.cpp | 1 - 24 files changed, 838 insertions(+), 90 deletions(-) create mode 100644 include/boost/redis/detail/receive_fsm.hpp create mode 100644 include/boost/redis/impl/receive_fsm.ipp create mode 100644 test/test_receive_fsm.cpp diff --git a/README.md b/README.md index 62a038963..5b71d904d 100644 --- a/README.md +++ b/README.md @@ -106,15 +106,17 @@ auto receiver(std::shared_ptr conn) -> asio::awaitable // You're now subscribed to 'mychannel'. Pushes sent over this channel will be stored // in resp. If the connection encounters a network error and reconnects to the server, // it will automatically subscribe to 'mychannel' again. This is transparent to the user. + // You need to use specialized request::subscribe() function (instead of request::push) + // to enable this behavior. // Loop to read Redis push messages. - for (error_code ec;;) { + while (conn->will_reconnect()) { // Wait for pushes - co_await conn->async_receive2(asio::redirect_error(ec)); + auto [ec] = co_await conn->async_receive2(asio::as_tuple); // Check for errors and cancellations - if (ec && (ec != asio::experimental::error::channel_cancelled || !conn->will_reconnect())) { - std::cerr << "Error during receive2: " << ec << std::endl; + if (ec) { + std::cerr << "Error during receive: " << ec << std::endl; break; } diff --git a/doc/modules/ROOT/pages/index.adoc b/doc/modules/ROOT/pages/index.adoc index b7194d1b9..f8c6e191b 100644 --- a/doc/modules/ROOT/pages/index.adoc +++ b/doc/modules/ROOT/pages/index.adoc @@ -117,15 +117,17 @@ auto receiver(std::shared_ptr conn) -> asio::awaitable // You're now subscribed to 'mychannel'. Pushes sent over this channel will be stored // in resp. If the connection encounters a network error and reconnects to the server, // it will automatically subscribe to 'mychannel' again. This is transparent to the user. + // You need to use specialized request::subscribe() function (instead of request::push) + // to enable this behavior. // Loop to read Redis push messages. - for (error_code ec;;) { + while (conn->will_reconnect()) { // Wait for pushes - co_await conn->async_receive2(asio::redirect_error(ec)); + auto [ec] = co_await conn->async_receive2(asio::as_tuple); // Check for errors and cancellations - if (ec && (ec != asio::experimental::error::channel_cancelled || !conn->will_reconnect())) { - std::cerr << "Error during receive2: " << ec << std::endl; + if (ec) { + std::cerr << "Error during receive: " << ec << std::endl; break; } diff --git a/example/cpp20_chat_room.cpp b/example/cpp20_chat_room.cpp index 4dd8180d7..2edb5cea4 100644 --- a/example/cpp20_chat_room.cpp +++ b/example/cpp20_chat_room.cpp @@ -6,12 +6,12 @@ #include +#include #include #include #include #include #include -#include #include #include @@ -30,7 +30,6 @@ using boost::asio::co_spawn; using boost::asio::consign; using boost::asio::detached; using boost::asio::dynamic_buffer; -using boost::asio::redirect_error; using boost::redis::config; using boost::redis::connection; using boost::redis::generic_flat_response; @@ -61,13 +60,13 @@ auto receiver(std::shared_ptr conn) -> awaitable req.subscribe({"channel"}); co_await conn->async_exec(req); - for (error_code ec;;) { + while (conn->will_reconnect()) { // Wait for pushes - co_await conn->async_receive2(asio::redirect_error(ec)); + auto [ec] = co_await conn->async_receive2(asio::as_tuple); // Check for errors and cancellations - if (ec && (ec != asio::experimental::error::channel_cancelled || !conn->will_reconnect())) { - std::cerr << "Error during receive2: " << ec << std::endl; + if (ec) { + std::cerr << "Error during receive: " << ec << std::endl; break; } diff --git a/example/cpp20_subscriber.cpp b/example/cpp20_subscriber.cpp index b3eb9cdd4..b8802b031 100644 --- a/example/cpp20_subscriber.cpp +++ b/example/cpp20_subscriber.cpp @@ -6,14 +6,12 @@ #include +#include #include #include #include #include -#include -#include #include -#include #include @@ -62,13 +60,13 @@ auto receiver(std::shared_ptr conn) -> asio::awaitable // to enable this behavior. // Loop to read Redis push messages. - for (error_code ec;;) { + while (conn->will_reconnect()) { // Wait for pushes - co_await conn->async_receive2(asio::redirect_error(ec)); + auto [ec] = co_await conn->async_receive2(asio::as_tuple); // Check for errors and cancellations - if (ec && (ec != asio::experimental::error::channel_cancelled || !conn->will_reconnect())) { - std::cerr << "Error during receive2: " << ec << std::endl; + if (ec) { + std::cerr << "Error during receive: " << ec << std::endl; break; } diff --git a/include/boost/redis/connection.hpp b/include/boost/redis/connection.hpp index 1fcf2d6b1..1801dd4dd 100644 --- a/include/boost/redis/connection.hpp +++ b/include/boost/redis/connection.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -154,7 +155,7 @@ struct connection_impl { { switch (op) { case operation::exec: st_.mpx.cancel_waiting(); break; - case operation::receive: receive_channel_.cancel(); break; + case operation::receive: cancel_receive_v2(); break; case operation::reconnection: st_.cfg.reconnect_wait_interval = std::chrono::seconds::zero(); break; @@ -165,7 +166,7 @@ struct connection_impl { case operation::health_check: cancel_run(); break; case operation::all: st_.mpx.cancel_waiting(); // exec - receive_channel_.cancel(); // receive + cancel_receive_v2(); // receive st_.cfg.reconnect_wait_interval = std::chrono::seconds::zero(); // reconnect cancel_run(); // run break; @@ -173,6 +174,14 @@ struct connection_impl { } } + void cancel_receive_v1() { receive_channel_.cancel(); } + + void cancel_receive_v2() + { + st_.receive2_cancelled = true; + cancel_receive_v1(); + } + void cancel_run() { // Individual operations should see a terminal cancellation, regardless @@ -184,8 +193,8 @@ struct connection_impl { stream_.cancel_resolve(); // Receive is technically not part of run, but we also cancel it for - // backwards compatibility. - receive_channel_.cancel(); + // backwards compatibility. Note that this intentionally affects v1 receive, only. + cancel_receive_v1(); } bool is_open() const noexcept { return stream_.is_open(); } @@ -234,32 +243,45 @@ struct connection_impl { return size; } +}; - template - auto async_receive2(CompletionToken&& token) +template +struct receive2_op { + connection_impl* conn_; + receive_fsm fsm_{}; + + void drain_receive_channel() { - // clang-format off - return - receive_channel_.async_receive( - asio::deferred( - [this](system::error_code ec, std::size_t) - { - if (!ec) { - auto f = [](system::error_code, std::size_t) { - // There is no point in checking for errors - // here since async_receive just completed - // without errors. - }; - - // We just want to drain the channel. - while (receive_channel_.try_receive(f)); - } - - return asio::deferred.values(ec); - } - ) - )(std::forward(token)); - // clang-format on + // We don't expect any errors here. The only errors + // that might appear in the channel are due to cancellations, + // and these don't make sense with try_receive + auto f = [](system::error_code, std::size_t) { }; + while (conn_->receive_channel_.try_receive(f)) + ; + } + + template + void operator()(Self& self, system::error_code ec = {}, std::size_t /* push_bytes */ = 0u) + { + receive_action act = fsm_.resume(conn_->st_, ec, self.get_cancellation_state().cancelled()); + + switch (act.type) { + case receive_action::action_type::setup_cancellation: + self.reset_cancellation_state(asio::enable_total_cancellation()); + (*this)(self); // this action does not require yielding + return; + case receive_action::action_type::wait: + conn_->receive_channel_.async_receive(std::move(self)); + return; + case receive_action::action_type::drain_channel: + drain_receive_channel(); + (*this)(self); // this action does not require yielding + return; + case receive_action::action_type::immediate: + asio::async_immediate(self.get_io_executor(), std::move(self)); + return; + case receive_action::action_type::done: self.complete(act.ec); return; + } } }; @@ -785,27 +807,68 @@ class basic_connection { return impl_->receive_channel_.async_receive(std::forward(token)); } - /** @brief Wait for server pushes asynchronously + /** @brief Wait for server pushes asynchronously. * - * This function suspends until a server push is received by the + * This function suspends until at least one server push is received by the * connection. On completion an unspecified number of pushes will * have been added to the response object set with @ref - * boost::redis::connection::set_receive_response. + * set_receive_response. Use the functions in the response object + * to know how many messages they were received and consume them. * * To prevent receiving an unbound number of pushes the connection * blocks further read operations on the socket when 256 pushes * accumulate internally (we don't make any commitment to this * exact number). When that happens any `async_exec`s and * health-checks won't make any progress and the connection may - * eventually timeout. To avoid that Apps should call - * `async_receive2` continuously in a loop. - * - * @Note To avoid deadlocks the task (e.g. coroutine) calling + * eventually timeout. To avoid this, apps that expect server pushes + * should call this function continuously in a loop. + * + * This function should be used instead of the deprecated @ref async_receive. + * It differs from `async_receive` in the following: + * + * @li `async_receive` is designed to consume a single push message at a time. + * This can be inefficient when receiving lots of server pushes. + * `async_receive2` is batch-oriented. All pushes that are available + * when `async_receive2` is called will be marked as consumed. + * @li `async_receive` is cancelled when a reconnection happens (e.g. because + * of a network error). This enabled the user to re-establish subscriptions + * using @ref async_exec before waiting for pushes again. With the introduction of + * functions like @ref request::subscribe, subscriptions are automatically + * re-established on reconnection. Thus, `async_receive2` is not cancelled + * on reconnection. + * @li `async_receive` passes the number of bytes that each received + * push message contains. This information is unreliable and not very useful. + * Equivalent information is available using functions in the response object. + * @li `async_receive` might get cancelled if `async_run` is cancelled. + * This doesn't happen with `async_receive2`. + * + * This function does *not* remove messages from the response object + * passed to @ref set_receive_response - use the functions in the response + * object to achieve this. + * + * Only a single instance of `async_receive2` may be outstanding + * for a given connection at any time. Trying to start a second one + * will fail with @ref error::already_running. + * + * @note To avoid deadlocks the task (e.g. coroutine) calling * `async_receive2` should not call `async_exec` in a way where - * they could block each other. + * they could block each other. This is, avoid the following pattern: + * + * @code + * asio::awaitable receiver() + * { + * // Do NOT do this!!! The receive buffer might get full while + * // async_exec runs, which will block all read operations until async_receive2 + * // is called. The two operations end up waiting each other, making the connection unresponsive. + * // If you need to do this, use two connections, instead. + * co_await conn.async_receive2(); + * co_await conn.async_exec(req, resp); + * } + * @endcode + * + * For an example see cpp20_subscriber.cpp. * - * For an example see cpp20_subscriber.cpp. The completion token - * must have the following signature + * The completion token must have the following signature: * * @code * void f(system::error_code); @@ -818,15 +881,15 @@ class basic_connection { * @li `asio::cancellation_type_t::partial`. * @li `asio::cancellation_type_t::total`. * - * Calling `basic_connection::cancel(operation::receive)` will - * also cancel any ongoing receive operations. - * * @param token Completion token. */ template > auto async_receive2(CompletionToken&& token = {}) { - return impl_->async_receive2(std::forward(token)); + return asio::async_compose( + detail::receive2_op{impl_.get()}, + token, + *impl_); } /** @brief (Deprecated) Receives server pushes synchronously without blocking. @@ -1223,7 +1286,9 @@ class connection { template auto async_receive2(CompletionToken&& token = {}) { - return impl_.async_receive2(std::forward(token)); + return asio::async_initiate( + initiation{this}, + token); } /// @copydoc basic_connection::receive @@ -1336,6 +1401,12 @@ class connection { { self->async_exec_impl(*req, std::move(adapter), std::forward(handler)); } + + template + void operator()(Handler&& handler) + { + self->async_receive2_impl(std::forward(handler)); + } }; void async_run_impl( @@ -1352,6 +1423,8 @@ class connection { any_adapter&& adapter, asio::any_completion_handler token); + void async_receive2_impl(asio::any_completion_handler token); + basic_connection impl_; }; diff --git a/include/boost/redis/detail/connection_state.hpp b/include/boost/redis/detail/connection_state.hpp index 75ddf1772..e8e091061 100644 --- a/include/boost/redis/detail/connection_state.hpp +++ b/include/boost/redis/detail/connection_state.hpp @@ -51,6 +51,7 @@ struct connection_state { request setup_req{}; request ping_req{}; subscription_tracker tracker{}; + bool receive2_running{false}, receive2_cancelled{false}; // Sentinel stuff lazy_random_engine eng{}; diff --git a/include/boost/redis/detail/receive_fsm.hpp b/include/boost/redis/detail/receive_fsm.hpp new file mode 100644 index 000000000..16989dce0 --- /dev/null +++ b/include/boost/redis/detail/receive_fsm.hpp @@ -0,0 +1,58 @@ +// +// Copyright (c) 2018-2026 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#ifndef BOOST_REDIS_RECEIVE_FSM_HPP +#define BOOST_REDIS_RECEIVE_FSM_HPP + +#include +#include + +// Sans-io algorithm for async_receive2, as a finite state machine + +namespace boost::redis::detail { + +struct connection_state; + +struct receive_action { + enum class action_type + { + setup_cancellation, // Set up the cancellation types supported by the composed operation + wait, // Wait for a message to appear in the receive channel + drain_channel, // Empty the receive channel + immediate, // Call async_immediate + done, // Complete + }; + + action_type type; + system::error_code ec; + + receive_action(action_type type) noexcept + : type{type} + { } + + receive_action(system::error_code ec) noexcept + : type{action_type::done} + , ec{ec} + { } +}; + +class receive_fsm { + int resume_point_{0}; + +public: + receive_fsm() = default; + + receive_action resume( + connection_state& st, + system::error_code ec, + asio::cancellation_type_t cancel_state); +}; + +} // namespace boost::redis::detail + +#endif diff --git a/include/boost/redis/error.hpp b/include/boost/redis/error.hpp index 346e3ad1d..55356635b 100644 --- a/include/boost/redis/error.hpp +++ b/include/boost/redis/error.hpp @@ -112,6 +112,10 @@ enum class error /// Expects a RESP3 array, but got a different data type. expects_resp3_array, + + /// A @ref basic_connection::async_receive2 operation is already running. + /// Only one of such operations might be running at any point in time. + already_running, }; /** diff --git a/include/boost/redis/impl/connection.ipp b/include/boost/redis/impl/connection.ipp index 088b1ac3b..9db49af36 100644 --- a/include/boost/redis/impl/connection.ipp +++ b/include/boost/redis/impl/connection.ipp @@ -51,6 +51,12 @@ void connection::async_exec_impl( impl_.async_exec(req, std::move(adapter), std::move(token)); } +void connection::async_receive2_impl( + asio::any_completion_handler token) +{ + impl_.async_receive2(std::move(token)); +} + void connection::cancel(operation op) { impl_.cancel(op); } } // namespace boost::redis diff --git a/include/boost/redis/impl/error.ipp b/include/boost/redis/impl/error.ipp index f70715064..04bfbdc27 100644 --- a/include/boost/redis/impl/error.ipp +++ b/include/boost/redis/impl/error.ipp @@ -68,6 +68,9 @@ struct error_category_impl : system::error_category { return "Expects a RESP3 string, but got a different data type."; case error::expects_resp3_array: return "Expects a RESP3 array, but got a different data type."; + case error::already_running: + return "An async_receive2 operation is already running. Only one of such operations " + "might be running at any point in time."; default: BOOST_ASSERT(false); return "Boost.Redis error."; } } diff --git a/include/boost/redis/impl/receive_fsm.ipp b/include/boost/redis/impl/receive_fsm.ipp new file mode 100644 index 000000000..ca98dcd8e --- /dev/null +++ b/include/boost/redis/impl/receive_fsm.ipp @@ -0,0 +1,85 @@ +// +// Copyright (c) 2018-2026 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#include +#include +#include +#include + +#include +#include +#include + +namespace boost::redis::detail { + +constexpr bool is_any_cancel(asio::cancellation_type_t type) +{ + return !!( + type & (asio::cancellation_type_t::terminal | asio::cancellation_type_t::partial | + asio::cancellation_type_t::total)); +} + +// We use the receive2_cancelled flag rather than will_reconnect() to +// avoid entanglement between async_run and async_receive2 cancellations. +// If we had used will_reconnect(), async_receive2 would be cancelled +// when disabling reconnection and async_run exits, and in an unpredictable fashion. +receive_action receive_fsm::resume( + connection_state& st, + system::error_code ec, + asio::cancellation_type_t cancel_state) +{ + switch (resume_point_) { + BOOST_REDIS_CORO_INITIAL + + // Parallel async_receive2 operations not supported + if (st.receive2_running) { + BOOST_REDIS_YIELD(resume_point_, 1, receive_action::action_type::immediate) + return system::error_code(error::already_running); + } + + // We're now running. Discard any previous cancellation state + st.receive2_running = true; + st.receive2_cancelled = false; + + // This operation supports total cancellation. Set it up + BOOST_REDIS_YIELD(resume_point_, 2, receive_action::action_type::setup_cancellation) + + while (true) { + // Wait at least once for a notification to arrive + BOOST_REDIS_YIELD(resume_point_, 3, receive_action::action_type::wait) + + // If the wait completed successfully, we have pushes. Drain the channel and exit + if (!ec) { + BOOST_REDIS_YIELD(resume_point_, 4, receive_action::action_type::drain_channel) + st.receive2_running = false; + return system::error_code(); + } + + // Check for cancellations + if (is_any_cancel(cancel_state) || st.receive2_cancelled) { + st.receive2_running = false; + return system::error_code(asio::error::operation_aborted); + } + + // If we get any unknown errors, propagate them (shouldn't happen, but just in case) + if (ec != asio::experimental::channel_errc::channel_cancelled) { + st.receive2_running = false; + return ec; + } + + // The channel was cancelled and no cancellation state is set. + // This is due to a reconnection. Ignore the notification + } + } + + // We should never get here + BOOST_ASSERT(false); + return receive_action{system::error_code()}; +} + +} // namespace boost::redis::detail diff --git a/include/boost/redis/operation.hpp b/include/boost/redis/operation.hpp index da087cc00..9d05d1d76 100644 --- a/include/boost/redis/operation.hpp +++ b/include/boost/redis/operation.hpp @@ -52,7 +52,12 @@ enum class operation /// Refers to `connection::async_run` operations. run, - /// Refers to `connection::async_receive` operations. + /** + * @brief (Deprecated) Refers to `async_receive` and `async_receive2` operations. + * + * To cancel `async_receive2`, use either @ref basic_connection::cancel with no arguments + * or per-operation cancellation. + */ receive, /** diff --git a/include/boost/redis/src.hpp b/include/boost/redis/src.hpp index 647e643de..196b2960a 100644 --- a/include/boost/redis/src.hpp +++ b/include/boost/redis/src.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9c6a97e0e..859dfc516 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -46,6 +46,7 @@ make_test(test_writer_fsm) make_test(test_reader_fsm) make_test(test_connect_fsm) make_test(test_sentinel_resolve_fsm) +make_test(test_receive_fsm) make_test(test_run_fsm) make_test(test_compose_setup_request) make_test(test_setup_adapter) diff --git a/test/Jamfile b/test/Jamfile index ccd47060c..346837292 100644 --- a/test/Jamfile +++ b/test/Jamfile @@ -62,6 +62,7 @@ local tests = test_writer_fsm test_reader_fsm test_sentinel_resolve_fsm + test_receive_fsm test_run_fsm test_connect_fsm test_compose_setup_request diff --git a/test/common.cpp b/test/common.cpp index cb7e1891d..54b3697f6 100644 --- a/test/common.cpp +++ b/test/common.cpp @@ -7,12 +7,14 @@ #include "common.hpp" +#include #include #include #include #include namespace net = boost::asio; +using namespace std::chrono_literals; struct run_callback { std::shared_ptr conn; @@ -55,6 +57,7 @@ boost::redis::config make_test_config() { boost::redis::config cfg; cfg.addr.host = get_server_hostname(); + cfg.reconnect_wait_interval = 50ms; // make tests involving reconnection faster return cfg; } diff --git a/test/test_conn_cancel_after.cpp b/test/test_conn_cancel_after.cpp index dd25a4a5f..b7f358005 100644 --- a/test/test_conn_cancel_after.cpp +++ b/test/test_conn_cancel_after.cpp @@ -93,6 +93,27 @@ void test_receive() BOOST_TEST(receive_finished); } +template +void test_receive2() +{ + // Setup + asio::io_context ioc; + Connection conn{ioc}; + bool receive_finished = false; + generic_response resp; + conn.set_receive_response(resp); + + // Call the function with a very short timeout. + conn.async_receive2(asio::cancel_after(1ms, [&](error_code ec) { + BOOST_TEST_EQ(ec, asio::error::operation_aborted); + receive_finished = true; + })); + + ioc.run_for(test_timeout); + + BOOST_TEST(receive_finished); +} + } // namespace int main() @@ -106,5 +127,8 @@ int main() test_receive>(); test_receive(); + test_receive2>(); + test_receive2(); + return boost::report_errors(); } diff --git a/test/test_conn_check_health.cpp b/test/test_conn_check_health.cpp index 78062e3b5..5d53b0844 100644 --- a/test/test_conn_check_health.cpp +++ b/test/test_conn_check_health.cpp @@ -51,7 +51,6 @@ void test_reconnection() // Make the test run faster auto cfg = make_test_config(); cfg.health_check_interval = 500ms; - cfg.reconnect_wait_interval = 100ms; bool run_finished = false, exec1_finished = false, exec2_finished = false; diff --git a/test/test_conn_push.cpp b/test/test_conn_push.cpp index 3b2645522..0ba3258f8 100644 --- a/test/test_conn_push.cpp +++ b/test/test_conn_push.cpp @@ -264,9 +264,7 @@ struct test_async_receive_cancelled_on_reconnection_impl { start_subscribe1(); - auto cfg = make_test_config(); - cfg.reconnect_wait_interval = 50ms; // make the test run faster - conn.async_run(cfg, [&](error_code ec) { + conn.async_run(make_test_config(), [&](error_code ec) { run_finished = true; BOOST_TEST_EQ(ec, net::error::operation_aborted); }); diff --git a/test/test_conn_push2.cpp b/test/test_conn_push2.cpp index 5c260f8c3..ab188d0d1 100644 --- a/test/test_conn_push2.cpp +++ b/test/test_conn_push2.cpp @@ -5,11 +5,17 @@ */ #include +#include #include +#include #include #include #include +#include +#include +#include +#include #include #include #include @@ -18,6 +24,7 @@ #include #include +#include #include #include #include @@ -126,6 +133,237 @@ void test_async_receive2_push_available() BOOST_TEST(run_finished); } +// async_receive2 blocks only once if several messages are received in a batch +void test_async_receive2_batch() +{ + // Setup + net::io_context ioc; + connection conn{ioc}; + resp3::flat_tree resp; + conn.set_receive_response(resp); + + // Cause two messages to be delivered. The PING ensures that + // the pushes have been read when exec completes + request req; + req.push("SUBSCRIBE", "test_async_receive2_batch"); + req.push("SUBSCRIBE", "test_async_receive2_batch"); + req.push("PING", "message"); + + bool receive_finished = false, run_finished = false; + + // 1. Trigger pushes + // 2. Receive both of them + // 3. Check that receive2 has consumed them by calling it again + auto on_receive2 = [&](error_code ec) { + BOOST_TEST_EQ(ec, net::error::operation_aborted); + receive_finished = true; + conn.cancel(); + }; + + auto on_receive1 = [&](error_code ec) { + BOOST_TEST_EQ(ec, error_code()); + BOOST_TEST_EQ(resp.get_total_msgs(), 2u); + conn.async_receive2(net::cancel_after(50ms, on_receive2)); + }; + + conn.async_exec(req, ignore, [&](error_code ec, std::size_t) { + BOOST_TEST_EQ(ec, error_code()); + conn.async_receive2(on_receive1); + }); + + conn.async_run(make_test_config(), [&](error_code ec) { + run_finished = true; + BOOST_TEST_EQ(ec, net::error::operation_aborted); + }); + + ioc.run_for(test_timeout); + + BOOST_TEST(receive_finished); + BOOST_TEST(run_finished); +} + +// async_receive2 can be called several times in a row +void test_async_receive2_subsequent_calls() +{ + struct impl { + net::io_context ioc{}; + connection conn{ioc}; + resp3::flat_tree resp{}; + request req{}; + bool receive_finished = false, run_finished = false; + + // Send a SUBSCRIBE, which will trigger a push + void start_subscribe1() + { + conn.async_exec(req, ignore, [this](error_code ec, std::size_t) { + BOOST_TEST_EQ(ec, error_code()); + start_receive1(); + }); + } + + // Receive the push + void start_receive1() + { + conn.async_receive2([this](error_code ec) { + BOOST_TEST_EQ(ec, error_code()); + BOOST_TEST_EQ(resp.get_total_msgs(), 1u); + resp.clear(); + start_subscribe2(); + }); + } + + // Send another SUBSCRIBE, which will trigger another push + void start_subscribe2() + { + conn.async_exec(req, ignore, [this](error_code ec, std::size_t) { + BOOST_TEST_EQ(ec, error_code()); + start_receive2(); + }); + } + + // End + void start_receive2() + { + conn.async_receive2([this](error_code ec) { + BOOST_TEST_EQ(ec, error_code()); + BOOST_TEST_EQ(resp.get_total_msgs(), 1u); + receive_finished = true; + conn.cancel(); + }); + } + + void run() + { + // Setup + conn.set_receive_response(resp); + req.push("SUBSCRIBE", "test_async_receive2_subsequent_calls"); + + start_subscribe1(); + conn.async_run(make_test_config(), [&](error_code ec) { + run_finished = true; + BOOST_TEST_EQ(ec, net::error::operation_aborted); + }); + + ioc.run_for(test_timeout); + + BOOST_TEST(receive_finished); + BOOST_TEST(run_finished); + } + }; + + impl{}.run(); +} + +// async_receive2 can be cancelled using per-operation cancellation, +// and supports all cancellation types +void test_async_receive2_per_operation_cancellation( + std::string_view name, + net::cancellation_type_t type) +{ + // Setup + net::io_context ioc; + connection conn{ioc}; + net::cancellation_signal sig; + bool receive_finished = false; + + conn.async_receive2(net::bind_cancellation_slot(sig.slot(), [&](error_code ec) { + if (!BOOST_TEST_EQ(ec, net::error::operation_aborted)) + std::cerr << "With cancellation type " << name << std::endl; + receive_finished = true; + })); + + sig.emit(type); + + ioc.run_for(test_timeout); + + if (!BOOST_TEST(receive_finished)) + std::cerr << "With cancellation type " << name << std::endl; +} + +// connection::cancel() cancels async_receive2 +void test_async_receive2_connection_cancel() +{ + // Setup + net::io_context ioc; + connection conn{ioc}; + net::cancellation_signal sig; + bool receive_finished = false; + + conn.async_receive2([&](error_code ec) { + BOOST_TEST_EQ(ec, net::error::operation_aborted); + receive_finished = true; + }); + + conn.cancel(); + + ioc.run_for(test_timeout); + + BOOST_TEST(receive_finished); +} + +// Reconnection doesn't cancel async_receive2 +void test_async_receive2_reconnection() +{ + // Setup + net::io_context ioc; + connection conn{ioc}; + resp3::flat_tree resp; + conn.set_receive_response(resp); + + // Causes the reconnection + request req_quit; + req_quit.push("QUIT"); + + // When this completes, the reconnection has happened + request req_ping; + req_ping.get_config().cancel_if_unresponded = false; + req_ping.push("PING", "test_async_receive2_connection"); + + // Generates a push + request req_subscribe; + req_subscribe.push("SUBSCRIBE", "test_async_receive2_connection"); + + bool exec_finished = false, receive_finished = false, run_finished = false; + + // Launch a receive operation, and in parallel + // 1. Trigger a reconnection + // 2. Wait for the reconnection and check that receive hasn't been cancelled + // 3. Trigger a push to make receive complete + auto on_subscribe = [&](error_code ec, std::size_t) { + // Will finish before receive2 because the command doesn't have a response + BOOST_TEST_EQ(ec, error_code()); + exec_finished = true; + }; + + auto on_ping = [&](error_code ec, std::size_t) { + // Reconnection has already happened here + BOOST_TEST_EQ(ec, error_code()); + BOOST_TEST_NOT(receive_finished); + conn.async_exec(req_subscribe, ignore, on_subscribe); + }; + + conn.async_exec(req_quit, ignore, [&](error_code, std::size_t) { + conn.async_exec(req_ping, ignore, on_ping); + }); + + conn.async_receive2([&](error_code ec) { + BOOST_TEST_EQ(ec, error_code()); + receive_finished = true; + conn.cancel(); + }); + + conn.async_run(make_test_config(), [&](error_code ec) { + run_finished = true; + BOOST_TEST_EQ(ec, net::error::operation_aborted); + }); + + ioc.run_for(test_timeout); + + BOOST_TEST(exec_finished); + BOOST_TEST(receive_finished); + BOOST_TEST(run_finished); +} + // A push may be interleaved between regular responses. // It is handed to the receive adapter (filtered out). void test_exec_push_interleaved() @@ -194,12 +432,12 @@ void test_push_adapter_error() req.push("SUBSCRIBE", "channel"); req.push("PING"); - bool push_received = false, exec_finished = false, run_finished = false; + bool receive_finished = false, exec_finished = false, run_finished = false; - // async_receive2 is cancelled every reconnection cycle + // We cancel receive when run exits conn.async_receive2([&](error_code ec) { - BOOST_TEST_EQ(ec, net::experimental::error::channel_cancelled); - push_received = true; + BOOST_TEST_EQ(ec, net::error::operation_aborted); + receive_finished = true; }); // The request is cancelled because the PING response isn't processed @@ -211,13 +449,14 @@ void test_push_adapter_error() auto cfg = make_test_config(); cfg.reconnect_wait_interval = 0s; // so we can validate the generated error - conn.async_run(cfg, [&run_finished](error_code ec) { + conn.async_run(cfg, [&](error_code ec) { BOOST_TEST_EQ(ec, error::incompatible_size); run_finished = true; + conn.cancel(); }); ioc.run_for(test_timeout); - BOOST_TEST(push_received); + BOOST_TEST(receive_finished); BOOST_TEST(exec_finished); BOOST_TEST(run_finished); } @@ -244,7 +483,7 @@ void test_push_adapter_error_reconnection() // async_receive2 is cancelled every reconnection cycle conn.async_receive2([&](error_code ec) { - BOOST_TEST_EQ(ec, net::experimental::error::channel_cancelled); + BOOST_TEST_EQ(ec, net::error::operation_aborted); push_received = true; }); @@ -262,9 +501,7 @@ void test_push_adapter_error_reconnection() conn.async_exec(req2, resp, on_exec2); }); - auto cfg = make_test_config(); - cfg.reconnect_wait_interval = 50ms; // make the test run faster - conn.async_run(cfg, [&run_finished](error_code ec) { + conn.async_run(make_test_config(), [&run_finished](error_code ec) { BOOST_TEST_EQ(ec, net::error::operation_aborted); run_finished = true; }); @@ -276,8 +513,8 @@ void test_push_adapter_error_reconnection() BOOST_TEST(run_finished); } -// After an async_receive2 operation finishes, another one can be issued -void test_consecutive_receives() +// Tests the usual push consumer pattern that we recommend in the examples +void test_push_consumer() { net::io_context ioc; connection conn{ioc}; @@ -287,7 +524,7 @@ void test_consecutive_receives() std::function launch_push_consumer = [&]() { conn.async_receive2([&](error_code ec) { if (ec) { - BOOST_TEST_EQ(ec, net::experimental::error::channel_cancelled); + BOOST_TEST_EQ(ec, net::error::operation_aborted); push_consumer_finished = true; resp.clear(); return; @@ -598,10 +835,17 @@ int main() { test_async_receive2_waiting_for_push(); test_async_receive2_push_available(); + test_async_receive2_batch(); + test_async_receive2_subsequent_calls(); + test_async_receive2_per_operation_cancellation("terminal", net::cancellation_type_t::terminal); + test_async_receive2_per_operation_cancellation("partial", net::cancellation_type_t::partial); + test_async_receive2_per_operation_cancellation("total", net::cancellation_type_t::total); + test_async_receive2_connection_cancel(); + test_async_receive2_reconnection(); test_exec_push_interleaved(); test_push_adapter_error(); test_push_adapter_error_reconnection(); - test_consecutive_receives(); + test_push_consumer(); test_unsubscribe(); test_pubsub_state_restoration(); diff --git a/test/test_conn_reconnect.cpp b/test/test_conn_reconnect.cpp index 2a9ea3377..7740c9b55 100644 --- a/test/test_conn_reconnect.cpp +++ b/test/test_conn_reconnect.cpp @@ -47,9 +47,7 @@ net::awaitable test_reconnect_impl() regular_req.get_config().cancel_if_unresponded = false; auto conn = std::make_shared(ex); - auto cfg = make_test_config(); - cfg.reconnect_wait_interval = 100ms; // make the test run faster - run(conn, std::move(cfg)); + run(conn, make_test_config()); for (int i = 0; i < 3; ++i) { BOOST_TEST_CONTEXT("i=" << i) diff --git a/test/test_conn_tls.cpp b/test/test_conn_tls.cpp index 43939b40a..8989e2e33 100644 --- a/test/test_conn_tls.cpp +++ b/test/test_conn_tls.cpp @@ -147,8 +147,6 @@ BOOST_AUTO_TEST_CASE(reconnection) net::io_context ioc; net::steady_timer timer{ioc}; connection conn{ioc}; - auto cfg = make_tls_config(); - cfg.reconnect_wait_interval = 10ms; // make the test run faster request ping_request; ping_request.push("PING", "some_value"); @@ -161,7 +159,7 @@ BOOST_AUTO_TEST_CASE(reconnection) bool exec_finished = false, run_finished = false; // Run the connection - conn.async_run(cfg, {}, [&](error_code ec) { + conn.async_run(make_test_config(), [&](error_code ec) { run_finished = true; BOOST_TEST(ec == net::error::operation_aborted); }); diff --git a/test/test_receive_fsm.cpp b/test/test_receive_fsm.cpp new file mode 100644 index 000000000..351af0b3a --- /dev/null +++ b/test/test_receive_fsm.cpp @@ -0,0 +1,246 @@ +// +// Copyright (c) 2026 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +namespace net = boost::asio; +using namespace boost::redis; +using net::cancellation_type_t; +using boost::system::error_code; +using net::cancellation_type_t; +using detail::receive_action; +using detail::receive_fsm; +using detail::connection_state; +namespace channel_errc = net::experimental::channel_errc; +using action_type = receive_action::action_type; + +// Operators +static const char* to_string(action_type type) +{ + switch (type) { + case action_type::setup_cancellation: return "setup_cancellation"; + case action_type::wait: return "wait"; + case action_type::drain_channel: return "drain_channel"; + case action_type::immediate: return "immediate"; + case action_type::done: return "done"; + default: return ""; + } +} + +namespace boost::redis::detail { + +std::ostream& operator<<(std::ostream& os, action_type type) { return os << to_string(type); } + +bool operator==(const receive_action& lhs, const receive_action& rhs) noexcept +{ + return lhs.type == rhs.type && lhs.ec == rhs.ec; +} + +std::ostream& operator<<(std::ostream& os, const receive_action& act) +{ + os << "action{ .type=" << act.type; + if (act.type == action_type::done) + os << ", ec=" << act.ec; + return os << " }"; +} + +} // namespace boost::redis::detail + +namespace { + +struct fixture { + connection_state st; + generic_response resp; +}; + +void test_success() +{ + connection_state st; + receive_fsm fsm; + + // Initiate + auto act = fsm.resume(st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, action_type::setup_cancellation); + act = fsm.resume(st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, action_type::wait); + + // At this point, the operation is now running + BOOST_TEST(st.receive2_running); + + // The wait finishes successfully (we were notified). Receive exits + act = fsm.resume(st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, action_type::drain_channel); + act = fsm.resume(st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code()); + + // The operation is no longer running + BOOST_TEST_NOT(st.receive2_running); +} + +// We might see spurious cancels during reconnection (v1 compatibility). +void test_cancelled_reconnection() +{ + connection_state st; + receive_fsm fsm; + + // Initiate + auto act = fsm.resume(st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, action_type::setup_cancellation); + act = fsm.resume(st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, action_type::wait); + + // Reconnection happens + act = fsm.resume(st, channel_errc::channel_cancelled, cancellation_type_t::none); + BOOST_TEST_EQ(act, action_type::wait); + BOOST_TEST(st.receive2_running); // still running + + // Another reconnection + act = fsm.resume(st, channel_errc::channel_cancelled, cancellation_type_t::none); + BOOST_TEST_EQ(act, action_type::wait); + BOOST_TEST(st.receive2_running); // still running + + // The wait finishes successfully (we were notified). Receive exits + act = fsm.resume(st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, action_type::drain_channel); + act = fsm.resume(st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code()); + + // The operation is no longer running + BOOST_TEST_NOT(st.receive2_running); +} + +// We might get cancellations due to connection::cancel() +void test_cancelled_connection_cancel() +{ + connection_state st; + receive_fsm fsm; + + // Initiate + auto act = fsm.resume(st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, action_type::setup_cancellation); + act = fsm.resume(st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, action_type::wait); + + // Simulate a connection::cancel() + st.receive2_cancelled = true; + act = fsm.resume(st, channel_errc::channel_cancelled, cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code(net::error::operation_aborted)); + BOOST_TEST_NOT(st.receive2_running); +} + +// Operations can still run after connection::cancel() +void test_after_connection_cancel() +{ + connection_state st; + receive_fsm fsm; + st.receive2_cancelled = true; + + // The operation initiates and runs normally + auto act = fsm.resume(st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, action_type::setup_cancellation); + act = fsm.resume(st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, action_type::wait); + BOOST_TEST(st.receive2_running); + + // Reconnection behavior not affected + act = fsm.resume(st, channel_errc::channel_cancelled, cancellation_type_t::none); + BOOST_TEST_EQ(act, action_type::wait); + BOOST_TEST(st.receive2_running); // still running + + // Simulate a connection::cancel() + st.receive2_cancelled = true; + act = fsm.resume(st, channel_errc::channel_cancelled, cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code(net::error::operation_aborted)); + BOOST_TEST_NOT(st.receive2_running); +} + +// Per-operation cancellation is supported +void test_per_operation_cancellation(std::string_view name, cancellation_type_t type) +{ + std::cerr << "Running cancellation case " << name << std::endl; + + connection_state st; + receive_fsm fsm; + + // The operation initiates and runs normally + auto act = fsm.resume(st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, action_type::setup_cancellation); + act = fsm.resume(st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, action_type::wait); + BOOST_TEST(st.receive2_running); + + // Cancellation is received + act = fsm.resume(st, channel_errc::channel_cancelled, type); + BOOST_TEST_EQ(act, error_code(net::error::operation_aborted)); + BOOST_TEST_NOT(st.receive2_running); +} + +// Only a single instance of async_receive2 can be running at the same time +void test_error_already_running() +{ + connection_state st; + receive_fsm fsm; + st.receive2_running = true; + + // The operation fails immediately + auto act = fsm.resume(st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, action_type::immediate); + BOOST_TEST(st.receive2_running); // not affected + act = fsm.resume(st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code(error::already_running)); + BOOST_TEST(st.receive2_running); // not affected +} + +// If an unknown error was obtained during channel receive, we propagate it +void test_error_unknown() +{ + connection_state st; + receive_fsm fsm; + + // Initiate + auto act = fsm.resume(st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, action_type::setup_cancellation); + act = fsm.resume(st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, action_type::wait); + BOOST_TEST(st.receive2_running); + + // We have an unknown error + act = fsm.resume(st, channel_errc::channel_closed, cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code(channel_errc::channel_closed)); + BOOST_TEST_NOT(st.receive2_running); +} + +} // namespace + +int main() +{ + test_success(); + test_cancelled_reconnection(); + test_cancelled_connection_cancel(); + test_after_connection_cancel(); + + test_per_operation_cancellation("terminal", cancellation_type_t::terminal); + test_per_operation_cancellation("partial", cancellation_type_t::partial); + test_per_operation_cancellation("total", cancellation_type_t::total); + test_per_operation_cancellation("all", cancellation_type_t::all); + + test_error_already_running(); + test_error_unknown(); + + return boost::report_errors(); +} diff --git a/test/test_unix_sockets.cpp b/test/test_unix_sockets.cpp index be4fa1394..ec53b5836 100644 --- a/test/test_unix_sockets.cpp +++ b/test/test_unix_sockets.cpp @@ -78,7 +78,6 @@ void test_reconnection() connection conn{ioc}; auto cfg = make_test_config(); cfg.unix_socket = unix_socket_path; - cfg.reconnect_wait_interval = 10ms; // make the test run faster request ping_request; ping_request.get_config().cancel_if_not_connected = false; From 18ee72830ba257d5d03bd50334c9a0683d225e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anarthal=20=28Rub=C3=A9n=20P=C3=A9rez=29?= <34971811+anarthal@users.noreply.github.com> Date: Sun, 18 Jan 2026 22:08:18 +0100 Subject: [PATCH 30/37] Makes flat_tree a proper container (#383) Removes flat_tree::get_view() Adds flat_tree{iterator, reverse_iterator, begin, end, rbegin, rend, data, operator[], at, front, back, size, empty} close #362 --- README.md | 2 +- doc/modules/ROOT/pages/index.adoc | 2 +- example/cpp20_chat_room.cpp | 2 +- example/cpp20_subscriber.cpp | 2 +- include/boost/redis/impl/flat_tree.ipp | 16 +- include/boost/redis/resp3/flat_tree.hpp | 172 +++++++++++++-- test/test_conn_exec.cpp | 2 +- test/test_conn_push2.cpp | 8 +- test/test_flat_tree.cpp | 264 +++++++++++++++++++++++- test/test_generic_flat_response.cpp | 6 +- 10 files changed, 432 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 5b71d904d..e79474dbb 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ auto receiver(std::shared_ptr conn) -> asio::awaitable // The response must be consumed without suspending the // coroutine i.e. without the use of async operations. - for (auto const& elem : resp.value().get_view()) + for (auto const& elem : resp.value()) std::cout << elem.value << "\n"; std::cout << std::endl; diff --git a/doc/modules/ROOT/pages/index.adoc b/doc/modules/ROOT/pages/index.adoc index f8c6e191b..ebfbd0446 100644 --- a/doc/modules/ROOT/pages/index.adoc +++ b/doc/modules/ROOT/pages/index.adoc @@ -133,7 +133,7 @@ auto receiver(std::shared_ptr conn) -> asio::awaitable // The response must be consumed without suspending the // coroutine i.e. without the use of async operations. - for (auto const& elem : resp.value().get_view()) + for (auto const& elem : resp.value()) std::cout << elem.value << "\n"; std::cout << std::endl; diff --git a/example/cpp20_chat_room.cpp b/example/cpp20_chat_room.cpp index 2edb5cea4..21aa0c3a6 100644 --- a/example/cpp20_chat_room.cpp +++ b/example/cpp20_chat_room.cpp @@ -72,7 +72,7 @@ auto receiver(std::shared_ptr conn) -> awaitable // The response must be consumed without suspending the // coroutine i.e. without the use of async operations. - for (auto const& elem : resp.value().get_view()) + for (auto const& elem : resp.value()) std::cout << elem.value << "\n"; std::cout << std::endl; diff --git a/example/cpp20_subscriber.cpp b/example/cpp20_subscriber.cpp index b8802b031..513f90acc 100644 --- a/example/cpp20_subscriber.cpp +++ b/example/cpp20_subscriber.cpp @@ -72,7 +72,7 @@ auto receiver(std::shared_ptr conn) -> asio::awaitable // The response must be consumed without suspending the // coroutine i.e. without the use of async operations. - for (auto const& elem : resp.value().get_view()) + for (auto const& elem : resp.value()) std::cout << elem.value << "\n"; std::cout << std::endl; diff --git a/include/boost/redis/impl/flat_tree.ipp b/include/boost/redis/impl/flat_tree.ipp index 7ee1aac4f..a891f6c8d 100644 --- a/include/boost/redis/impl/flat_tree.ipp +++ b/include/boost/redis/impl/flat_tree.ipp @@ -11,10 +11,12 @@ #include #include +#include #include #include #include +#include #include namespace boost::redis::resp3 { @@ -228,15 +230,19 @@ void flat_tree::notify_done() data_tmp_offset_ = data_.size; } +const node_view& flat_tree::at(std::size_t i) const +{ + if (i >= size()) + BOOST_THROW_EXCEPTION(std::out_of_range("flat_tree::at")); + return view_tree_[i]; +} + bool operator==(flat_tree const& a, flat_tree const& b) { // data is already taken into account by comparing the nodes. // Only committed nodes should be taken into account. - auto a_nodes = a.get_view(); - auto b_nodes = b.get_view(); - return a_nodes.size() == b_nodes.size() && - std::equal(a_nodes.begin(), a_nodes.end(), b_nodes.begin()) && - a.total_msgs_ == b.total_msgs_; + return a.size() == b.size() && std::equal(a.begin(), a.end(), b.begin()) && + a.get_total_msgs() == b.get_total_msgs(); } } // namespace boost::redis::resp3 diff --git a/include/boost/redis/resp3/flat_tree.hpp b/include/boost/redis/resp3/flat_tree.hpp index 4c6158c53..2783e9adf 100644 --- a/include/boost/redis/resp3/flat_tree.hpp +++ b/include/boost/redis/resp3/flat_tree.hpp @@ -15,6 +15,7 @@ #include #include +#include #include namespace boost::redis { @@ -45,8 +46,12 @@ struct flat_buffer { * to obtain how many responses this object contains. * * Objects are typically created by the user and passed to @ref connection::async_exec - * to be used as response containers. Call @ref get_view to access the actual RESP3 nodes. - * Once populated, `flat_tree` can't be modified, except for @ref clear and assignment. + * to be used as response containers. Once populated, they can be used as a const range + * of @ref resp3::node_view objects. The usual random access range methods (like @ref at, @ref size or + * @ref front) are provided. Once populated, `flat_tree` can't be modified, + * except for @ref clear and assignment. + * + * `flat_tree` models `std::ranges::contiguous_range`. * * A `flat_tree` is conceptually similar to a pair of `std::vector` objects, one holding * @ref resp3::node_view objects, and another owning the the string data that these views @@ -54,6 +59,22 @@ struct flat_buffer { */ class flat_tree { public: + /** + * @brief The type of the iterators returned by @ref begin and @ref end. + * + * It is guaranteed to be a contiguous iterator. While this is currently a pointer, + * users shouldn't rely on this fact, as the exact implementation may change between releases. + */ + using iterator = const node_view*; + + /** + * @brief The type of the iterators returned by @ref rbegin and @ref rend. + * + * As with @ref iterator, users should treat this type as an unspecified + * contiguous iterator type rather than assuming a specific type. + */ + using reverse_iterator = std::reverse_iterator; + /** * @brief Default constructor. * @@ -70,7 +91,7 @@ class flat_tree { * Constructs a tree by taking ownership of the nodes in `other`. * * @par Object lifetimes - * References to the nodes and strings in `other` remain valid. + * Iterators, pointers and references to the nodes and strings in `other` remain valid. * * @par Exception safety * No-throw guarantee. @@ -95,8 +116,8 @@ class flat_tree { * `other` is left in a valid but unspecified state. * * @par Object lifetimes - * References to the nodes and strings in `other` remain valid. - * References to the nodes and strings in `*this` are invalidated. + * Iterators, pointers and references to the nodes and strings in `other` remain valid. + * Iterators, pointers and references to the nodes and strings in `*this` are invalidated. * * @par Exception safety * No-throw guarantee. @@ -110,16 +131,137 @@ class flat_tree { * After the copy, `*this` and `other` have independent lifetimes (usual copy semantics). * * @par Object lifetimes - * References to the nodes and strings in `*this` are invalidated. + * Iterators, pointers and references to the nodes and strings in `*this` are invalidated. * * @par Exception safety * Basic guarantee. Memory allocations might throw. */ flat_tree& operator=(const flat_tree& other); - friend bool operator==(flat_tree const&, flat_tree const&); + /** + * @brief Returns an iterator to the first element of the node range. + * + * @par Exception safety + * No-throw guarantee. + * + * @returns An iterator to the first node. + */ + iterator begin() const noexcept { return data(); } - friend bool operator!=(flat_tree const&, flat_tree const&); + /** + * @brief Returns an iterator past the last element in the node range. + * + * @par Exception safety + * No-throw guarantee. + * + * @returns An iterator past the last element in the node range. + */ + iterator end() const noexcept { return data() + size(); } + + /** + * @brief Returns an iterator to the first element of the reversed node range. + * + * Allows iterating the range of nodes in reverse order. + * + * @par Exception safety + * No-throw guarantee. + * + * @returns An iterator to the first node of the reversed range. + */ + reverse_iterator rbegin() const noexcept { return reverse_iterator{end()}; } + + /** + * @brief Returns an iterator past the last element of the reversed node range. + * + * Allows iterating the range of nodes in reverse order. + * + * @par Exception safety + * No-throw guarantee. + * + * @returns An iterator past the last element of the reversed node range. + */ + reverse_iterator rend() const noexcept { return reverse_iterator{begin()}; } + + /** + * @brief Returns a reference to the node at the specified position (checked access). + * + * @par Exception safety + * Strong guarantee. Throws `std::out_of_range` if `i >= size()`. + * + * @param i Position of the node to return. + * @returns A reference to the node at position `i`. + */ + const node_view& at(std::size_t i) const; + + /** + * @brief Returns a reference to the node at the specified position (unchecked access). + * + * @par Precondition + * `i < size()`. + * + * @par Exception safety + * No-throw guarantee. + * + * @param i Position of the node to return. + * @returns A reference to the node at position `i`. + */ + const node_view& operator[](std::size_t i) const noexcept { return get_view()[i]; } + + /** + * @brief Returns a reference to the first node. + * + * @par Precondition + * `!empty()`. + * + * @par Exception safety + * No-throw guarantee. + * + * @returns A reference to the first node. + */ + const node_view& front() const noexcept { return get_view().front(); } + + /** + * @brief Returns a reference to the last node. + * + * @par Precondition + * `!empty()`. + * + * @par Exception safety + * No-throw guarantee. + * + * @returns A reference to the last node. + */ + const node_view& back() const noexcept { return get_view().back(); } + + /** + * @brief Returns a pointer to the underlying node storage. + * + * @par Exception safety + * No-throw guarantee. + * + * @returns A pointer to the underlying node array. + */ + const node_view* data() const noexcept { return view_tree_.data(); } + + /** + * @brief Checks whether the tree is empty. + * + * @par Exception safety + * No-throw guarantee. + * + * @returns `true` if the tree contains no nodes, `false` otherwise. + */ + bool empty() const noexcept { return size() == 0u; } + + /** + * @brief Returns the number of nodes in the tree. + * + * @par Exception safety + * No-throw guarantee. + * + * @returns The number of nodes. + */ + std::size_t size() const noexcept { return node_tmp_offset_; } /** @brief Reserves capacity for incoming data. * @@ -142,7 +284,7 @@ class flat_tree { /** @brief Clears the tree so it contains no nodes. * * Calling this function removes every node, making - * @ref get_view return empty and @ref get_total_msgs + * the range contain no nodes, and @ref get_total_msgs * return zero. It does not modify the object's capacity. * * To re-use a `flat_tree` for several requests, @@ -189,17 +331,6 @@ class flat_tree { */ auto data_capacity() const noexcept -> std::size_t { return data_.capacity; } - /** @brief Returns a vector with the nodes in the tree. - * - * This is the main way to access the contents of the tree. - * - * @par Exception safety - * No-throw guarantee. - * - * @returns The nodes in the tree. - */ - span get_view() const noexcept { return {view_tree_.data(), node_tmp_offset_}; } - /** @brief Returns the number of memory reallocations that took place in the data buffer. * * This function returns how many reallocations in the data buffer were performed and @@ -226,6 +357,7 @@ class flat_tree { private: template friend class adapter::detail::general_aggregate; + span get_view() const noexcept { return {data(), size()}; } void notify_init(); void notify_done(); diff --git a/test/test_conn_exec.cpp b/test/test_conn_exec.cpp index 22216f9b8..715d844fa 100644 --- a/test/test_conn_exec.cpp +++ b/test/test_conn_exec.cpp @@ -205,7 +205,7 @@ BOOST_AUTO_TEST_CASE(exec_generic_flat_response) BOOST_TEST_REQUIRE(finished); BOOST_TEST(resp.has_value()); - BOOST_TEST(resp->get_view().front().value == "PONG"); + BOOST_TEST(resp.value().front().value == "PONG"); } } // namespace diff --git a/test/test_conn_push2.cpp b/test/test_conn_push2.cpp index ab188d0d1..c76191ddc 100644 --- a/test/test_conn_push2.cpp +++ b/test/test_conn_push2.cpp @@ -682,15 +682,15 @@ struct test_pubsub_state_restoration_impl { { // Checks for the expected subscriptions and patterns after restoration std::set seen_channels, seen_patterns; - for (auto it = resp_push.get_view().begin(); it != resp_push.get_view().end();) { + for (auto it = resp_push.begin(); it != resp_push.end();) { // The root element should be a push BOOST_TEST_EQ(it->data_type, type::push); BOOST_TEST_GE(it->aggregate_size, 2u); - BOOST_TEST(++it != resp_push.get_view().end()); + BOOST_TEST(++it != resp_push.end()); // The next element should be the message type std::string_view msg_type = it->value; - BOOST_TEST(++it != resp_push.get_view().end()); + BOOST_TEST(++it != resp_push.end()); // The next element is the channel or pattern if (msg_type == "subscribe") @@ -699,7 +699,7 @@ struct test_pubsub_state_restoration_impl { seen_patterns.insert(it->value); // Skip the rest of the nodes - while (it != resp_push.get_view().end() && it->depth != 0u) + while (it != resp_push.end() && it->depth != 0u) ++it; } diff --git a/test/test_flat_tree.cpp b/test/test_flat_tree.cpp index 87b7f804a..bc8a65457 100644 --- a/test/test_flat_tree.cpp +++ b/test/test_flat_tree.cpp @@ -11,19 +11,29 @@ #include #include +#include // for a safe #include #include #include #include "print_node.hpp" #include +#include #include #include +#include +#include #include +#include #include #include #include +#if (__cpp_lib_ranges >= 201911L) && (__cpp_lib_concepts >= 202002L) +#define BOOST_REDIS_TEST_RANGE_CONCEPTS +#include +#endif + using boost::redis::adapter::adapt2; using boost::redis::adapter::result; using boost::redis::resp3::tree; @@ -70,11 +80,7 @@ void check_nodes( boost::span expected, boost::source_location loc = BOOST_CURRENT_LOCATION) { - if (!BOOST_TEST_ALL_EQ( - tree.get_view().begin(), - tree.get_view().end(), - expected.begin(), - expected.end())) + if (!BOOST_TEST_ALL_EQ(tree.begin(), tree.end(), expected.begin(), expected.end())) std::cerr << "Called from " << loc << std::endl; } @@ -1090,6 +1096,233 @@ void test_move_assign_tmp() BOOST_TEST_EQ(t.get_total_msgs(), 2u); } +// --- Iterators --- +// We can obtain iterators using begin() and end() and use them to iterate +void test_iterators() +{ + // Setup + flat_tree t; + add_nodes(t, "+node1\r\n"); + add_nodes(t, ":200\r\n"); + constexpr node_view node1{type::simple_string, 1u, 0u, "node1"}; + constexpr node_view node2{type::number, 1u, 0u, "200"}; + + // These methods are const + const auto& tconst = t; + auto it = tconst.begin(); + auto end = tconst.end(); + + // Iteration using iterators + BOOST_TEST_NE(it, end); + BOOST_TEST_EQ(*it, node1); + BOOST_TEST_NE(++it, end); + BOOST_TEST_EQ(*it, node2); + BOOST_TEST_EQ(++it, end); + + // Iteration using range for + std::vector nodes; + for (const auto& n : t) + nodes.push_back(n); + constexpr std::array expected_nodes{node1, node2}; + BOOST_TEST_ALL_EQ(nodes.begin(), nodes.end(), expected_nodes.begin(), expected_nodes.end()); +} + +// Empty ranges don't cause trouble +void test_iterators_empty() +{ + flat_tree t; + BOOST_TEST_EQ(t.begin(), t.end()); +} + +// Tmp area is not included in the range +// More or less tested with the add_nodes tests +void test_iterators_tmp() +{ + parser p; + flat_tree t; + BOOST_TEST_NOT(parse_checked(t, p, "*2\r\n+hello\r\n")); + BOOST_TEST_EQ(t.begin(), t.end()); +} + +// The iterator should be contiguous +#ifdef BOOST_REDIS_TEST_RANGE_CONCEPTS +static_assert(std::contiguous_iterator); +#endif + +// --- Reverse iterators --- +// We can obtain iterators using rbegin() and rend() and use them to iterate +void test_reverse_iterators() +{ + // Setup + flat_tree t; + add_nodes(t, "+node1\r\n"); + add_nodes(t, ":200\r\n"); + + // These methods are const + const auto& tconst = t; + + constexpr node_view expected_nodes[] = { + {type::number, 1u, 0u, "200" }, + {type::simple_string, 1u, 0u, "node1"}, + }; + BOOST_TEST_ALL_EQ( + tconst.rbegin(), + tconst.rend(), + std::begin(expected_nodes), + std::end(expected_nodes)); +} + +// Empty ranges don't cause trouble +void test_reverse_iterators_empty() +{ + flat_tree t; + BOOST_TEST(t.rbegin() == t.rend()); +} + +// Tmp area is not included in the range +void test_reverse_iterators_tmp() +{ + parser p; + flat_tree t; + + // Add one full message and a partial one + add_nodes(t, "*1\r\n+node1\r\n"); + BOOST_TEST_NOT(parse_checked(t, p, "*2\r\n+hello\r\n")); + + // Only the full message appears in the reversed range + constexpr node_view expected_nodes[] = { + {type::simple_string, 1u, 1u, "node1"}, + {type::array, 1u, 0u, "" }, + }; + BOOST_TEST_ALL_EQ(t.rbegin(), t.rend(), std::begin(expected_nodes), std::end(expected_nodes)); +} + +// --- at --- +void test_at() +{ + parser p; + flat_tree t; + + // Add one full message and a partial one + add_nodes(t, "*1\r\n+node1\r\n"); + BOOST_TEST_NOT(parse_checked(t, p, "*2\r\n+hello\r\n")); + + // Nodes in the range can be accessed with at() + constexpr node_view n0{type::array, 1u, 0u, ""}; + constexpr node_view n1{type::simple_string, 1u, 1u, "node1"}; + BOOST_TEST_EQ(t.at(0u), n0); + BOOST_TEST_EQ(t.at(1u), n1); + + // Nodes in the tmp area are not considered in range + BOOST_TEST_THROWS(t.at(2u), std::out_of_range); + BOOST_TEST_THROWS(t.at(3u), std::out_of_range); + + // Indices out of range throw + BOOST_TEST_THROWS(t.at(4u), std::out_of_range); + BOOST_TEST_THROWS(t.at(5u), std::out_of_range); + BOOST_TEST_THROWS(t.at((std::numeric_limits::max)()), std::out_of_range); +} + +// Empty ranges don't cause trouble +void test_at_empty() +{ + flat_tree t; + BOOST_TEST_THROWS(t.at(0u), std::out_of_range); + BOOST_TEST_THROWS(t.at(2u), std::out_of_range); + BOOST_TEST_THROWS(t.at((std::numeric_limits::max)()), std::out_of_range); +} + +// --- operator[], front, back --- +void test_unchecked_access() +{ + flat_tree t; + add_nodes(t, "*2\r\n+node1\r\n+node2\r\n"); + + constexpr node_view n0{type::array, 2u, 0u, ""}; + constexpr node_view n1{type::simple_string, 1u, 1u, "node1"}; + constexpr node_view n2{type::simple_string, 1u, 1u, "node2"}; + + // operator [] + BOOST_TEST_EQ(t[0u], n0); + BOOST_TEST_EQ(t[1u], n1); + BOOST_TEST_EQ(t[2u], n2); + + // Front and back + BOOST_TEST_EQ(t.front(), n0); + BOOST_TEST_EQ(t.back(), n2); +} + +// --- data --- +void test_data() +{ + flat_tree t; + add_nodes(t, "*1\r\n+node1\r\n"); + + constexpr node_view expected_nodes[] = { + {type::array, 1u, 0u, "" }, + {type::simple_string, 1u, 1u, "node1"}, + }; + + BOOST_TEST_NE(t.data(), nullptr); + BOOST_TEST_ALL_EQ(t.data(), t.data() + 2u, std::begin(expected_nodes), std::end(expected_nodes)); +} + +// Empty ranges don't cause trouble +void test_data_empty() +{ + flat_tree t; + BOOST_TEST_EQ(t.data(), nullptr); +} + +// --- size and empty --- +void test_size() +{ + flat_tree t; + add_nodes(t, "*1\r\n+node1\r\n"); + + BOOST_TEST_EQ(t.size(), 2u); + BOOST_TEST_NOT(t.empty()); +} + +void test_size_empty() +{ + flat_tree t; + + BOOST_TEST_EQ(t.size(), 0u); + BOOST_TEST(t.empty()); +} + +// Tmp area not taken into account +void test_size_tmp() +{ + parser p; + flat_tree t; + + // Add one full message and a partial one + add_nodes(t, "*1\r\n+node1\r\n"); + BOOST_TEST_NOT(parse_checked(t, p, "*2\r\n+hello\r\n")); + + BOOST_TEST_EQ(t.size(), 2u); + BOOST_TEST_NOT(t.empty()); +} + +void test_size_tmp_only() +{ + parser p; + flat_tree t; + + // Add one partial message + BOOST_TEST_NOT(parse_checked(t, p, "*2\r\n+hello\r\n")); + + BOOST_TEST_EQ(t.size(), 0u); + BOOST_TEST(t.empty()); +} + +// The range should model contiguous range +#ifdef BOOST_REDIS_TEST_RANGE_CONCEPTS +static_assert(std::ranges::contiguous_range); +#endif + // --- Comparison --- void test_comparison_different() { @@ -1310,6 +1543,27 @@ int main() test_move_assign_both_empty(); test_move_assign_tmp(); + test_iterators(); + test_iterators_empty(); + test_iterators_tmp(); + + test_reverse_iterators(); + test_reverse_iterators_empty(); + test_reverse_iterators_tmp(); + + test_at(); + test_at_empty(); + + test_unchecked_access(); + + test_data(); + test_data_empty(); + + test_size(); + test_size_empty(); + test_size_tmp(); + test_size_tmp_only(); + test_comparison_different(); test_comparison_different_node_types(); test_comparison_equal(); diff --git a/test/test_generic_flat_response.cpp b/test/test_generic_flat_response.cpp index ff9e2b863..b667f5bd8 100644 --- a/test/test_generic_flat_response.cpp +++ b/test/test_generic_flat_response.cpp @@ -36,11 +36,7 @@ void test_success() std::vector expected_nodes{ {type::simple_string, 1u, 0u, "hello"}, }; - BOOST_TEST_ALL_EQ( - resp->get_view().begin(), - resp->get_view().end(), - expected_nodes.begin(), - expected_nodes.end()); + BOOST_TEST_ALL_EQ(resp->begin(), resp->end(), expected_nodes.begin(), expected_nodes.end()); } // If an error of any kind appears, we set the overall result to error From d3013c449b9eb92b2eb77f81d07ea5003a9aebf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anarthal=20=28Rub=C3=A9n=20P=C3=A9rez=29?= <34971811+anarthal@users.noreply.github.com> Date: Wed, 28 Jan 2026 11:07:51 +0100 Subject: [PATCH 31/37] Refactors the Sentinel response parser to use flat_tree (#391) The code now uses checked access via flat_tree::at, for hardening. --- .../boost/redis/detail/connection_state.hpp | 4 +- .../boost/redis/impl/sentinel_resolve_fsm.ipp | 1 - include/boost/redis/impl/sentinel_utils.hpp | 233 +++++++++--------- test/sansio_utils.cpp | 11 +- test/sansio_utils.hpp | 5 +- test/test_parse_sentinel_response.cpp | 57 ++++- test/test_sentinel_resolve_fsm.cpp | 32 +-- 7 files changed, 187 insertions(+), 156 deletions(-) diff --git a/include/boost/redis/detail/connection_state.hpp b/include/boost/redis/detail/connection_state.hpp index e8e091061..4c938a292 100644 --- a/include/boost/redis/detail/connection_state.hpp +++ b/include/boost/redis/detail/connection_state.hpp @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include @@ -56,7 +56,7 @@ struct connection_state { // Sentinel stuff lazy_random_engine eng{}; std::vector
sentinels{}; - std::vector sentinel_resp_nodes{}; // for parsing + resp3::flat_tree sentinel_resp_nodes{}; // for parsing }; } // namespace boost::redis::detail diff --git a/include/boost/redis/impl/sentinel_resolve_fsm.ipp b/include/boost/redis/impl/sentinel_resolve_fsm.ipp index 96cf28f92..c4359e63b 100644 --- a/include/boost/redis/impl/sentinel_resolve_fsm.ipp +++ b/include/boost/redis/impl/sentinel_resolve_fsm.ipp @@ -141,7 +141,6 @@ sentinel_action sentinel_resolve_fsm::resume( update_sentinel_list(st.sentinels, idx_, resp.sentinels, st.cfg.sentinel.addresses); - st.sentinel_resp_nodes.clear(); // reduce memory consumption return system::error_code(); } diff --git a/include/boost/redis/impl/sentinel_utils.hpp b/include/boost/redis/impl/sentinel_utils.hpp index b3df430ba..6b52ce7ea 100644 --- a/include/boost/redis/impl/sentinel_utils.hpp +++ b/include/boost/redis/impl/sentinel_utils.hpp @@ -11,11 +11,11 @@ #include #include +#include #include #include #include -#include #include #include @@ -43,71 +43,131 @@ inline void compose_sentinel_request(config& cfg) // Note that we don't care about request flags because this is a one-time request } +// Helper +inline system::error_code sentinel_check_errors(const resp3::node_view& nd, std::string& diag) +{ + switch (nd.data_type) { + case resp3::type::simple_error: diag = nd.value; return error::resp3_simple_error; + case resp3::type::blob_error: diag = nd.value; return error::resp3_blob_error; + default: return system::error_code(); + } +}; + // Parses a list of replicas or sentinels inline system::error_code parse_server_list( - const resp3::node*& first, - const resp3::node* last, + const resp3::flat_tree& tree, + std::size_t& index, + std::string& diag, std::vector
& out) { - const auto* it = first; - ignore_unused(last); + const auto& root = tree.at(index); + BOOST_ASSERT(root.depth == 0u); + + // If the command failed, this will be an error + if (auto ec = sentinel_check_errors(root, diag)) + return ec; // The root node must be an array - BOOST_ASSERT(it != last); - BOOST_ASSERT(it->depth == 0u); - if (it->data_type != resp3::type::array) - return {error::expects_resp3_array}; - const std::size_t num_servers = it->aggregate_size; - ++it; + if (root.data_type != resp3::type::array) + return error::expects_resp3_array; + const std::size_t num_servers = root.aggregate_size; + ++index; // Each element in the array represents a server out.resize(num_servers); for (std::size_t i = 0u; i < num_servers; ++i) { // A server is a map (resp3) or array (resp2, currently unsupported) - BOOST_ASSERT(it != last); - BOOST_ASSERT(it->depth == 1u); - if (it->data_type != resp3::type::map) - return {error::expects_resp3_map}; - const std::size_t num_key_values = it->aggregate_size; - ++it; + const auto& server_node = tree.at(index); + BOOST_ASSERT(server_node.depth == 1u); + if (server_node.data_type != resp3::type::map) + return error::expects_resp3_map; + const std::size_t num_key_values = server_node.aggregate_size; + ++index; // The server object is composed by a set of key/value pairs. // Skip everything except for the ones we care for. bool ip_seen = false, port_seen = false; for (std::size_t j = 0; j < num_key_values; ++j) { // Key. It should be a string - BOOST_ASSERT(it != last); - BOOST_ASSERT(it->depth == 2u); - if (it->data_type != resp3::type::blob_string) - return {error::expects_resp3_string}; - const std::string_view key = it->value; - ++it; + const auto& key_node = tree.at(index); + BOOST_ASSERT(key_node.depth == 2u); + if (key_node.data_type != resp3::type::blob_string) + return error::expects_resp3_string; + const std::string_view key = key_node.value; + ++index; // Value. All values seem to be strings, too. - BOOST_ASSERT(it != last); - BOOST_ASSERT(it->depth == 2u); - if (it->data_type != resp3::type::blob_string) - return {error::expects_resp3_string}; + const auto& value_node = tree.at(index); + BOOST_ASSERT(value_node.depth == 2u); + if (value_node.data_type != resp3::type::blob_string) + return error::expects_resp3_string; // Record it if (key == "ip") { ip_seen = true; - out[i].host = it->value; + out[i].host = value_node.value; } else if (key == "port") { port_seen = true; - out[i].port = it->value; + out[i].port = value_node.value; } - ++it; + ++index; } // Check that the response actually contained the fields we wanted if (!ip_seen || !port_seen) - return {error::empty_field}; + return error::empty_field; } // Done - first = it; + return system::error_code(); +} + +// Parses the output of SENTINEL GET-MASTER-ADDR-BY-NAME +inline system::error_code parse_get_master_addr_by_name( + const resp3::flat_tree& tree, + std::size_t& index, + std::string& diag, + address& out) +{ + const auto& root_node = tree.at(index); + BOOST_ASSERT(root_node.depth == 0u); + + // Check for errors + if (auto ec = sentinel_check_errors(root_node, diag)) + return ec; + + // If the root node is NULL, Sentinel doesn't know about this master. + // We use resp3_null to signal this fact. This doesn't reach the end user. + // If this is the case, SENTINEL REPLICAS and SENTINEL SENTINELS will fail. + // We exit here so the diagnostic is clean. + if (root_node.data_type == resp3::type::null) { + return error::resp3_null; + } + + // If the root node is an array, an IP and port follow + if (root_node.data_type != resp3::type::array) + return error::expects_resp3_array; + if (root_node.aggregate_size != 2u) + return error::incompatible_size; + ++index; + + // IP + const auto& ip_node = tree.at(index); + BOOST_ASSERT(ip_node.depth == 1u); + if (ip_node.data_type != resp3::type::blob_string) + return error::expects_resp3_string; + out.host = ip_node.value; + ++index; + + // Port + const auto& port_node = tree.at(index); + BOOST_ASSERT(port_node.depth == 1u); + if (port_node.data_type != resp3::type::blob_string) + return error::expects_resp3_string; + out.port = port_node.value; + ++index; + return system::error_code(); } @@ -130,114 +190,53 @@ struct sentinel_response { // This means that we can't use generic_response, since its adapter errors on error nodes. // SENTINEL GET-MASTER-ADDR-BY-NAME is sent even when connecting to replicas // for better diagnostics when the master name is unknown. -// Preconditions: -// * There are at least 2 (master)/3 (replica) root nodes. -// * The node array originates from parsing a valid RESP3 message. -// E.g. we won't check that the first node has depth 0. +// Note that the tree should originate from a valid RESP3 message +// (i.e. we won't check that the first node has depth 0.) inline system::error_code parse_sentinel_response( - span nodes, + const resp3::flat_tree& tree, role server_role, sentinel_response& out) { - auto check_errors = [&out](const resp3::node& nd) { - switch (nd.data_type) { - case resp3::type::simple_error: - out.diagnostic = nd.value; - return system::error_code(error::resp3_simple_error); - case resp3::type::blob_error: - out.diagnostic = nd.value; - return system::error_code(error::resp3_blob_error); - default: return system::error_code(); - } - }; - // Clear the output out.diagnostic.clear(); out.sentinels.clear(); out.replicas.clear(); - // Find the first root node of interest. It's the 2nd or 3rd, starting with the end - auto find_first = [nodes, server_role] { - const std::size_t expected_roots = server_role == role::master ? 2u : 3u; - std::size_t roots_seen = 0u; - for (auto it = nodes.rbegin();; ++it) { - BOOST_ASSERT(it != nodes.rend()); - if (it->depth == 0u && ++roots_seen == expected_roots) - return &*it; - } - }; - const resp3::node* lib_first = find_first(); + // Index-based access + std::size_t index = 0; - // Iterators - const resp3::node* it = nodes.begin(); - const resp3::node* last = nodes.end(); - ignore_unused(last); + // User-supplied commands are before the ones added by us. + // Find out how many responses should we skip + const std::size_t num_lib_msgs = server_role == role::master ? 2u : 3u; + if (tree.get_total_msgs() < num_lib_msgs) + return error::incompatible_size; + const std::size_t num_user_msgs = tree.get_total_msgs() - num_lib_msgs; // Go through all the responses to user-supplied requests checking for errors - for (; it != lib_first; ++it) { - if (auto ec = check_errors(*it)) + BOOST_ASSERT(tree.at(index).depth == 0u); + for (std::size_t remaining_roots = num_user_msgs + 1u;; ++index) { + // Exit at node N+1 + const auto& node = tree.at(index); + if (node.depth == 0u && --remaining_roots == 0u) + break; + + // This is a user-supplied message. Check for errors + if (auto ec = sentinel_check_errors(node, out.diagnostic)) return ec; } // SENTINEL GET-MASTER-ADDR-BY-NAME - - // Check for errors - if (auto ec = check_errors(*it)) + if (auto ec = parse_get_master_addr_by_name(tree, index, out.diagnostic, out.master_addr)) return ec; - // If the root node is NULL, Sentinel doesn't know about this master. - // We use resp3_null to signal this fact. This doesn't reach the end user. - if (it->data_type == resp3::type::null) { - return {error::resp3_null}; - } - - // If the root node is an array, an IP and port follow - if (it->data_type != resp3::type::array) - return {error::expects_resp3_array}; - if (it->aggregate_size != 2u) - return {error::incompatible_size}; - ++it; - - // IP - BOOST_ASSERT(it != last); - BOOST_ASSERT(it->depth == 1u); - if (it->data_type != resp3::type::blob_string) - return {error::expects_resp3_string}; - out.master_addr.host = it->value; - ++it; - - // Port - BOOST_ASSERT(it != last); - BOOST_ASSERT(it->depth == 1u); - if (it->data_type != resp3::type::blob_string) - return {error::expects_resp3_string}; - out.master_addr.port = it->value; - ++it; - + // SENTINEL REPLICAS if (server_role == role::replica) { - // SENTINEL REPLICAS - - // This request fails if Sentinel doesn't know about this master. - // However, that's not the case if we got here. - // Check for other errors. - if (auto ec = check_errors(*it)) - return ec; - - // Actual parsing - if (auto ec = parse_server_list(it, last, out.replicas)) + if (auto ec = parse_server_list(tree, index, out.diagnostic, out.replicas)) return ec; } // SENTINEL SENTINELS - - // This request fails if Sentinel doesn't know about this master. - // However, that's not the case if we got here. - // Check for other errors. - if (auto ec = check_errors(*it)) - return ec; - - // Actual parsing - if (auto ec = parse_server_list(it, last, out.sentinels)) + if (auto ec = parse_server_list(tree, index, out.diagnostic, out.sentinels)) return ec; // Done diff --git a/test/sansio_utils.cpp b/test/sansio_utils.cpp index b3e446ac0..90e5529a9 100644 --- a/test/sansio_utils.cpp +++ b/test/sansio_utils.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -74,12 +75,10 @@ logger log_fixture::make_logger() }); } -std::vector nodes_from_resp3( - const std::vector& msgs, - source_location loc) +resp3::flat_tree tree_from_resp3(const std::vector& msgs, source_location loc) { - std::vector nodes; - any_adapter adapter{nodes}; + resp3::flat_tree res; + any_adapter adapter{res}; for (std::string_view resp : msgs) { resp3::parser p; @@ -91,7 +90,7 @@ std::vector nodes_from_resp3( std::cerr << "Called from " << loc << std::endl; } - return nodes; + return res; } } // namespace boost::redis::detail diff --git a/test/sansio_utils.hpp b/test/sansio_utils.hpp index cd4d7ebc6..aba0f7f59 100644 --- a/test/sansio_utils.hpp +++ b/test/sansio_utils.hpp @@ -8,6 +8,7 @@ #define BOOST_REDIS_TEST_SANSIO_UTILS_HPP #include +#include #include #include @@ -52,10 +53,10 @@ constexpr auto to_milliseconds(std::chrono::steady_clock::duration d) return std::chrono::duration_cast(d).count(); } -// Creates a vector of nodes from a set of RESP3 messages. +// Creates a collection of nodes from a set of RESP3 messages. // Using the raw RESP values ensures that the correct // node tree is built, which is not always obvious -std::vector nodes_from_resp3( +resp3::flat_tree tree_from_resp3( const std::vector& msgs, source_location loc = BOOST_CURRENT_LOCATION); diff --git a/test/test_parse_sentinel_response.cpp b/test/test_parse_sentinel_response.cpp index 00a83d752..b77c6abf5 100644 --- a/test/test_parse_sentinel_response.cpp +++ b/test/test_parse_sentinel_response.cpp @@ -25,7 +25,7 @@ #include using namespace boost::redis; -using detail::nodes_from_resp3; +using detail::tree_from_resp3; using detail::parse_sentinel_response; using detail::sentinel_response; using boost::system::error_code; @@ -80,7 +80,7 @@ void test_master() { // Setup fixture fix; - auto nodes = nodes_from_resp3({ + auto nodes = tree_from_resp3({ // clang-format off "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", "*2\r\n" @@ -116,7 +116,7 @@ void test_master_no_sentinels() { // Setup fixture fix; - auto nodes = nodes_from_resp3({ + auto nodes = tree_from_resp3({ "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", "*0\r\n", }); @@ -132,7 +132,7 @@ void test_master_setup_request() { // Setup fixture fix; - auto nodes = nodes_from_resp3({ + auto nodes = tree_from_resp3({ // clang-format off "+OK\r\n", "%6\r\n$6\r\nserver\r\n$5\r\nredis\r\n$7\r\nversion\r\n$5\r\n7.4.2\r\n$5\r\nproto\r\n:3\r\n$2\r\nid\r\n:3\r\n$4\r\nmode\r\n$8\r\nsentinel\r\n$7\r\nmodules\r\n*0\r\n", @@ -170,7 +170,7 @@ void test_master_ip_port_out_of_order() { // Setup fixture fix; - auto nodes = nodes_from_resp3({ + auto nodes = tree_from_resp3({ // clang-format off "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", "*1\r\n" @@ -195,7 +195,7 @@ void test_replica() { // Setup fixture fix; - auto nodes = nodes_from_resp3({ + auto nodes = tree_from_resp3({ // clang-format off "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", "*2\r\n" @@ -254,7 +254,7 @@ void test_replica_no_sentinels() { // Setup fixture fix; - auto nodes = nodes_from_resp3({ + auto nodes = tree_from_resp3({ // clang-format off "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", "*2\r\n" @@ -283,7 +283,7 @@ void test_replica_no_replicas() { // Setup fixture fix; - auto nodes = nodes_from_resp3({ + auto nodes = tree_from_resp3({ // clang-format off "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", "*0\r\n", @@ -304,7 +304,7 @@ void test_replica_setup_request() { // Setup fixture fix; - auto nodes = nodes_from_resp3({ + auto nodes = tree_from_resp3({ // clang-format off "*2\r\n+OK\r\n+OK\r\n", "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", @@ -342,7 +342,7 @@ void test_replica_ip_port_out_of_order() { // Setup fixture fix; - auto nodes = nodes_from_resp3({ + auto nodes = tree_from_resp3({ // clang-format off "*2\r\n$9\r\ntest.host\r\n$4\r\n6389\r\n", "*1\r\n" @@ -688,7 +688,40 @@ void test_errors() }, "", error::empty_field - } + }, + { + "not_enough_messages_1", + role::master, + {}, + "", + error::incompatible_size + }, + { + "not_enough_messages_2", + role::master, + { + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + }, + "", + error::incompatible_size + }, + { + "not_enough_messages_replica_1", + role::replica, + {}, + "", + error::incompatible_size + }, + { + "not_enough_messages_replica_2", + role::replica, + { + "*2\r\n$9\r\nlocalhost\r\n$4\r\n6380\r\n", + "*0\r\n", + }, + "", + error::incompatible_size + }, // clang-format on }; @@ -697,7 +730,7 @@ void test_errors() // Setup std::cerr << "Running error test case: " << tc.name << std::endl; fixture fix; - auto nodes = nodes_from_resp3(tc.responses); + auto nodes = tree_from_resp3(tc.responses); // Call the function auto ec = parse_sentinel_response(nodes, tc.server_role, fix.resp); diff --git a/test/test_sentinel_resolve_fsm.cpp b/test/test_sentinel_resolve_fsm.cpp index 4eb847d93..d3046819a 100644 --- a/test/test_sentinel_resolve_fsm.cpp +++ b/test/test_sentinel_resolve_fsm.cpp @@ -26,7 +26,7 @@ namespace asio = boost::asio; using detail::sentinel_resolve_fsm; using detail::sentinel_action; using detail::connection_state; -using detail::nodes_from_resp3; +using detail::tree_from_resp3; using boost::system::error_code; using boost::asio::cancellation_type_t; @@ -115,7 +115,7 @@ void test_success() // Now send the request act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); BOOST_TEST_EQ(act, sentinel_action::request()); - fix.st.sentinel_resp_nodes = nodes_from_resp3({ + fix.st.sentinel_resp_nodes = tree_from_resp3({ // clang-format off "*2\r\n$9\r\ntest.host\r\n$4\r\n6380\r\n", "*1\r\n" @@ -166,7 +166,7 @@ void test_success_replica() // Now send the request act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); BOOST_TEST_EQ(act, sentinel_action::request()); - fix.st.sentinel_resp_nodes = nodes_from_resp3({ + fix.st.sentinel_resp_nodes = tree_from_resp3({ // clang-format off "*2\r\n$9\r\ntest.host\r\n$4\r\n6380\r\n", "*3\r\n" @@ -215,7 +215,7 @@ void test_one_connect_error() // Now send the request act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); BOOST_TEST_EQ(act, sentinel_action::request()); - fix.st.sentinel_resp_nodes = nodes_from_resp3({ + fix.st.sentinel_resp_nodes = tree_from_resp3({ "*2\r\n$9\r\ntest.host\r\n$4\r\n6380\r\n", "*0\r\n", }); @@ -257,7 +257,7 @@ void test_one_request_network_error() BOOST_TEST_EQ(act, (address{"host2", "2000"})); act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); BOOST_TEST_EQ(act, sentinel_action::request()); - fix.st.sentinel_resp_nodes = nodes_from_resp3({ + fix.st.sentinel_resp_nodes = tree_from_resp3({ "*2\r\n$9\r\ntest.host\r\n$4\r\n6380\r\n", "*0\r\n", }); @@ -292,7 +292,7 @@ void test_one_request_parse_error() BOOST_TEST_EQ(act, (address{"host1", "1000"})); act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); BOOST_TEST_EQ(act, sentinel_action::request()); - fix.st.sentinel_resp_nodes = nodes_from_resp3({ + fix.st.sentinel_resp_nodes = tree_from_resp3({ "+OK\r\n", "+OK\r\n", }); @@ -302,7 +302,7 @@ void test_one_request_parse_error() BOOST_TEST_EQ(act, (address{"host2", "2000"})); act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); BOOST_TEST_EQ(act, sentinel_action::request()); - fix.st.sentinel_resp_nodes = nodes_from_resp3({ + fix.st.sentinel_resp_nodes = tree_from_resp3({ "*2\r\n$9\r\ntest.host\r\n$4\r\n6380\r\n", "*0\r\n", }); @@ -338,7 +338,7 @@ void test_one_request_error_node() BOOST_TEST_EQ(act, (address{"host1", "1000"})); act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); BOOST_TEST_EQ(act, sentinel_action::request()); - fix.st.sentinel_resp_nodes = nodes_from_resp3({ + fix.st.sentinel_resp_nodes = tree_from_resp3({ "-ERR needs authentication\r\n", "-ERR needs authentication\r\n", }); @@ -348,7 +348,7 @@ void test_one_request_error_node() BOOST_TEST_EQ(act, (address{"host2", "2000"})); act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); BOOST_TEST_EQ(act, sentinel_action::request()); - fix.st.sentinel_resp_nodes = nodes_from_resp3({ + fix.st.sentinel_resp_nodes = tree_from_resp3({ "*2\r\n$9\r\ntest.host\r\n$4\r\n6380\r\n", "*0\r\n", }); @@ -383,7 +383,7 @@ void test_one_master_unknown() BOOST_TEST_EQ(act, (address{"host1", "1000"})); act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); BOOST_TEST_EQ(act, sentinel_action::request()); - fix.st.sentinel_resp_nodes = nodes_from_resp3({ + fix.st.sentinel_resp_nodes = tree_from_resp3({ "_\r\n", "-ERR unknown master\r\n", }); @@ -394,7 +394,7 @@ void test_one_master_unknown() BOOST_TEST_EQ(act, (address{"host2", "2000"})); act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); BOOST_TEST_EQ(act, sentinel_action::request()); - fix.st.sentinel_resp_nodes = nodes_from_resp3({ + fix.st.sentinel_resp_nodes = tree_from_resp3({ "*2\r\n$9\r\ntest.host\r\n$4\r\n6380\r\n", "*0\r\n", }); @@ -430,7 +430,7 @@ void test_one_no_replicas() BOOST_TEST_EQ(act, (address{"host1", "1000"})); act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); BOOST_TEST_EQ(act, sentinel_action::request()); - fix.st.sentinel_resp_nodes = nodes_from_resp3({ + fix.st.sentinel_resp_nodes = tree_from_resp3({ "*2\r\n$9\r\ntest.host\r\n$4\r\n6380\r\n", "*0\r\n", "*0\r\n", @@ -441,7 +441,7 @@ void test_one_no_replicas() BOOST_TEST_EQ(act, (address{"host2", "2000"})); act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); BOOST_TEST_EQ(act, sentinel_action::request()); - fix.st.sentinel_resp_nodes = nodes_from_resp3({ + fix.st.sentinel_resp_nodes = tree_from_resp3({ // clang-format off "*2\r\n$9\r\ntest.host\r\n$4\r\n6380\r\n", "*1\r\n" @@ -481,7 +481,7 @@ void test_error() BOOST_TEST_EQ(act, (address{"host1", "1000"})); act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); BOOST_TEST_EQ(act, sentinel_action::request()); - fix.st.sentinel_resp_nodes = nodes_from_resp3({ + fix.st.sentinel_resp_nodes = tree_from_resp3({ "_\r\n", "-ERR unknown master\r\n", }); @@ -495,7 +495,7 @@ void test_error() BOOST_TEST_EQ(act, (address{"host3", "3000"})); act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); BOOST_TEST_EQ(act, sentinel_action::request()); - fix.st.sentinel_resp_nodes = nodes_from_resp3({ + fix.st.sentinel_resp_nodes = tree_from_resp3({ "-ERR unauthorized\r\n", "-ERR unauthorized\r\n", }); @@ -546,7 +546,7 @@ void test_error_replica() BOOST_TEST_EQ(act, (address{"host1", "1000"})); act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); BOOST_TEST_EQ(act, sentinel_action::request()); - fix.st.sentinel_resp_nodes = nodes_from_resp3({ + fix.st.sentinel_resp_nodes = tree_from_resp3({ "*2\r\n$9\r\ntest.host\r\n$4\r\n6380\r\n", "*0\r\n", "*0\r\n", From bd1e22fc8c8a27b898b185e45e1b33874feabadd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anarthal=20=28Rub=C3=A9n=20P=C3=A9rez=29?= <34971811+anarthal@users.noreply.github.com> Date: Fri, 30 Jan 2026 19:30:59 +0100 Subject: [PATCH 32/37] Adds request::hello and deprecates config::username, password (#392) Adds request::hello and request::hello_setname Deprecates config::{username, password, client_name, database_index} Adds a documentation page on authentication --- doc/modules/ROOT/nav.adoc | 1 + doc/modules/ROOT/pages/auth.adoc | 49 +++++++++++++++++++ include/boost/redis/config.hpp | 47 ++++++++++++++++-- include/boost/redis/impl/request.ipp | 30 ++++++++++-- .../boost/redis/impl/setup_request_utils.hpp | 8 +-- include/boost/redis/request.hpp | 45 +++++++++++++++++ test/test_request.cpp | 41 ++++++++++++++++ 7 files changed, 210 insertions(+), 11 deletions(-) create mode 100644 doc/modules/ROOT/pages/auth.adoc diff --git a/doc/modules/ROOT/nav.adoc b/doc/modules/ROOT/nav.adoc index 8fd44b67f..034322e8f 100644 --- a/doc/modules/ROOT/nav.adoc +++ b/doc/modules/ROOT/nav.adoc @@ -2,6 +2,7 @@ * xref:requests_responses.adoc[] * xref:cancellation.adoc[] * xref:serialization.adoc[] +* xref:auth.adoc[] * xref:logging.adoc[] * xref:sentinel.adoc[] * xref:benchmarks.adoc[] diff --git a/doc/modules/ROOT/pages/auth.adoc b/doc/modules/ROOT/pages/auth.adoc new file mode 100644 index 000000000..59e1644fc --- /dev/null +++ b/doc/modules/ROOT/pages/auth.adoc @@ -0,0 +1,49 @@ +// +// Copyright (c) 2026 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + += Authentication + +Boost.Redis supports connecting to servers that require authentication. +To achieve it, you must send a +https://redis.io/commands/hello/[HELLO] command with the appropriate +`AUTH` parameters during connection establishment. Boost.Redis allows you +to customize this handshake by using a _setup request_: a request that +is run automatically before any other request, +every time a physical connection to the server is established. + +Configuration is done on xref:reference:boost/redis/config.adoc[`config`]. +Set `use_setup` to `true` (required for backwards compatibility) +and build the desired setup request in +`config::setup`. By default, the library sends a plain `HELLO 3` (RESP3 +without auth). To authenticate, clear the default setup and push a +`HELLO` command that includes your credentials: + +[source,cpp] +---- +config cfg; +cfg.use_setup = true; +cfg.setup.clear(); // Remove the default HELLO 3 +cfg.setup.hello("my_username", "my_password"); + +co_await conn.async_run(cfg); +---- + +Authentication is just one use of this mechanism. +For example, to select a particular logical database (see the Redis +https://redis.io/commands/select/[SELECT] command), add a `SELECT` +command after the `HELLO`: + +[source,cpp] +---- +config cfg; +cfg.use_setup = true; +cfg.setup.push("SELECT", 42); // select the logical database 42 after the default HELLO 3 + +co_await conn.async_run(cfg); +---- + diff --git a/include/boost/redis/config.hpp b/include/boost/redis/config.hpp index 4e9806794..98bb76247 100644 --- a/include/boost/redis/config.hpp +++ b/include/boost/redis/config.hpp @@ -164,7 +164,7 @@ struct config { */ std::string unix_socket; - /** @brief Username used for authentication during connection establishment. + /** @brief (Deprecated) Username used for authentication during connection establishment. * * If @ref use_setup is false (the default), during connection establishment, * authentication is performed by sending a `HELLO` command. @@ -176,10 +176,20 @@ struct config { * * When using Sentinel, this setting applies to masters and replicas. * Use @ref sentinel_config::setup to configure authorization for Sentinels. + * + * @par Deprecated + * This setting is deprecated and will be removed in a subsequent release. + * Please set @ref setup, instead: + * + * @code + * cfg.use_setup = true; + * cfg.setup.clear(); + * cfg.setup.hello("my_username", "my_password"); + * @endcode */ std::string username = "default"; - /** @brief Password used for authentication during connection establishment. + /** @brief (Deprecated) Password used for authentication during connection establishment. * * If @ref use_setup is false (the default), during connection establishment, * authentication is performed by sending a `HELLO` command. @@ -191,10 +201,20 @@ struct config { * * When using Sentinel, this setting applies to masters and replicas. * Use @ref sentinel_config::setup to configure authorization for Sentinels. + * + * @par Deprecated + * This setting is deprecated and will be removed in a subsequent release. + * Please set @ref setup, instead: + * + * @code + * cfg.use_setup = true; + * cfg.setup.clear(); + * cfg.setup.hello("my_username", "my_password"); + * @endcode */ std::string password; - /** @brief Client name parameter to use during connection establishment. + /** @brief (Deprecated) Client name parameter to use during connection establishment. * * If @ref use_setup is false (the default), during connection establishment, * a `HELLO` command is sent. If this field is not empty, the `HELLO` command @@ -202,10 +222,20 @@ struct config { * * When using Sentinel, this setting applies to masters and replicas. * Use @ref sentinel_config::setup to configure this value for Sentinels. + * + * @par Deprecated + * This setting is deprecated and will be removed in a subsequent release. + * Please set @ref setup, instead: + * + * @code + * cfg.use_setup = true; + * cfg.setup.clear(); + * cfg.setup.hello_setname("my_client_name"); + * @endcode */ std::string clientname = "Boost.Redis"; - /** @brief Database index to pass to the `SELECT` command during connection establishment. + /** @brief (Deprecated) Database index to pass to the `SELECT` command during connection establishment. * * If @ref use_setup is false (the default), and this field is set to a * non-empty optional, and its value is different than zero, @@ -213,6 +243,15 @@ struct config { * database index. By default, no `SELECT` command is sent. * * When using Sentinel, this setting applies to masters and replicas. + * + * @par Deprecated + * This setting is deprecated and will be removed in a subsequent release. + * Please set @ref setup, instead: + * + * @code + * cfg.use_setup = true; + * cfg.setup.push("SELECT", 4); // select database index 4 + * @endcode */ std::optional database_index = 0; diff --git a/include/boost/redis/impl/request.ipp b/include/boost/redis/impl/request.ipp index 72653897b..e00d0f76f 100644 --- a/include/boost/redis/impl/request.ipp +++ b/include/boost/redis/impl/request.ipp @@ -28,13 +28,15 @@ auto has_response(std::string_view cmd) -> bool request make_hello_request() { request req; - req.push("HELLO", "3"); + req.hello(); return req; } } // namespace boost::redis::detail -void boost::redis::request::append(const request& other) +namespace boost::redis { + +void request::append(const request& other) { // Remember the old payload size, to update offsets std::size_t old_offset = payload_.size(); @@ -55,7 +57,7 @@ void boost::redis::request::append(const request& other) } } -void boost::redis::request::add_pubsub_arg(detail::pubsub_change_type type, std::string_view value) +void request::add_pubsub_arg(detail::pubsub_change_type type, std::string_view value) { // Add the argument resp3::add_bulk(payload_, value); @@ -65,3 +67,25 @@ void boost::redis::request::add_pubsub_arg(detail::pubsub_change_type type, std: std::size_t offset = payload_.size() - value.size() - 2u; pubsub_changes_.push_back({type, offset, value.size()}); } + +void request::hello() { push("HELLO", "3"); } + +void request::hello(std::string_view username, std::string_view password) +{ + push("HELLO", "3", "AUTH", username, password); +} + +void request::hello_setname(std::string_view client_name) +{ + push("HELLO", "3", "SETNAME", client_name); +} + +void request::hello_setname( + std::string_view username, + std::string_view password, + std::string_view client_name) +{ + push("HELLO", "3", "AUTH", username, password, "SETNAME", client_name); +} + +} // namespace boost::redis diff --git a/include/boost/redis/impl/setup_request_utils.hpp b/include/boost/redis/impl/setup_request_utils.hpp index 2fb0c6033..33451bae7 100644 --- a/include/boost/redis/impl/setup_request_utils.hpp +++ b/include/boost/redis/impl/setup_request_utils.hpp @@ -53,13 +53,13 @@ inline void compose_setup_request( // Gather everything we can in a HELLO command if (send_auth && send_setname) - req.push("HELLO", "3", "AUTH", cfg.username, cfg.password, "SETNAME", cfg.clientname); + req.hello_setname(cfg.username, cfg.password, cfg.clientname); else if (send_auth) - req.push("HELLO", "3", "AUTH", cfg.username, cfg.password); + req.hello(cfg.username, cfg.password); else if (send_setname) - req.push("HELLO", "3", "SETNAME", cfg.clientname); + req.hello_setname(cfg.clientname); else - req.push("HELLO", "3"); + req.hello(); // SELECT is independent of HELLO if (cfg.database_index && cfg.database_index.value() != 0) diff --git a/include/boost/redis/request.hpp b/include/boost/redis/request.hpp index d3104a785..c6a4ef21b 100644 --- a/include/boost/redis/request.hpp +++ b/include/boost/redis/request.hpp @@ -724,6 +724,51 @@ class request { patterns_end); } + /** @brief Appends a HELLO 3 command to the end of the request. + * + * Equivalent to adding the Redis command `HELLO 3`. + */ + void hello(); + + /** @brief Appends a HELLO 3 command with AUTH to the end of the request. + * + * Equivalent to the adding the following Redis command: + * @code + * HELLO 3 AUTH + * @endcode + * + * @param username The ACL username. + * @param password The password for the user. + */ + void hello(std::string_view username, std::string_view password); + + /** @brief Appends a HELLO 3 command with SETNAME to the end of the request. + * + * Equivalent to adding the following Redis command: + * @code + * HELLO 3 SETNAME + * @endcode + * + * @param client_name The client name (visible in CLIENT LIST). + */ + void hello_setname(std::string_view client_name); + + /** @brief Appends a HELLO 3 command with AUTH and SETNAME to the end of the request. + * + * Equivalent to adding the following Redis command: + * @code + * HELLO 3 AUTH SETNAME + * @endcode + * + * @param username The ACL username. + * @param password The password for the user. + * @param client_name The client name (visible in CLIENT LIST). + */ + void hello_setname( + std::string_view username, + std::string_view password, + std::string_view client_name); + private: void check_cmd(std::string_view cmd) { diff --git a/test/test_request.cpp b/test/test_request.cpp index 9777e790c..a43469fbc 100644 --- a/test/test_request.cpp +++ b/test/test_request.cpp @@ -467,6 +467,42 @@ void test_mix_pubsub_regular() check_pubsub_changes(req, expected_changes); } +// --- hello --- +void test_hello() +{ + request req; + req.hello(); + BOOST_TEST_EQ(req.payload(), "*2\r\n$5\r\nHELLO\r\n$1\r\n3\r\n"); +} + +void test_hello_auth() +{ + request req; + req.hello("user", "pass"); + BOOST_TEST_EQ( + req.payload(), + "*5\r\n$5\r\nHELLO\r\n$1\r\n3\r\n$4\r\nAUTH\r\n$4\r\nuser\r\n$4\r\npass\r\n"); +} + +void test_hello_setname() +{ + request req; + req.hello_setname("myclient"); + BOOST_TEST_EQ( + req.payload(), + "*4\r\n$5\r\nHELLO\r\n$1\r\n3\r\n$7\r\nSETNAME\r\n$8\r\nmyclient\r\n"); +} + +void test_hello_setname_auth() +{ + request req; + req.hello_setname("user", "pass", "myclient"); + BOOST_TEST_EQ( + req.payload(), + "*7\r\n$5\r\nHELLO\r\n$1\r\n3\r\n$4\r\nAUTH\r\n$4\r\nuser\r\n$4\r\npass\r\n" + "$7\r\nSETNAME\r\n$8\r\nmyclient\r\n"); +} + // --- append --- void test_append() { @@ -703,6 +739,11 @@ int main() test_mix_pubsub_regular(); + test_hello(); + test_hello_auth(); + test_hello_setname(); + test_hello_setname_auth(); + test_append(); test_append_no_response(); test_append_flags(); From b7b93c96dae038f7817d653a35e7225b83d30ae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anarthal=20=28Rub=C3=A9n=20P=C3=A9rez=29?= <34971811+anarthal@users.noreply.github.com> Date: Fri, 30 Jan 2026 23:41:28 +0100 Subject: [PATCH 33/37] Increases the level at which error messages are logged (#389) close #377 --- include/boost/redis/impl/reader_fsm.ipp | 19 +++-- include/boost/redis/impl/run_fsm.ipp | 10 ++- include/boost/redis/impl/writer_fsm.ipp | 5 +- test/test_reader_fsm.cpp | 97 ++++++++++++++++++------- test/test_run_fsm.cpp | 67 ++++++++++++++--- test/test_writer_fsm.cpp | 14 ++-- 6 files changed, 155 insertions(+), 57 deletions(-) diff --git a/include/boost/redis/impl/reader_fsm.ipp b/include/boost/redis/impl/reader_fsm.ipp index 5c1539eb2..f04dff4d7 100644 --- a/include/boost/redis/impl/reader_fsm.ipp +++ b/include/boost/redis/impl/reader_fsm.ipp @@ -29,7 +29,7 @@ reader_fsm::action reader_fsm::resume( // Prepare the buffer for the read operation ec = st.mpx.prepare_read(); if (ec) { - log_debug(st.logger, "Reader task: error in prepare_read: ", ec); + log_err(st.logger, "Error preparing the read buffer: ", ec); return {ec}; } @@ -53,10 +53,9 @@ reader_fsm::action reader_fsm::resume( } // Log what we read + log_debug(st.logger, "Reader task: ", bytes_read, " bytes read"); if (ec) { - log_debug(st.logger, "Reader task: ", bytes_read, " bytes read, error: ", ec); - } else { - log_debug(st.logger, "Reader task: ", bytes_read, " bytes read"); + log_err(st.logger, "Error reading data from the server: ", ec); } // Process the bytes read, even if there was an error @@ -77,7 +76,12 @@ reader_fsm::action reader_fsm::resume( if (ec) { // TODO: Perhaps log what has not been consumed to aid // debugging. - log_debug(st.logger, "Reader task: error processing message: ", ec); + if (ec == error::resp3_hello) { + // This is already logged in the setup adapter + log_debug(st.logger, "Error processing message: setup request error"); + } else { + log_err(st.logger, "Error processing message: ", ec); + } return ec; } @@ -94,9 +98,10 @@ reader_fsm::action reader_fsm::resume( return system::error_code(asio::error::operation_aborted); } - // Check for other errors + // Check for other errors. + // We should't get any in the real world, but just in case. if (ec) { - log_debug(st.logger, "Reader task: error notifying push receiver: ", ec); + log_err(st.logger, "Error notifying push receiver: ", ec); return ec; } } else { diff --git a/include/boost/redis/impl/run_fsm.ipp b/include/boost/redis/impl/run_fsm.ipp index d477d549d..8315665e9 100644 --- a/include/boost/redis/impl/run_fsm.ipp +++ b/include/boost/redis/impl/run_fsm.ipp @@ -50,10 +50,12 @@ inline void on_setup_done(const multiplexer::elem& elm, connection_state& st) { const auto ec = elm.get_error(); if (ec) { - if (st.diagnostic.empty()) { - log_info(st.logger, "Setup request execution: ", ec); + if (ec == error::resp3_hello) { + // This is the most common case, and the only one that generates a string diagnostic + log_err(st.logger, "Setup request execution failed: ", st.diagnostic); } else { - log_info(st.logger, "Setup request execution: ", ec, " (", st.diagnostic, ")"); + // Something else went wrong (e.g. network error while running the request). + log_err(st.logger, "Setup request execution failed: ", ec); } } else { log_info(st.logger, "Setup request execution: success"); @@ -144,7 +146,7 @@ run_action run_fsm::resume( if (ec) { // There was an error. Skip to the reconnection loop - log_info( + log_err( st.logger, "Failed to connect to Redis server at ", get_server_address(st), diff --git a/include/boost/redis/impl/writer_fsm.ipp b/include/boost/redis/impl/writer_fsm.ipp index f4d3f779d..460f21c68 100644 --- a/include/boost/redis/impl/writer_fsm.ipp +++ b/include/boost/redis/impl/writer_fsm.ipp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -39,7 +40,7 @@ inline void process_ping_node( } if (ec) { - log_info(lgr, "Health checker: server answered ping with an error: ", nd.value); + log_err(lgr, "Health checker: server answered ping with an error: ", nd.value); } } @@ -89,7 +90,7 @@ writer_action writer_fsm::resume( if (ec == asio::error::operation_aborted) { log_debug(st.logger, "Writer task: cancelled (1)."); } else { - log_debug(st.logger, "Writer task error: ", ec); + log_err(st.logger, "Error writing data to the server: ", ec); } return ec; } diff --git a/test/test_reader_fsm.cpp b/test/test_reader_fsm.cpp index 75664aee1..b5cc60c04 100644 --- a/test/test_reader_fsm.cpp +++ b/test/test_reader_fsm.cpp @@ -7,9 +7,12 @@ #include #include +#include #include #include #include +#include +#include #include #include @@ -19,20 +22,17 @@ #include "sansio_utils.hpp" #include +#include #include +using namespace boost::redis; namespace net = boost::asio; -namespace redis = boost::redis; using boost::system::error_code; using net::cancellation_type_t; -using redis::detail::reader_fsm; -using redis::detail::multiplexer; -using redis::generic_response; -using redis::any_adapter; -using redis::config; -using redis::detail::connection_state; -using action = redis::detail::reader_fsm::action; -using redis::logger; +using detail::reader_fsm; +using detail::multiplexer; +using detail::connection_state; +using action = detail::reader_fsm::action; using namespace std::chrono_literals; // Operators @@ -95,7 +95,7 @@ void copy_to(multiplexer& mpx, std::string_view data) std::copy(data.cbegin(), data.cend(), buffer.begin()); } -struct fixture : redis::detail::log_fixture { +struct fixture : detail::log_fixture { connection_state st{{make_logger()}}; generic_response resp; @@ -246,14 +246,15 @@ void test_read_error() copy_to(fix.st.mpx, payload); // Deliver the data - act = fsm.resume(fix.st, payload.size(), {redis::error::empty_field}, cancellation_type_t::none); - BOOST_TEST_EQ(act, error_code{redis::error::empty_field}); + act = fsm.resume(fix.st, payload.size(), {error::empty_field}, cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code{error::empty_field}); // Check logging fix.check_log({ // clang-format off - {logger::level::debug, "Reader task: issuing read" }, - {logger::level::debug, "Reader task: 11 bytes read, error: Expected field value is empty. [boost.redis:5]"}, + {logger::level::debug, "Reader task: issuing read" }, + {logger::level::debug, "Reader task: 11 bytes read" }, + {logger::level::err, "Error reading data from the server: Expected field value is empty. [boost.redis:5]"}, // clang-format on }); } @@ -270,13 +271,14 @@ void test_read_timeout() // Timeout act = fsm.resume(fix.st, 0, {net::error::operation_aborted}, cancellation_type_t::none); - BOOST_TEST_EQ(act, error_code{redis::error::pong_timeout}); + BOOST_TEST_EQ(act, error_code{error::pong_timeout}); // Check logging fix.check_log({ // clang-format off - {logger::level::debug, "Reader task: issuing read" }, - {logger::level::debug, "Reader task: 0 bytes read, error: Pong timeout. [boost.redis:19]"}, + {logger::level::debug, "Reader task: issuing read" }, + {logger::level::debug, "Reader task: 0 bytes read" }, + {logger::level::err, "Error reading data from the server: Pong timeout. [boost.redis:19]"}, // clang-format on }); } @@ -296,18 +298,60 @@ void test_parse_error() // Deliver the data act = fsm.resume(fix.st, payload.size(), {}, cancellation_type_t::none); - BOOST_TEST_EQ(act, error_code{redis::error::not_a_number}); + BOOST_TEST_EQ(act, error_code{error::not_a_number}); // Check logging fix.check_log({ {logger::level::debug, "Reader task: issuing read"}, {logger::level::debug, "Reader task: 4 bytes read"}, - {logger::level::debug, - "Reader task: error processing message: Can't convert string to number (maybe forgot to " + {logger::level::err, + "Error processing message: Can't convert string to number (maybe forgot to " "upgrade to RESP3?). [boost.redis:2]" }, }); } +// A setup request error is similar to a parse error. +// The adapter would return error::resp3_hello. +// We log this somewhere else, so it shouldn't be logged here by default. +void test_setup_request_error() +{ + // Setup + fixture fix; + reader_fsm fsm; + request req; + req.push("PING"); // should have 1 command + auto elem = std::make_shared( + req, + any_adapter{[](any_adapter::parse_event, const resp3::node_view&, error_code& ec) { + ec = error::resp3_hello; + }}); + elem->set_done_callback([] { }); + + // Add the request to the multiplexer and simulate a successful write + fix.st.mpx.add(elem); + BOOST_TEST_EQ(fix.st.mpx.prepare_write(), 1u); + BOOST_TEST(fix.st.mpx.commit_write(fix.st.mpx.get_write_buffer().size())); + + // Initiate + auto act = fsm.resume(fix.st, 0, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, action::read_some(6s)); + + // The fsm is asking for data. + std::string const payload = "-ERR: bad\r\n"; + copy_to(fix.st.mpx, payload); + + // Deliver the data + act = fsm.resume(fix.st, payload.size(), {}, cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code{error::resp3_hello}); + + // Check logging + fix.check_log({ + {logger::level::debug, "Reader task: issuing read" }, + {logger::level::debug, "Reader task: 11 bytes read" }, + {logger::level::debug, "Error processing message: setup request error"}, + }); +} + void test_push_deliver_error() { fixture fix; @@ -326,15 +370,15 @@ void test_push_deliver_error() BOOST_TEST_EQ(act, action::notify_push_receiver(11u)); // Resumes from notifying a push with an error. - act = fsm.resume(fix.st, 0, redis::error::empty_field, cancellation_type_t::none); - BOOST_TEST_EQ(act, error_code{redis::error::empty_field}); + act = fsm.resume(fix.st, 0, error::empty_field, cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code{error::empty_field}); // Check logging fix.check_log({ // clang-format off {logger::level::debug, "Reader task: issuing read" }, {logger::level::debug, "Reader task: 11 bytes read" }, - {logger::level::debug, "Reader task: error notifying push receiver: Expected field value is empty. [boost.redis:5]"}, + {logger::level::err, "Error notifying push receiver: Expected field value is empty. [boost.redis:5]"}, // clang-format on }); } @@ -355,15 +399,15 @@ void test_max_read_buffer_size() std::string const part1 = ">3\r\n"; copy_to(fix.st.mpx, part1); act = fsm.resume(fix.st, part1.size(), error_code(), cancellation_type_t::none); - BOOST_TEST_EQ(act, error_code(redis::error::exceeds_maximum_read_buffer_size)); + BOOST_TEST_EQ(act, error_code(error::exceeds_maximum_read_buffer_size)); // Check logging fix.check_log({ {logger::level::debug, "Reader task: issuing read" }, {logger::level::debug, "Reader task: 4 bytes read" }, {logger::level::debug, "Reader task: incomplete message received"}, - {logger::level::debug, - "Reader task: error in prepare_read: Reading data from the socket would exceed the maximum " + {logger::level::err, + "Error preparing the read buffer: Reading data from the socket would exceed the maximum " "size allowed of the read buffer. [boost.redis:26]" }, }); } @@ -494,6 +538,7 @@ int main() test_read_error(); test_read_timeout(); test_parse_error(); + test_setup_request_error(); test_push_deliver_error(); test_max_read_buffer_size(); diff --git a/test/test_run_fsm.cpp b/test/test_run_fsm.cpp index 53f413f6f..25558fd5f 100644 --- a/test/test_run_fsm.cpp +++ b/test/test_run_fsm.cpp @@ -191,7 +191,7 @@ void test_connect_error() fix.check_log({ // clang-format off {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)" }, - {logger::level::info, "Failed to connect to Redis server at 127.0.0.1:6379 (TLS disabled): Connect timeout. [boost.redis:18]"}, + {logger::level::err, "Failed to connect to Redis server at 127.0.0.1:6379 (TLS disabled): Connect timeout. [boost.redis:18]"}, {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)" }, {logger::level::info, "Connected to Redis server at 127.0.0.1:6379 (TLS disabled)" }, // clang-format on @@ -224,7 +224,7 @@ void test_connect_error_ssl() fix.check_log({ // clang-format off {logger::level::info, "Trying to connect to Redis server at my_hostname:10000 (TLS enabled)" }, - {logger::level::info, "Failed to connect to Redis server at my_hostname:10000 (TLS enabled): Connect timeout. [boost.redis:18]"}, + {logger::level::err, "Failed to connect to Redis server at my_hostname:10000 (TLS enabled): Connect timeout. [boost.redis:18]"}, {logger::level::info, "Trying to connect to Redis server at my_hostname:10000 (TLS enabled)" }, {logger::level::info, "Connected to Redis server at my_hostname:10000 (TLS enabled)" }, // clang-format on @@ -256,7 +256,7 @@ void test_connect_error_unix() fix.check_log({ // clang-format off {logger::level::info, "Trying to connect to Redis server at '/tmp/sock'" }, - {logger::level::info, "Failed to connect to Redis server at '/tmp/sock': Connect timeout. [boost.redis:18]"}, + {logger::level::err, "Failed to connect to Redis server at '/tmp/sock': Connect timeout. [boost.redis:18]"}, {logger::level::info, "Trying to connect to Redis server at '/tmp/sock'" }, {logger::level::info, "Connected to Redis server at '/tmp/sock'" }, // clang-format on @@ -282,7 +282,7 @@ void test_connect_error_no_reconnect() fix.check_log({ // clang-format off {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)" }, - {logger::level::info, "Failed to connect to Redis server at 127.0.0.1:6379 (TLS disabled): Connect timeout. [boost.redis:18]"}, + {logger::level::err, "Failed to connect to Redis server at 127.0.0.1:6379 (TLS disabled): Connect timeout. [boost.redis:18]"}, // clang-format on }); } @@ -533,10 +533,10 @@ void test_several_reconnections() // Log fix.check_log({ // clang-format off - {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)" }, - {logger::level::info, "Failed to connect to Redis server at 127.0.0.1:6379 (TLS disabled): Connect timeout. [boost.redis:18]"}, - {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)" }, - {logger::level::info, "Connected to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + {logger::level::err, "Failed to connect to Redis server at 127.0.0.1:6379 (TLS disabled): Connect timeout. [boost.redis:18]"}, + {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + {logger::level::info, "Connected to Redis server at 127.0.0.1:6379 (TLS disabled)" }, {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)"}, {logger::level::info, "Connected to Redis server at 127.0.0.1:6379 (TLS disabled)" }, {logger::level::debug, "Run: cancelled (2)" } // clang-format on @@ -672,9 +672,51 @@ void test_setup_request_server_error() fix.check_log({ {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)"}, {logger::level::info, "Connected to Redis server at 127.0.0.1:6379 (TLS disabled)" }, - {logger::level::info, - "Setup request execution: The server response to the setup request sent during connection " - "establishment contains an error. [boost.redis:23] (ERR: wrong command)" } + {logger::level::err, "Setup request execution failed: ERR: wrong command" } + }); +} + +// If the setup request finishes with another error (e.g. network error), we log it +void test_setup_request_other_error() +{ + // Setup + fixture fix; + fix.st.diagnostic = "leftover"; // simulate a leftover from previous runs + fix.st.cfg.setup.clear(); + fix.st.cfg.setup.push("HELLO", 3); + fix.st.cfg.reconnect_wait_interval = 0s; + + // Run the operation. We connect and launch the tasks + auto act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::connect); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::parallel_group); + + // At this point, the setup request should be already queued. + // Simulate the writer + BOOST_TEST_EQ(fix.st.mpx.prepare_write(), 1u); + BOOST_TEST(fix.st.mpx.commit_write(fix.st.mpx.get_write_buffer().size())); + + // Simulate a read error + read(fix.st.mpx, "*malformed\r\n"); + error_code ec; + auto res = fix.st.mpx.consume(ec); + BOOST_TEST_EQ(ec, error::not_a_number); + BOOST_TEST(res.first == detail::consume_result::got_response); + + // This will cause the writer to exit + act = fix.fsm.resume(fix.st, error::not_a_number, cancellation_type_t::none); + BOOST_TEST_EQ(act, run_action_type::cancel_receive); + act = fix.fsm.resume(fix.st, error_code(), cancellation_type_t::none); + BOOST_TEST_EQ(act, error_code(error::not_a_number)); + + // Check log + fix.check_log({ + // clang-format off + {logger::level::info, "Trying to connect to Redis server at 127.0.0.1:6379 (TLS disabled)"}, + {logger::level::info, "Connected to Redis server at 127.0.0.1:6379 (TLS disabled)" }, + {logger::level::err, "Setup request execution failed: Can't convert string to number (maybe forgot to upgrade to RESP3?). [boost.redis:2]" }, + // clang-format on }); } @@ -733,7 +775,7 @@ void test_sentinel_reconnection() fix.check_log({ // clang-format off {logger::level::info, "Trying to connect to Redis server at host1:1000 (TLS disabled)"}, - {logger::level::info, "Failed to connect to Redis server at host1:1000 (TLS disabled): Connect timeout. [boost.redis:18]"}, + {logger::level::err, "Failed to connect to Redis server at host1:1000 (TLS disabled): Connect timeout. [boost.redis:18]"}, {logger::level::info, "Trying to connect to Redis server at host2:2000 (TLS disabled)"}, {logger::level::info, "Connected to Redis server at host2:2000 (TLS disabled)"}, {logger::level::info, "Setup request execution: success"}, @@ -847,6 +889,7 @@ int main() test_setup_request_success(); test_setup_request_empty(); test_setup_request_server_error(); + test_setup_request_other_error(); test_sentinel_reconnection(); test_sentinel_resolve_error(); diff --git a/test/test_writer_fsm.cpp b/test/test_writer_fsm.cpp index da7f185d7..0c151a3f7 100644 --- a/test/test_writer_fsm.cpp +++ b/test/test_writer_fsm.cpp @@ -366,7 +366,7 @@ void test_ping_error() // Logs fix.check_log({ {logger::level::debug, "Writer task: 28 bytes written." }, - {logger::level::info, "Health checker: server answered ping with an error: ERR: bad command"}, + {logger::level::err, "Health checker: server answered ping with an error: ERR: bad command"}, }); } @@ -394,8 +394,9 @@ void test_write_error() // Logs fix.check_log({ - {logger::level::debug, "Writer task: 2 bytes written." }, - {logger::level::debug, "Writer task error: Expected field value is empty. [boost.redis:5]"}, + {logger::level::debug, "Writer task: 2 bytes written." }, + {logger::level::err, + "Error writing data to the server: Expected field value is empty. [boost.redis:5]"}, }); } @@ -420,9 +421,10 @@ void test_write_timeout() // Logs fix.check_log({ - {logger::level::debug, "Writer task: 0 bytes written." }, - {logger::level::debug, - "Writer task error: Timeout while writing data to the server. [boost.redis:27]"}, + // clang-format off + {logger::level::debug, "Writer task: 0 bytes written."}, + {logger::level::err, "Error writing data to the server: Timeout while writing data to the server. [boost.redis:27]" }, + // clang-format on }); } From 1360155baa5de07d68361145f7b520fc6bde0fe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anarthal=20=28Rub=C3=A9n=20P=C3=A9rez=29?= <34971811+anarthal@users.noreply.github.com> Date: Wed, 18 Feb 2026 11:15:45 +0100 Subject: [PATCH 34/37] Generates TLS certificates dynamically in CI (#396) This should avoid compliance errors in scanners. Certificates are generated by default to /opt/ci-tls. Added BOOST_REDIS_CA_PATH in tests to override this default close #394 --- .github/workflows/ci.yml | 4 +++ test/common.cpp | 2 +- test/common.hpp | 2 ++ test/test_conn_tls.cpp | 52 +++++++++++++++++-------------------- tools/docker-compose.yml | 49 +++++++++++++++++----------------- tools/docker/tls/ca.crt | 21 --------------- tools/docker/tls/ca.key | 28 -------------------- tools/docker/tls/server.crt | 19 -------------- tools/docker/tls/server.key | 28 -------------------- tools/gen-certificates.sh | 10 ++++++- 10 files changed, 65 insertions(+), 150 deletions(-) delete mode 100644 tools/docker/tls/ca.crt delete mode 100644 tools/docker/tls/ca.key delete mode 100644 tools/docker/tls/server.crt delete mode 100644 tools/docker/tls/server.key diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34af5e4a5..14d5b4210 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -233,6 +233,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Generate TLS certificates + run: | + ./tools/gen-certificates.sh + - name: Set up the required containers run: | BUILDER_IMAGE=${{ matrix.container }} SERVER_IMAGE=${{ matrix.server }} docker compose -f tools/docker-compose.yml up -d --wait || (docker compose logs; exit 1) diff --git a/test/common.cpp b/test/common.cpp index 54b3697f6..e13de33a6 100644 --- a/test/common.cpp +++ b/test/common.cpp @@ -37,7 +37,7 @@ void run( conn->async_run(cfg, run_callback{conn, op, ec}); } -static std::string safe_getenv(const char* name, const char* default_value) +std::string safe_getenv(const char* name, const char* default_value) { // MSVC doesn't like getenv #ifdef BOOST_MSVC diff --git a/test/common.hpp b/test/common.hpp index 569a7bcd5..1dafbe260 100644 --- a/test/common.hpp +++ b/test/common.hpp @@ -47,3 +47,5 @@ std::string_view find_client_info(std::string_view client_info, std::string_view void create_user(std::string_view port, std::string_view username, std::string_view password); boost::redis::logger make_string_logger(std::string& to); + +std::string safe_getenv(const char* name, const char* default_value); diff --git a/test/test_conn_tls.cpp b/test/test_conn_tls.cpp index 8989e2e33..9e588884e 100644 --- a/test/test_conn_tls.cpp +++ b/test/test_conn_tls.cpp @@ -10,8 +10,12 @@ #include #include #include +#include +#include #include +#include +#include #include #define BOOST_TEST_MODULE conn_tls #include @@ -25,37 +29,28 @@ using boost::system::error_code; namespace { -// CA certificate that signed the test server's certificate. -// This is a self-signed CA created for testing purposes. -// This must match tools/tls/ca.crt contents -static constexpr const char* ca_certificate = R"%(-----BEGIN CERTIFICATE----- -MIIDhzCCAm+gAwIBAgIUZGttu4o/Exs08EHCneeD3gHw7KkwDQYJKoZIhvcNAQEL -BQAwUjELMAkGA1UEBhMCRVMxGjAYBgNVBAoMEUJvb3N0LlJlZGlzIENJIENBMQsw -CQYDVQQLDAJJVDEaMBgGA1UEAwwRYm9vc3QtcmVkaXMtY2ktY2EwIBcNMjUwNjA3 -MTI0NzUwWhgPMjA4MDAzMTAxMjQ3NTBaMFIxCzAJBgNVBAYTAkVTMRowGAYDVQQK -DBFCb29zdC5SZWRpcyBDSSBDQTELMAkGA1UECwwCSVQxGjAYBgNVBAMMEWJvb3N0 -LXJlZGlzLWNpLWNhMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu7XV -sOoHB2J/5VtyJmMOzxhBbHKyQgW1YnMvYIb1JqIm7VuICA831SUw76n3j8mIK3zz -FfK2eYyUWf4Uo2j3uxmXDyjujqzIaUJNLcB53CQXkmIbqDigNhzUTPZ5A2MQ7xT+ -t1eDbjsZ7XIM+aTShgtrpyxiccsgPJ3/XXme2RrqKeNvYsTYY6pquWZdyLOg/LOH -IeSJyL1/eQDRu/GsZjnR8UOE6uHfbjrLWls7Tifj/1IueVYCEhQZpJSWS8aUMLBZ -fi+t9YMCCK4DGy+6QlznGgVqdFFbTUt2C7tzqz+iF5dxJ8ogKMUPEeFrWiZpozoS -t60jV8fKwdXz854jLQIDAQABo1MwUTAdBgNVHQ4EFgQU2SoWvvZUW8JiDXtyuXZK -deaYYBswHwYDVR0jBBgwFoAU2SoWvvZUW8JiDXtyuXZKdeaYYBswDwYDVR0TAQH/ -BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAqY4hGcdCFFPL4zveSDhR9H/akjae -uXbpo/9sHZd8e3Y4BtD8K05xa3417H9u5+S2XtyLQg5MON6J2LZueQEtE3wiR3ja -QIWbizqp8W54O5hTLQs6U/mWggfuL2R/HUw7ab4M8JobwHNEMK/WKZW71z0So/kk -W3wC0+1RH2PjMOZrCIflsD7EXYKIIr9afypAbhCQmCfu/GELuNx+LmaPi5JP4TTE -tDdhzWL04JLcZnA0uXb2Mren1AR9yKYH2I5tg5kQ3Bn/6v9+JiUhiejP3Vcbw84D -yFwRzN54bLanrJNILJhHPwnNIABXOtGUV05SZbYazJpiMst1a6eqDZhv/Q== ------END CERTIFICATE-----)%"; - -static config make_tls_config() +// Loads the CA certificate that signed the certificate used by the server. +// Should be in /tmp/ +std::string load_ca_certificate() +{ + auto ca_path = safe_getenv("BOOST_REDIS_CA_PATH", "/opt/ci-tls/ca.crt"); + std::ifstream f(ca_path); + if (!f) { + throw boost::system::system_error( + errno, + boost::system::system_category(), + "Failed to open CA certificate file '" + ca_path + "'"); + } + + return std::string(std::istreambuf_iterator(f), std::istreambuf_iterator()); +} + +config make_tls_config() { config cfg; cfg.use_ssl = true; cfg.addr.host = get_server_hostname(); - cfg.addr.port = "16380"; + cfg.addr.port = "16379"; return cfg; } @@ -100,6 +95,7 @@ BOOST_AUTO_TEST_CASE(exec_default_ssl_context) // Users can pass a custom context with TLS config BOOST_AUTO_TEST_CASE(exec_custom_ssl_context) { + std::string ca_pem = load_ca_certificate(); auto const cfg = make_tls_config(); constexpr std::string_view ping_value = "Kabuf"; @@ -113,7 +109,7 @@ BOOST_AUTO_TEST_CASE(exec_custom_ssl_context) // Configure the SSL context to trust the CA that signed the server's certificate. // The test certificate uses "redis" as its common name, regardless of the actual server's hostname - ctx.add_certificate_authority(net::const_buffer(ca_certificate, std::strlen(ca_certificate))); + ctx.add_certificate_authority(net::buffer(ca_pem)); ctx.set_verify_mode(net::ssl::verify_peer); ctx.set_verify_callback(net::ssl::host_name_verification("redis")); diff --git a/tools/docker-compose.yml b/tools/docker-compose.yml index c524dcf18..14c3f45af 100644 --- a/tools/docker-compose.yml +++ b/tools/docker-compose.yml @@ -9,14 +9,14 @@ services: --replica-announce-ip localhost \ --port 6379 \ --tls-port 16379 \ - --tls-cert-file /docker/tls/server.crt \ - --tls-key-file /docker/tls/server.key \ - --tls-ca-cert-file /docker/tls/ca.crt \ + --tls-cert-file /tls/server.crt \ + --tls-key-file /tls/server.key \ + --tls-ca-cert-file /tls/ca.crt \ --tls-auth-clients no \ --unixsocket /tmp/redis-socks/redis.sock \ --unixsocketperm 777' volumes: - - ./docker:/docker + - /opt/ci-tls:/tls - /tmp/redis-socks:/tmp/redis-socks redis-replica-1: @@ -30,13 +30,13 @@ services: "--replicaof", "localhost", "6379", "--port", "6380", "--tls-port", "16380", - "--tls-cert-file", "/docker/tls/server.crt", - "--tls-key-file", "/docker/tls/server.key", - "--tls-ca-cert-file", "/docker/tls/ca.crt", + "--tls-cert-file", "/tls/server.crt", + "--tls-key-file", "/tls/server.key", + "--tls-ca-cert-file", "/tls/ca.crt", "--tls-auth-clients", "no", ] volumes: - - ./docker:/docker + - /opt/ci-tls:/tls redis-replica-2: @@ -50,13 +50,13 @@ services: "--replicaof", "localhost", "6379", "--port", "6381", "--tls-port", "16381", - "--tls-cert-file", "/docker/tls/server.crt", - "--tls-key-file", "/docker/tls/server.key", - "--tls-ca-cert-file", "/docker/tls/ca.crt", + "--tls-cert-file", "/tls/server.crt", + "--tls-key-file", "/tls/server.key", + "--tls-ca-cert-file", "/tls/ca.crt", "--tls-auth-clients", "no", ] volumes: - - ./docker:/docker + - /opt/ci-tls:/tls sentinel-1: @@ -67,9 +67,9 @@ services: sh -c 'cat << EOF > /etc/sentinel.conf && redis-sentinel /etc/sentinel.conf port 26379 tls-port 36379 - tls-cert-file /docker/tls/server.crt - tls-key-file /docker/tls/server.key - tls-ca-cert-file /docker/tls/ca.crt + tls-cert-file /tls/server.crt + tls-key-file /tls/server.key + tls-ca-cert-file /tls/ca.crt tls-auth-clients no sentinel resolve-hostnames yes sentinel announce-hostnames yes @@ -80,7 +80,7 @@ services: sentinel parallel-syncs mymaster 1 EOF' volumes: - - ./docker:/docker + - /opt/ci-tls:/tls sentinel-2: @@ -91,9 +91,9 @@ services: sh -c 'cat << EOF > /etc/sentinel.conf && redis-sentinel /etc/sentinel.conf port 26380 tls-port 36380 - tls-cert-file /docker/tls/server.crt - tls-key-file /docker/tls/server.key - tls-ca-cert-file /docker/tls/ca.crt + tls-cert-file /tls/server.crt + tls-key-file /tls/server.key + tls-ca-cert-file /tls/ca.crt tls-auth-clients no sentinel resolve-hostnames yes sentinel announce-hostnames yes @@ -104,7 +104,7 @@ services: sentinel parallel-syncs mymaster 1 EOF' volumes: - - ./docker:/docker + - /opt/ci-tls:/tls sentinel-3: container_name: sentinel-3 @@ -114,9 +114,9 @@ services: sh -c 'cat << EOF > /etc/sentinel.conf && redis-sentinel /etc/sentinel.conf port 26381 tls-port 36381 - tls-cert-file /docker/tls/server.crt - tls-key-file /docker/tls/server.key - tls-ca-cert-file /docker/tls/ca.crt + tls-cert-file /tls/server.crt + tls-key-file /tls/server.key + tls-ca-cert-file /tls/ca.crt tls-auth-clients no sentinel resolve-hostnames yes sentinel announce-hostnames yes @@ -127,7 +127,7 @@ services: sentinel parallel-syncs mymaster 1 EOF' volumes: - - ./docker:/docker + - /opt/ci-tls:/tls builder: @@ -137,4 +137,5 @@ services: tty: true volumes: - ../:/boost-redis + - /opt/ci-tls:/opt/ci-tls - /tmp/redis-socks:/tmp/redis-socks diff --git a/tools/docker/tls/ca.crt b/tools/docker/tls/ca.crt deleted file mode 100644 index ac241b89c..000000000 --- a/tools/docker/tls/ca.crt +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDhzCCAm+gAwIBAgIUZGttu4o/Exs08EHCneeD3gHw7KkwDQYJKoZIhvcNAQEL -BQAwUjELMAkGA1UEBhMCRVMxGjAYBgNVBAoMEUJvb3N0LlJlZGlzIENJIENBMQsw -CQYDVQQLDAJJVDEaMBgGA1UEAwwRYm9vc3QtcmVkaXMtY2ktY2EwIBcNMjUwNjA3 -MTI0NzUwWhgPMjA4MDAzMTAxMjQ3NTBaMFIxCzAJBgNVBAYTAkVTMRowGAYDVQQK -DBFCb29zdC5SZWRpcyBDSSBDQTELMAkGA1UECwwCSVQxGjAYBgNVBAMMEWJvb3N0 -LXJlZGlzLWNpLWNhMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu7XV -sOoHB2J/5VtyJmMOzxhBbHKyQgW1YnMvYIb1JqIm7VuICA831SUw76n3j8mIK3zz -FfK2eYyUWf4Uo2j3uxmXDyjujqzIaUJNLcB53CQXkmIbqDigNhzUTPZ5A2MQ7xT+ -t1eDbjsZ7XIM+aTShgtrpyxiccsgPJ3/XXme2RrqKeNvYsTYY6pquWZdyLOg/LOH -IeSJyL1/eQDRu/GsZjnR8UOE6uHfbjrLWls7Tifj/1IueVYCEhQZpJSWS8aUMLBZ -fi+t9YMCCK4DGy+6QlznGgVqdFFbTUt2C7tzqz+iF5dxJ8ogKMUPEeFrWiZpozoS -t60jV8fKwdXz854jLQIDAQABo1MwUTAdBgNVHQ4EFgQU2SoWvvZUW8JiDXtyuXZK -deaYYBswHwYDVR0jBBgwFoAU2SoWvvZUW8JiDXtyuXZKdeaYYBswDwYDVR0TAQH/ -BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAqY4hGcdCFFPL4zveSDhR9H/akjae -uXbpo/9sHZd8e3Y4BtD8K05xa3417H9u5+S2XtyLQg5MON6J2LZueQEtE3wiR3ja -QIWbizqp8W54O5hTLQs6U/mWggfuL2R/HUw7ab4M8JobwHNEMK/WKZW71z0So/kk -W3wC0+1RH2PjMOZrCIflsD7EXYKIIr9afypAbhCQmCfu/GELuNx+LmaPi5JP4TTE -tDdhzWL04JLcZnA0uXb2Mren1AR9yKYH2I5tg5kQ3Bn/6v9+JiUhiejP3Vcbw84D -yFwRzN54bLanrJNILJhHPwnNIABXOtGUV05SZbYazJpiMst1a6eqDZhv/Q== ------END CERTIFICATE----- diff --git a/tools/docker/tls/ca.key b/tools/docker/tls/ca.key deleted file mode 100644 index ab64d6625..000000000 --- a/tools/docker/tls/ca.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC7tdWw6gcHYn/l -W3ImYw7PGEFscrJCBbVicy9ghvUmoibtW4gIDzfVJTDvqfePyYgrfPMV8rZ5jJRZ -/hSjaPe7GZcPKO6OrMhpQk0twHncJBeSYhuoOKA2HNRM9nkDYxDvFP63V4NuOxnt -cgz5pNKGC2unLGJxyyA8nf9deZ7ZGuop429ixNhjqmq5Zl3Is6D8s4ch5InIvX95 -ANG78axmOdHxQ4Tq4d9uOstaWztOJ+P/Ui55VgISFBmklJZLxpQwsFl+L631gwII -rgMbL7pCXOcaBWp0UVtNS3YLu3OrP6IXl3EnyiAoxQ8R4WtaJmmjOhK3rSNXx8rB -1fPzniMtAgMBAAECggEAffDzTf7G9oJ08yrWvMCrl/FbO+r8hOWVnFdSCz6hTulu -msRV0ap8OGr6mWJ9ciCyVxM/eZNeMNFMW9DogfcUd5kkzAwXeuycjlros3C3kic5 -FYLzsXtEqunO21BnQwl9vBtnBxYVXakTrUdfQ0P95+fTs9dIubaii2kqg8ns9RY+ -ebz2vnviNQaVL4WK3ohX+PJ7pimAM8KAwHl7e1RQn/ORghSycr9TAZKEqeZRlvtW -2TJBTknyr0Uo8KNi5L4nxi1qspgm9W7lFcmmv2cIkQsWFZvRY3a7iYsD6DOt2kaA -yFYlW6+n2V0uE6sYxZKe++VnSsueSDPzhmpRsiQONQKBgQDjV+vyoWqEaZoTVIpI -Ody1cOWwaqp0xnzaB/xsGYCWiW2wTWkq5Xpkp+Ia9w3YhHSEp7y6QYzNtofF8008 -LNKWynrhvq9isfrxcz5aqqf3ZYpFbJrxLJMzmF+H70G0HO/cXKg0FAYNnq1arkUZ -kQHWV8u/Bov5mLv9tVpmhzP7FwKBgQDTXwCqdJ1vNnZyXGhgCGVvMp6l1gw7RoF6 -LxrP4bR5vNhTa5xs8ibxpj2jd2ZmdzV6Y1r2imedsbKNlnvgDMVIWBtYWBFqjz9q -I88xtfiHLj1rFI21aI4TrWhxafmSBC6gli+1I840l6DtVE8xT/qCDveP/Umzivj8 -XLd+y8BuWwKBgQDXzWPZw0ObQarR4pQJD3Pkf7BokDgR9UAary37ZxHa08Vdb33/ -DCnsVjiZJB7ugn0gVyEdJJAFzdiAHP4ZuuqD3NxcYWeWph/xBlYQNqKOgsKIOBm0 -CX4JogA8xu51jGpboVDBbqh4UUF5LKfHJxC5aEmtoQdJ/KOmp3mSjZDYLQKBgFtW -klMWUXHddxG0HOZiunJyzVucQ2zZ6tmBwXRTdEmm8VQbDF2Hxoxl6fzZe8aLfPCG -PqiK5nuebioA/Ua0PgwlBqwGYoBJpn2XO9GfcOX5dVDwcMwTglG+fYOE5/PRGtUK -EVOVRWY0n7Xu2MnWZcoN7ayrJ04On8ltx11jbqRBAoGBAKvtJS0dKpsP6WvIrjQR -pqVxrpxnhWsgbSRNqCAFsqKwEGsrIXXVQIcOyrNSm1l6GxCWm7lnmkEcvddGy67f -0H/LqMItt1G5Dex96Zslhainz0oEE2yVX1x2H4qb0A1vEjviC/RVxFBheZrkWtEP -zQx9D/Gk2S471503xdYgUAv3 ------END PRIVATE KEY----- diff --git a/tools/docker/tls/server.crt b/tools/docker/tls/server.crt deleted file mode 100644 index a91efaea4..000000000 --- a/tools/docker/tls/server.crt +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDITCCAgkCFHexGHCisYJHDP/5HAFcT32jXvf8MA0GCSqGSIb3DQEBCwUAMFIx -CzAJBgNVBAYTAkVTMRowGAYDVQQKDBFCb29zdC5SZWRpcyBDSSBDQTELMAkGA1UE -CwwCSVQxGjAYBgNVBAMMEWJvb3N0LXJlZGlzLWNpLWNhMCAXDTI1MDYwNzEyNDc1 -MFoYDzIwODAwMzEwMTI0NzUwWjBGMQswCQYDVQQGEwJFUzEaMBgGA1UECgwRQm9v -c3QuUmVkaXMgQ0kgQ0ExCzAJBgNVBAsMAklUMQ4wDAYDVQQDDAVyZWRpczCCASIw -DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJeG8SyA6sOzdWn/G0KlugX1Se1e -S3xNSjk6pgaB6OELU1T8NRz3wpHlc2xGY6mEbZjBx/CEG7kukU6SLPPrwEu3cxDZ -wgxVyYaoVE5lD5Ue5lsowegO/jYsuDtB1ZbOby39LO3fLAte+aoLAoZam9Vpoatm -QvmYqgITSGGEPJSOapdu7UQBt0mcy1vY1eD+vNcZ/epk/cWBA5MU6hbGCa+8Mkky -tsDZ72D+OmdqZUJ4sSVU0fVZxjsQFhideUAu17UGYPqrQrvtzXMxDtMY7p4aBa+b -7QNfZ3wjNXnskXiDbiJ2F9UHamtTSd6IdXfU520davfeYJHfpr0NNZeS3IMCAwEA -ATANBgkqhkiG9w0BAQsFAAOCAQEAZitqMQOMwkYp9vfb4qdkuxoOIBq6Sx6aSXen -rS1N2g5eIhOV7mDyOgxPLVT3kZDsGKYGpbrjHFoYd9zALO0ZY05Vgm2Hlg10oCjq -iEdWr+PDDSRH762n4MNXZToG3ijPXNfNbMwDuXg0fG96P9D19dOsGwRUBWnaG8F0 -v3K+rEOXZNVZU4v7FhyNUmyqdpk2TQpj+k5aBwdOAWGfExeOo36AGJ5+JRR/85DA -rEPISY29eUwH8q+Pmj2DZ3YNee+6f/YvkO4+Ms9h74KqaIr/R/jeLnlUPx7szFmu -Ko9+AB0KA84HwkZhf1lPZrxouEqD5JZQ8xvjqhSFG/BxvqgW+w== ------END CERTIFICATE----- diff --git a/tools/docker/tls/server.key b/tools/docker/tls/server.key deleted file mode 100644 index d425eaba6..000000000 --- a/tools/docker/tls/server.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCXhvEsgOrDs3Vp -/xtCpboF9UntXkt8TUo5OqYGgejhC1NU/DUc98KR5XNsRmOphG2YwcfwhBu5LpFO -kizz68BLt3MQ2cIMVcmGqFROZQ+VHuZbKMHoDv42LLg7QdWWzm8t/Szt3ywLXvmq -CwKGWpvVaaGrZkL5mKoCE0hhhDyUjmqXbu1EAbdJnMtb2NXg/rzXGf3qZP3FgQOT -FOoWxgmvvDJJMrbA2e9g/jpnamVCeLElVNH1WcY7EBYYnXlALte1BmD6q0K77c1z -MQ7TGO6eGgWvm+0DX2d8IzV57JF4g24idhfVB2prU0neiHV31OdtHWr33mCR36a9 -DTWXktyDAgMBAAECggEAFHUTen6tM163K2hVtdnKE6PaJ0HjzxRgQPm6EIbZegid -Z4vlX9PTYP8pZiTar7hBU65V4BL16zpuQ71YdFYP0CkkT9IWcLPkUVnIiAz83ZUe -ZJBMHZhomcfP17A7tfCI8hyMRFEtERvxeROjc6AoCdgJC6ryv9/sk/smeoPv6d5b -NyRBA1TbQ7yV/Z1yTymkGsUHDSzqnogrB3/t8DCpbGHWj5mUCiQP0rTFAhjQNG0g -xaOfJEf/zyQNBREz2G/0d00ZhPx8CdwRuKVCNX+z/3hrJm4BGyiN8/B246TBZR6B -8Bv3bXwl7udCWgbc8yLxhDQQnxmr/0hEz3uKoMBdgQKBgQDFxHJy6Ko+2b4RICir -zCtx4xM5cU5MoFVvJPatYxmPBnDUGGgkxAtuKyHzerI0dX6yk9BJtR7Uy0HyTLv+ -2iCBcukq1ssI6GFgV+SCoqFq+DZfDEho11UlAeXtvYLsUxOTdlbyqg72q74gh1rb -jD9kG+9bycwWlhnWbkyNDT0aJwKBgQDEJPCrNTYuHqFW3wC3X3lj0uTrwh3fkeDF -gqIDwhCoHSnzrNo0XkAY2v5PhYyb0ThW6Gvbz/6k2MdOlcT7Ru4Ff6Nv8isw2N5+ -GT7PALrTwjtM3O10KtWvXgvleeB44dofsFlBmJTSysYyGVeGppxSgLM+TqRA7cLq -7up+DuCwRQKBgQCBeNeYhNuX5AM7wPr/Zd33ZDrcu2IQZlMoPWHFJ4C/Eu9g202q -7DGzgUdr2CK8l0NH899DsfCqTRsyXqwg73qiZLFjm1U20rOVWFVAdOoKVs95sFfj -Uz3pyXFXEF+bCdRiEPJhUZYURWFmeFayLwH9LxCn3dff3YqyyGkTWtxqtwKBgQCg -F5KD+zuMqx1+nSelg/y9WF3We6sMVrHCI9x++r/Dp4IdTKERCzh/0Qau+08Hwt2c -OboHQJ4UCIesgZu0iHEv9bz9Wwibpvb6rzpPlMXonujt1IjPP0MIDtfg5fgsUhNJ -uBMx8grOfgkEzSBeW9DNmhQyr9dq02U7gePNHfGQjQKBgQCw4LlrJHaWIAvh9hgH -MAYtlkxJkZwASWCntab241ZXQvD27+Nuc/wlBWN/unGJ2ip++IOHpG0rRtg58/lH -6/lzJ9wBISD5JOPU7av/iBirz9r3A5j7es9V3qer/69D+qh7lZHuKWHkOdgorgH0 -3XVmr7yPNQQs4Jw/JKrMmVHX6A== ------END PRIVATE KEY----- diff --git a/tools/gen-certificates.sh b/tools/gen-certificates.sh index 491e50324..98cdabd62 100755 --- a/tools/gen-certificates.sh +++ b/tools/gen-certificates.sh @@ -7,10 +7,14 @@ # # Generates the ca and certificates used for CI testing. -# Run this in the directory where you want the certificates to be generated. +# Usage: gen-certificates.sh [output-dir] set -e +OUTPUT_DIR="${1:-/opt/ci-tls}" +mkdir -p "$OUTPUT_DIR" +cd "$OUTPUT_DIR" + # CA private key openssl genpkey -algorithm RSA -out ca.key -pkeyopt rsa_keygen_bits:2048 @@ -28,3 +32,7 @@ openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial \ -out server.crt -days 20000 -sha256 rm server.csr rm ca.srl + +# Required when running with Docker because of mismatched user IDs +chmod 444 * + From fb9b40e8839aa5615f1f26cb00ffb8dc952462f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anarthal=20=28Rub=C3=A9n=20P=C3=A9rez=29?= <34971811+anarthal@users.noreply.github.com> Date: Mon, 23 Feb 2026 15:59:41 +0100 Subject: [PATCH 35/37] Adds a convenient way of iterating PubSub messages (#390) Adds push_view and push_parser. This is an input view range that allows parsing PubSub messages from a node tree. Adds a doc page on how to use server pushes. Adds reference docs to set_receive_response. close #349 --- README.md | 15 +- doc/modules/ROOT/nav.adoc | 1 + doc/modules/ROOT/pages/index.adoc | 58 +-- doc/modules/ROOT/pages/pushes.adoc | 187 +++++++ doc/modules/ROOT/pages/reference.adoc | 4 + doc/mrdocs.yml | 1 + example/cpp20_chat_room.cpp | 16 +- example/cpp20_subscriber.cpp | 20 +- include/boost/redis.hpp | 11 +- include/boost/redis/connection.hpp | 30 +- include/boost/redis/impl/push_parser.ipp | 118 +++++ include/boost/redis/push_parser.hpp | 188 +++++++ include/boost/redis/src.hpp | 1 + test/CMakeLists.txt | 1 + test/Jamfile | 1 + test/test_push_parser.cpp | 595 +++++++++++++++++++++++ 16 files changed, 1169 insertions(+), 78 deletions(-) create mode 100644 doc/modules/ROOT/pages/pushes.adoc create mode 100644 include/boost/redis/impl/push_parser.ipp create mode 100644 include/boost/redis/push_parser.hpp create mode 100644 test/test_push_parser.cpp diff --git a/README.md b/README.md index e79474dbb..dadb563ca 100644 --- a/README.md +++ b/README.md @@ -120,12 +120,19 @@ auto receiver(std::shared_ptr conn) -> asio::awaitable break; } + // This can happen if a SUBSCRIBE command errored (e.g. insufficient permissions) + if (resp.has_error()) { + std::cerr << "The receive response contains an error: " << resp.error().diagnostic + << std::endl; + break; + } + // The response must be consumed without suspending the // coroutine i.e. without the use of async operations. - for (auto const& elem : resp.value()) - std::cout << elem.value << "\n"; - - std::cout << std::endl; + for (push_view elem : push_parser(resp.value())) { + std::cout << "Received message from channel " << elem.channel << ": " << elem.payload + << "\n"; + } resp.value().clear(); } diff --git a/doc/modules/ROOT/nav.adoc b/doc/modules/ROOT/nav.adoc index 034322e8f..34c9255a7 100644 --- a/doc/modules/ROOT/nav.adoc +++ b/doc/modules/ROOT/nav.adoc @@ -1,5 +1,6 @@ * xref:index.adoc[Introduction] * xref:requests_responses.adoc[] +* xref:pushes.adoc[Server pushes] * xref:cancellation.adoc[] * xref:serialization.adoc[] * xref:auth.adoc[] diff --git a/doc/modules/ROOT/pages/index.adoc b/doc/modules/ROOT/pages/index.adoc index ebfbd0446..875ca9f97 100644 --- a/doc/modules/ROOT/pages/index.adoc +++ b/doc/modules/ROOT/pages/index.adoc @@ -87,68 +87,14 @@ The roles played by the `async_run` and `async_exec` functions are: be called from multiple places in your code concurrently. * xref:reference:boost/redis/basic_connection/async_run-04.adoc[`connection::async_run`]: keeps the connection healthy. It takes care of hostname resolution, session establishment, health-checks, reconnection and coordination of low-level read and write operations. It should be called only once per connection, regardless of the number of requests to execute. -== Server pushes - -Redis servers can also send a variety of pushes to the client. Some of -them are: - -* https://redis.io/docs/manual/pubsub/[Pubsub messages]. -* https://redis.io/docs/manual/keyspace-notifications/[Keyspace notifications]. -* https://redis.io/docs/manual/client-side-caching/[Client-side caching]. - -The connection class supports server pushes by means of the -xref:reference:boost/redis/basic_connection/async_receive.adoc[`connection::async_receive2`] function, which can be -called in the same connection that is being used to execute commands. -The coroutine below shows how to use it - - -[source,cpp] ----- -auto receiver(std::shared_ptr conn) -> asio::awaitable -{ - generic_flat_response resp; - conn->set_receive_response(resp); - - // Subscribe to the channel 'mychannel'. You can add any number of channels here. - request req; - req.subscribe({"mychannel"}); - co_await conn->async_exec(req); - - // You're now subscribed to 'mychannel'. Pushes sent over this channel will be stored - // in resp. If the connection encounters a network error and reconnects to the server, - // it will automatically subscribe to 'mychannel' again. This is transparent to the user. - // You need to use specialized request::subscribe() function (instead of request::push) - // to enable this behavior. - - // Loop to read Redis push messages. - while (conn->will_reconnect()) { - // Wait for pushes - auto [ec] = co_await conn->async_receive2(asio::as_tuple); - - // Check for errors and cancellations - if (ec) { - std::cerr << "Error during receive: " << ec << std::endl; - break; - } - - // The response must be consumed without suspending the - // coroutine i.e. without the use of async operations. - for (auto const& elem : resp.value()) - std::cout << elem.value << "\n"; - - std::cout << std::endl; - - resp.value().clear(); - } -} ----- == Further reading Here is a list of topics that you might be interested in: -* xref:cancellation.adoc[Setting timeouts to requests and managing cancellation]. * xref:requests_responses.adoc[More on requests and responses]. +* xref:pushes.adoc[Reading server pushes]. +* xref:cancellation.adoc[Setting timeouts to requests and managing cancellation]. * xref:serialization.adoc[Serializing and parsing into custom types]. * xref:logging.adoc[Configuring logging]. * xref:examples.adoc[Examples]. diff --git a/doc/modules/ROOT/pages/pushes.adoc b/doc/modules/ROOT/pages/pushes.adoc new file mode 100644 index 000000000..efbe7a4ec --- /dev/null +++ b/doc/modules/ROOT/pages/pushes.adoc @@ -0,0 +1,187 @@ +// +// Copyright (c) 2025 Marcelo Zimbres Silva (mzimbres@gmail.com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +[#pushes] += Server pushes + +Redis servers can send various push messages to the client. Boost.Redis supports these +as first-class operations. This page describes how to receive and handle them. + +The most common case is +https://redis.io/docs/latest/develop/pubsub/[Pub/Sub messages] +triggered by `PUBLISH`. The following example shows a typical receiver: + +[source,cpp] +---- +auto receiver(std::shared_ptr conn) -> asio::awaitable +{ + generic_flat_response resp; + conn->set_receive_response(resp); + + // Subscribe to the channel 'mychannel'. You can add any number of channels here. + request req; + req.subscribe({"mychannel"}); + co_await conn->async_exec(req); + + // You're now subscribed to 'mychannel'. Pushes sent over this channel will be stored + // in resp. If the connection encounters a network error and reconnects to the server, + // it will automatically subscribe to 'mychannel' again. This is transparent to the user. + // You need to use specialized request::subscribe() function (instead of request::push) + // to enable this behavior. + + // Loop to read Redis push messages. + while (conn->will_reconnect()) { + // Wait for pushes + auto [ec] = co_await conn->async_receive2(asio::as_tuple); + + // Check for errors and cancellations + if (ec) { + std::cerr << "Error during receive: " << ec << std::endl; + break; + } + + // This can happen if a SUBSCRIBE command errored (e.g. insufficient permissions) + if (resp.has_error()) { + std::cerr << "The receive response contains an error: " << resp.error().diagnostic + << std::endl; + break; + } + + // The response must be consumed without suspending the + // coroutine i.e. without the use of async operations. + for (push_view elem : push_parser(resp.value())) { + std::cout << "Received message from channel " << elem.channel << ": " << elem.payload + << "\n"; + } + + resp.value().clear(); + } +} +---- + +Summary of the steps: + +* Call xref:reference:boost/redis/basic_connection.adoc[`connection::set_receive_response`] + before any other receive-related calls so that the connection stores incoming pushes in the + given object. The library does not copy the response; you must keep it alive for the + duration of the receive loop. +* Build a request with xref:reference:boost/redis/request.adoc[`request::subscribe`] + (or `psubscribe`) and execute it. If the connection drops and is re-established, + an equivalent `SUBSCRIBE` is sent automatically. +* Loop while xref:reference:boost/redis/basic_connection.adoc[`connection::will_reconnect`] + is true (i.e. until the connection is cancelled). +* Call xref:reference:boost/redis/basic_connection.adoc[`connection::async_receive2`] + to wait until at least one push is available. This function also participates in + push flow control (see <>). +* After completion, `resp` holds the raw RESP3 nodes for the received pushes + (xref:reference:boost/redis/resp3/node_view.adoc[`resp3::node_view`]). + For Pub/Sub messages produced by `PUBLISH`, use xref:reference:boost/redis/push_parser.adoc[`push_parser`] + to iterate over them. Each parsed message exposes the channel, the payload, and, for + pattern subscriptions (`PSUBSCRIBE`), the matched pattern. +* Call `resp.value().clear()` to discard the current push data and make room for the + next batch. + +WARNING: Consume all push data *without suspending the current coroutine* (or otherwise re-entering +the event loop). See <> for how to handle cases where you must +suspend. + +NOTE: You can use the same connection for both pushes and normal request/response +traffic. Boost.Redis relies on RESP3 to multiplex them on a single connection. + +== Dynamic Pub/Sub tracking + +When you execute a request built with xref:reference:boost/redis/request.adoc[`request::subscribe`] +(or `psubscribe`, `unsubscribe`, `punsubscribe`), the connection tracks it. After a reconnect, it +re-issues the subscribe commands to restore any subscriptions that were active before the reconnect. +You can dynamically subscribe and unsubscribe to channels; tracking is fully dynamic. + +[#flow-control] +== Flow control + +If the server sends pushes faster than the client consumes them, the connection applies +flow control. After a certain number of pushes have been read and not yet consumed, it +stops reading until the application drains the receive buffer. This creates backpressure +at the TCP level. + +The pending count is reset each time `async_receive2` completes. You must still free +memory by calling `resp3::flat_tree::clear()` (or equivalent) on the response, as in +the example. + +WARNING: Do not call xref:reference:boost/redis/basic_connection/async_exec-02.adoc[`async_exec`] from within the receiver loop. The response to your request may be behind enough pushes might to trigger the flow control mechanism, causing a deadlock. + + +== Subscribe confirmations + +Every time you subscribe to a channel, Redis sends a push as a confirmation. These confirmations +are stored in your receive response and cause `async_receive2` to complete like any +other push. `push_parser` skips these messages, since they don't contain application-level information. + +As a result, *`async_receive2` will complete while the `push_parser` range is empty* +when only confirmations are received. Your code should handle an empty range correctly. + +To work with subscribe confirmations or other push shapes, use the raw nodes in +xref:reference:boost/redis/generic_flat_response.adoc[`generic_flat_response`] +directly (see <>). + +[#suspending-the-receiver] +== Suspending the receiver + +You must not suspend the receiver coroutine before fully consuming the current push data. If you +need to perform I/O with that data (e.g. send it over a WebSocket), first copy or +serialize it without suspending, then clear the response, and only then perform the +async operation. + +For example, the following is incorrect: + +[source,cpp] +---- +auto [ec] = co_await conn->async_receive2(asio::as_tuple); +// DON'T DO THIS: more pushes may be read while the WebSocket send runs, +// and they are discarded when you clear(), causing a race. +co_await websocket.async_send(push_parser(resp.value())); +resp.value().clear(); +---- + +Instead, copy or compose the message without I/O, then clear, then send: + +[source,cpp] +---- +auto [ec] = co_await conn->async_receive2(asio::as_tuple); + +// Compose the WebSocket message without any I/O (no suspension). +// msg must not reference resp. +auto msg = compose_websocket_message(push_parser(resp.value())); + +resp.value().clear(); + +// Now it's OK to suspend the coroutine +co_await websocket.async_write(msg); +---- + +== SUBSCRIBE errors + +`SUBSCRIBE` can fail (e.g. due to ACL rules). Because of how the protocol works, such +errors may be delivered as push-like data. Using +xref:reference:boost/redis/generic_flat_response.adoc[`generic_flat_response`] +(an alias for `adapter::result`, similar to `std::expected`) lets you detect them: check +`resp.has_error()` and handle the error before iterating. See the example at the top +of this page. + +[#advanced-scenarios] +== Advanced scenarios + +`push_parser` only recognizes standard +Pub/Sub `message` and `pmessage` pushes. Other push types are skipped, including: + +* Subscribe/unsubscribe confirmations. +* https://redis.io/docs/latest/develop/reference/client-side-caching/[Client-side caching] + invalidation messages. +* Output of the `MONITOR` command. + +If you need to handle these, work directly with the nodes in `generic_flat_response` +instead of `push_parser`. The overall receiver pattern (set response, subscribe, +loop with `async_receive2`, consume, clear) stays the same. diff --git a/doc/modules/ROOT/pages/reference.adoc b/doc/modules/ROOT/pages/reference.adoc index 6281911ee..085ea3cbc 100644 --- a/doc/modules/ROOT/pages/reference.adoc +++ b/doc/modules/ROOT/pages/reference.adoc @@ -59,6 +59,10 @@ xref:reference:boost/redis/generic_flat_response.adoc[`generic_flat_response`] xref:reference:boost/redis/consume_one-08.adoc[`consume_one`] +xref:reference:boost/redis/push_view.adoc[`push_view`] + +xref:reference:boost/redis/push_parser.adoc[`push_parser`] + | xref:reference:boost/redis/adapter/boost_redis_adapt.adoc[`boost_redis_adapt`] diff --git a/doc/mrdocs.yml b/doc/mrdocs.yml index 36510918f..32af97526 100644 --- a/doc/mrdocs.yml +++ b/doc/mrdocs.yml @@ -23,6 +23,7 @@ exclude-symbols: - "boost::redis::request::has_hello_priority" see-below: - "boost::redis::adapter::ignore" + - "boost::redis::push_parser::iterator" sort-members: false base-url: https://github.com/boostorg/redis/blob/master/include/ use-system-libc: true diff --git a/example/cpp20_chat_room.cpp b/example/cpp20_chat_room.cpp index 21aa0c3a6..8c056b0b6 100644 --- a/example/cpp20_chat_room.cpp +++ b/example/cpp20_chat_room.cpp @@ -5,6 +5,7 @@ */ #include +#include #include #include @@ -35,6 +36,8 @@ using boost::redis::connection; using boost::redis::generic_flat_response; using boost::redis::request; using boost::system::error_code; +using boost::redis::push_parser; +using boost::redis::push_view; using namespace std::chrono_literals; // Chat over Redis pubsub. To test, run this program from multiple @@ -70,10 +73,19 @@ auto receiver(std::shared_ptr conn) -> awaitable break; } + // This can happen if a SUBSCRIBE command errored (e.g. insufficient permissions) + if (resp.has_error()) { + std::cerr << "The receive response contains an error: " << resp.error().diagnostic + << std::endl; + break; + } + // The response must be consumed without suspending the // coroutine i.e. without the use of async operations. - for (auto const& elem : resp.value()) - std::cout << elem.value << "\n"; + for (push_view elem : push_parser(resp.value())) { + std::cout << "Received message from channel " << elem.channel << ": " << elem.payload + << "\n"; + } std::cout << std::endl; diff --git a/example/cpp20_subscriber.cpp b/example/cpp20_subscriber.cpp index 513f90acc..aae8bbc03 100644 --- a/example/cpp20_subscriber.cpp +++ b/example/cpp20_subscriber.cpp @@ -5,6 +5,7 @@ */ #include +#include #include #include @@ -22,8 +23,10 @@ using namespace std::chrono_literals; using boost::redis::request; using boost::redis::generic_flat_response; using boost::redis::config; -using boost::system::error_code; using boost::redis::connection; +using boost::redis::push_parser; +using boost::redis::push_view; +using boost::system::error_code; using asio::signal_set; /* This example will subscribe and read pushes indefinitely. @@ -70,12 +73,19 @@ auto receiver(std::shared_ptr conn) -> asio::awaitable break; } + // This can happen if a SUBSCRIBE command errored (e.g. insufficient permissions) + if (resp.has_error()) { + std::cerr << "The receive response contains an error: " << resp.error().diagnostic + << std::endl; + break; + } + // The response must be consumed without suspending the // coroutine i.e. without the use of async operations. - for (auto const& elem : resp.value()) - std::cout << elem.value << "\n"; - - std::cout << std::endl; + for (push_view elem : push_parser(resp.value())) { + std::cout << "Received message from channel " << elem.channel << ": " << elem.payload + << "\n"; + } resp.value().clear(); } diff --git a/include/boost/redis.hpp b/include/boost/redis.hpp index 346b6f34e..281539528 100644 --- a/include/boost/redis.hpp +++ b/include/boost/redis.hpp @@ -12,17 +12,8 @@ #include #include #include +#include #include #include -/** @defgroup high-level-api Reference - * - * This page contains the documentation of the Aedis high-level API. - */ - -/** @defgroup low-level-api Reference - * - * This page contains the documentation of the Aedis low-level API. - */ - #endif // BOOST_REDIS_HPP diff --git a/include/boost/redis/connection.hpp b/include/boost/redis/connection.hpp index 1801dd4dd..497ff06fc 100644 --- a/include/boost/redis/connection.hpp +++ b/include/boost/redis/connection.hpp @@ -1085,7 +1085,35 @@ class basic_connection { "the other member functions to interact with the connection.") auto const& next_layer() const noexcept { return impl_->stream_.next_layer(); } - /// Sets the response object of @ref async_receive2 operations. + /** + * @brief Sets the response object of @ref async_receive2 operations. + * + * Pushes received by the connection (concretely, by @ref async_run) + * will be stored in `resp`. This happens even if @ref async_receive2 + * is not being called. + * + * `resp` should be able to accommodate the following message types: + * + * @li Any kind of RESP3 pushes that the application might expect. + * This usually involves Pub/Sub messages of type `message`, + * `subscribe`, `unsubscribe`, `psubscribe` and `punsubscribe`. + * See this page + * for more info. + * @li Any errors caused by failed `SUBSCRIBE` commands. + * Because of protocol oddities, these are placed in the receive buffer, + * rather than handed to `async_exec`. + * @li If your application is using `MONITOR`, simple strings. + * + * Because receive responses need to accommodate many different kind + * of messages, it's advised to use one of the generic responses + * (like @ref generic_flat_response). If a response can't accommodate + * one of the received types, @ref async_run will exit with an error. + * + * Messages received before this function is called are discarded. + * + * @par Object lifetimes + * `resp` should be kept alive until @ref async_run completes. + */ template void set_receive_response(Response& resp) { diff --git a/include/boost/redis/impl/push_parser.ipp b/include/boost/redis/impl/push_parser.ipp new file mode 100644 index 000000000..ebac60bdc --- /dev/null +++ b/include/boost/redis/impl/push_parser.ipp @@ -0,0 +1,118 @@ +// +// Copyright (c) 2026 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#include +#include + +#include +#include +#include + +namespace boost::redis { + +namespace detail { + +// Advances the iterator past all nodes belonging to the current root message. +// Returns a pointer to the next root node (depth == 0) or end. +inline const resp3::node_view* skip_current_message( + const resp3::node_view* current, + const resp3::node_view* end) +{ + return std::find_if(current, end, [](const resp3::node_view& n) { + return n.depth == 0u; + }); +} + +// Attempts to parse a push message starting at 'current'. +// If successful, populates 'msg' and returns true. +// If not a valid pubsub message (message/pmessage), returns false. +inline bool try_parse_push_view( + const resp3::node_view*& current, + const resp3::node_view* end, + push_view& msg) +{ + if (current == end) + return false; + + // Root must be a push type + if (current->data_type != resp3::type::push || current->depth != 0u) { + current = skip_current_message(current + 1, end); + return false; + } + + // Move to first child (message type) + ++current; + if (current == end || current->depth != 1u || current->data_type != resp3::type::blob_string) { + current = skip_current_message(current, end); + return false; + } + + // Determine the message type + bool is_pmessage; + if (current->value == "message") { + is_pmessage = false; + } else if (current->value == "pmessage") { + is_pmessage = true; + } else { + // Not interested in this message + current = skip_current_message(current, end); + return false; + } + + // For pmessage, the matched pattern goes next + if (is_pmessage) { + ++current; + if ( + current == end || current->depth != 1u || current->data_type != resp3::type::blob_string) { + current = skip_current_message(current, end); + return false; + } + msg.pattern = current->value; + } else { + msg.pattern = std::nullopt; + } + + // Channel + ++current; + if (current == end || current->depth != 1 || current->data_type != resp3::type::blob_string) { + current = skip_current_message(current, end); + return false; + } + msg.channel = current->value; + + // Payload + ++current; + if (current == end || current->depth != 1 || current->data_type != resp3::type::blob_string) { + current = skip_current_message(current, end); + return false; + } + msg.payload = current->value; + + // We're done. We should be at the end of the message + ++current; + if (current != end && current->depth != 0u) { + current = skip_current_message(current, end); + return false; + } + + return true; +} + +} // namespace detail + +void push_parser::advance() noexcept +{ + while (first_ != last_) { + if (detail::try_parse_push_view(first_, last_, current_)) { + return; + } + } + done_ = true; +} + +} // namespace boost::redis diff --git a/include/boost/redis/push_parser.hpp b/include/boost/redis/push_parser.hpp new file mode 100644 index 000000000..0f526f851 --- /dev/null +++ b/include/boost/redis/push_parser.hpp @@ -0,0 +1,188 @@ +// +// Copyright (c) 2026 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#ifndef BOOST_REDIS_PUSH_PARSER_HPP +#define BOOST_REDIS_PUSH_PARSER_HPP + +#include + +#include + +#include +#include +#include + +namespace boost::redis { + +/** + * @brief A Pub/Sub message received from the server. + * + * Can represent messages from both regular subscriptions (`message`) and + * pattern subscriptions (`pmessage`) with string payloads. + * + * @par Object lifetimes + * This object contains views pointing into external storage + * (typically a @ref resp3::flat_tree). + */ +struct push_view { + /// @brief The channel where the message was published. + std::string_view channel; + + /** + * @brief The pattern that matched the channel. + * + * Set only for messages received through pattern subscriptions (`pmessage`); + * @ref std::nullopt otherwise. + */ + std::optional pattern; + + /// The message payload. + std::string_view payload; +}; + +/** + * @brief A range that parses Pub/Sub messages from a sequence of RESP3 nodes. + * + * Given an input range of RESP3 nodes, this class attempts to parse them + * as Pub/Sub messages. Parsing occurs incrementally, while advancing the iterators. + * + * This type models `std::input_range`. + * + * This class handles pushes with type `message` and `pmessage` with string payloads. + * These are the messages containing the actual information in the usual Pub/Sub workflow. + * Any RESP3 message with a different shape will be skipped. In particular, the following + * message types are skipped: + * + * @li Messages with type other than @ref resp3::type::push. This includes + * the output of the `MONITOR` command and errors. + * @li Subscribe and unsubscribe confirmation messages. + * These happen every time a `SUBSCRIBE`, `UNSUBSCRIBE`, `PSUBSCRIBE` or `PUNSUBSCRIBE` + * command is successfully executed, and don't carry application-level information. + * @li Pushes with type `message` and `pmessage` with a payload type different to + * @ref resp3::type::bulk_string. Client-side caching generates this kind of messages. + * + * If you need to handle any of these, use the raw RESP3 nodes rather than this class. + * + * @par Object lifetimes + * Iteration state is held by the parser. Iterators reference the parser. + * The parser must be kept alive for the duration of the iteration. + * + * No copies of the input range are made. The returned @ref push_view + * values are views. + * + * @par Example + * @code + * for (const push_view& msg : push_parser(tree)) { + * std::cout << "Channel: " << msg.channel << ", Payload: " << msg.payload << "\n"; + * } + * @endcode + */ +class push_parser { + const resp3::node_view* first_{}; + const resp3::node_view* last_{}; + push_view current_{}; + bool done_{false}; + + void advance() noexcept; + +public: + /** + * @brief Iterator over parsed Pub/Sub messages. + * + * Models `std::input_iterator`. Iterators are invalidated when the + * @ref push_parser they refer to is destroyed or when the underlying + * node range is modified. + * + * The exact iterator type is unspecified. + */ + class iterator { + push_parser* gen_{}; + + friend class push_parser; + + explicit iterator(push_parser* gen) noexcept + : gen_{gen} + { } + + public: + using value_type = push_view; + using difference_type = std::ptrdiff_t; + using reference = push_view const&; + using pointer = push_view const*; + using iterator_category = std::input_iterator_tag; + + iterator() = default; + + reference operator*() const noexcept { return gen_->current_; } + pointer operator->() const noexcept { return &gen_->current_; } + + iterator& operator++() noexcept + { + BOOST_ASSERT(gen_); + gen_->advance(); + if (gen_->done_) + gen_ = nullptr; + return *this; + } + + void operator++(int) noexcept { ++*this; } + + friend bool operator==(const iterator& a, const iterator& b) noexcept + { + return a.gen_ == b.gen_; + } + + friend bool operator!=(const iterator& a, const iterator& b) noexcept { return !(a == b); } + }; + + /** + * @brief Constructs a parser over a range of RESP3 nodes. + * + * @par Object lifetimes + * No copy of `nodes` is made. The underlying `nodes` range + * must remain valid for the lifetime of this parser. + * + * @par Exception safety + * No-throw guarantee. + * + * @param nodes A contiguous range of @ref resp3::node_view to parse. + */ + explicit push_parser(span nodes) noexcept + : first_{nodes.data()} + , last_{nodes.data() + nodes.size()} + { + advance(); + } + + /** + * @brief Returns an iterator to the first parsed Pub/Sub message. + * + * If the node range contains no valid `message` or `pmessage` pushes, + * returns an iterator equal to @ref end. + * + * @par Exception safety + * No-throw guarantee. + * + * @returns An iterator to the first @ref push_view, or past-the-end if none. + */ + iterator begin() noexcept { return done_ ? iterator() : iterator(this); } + + /** + * @brief Returns a past-the-end iterator. + * + * @par Exception safety + * No-throw guarantee. + * + * @returns A past-the-end iterator for this parser. + */ + iterator end() noexcept { return iterator(); } +}; + +} // namespace boost::redis + +#endif diff --git a/include/boost/redis/src.hpp b/include/boost/redis/src.hpp index 196b2960a..ecc94d5be 100644 --- a/include/boost/redis/src.hpp +++ b/include/boost/redis/src.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 859dfc516..4eb460add 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -57,6 +57,7 @@ make_test(test_flat_tree) make_test(test_generic_flat_response) make_test(test_read_buffer) make_test(test_subscription_tracker) +make_test(test_push_parser) # Tests that require a real Redis server make_test(test_conn_quit) diff --git a/test/Jamfile b/test/Jamfile index 346837292..adaa257f3 100644 --- a/test/Jamfile +++ b/test/Jamfile @@ -74,6 +74,7 @@ local tests = test_generic_flat_response test_read_buffer test_subscription_tracker + test_push_parser ; # Build and run the tests diff --git a/test/test_push_parser.cpp b/test/test_push_parser.cpp new file mode 100644 index 000000000..e131990f5 --- /dev/null +++ b/test/test_push_parser.cpp @@ -0,0 +1,595 @@ +// +// Copyright (c) 2026 Marcelo Zimbres Silva (mzimbres@gmail.com), +// Ruben Perez Hidalgo (rubenperez038 at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#include + +#include +#include + +#include "sansio_utils.hpp" + +#include +#include +#include +#if (__cpp_lib_ranges >= 201911L) && (__cpp_lib_concepts >= 202002L) +#define BOOST_REDIS_TEST_RANGE_CONCEPTS +#include +#endif + +using namespace boost::redis; +using detail::tree_from_resp3; + +// Operators +namespace boost::redis { + +std::ostream& operator<<(std::ostream& os, const push_view& v) +{ + os << "push_view { .channel=" << v.channel; + if (v.pattern) + os << ", .pattern=" << *v.pattern; + return os << ", .payload=" << v.payload << " }"; +} + +bool operator==(const push_view& lhs, const push_view& rhs) noexcept +{ + return lhs.channel == rhs.channel && lhs.pattern == rhs.pattern && lhs.payload == rhs.payload; +} + +} // namespace boost::redis + +namespace { + +// --- Only valid messages --- +void test_one_message() +{ + auto nodes = tree_from_resp3({">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n"}); + push_parser p{nodes}; + + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +void test_several_messages() +{ + auto nodes = tree_from_resp3({ + ">3\r\n$7\r\nmessage\r\n$5\r\nfirst\r\n$2\r\nHi\r\n", + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$3\r\nBye\r\n", + }); + push_parser p{nodes}; + + const push_view expected[] = { + {"first", std::nullopt, "Hi" }, + {"second", std::nullopt, "Bye"}, + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +void test_one_pmessage() +{ + auto nodes = tree_from_resp3( + {">4\r\n$8\r\npmessage\r\n$6\r\nmycha*\r\n$6\r\nmychan\r\n$5\r\nHello\r\n"}); + push_parser p{nodes}; + + const push_view expected[] = { + {"mychan", "mycha*", "Hello"}, + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +void test_mixed_valid() +{ + auto nodes = tree_from_resp3({ + ">3\r\n$7\r\nmessage\r\n$3\r\nch1\r\n$4\r\nmsg1\r\n", + ">4\r\n$8\r\npmessage\r\n$1\r\n*\r\n$3\r\nch2\r\n$4\r\nmsg2\r\n", + ">3\r\n$7\r\nmessage\r\n$3\r\nch3\r\n$4\r\nmsg3\r\n", + }); + push_parser p{nodes}; + + const push_view expected[] = { + {"ch1", std::nullopt, "msg1"}, + {"ch2", "*", "msg2"}, + {"ch3", std::nullopt, "msg3"}, + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +void test_message_empty_fields() +{ + auto nodes = tree_from_resp3({">3\r\n$7\r\nmessage\r\n$0\r\n\r\n$0\r\n\r\n"}); + push_parser p{nodes}; + + const push_view expected[] = { + {"", std::nullopt, ""}, + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +void test_pmessage_empty_fields() +{ + auto nodes = tree_from_resp3({">4\r\n$8\r\npmessage\r\n$0\r\n\r\n$0\r\n\r\n$0\r\n\r\n"}); + push_parser p{nodes}; + + const push_view expected[] = { + {"", "", ""}, + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +// --- Message skipping (valid, expected messages we don't care about) --- + +// Anything that is not a RESP3 push is skipped. +// Concretely, pushes in RESP2 used to be arrays. We don't support these +void test_skip_type_array() +{ + auto nodes = tree_from_resp3({ + "*3\r\n$7\r\nmessage\r\n$5\r\nfirst\r\n$5\r\nValue\r\n", + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + }); + push_parser p{nodes}; + + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +// Might happen if a SUBSCRIBE call fails +void test_skip_type_simple_error() +{ + auto nodes = tree_from_resp3({ + "-ERR foo\r\n", + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + }); + push_parser p{nodes}; + + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +// Might happen when using MONITOR +void test_skip_type_simple_string() +{ + auto nodes = tree_from_resp3({ + "+MONITOR output\r\n", + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + }); + push_parser p{nodes}; + + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +// Edge case: other RESP3 types are skipped +void test_skip_type_other() +{ + auto nodes = tree_from_resp3({ + "%1\r\n$3\r\nkey\r\n$5\r\nvalue\r\n", + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + }); + push_parser p{nodes}; + + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +// The push first field is not 'message' or 'pmessage' (e.g. subscribe confirmations) +void test_skip_msgtype_subscribe_confirmations() +{ + auto nodes = tree_from_resp3({ + ">3\r\n$9\r\nsubscribe\r\n$6\r\nmychan\r\n:1\r\n", + ">3\r\n$11\r\nunsubscribe\r\n$6\r\nmychan\r\n:1\r\n", + ">3\r\n$10\r\npsubscribe\r\n$5\r\np*tt*\r\n:1\r\n", + ">3\r\n$12\r\npunsubscribe\r\n$5\r\np*tt*\r\n:1\r\n", + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + }); + push_parser p{nodes}; + + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +// Message type string, but with an unknown value (not 'message' or 'pmessage') +void test_skip_msgtype_unknown() +{ + auto nodes = tree_from_resp3({ + ">3\r\n$7\r\nunknown\r\n$4\r\nchan\r\n$4\r\nbody\r\n", + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + }); + push_parser p{nodes}; + + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +// Message type is an empty string +void test_skip_msgtype_empty() +{ + auto nodes = tree_from_resp3({ + ">3\r\n$0\r\n\r\n$4\r\nchan\r\n$4\r\nbody\r\n", + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + }); + push_parser p{nodes}; + + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +// Message type is not a string (e.g. array) +void test_skip_msgtype_not_string() +{ + auto nodes = tree_from_resp3({ + ">3\r\n*1\r\n$3\r\nfoo\r\n$5\r\nHello\r\n$5\r\nworld\r\n", + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + }); + push_parser p{nodes}; + + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +// The message ends before the message type +void test_skip_msgtype_eof() +{ + auto nodes = tree_from_resp3({ + ">0\r\n", // end of message manifests as another message starting + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + ">0\r\n", // end of message manifests as the end of the range + }); + push_parser p{nodes}; + + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +// Pattern is not a string +void test_skip_pattern_not_string() +{ + auto nodes = tree_from_resp3({ + ">4\r\n$8\r\npmessage\r\n*1\r\n$3\r\nfoo\r\n$6\r\nmychan\r\n$5\r\nHello\r\n", + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + }); + push_parser p{nodes}; + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +// The message ends before the pattern field +void test_skip_pattern_eof() +{ + auto nodes = tree_from_resp3({ + ">1\r\n$8\r\npmessage\r\n", + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + ">1\r\n$8\r\npmessage\r\n", + }); + push_parser p{nodes}; + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +// Channel not a string +void test_skip_channel_not_string_message() +{ + auto nodes = tree_from_resp3({ + ">3\r\n$7\r\nmessage\r\n*1\r\n$3\r\nfoo\r\n$5\r\nHello\r\n", + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + }); + push_parser p{nodes}; + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +void test_skip_channel_not_string_pmessage() +{ + auto nodes = tree_from_resp3({ + ">4\r\n$8\r\npmessage\r\n$1\r\n*\r\n*1\r\n$3\r\nfoo\r\n$5\r\nHello\r\n", + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + }); + push_parser p{nodes}; + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +// Message ends before channel +void test_skip_channel_eof_message() +{ + auto nodes = tree_from_resp3({ + ">1\r\n$7\r\nmessage\r\n", + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + ">1\r\n$7\r\nmessage\r\n", + }); + push_parser p{nodes}; + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +void test_skip_channel_eof_pmessage() +{ + auto nodes = tree_from_resp3({ + ">2\r\n$8\r\npmessage\r\n$1\r\n*\r\n", + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + ">2\r\n$8\r\npmessage\r\n$1\r\n*\r\n", + }); + push_parser p{nodes}; + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +// Payload not a string +void test_skip_payload_not_string_message() +{ + auto nodes = tree_from_resp3({ + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n*1\r\n$3\r\nfoo\r\n", + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + }); + push_parser p{nodes}; + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +void test_skip_payload_not_string_pmessage() +{ + auto nodes = tree_from_resp3({ + ">4\r\n$8\r\npmessage\r\n$1\r\n*\r\n$6\r\nsecond\r\n*1\r\n$3\r\nfoo\r\n", + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + }); + push_parser p{nodes}; + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +// Message ends before payload +void test_skip_payload_eof_pmessage() +{ + auto nodes = tree_from_resp3({ + ">3\r\n$8\r\npmessage\r\n$1\r\n*\r\n$3\r\nch2\r\n", + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + ">3\r\n$8\r\npmessage\r\n$1\r\n*\r\n$3\r\nch2\r\n", + }); + push_parser p{nodes}; + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +void test_skip_payload_eof_message() +{ + auto nodes = tree_from_resp3({ + ">2\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n", + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + ">2\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n", + }); + push_parser p{nodes}; + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +// Message is longer than expected +void test_skip_longer_message() +{ + auto nodes = tree_from_resp3({ + ">4\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n$1\r\nx\r\n", + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + }); + push_parser p{nodes}; + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +void test_skip_longer_pmessage() +{ + auto nodes = tree_from_resp3({ + ">5\r\n$8\r\npmessage\r\n$1\r\n*\r\n$3\r\nch2\r\n$4\r\nmsg2\r\n$1\r\nx\r\n", + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + }); + push_parser p{nodes}; + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +// --- Mixes of valid and skipped messages --- +void test_only_skipped() +{ + auto nodes = tree_from_resp3({ + "+foo\r\n", + }); + push_parser p{nodes}; + std::vector expected; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), expected.begin(), expected.end()); +} + +void test_valid_skipped() +{ + auto nodes = tree_from_resp3({ + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + "-ERR foo\r\n", + }); + push_parser p{nodes}; + constexpr push_view expected[] = { + {"second", std::nullopt, "Hello"} + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +void test_valid_skipped_mix() +{ + auto nodes = tree_from_resp3({ + ">3\r\n$7\r\nmessage\r\n$3\r\nch1\r\n$4\r\nmsg1\r\n", + "+MONITOR\r\n", + ">3\r\n$9\r\nsubscribe\r\n$4\r\nchan\r\n:1\r\n", + ">3\r\n$7\r\nmessage\r\n$3\r\nch2\r\n$4\r\nmsg2\r\n", + ">3\r\n$7\r\nmessage\r\n$3\r\nch3\r\n$4\r\nmsg3\r\n", + "%1\r\n$3\r\nkey\r\n$5\r\nvalue\r\n", + }); + push_parser p{nodes}; + const push_view expected[] = { + {"ch1", std::nullopt, "msg1"}, + {"ch2", std::nullopt, "msg2"}, + {"ch3", std::nullopt, "msg3"}, + }; + BOOST_TEST_ALL_EQ(p.begin(), p.end(), std::begin(expected), std::end(expected)); +} + +// --- Edge cases --- +void test_empty() +{ + auto nodes = tree_from_resp3({}); + push_parser p{nodes}; + std::vector expected; + + BOOST_TEST_ALL_EQ(p.begin(), p.end(), expected.begin(), expected.end()); +} + +// --- The range interface works as expected --- +#ifdef BOOST_REDIS_TEST_RANGE_CONCEPTS +static_assert(std::ranges::input_range); +static_assert(std::input_iterator); +#endif + +void test_range_interface() +{ + auto nodes = tree_from_resp3({ + ">3\r\n$7\r\nmessage\r\n$6\r\nsecond\r\n$5\r\nHello\r\n", + }); + + // Iterator default construction + { + push_parser p{nodes}; + push_parser::iterator it; + BOOST_TEST(it == p.end()); + } + + // Iterator comparison + { + push_parser p{nodes}; + BOOST_TEST(p.begin() == p.begin()); + BOOST_TEST(p.begin() != p.end()); + BOOST_TEST(p.end() == p.end()); + } + + // Iterator prefix increment + { + push_parser p{nodes}; + auto it = p.begin(); + ++it; + BOOST_TEST(it == p.end()); + } + + // Iterator postfix increment + { + push_parser p{nodes}; + auto it = p.begin(); + it++; + BOOST_TEST(it == p.end()); + } + + // Iterator dereference + { + push_parser p{nodes}; + auto it = p.begin(); + auto value = *it; + BOOST_TEST_EQ(value.payload, "Hello"); + } + + // Iterator operator-> + { + push_parser p{nodes}; + auto it = p.begin(); + BOOST_TEST_EQ(it->payload, "Hello"); + } +} + +} // namespace + +int main() +{ + test_one_message(); + test_several_messages(); + test_one_pmessage(); + test_mixed_valid(); + test_message_empty_fields(); + test_pmessage_empty_fields(); + + test_skip_type_array(); + test_skip_type_simple_error(); + test_skip_type_simple_string(); + test_skip_type_other(); + + test_skip_msgtype_subscribe_confirmations(); + test_skip_msgtype_unknown(); + test_skip_msgtype_empty(); + test_skip_msgtype_not_string(); + test_skip_msgtype_eof(); + + test_skip_pattern_not_string(); + test_skip_pattern_eof(); + + test_skip_channel_not_string_message(); + test_skip_channel_not_string_pmessage(); + test_skip_channel_eof_message(); + test_skip_channel_eof_pmessage(); + + test_skip_payload_not_string_message(); + test_skip_payload_not_string_pmessage(); + test_skip_payload_eof_message(); + test_skip_payload_eof_pmessage(); + + test_skip_longer_message(); + test_skip_longer_pmessage(); + test_skip_longer_pmessage(); + + test_only_skipped(); + test_valid_skipped(); + test_valid_skipped_mix(); + + test_empty(); + + test_range_interface(); + + return boost::report_errors(); +} From 3a45ebb47919233b2c279415ef9c2cd4cf839874 Mon Sep 17 00:00:00 2001 From: Andrey Semashev Date: Tue, 24 Feb 2026 13:42:59 +0300 Subject: [PATCH 36/37] Remove dependencies on Boost.StaticAssert. (#386) Boost.StaticAssert has been merged into Boost.Config, so remove the dependency. --- CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 967f1ccd7..a9baacf4e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,7 +38,6 @@ if (BOOST_REDIS_MAIN_PROJECT) align context core - static_assert pool date_time smart_ptr From 455b2e136d4e358859a6f84fcd261a67cbb86c3a Mon Sep 17 00:00:00 2001 From: Ruben Perez Date: Wed, 25 Feb 2026 12:22:44 +0100 Subject: [PATCH 37/37] Revert "Remove dependencies on Boost.StaticAssert. (#386)" This reverts commit 3a45ebb47919233b2c279415ef9c2cd4cf839874. Some dependencies have still not moved away from StaticAssert. This has effect only when Boost.Redis is the main project. --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index a9baacf4e..967f1ccd7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,6 +38,7 @@ if (BOOST_REDIS_MAIN_PROJECT) align context core + static_assert pool date_time smart_ptr