chore: import upstream snapshot with attribution

example/BUILD.bazel

@@ -0,0 +1,122 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("@rules_cc//cc:defs.bzl", "cc_binary")
+load("//bazel/tools:brpc_proto_library.bzl", "brpc_proto_library")
+
+COPTS = [
+    "-D__STDC_FORMAT_MACROS",
+    "-DBTHREAD_USE_FAST_PTHREAD_MUTEX",
+    "-D__const__=__unused__",
+    "-D_GNU_SOURCE",
+    "-DUSE_SYMBOLIZE",
+    "-DNO_TCMALLOC",
+    "-D__STDC_LIMIT_MACROS",
+    "-D__STDC_CONSTANT_MACROS",
+    "-fPIC",
+    "-Wno-unused-parameter",
+    "-fno-omit-frame-pointer",
+] + select({
+    "//bazel/config:brpc_with_glog": ["-DBRPC_WITH_GLOG=1"],
+    "//conditions:default": ["-DBRPC_WITH_GLOG=0"],
+}) + select({
+    "//bazel/config:brpc_with_rdma": ["-DBRPC_WITH_RDMA=1"],
+    "//conditions:default": [""],
+})
+
+brpc_proto_library(
+    name = "cc_echo_c++_proto",
+    srcs = ["echo_c++/echo.proto"],
+    include = "echo_c++",
+    proto_deps = [],
+)
+
+brpc_proto_library(
+    name = "cc_rdma_performance_proto",
+    srcs = ["rdma_performance/test.proto"],
+    include = "rdma_performance",
+    proto_deps = [],
+)
+
+cc_binary(
+    name = "echo_c++_server",
+    srcs = [
+        "echo_c++/server.cpp",
+    ],
+    copts = COPTS,
+    includes = [
+        "echo_c++",
+    ],
+    deps = [
+        ":cc_echo_c++_proto",
+        "//:brpc",
+    ],
+)
+
+cc_binary(
+    name = "echo_c++_client",
+    srcs = [
+        "echo_c++/client.cpp",
+    ],
+    copts = COPTS,
+    includes = [
+        "echo_c++",
+    ],
+    deps = [
+        ":cc_echo_c++_proto",
+        "//:brpc",
+    ],
+)
+
+cc_binary(
+    name = "rdma_performance_server",
+    srcs = [
+        "rdma_performance/server.cpp",
+    ],
+    includes = [
+        "rdma_performance",
+    ],
+    copts = COPTS + ["-DBRPC_WITH_RDMA=1"],
+    deps = [
+        ":cc_rdma_performance_proto",
+        "//:brpc",
+    ],
+)
+
+cc_binary(
+    name = "rdma_performance_client",
+    srcs = [
+        "rdma_performance/client.cpp",
+    ],
+    includes = [
+        "rdma_performance",
+    ],
+    copts = COPTS + ["-DBRPC_WITH_RDMA=1"],
+    deps = [
+        ":cc_rdma_performance_proto",
+        "//:brpc",
+    ],
+)
+
+cc_binary(
+    name = "redis_c++_server",
+    srcs = [
+        "redis_c++/redis_server.cpp",
+    ],
+    copts = COPTS,
+    deps = [
+        "//:brpc",
+    ],
+)

example/asynchronous_echo_c++/CMakeLists.txt

@@ -0,0 +1,110 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.16...3.28)
+project(asynchronous_echo_c++ C CXX)
+
+include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake)
+
+option(LINK_SO "Whether examples are linked dynamically" OFF)
+
+execute_process(
+    COMMAND bash -c "find ${PROJECT_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'"
+    OUTPUT_VARIABLE OUTPUT_PATH
+)
+
+if(OUTPUT_PATH)
+    list(PREPEND CMAKE_PREFIX_PATH ${OUTPUT_PATH})
+endif()
+
+find_package(Threads REQUIRED)
+find_package(Protobuf REQUIRED)
+protobuf_generate_cpp(PROTO_SRC PROTO_HEADER echo.proto)
+
+# Search for libthrift* by best effort. If it is not found and brpc is
+# compiled with thrift protocol enabled, a link error would be reported.
+find_library(THRIFT_LIB NAMES thrift)
+if (NOT THRIFT_LIB)
+    set(THRIFT_LIB "")
+endif()
+
+find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h)
+if(LINK_SO)
+    find_library(BRPC_LIB NAMES brpc)
+else()
+    find_library(BRPC_LIB NAMES libbrpc.a brpc)
+endif()
+if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB))
+    message(FATAL_ERROR "Fail to find brpc")
+endif()
+
+find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h)
+find_library(GFLAGS_LIBRARY NAMES gflags libgflags)
+if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY))
+    message(FATAL_ERROR "Fail to find gflags")
+endif()
+
+find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h)
+find_library(LEVELDB_LIB NAMES leveldb)
+if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB))
+    message(FATAL_ERROR "Fail to find leveldb")
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT OPENSSL_ROOT_DIR)
+    set(OPENSSL_ROOT_DIR
+        "/usr/local/opt/openssl"    # Homebrew installed OpenSSL
+        )
+endif()
+
+find_package(OpenSSL REQUIRED)
+
+set(DYNAMIC_LIB
+    Threads::Threads
+    ${GFLAGS_LIBRARY}
+    ${PROTOBUF_LIBRARIES}
+    ${LEVELDB_LIB}
+    ${OPENSSL_CRYPTO_LIBRARY}
+    ${OPENSSL_SSL_LIBRARY}
+    ${THRIFT_LIB}
+    dl
+    )
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    set(DYNAMIC_LIB ${DYNAMIC_LIB}
+        pthread
+        "-framework CoreFoundation"
+        "-framework CoreGraphics"
+        "-framework CoreData"
+        "-framework CoreText"
+        "-framework Security"
+        "-framework Foundation"
+        "-Wl,-U,_MallocExtension_ReleaseFreeMemory"
+        "-Wl,-U,_ProfilerStart"
+        "-Wl,-U,_ProfilerStop"
+        "-Wl,-U,__Z13GetStackTracePPvii"
+        "-Wl,-U,_mallctl"
+        "-Wl,-U,_malloc_stats_print"
+    )
+endif()
+
+add_executable(asynchronous_echo_client client.cpp ${PROTO_SRC})
+brpc_example_configure_target(asynchronous_echo_client)
+add_executable(asynchronous_echo_server server.cpp ${PROTO_SRC})
+brpc_example_configure_target(asynchronous_echo_server)
+
+target_link_libraries(asynchronous_echo_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})
+target_link_libraries(asynchronous_echo_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})

example/asynchronous_echo_c++/client.cpp

@@ -0,0 +1,110 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A client sending requests to server asynchronously every 1 second.
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <butil/time.h>
+#include <brpc/channel.h>
+#include "echo.pb.h"
+
+DEFINE_bool(send_attachment, true, "Carry attachment along with requests");
+DEFINE_string(protocol, "baidu_std", "Protocol type. Defined in src/brpc/options.proto");
+DEFINE_string(connection_type, "", "Connection type. Available values: single, pooled, short");
+DEFINE_string(server, "0.0.0.0:8003", "IP Address of server");
+DEFINE_string(load_balancer, "", "The algorithm for load balancing");
+DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); 
+
+void HandleEchoResponse(
+        brpc::Controller* cntl,
+        example::EchoResponse* response) {
+    // std::unique_ptr makes sure cntl/response will be deleted before returning.
+    std::unique_ptr<brpc::Controller> cntl_guard(cntl);
+    std::unique_ptr<example::EchoResponse> response_guard(response);
+
+    if (cntl->Failed()) {
+        LOG(WARNING) << "Fail to send EchoRequest, " << cntl->ErrorText();
+        return;
+    }
+    LOG(INFO) << "Received response from " << cntl->remote_side()
+        << ": " << response->message() << " (attached="
+        << cntl->response_attachment() << ")"
+        << " latency=" << cntl->latency_us() << "us";
+}
+                        
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    // A Channel represents a communication line to a Server. Notice that 
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::Channel channel;
+
+    // Initialize the channel, NULL means using default options.
+    brpc::ChannelOptions options;
+    options.protocol = FLAGS_protocol;
+    options.connection_type = FLAGS_connection_type;
+    options.timeout_ms = FLAGS_timeout_ms/*milliseconds*/;
+    options.max_retry = FLAGS_max_retry;
+    if (channel.Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), &options) != 0) {
+        LOG(ERROR) << "Fail to initialize channel";
+        return -1;
+    }
+
+    // Normally, you should not call a Channel directly, but instead construct
+    // a stub Service wrapping it. stub can be shared by all threads as well.
+    example::EchoService_Stub stub(&channel);
+    
+    // Send a request and wait for the response every 1 second.
+    int log_id = 0;
+    while (!brpc::IsAskedToQuit()) {
+        // Since we are sending asynchronous RPC (`done' is not NULL),
+        // these objects MUST remain valid until `done' is called.
+        // As a result, we allocate these objects on heap
+        example::EchoResponse* response = new example::EchoResponse();
+        brpc::Controller* cntl = new brpc::Controller();
+
+        // Notice that you don't have to new request, which can be modified
+        // or destroyed just after stub.Echo is called.
+        example::EchoRequest request;
+        request.set_message("hello world");
+
+        cntl->set_log_id(log_id ++);  // set by user
+        if (FLAGS_send_attachment) {
+            // Set attachment which is wired to network directly instead of 
+            // being serialized into protobuf messages.
+            cntl->request_attachment().append("foo");
+        }
+
+        // We use protobuf utility `NewCallback' to create a closure object
+        // that will call our callback `HandleEchoResponse'. This closure
+        // will automatically delete itself after being called once
+        google::protobuf::Closure* done = brpc::NewCallback(
+            &HandleEchoResponse, cntl, response);
+        stub.Echo(cntl, &request, response, done);
+
+        // This is an asynchronous RPC, so we can only fetch the result
+        // inside the callback
+        sleep(1);
+    }
+
+    LOG(INFO) << "EchoClient is going to quit";
+    return 0;
+}

example/asynchronous_echo_c++/echo.proto

@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+syntax="proto2";
+package example;
+
+option cc_generic_services = true;
+
+message EchoRequest {
+      required string message = 1;
+};
+
+message EchoResponse {
+      required string message = 1;
+};
+
+service EchoService {
+      rpc Echo(EchoRequest) returns (EchoResponse);
+};

example/asynchronous_echo_c++/server.cpp

@@ -0,0 +1,118 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A server to receive EchoRequest and send back EchoResponse.
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <brpc/server.h>
+#include <json2pb/pb_to_json.h>
+#include "echo.pb.h"
+
+DEFINE_bool(send_attachment, true, "Carry attachment along with response");
+DEFINE_int32(port, 8003, "TCP Port of this server");
+DEFINE_int32(idle_timeout_s, -1, "Connection will be closed if there is no "
+             "read/write operations during the last `idle_timeout_s'");
+
+// Your implementation of example::EchoService
+class EchoServiceImpl : public example::EchoService {
+public:
+    EchoServiceImpl() {}
+    virtual ~EchoServiceImpl() {}
+    virtual void Echo(google::protobuf::RpcController* cntl_base,
+                      const example::EchoRequest* request,
+                      example::EchoResponse* response,
+                      google::protobuf::Closure* done) {
+        // This object helps you to call done->Run() in RAII style. If you need
+        // to process the request asynchronously, pass done_guard.release().
+        brpc::ClosureGuard done_guard(done);
+
+        brpc::Controller* cntl =
+            static_cast<brpc::Controller*>(cntl_base);
+
+        // optional: set a callback function which is called after response is sent
+        // and before cntl/req/res is destructed.
+        cntl->set_after_rpc_resp_fn(std::bind(&EchoServiceImpl::CallAfterRpc,
+            std::placeholders::_1, std::placeholders::_2, std::placeholders::_3));
+
+        // The purpose of following logs is to help you to understand
+        // how clients interact with servers more intuitively. You should 
+        // remove these logs in performance-sensitive servers.
+        LOG(INFO) << "Received request[log_id=" << cntl->log_id() 
+                  << "] from " << cntl->remote_side()
+                  << ": " << request->message()
+                  << " (attached=" << cntl->request_attachment() << ")";
+
+        // Fill response.
+        response->set_message(request->message());
+
+        // You can compress the response by setting Controller, but be aware
+        // that compression may be costly, evaluate before turning on.
+        // cntl->set_response_compress_type(brpc::COMPRESS_TYPE_GZIP);
+
+        if (FLAGS_send_attachment) {
+            // Set attachment which is wired to network directly instead of
+            // being serialized into protobuf messages.
+            cntl->response_attachment().append("bar");
+        }
+    }
+
+    // optional
+    static void CallAfterRpc(brpc::Controller* cntl,
+                        const google::protobuf::Message* req,
+                        const google::protobuf::Message* res) {
+        // at this time res is already sent to client, but cntl/req/res is not destructed
+        std::string req_str;
+        std::string res_str;
+        json2pb::ProtoMessageToJson(*req, &req_str, NULL);
+        json2pb::ProtoMessageToJson(*res, &res_str, NULL);
+        LOG(INFO) << "req:" << req_str
+                    << " res:" << res_str;
+    }
+};
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    // Generally you only need one Server.
+    brpc::Server server;
+
+    // Instance of your service.
+    EchoServiceImpl echo_service_impl;
+
+    // Add the service into server. Notice the second parameter, because the
+    // service is put on stack, we don't want server to delete it, otherwise
+    // use brpc::SERVER_OWNS_SERVICE.
+    if (server.AddService(&echo_service_impl, 
+                          brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
+        LOG(ERROR) << "Fail to add service";
+        return -1;
+    }
+
+    // Start the server.
+    brpc::ServerOptions options;
+    options.idle_timeout_sec = FLAGS_idle_timeout_s;
+    if (server.Start(FLAGS_port, &options) != 0) {
+        LOG(ERROR) << "Fail to start EchoServer";
+        return -1;
+    }
+
+    // Wait until Ctrl-C is pressed, then Stop() and Join() the server.
+    server.RunUntilAskedToQuit();
+    return 0;
+}

example/auto_concurrency_limiter/CMakeLists.txt

@@ -0,0 +1,104 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.16...3.28)
+project(asynchronous_echo_c++ C CXX)
+
+include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake)
+
+option(LINK_SO "Whether examples are linked dynamically" OFF)
+
+execute_process(
+    COMMAND bash -c "find ${CMAKE_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'"
+    OUTPUT_VARIABLE OUTPUT_PATH
+)
+
+if(OUTPUT_PATH)
+    list(PREPEND CMAKE_PREFIX_PATH ${OUTPUT_PATH})
+endif()
+
+find_package(Threads REQUIRED)
+find_package(Protobuf REQUIRED)
+protobuf_generate_cpp(PROTO_SRC PROTO_HEADER cl_test.proto)
+
+find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h)
+if(LINK_SO)
+    find_library(BRPC_LIB NAMES brpc)
+else()
+    find_library(BRPC_LIB NAMES libbrpc.a brpc)
+endif()
+if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB))
+    message(FATAL_ERROR "Fail to find brpc")
+endif()
+
+find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h)
+find_library(GFLAGS_LIBRARY NAMES gflags libgflags)
+if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY))
+    message(FATAL_ERROR "Fail to find gflags")
+endif()
+
+find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h)
+find_library(LEVELDB_LIB NAMES leveldb)
+if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB))
+    message(FATAL_ERROR "Fail to find leveldb")
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT OPENSSL_ROOT_DIR)
+    set(OPENSSL_ROOT_DIR
+        "/usr/local/opt/openssl"    # Homebrew installed OpenSSL
+        )
+endif()
+
+find_package(OpenSSL REQUIRED)
+
+set(DYNAMIC_LIB
+    Threads::Threads
+    ${GFLAGS_LIBRARY}
+    ${PROTOBUF_LIBRARIES}
+    ${LEVELDB_LIB}
+    ${OPENSSL_CRYPTO_LIBRARY}
+    ${OPENSSL_SSL_LIBRARY}
+    ${THRIFT_LIB}
+    dl
+    )
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    set(DYNAMIC_LIB ${DYNAMIC_LIB}
+        pthread
+        "-framework CoreFoundation"
+        "-framework CoreGraphics"
+        "-framework CoreData"
+        "-framework CoreText"
+        "-framework Security"
+        "-framework Foundation"
+        "-Wl,-U,_MallocExtension_ReleaseFreeMemory"
+        "-Wl,-U,_ProfilerStart"
+        "-Wl,-U,_ProfilerStop"
+        "-Wl,-U,__Z13GetStackTracePPvii"
+        "-Wl,-U,_RegisterThriftProtocol"
+        "-Wl,-U,_mallctl"
+        "-Wl,-U,_malloc_stats_print"
+    )
+endif()
+
+add_executable(asynchronous_echo_client client.cpp ${PROTO_SRC})
+brpc_example_configure_target(asynchronous_echo_client)
+add_executable(asynchronous_echo_server server.cpp ${PROTO_SRC})
+brpc_example_configure_target(asynchronous_echo_server)
+
+target_link_libraries(asynchronous_echo_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})
+target_link_libraries(asynchronous_echo_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})

example/auto_concurrency_limiter/cl_test.proto

@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+syntax="proto2";
+package test;
+
+option cc_generic_services = true;
+
+message NotifyRequest {
+      required string message = 1;
+};
+
+message NotifyResponse {
+      required string message = 1;
+};
+
+enum ChangeType {
+    FLUCTUATE = 1;  // Fluctuating between upper and lower bound 
+    SMOOTH = 2;     // Smoothly rising from the lower bound to the upper bound 
+}
+
+message Stage {
+    required int32 lower_bound = 1;
+    required int32 upper_bound = 2;
+    required int32 duration_sec = 3;
+    required ChangeType type = 4; 
+}
+
+message TestCase {
+    required string case_name = 1;
+    required string max_concurrency = 2;
+    repeated Stage qps_stage_list = 3;
+    repeated Stage latency_stage_list = 4;
+}
+
+message TestCaseSet {
+    repeated TestCase test_case = 1;
+}
+
+service ControlService {
+    rpc Notify(NotifyRequest) returns (NotifyResponse);
+}
+
+service EchoService {
+    rpc Echo(NotifyRequest) returns (NotifyResponse);
+};
+

example/auto_concurrency_limiter/client.cpp

@@ -0,0 +1,247 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A client sending requests to server asynchronously every 1 second.
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <butil/time.h>
+#include <brpc/channel.h>
+#include <bvar/bvar.h>
+#include <bthread/timer_thread.h>
+#include <json2pb/json_to_pb.h>
+
+#include <fstream>
+#include "cl_test.pb.h"
+
+DEFINE_string(protocol, "baidu_std", "Protocol type. Defined in src/brpc/options.proto");
+DEFINE_string(connection_type, "", "Connection type. Available values: single, pooled, short");
+DEFINE_string(cntl_server, "0.0.0.0:9000", "IP Address of server");
+DEFINE_string(echo_server, "0.0.0.0:9001", "IP Address of server");
+DEFINE_int32(timeout_ms, 3000, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 0, "Max retries(not including the first RPC)"); 
+DEFINE_int32(case_interval, 20, "Intervals for different test cases");
+DEFINE_int32(client_qps_change_interval_us, 50000, 
+             "The interval for client changes the sending speed");
+DEFINE_string(case_file, "", "File path for test_cases");
+
+void DisplayStage(const test::Stage& stage) {
+    std::string type;
+    switch(stage.type()) {
+        case test::FLUCTUATE: 
+            type = "Fluctuate";
+            break;
+        case test::SMOOTH:
+            type = "Smooth";
+            break;
+        default:
+            type = "Unknown";
+    }
+    std::stringstream ss;
+    ss 
+        << "Stage:[" << stage.lower_bound() << ':' 
+        << stage.upper_bound() <<  "]"
+        << " , Type:" << type;
+    LOG(INFO) << ss.str();
+}
+
+uint32_t cast_func(void* arg) {
+    return *(uint32_t*)arg;
+}
+
+butil::atomic<uint32_t> g_timeout(0);
+butil::atomic<uint32_t> g_error(0);
+butil::atomic<uint32_t> g_succ(0);
+bvar::PassiveStatus<uint32_t> g_timeout_bvar(cast_func, &g_timeout);
+bvar::PassiveStatus<uint32_t> g_error_bvar(cast_func, &g_error);
+bvar::PassiveStatus<uint32_t> g_succ_bvar(cast_func, &g_succ);
+bvar::LatencyRecorder g_latency_rec;
+
+void LoadCaseSet(test::TestCaseSet* case_set, const std::string& file_path) {
+    std::ifstream ifs(file_path.c_str(), std::ios::in);  
+    if (!ifs) {
+        LOG(FATAL) << "Fail to open case set file: " << file_path;
+    }
+    std::string case_set_json((std::istreambuf_iterator<char>(ifs)),  
+                              std::istreambuf_iterator<char>()); 
+    std::string err;
+    if (!json2pb::JsonToProtoMessage(case_set_json, case_set, &err)) {
+        LOG(FATAL) 
+            << "Fail to trans case_set from json to protobuf message: "
+            << err;
+    }
+}
+
+void HandleEchoResponse(
+        brpc::Controller* cntl,
+        test::NotifyResponse* response) {
+    // std::unique_ptr makes sure cntl/response will be deleted before returning.
+    std::unique_ptr<brpc::Controller> cntl_guard(cntl);
+    std::unique_ptr<test::NotifyResponse> response_guard(response);
+
+    if (cntl->Failed() && cntl->ErrorCode() == brpc::ERPCTIMEDOUT) {
+        g_timeout.fetch_add(1, butil::memory_order_relaxed);
+        LOG_EVERY_N(INFO, 1000) << cntl->ErrorText();
+    } else if (cntl->Failed()) {
+        g_error.fetch_add(1, butil::memory_order_relaxed);
+        LOG_EVERY_N(INFO, 1000) << cntl->ErrorText();
+    } else {
+        g_succ.fetch_add(1, butil::memory_order_relaxed);
+        g_latency_rec << cntl->latency_us();
+    }
+
+}
+
+void Expose() {
+    g_timeout_bvar.expose_as("cl", "timeout");
+    g_error_bvar.expose_as("cl", "failed");
+    g_succ_bvar.expose_as("cl", "succ");
+    g_latency_rec.expose("cl");
+}
+
+struct TestCaseContext {
+    TestCaseContext(const test::TestCase& tc) 
+        : running(true)
+        , stage_index(0)
+        , test_case(tc)
+        , next_stage_sec(test_case.qps_stage_list(0).duration_sec() + 
+                         butil::cpuwide_time_s()) {
+        DisplayStage(test_case.qps_stage_list(stage_index));
+        Update();
+    }
+
+    bool Update() {
+        if (butil::cpuwide_time_s() >= next_stage_sec) {
+            ++stage_index;
+            if (stage_index < test_case.qps_stage_list_size()) {
+                next_stage_sec += test_case.qps_stage_list(stage_index).duration_sec(); 
+                DisplayStage(test_case.qps_stage_list(stage_index));
+            } else {
+                return false;
+            }
+        }
+
+        int qps = 0;
+        const test::Stage& qps_stage = test_case.qps_stage_list(stage_index);
+        const int lower_bound = qps_stage.lower_bound();
+        const int upper_bound = qps_stage.upper_bound();
+        if (qps_stage.type() == test::FLUCTUATE) {
+            qps = butil::fast_rand_less_than(upper_bound - lower_bound) + lower_bound;
+        } else if (qps_stage.type() == test::SMOOTH) {
+            qps = lower_bound + (upper_bound - lower_bound) / 
+                double(qps_stage.duration_sec()) * (qps_stage.duration_sec() - next_stage_sec
+                + butil::cpuwide_time_s());
+        }
+        interval_us.store(1.0 / qps * 1000000, butil::memory_order_relaxed);
+        return true;
+    }
+
+    butil::atomic<bool> running;
+    butil::atomic<int64_t> interval_us;
+    int stage_index;
+    const test::TestCase test_case;
+    int next_stage_sec;
+};
+
+void RunUpdateTask(void* data) {
+    TestCaseContext* context = (TestCaseContext*)data;
+    bool should_continue = context->Update();
+    if (should_continue) {
+        bthread::get_global_timer_thread()->schedule(RunUpdateTask, data, 
+            butil::microseconds_from_now(FLAGS_client_qps_change_interval_us));
+    } else {
+        context->running.store(false, butil::memory_order_release);
+    }
+}
+
+void RunCase(test::ControlService_Stub &cntl_stub, 
+             const test::TestCase& test_case) {
+    LOG(INFO) << "Running case:`" << test_case.case_name() << '\'';
+    brpc::Channel channel;
+    brpc::ChannelOptions options;
+    options.protocol = FLAGS_protocol;
+    options.connection_type = FLAGS_connection_type;
+    options.timeout_ms = FLAGS_timeout_ms;
+    options.max_retry = FLAGS_max_retry;
+    if (channel.Init(FLAGS_echo_server.c_str(), &options) != 0) {
+        LOG(FATAL) << "Fail to initialize channel";
+    }
+    test::EchoService_Stub echo_stub(&channel);
+
+    test::NotifyRequest cntl_req;
+    test::NotifyResponse cntl_rsp;
+    brpc::Controller cntl;
+    cntl_req.set_message("StartCase");
+    cntl_stub.Notify(&cntl, &cntl_req, &cntl_rsp, NULL);
+    CHECK(!cntl.Failed()) << "control failed";
+
+    TestCaseContext context(test_case);
+    bthread::get_global_timer_thread()->schedule(RunUpdateTask, &context, 
+        butil::microseconds_from_now(FLAGS_client_qps_change_interval_us));
+
+    while (context.running.load(butil::memory_order_acquire)) {
+        test::NotifyRequest echo_req;
+        echo_req.set_message("hello");
+        brpc::Controller* echo_cntl = new brpc::Controller;
+        test::NotifyResponse* echo_rsp = new test::NotifyResponse;
+        google::protobuf::Closure* done = brpc::NewCallback(
+            &HandleEchoResponse, echo_cntl, echo_rsp);
+        echo_stub.Echo(echo_cntl, &echo_req, echo_rsp, done);
+        ::usleep(context.interval_us.load(butil::memory_order_relaxed));
+    }
+
+    LOG(INFO) << "Waiting to stop case: `" << test_case.case_name() << '\'';
+    ::sleep(FLAGS_case_interval);
+    cntl.Reset();
+    cntl_req.set_message("StopCase");
+    cntl_stub.Notify(&cntl, &cntl_req, &cntl_rsp, NULL);
+    CHECK(!cntl.Failed()) << "control failed";
+    LOG(INFO) << "Case `" << test_case.case_name() << "' finshed:";
+}
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+    Expose();
+
+    brpc::Channel channel;
+    brpc::ChannelOptions options;
+    options.protocol = FLAGS_protocol;
+    options.connection_type = FLAGS_connection_type;
+    options.timeout_ms = FLAGS_timeout_ms;
+
+    if (channel.Init(FLAGS_cntl_server.c_str(), &options) != 0) {
+        LOG(ERROR) << "Fail to initialize channel";
+        return -1;
+    }
+    test::ControlService_Stub cntl_stub(&channel);
+
+    test::TestCaseSet case_set;
+    LoadCaseSet(&case_set, FLAGS_case_file);
+
+    brpc::Controller cntl;
+    test::NotifyRequest cntl_req;
+    test::NotifyResponse cntl_rsp;
+    cntl_req.set_message("ResetCaseSet");
+    cntl_stub.Notify(&cntl, &cntl_req, &cntl_rsp, NULL);
+    CHECK(!cntl.Failed()) << "Cntl Failed";
+    for (int i = 0; i < case_set.test_case_size(); ++i) {
+        RunCase(cntl_stub, case_set.test_case(i));
+    }
+    LOG(INFO) << "EchoClient is going to quit";
+    return 0;
+}

example/auto_concurrency_limiter/dummy_server.port

@@ -0,0 +1 @@
+9999

example/auto_concurrency_limiter/server.cpp

@@ -0,0 +1,295 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A server to receive EchoRequest and send back EchoResponse.
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <brpc/server.h>
+#include <butil/atomicops.h>
+#include <butil/time.h>
+#include <butil/logging.h>
+#include <json2pb/json_to_pb.h>
+#include <bthread/timer_thread.h>
+#include <bthread/bthread.h>
+
+#include <cstdlib>
+#include <fstream>
+#include "cl_test.pb.h"
+
+DEFINE_int32(server_bthread_concurrency, 4, 
+             "Configuring the value of bthread_concurrency, For compute max qps, ");
+DEFINE_int32(server_sync_sleep_us, 2500, 
+             "Usleep time, each request will be executed once, For compute max qps");
+// max qps = 1000 / 2.5 * 4 
+
+DEFINE_int32(control_server_port, 9000, "");
+DEFINE_int32(echo_port, 9001, "TCP Port of echo server");
+DEFINE_int32(cntl_port, 9000, "TCP Port of controller server");
+DEFINE_string(case_file, "", "File path for test_cases");
+DEFINE_int32(latency_change_interval_us, 50000, "Intervalt for server side changes the latency");
+DEFINE_int32(server_max_concurrency, 0, "Echo Server's max_concurrency");
+DEFINE_bool(use_usleep, false, 
+            "EchoServer uses ::usleep or bthread_usleep to simulate latency "
+            "when processing requests");
+
+
+bthread::TimerThread g_timer_thread;
+
+int cast_func(void* arg) {
+    return *(int*)arg;
+}
+
+void DisplayStage(const test::Stage& stage) {
+    std::string type;
+    switch(stage.type()) {
+        case test::FLUCTUATE: 
+            type = "Fluctuate";
+            break;
+        case test::SMOOTH:
+            type = "Smooth";
+            break;
+        default:
+            type = "Unknown";
+    }
+    std::stringstream ss;
+    ss 
+        << "Stage:[" << stage.lower_bound() << ':' 
+        << stage.upper_bound() <<  "]"
+        << " , Type:" << type;
+    LOG(INFO) << ss.str();
+}
+
+butil::atomic<int> cnt(0);
+butil::atomic<int> atomic_sleep_time(0);
+bvar::PassiveStatus<int> atomic_sleep_time_bvar(cast_func, &atomic_sleep_time);
+
+namespace bthread {
+DECLARE_int32(bthread_concurrency);
+}
+
+void TimerTask(void* data);
+
+class EchoServiceImpl : public test::EchoService {
+public:
+    EchoServiceImpl() 
+        : _stage_index(0)
+        , _running_case(false) {
+    };
+
+    virtual ~EchoServiceImpl() {}
+
+    void SetTestCase(const test::TestCase& test_case) {
+        _test_case = test_case;
+        _next_stage_start = _test_case.latency_stage_list(0).duration_sec() + 
+            butil::cpuwide_time_s();
+        _stage_index = 0;
+        _running_case = false;
+        DisplayStage(_test_case.latency_stage_list(_stage_index));
+    }
+
+    void StartTestCase() {
+        CHECK(!_running_case);
+        _running_case = true;
+        UpdateLatency();
+    }
+
+    void StopTestCase() {
+        _running_case = false;
+    }
+    
+    void UpdateLatency() {
+        if (!_running_case) {
+            return;
+        }
+        ComputeLatency();
+        g_timer_thread.schedule(TimerTask, (void*)this, 
+            butil::microseconds_from_now(FLAGS_latency_change_interval_us));
+    }
+
+    virtual void Echo(google::protobuf::RpcController* cntl_base,
+                      const test::NotifyRequest* request,
+                      test::NotifyResponse* response,
+                      google::protobuf::Closure* done) {
+        brpc::ClosureGuard done_guard(done); 
+        response->set_message("hello");
+        ::usleep(FLAGS_server_sync_sleep_us);
+        if (FLAGS_use_usleep) {
+            ::usleep(_latency.load(butil::memory_order_relaxed));
+        } else {
+            bthread_usleep(_latency.load(butil::memory_order_relaxed));
+        }
+    }
+
+    void ComputeLatency() {
+        if (_stage_index < _test_case.latency_stage_list_size() &&
+            butil::cpuwide_time_s() > _next_stage_start) {
+            ++_stage_index;
+            if (_stage_index < _test_case.latency_stage_list_size()) {
+                _next_stage_start += _test_case.latency_stage_list(_stage_index).duration_sec();
+                DisplayStage(_test_case.latency_stage_list(_stage_index));
+            }
+        }
+
+        if (_stage_index == _test_case.latency_stage_list_size()) {
+            const test::Stage& latency_stage = 
+                _test_case.latency_stage_list(_stage_index - 1);
+            if (latency_stage.type() == test::ChangeType::FLUCTUATE) {
+                _latency.store((latency_stage.lower_bound() + latency_stage.upper_bound()) / 2,
+                               butil::memory_order_relaxed);
+            } else if (latency_stage.type() == test::ChangeType::SMOOTH) {
+                _latency.store(latency_stage.upper_bound(), butil::memory_order_relaxed);
+            }
+            return;
+        }
+        
+        const test::Stage& latency_stage = _test_case.latency_stage_list(_stage_index);
+        const int lower_bound = latency_stage.lower_bound();
+        const int upper_bound = latency_stage.upper_bound();
+        if (latency_stage.type() == test::FLUCTUATE) {
+            _latency.store(butil::fast_rand_less_than(upper_bound - lower_bound) + lower_bound,
+                           butil::memory_order_relaxed); 
+        } else if (latency_stage.type() == test::SMOOTH) {
+            int latency = lower_bound + (upper_bound - lower_bound) / 
+                double(latency_stage.duration_sec()) * 
+                (latency_stage.duration_sec() - _next_stage_start + 
+                butil::cpuwide_time_s());
+            _latency.store(latency, butil::memory_order_relaxed);
+        } else {
+            LOG(FATAL) << "Wrong Type:" << latency_stage.type();
+        }
+    }
+
+private:
+    int _stage_index;
+    int _next_stage_start;
+    butil::atomic<int> _latency;
+    test::TestCase _test_case;
+    bool _running_case;
+};
+
+void TimerTask(void* data) {
+    EchoServiceImpl* echo_service = (EchoServiceImpl*)data;
+    echo_service->UpdateLatency();
+}
+
+class ControlServiceImpl : public test::ControlService {
+public:
+    ControlServiceImpl() 
+        : _case_index(0) {
+        LoadCaseSet(FLAGS_case_file);
+        _echo_service = new EchoServiceImpl;
+        if (_server.AddService(_echo_service,
+                                brpc::SERVER_OWNS_SERVICE) != 0) {
+            LOG(FATAL) << "Fail to add service";
+        }
+        g_timer_thread.start(NULL);
+    }
+    virtual ~ControlServiceImpl() { 
+        _echo_service->StopTestCase();
+        g_timer_thread.stop_and_join(); 
+    };
+
+    virtual void Notify(google::protobuf::RpcController* cntl_base,
+                        const test::NotifyRequest* request,
+                        test::NotifyResponse* response,
+                        google::protobuf::Closure* done) {
+        brpc::ClosureGuard done_guard(done);
+        const std::string& message = request->message();
+        LOG(INFO) << message;
+        if (message == "ResetCaseSet") {
+            _server.Stop(0);
+            _server.Join();
+            _echo_service->StopTestCase();
+
+            LoadCaseSet(FLAGS_case_file);
+            _case_index = 0;
+            response->set_message("CaseSetReset");
+        } else if (message == "StartCase") {
+            CHECK(!_server.IsRunning()) << "Continuous StartCase";
+            const test::TestCase& test_case = _case_set.test_case(_case_index++);
+            _echo_service->SetTestCase(test_case);
+            brpc::ServerOptions options;
+            options.max_concurrency = FLAGS_server_max_concurrency;
+            _server.MaxConcurrencyOf("test.EchoService.Echo") = test_case.max_concurrency();
+
+            _server.Start(FLAGS_echo_port, &options);            
+            _echo_service->StartTestCase();
+            response->set_message("CaseStarted");
+        } else if (message == "StopCase") {
+            CHECK(_server.IsRunning()) << "Continuous StopCase";
+            _server.Stop(0);
+            _server.Join();
+
+            _echo_service->StopTestCase();
+            response->set_message("CaseStopped");
+        } else {
+            LOG(FATAL) << "Invalid message:" << message;
+            response->set_message("Invalid Cntl Message");
+        }
+    }
+
+private:
+    void LoadCaseSet(const std::string& file_path) {
+        std::ifstream ifs(file_path.c_str(), std::ios::in);  
+        if (!ifs) {
+            LOG(FATAL) << "Fail to open case set file: " << file_path;
+        }
+        std::string case_set_json((std::istreambuf_iterator<char>(ifs)),  
+                                  std::istreambuf_iterator<char>()); 
+        test::TestCaseSet case_set;
+        std::string err;
+        if (!json2pb::JsonToProtoMessage(case_set_json, &case_set, &err)) {
+            LOG(FATAL) 
+                << "Fail to trans case_set from json to protobuf message: "
+                << err;
+        }
+        _case_set = case_set;
+        ifs.close();
+    }
+
+    brpc::Server _server;
+    EchoServiceImpl* _echo_service;
+    test::TestCaseSet _case_set;
+    int _case_index;
+};
+
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+    bthread::FLAGS_bthread_concurrency= FLAGS_server_bthread_concurrency;
+
+    brpc::Server server;
+
+    ControlServiceImpl control_service_impl;
+
+    if (server.AddService(&control_service_impl, 
+                          brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
+        LOG(ERROR) << "Fail to add service";
+        return -1;
+    }
+
+    if (server.Start(FLAGS_cntl_port, NULL) != 0) {
+        LOG(ERROR) << "Fail to start EchoServer";
+        return -1;
+    }
+
+    server.RunUntilAskedToQuit();
+    return 0;
+}
+

example/auto_concurrency_limiter/settings.flags

@@ -0,0 +1,25 @@
+#client settings
+--case_file=test_case.json
+--client_qps_change_interval_us=50000
+--max_retry=0
+
+#auto_cl settings
+--auto_cl_initial_max_concurrency=40
+--auto_cl_max_explore_ratio=0.3
+--auto_cl_min_explore_ratio=0.06
+--auto_cl_change_rate_of_explore_ratio=0.02
+--auto_cl_reduce_ratio_while_remeasure=0.9
+--auto_cl_latency_fluctuation_correction_factor=2
+
+#server setings for async sleep
+--latency_change_interval_us=50000
+--server_bthread_concurrency=4
+--server_sync_sleep_us=2500
+--use_usleep=false
+
+#server setings for sync sleep
+#--latency_change_interval_us=50000
+#--server_bthread_concurrency=16
+#--server_max_concurrency=15
+#--server_sync_sleep_us=2500
+#--use_usleep=true

example/auto_concurrency_limiter/test_case.json

@@ -0,0 +1,253 @@
+{"test_case":[
+
+{
+    "case_name":"CheckPeakQps",
+    "max_concurrency":"140",
+    "qps_stage_list":
+    [
+        {
+            "lower_bound":3000,
+            "upper_bound":3000,
+            "duration_sec":30,
+            "type":2
+        }
+    ],
+    "latency_stage_list":
+    [
+        {
+            "lower_bound":20000,
+            "upper_bound":20000,
+            "duration_sec":30,
+            "type":2
+        }
+    ]
+},
+
+
+{
+    "case_name":"qps_stable_noload, latency_raise_smooth",
+    "max_concurrency":"auto",
+    "qps_stage_list":
+    [
+        {
+            "lower_bound":1500,
+            "upper_bound":1500,
+            "duration_sec":190,
+            "type":2
+        }
+    ],
+    "latency_stage_list":
+    [
+        {
+            "lower_bound":2000,
+            "upper_bound":90000,
+            "duration_sec":200,
+            "type":2
+        }
+    ]
+},
+
+
+{
+    "case_name":"qps_fluctuate_noload, latency_stable",
+    "max_concurrency":"auto",
+    "qps_stage_list":
+    [
+        {
+            "lower_bound":300,
+            "upper_bound":1800,
+            "duration_sec":290,
+            "type":1
+        }
+    ],
+    "latency_stage_list":
+    [
+        {
+            "lower_bound":40000,
+            "upper_bound":40000,
+            "duration_sec":300,
+            "type":1
+        }
+    ]
+},
+
+
+{
+    "case_name":"qps_stable_overload, latency_stable",
+    "max_concurrency":"auto",
+    "qps_stage_list":
+    [
+        {
+            "lower_bound":3000,
+            "upper_bound":3000,
+            "duration_sec":180,
+            "type":2
+        }
+    ],
+    "latency_stage_list":
+    [
+        {
+            "lower_bound":40000,
+            "upper_bound":40000,
+            "duration_sec":200,
+            "type":2
+        }
+    ]
+},
+
+{
+    "case_name":"qps_stable_overload, latency_raise_smooth",
+    "max_concurrency":"auto",
+    "qps_stage_list":
+    [
+        {
+            "lower_bound":3000,
+            "upper_bound":3000,
+            "duration_sec":180,
+            "type":2
+        }
+    ],
+    "latency_stage_list":
+    [
+        {
+            "lower_bound":30000,
+            "upper_bound":80000,
+            "duration_sec":200,
+            "type":2
+        }
+    ]
+},
+
+{
+    "case_name":"qps_overload_then_noload, latency_stable",
+    "max_concurrency":"auto",
+    "qps_stage_list":
+    [
+        {
+            "lower_bound":200,
+            "upper_bound":2500,
+            "duration_sec":20,
+            "type":2
+        },
+        {
+            "lower_bound":2500,
+            "upper_bound":2500,
+            "duration_sec":150,
+            "type":2
+        },
+        {
+            "lower_bound":2500,
+            "upper_bound":1000,
+            "duration_sec":20,
+            "type":2
+        },
+        {
+            "lower_bound":1000,
+            "upper_bound":1000,
+            "duration_sec":150,
+            "type":2
+        }
+    ],
+    "latency_stage_list":
+    [
+        {
+            "lower_bound":30000,
+            "upper_bound":30000,
+            "duration_sec":200,
+            "type":2
+        }
+    ]
+},
+
+
+{
+    "case_name":"qps_noload_to_overload, latency_stable",
+    "max_concurrency":"auto",
+    "qps_stage_list":
+    [
+        {
+            "lower_bound":200,
+            "upper_bound":3000,
+            "duration_sec":150,
+            "type":2
+        },
+        {
+            "lower_bound":3000,
+            "upper_bound":3000,
+            "duration_sec":150,
+            "type":2
+        }
+   ],
+    "latency_stage_list":
+    [
+        {
+            "lower_bound":30000,
+            "upper_bound":30000,
+            "duration_sec":200,
+            "type":2
+        }
+    ]
+},
+
+{
+    "case_name":"qps_stable_noload, latency_leap_raise",
+    "max_concurrency":"auto",
+    "qps_stage_list":
+    [
+        {
+            "lower_bound":300,
+            "upper_bound":1800,
+            "duration_sec":20,
+            "type":2
+        },
+        {
+            "lower_bound":1800,
+            "upper_bound":1800,
+            "duration_sec":220,
+            "type":2
+        }
+    ],
+    "latency_stage_list":
+    [
+        {
+            "lower_bound":30000,
+            "upper_bound":30000,
+            "duration_sec":100,
+            "type":2
+        },
+        {
+            "lower_bound":50000,
+            "upper_bound":50000,
+            "duration_sec":100,
+            "type":2
+        }
+    ]
+},
+
+{
+    "case_name":"qps_fluctuate_noload, latency_fluctuate_noload",
+    "max_concurrency":"auto",
+    "qps_stage_list":
+    [
+        {
+            "lower_bound":300,
+            "upper_bound":1800,
+            "duration_sec":190,
+            "type":1
+        }
+    ],
+    "latency_stage_list":
+    [
+        {
+            "lower_bound":30000,
+            "upper_bound":50000,
+            "duration_sec":200,
+            "type":1
+        }
+    ]
+}
+
+
+
+
+]}

example/backup_request_c++/CMakeLists.txt

@@ -0,0 +1,110 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.16...3.28)
+project(backup_request_c++ C CXX)
+
+include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake)
+
+option(LINK_SO "Whether examples are linked dynamically" OFF)
+
+execute_process(
+    COMMAND bash -c "find ${PROJECT_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'"
+    OUTPUT_VARIABLE OUTPUT_PATH
+)
+
+if(OUTPUT_PATH)
+    list(PREPEND CMAKE_PREFIX_PATH ${OUTPUT_PATH})
+endif()
+
+find_package(Threads REQUIRED)
+find_package(Protobuf REQUIRED)
+protobuf_generate_cpp(PROTO_SRC PROTO_HEADER echo.proto)
+
+# Search for libthrift* by best effort. If it is not found and brpc is
+# compiled with thrift protocol enabled, a link error would be reported.
+find_library(THRIFT_LIB NAMES thrift)
+if (NOT THRIFT_LIB)
+    set(THRIFT_LIB "")
+endif()
+
+find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h)
+if(LINK_SO)
+    find_library(BRPC_LIB NAMES brpc)
+else()
+    find_library(BRPC_LIB NAMES libbrpc.a brpc)
+endif()
+if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB))
+    message(FATAL_ERROR "Fail to find brpc")
+endif()
+
+find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h)
+find_library(GFLAGS_LIBRARY NAMES gflags libgflags)
+if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY))
+    message(FATAL_ERROR "Fail to find gflags")
+endif()
+
+find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h)
+find_library(LEVELDB_LIB NAMES leveldb)
+if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB))
+    message(FATAL_ERROR "Fail to find leveldb")
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT OPENSSL_ROOT_DIR)
+    set(OPENSSL_ROOT_DIR
+        "/usr/local/opt/openssl"    # Homebrew installed OpenSSL
+        )
+endif()
+
+find_package(OpenSSL REQUIRED)
+
+set(DYNAMIC_LIB
+    Threads::Threads
+    ${GFLAGS_LIBRARY}
+    ${PROTOBUF_LIBRARIES}
+    ${LEVELDB_LIB}
+    ${OPENSSL_CRYPTO_LIBRARY}
+    ${OPENSSL_SSL_LIBRARY}
+    ${THRIFT_LIB}
+    dl
+    )
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    set(DYNAMIC_LIB ${DYNAMIC_LIB}
+        pthread
+        "-framework CoreFoundation"
+        "-framework CoreGraphics"
+        "-framework CoreData"
+        "-framework CoreText"
+        "-framework Security"
+        "-framework Foundation"
+        "-Wl,-U,_MallocExtension_ReleaseFreeMemory"
+        "-Wl,-U,_ProfilerStart"
+        "-Wl,-U,_ProfilerStop"
+        "-Wl,-U,__Z13GetStackTracePPvii"
+        "-Wl,-U,_mallctl"
+        "-Wl,-U,_malloc_stats_print"
+    )
+endif()
+
+add_executable(backup_request_client client.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(backup_request_client)
+add_executable(backup_request_server server.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(backup_request_server)
+
+target_link_libraries(backup_request_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})
+target_link_libraries(backup_request_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})

example/backup_request_c++/client.cpp

@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A client sending requests to server every 1 second. If the response does
+// not come back within FLAGS_backup_request_ms, it sends another request
+// and ends the RPC when any response comes back.
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <butil/time.h>
+#include <brpc/channel.h>
+#include "echo.pb.h"
+
+DEFINE_string(protocol, "baidu_std", "Protocol type. Defined in src/brpc/options.proto");
+DEFINE_string(connection_type, "", "Connection type. Available values: single, pooled, short");
+DEFINE_string(server, "0.0.0.0:8000", "IP Address of server");
+DEFINE_string(load_balancer, "", "The algorithm for load balancing");
+DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)");
+DEFINE_int32(backup_request_ms, 2, "Timeout for sending backup request");
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+    
+    // A Channel represents a communication line to a Server. Notice that 
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::Channel channel;
+
+    // Initialize the channel, NULL means using default options.
+    brpc::ChannelOptions options;
+    options.protocol = FLAGS_protocol;
+    options.connection_type = FLAGS_connection_type;
+    options.timeout_ms = FLAGS_timeout_ms/*milliseconds*/;
+    options.max_retry = FLAGS_max_retry;
+    options.backup_request_ms = FLAGS_backup_request_ms;
+    if (channel.Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), &options) != 0) {
+        LOG(ERROR) << "Fail to initialize channel";
+        return -1;
+    }
+
+    // Normally, you should not call a Channel directly, but instead construct
+    // a stub Service wrapping it. stub can be shared by all threads as well.
+    example::EchoService_Stub stub(&channel);
+
+    // Send a request and wait for the response every 1 second.
+    int counter = 0;
+    while (!brpc::IsAskedToQuit()) {
+        // We will receive response synchronously, safe to put variables
+        // on stack.
+        example::EchoRequest request;
+        example::EchoResponse response;
+        brpc::Controller cntl;
+
+        request.set_index(++counter);
+
+        // Because `done'(last parameter) is NULL, this function waits until
+        // the response comes back or error occurs(including timedout).
+        stub.Echo(&cntl, &request, &response, NULL);
+        if (!cntl.Failed()) {
+            LOG(INFO) << "Received response[index=" << response.index()
+                      << "] from " << cntl.remote_side()
+                      << " to " << cntl.local_side()
+                      << " latency=" << cntl.latency_us() << "us";
+        } else {
+            LOG(WARNING) << cntl.ErrorText();
+        }
+        sleep(1);
+    }
+
+    LOG(INFO) << "EchoClient is going to quit";
+    return 0;
+}

example/backup_request_c++/echo.proto

@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+syntax="proto2";
+package example;
+
+option cc_generic_services = true;
+
+message EchoRequest {
+  required int32 index = 1;
+};
+
+message EchoResponse {
+  required int32 index = 1;
+};
+
+service EchoService {
+  rpc Echo(EchoRequest) returns (EchoResponse);
+};

example/backup_request_c++/server.cpp

@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A server sleeping for even-th requests to trigger backup request of client.
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <brpc/server.h>
+#include "echo.pb.h"
+
+DEFINE_bool(echo_attachment, true, "Echo attachment as well");
+DEFINE_int32(port, 8000, "TCP Port of this server");
+DEFINE_int32(idle_timeout_s, -1, "Connection will be closed if there is no "
+             "read/write operations during the last `idle_timeout_s'");
+DEFINE_int32(sleep_ms, 20, "Sleep so many milliseconds on even-th requests");
+
+// Your implementation of example::EchoService
+// Notice that implementing brpc::Describable grants the ability to put
+// additional information in /status.
+namespace example {
+class SleepyEchoService : public EchoService
+                        , public brpc::Describable {
+public:
+    SleepyEchoService() : _count(0) {}
+    virtual ~SleepyEchoService() {}
+    virtual void Echo(google::protobuf::RpcController* cntl_base,
+                      const EchoRequest* request,
+                      EchoResponse* response,
+                      google::protobuf::Closure* done) {
+        // This object helps you to call done->Run() in RAII style. If you need
+        // to process the request asynchronously, pass done_guard.release().
+        brpc::ClosureGuard done_guard(done);
+
+        brpc::Controller* cntl =
+            static_cast<brpc::Controller*>(cntl_base);
+
+        // The purpose of following logs is to help you to understand
+        // how clients interact with servers more intuitively. You should 
+        // remove these logs in performance-sensitive servers.
+        // The noflush prevents the log from being flushed immediately.
+        LOG(INFO) << "Received request[index=" << request->index()
+                  << "] from " << cntl->remote_side() 
+                  << " to " << cntl->local_side() << noflush;
+        // Sleep a while for 0th, 2nd, 4th, 6th ... requests to trigger backup request
+        // at client-side.
+        bool do_sleep = (_count.fetch_add(1, butil::memory_order_relaxed) % 2 == 0);
+        if (do_sleep) {
+            LOG(INFO) << ", sleep " << FLAGS_sleep_ms 
+                      << " ms to trigger backup request" << noflush;
+        }
+        LOG(INFO);
+
+        // Fill response.
+        response->set_index(request->index());
+
+        if (do_sleep) {
+            bthread_usleep(FLAGS_sleep_ms * 1000);
+        }
+    }
+private:
+    butil::atomic<int> _count;
+};
+}  // namespace example
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    // Generally you only need one Server.
+    brpc::Server server;
+
+    // Instance of your service.
+    example::SleepyEchoService echo_service_impl;
+
+    // Add the service into server. Notice the second parameter, because the
+    // service is put on stack, we don't want server to delete it, otherwise
+    // use brpc::SERVER_OWNS_SERVICE.
+    if (server.AddService(&echo_service_impl, 
+                          brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
+        LOG(ERROR) << "Fail to add service";
+        return -1;
+    }
+
+    // Start the server.
+    brpc::ServerOptions options;
+    options.idle_timeout_sec = FLAGS_idle_timeout_s;
+    if (server.Start(FLAGS_port, &options) != 0) {
+        LOG(ERROR) << "Fail to start EchoServer";
+        return -1;
+    }
+
+    // Wait until Ctrl-C is pressed, then Stop() and Join() the server.
+    server.RunUntilAskedToQuit();
+    return 0;
+}

example/baidu_proxy_and_generic_call/CMakeLists.txt

@@ -0,0 +1,105 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.16...3.28)
+project(baidu_proxy_and_generic_call C CXX)
+
+include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake)
+
+option(LINK_SO "Whether examples are linked dynamically" OFF)
+
+execute_process(
+    COMMAND bash -c "find ${PROJECT_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'"
+    OUTPUT_VARIABLE OUTPUT_PATH
+)
+
+if(OUTPUT_PATH)
+    list(PREPEND CMAKE_PREFIX_PATH ${OUTPUT_PATH})
+endif()
+
+find_package(Threads REQUIRED)
+find_package(Protobuf REQUIRED)
+protobuf_generate_cpp(PROTO_SRC PROTO_HEADER echo.proto)
+
+find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h)
+if(LINK_SO)
+    find_library(BRPC_LIB NAMES brpc)
+else()
+    find_library(BRPC_LIB NAMES libbrpc.a brpc)
+endif()
+if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB))
+    message(FATAL_ERROR "Fail to find brpc")
+endif()
+
+find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h)
+find_library(GFLAGS_LIBRARY NAMES gflags libgflags)
+if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY))
+    message(FATAL_ERROR "Fail to find gflags")
+endif()
+
+find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h)
+find_library(LEVELDB_LIB NAMES leveldb)
+if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB))
+    message(FATAL_ERROR "Fail to find leveldb")
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT OPENSSL_ROOT_DIR)
+    set(OPENSSL_ROOT_DIR
+        "/usr/local/opt/openssl"    # Homebrew installed OpenSSL
+        )
+endif()
+
+find_package(OpenSSL REQUIRED)
+
+set(DYNAMIC_LIB
+    Threads::Threads
+    ${GFLAGS_LIBRARY}
+    ${PROTOBUF_LIBRARIES}
+    ${LEVELDB_LIB}
+    ${OPENSSL_CRYPTO_LIBRARY}
+    ${OPENSSL_SSL_LIBRARY}
+    dl
+    )
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    set(DYNAMIC_LIB ${DYNAMIC_LIB}
+        pthread
+        "-framework CoreFoundation"
+        "-framework CoreGraphics"
+        "-framework CoreData"
+        "-framework CoreText"
+        "-framework Security"
+        "-framework Foundation"
+        "-Wl,-U,_MallocExtension_ReleaseFreeMemory"
+        "-Wl,-U,_ProfilerStart"
+        "-Wl,-U,_ProfilerStop"
+        "-Wl,-U,__Z13GetStackTracePPvii"
+        "-Wl,-U,_mallctl"
+        "-Wl,-U,_malloc_stats_print"
+    )
+endif()
+
+add_executable(echo_client client.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(echo_client)
+add_executable(proxy proxy.cpp)
+brpc_example_configure_target(proxy)
+add_executable(echo_server server.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(echo_server)
+
+target_link_libraries(echo_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})
+target_link_libraries(proxy PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})
+target_link_libraries(echo_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})

example/baidu_proxy_and_generic_call/client.cpp

@@ -0,0 +1,94 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A client sending requests to server every 1 second.
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <butil/time.h>
+#include <brpc/channel.h>
+#include "echo.pb.h"
+
+DEFINE_int32(compress_type, 2, "The compress type of request");
+DEFINE_string(attachment, "", "Carry this along with requests");
+DEFINE_string(connection_type, "", "Connection type. Available values: single, pooled, short");
+DEFINE_string(proxy_address, "0.0.0.0:8000", "IP Address of proxy");
+DEFINE_string(load_balancer, "", "The algorithm for load balancing");
+DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); 
+DEFINE_int32(interval_ms, 1000, "Milliseconds between consecutive requests");
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+    
+    // A Channel represents a communication line to a Server. Notice that 
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::Channel channel;
+    
+    // Initialize the channel, NULL means using default options.
+    brpc::ChannelOptions options;
+    options.protocol = brpc::PROTOCOL_BAIDU_STD;
+    options.connection_type = FLAGS_connection_type;
+    options.timeout_ms = FLAGS_timeout_ms/*milliseconds*/;
+    options.max_retry = FLAGS_max_retry;
+    if (channel.Init(FLAGS_proxy_address.c_str(),
+                     FLAGS_load_balancer.c_str(), &options) != 0) {
+        LOG(ERROR) << "Fail to initialize channel";
+        return -1;
+    }
+
+    // Normally, you should not call a Channel directly, but instead construct
+    // a stub Service wrapping it. stub can be shared by all threads as well.
+    example::EchoService_Stub stub(&channel);
+
+    // Send a request and wait for the response every 1 second.
+    int log_id = 0;
+    while (!brpc::IsAskedToQuit()) {
+        // We will receive response synchronously, safe to put variables
+        // on stack.
+        example::EchoRequest request;
+        example::EchoResponse response;
+        brpc::Controller cntl;
+
+        request.set_message("hello world");
+        cntl.set_request_compress_type((brpc::CompressType)FLAGS_compress_type);
+
+        cntl.set_log_id(log_id++);  // set by user
+        // Set attachment which is wired to network directly instead of 
+        // being serialized into protobuf messages.
+        cntl.request_attachment().append(FLAGS_attachment);
+
+        // Because `done'(last parameter) is NULL, this function waits until
+        // the response comes back or error occurs(including timedout).
+        stub.Echo(&cntl, &request, &response, NULL);
+        if (!cntl.Failed()) {
+            LOG(INFO) << "Received response from " << cntl.remote_side()
+                << " to " << cntl.local_side()
+                << ": " << response.message()
+                << ", response compress type=" << cntl.response_compress_type()
+                << ", attached=" << cntl.response_attachment()
+                << ", latency=" << cntl.latency_us() << "us";
+        } else {
+            LOG(WARNING) << cntl.ErrorText();
+        }
+        usleep(FLAGS_interval_ms * 1000L);
+    }
+
+    LOG(INFO) << "EchoClient is going to quit";
+    return 0;
+}

example/baidu_proxy_and_generic_call/echo.proto

@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+syntax="proto2";
+package example;
+
+option cc_generic_services = true;
+
+message EchoRequest {
+      required string message = 1;
+};
+
+message EchoResponse {
+      required string message = 1;
+};
+
+service EchoService {
+      rpc Echo(EchoRequest) returns (EchoResponse);
+};

example/baidu_proxy_and_generic_call/proxy.cpp

@@ -0,0 +1,142 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// todo
+// A proxy to receive EchoRequest and send back EchoResponse.
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <butil/strings/string_number_conversions.h>
+#include <brpc/server.h>
+#include <brpc/controller.h>
+#include <brpc/channel.h>
+#include <json2pb/pb_to_json.h>
+
+DEFINE_int32(port, 8000, "TCP Port of this server");
+DEFINE_string(listen_addr, "", "Server listen address, may be IPV4/IPV6/UDS."
+            " If this is set, the flag port will be ignored");
+DEFINE_int32(idle_timeout_s, -1, "Connection will be closed if there is no "
+             "read/write operations during the last `idle_timeout_s'");
+DEFINE_string(connection_type, "", "Connection type. Available values: single, pooled, short");
+DEFINE_string(server_address, "0.0.0.0:8001", "IP Address of server");
+DEFINE_string(load_balancer, "", "The algorithm for load balancing");
+DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)");
+DEFINE_int32(interval_ms, 1000, "Milliseconds between consecutive requests");
+
+// Your implementation of example::EchoService
+// Notice that implementing brpc::Describable grants the ability to put
+// additional information in /status.
+namespace example {
+class BaiduMasterServiceImpl : public brpc::BaiduMasterService {
+public:
+    void ProcessRpcRequest(brpc::Controller* cntl,
+                           const brpc::SerializedRequest* request,
+                           brpc::SerializedResponse* response,
+                           ::google::protobuf::Closure* done) override {
+        // This object helps you to call done->Run() in RAII style. If you need
+        // to process the request asynchronously, pass done_guard.release().
+        brpc::ClosureGuard done_guard(done);
+
+        // A Channel represents a communication line to a Server. Notice that
+        // Channel is thread-safe and can be shared by all threads in your program.
+        brpc::Channel channel;
+
+        // Initialize the channel, NULL means using default options.
+        brpc::ChannelOptions options;
+        options.protocol = brpc::PROTOCOL_BAIDU_STD;
+        options.connection_type = FLAGS_connection_type;
+        options.timeout_ms = FLAGS_timeout_ms/*milliseconds*/;
+        options.max_retry = FLAGS_max_retry;
+        if (channel.Init(FLAGS_server_address.c_str(),
+            FLAGS_load_balancer.c_str(), &options) != 0) {
+            LOG(ERROR) << "Fail to initialize channel";
+            (*cntl->response_user_fields())["x-bd-proxy-error-code"] =
+                butil::IntToString(brpc::EINTERNAL);
+            (*cntl->response_user_fields())["x-bd-proxy-error-text"] =
+                "Fail to initialize channel";
+            return;
+        }
+
+        LOG(INFO) << "Received request[log_id=" << cntl->log_id()
+                  << "] from " << cntl->remote_side()
+                  << " to " << cntl->local_side()
+                  << ", serialized request size=" << request->serialized_data().size()
+                  << ", request compress type=" << cntl->request_compress_type()
+                  << " (attached=" << cntl->request_attachment() << ")";
+
+        brpc::Controller call_cntl;
+        call_cntl.set_log_id(cntl->log_id());
+        call_cntl.request_attachment().swap(cntl->request_attachment());
+        call_cntl.set_request_compress_type(cntl->request_compress_type());
+        call_cntl.reset_sampled_request(cntl->release_sampled_request());
+        // It is ok to use request and response for sync rpc.
+        channel.CallMethod(NULL, &call_cntl, request, response, NULL);
+        (*cntl->response_user_fields())["x-bd-proxy-error-code"] =
+            butil::IntToString(call_cntl.ErrorCode());
+        if (call_cntl.Failed()) {
+            (*cntl->response_user_fields())["x-bd-proxy-error-text"] =
+                call_cntl.ErrorText();
+            LOG(ERROR) << "Fail to call service=" << call_cntl.sampled_request()->meta.service_name()
+                       << ", method=" << call_cntl.sampled_request()->meta.method_name()
+                       << ", error_code=" << call_cntl.ErrorCode()
+                       << ", error_text=" << call_cntl.ErrorCode();
+            return;
+        } else {
+            LOG(INFO) << "Received response from " << call_cntl.remote_side()
+                      << " to " << call_cntl.local_side()
+                      << ", serialized response size=" << response->serialized_data().size()
+                      << ", response compress type=" << call_cntl.response_compress_type()
+                      << ", attached=" << call_cntl.response_attachment()
+                      << ", latency=" << call_cntl.latency_us() << "us";
+        }
+        cntl->response_attachment().swap(call_cntl.response_attachment());
+        cntl->set_response_compress_type(call_cntl.response_compress_type());
+    }
+};
+}  // namespace example
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    // Generally you only need one Server.
+    brpc::Server server;
+
+    butil::EndPoint point;
+    if (!FLAGS_listen_addr.empty()) {
+        if (butil::str2endpoint(FLAGS_listen_addr.c_str(), &point) < 0) {
+            LOG(ERROR) << "Invalid listen address:" << FLAGS_listen_addr;
+            return -1;
+        }
+    } else {
+        point = butil::EndPoint(butil::IP_ANY, FLAGS_port);
+    }
+    // Start the server.
+    brpc::ServerOptions options;
+    // Add the baidu master service into server.
+    // Notice new operator, because server will delete it in dtor of Server.
+    options.baidu_master_service = new example::BaiduMasterServiceImpl();
+    options.idle_timeout_sec = FLAGS_idle_timeout_s;
+    if (server.Start(point, &options) != 0) {
+        LOG(ERROR) << "Fail to start EchoServer";
+        return -1;
+    }
+
+    // Wait until Ctrl-C is pressed, then Stop() and Join() the server.
+    server.RunUntilAskedToQuit();
+    return 0;
+}

example/baidu_proxy_and_generic_call/server.cpp

@@ -0,0 +1,118 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A server to receive EchoRequest and send back EchoResponse.
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <brpc/server.h>
+#include <json2pb/pb_to_json.h>
+#include "echo.pb.h"
+
+DEFINE_int32(compress_type, 2, "The compress type of response");
+DEFINE_bool(echo_attachment, true, "Echo attachment as well");
+DEFINE_int32(port, 8001, "TCP Port of this server");
+DEFINE_string(listen_addr, "", "Server listen address, may be IPV4/IPV6/UDS."
+            " If this is set, the flag port will be ignored");
+DEFINE_int32(idle_timeout_s, -1, "Connection will be closed if there is no "
+             "read/write operations during the last `idle_timeout_s'");
+
+// Your implementation of example::EchoService
+// Notice that implementing brpc::Describable grants the ability to put
+// additional information in /status.
+namespace example {
+class EchoServiceImpl : public EchoService {
+public:
+    EchoServiceImpl() = default;
+    ~EchoServiceImpl() override = default;
+    void Echo(google::protobuf::RpcController* cntl_base,
+              const EchoRequest* request,
+              EchoResponse* response,
+              google::protobuf::Closure* done) override {
+        // This object helps you to call done->Run() in RAII style. If you need
+        // to process the request asynchronously, pass done_guard.release().
+        brpc::ClosureGuard done_guard(done);
+
+        auto cntl = static_cast<brpc::Controller*>(cntl_base);
+
+        // The purpose of following logs is to help you to understand
+        // how clients interact with servers more intuitively. You should 
+        // remove these logs in performance-sensitive servers.
+        LOG(INFO) << "Received request[log_id=" << cntl->log_id() 
+                  << "] from " << cntl->remote_side() 
+                  << " to " << cntl->local_side()
+                  << ": " << request->message()
+                  << ", request compress type=" << cntl->request_compress_type()
+                  << ", attached=" << cntl->request_attachment();
+
+        // Fill response.
+        response->set_message(request->message());
+        cntl->set_response_compress_type((brpc::CompressType)FLAGS_compress_type);
+
+        // You can compress the response by setting Controller, but be aware
+        // that compression may be costly, evaluate before turning on.
+        // cntl->set_response_compress_type(brpc::COMPRESS_TYPE_GZIP);
+
+        if (FLAGS_echo_attachment) {
+            // Set attachment which is wired to network directly instead of
+            // being serialized into protobuf messages.
+            cntl->response_attachment().append(cntl->request_attachment());
+        }
+    }
+};
+}  // namespace example
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    // Generally you only need one Server.
+    brpc::Server server;
+
+    // Instance of your service.
+    example::EchoServiceImpl echo_service_impl;
+
+    // Add the service into server. Notice the second parameter, because the
+    // service is put on stack, we don't want server to delete it, otherwise
+    // use brpc::SERVER_OWNS_SERVICE.
+    if (server.AddService(&echo_service_impl, 
+                          brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
+        LOG(ERROR) << "Fail to add service";
+        return -1;
+    }
+
+    butil::EndPoint point;
+    if (!FLAGS_listen_addr.empty()) {
+        if (butil::str2endpoint(FLAGS_listen_addr.c_str(), &point) < 0) {
+            LOG(ERROR) << "Invalid listen address:" << FLAGS_listen_addr;
+            return -1;
+        }
+    } else {
+        point = butil::EndPoint(butil::IP_ANY, FLAGS_port);
+    }
+    // Start the server.
+    brpc::ServerOptions options;
+    options.idle_timeout_sec = FLAGS_idle_timeout_s;
+    if (server.Start(point, &options) != 0) {
+        LOG(ERROR) << "Fail to start EchoServer";
+        return -1;
+    }
+
+    // Wait until Ctrl-C is pressed, then Stop() and Join() the server.
+    server.RunUntilAskedToQuit();
+    return 0;
+}

example/bthread_tag_echo_c++/CMakeLists.txt

@@ -0,0 +1,115 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.16...3.28)
+project(bthread_tag_echo_c++ C CXX)
+
+include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake)
+
+option(LINK_SO "Whether examples are linked dynamically" OFF)
+
+execute_process(
+    COMMAND bash -c "find ${PROJECT_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'"
+    OUTPUT_VARIABLE OUTPUT_PATH
+)
+
+if(OUTPUT_PATH)
+    list(PREPEND CMAKE_PREFIX_PATH ${OUTPUT_PATH})
+endif()
+
+find_package(Threads REQUIRED)
+find_package(Protobuf REQUIRED)
+protobuf_generate_cpp(PROTO_SRC PROTO_HEADER echo.proto)
+
+# Search for libthrift* by best effort. If it is not found and brpc is
+# compiled with thrift protocol enabled, a link error would be reported.
+find_library(THRIFT_LIB NAMES thrift)
+if (NOT THRIFT_LIB)
+    set(THRIFT_LIB "")
+endif()
+
+find_path(GPERFTOOLS_INCLUDE_DIR NAMES gperftools/heap-profiler.h)
+find_library(GPERFTOOLS_LIBRARIES NAMES tcmalloc_and_profiler)
+
+find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h)
+if(LINK_SO)
+    find_library(BRPC_LIB NAMES brpc)
+else()
+    find_library(BRPC_LIB NAMES libbrpc.a brpc)
+endif()
+if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB))
+    message(FATAL_ERROR "Fail to find brpc")
+endif()
+
+find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h)
+find_library(GFLAGS_LIBRARY NAMES gflags libgflags)
+if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY))
+    message(FATAL_ERROR "Fail to find gflags")
+endif()
+
+set(BRPC_EXAMPLE_ENABLE_CPU_PROFILER ON)
+
+find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h)
+find_library(LEVELDB_LIB NAMES leveldb)
+if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB))
+    message(FATAL_ERROR "Fail to find leveldb")
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT OPENSSL_ROOT_DIR)
+    set(OPENSSL_ROOT_DIR
+        "/usr/local/opt/openssl"    # Homebrew installed OpenSSL
+        )
+endif()
+
+find_package(OpenSSL REQUIRED)
+
+set(DYNAMIC_LIB
+    Threads::Threads
+    ${GFLAGS_LIBRARY}
+    ${PROTOBUF_LIBRARIES}
+    ${LEVELDB_LIB}
+    ${OPENSSL_CRYPTO_LIBRARY}
+    ${OPENSSL_SSL_LIBRARY}
+    ${THRIFT_LIB}
+    dl
+    )
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    set(DYNAMIC_LIB ${DYNAMIC_LIB}
+        pthread
+        "-framework CoreFoundation"
+        "-framework CoreGraphics"
+        "-framework CoreData"
+        "-framework CoreText"
+        "-framework Security"
+        "-framework Foundation"
+        "-Wl,-U,_MallocExtension_ReleaseFreeMemory"
+        "-Wl,-U,_ProfilerStart"
+        "-Wl,-U,_ProfilerStop"
+        "-Wl,-U,__Z13GetStackTracePPvii"
+        "-Wl,-U,_mallctl"
+        "-Wl,-U,_malloc_stats_print"
+    )
+endif()
+
+add_executable(echo_client client.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(echo_client)
+add_executable(echo_server server.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(echo_server)
+
+target_link_libraries(echo_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB} ${GPERFTOOLS_LIBRARIES})
+target_link_libraries(echo_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB} ${GPERFTOOLS_LIBRARIES})

example/bthread_tag_echo_c++/client.cpp

@@ -0,0 +1,154 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A client sending requests to server by multiple threads.
+
+#include <gflags/gflags.h>
+#include <bthread/bthread.h>
+#include <butil/logging.h>
+#include <brpc/server.h>
+#include <brpc/channel.h>
+#include "echo.pb.h"
+#include <bvar/bvar.h>
+
+DEFINE_int32(thread_num, 50, "Number of threads to send requests");
+DEFINE_bool(use_bthread, false, "Use bthread to send requests");
+DEFINE_int32(attachment_size, 0, "Carry so many byte attachment along with requests");
+DEFINE_int32(request_size, 16, "Bytes of each request");
+DEFINE_string(protocol, "baidu_std", "Protocol type. Defined in src/brpc/options.proto");
+DEFINE_string(connection_type, "", "Connection type. Available values: single, pooled, short");
+DEFINE_string(server, "0.0.0.0:8002", "IP Address of server");
+DEFINE_string(load_balancer, "", "The algorithm for load balancing");
+DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)");
+DEFINE_bool(dont_fail, false, "Print fatal when some call failed");
+DEFINE_int32(dummy_port, -1, "Launch dummy server at this port");
+
+std::string g_request;
+std::string g_attachment;
+
+bvar::LatencyRecorder g_latency_recorder("client");
+bvar::Adder<int> g_error_count("client_error_count");
+
+static void* sender(void* arg) {
+    // Normally, you should not call a Channel directly, but instead construct
+    // a stub Service wrapping it. stub can be shared by all threads as well.
+    example::EchoService_Stub stub(static_cast<google::protobuf::RpcChannel*>(arg));
+
+    int log_id = 0;
+    while (!brpc::IsAskedToQuit()) {
+        // We will receive response synchronously, safe to put variables
+        // on stack.
+        example::EchoRequest request;
+        example::EchoResponse response;
+        brpc::Controller cntl;
+
+        request.set_message(g_request);
+        cntl.set_log_id(log_id++);  // set by user
+        // Set attachment which is wired to network directly instead of
+        // being serialized into protobuf messages.
+        cntl.request_attachment().append(g_attachment);
+
+        // Because `done'(last parameter) is NULL, this function waits until
+        // the response comes back or error occurs(including timedout).
+        stub.Echo(&cntl, &request, &response, NULL);
+        if (!cntl.Failed()) {
+            g_latency_recorder << cntl.latency_us();
+        } else {
+            g_error_count << 1;
+            CHECK(brpc::IsAskedToQuit() || !FLAGS_dont_fail)
+                << "error=" << cntl.ErrorText() << " latency=" << cntl.latency_us();
+            // We can't connect to the server, sleep a while. Notice that this
+            // is a specific sleeping to prevent this thread from spinning too
+            // fast. You should continue the business logic in a production
+            // server rather than sleeping.
+            bthread_usleep(50000);
+        }
+    }
+    return NULL;
+}
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    // A Channel represents a communication line to a Server. Notice that
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::Channel channel;
+
+    // Initialize the channel, NULL means using default options.
+    brpc::ChannelOptions options;
+    options.protocol = FLAGS_protocol;
+    options.connection_type = FLAGS_connection_type;
+    options.connect_timeout_ms = std::min(FLAGS_timeout_ms / 2, 100);
+    options.timeout_ms = FLAGS_timeout_ms;
+    options.max_retry = FLAGS_max_retry;
+    if (channel.Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), &options) != 0) {
+        LOG(ERROR) << "Fail to initialize channel";
+        return -1;
+    }
+
+    if (FLAGS_attachment_size > 0) {
+        g_attachment.resize(FLAGS_attachment_size, 'a');
+    }
+    if (FLAGS_request_size <= 0) {
+        LOG(ERROR) << "Bad request_size=" << FLAGS_request_size;
+        return -1;
+    }
+    g_request.resize(FLAGS_request_size, 'r');
+
+    if (FLAGS_dummy_port >= 0) {
+        brpc::StartDummyServerAt(FLAGS_dummy_port);
+    }
+
+    std::vector<bthread_t> bids;
+    std::vector<pthread_t> pids;
+    if (!FLAGS_use_bthread) {
+        pids.resize(FLAGS_thread_num);
+        for (int i = 0; i < FLAGS_thread_num; ++i) {
+            if (pthread_create(&pids[i], NULL, sender, &channel) != 0) {
+                LOG(ERROR) << "Fail to create pthread";
+                return -1;
+            }
+        }
+    } else {
+        bids.resize(FLAGS_thread_num);
+        for (int i = 0; i < FLAGS_thread_num; ++i) {
+            if (bthread_start_background(&bids[i], nullptr, sender, &channel) != 0) {
+                LOG(ERROR) << "Fail to create bthread";
+                return -1;
+            }
+        }
+    }
+
+    while (!brpc::IsAskedToQuit()) {
+        sleep(1);
+        LOG(INFO) << "Sending EchoRequest at qps=" << g_latency_recorder.qps(1)
+                  << " latency=" << g_latency_recorder.latency(1);
+    }
+
+    LOG(INFO) << "EchoClient is going to quit";
+    for (int i = 0; i < FLAGS_thread_num; ++i) {
+        if (!FLAGS_use_bthread) {
+            pthread_join(pids[i], NULL);
+        } else {
+            bthread_join(bids[i], NULL);
+        }
+    }
+
+    return 0;
+}

example/bthread_tag_echo_c++/echo.proto

@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+syntax="proto2";
+option cc_generic_services = true;
+
+package example;
+
+message EchoRequest {
+      required string message = 1;
+};
+
+message EchoResponse {
+      required string message = 1;
+};
+
+service EchoService {
+      rpc Echo(EchoRequest) returns (EchoResponse);
+};

example/bthread_tag_echo_c++/server.cpp

@@ -0,0 +1,153 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A server to receive EchoRequest and send back EchoResponse.
+
+#include <brpc/server.h>
+#include <bthread/unstable.h>
+#include <butil/logging.h>
+#include <gflags/gflags.h>
+#include "echo.pb.h"
+
+DEFINE_bool(echo_attachment, true, "Echo attachment as well");
+DEFINE_int32(port1, 8002, "TCP Port of this server");
+DEFINE_int32(port2, 8003, "TCP Port of this server");
+DEFINE_int32(tag1, 0, "Server1 tag");
+DEFINE_int32(tag2, 1, "Server2 tag");
+DEFINE_int32(tag3, 2, "Background task tag");
+DEFINE_int32(num_threads1, 6, "Thread number of server1");
+DEFINE_int32(num_threads2, 16, "Thread number of server2");
+DEFINE_int32(idle_timeout_s, -1,
+             "Connection will be closed if there is no "
+             "read/write operations during the last `idle_timeout_s'");
+DEFINE_int32(max_concurrency, 0, "Limit of request processing in parallel");
+DEFINE_int32(internal_port1, -1, "Only allow builtin services at this port");
+DEFINE_int32(internal_port2, -1, "Only allow builtin services at this port");
+
+namespace example {
+// Your implementation of EchoService
+class EchoServiceImpl : public EchoService {
+public:
+    EchoServiceImpl() {}
+    ~EchoServiceImpl() {}
+    void Echo(google::protobuf::RpcController* cntl_base, const EchoRequest* request,
+              EchoResponse* response, google::protobuf::Closure* done) {
+        brpc::ClosureGuard done_guard(done);
+        brpc::Controller* cntl = static_cast<brpc::Controller*>(cntl_base);
+
+        // Echo request and its attachment
+        response->set_message(request->message());
+        if (FLAGS_echo_attachment) {
+            cntl->response_attachment().append(cntl->request_attachment());
+        }
+    }
+};
+}  // namespace example
+
+DEFINE_bool(h, false, "print help information");
+
+static void my_tagged_worker_start_fn(bthread_tag_t tag) {
+    LOG(INFO) << "run tagged worker start function tag=" << tag;
+}
+
+static void* my_background_task(void*) {
+    while (true) {
+        LOG(INFO) << "run background task tag=" << bthread_self_tag();
+        bthread_usleep(1000000UL);
+    }
+    return nullptr;
+}
+
+int main(int argc, char* argv[]) {
+    std::string help_str = "dummy help infomation";
+    GFLAGS_NAMESPACE::SetUsageMessage(help_str);
+
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    if (FLAGS_h) {
+        fprintf(stderr, "%s\n%s\n%s", help_str.c_str(), help_str.c_str(), help_str.c_str());
+        return 0;
+    }
+
+    // Set tagged worker function
+    bthread_set_tagged_worker_startfn(my_tagged_worker_start_fn);
+
+    // Generally you only need one Server.
+    brpc::Server server1;
+
+    // Instance of your service.
+    example::EchoServiceImpl echo_service_impl1;
+
+    // Add the service into server. Notice the second parameter, because the
+    // service is put on stack, we don't want server to delete it, otherwise
+    // use brpc::SERVER_OWNS_SERVICE.
+    if (server1.AddService(&echo_service_impl1, brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
+        LOG(ERROR) << "Fail to add service";
+        return -1;
+    }
+
+    // Start the server.
+    brpc::ServerOptions options1;
+    options1.idle_timeout_sec = FLAGS_idle_timeout_s;
+    options1.max_concurrency = FLAGS_max_concurrency;
+    options1.internal_port = FLAGS_internal_port1;
+    options1.bthread_tag = FLAGS_tag1;
+    options1.num_threads = FLAGS_num_threads1;
+    if (server1.Start(FLAGS_port1, &options1) != 0) {
+        LOG(ERROR) << "Fail to start EchoServer";
+        return -1;
+    }
+
+    // Generally you only need one Server.
+    brpc::Server server2;
+
+    // Instance of your service.
+    example::EchoServiceImpl echo_service_impl2;
+
+    // Add the service into server. Notice the second parameter, because the
+    // service is put on stack, we don't want server to delete it, otherwise
+    // use brpc::SERVER_OWNS_SERVICE.
+    if (server2.AddService(&echo_service_impl2, brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
+        LOG(ERROR) << "Fail to add service";
+        return -1;
+    }
+
+    // Start the server.
+    brpc::ServerOptions options2;
+    options2.idle_timeout_sec = FLAGS_idle_timeout_s;
+    options2.max_concurrency = FLAGS_max_concurrency;
+    options2.internal_port = FLAGS_internal_port2;
+    options2.bthread_tag = FLAGS_tag2;
+    options2.num_threads = FLAGS_num_threads2;
+    if (server2.Start(FLAGS_port2, &options2) != 0) {
+        LOG(ERROR) << "Fail to start EchoServer";
+        return -1;
+    }
+
+    // Start backgroup task
+    bthread_t tid;
+    bthread_attr_t attr = BTHREAD_ATTR_NORMAL;
+    attr.tag = FLAGS_tag3;
+    bthread_start_background(&tid, &attr, my_background_task, nullptr);
+
+    // Wait until Ctrl-C is pressed, then Stop() and Join() the server.
+    server1.RunUntilAskedToQuit();
+    server2.RunUntilAskedToQuit();
+
+    return 0;
+}

example/build_with_bazel/BUILD.bazel

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Thie empty BUILD.bazel file is required to make Bazel treat
+# this directory as a package.
+
+
+cc_binary(
+    name = "test",
+    srcs = ["test.cc"],
+    deps = [
+        "@apache_brpc//:brpc",
+        "@apache_brpc//:bthread",
+        "@apache_brpc//:bvar",
+        "@apache_brpc//:butil",
+    ],
+)

example/build_with_bazel/brpc_workspace.bzl

@@ -0,0 +1,94 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+
+BAZEL_SKYLIB_VERSION = "1.1.1"  # 2021-09-27T17:33:49Z
+
+BAZEL_SKYLIB_SHA256 = "c6966ec828da198c5d9adbaa94c05e3a1c7f21bd012a0b29ba8ddbccb2c93b0d"
+
+def brpc_workspace():
+    http_archive(
+        name = "bazel_skylib",
+        sha256 = BAZEL_SKYLIB_SHA256,
+        urls = [
+            "https://github.com/bazelbuild/bazel-skylib/releases/download/{version}/bazel-skylib-{version}.tar.gz".format(version = BAZEL_SKYLIB_VERSION),
+            "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/{version}/bazel-skylib-{version}.tar.gz".format(version = BAZEL_SKYLIB_VERSION),
+        ],
+    )
+
+    http_archive(
+        name = "com_google_protobuf",  # 2021-10-29T00:04:02Z
+        build_file = "//:protobuf.BUILD",
+        patch_cmds = [
+            "sed -i protobuf.bzl -re '4,4d;417,508d'",
+        ],
+        patch_cmds_win = [
+            """$content = Get-Content 'protobuf.bzl' | Where-Object {
+            -not ($_.ReadCount -ne 4) -and
+            -not ($_.ReadCount -ge 418 -and $_.ReadCount -le 509)
+        }
+        Set-Content protobuf.bzl -Value $content -Encoding UTF8
+        """,
+        ],
+        sha256 = "87407cd28e7a9c95d9f61a098a53cf031109d451a7763e7dd1253abf8b4df422",
+        strip_prefix = "protobuf-3.19.1",
+        urls = ["https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.19.1.tar.gz"],
+    )
+
+
+    http_archive(
+        name = "com_github_google_leveldb",
+        build_file = "//:leveldb.BUILD",
+        strip_prefix = "leveldb-a53934a3ae1244679f812d998a4f16f2c7f309a6",
+        url = "https://github.com/google/leveldb/archive/a53934a3ae1244679f812d998a4f16f2c7f309a6.tar.gz"
+    )
+
+
+
+    http_archive(
+        name = "com_github_madler_zlib",  # 2017-01-15T17:57:23Z
+        build_file = "//:zlib.BUILD",
+        sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
+        strip_prefix = "zlib-1.2.11",
+        urls = [
+            "https://downloads.sourceforge.net/project/libpng/zlib/1.2.11/zlib-1.2.11.tar.gz",
+            "https://zlib.net/fossils/zlib-1.2.11.tar.gz",
+        ],
+    )
+
+    native.new_local_repository(
+        name = "openssl",
+        path = "/usr",
+        build_file = "//:openssl.BUILD",
+    )
+
+    http_archive(
+        name = "com_github_gflags_gflags",
+        strip_prefix = "gflags-46f73f88b18aee341538c0dfc22b1710a6abedef",
+        url = "https://github.com/gflags/gflags/archive/46f73f88b18aee341538c0dfc22b1710a6abedef.tar.gz",
+    )
+
+    http_archive(
+        name = "apache_brpc",
+        strip_prefix = "brpc-1.3.0",
+        url = "https://github.com/apache/brpc/archive/refs/tags/1.3.0.tar.gz"
+    )
+

example/build_with_bazel/leveldb.BUILD

@@ -0,0 +1,97 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//visibility:public"])
+
+
+config_setting(
+    name = "darwin",
+    values = {"cpu": "darwin"},
+    visibility = ["//visibility:public"],
+)
+
+SOURCES = ["db/builder.cc",
+         "db/c.cc",
+         "db/dbformat.cc",
+         "db/db_impl.cc",
+         "db/db_iter.cc",
+         "db/dumpfile.cc",
+         "db/filename.cc",
+         "db/log_reader.cc",
+         "db/log_writer.cc",
+         "db/memtable.cc",
+         "db/repair.cc",
+         "db/table_cache.cc",
+         "db/version_edit.cc",
+         "db/version_set.cc",
+         "db/write_batch.cc",
+         "table/block_builder.cc",
+         "table/block.cc",
+         "table/filter_block.cc",
+         "table/format.cc",
+         "table/iterator.cc",
+         "table/merger.cc",
+         "table/table_builder.cc",
+         "table/table.cc",
+         "table/two_level_iterator.cc",
+         "util/arena.cc",
+         "util/bloom.cc",
+         "util/cache.cc",
+         "util/coding.cc",
+         "util/comparator.cc",
+         "util/crc32c.cc",
+         "util/env.cc",
+         "util/env_posix.cc",
+         "util/filter_policy.cc",
+         "util/hash.cc",
+         "util/histogram.cc",
+         "util/logging.cc",
+         "util/options.cc",
+         "util/status.cc",
+         "port/port_posix.cc",
+         "port/port_posix_sse.cc",
+         "helpers/memenv/memenv.cc",
+    ]
+
+cc_library(
+    name = "leveldb",
+    srcs = SOURCES,
+    hdrs = glob([ 
+        "helpers/memenv/*.h",
+        "util/*.h",
+        "port/*.h",
+        "port/win/*.h",
+        "table/*.h",
+        "db/*.h",
+        "include/leveldb/*.h"
+    ],
+    exclude = [
+            "**/*test.*",
+    ]),
+    includes = [
+        "include/",
+    ],
+    copts = [
+        "-fno-builtin-memcmp",
+        "-DLEVELDB_PLATFORM_POSIX=1",
+        "-DLEVELDB_ATOMIC_PRESENT",
+    ],
+    defines = [
+        "LEVELDB_PLATFORM_POSIX",
+    ] + select({
+        ":darwin": ["OS_MACOSX"],
+        "//conditions:default": [],
+    }),
+)

example/build_with_bazel/openssl.BUILD

@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(
+    default_visibility=["//visibility:public"]
+)
+
+config_setting(
+    name = "macos",
+    values = {
+        "cpu": "darwin",
+    },
+    visibility = ["//visibility:private"],
+)
+
+cc_library(
+    name = "crypto",
+    srcs = select({
+        ":macos": ["lib/libcrypto.dylib"],
+        "//conditions:default": []
+    }),
+    linkopts = select({
+        ":macos" : [],
+        "//conditions:default": ["-lcrypto"],
+    }),
+)
+
+cc_library(
+    name = "ssl",
+    hdrs = select({
+        ":macos": glob(["include/openssl/*.h"]),
+        "//conditions:default": []
+    }),
+    srcs = select ({
+        ":macos": ["lib/libssl.dylib"],
+        "//conditions:default": []
+    }),
+    includes = ["include"],
+    linkopts = select({
+        ":macos" : [],
+        "//conditions:default": ["-lssl"],
+    }),
+    deps = [":crypto"]
+)

example/build_with_bazel/protobuf.BUILD

@@ -0,0 +1,498 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Copyright 2008 Google Inc.  All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Copied from https://github.com/protocolbuffers/protobuf/blob/v3.19.1/BUILD
+#
+# Modifications:
+# 1. Remove all non-cxx rules.
+# 2. Remove android support.
+# 3. zlib use @com_github_madler_zlib//:zlib
+
+# Bazel (https://bazel.build/) BUILD file for Protobuf.
+
+load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", native_cc_proto_library = "cc_proto_library")
+load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain", "proto_library")
+load(":compiler_config_setting.bzl", "create_compiler_config_setting")
+load(
+    ":protobuf.bzl",
+    "adapt_proto_library",
+)
+
+licenses(["notice"])
+
+exports_files(["LICENSE"])
+
+################################################################################
+# build configuration
+################################################################################
+
+################################################################################
+# ZLIB configuration
+################################################################################
+
+ZLIB_DEPS = ["@com_github_madler_zlib//:zlib"]
+
+################################################################################
+# Protobuf Runtime Library
+################################################################################
+
+MSVC_COPTS = [
+    "/wd4018",  # -Wno-sign-compare
+    "/wd4065",  # switch statement contains 'default' but no 'case' labels
+    "/wd4146",  # unary minus operator applied to unsigned type, result still unsigned
+    "/wd4244",  # 'conversion' conversion from 'type1' to 'type2', possible loss of data
+    "/wd4251",  # 'identifier' : class 'type' needs to have dll-interface to be used by clients of class 'type2'
+    "/wd4267",  # 'var' : conversion from 'size_t' to 'type', possible loss of data
+    "/wd4305",  # 'identifier' : truncation from 'type1' to 'type2'
+    "/wd4307",  # 'operator' : integral constant overflow
+    "/wd4309",  # 'conversion' : truncation of constant value
+    "/wd4334",  # 'operator' : result of 32-bit shift implicitly converted to 64 bits (was 64-bit shift intended?)
+    "/wd4355",  # 'this' : used in base member initializer list
+    "/wd4506",  # no definition for inline function 'function'
+    "/wd4800",  # 'type' : forcing value to bool 'true' or 'false' (performance warning)
+    "/wd4996",  # The compiler encountered a deprecated declaration.
+]
+
+COPTS = select({
+    ":msvc": MSVC_COPTS,
+    "//conditions:default": [
+        "-DHAVE_ZLIB",
+        "-Wmissing-field-initializers",
+        "-Woverloaded-virtual",
+        "-Wno-sign-compare",
+    ],
+})
+
+create_compiler_config_setting(
+    name = "msvc",
+    value = "msvc-cl",
+    visibility = [
+        # Public, but Protobuf only visibility.
+        "//:__subpackages__",
+    ],
+)
+
+# Android and MSVC builds do not need to link in a separate pthread library.
+LINK_OPTS = select({
+    ":msvc": [
+        # Suppress linker warnings about files with no symbols defined.
+        "-ignore:4221",
+    ],
+    "//conditions:default": [
+        "-lpthread",
+        "-lm",
+    ],
+})
+
+cc_library(
+    name = "protobuf_lite",
+    srcs = [
+        # AUTOGEN(protobuf_lite_srcs)
+        "src/google/protobuf/any_lite.cc",
+        "src/google/protobuf/arena.cc",
+        "src/google/protobuf/arenastring.cc",
+        "src/google/protobuf/extension_set.cc",
+        "src/google/protobuf/generated_enum_util.cc",
+        "src/google/protobuf/generated_message_table_driven_lite.cc",
+        "src/google/protobuf/generated_message_tctable_lite.cc",
+        "src/google/protobuf/generated_message_util.cc",
+        "src/google/protobuf/implicit_weak_message.cc",
+        "src/google/protobuf/inlined_string_field.cc",
+        "src/google/protobuf/io/coded_stream.cc",
+        "src/google/protobuf/io/io_win32.cc",
+        "src/google/protobuf/io/strtod.cc",
+        "src/google/protobuf/io/zero_copy_stream.cc",
+        "src/google/protobuf/io/zero_copy_stream_impl.cc",
+        "src/google/protobuf/io/zero_copy_stream_impl_lite.cc",
+        "src/google/protobuf/map.cc",
+        "src/google/protobuf/message_lite.cc",
+        "src/google/protobuf/parse_context.cc",
+        "src/google/protobuf/repeated_field.cc",
+        "src/google/protobuf/repeated_ptr_field.cc",
+        "src/google/protobuf/stubs/bytestream.cc",
+        "src/google/protobuf/stubs/common.cc",
+        "src/google/protobuf/stubs/int128.cc",
+        "src/google/protobuf/stubs/status.cc",
+        "src/google/protobuf/stubs/statusor.cc",
+        "src/google/protobuf/stubs/stringpiece.cc",
+        "src/google/protobuf/stubs/stringprintf.cc",
+        "src/google/protobuf/stubs/structurally_valid.cc",
+        "src/google/protobuf/stubs/strutil.cc",
+        "src/google/protobuf/stubs/time.cc",
+        "src/google/protobuf/wire_format_lite.cc",
+    ],
+    hdrs = glob([
+        "src/google/protobuf/**/*.h",
+        "src/google/protobuf/**/*.inc",
+    ]),
+    copts = COPTS,
+    includes = ["src/"],
+    linkopts = LINK_OPTS,
+    visibility = ["//visibility:public"],
+)
+
+PROTOBUF_DEPS = select({
+    ":msvc": [],
+    "//conditions:default": ZLIB_DEPS,
+})
+
+cc_library(
+    name = "protobuf",
+    srcs = [
+        # AUTOGEN(protobuf_srcs)
+        "src/google/protobuf/any.cc",
+        "src/google/protobuf/any.pb.cc",
+        "src/google/protobuf/api.pb.cc",
+        "src/google/protobuf/compiler/importer.cc",
+        "src/google/protobuf/compiler/parser.cc",
+        "src/google/protobuf/descriptor.cc",
+        "src/google/protobuf/descriptor.pb.cc",
+        "src/google/protobuf/descriptor_database.cc",
+        "src/google/protobuf/duration.pb.cc",
+        "src/google/protobuf/dynamic_message.cc",
+        "src/google/protobuf/empty.pb.cc",
+        "src/google/protobuf/extension_set_heavy.cc",
+        "src/google/protobuf/field_mask.pb.cc",
+        "src/google/protobuf/generated_message_bases.cc",
+        "src/google/protobuf/generated_message_reflection.cc",
+        "src/google/protobuf/generated_message_table_driven.cc",
+        "src/google/protobuf/generated_message_tctable_full.cc",
+        "src/google/protobuf/io/gzip_stream.cc",
+        "src/google/protobuf/io/printer.cc",
+        "src/google/protobuf/io/tokenizer.cc",
+        "src/google/protobuf/map_field.cc",
+        "src/google/protobuf/message.cc",
+        "src/google/protobuf/reflection_ops.cc",
+        "src/google/protobuf/service.cc",
+        "src/google/protobuf/source_context.pb.cc",
+        "src/google/protobuf/struct.pb.cc",
+        "src/google/protobuf/stubs/substitute.cc",
+        "src/google/protobuf/text_format.cc",
+        "src/google/protobuf/timestamp.pb.cc",
+        "src/google/protobuf/type.pb.cc",
+        "src/google/protobuf/unknown_field_set.cc",
+        "src/google/protobuf/util/delimited_message_util.cc",
+        "src/google/protobuf/util/field_comparator.cc",
+        "src/google/protobuf/util/field_mask_util.cc",
+        "src/google/protobuf/util/internal/datapiece.cc",
+        "src/google/protobuf/util/internal/default_value_objectwriter.cc",
+        "src/google/protobuf/util/internal/error_listener.cc",
+        "src/google/protobuf/util/internal/field_mask_utility.cc",
+        "src/google/protobuf/util/internal/json_escaping.cc",
+        "src/google/protobuf/util/internal/json_objectwriter.cc",
+        "src/google/protobuf/util/internal/json_stream_parser.cc",
+        "src/google/protobuf/util/internal/object_writer.cc",
+        "src/google/protobuf/util/internal/proto_writer.cc",
+        "src/google/protobuf/util/internal/protostream_objectsource.cc",
+        "src/google/protobuf/util/internal/protostream_objectwriter.cc",
+        "src/google/protobuf/util/internal/type_info.cc",
+        "src/google/protobuf/util/internal/utility.cc",
+        "src/google/protobuf/util/json_util.cc",
+        "src/google/protobuf/util/message_differencer.cc",
+        "src/google/protobuf/util/time_util.cc",
+        "src/google/protobuf/util/type_resolver_util.cc",
+        "src/google/protobuf/wire_format.cc",
+        "src/google/protobuf/wrappers.pb.cc",
+    ],
+    hdrs = glob([
+        "src/**/*.h",
+        "src/**/*.inc",
+    ]),
+    copts = COPTS,
+    includes = ["src/"],
+    linkopts = LINK_OPTS,
+    visibility = ["//visibility:public"],
+    deps = [":protobuf_lite"] + PROTOBUF_DEPS,
+)
+
+# This provides just the header files for use in projects that need to build
+# shared libraries for dynamic loading. This target is available until Bazel
+# adds native support for such use cases.
+# TODO(keveman): Remove this target once the support gets added to Bazel.
+cc_library(
+    name = "protobuf_headers",
+    hdrs = glob([
+        "src/**/*.h",
+        "src/**/*.inc",
+    ]),
+    includes = ["src/"],
+    visibility = ["//visibility:public"],
+)
+
+# Map of all well known protos.
+# name => (include path, imports)
+WELL_KNOWN_PROTO_MAP = {
+    "any": ("src/google/protobuf/any.proto", []),
+    "api": (
+        "src/google/protobuf/api.proto",
+        [
+            "source_context",
+            "type",
+        ],
+    ),
+    "compiler_plugin": (
+        "src/google/protobuf/compiler/plugin.proto",
+        ["descriptor"],
+    ),
+    "descriptor": ("src/google/protobuf/descriptor.proto", []),
+    "duration": ("src/google/protobuf/duration.proto", []),
+    "empty": ("src/google/protobuf/empty.proto", []),
+    "field_mask": ("src/google/protobuf/field_mask.proto", []),
+    "source_context": ("src/google/protobuf/source_context.proto", []),
+    "struct": ("src/google/protobuf/struct.proto", []),
+    "timestamp": ("src/google/protobuf/timestamp.proto", []),
+    "type": (
+        "src/google/protobuf/type.proto",
+        [
+            "any",
+            "source_context",
+        ],
+    ),
+    "wrappers": ("src/google/protobuf/wrappers.proto", []),
+}
+
+WELL_KNOWN_PROTOS = [value[0] for value in WELL_KNOWN_PROTO_MAP.values()]
+
+LITE_WELL_KNOWN_PROTO_MAP = {
+    "any": ("src/google/protobuf/any.proto", []),
+    "api": (
+        "src/google/protobuf/api.proto",
+        [
+            "source_context",
+            "type",
+        ],
+    ),
+    "duration": ("src/google/protobuf/duration.proto", []),
+    "empty": ("src/google/protobuf/empty.proto", []),
+    "field_mask": ("src/google/protobuf/field_mask.proto", []),
+    "source_context": ("src/google/protobuf/source_context.proto", []),
+    "struct": ("src/google/protobuf/struct.proto", []),
+    "timestamp": ("src/google/protobuf/timestamp.proto", []),
+    "type": (
+        "src/google/protobuf/type.proto",
+        [
+            "any",
+            "source_context",
+        ],
+    ),
+    "wrappers": ("src/google/protobuf/wrappers.proto", []),
+}
+
+LITE_WELL_KNOWN_PROTOS = [value[0] for value in LITE_WELL_KNOWN_PROTO_MAP.values()]
+
+filegroup(
+    name = "well_known_protos",
+    srcs = WELL_KNOWN_PROTOS,
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "lite_well_known_protos",
+    srcs = LITE_WELL_KNOWN_PROTOS,
+    visibility = ["//visibility:public"],
+)
+
+adapt_proto_library(
+    name = "cc_wkt_protos_genproto",
+    visibility = ["//visibility:public"],
+    deps = [proto + "_proto" for proto in WELL_KNOWN_PROTO_MAP.keys()],
+)
+
+cc_library(
+    name = "cc_wkt_protos",
+    deprecation = "Only for backward compatibility. Do not use.",
+    visibility = ["//visibility:public"],
+)
+
+################################################################################
+# Well Known Types Proto Library Rules
+#
+# These proto_library rules can be used with one of the language specific proto
+# library rules i.e. java_proto_library:
+#
+# java_proto_library(
+#   name = "any_java_proto",
+#   deps = ["@com_google_protobuf//:any_proto],
+# )
+################################################################################
+
+[proto_library(
+    name = proto[0] + "_proto",
+    srcs = [proto[1][0]],
+    strip_import_prefix = "src",
+    visibility = ["//visibility:public"],
+    deps = [dep + "_proto" for dep in proto[1][1]],
+) for proto in WELL_KNOWN_PROTO_MAP.items()]
+
+[native_cc_proto_library(
+    name = proto + "_cc_proto",
+    visibility = ["//visibility:private"],
+    deps = [proto + "_proto"],
+) for proto in WELL_KNOWN_PROTO_MAP.keys()]
+
+################################################################################
+# Protocol Buffers Compiler
+################################################################################
+
+cc_library(
+    name = "protoc_lib",
+    srcs = [
+        # AUTOGEN(protoc_lib_srcs)
+        "src/google/protobuf/compiler/code_generator.cc",
+        "src/google/protobuf/compiler/command_line_interface.cc",
+        "src/google/protobuf/compiler/cpp/cpp_enum.cc",
+        "src/google/protobuf/compiler/cpp/cpp_enum_field.cc",
+        "src/google/protobuf/compiler/cpp/cpp_extension.cc",
+        "src/google/protobuf/compiler/cpp/cpp_field.cc",
+        "src/google/protobuf/compiler/cpp/cpp_file.cc",
+        "src/google/protobuf/compiler/cpp/cpp_generator.cc",
+        "src/google/protobuf/compiler/cpp/cpp_helpers.cc",
+        "src/google/protobuf/compiler/cpp/cpp_map_field.cc",
+        "src/google/protobuf/compiler/cpp/cpp_message.cc",
+        "src/google/protobuf/compiler/cpp/cpp_message_field.cc",
+        "src/google/protobuf/compiler/cpp/cpp_padding_optimizer.cc",
+        "src/google/protobuf/compiler/cpp/cpp_parse_function_generator.cc",
+        "src/google/protobuf/compiler/cpp/cpp_primitive_field.cc",
+        "src/google/protobuf/compiler/cpp/cpp_service.cc",
+        "src/google/protobuf/compiler/cpp/cpp_string_field.cc",
+        "src/google/protobuf/compiler/csharp/csharp_doc_comment.cc",
+        "src/google/protobuf/compiler/csharp/csharp_enum.cc",
+        "src/google/protobuf/compiler/csharp/csharp_enum_field.cc",
+        "src/google/protobuf/compiler/csharp/csharp_field_base.cc",
+        "src/google/protobuf/compiler/csharp/csharp_generator.cc",
+        "src/google/protobuf/compiler/csharp/csharp_helpers.cc",
+        "src/google/protobuf/compiler/csharp/csharp_map_field.cc",
+        "src/google/protobuf/compiler/csharp/csharp_message.cc",
+        "src/google/protobuf/compiler/csharp/csharp_message_field.cc",
+        "src/google/protobuf/compiler/csharp/csharp_primitive_field.cc",
+        "src/google/protobuf/compiler/csharp/csharp_reflection_class.cc",
+        "src/google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc",
+        "src/google/protobuf/compiler/csharp/csharp_repeated_message_field.cc",
+        "src/google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc",
+        "src/google/protobuf/compiler/csharp/csharp_source_generator_base.cc",
+        "src/google/protobuf/compiler/csharp/csharp_wrapper_field.cc",
+        "src/google/protobuf/compiler/java/java_context.cc",
+        "src/google/protobuf/compiler/java/java_doc_comment.cc",
+        "src/google/protobuf/compiler/java/java_enum.cc",
+        "src/google/protobuf/compiler/java/java_enum_field.cc",
+        "src/google/protobuf/compiler/java/java_enum_field_lite.cc",
+        "src/google/protobuf/compiler/java/java_enum_lite.cc",
+        "src/google/protobuf/compiler/java/java_extension.cc",
+        "src/google/protobuf/compiler/java/java_extension_lite.cc",
+        "src/google/protobuf/compiler/java/java_field.cc",
+        "src/google/protobuf/compiler/java/java_file.cc",
+        "src/google/protobuf/compiler/java/java_generator.cc",
+        "src/google/protobuf/compiler/java/java_generator_factory.cc",
+        "src/google/protobuf/compiler/java/java_helpers.cc",
+        "src/google/protobuf/compiler/java/java_kotlin_generator.cc",
+        "src/google/protobuf/compiler/java/java_map_field.cc",
+        "src/google/protobuf/compiler/java/java_map_field_lite.cc",
+        "src/google/protobuf/compiler/java/java_message.cc",
+        "src/google/protobuf/compiler/java/java_message_builder.cc",
+        "src/google/protobuf/compiler/java/java_message_builder_lite.cc",
+        "src/google/protobuf/compiler/java/java_message_field.cc",
+        "src/google/protobuf/compiler/java/java_message_field_lite.cc",
+        "src/google/protobuf/compiler/java/java_message_lite.cc",
+        "src/google/protobuf/compiler/java/java_name_resolver.cc",
+        "src/google/protobuf/compiler/java/java_primitive_field.cc",
+        "src/google/protobuf/compiler/java/java_primitive_field_lite.cc",
+        "src/google/protobuf/compiler/java/java_service.cc",
+        "src/google/protobuf/compiler/java/java_shared_code_generator.cc",
+        "src/google/protobuf/compiler/java/java_string_field.cc",
+        "src/google/protobuf/compiler/java/java_string_field_lite.cc",
+        "src/google/protobuf/compiler/js/js_generator.cc",
+        "src/google/protobuf/compiler/js/well_known_types_embed.cc",
+        "src/google/protobuf/compiler/objectivec/objectivec_enum.cc",
+        "src/google/protobuf/compiler/objectivec/objectivec_enum_field.cc",
+        "src/google/protobuf/compiler/objectivec/objectivec_extension.cc",
+        "src/google/protobuf/compiler/objectivec/objectivec_field.cc",
+        "src/google/protobuf/compiler/objectivec/objectivec_file.cc",
+        "src/google/protobuf/compiler/objectivec/objectivec_generator.cc",
+        "src/google/protobuf/compiler/objectivec/objectivec_helpers.cc",
+        "src/google/protobuf/compiler/objectivec/objectivec_map_field.cc",
+        "src/google/protobuf/compiler/objectivec/objectivec_message.cc",
+        "src/google/protobuf/compiler/objectivec/objectivec_message_field.cc",
+        "src/google/protobuf/compiler/objectivec/objectivec_oneof.cc",
+        "src/google/protobuf/compiler/objectivec/objectivec_primitive_field.cc",
+        "src/google/protobuf/compiler/php/php_generator.cc",
+        "src/google/protobuf/compiler/plugin.cc",
+        "src/google/protobuf/compiler/plugin.pb.cc",
+        "src/google/protobuf/compiler/python/python_generator.cc",
+        "src/google/protobuf/compiler/ruby/ruby_generator.cc",
+        "src/google/protobuf/compiler/subprocess.cc",
+        "src/google/protobuf/compiler/zip_writer.cc",
+    ],
+    copts = COPTS,
+    includes = ["src/"],
+    linkopts = LINK_OPTS,
+    visibility = ["//visibility:public"],
+    deps = [":protobuf"],
+)
+
+cc_binary(
+    name = "protoc",
+    srcs = ["src/google/protobuf/compiler/main.cc"],
+    linkopts = LINK_OPTS,
+    visibility = ["//visibility:public"],
+    deps = [":protoc_lib"],
+)
+
+proto_lang_toolchain(
+    name = "cc_toolchain",
+    blacklisted_protos = [proto + "_proto" for proto in WELL_KNOWN_PROTO_MAP.keys()],
+    command_line = "--cpp_out=$(OUT)",
+    runtime = ":protobuf",
+    visibility = ["//visibility:public"],
+)
+
+alias(
+    name = "objectivec",
+    actual = "//objectivec",
+    visibility = ["//visibility:public"],
+)
+
+alias(
+    name = "protobuf_objc",
+    actual = "//objectivec",
+    visibility = ["//visibility:public"],
+)

example/build_with_bazel/test.cc

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <bthread/bthread.h>
+
+void *PrintHellobRPC(void *arg) {
+    printf("I Love bRPC");
+    return nullptr;
+}
+
+int main(int argc, char **argv) {
+    bthread_t th_1;
+    bthread_start_background(&th_1, nullptr, PrintHellobRPC, nullptr);
+    bthread_join(th_1, nullptr);
+    return 0;
+}

example/build_with_bazel/zlib.BUILD

@@ -0,0 +1,110 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Copyright 2008 Google Inc.  All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Copied from https://github.com/protocolbuffers/protobuf/blob/v3.9.1/third_party/zlib.BUILD
+load("@rules_cc//cc:defs.bzl", "cc_library")
+
+_ZLIB_HEADERS = [
+    "crc32.h",
+    "deflate.h",
+    "gzguts.h",
+    "inffast.h",
+    "inffixed.h",
+    "inflate.h",
+    "inftrees.h",
+    "trees.h",
+    "zconf.h",
+    "zlib.h",
+    "zutil.h",
+]
+
+_ZLIB_PREFIXED_HEADERS = ["zlib/include/" + hdr for hdr in _ZLIB_HEADERS]
+
+# In order to limit the damage from the `includes` propagation
+# via `:zlib`, copy the public headers to a subdirectory and
+# expose those.
+genrule(
+    name = "copy_public_headers",
+    srcs = _ZLIB_HEADERS,
+    outs = _ZLIB_PREFIXED_HEADERS,
+    cmd = "cp $(SRCS) $(@D)/zlib/include/",
+    visibility = ["//visibility:private"],
+)
+
+cc_library(
+    name = "zlib",
+    srcs = [
+        "adler32.c",
+        "compress.c",
+        "crc32.c",
+        "deflate.c",
+        "gzclose.c",
+        "gzlib.c",
+        "gzread.c",
+        "gzwrite.c",
+        "infback.c",
+        "inffast.c",
+        "inflate.c",
+        "inftrees.c",
+        "trees.c",
+        "uncompr.c",
+        "zutil.c",
+    ],
+    hdrs = _ZLIB_PREFIXED_HEADERS,
+    copts = select({
+        ":windows": [],
+        "//conditions:default": [
+            "-Wno-unused-variable",
+            "-Wno-implicit-function-declaration",
+        ],
+    }),
+    includes = ["zlib/include/"],
+    visibility = ["//visibility:public"],
+)
+
+config_setting(
+    name = "windows",
+    constraint_values = [
+        "@platforms//os:windows",
+    ],
+)

example/build_with_old_bazel/BUILD.bazel

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+genrule(
+    name = "empty_cc",
+    outs = ["empty.cc"],
+    cmd = "echo 'int main(){return 0;}' > $@",
+)
+
+cc_binary(
+    name = "empty",
+    srcs = [":empty_cc"],
+    deps = [
+        "@com_github_brpc_brpc//:brpc",
+    ],
+)

example/build_with_old_bazel/leveldb.BUILD

@@ -0,0 +1,97 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//visibility:public"])
+
+
+config_setting(
+    name = "darwin",
+    values = {"cpu": "darwin"},
+    visibility = ["//visibility:public"],
+)
+
+SOURCES = ["db/builder.cc",
+         "db/c.cc",
+         "db/dbformat.cc",
+         "db/db_impl.cc",
+         "db/db_iter.cc",
+         "db/dumpfile.cc",
+         "db/filename.cc",
+         "db/log_reader.cc",
+         "db/log_writer.cc",
+         "db/memtable.cc",
+         "db/repair.cc",
+         "db/table_cache.cc",
+         "db/version_edit.cc",
+         "db/version_set.cc",
+         "db/write_batch.cc",
+         "table/block_builder.cc",
+         "table/block.cc",
+         "table/filter_block.cc",
+         "table/format.cc",
+         "table/iterator.cc",
+         "table/merger.cc",
+         "table/table_builder.cc",
+         "table/table.cc",
+         "table/two_level_iterator.cc",
+         "util/arena.cc",
+         "util/bloom.cc",
+         "util/cache.cc",
+         "util/coding.cc",
+         "util/comparator.cc",
+         "util/crc32c.cc",
+         "util/env.cc",
+         "util/env_posix.cc",
+         "util/filter_policy.cc",
+         "util/hash.cc",
+         "util/histogram.cc",
+         "util/logging.cc",
+         "util/options.cc",
+         "util/status.cc",
+         "port/port_posix.cc",
+         "port/port_posix_sse.cc",
+         "helpers/memenv/memenv.cc",
+    ]
+
+cc_library(
+    name = "leveldb",
+    srcs = SOURCES,
+    hdrs = glob([ 
+        "helpers/memenv/*.h",
+        "util/*.h",
+        "port/*.h",
+        "port/win/*.h",
+        "table/*.h",
+        "db/*.h",
+        "include/leveldb/*.h"
+    ],
+    exclude = [
+            "**/*test.*",
+    ]),
+    includes = [
+        "include/",
+    ],
+    copts = [
+        "-fno-builtin-memcmp",
+        "-DLEVELDB_PLATFORM_POSIX=1",
+        "-DLEVELDB_ATOMIC_PRESENT",
+    ],
+    defines = [
+        "LEVELDB_PLATFORM_POSIX",
+    ] + select({
+        ":darwin": ["OS_MACOSX"],
+        "//conditions:default": [],
+    }),
+)

example/build_with_old_bazel/openssl.BUILD

@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(
+    default_visibility=["//visibility:public"]
+)
+
+config_setting(
+    name = "macos",
+    values = {
+        "cpu": "darwin",
+    },
+    visibility = ["//visibility:private"],
+)
+
+cc_library(
+    name = "crypto",
+    srcs = select({
+        ":macos": ["lib/libcrypto.dylib"],
+        "//conditions:default": []
+    }),
+    linkopts = select({
+        ":macos" : [],
+        "//conditions:default": ["-lcrypto"],
+    }),
+)
+
+cc_library(
+    name = "ssl",
+    hdrs = select({
+        ":macos": glob(["include/openssl/*.h"]),
+        "//conditions:default": []
+    }),
+    srcs = select ({
+        ":macos": ["lib/libssl.dylib"],
+        "//conditions:default": []
+    }),
+    includes = ["include"],
+    linkopts = select({
+        ":macos" : [],
+        "//conditions:default": ["-lssl"],
+    }),
+    deps = [":crypto"]
+)

example/build_with_old_bazel/zlib.BUILD

@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(
+    default_visibility=["//visibility:public"]
+)
+
+cc_library(
+    name = "zlib",
+    linkopts = ["-lz"],
+)

example/cancel_c++/CMakeLists.txt

@@ -0,0 +1,110 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.16...3.28)
+project(cancel_c++ C CXX)
+
+include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake)
+
+option(LINK_SO "Whether examples are linked dynamically" OFF)
+
+execute_process(
+    COMMAND bash -c "find ${PROJECT_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'"
+    OUTPUT_VARIABLE OUTPUT_PATH
+)
+
+if(OUTPUT_PATH)
+    list(PREPEND CMAKE_PREFIX_PATH ${OUTPUT_PATH})
+endif()
+
+find_package(Threads REQUIRED)
+find_package(Protobuf REQUIRED)
+protobuf_generate_cpp(PROTO_SRC PROTO_HEADER echo.proto)
+
+# Search for libthrift* by best effort. If it is not found and brpc is
+# compiled with thrift protocol enabled, a link error would be reported.
+find_library(THRIFT_LIB NAMES thrift)
+if (NOT THRIFT_LIB)
+    set(THRIFT_LIB "")
+endif()
+
+find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h)
+if(LINK_SO)
+    find_library(BRPC_LIB NAMES brpc)
+else()
+    find_library(BRPC_LIB NAMES libbrpc.a brpc)
+endif()
+if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB))
+    message(FATAL_ERROR "Fail to find brpc")
+endif()
+
+find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h)
+find_library(GFLAGS_LIBRARY NAMES gflags libgflags)
+if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY))
+    message(FATAL_ERROR "Fail to find gflags")
+endif()
+
+find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h)
+find_library(LEVELDB_LIB NAMES leveldb)
+if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB))
+    message(FATAL_ERROR "Fail to find leveldb")
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT OPENSSL_ROOT_DIR)
+    set(OPENSSL_ROOT_DIR
+        "/usr/local/opt/openssl"    # Homebrew installed OpenSSL
+        )
+endif()
+
+find_package(OpenSSL REQUIRED)
+
+set(DYNAMIC_LIB
+    Threads::Threads
+    ${GFLAGS_LIBRARY}
+    ${PROTOBUF_LIBRARIES}
+    ${LEVELDB_LIB}
+    ${OPENSSL_CRYPTO_LIBRARY}
+    ${OPENSSL_SSL_LIBRARY}
+    ${THRIFT_LIB}
+    dl
+    )
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    set(DYNAMIC_LIB ${DYNAMIC_LIB}
+        pthread
+        "-framework CoreFoundation"
+        "-framework CoreGraphics"
+        "-framework CoreData"
+        "-framework CoreText"
+        "-framework Security"
+        "-framework Foundation"
+        "-Wl,-U,_MallocExtension_ReleaseFreeMemory"
+        "-Wl,-U,_ProfilerStart"
+        "-Wl,-U,_ProfilerStop"
+        "-Wl,-U,__Z13GetStackTracePPvii"
+        "-Wl,-U,_mallctl"
+        "-Wl,-U,_malloc_stats_print"
+    )
+endif()
+
+add_executable(cancel_client client.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(cancel_client)
+add_executable(cancel_server server.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(cancel_server)
+
+target_link_libraries(cancel_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})
+target_link_libraries(cancel_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})

example/cancel_c++/client.cpp

@@ -0,0 +1,129 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A client to send 2 requests to server and accept the first returned response.
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <butil/time.h>
+#include <brpc/channel.h>
+#include "echo.pb.h"
+
+DEFINE_string(protocol, "baidu_std", "Protocol type. Defined in src/brpc/options.proto");
+DEFINE_string(connection_type, "", "Connection type. Available values: single, pooled, short");
+DEFINE_string(server, "0.0.0.0:8000", "IP Address of server");
+DEFINE_string(load_balancer, "", "The algorithm for load balancing");
+DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)");
+
+// A special done for canceling another RPC.
+class CancelRPC : public google::protobuf::Closure {
+public:
+    explicit CancelRPC(brpc::CallId rpc_id) : _rpc_id(rpc_id) {}
+    
+    void Run() {
+        brpc::StartCancel(_rpc_id);
+    }
+    
+private:
+    brpc::CallId _rpc_id;
+};
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+    
+    // A Channel represents a communication line to a Server. Notice that 
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::Channel channel;
+
+    // Initialize the channel, NULL means using default options. 
+    brpc::ChannelOptions options;
+    options.protocol = FLAGS_protocol;
+    options.connection_type = FLAGS_connection_type;
+    options.timeout_ms = FLAGS_timeout_ms/*milliseconds*/;
+    options.max_retry = FLAGS_max_retry;
+    if (channel.Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), &options) != 0) {
+        LOG(ERROR) << "Fail to initialize channel";
+        return -1;
+    }
+
+    // Normally, you should not call a Channel directly, but instead construct
+    // a stub Service wrapping it. stub can be shared by all threads as well.
+    example::EchoService_Stub stub(&channel);
+
+    // Send a request and wait for the response every 1 second.
+    int log_id = 0;
+    while (!brpc::IsAskedToQuit()) {
+        example::EchoRequest request1;
+        example::EchoResponse response1;
+        brpc::Controller cntl1;
+
+        example::EchoRequest request2;
+        example::EchoResponse response2;
+        brpc::Controller cntl2;
+
+        request1.set_message("hello1");
+        request2.set_message("hello2");
+        
+        cntl1.set_log_id(log_id ++);  // set by user
+        cntl2.set_log_id(log_id ++);
+
+        const brpc::CallId id1 = cntl1.call_id();
+        const brpc::CallId id2 = cntl2.call_id();
+        CancelRPC done1(id2);
+        CancelRPC done2(id1);
+        
+        butil::Timer tm;
+        tm.start();
+        // Send 2 async calls and join them. They will cancel each other in
+        // their done which is run before the RPC being Join()-ed. Canceling
+        // a finished RPC has no effect. 
+        // For example:
+        //  Time       RPC1                      RPC2
+        //   1     response1 comes back. 
+        //   2     running done1.
+        //   3     cancel RPC2   
+        //   4                              running done2 (NOTE: done also runs)
+        //   5                              cancel RPC1 (no effect)
+        stub.Echo(&cntl1, &request1, &response1, &done1);
+        stub.Echo(&cntl2, &request2, &response2, &done2);
+        brpc::Join(id1);
+        brpc::Join(id2);
+        tm.stop();
+        if (cntl1.Failed() && cntl2.Failed()) {
+            LOG(WARNING) << "Both failed. rpc1:" << cntl1.ErrorText()
+                         << ", rpc2: " << cntl2.ErrorText();
+        } else if (!cntl1.Failed()) {
+            LOG(INFO) << "Received `" << response1.message() << "' from rpc1="
+                      << id1.value << '@' << cntl1.remote_side()
+                      << " latency=" << tm.u_elapsed() << "us"
+                      << " rpc1_latency=" << cntl1.latency_us() << "us"
+                      << " rpc2_latency=" << cntl2.latency_us() << "us";
+        } else {
+            LOG(INFO) << "Received `" << response2.message() << "' from rpc2="
+                      << id2.value << '@' << cntl2.remote_side()
+                      << " latency=" << tm.u_elapsed() << "us"
+                      << " rpc1_latency=" << cntl1.latency_us() << "us"
+                      << " rpc2_latency=" << cntl2.latency_us() << "us";
+        }
+        sleep(1);
+    }
+
+    LOG(INFO) << "EchoClient is going to quit";
+    return 0;
+}

example/cancel_c++/echo.proto

@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+syntax="proto2";
+package example;
+
+option cc_generic_services = true;
+
+message EchoRequest {
+      required string message = 1;
+};
+
+message EchoResponse {
+      required string message = 1;
+};
+
+service EchoService {
+      rpc Echo(EchoRequest) returns (EchoResponse);
+};

example/cancel_c++/server.cpp

@@ -0,0 +1,104 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A server to receive EchoRequest and send back EchoResponse.
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <brpc/server.h>
+#include "echo.pb.h"
+
+DEFINE_bool(echo_attachment, true, "Echo attachment as well");
+DEFINE_int32(port, 8000, "TCP Port of this server");
+DEFINE_int32(idle_timeout_s, -1, "Connection will be closed if there is no "
+             "read/write operations during the last `idle_timeout_s'");
+
+// Your implementation of example::EchoService
+// Notice that implementing brpc::Describable grants the ability to put
+// additional information in /status.
+namespace example {
+class EchoServiceImpl : public EchoService {
+public:
+    EchoServiceImpl() {}
+    virtual ~EchoServiceImpl() {}
+    virtual void Echo(google::protobuf::RpcController* cntl_base,
+                      const EchoRequest* request,
+                      EchoResponse* response,
+                      google::protobuf::Closure* done) {
+        // This object helps you to call done->Run() in RAII style. If you need
+        // to process the request asynchronously, pass done_guard.release().
+        brpc::ClosureGuard done_guard(done);
+
+        brpc::Controller* cntl =
+            static_cast<brpc::Controller*>(cntl_base);
+
+        // The purpose of following logs is to help you to understand
+        // how clients interact with servers more intuitively. You should 
+        // remove these logs in performance-sensitive servers.
+        LOG(INFO) << "Received request[log_id=" << cntl->log_id() 
+                  << "] from " << cntl->remote_side() 
+                  << " to " << cntl->local_side()
+                  << ": " << request->message()
+                  << " (attached=" << cntl->request_attachment() << ")";
+
+        // Fill response.
+        response->set_message(request->message());
+
+        // You can compress the response by setting Controller, but be aware
+        // that compression may be costly, evaluate before turning on.
+        // cntl->set_response_compress_type(brpc::COMPRESS_TYPE_GZIP);
+
+        if (FLAGS_echo_attachment) {
+            // Set attachment which is wired to network directly instead of
+            // being serialized into protobuf messages.
+            cntl->response_attachment().append(cntl->request_attachment());
+        }
+    }
+};
+}  // namespace example
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    // Generally you only need one Server.
+    brpc::Server server;
+
+    // Instance of your service.
+    example::EchoServiceImpl echo_service_impl;
+
+    // Add the service into server. Notice the second parameter, because the
+    // service is put on stack, we don't want server to delete it, otherwise
+    // use brpc::SERVER_OWNS_SERVICE.
+    if (server.AddService(&echo_service_impl, 
+                          brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
+        LOG(ERROR) << "Fail to add service";
+        return -1;
+    }
+
+    // Start the server.
+    brpc::ServerOptions options;
+    options.idle_timeout_sec = FLAGS_idle_timeout_s;
+    if (server.Start(FLAGS_port, &options) != 0) {
+        LOG(ERROR) << "Fail to start EchoServer";
+        return -1;
+    }
+
+    // Wait until Ctrl-C is pressed, then Stop() and Join() the server.
+    server.RunUntilAskedToQuit();
+    return 0;
+}

example/cascade_echo_c++/CMakeLists.txt

@@ -0,0 +1,109 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.16...3.28)
+project(cascade_echo_c++ C CXX)
+
+include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake)
+
+option(LINK_SO "Whether examples are linked dynamically" OFF)
+
+execute_process(
+    COMMAND bash -c "find ${PROJECT_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'"
+    OUTPUT_VARIABLE OUTPUT_PATH
+)
+
+if(OUTPUT_PATH)
+    list(PREPEND CMAKE_PREFIX_PATH ${OUTPUT_PATH})
+endif()
+find_package(Threads REQUIRED)
+find_package(Protobuf REQUIRED)
+protobuf_generate_cpp(PROTO_SRC PROTO_HEADER echo.proto)
+
+# Search for libthrift* by best effort. If it is not found and brpc is
+# compiled with thrift protocol enabled, a link error would be reported.
+find_library(THRIFT_LIB NAMES thrift)
+if (NOT THRIFT_LIB)
+    set(THRIFT_LIB "")
+endif()
+
+find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h)
+if(LINK_SO)
+    find_library(BRPC_LIB NAMES brpc)
+else()
+    find_library(BRPC_LIB NAMES libbrpc.a brpc)
+endif()
+if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB))
+    message(FATAL_ERROR "Fail to find brpc")
+endif()
+
+find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h)
+find_library(GFLAGS_LIBRARY NAMES gflags libgflags)
+if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY))
+    message(FATAL_ERROR "Fail to find gflags")
+endif()
+
+find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h)
+find_library(LEVELDB_LIB NAMES leveldb)
+if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB))
+    message(FATAL_ERROR "Fail to find leveldb")
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT OPENSSL_ROOT_DIR)
+    set(OPENSSL_ROOT_DIR
+        "/usr/local/opt/openssl"    # Homebrew installed OpenSSL
+        )
+endif()
+
+find_package(OpenSSL REQUIRED)
+
+set(DYNAMIC_LIB
+    Threads::Threads
+    ${GFLAGS_LIBRARY}
+    ${PROTOBUF_LIBRARIES}
+    ${LEVELDB_LIB}
+    ${OPENSSL_CRYPTO_LIBRARY}
+    ${OPENSSL_SSL_LIBRARY}
+    ${THRIFT_LIB}
+    dl
+    )
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    set(DYNAMIC_LIB ${DYNAMIC_LIB}
+        pthread
+        "-framework CoreFoundation"
+        "-framework CoreGraphics"
+        "-framework CoreData"
+        "-framework CoreText"
+        "-framework Security"
+        "-framework Foundation"
+        "-Wl,-U,_MallocExtension_ReleaseFreeMemory"
+        "-Wl,-U,_ProfilerStart"
+        "-Wl,-U,_ProfilerStop"
+        "-Wl,-U,__Z13GetStackTracePPvii"
+        "-Wl,-U,_mallctl"
+        "-Wl,-U,_malloc_stats_print"
+    )
+endif()
+
+add_executable(cascade_echo_client client.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(cascade_echo_client)
+add_executable(cascade_echo_server server.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(cascade_echo_server)
+
+target_link_libraries(cascade_echo_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})
+target_link_libraries(cascade_echo_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})

example/cascade_echo_c++/client.cpp

@@ -0,0 +1,149 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A client sending requests to server which will send the request to itself
+// again according to the field `depth'
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <butil/time.h>
+#include <bthread/bthread.h>
+#include <brpc/channel.h>
+#include <brpc/server.h>
+#include "echo.pb.h"
+#include <bvar/bvar.h>
+#include <butil/fast_rand.h>
+
+DEFINE_int32(thread_num, 2, "Number of threads to send requests");
+DEFINE_bool(use_bthread, false, "Use bthread to send requests");
+DEFINE_string(attachment, "foo", "Carry this along with requests");
+DEFINE_string(connection_type, "", "Connection type. Available values: single, pooled, short");
+DEFINE_string(server, "0.0.0.0:8000", "IP Address of server");
+DEFINE_string(load_balancer, "", "The algorithm for load balancing");
+DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); 
+DEFINE_string(protocol, "baidu_std", "Protocol type. Defined in src/brpc/options.proto");
+DEFINE_int32(depth, 0, "number of loop calls");
+// Don't send too frequently in this example
+DEFINE_int32(sleep_ms, 1000, "milliseconds to sleep after each RPC");
+DEFINE_int32(dummy_port, -1, "Launch dummy server at this port");
+
+bvar::LatencyRecorder g_latency_recorder("client");
+
+void* sender(void* arg) {
+    brpc::Channel* chan = (brpc::Channel*)arg;
+    // Normally, you should not call a Channel directly, but instead construct
+    // a stub Service wrapping it. stub can be shared by all threads as well.
+    example::EchoService_Stub stub(chan);
+
+    // Send a request and wait for the response every 1 second.
+    while (!brpc::IsAskedToQuit()) {
+        // We will receive response synchronously, safe to put variables
+        // on stack.
+        example::EchoRequest request;
+        example::EchoResponse response;
+        brpc::Controller cntl;
+
+        request.set_message("hello world");
+        if (FLAGS_depth > 0) {
+            request.set_depth(FLAGS_depth);
+        }
+
+        // Set request_id to be a random string
+        cntl.set_request_id(butil::fast_rand_printable(9));
+
+        // Set attachment which is wired to network directly instead of 
+        // being serialized into protobuf messages.
+        cntl.request_attachment().append(FLAGS_attachment);
+
+        // Because `done'(last parameter) is NULL, this function waits until
+        // the response comes back or error occurs(including timedout).
+        stub.Echo(&cntl, &request, &response, NULL);
+        if (cntl.Failed()) {
+            //LOG_EVERY_SECOND(WARNING) << "Fail to send EchoRequest, " << cntl.ErrorText();
+        } else {
+            g_latency_recorder << cntl.latency_us();
+        }
+        if (FLAGS_sleep_ms != 0) {
+            bthread_usleep(FLAGS_sleep_ms * 1000L);
+        }
+    }
+    return NULL;
+}
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::SetUsageMessage("Send EchoRequest to server every second");
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    // A Channel represents a communication line to a Server. Notice that 
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::Channel channel;
+    brpc::ChannelOptions options;
+    options.protocol = FLAGS_protocol;
+    options.connection_type = FLAGS_connection_type;
+    options.timeout_ms = FLAGS_timeout_ms/*milliseconds*/;
+    options.max_retry = FLAGS_max_retry;
+    
+    // Initialize the channel, NULL means using default options. 
+    // options, see `brpc/channel.h'.
+    if (channel.Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), &options) != 0) {
+        LOG(ERROR) << "Fail to initialize channel";
+        return -1;
+    }
+
+    std::vector<bthread_t> bids;
+    std::vector<pthread_t> pids;
+    if (!FLAGS_use_bthread) {
+        pids.resize(FLAGS_thread_num);
+        for (int i = 0; i < FLAGS_thread_num; ++i) {
+            if (pthread_create(&pids[i], NULL, sender, &channel) != 0) {
+                LOG(ERROR) << "Fail to create pthread";
+                return -1;
+            }
+        }
+    } else {
+        bids.resize(FLAGS_thread_num);
+        for (int i = 0; i < FLAGS_thread_num; ++i) {
+            if (bthread_start_background(
+                    &bids[i], NULL, sender, &channel) != 0) {
+                LOG(ERROR) << "Fail to create bthread";
+                return -1;
+            }
+        }
+    }
+
+    if (FLAGS_dummy_port >= 0) {
+        brpc::StartDummyServerAt(FLAGS_dummy_port);
+    }
+
+    while (!brpc::IsAskedToQuit()) {
+        sleep(1);
+        LOG(INFO) << "Sending EchoRequest at qps=" << g_latency_recorder.qps(1)
+                  << " latency=" << g_latency_recorder.latency(1);
+    }
+
+    LOG(INFO) << "EchoClient is going to quit";
+    for (int i = 0; i < FLAGS_thread_num; ++i) {
+        if (!FLAGS_use_bthread) {
+            pthread_join(pids[i], NULL);
+        } else {
+            bthread_join(bids[i], NULL);
+        }
+    }
+    return 0;
+}

example/cascade_echo_c++/echo.proto

@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+syntax="proto2";
+package example;
+
+option cc_generic_services = true;
+
+message EchoRequest {
+    required string message = 1;
+    optional int32 depth = 2;
+};
+
+message EchoResponse {
+    required string message = 1;
+};
+
+service EchoService {
+    rpc Echo(EchoRequest) returns (EchoResponse);
+};

example/cascade_echo_c++/server.cpp

@@ -0,0 +1,139 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <brpc/channel.h>
+#include <brpc/server.h>
+#include "echo.pb.h"
+
+DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); 
+
+DEFINE_bool(echo_attachment, true, "Echo attachment as well");
+DEFINE_int32(port, 8000, "TCP Port of this server");
+DEFINE_string(server, "", "The server to connect, localhost:FLAGS_port as default");
+DEFINE_string(load_balancer, "", "The algorithm for load balancing");
+DEFINE_bool(use_http, false, "Use http protocol to transfer messages");
+DEFINE_int32(idle_timeout_s, -1, "Connection will be closed if there is no "
+             "read/write operations during the last `idle_timeout_s'");
+
+brpc::Channel channel;
+
+// Your implementation of example::EchoService
+namespace example {
+class CascadeEchoService : public EchoService {
+public:
+    CascadeEchoService() {}
+    virtual ~CascadeEchoService() {}
+    virtual void Echo(google::protobuf::RpcController* cntl_base,
+                      const EchoRequest* request,
+                      EchoResponse* response,
+                      google::protobuf::Closure* done) {
+        // This object helps you to call done->Run() in RAII style. If you need
+        // to process the request asynchronously, pass done_guard.release().
+        brpc::ClosureGuard done_guard(done);
+
+        brpc::Controller* cntl =
+            static_cast<brpc::Controller*>(cntl_base);
+
+        if (request->depth() > 0) {
+            CLOGI(cntl) << "I'm about to call myself for another time, depth=" << request->depth();
+            example::EchoService_Stub stub(&channel);
+            example::EchoRequest request2;
+            example::EchoResponse response2;
+            brpc::Controller cntl2(cntl->inheritable());
+            request2.set_message(request->message());
+            request2.set_depth(request->depth() - 1);
+
+            cntl2.set_timeout_ms(FLAGS_timeout_ms);
+            cntl2.set_max_retry(FLAGS_max_retry);
+            stub.Echo(&cntl2, &request2, &response2, NULL);
+            if (cntl2.Failed()) {
+                CLOGE(&cntl2) << "Fail to send EchoRequest, " << cntl2.ErrorText();
+                cntl->SetFailed(cntl2.ErrorCode(), "%s", cntl2.ErrorText().c_str());
+                return;
+            }
+            response->set_message(response2.message());
+        } else {
+            CLOGI(cntl) << "I'm the last call";
+            response->set_message(request->message());
+        }
+        
+        if (FLAGS_echo_attachment && !FLAGS_use_http) {
+            // Set attachment which is wired to network directly instead of
+            // being serialized into protobuf messages.
+            cntl->response_attachment().append(cntl->request_attachment());
+        }
+    }
+};
+}  // namespace example
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::SetUsageMessage("A server that may call itself");
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    // A Channel represents a communication line to a Server. Notice that 
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::ChannelOptions coption;
+    if (FLAGS_use_http) {
+        coption.protocol = brpc::PROTOCOL_HTTP;
+    }
+    
+    // Initialize the channel, NULL means using default options. 
+    // options, see `brpc/channel.h'.
+    if (FLAGS_server.empty()) {
+        if (channel.Init("localhost", FLAGS_port, &coption) != 0) {
+            LOG(ERROR) << "Fail to initialize channel";
+            return -1;
+        }
+    } else {
+        if (channel.Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), &coption) != 0) {
+            LOG(ERROR) << "Fail to initialize channel";
+            return -1;
+        }
+    }
+
+    // Generally you only need one Server.
+    brpc::Server server;
+    // For more options see `brpc/server.h'.
+    brpc::ServerOptions options;
+    options.idle_timeout_sec = FLAGS_idle_timeout_s;
+
+    // Instance of your service.
+    example::CascadeEchoService echo_service_impl;
+
+    // Add the service into server. Notice the second parameter, because the
+    // service is put on stack, we don't want server to delete it, otherwise
+    // use brpc::SERVER_OWNS_SERVICE.
+    if (server.AddService(&echo_service_impl, 
+                          brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
+        LOG(ERROR) << "Fail to add service";
+        return -1;
+    }
+
+    // Start the server.
+    if (server.Start(FLAGS_port, &options) != 0) {
+        LOG(ERROR) << "Fail to start EchoServer";
+        return -1;
+    }
+    
+    // Wait until Ctrl-C is pressed, then Stop() and Join() the server.
+    server.RunUntilAskedToQuit();
+    return 0;
+}

example/cmake/BrpcExample.cmake

@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+include_guard(GLOBAL)
+
+function(brpc_example_append_existing_dirs out_var)
+    set(_dirs)
+    foreach(_dir IN LISTS ARGN)
+        if(_dir AND NOT _dir MATCHES "-NOTFOUND$")
+            list(APPEND _dirs ${_dir})
+        endif()
+    endforeach()
+    set(${out_var} ${_dirs} PARENT_SCOPE)
+endfunction()
+
+function(brpc_example_configure_target target_name)
+    brpc_example_append_existing_dirs(_include_dirs
+        ${CMAKE_CURRENT_BINARY_DIR}
+        ${BRPC_INCLUDE_PATH}
+        ${GFLAGS_INCLUDE_PATH}
+        ${LEVELDB_INCLUDE_PATH}
+        ${OPENSSL_INCLUDE_DIR}
+        ${GPERFTOOLS_INCLUDE_DIR}
+        ${RDMA_INCLUDE_PATH}
+    )
+
+    if(_include_dirs)
+        target_include_directories(${target_name} PRIVATE ${_include_dirs})
+    endif()
+
+    target_compile_features(${target_name} PRIVATE cxx_std_11)
+    target_compile_definitions(${target_name} PRIVATE
+        NDEBUG
+        __const__=__unused__
+    )
+    target_compile_options(${target_name} PRIVATE
+        -O2
+        -pipe
+        -W
+        -Wall
+        -Wno-unused-parameter
+        -fPIC
+        -fno-omit-frame-pointer
+    )
+
+    if(BRPC_EXAMPLE_ENABLE_CPU_PROFILER)
+        target_compile_definitions(${target_name} PRIVATE BRPC_ENABLE_CPU_PROFILER)
+    endif()
+
+    if(BRPC_EXAMPLE_WITH_RDMA)
+        target_compile_definitions(${target_name} PRIVATE BRPC_WITH_RDMA=1)
+    endif()
+
+    if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+        include(CheckFunctionExists)
+        check_function_exists(clock_gettime BRPC_EXAMPLE_HAVE_CLOCK_GETTIME)
+        if(NOT BRPC_EXAMPLE_HAVE_CLOCK_GETTIME)
+            target_compile_definitions(${target_name} PRIVATE NO_CLOCK_GETTIME_IN_MAC)
+        endif()
+    endif()
+endfunction()

example/coroutine/coroutine_server.cpp

@@ -0,0 +1,145 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A server to receive EchoRequest and send back EchoResponse.
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <brpc/server.h>
+#include <brpc/channel.h>
+#include <brpc/coroutine.h>
+#include "echo.pb.h"
+
+DEFINE_int32(port, 8000, "TCP Port of this server");
+DEFINE_int32(sleep_us, 1000000, "Server sleep us");
+DEFINE_bool(enable_coroutine, true, "Enable coroutine");
+
+using brpc::experimental::Awaitable;
+using brpc::experimental::AwaitableDone;
+using brpc::experimental::Coroutine;
+
+namespace example {
+class EchoServiceImpl : public EchoService {
+public:
+    EchoServiceImpl() {
+        brpc::ChannelOptions options;
+        options.timeout_ms = FLAGS_sleep_us / 1000 * 2 + 100;
+        options.max_retry = 0;
+        CHECK(_channel.Init(butil::EndPoint(butil::IP_ANY, FLAGS_port), &options) == 0);
+    }
+
+    virtual ~EchoServiceImpl() {}
+
+    void Echo(google::protobuf::RpcController* cntl_base,
+              const EchoRequest* request,
+              EchoResponse* response,
+              google::protobuf::Closure* done) override {
+        // brpc::Controller* cntl =
+        //     static_cast<brpc::Controller*>(cntl_base);
+
+        if (FLAGS_enable_coroutine) {
+            Coroutine(EchoAsync(request, response, done), true);
+        } else {
+            brpc::ClosureGuard done_guard(done);
+            bthread_usleep(FLAGS_sleep_us);
+            response->set_message(request->message());
+        }
+    }
+
+    Awaitable<void> EchoAsync(const EchoRequest* request,
+               EchoResponse* response,
+               google::protobuf::Closure* done) {
+        brpc::ClosureGuard done_guard(done);
+        co_await Coroutine::usleep(FLAGS_sleep_us);
+        response->set_message(request->message());
+    }
+
+    void Proxy(google::protobuf::RpcController* cntl_base,
+               const EchoRequest* request,
+               EchoResponse* response,
+               google::protobuf::Closure* done) override {
+        // brpc::Controller* cntl =
+        //     static_cast<brpc::Controller*>(cntl_base);
+
+        if (FLAGS_enable_coroutine) {
+            Coroutine(ProxyAsync(request, response, done), true);
+        } else {
+            brpc::ClosureGuard done_guard(done);
+            EchoService_Stub stub(&_channel);
+            brpc::Controller cntl;
+            stub.Echo(&cntl, request, response, NULL);
+            if (cntl.Failed()) {
+                response->set_message(cntl.ErrorText());
+            }
+        }
+    }
+
+    Awaitable<void> ProxyAsync(const EchoRequest* request,
+                    EchoResponse* response,
+                    google::protobuf::Closure* done) {
+        brpc::ClosureGuard done_guard(done);
+        EchoService_Stub stub(&_channel);
+        brpc::Controller cntl;
+        AwaitableDone done2;
+        stub.Echo(&cntl, request, response, &done2);
+        co_await done2.awaitable();
+        if (cntl.Failed()) {
+            response->set_message(cntl.ErrorText());
+        }
+    }    
+
+private:
+    brpc::Channel _channel;
+};
+}  // namespace example
+
+int main(int argc, char* argv[]) {
+    bthread_setconcurrency(BTHREAD_MIN_CONCURRENCY);
+
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+    if (FLAGS_enable_coroutine) {
+        GFLAGS_NAMESPACE::SetCommandLineOption("usercode_in_coroutine", "true");
+    }
+
+    // Generally you only need one Server.
+    brpc::Server server;
+
+    // Instance of your service.
+    example::EchoServiceImpl echo_service_impl;
+
+    // Add the service into server. Notice the second parameter, because the
+    // service is put on stack, we don't want server to delete it, otherwise
+    // use brpc::SERVER_OWNS_SERVICE.
+    if (server.AddService(&echo_service_impl, 
+                          brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
+        LOG(ERROR) << "Fail to add service";
+        return -1;
+    }
+
+    // Start the server.
+    brpc::ServerOptions options;
+    options.num_threads = BTHREAD_MIN_CONCURRENCY;
+    if (server.Start(FLAGS_port, &options) != 0) {
+        LOG(ERROR) << "Fail to start EchoServer";
+        return -1;
+    }
+
+    // Wait until Ctrl-C is pressed, then Stop() and Join() the server.
+    server.RunUntilAskedToQuit();
+    return 0;
+}

example/coroutine/echo.proto

@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+syntax="proto2";
+package example;
+
+option cc_generic_services = true;
+
+message EchoRequest {
+      required string message = 1;
+};
+
+message EchoResponse {
+      required string message = 1;
+};
+
+service EchoService {
+      rpc Echo(EchoRequest) returns (EchoResponse);
+      rpc Proxy(EchoRequest) returns (EchoResponse);
+};

example/couchbase_c++/couchbase_client.cpp

@@ -0,0 +1,451 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <brpc/couchbase.h>
+#include <butil/logging.h>
+#include <gflags/gflags.h>
+
+#include <iostream>
+#include <string>
+
+// ANSI color codes for console output
+#define GREEN "\033[32m"
+#define RED "\033[31m"
+#define RESET "\033[0m"
+
+DEFINE_string(server, "localhost:11210", "IP Address of server");
+int performOperations(brpc::CouchbaseOperations& couchbase_ops) {
+  std::string add_key = "user::test_brpc_binprot";
+  std::string add_value =
+      R"({"name": "John Doe", "age": 30, "email": "john@example.com"})";
+
+  brpc::CouchbaseOperations::Result add_result =
+      couchbase_ops.add(add_key, add_value);
+  if (add_result.success) {
+    std::cout << GREEN << "ADD operation successful" << RESET << std::endl;
+  } else {
+    std::cout << RED << "ADD operation failed: " << add_result.error_message
+              << RESET << std::endl;
+  }
+
+  // Try to ADD the same key again (should fail with key exists)
+  brpc::CouchbaseOperations::Result add_result2 =
+      couchbase_ops.add(add_key, add_value);
+  if (add_result2.success) {
+    std::cout << GREEN << "Second ADD operation unexpectedly successful"
+              << RESET << std::endl;
+  } else {
+    std::cout << RED << "Second ADD operation failed as expected: "
+              << add_result2.error_message << RESET << std::endl;
+  }
+  // Get operation using high-level method
+  brpc::CouchbaseOperations::Result get_result = couchbase_ops.get(add_key);
+  if (get_result.success) {
+    std::cout << GREEN << "GET operation successful" << RESET << std::endl;
+    std::cout << "GET value: " << get_result.value << std::endl;
+  } else {
+    std::cout << RED << "GET operation failed: " << get_result.error_message
+              << RESET << std::endl;
+  }
+
+  // Add binprot item1 using high-level method
+  std::string item1_key = "binprot_item1";
+  brpc::CouchbaseOperations::Result item1_result =
+      couchbase_ops.add(item1_key, add_value);
+  if (item1_result.success) {
+    std::cout << GREEN << "ADD binprot item1 successful" << RESET << std::endl;
+  } else {
+    std::cout << RED
+              << "ADD binprot item1 failed: " << item1_result.error_message
+              << RESET << std::endl;
+  }
+
+  // Add binprot item2 using high-level method
+  std::string item2_key = "binprot_item2";
+  brpc::CouchbaseOperations::Result item2_result =
+      couchbase_ops.add(item2_key, add_value);
+  if (item2_result.success) {
+    std::cout << GREEN << "ADD binprot item2 successful" << RESET << std::endl;
+  } else {
+    std::cout << RED
+              << "ADD binprot item2 failed: " << item2_result.error_message
+              << RESET << std::endl;
+  }
+
+  // Add binprot item3 using high-level method
+  std::string item3_key = "binprot_item3";
+  brpc::CouchbaseOperations::Result item3_result =
+      couchbase_ops.add(item3_key, add_value);
+  if (item3_result.success) {
+    std::cout << GREEN << "ADD binprot item3 successful" << RESET << std::endl;
+  } else {
+    std::cout << RED
+              << "ADD binprot item3 failed: " << item3_result.error_message
+              << RESET << std::endl;
+  }
+
+  // Perform an UPSERT on the existing key using high-level method
+  std::string upsert_key = "user::test_brpc_binprot";
+  std::string upsert_value =
+      R"({"name": "Upserted Jane Doe", "age": 28, "email": "upserted.doe@example.com"})";
+  brpc::CouchbaseOperations::Result upsert_result =
+      couchbase_ops.upsert(upsert_key, upsert_value);
+  if (upsert_result.success) {
+    std::cout
+        << GREEN
+        << "UPSERT operation successful when the document exists in the server"
+        << RESET << std::endl;
+  } else {
+    std::cout
+        << RED
+        << "UPSERT operation failed when the document exists in the server: "
+        << upsert_result.error_message << RESET << std::endl;
+  }
+  // Do UPSERT operation on a new document using high-level method
+  std::string new_upsert_key = "user::test_brpc_new_upsert";
+  std::string new_upsert_value =
+      R"({"name": "Jane Doe", "age": 28, "email": "jane.doe@example.com"})";
+  brpc::CouchbaseOperations::Result new_upsert_result =
+      couchbase_ops.upsert(new_upsert_key, new_upsert_value);
+  if (new_upsert_result.success) {
+    std::cout << GREEN
+              << "UPSERT operation successful when the document doesn't exist "
+                 "in the server"
+              << RESET << std::endl;
+  } else {
+    std::cout << RED
+              << "UPSERT operation failed when document does not exist in the "
+                 "server: "
+              << new_upsert_result.error_message << RESET << std::endl;
+  }
+
+  // Check the upserted data using high-level method
+  std::string check_key = "user::test_brpc_new_upsert";
+  brpc::CouchbaseOperations::Result check_result = couchbase_ops.get(check_key);
+  if (check_result.success) {
+    std::cout << GREEN << "GET after UPSERT operation successful - Value: "
+              << check_result.value << RESET << std::endl;
+  } else {
+    std::cout << RED << "GET after UPSERT operation failed: "
+              << check_result.error_message << RESET << std::endl;
+  }
+
+  // Delete a non-existent key using high-level method
+  std::string delete_key = "Nonexistent_key";
+  brpc::CouchbaseOperations::Result delete_result =
+      couchbase_ops.delete_(delete_key);
+  if (delete_result.success) {
+    std::cout << GREEN << "DELETE operation successful" << RESET << std::endl;
+  } else {
+    std::cout << RED << "DELETE operation failed: as expected "
+              << delete_result.error_message << RESET << std::endl;
+  }
+
+  // Delete the existing key using high-level method
+  std::string delete_existing_key = "user::test_brpc_binprot";
+  brpc::CouchbaseOperations::Result delete_existing_result =
+      couchbase_ops.delete_(delete_existing_key);
+  if (delete_existing_result.success) {
+    std::cout << GREEN << "DELETE operation successful" << RESET << std::endl;
+  } else {
+    std::cout << RED << "DELETE operation failed: "
+              << delete_existing_result.error_message << RESET << std::endl;
+  }
+
+  // Retrieve Collection ID for scope `_default` and collection
+  // `col1`
+  const std::string scope_name = "_default";  // default scope
+  std::string collection_name = "col1";       // target collection
+  // ------------------------------------------------------------------
+  // Collection-scoped CRUD operations (only if collection id was retrieved)
+  // ------------------------------------------------------------------
+  // 1. ADD in collection using high-level method
+  std::string coll_key = "user::collection_doc";
+  std::string coll_value = R"({"type":"collection","op":"add","v":1})";
+  brpc::CouchbaseOperations::Result coll_add_result =
+      couchbase_ops.add(coll_key, coll_value, collection_name);
+  if (coll_add_result.success) {
+    std::cout << GREEN << "Collection ADD success" << RESET << std::endl;
+  } else {
+    std::cout << RED
+              << "Collection ADD failed: " << coll_add_result.error_message
+              << RESET << std::endl;
+  }
+  // 2. GET from collection using high-level method
+  brpc::CouchbaseOperations::Result coll_get_result =
+      couchbase_ops.get(coll_key, collection_name);
+  if (coll_get_result.success) {
+    std::cout << GREEN
+              << "Collection GET success value=" << coll_get_result.value
+              << RESET << std::endl;
+  } else {
+    std::cout << RED
+              << "Collection GET failed: " << coll_get_result.error_message
+              << RESET << std::endl;
+  }
+
+  // 3. UPSERT in collection using high-level method
+  std::string coll_upsert_value =
+      R"({"type":"collection","op":"upsert","v":2})";
+  brpc::CouchbaseOperations::Result coll_upsert_result =
+      couchbase_ops.upsert(coll_key, coll_upsert_value, collection_name);
+  if (coll_upsert_result.success) {
+    std::cout << GREEN << "Collection UPSERT success" << RESET << std::endl;
+  } else {
+    std::cout << RED << "Collection UPSERT failed: "
+              << coll_upsert_result.error_message << RESET << std::endl;
+  }
+
+  // 4. GET again to verify upsert using high-level method
+  brpc::CouchbaseOperations::Result coll_get2_result =
+      couchbase_ops.get(coll_key, collection_name);
+  if (coll_get2_result.success) {
+    std::cout << GREEN
+              << "Collection GET(after upsert) value=" << coll_get2_result.value
+              << RESET << std::endl;
+  }
+
+  // 5. DELETE from collection using high-level method
+  brpc::CouchbaseOperations::Result coll_del_result =
+      couchbase_ops.delete_(coll_key, collection_name);
+  if (coll_del_result.success) {
+    std::cout << GREEN << "Collection DELETE success" << RESET << std::endl;
+  } else {
+    std::cout << RED
+              << "Collection DELETE failed: " << coll_del_result.error_message
+              << RESET << std::endl;
+  }
+
+  // ------------------------------------------------------------------
+  // Pipeline Operations Demo
+  // ------------------------------------------------------------------
+  std::cout << GREEN << "\n=== Pipeline Operations Demo ===" << RESET
+            << std::endl;
+
+  // Begin a new pipeline
+  if (!couchbase_ops.beginPipeline()) {
+    std::cout << RED << "Failed to begin pipeline" << RESET << std::endl;
+    return -1;
+  }
+
+  std::cout << "Pipeline started. Adding multiple operations..." << std::endl;
+
+  // Add multiple operations to the pipeline
+  std::string pipeline_key1 = "pipeline::doc1";
+  std::string pipeline_key2 = "pipeline::doc2";
+  std::string pipeline_key3 = "pipeline::doc3";
+  std::string pipeline_value1 = R"({"operation": "pipeline_add", "id": 1})";
+  std::string pipeline_value2 = R"({"operation": "pipeline_upsert", "id": 2})";
+  std::string pipeline_value3 = R"({"operation": "pipeline_add", "id": 3})";
+
+  // Pipeline operations - all prepared but not yet executed
+  bool pipeline_success = true;
+  pipeline_success &= couchbase_ops.pipelineRequest(
+      brpc::CouchbaseOperations::ADD, pipeline_key1, pipeline_value1);
+  pipeline_success &= couchbase_ops.pipelineRequest(
+      brpc::CouchbaseOperations::UPSERT, pipeline_key2, pipeline_value2);
+  pipeline_success &= couchbase_ops.pipelineRequest(
+      brpc::CouchbaseOperations::ADD, pipeline_key3, pipeline_value3);
+  pipeline_success &= couchbase_ops.pipelineRequest(
+      brpc::CouchbaseOperations::GET, pipeline_key1);
+  pipeline_success &= couchbase_ops.pipelineRequest(
+      brpc::CouchbaseOperations::GET, pipeline_key2);
+
+  if (!pipeline_success) {
+    std::cout << RED << "Failed to add operations to pipeline" << RESET
+              << std::endl;
+    couchbase_ops.clearPipeline();
+    return -1;
+  }
+
+  std::cout << "Added " << couchbase_ops.getPipelineSize()
+            << " operations to pipeline" << std::endl;
+
+  // Execute all operations in a single network call
+  std::cout << "Executing pipeline operations..." << std::endl;
+  std::vector<brpc::CouchbaseOperations::Result> pipeline_results =
+      couchbase_ops.executePipeline();
+
+  // Process results in order
+  std::cout << GREEN << "Pipeline execution completed. Results:" << RESET
+            << std::endl;
+  for (size_t i = 0; i < pipeline_results.size(); ++i) {
+    const auto& result = pipeline_results[i];
+    if (result.success) {
+      if (!result.value.empty()) {
+        std::cout << GREEN << "  Operation " << (i + 1)
+                  << " SUCCESS - Value: " << result.value << RESET << std::endl;
+      } else {
+        std::cout << GREEN << "  Operation " << (i + 1) << " SUCCESS" << RESET
+                  << std::endl;
+      }
+    } else {
+      std::cout << RED << "  Operation " << (i + 1)
+                << " FAILED: " << result.error_message << RESET << std::endl;
+    }
+  }
+
+  // Demonstrate pipeline with collection operations
+  std::cout << GREEN << "\n=== Pipeline with Collection Operations ===" << RESET
+            << std::endl;
+
+  if (!couchbase_ops.beginPipeline()) {
+    std::cout << RED << "Failed to begin collection pipeline" << RESET
+              << std::endl;
+    return -1;
+  }
+
+  std::string coll_pipeline_key1 = "coll_pipeline::doc1";
+  std::string coll_pipeline_key2 = "coll_pipeline::doc2";
+  std::string coll_pipeline_value1 =
+      R"({"collection_operation": "pipeline_add", "id": 1})";
+  std::string coll_pipeline_value2 =
+      R"({"collection_operation": "pipeline_upsert", "id": 2})";
+
+  // Add collection-scoped operations to pipeline
+  bool coll_pipeline_success = true;
+  coll_pipeline_success &= couchbase_ops.pipelineRequest(
+      brpc::CouchbaseOperations::ADD, coll_pipeline_key1, coll_pipeline_value1,
+      collection_name);
+  coll_pipeline_success &= couchbase_ops.pipelineRequest(
+      brpc::CouchbaseOperations::UPSERT, coll_pipeline_key2,
+      coll_pipeline_value2, collection_name);
+  coll_pipeline_success &= couchbase_ops.pipelineRequest(
+      brpc::CouchbaseOperations::GET, coll_pipeline_key1, "", collection_name);
+  coll_pipeline_success &=
+      couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::DELETE,
+                                    coll_pipeline_key1, "", collection_name);
+
+  if (!coll_pipeline_success) {
+    std::cout << RED << "Failed to add collection operations to pipeline"
+              << RESET << std::endl;
+    couchbase_ops.clearPipeline();
+    return -1;
+  }
+
+  // Execute collection pipeline
+  std::vector<brpc::CouchbaseOperations::Result> coll_pipeline_results =
+      couchbase_ops.executePipeline();
+
+  std::cout << GREEN
+            << "Collection pipeline execution completed. Results:" << RESET
+            << std::endl;
+  for (size_t i = 0; i < coll_pipeline_results.size(); ++i) {
+    const auto& result = coll_pipeline_results[i];
+    if (result.success) {
+      if (!result.value.empty()) {
+        std::cout << GREEN << "  Collection Operation " << (i + 1)
+                  << " SUCCESS - Value: " << result.value << RESET << std::endl;
+      } else {
+        std::cout << GREEN << "  Collection Operation " << (i + 1) << " SUCCESS"
+                  << RESET << std::endl;
+      }
+    } else {
+      std::cout << RED << "  Collection Operation " << (i + 1)
+                << " FAILED: " << result.error_message << RESET << std::endl;
+    }
+  }
+
+  // Clean up remaining pipeline documents
+  std::cout << GREEN << "\n=== Cleanup Pipeline Demo ===" << RESET << std::endl;
+  if (couchbase_ops.beginPipeline()) {
+    couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::DELETE,
+                                  pipeline_key1);
+    couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::DELETE,
+                                  pipeline_key2);
+    couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::DELETE,
+                                  pipeline_key3);
+    couchbase_ops.pipelineRequest(brpc::CouchbaseOperations::DELETE,
+                                  coll_pipeline_key2, "", collection_name);
+
+    std::vector<brpc::CouchbaseOperations::Result> cleanup_results =
+        couchbase_ops.executePipeline();
+    std::cout << "Cleanup completed (" << cleanup_results.size()
+              << " operations)" << std::endl;
+  }
+
+  std::cout << GREEN
+            << "\n=== All operations completed successfully! ===" << RESET
+            << std::endl;
+}
+int main() {
+  // Create CouchbaseOperations instance for high-level operations
+  brpc::CouchbaseOperations couchbase_ops;
+
+  // std::cout << GREEN << "Using high-level CouchbaseOperations interface"
+  //           << RESET << std::endl;
+
+  // Ask username and password for authentication
+  std::string username = "Administrator";
+  std::string password = "password";
+  while (username.empty() || password.empty()) {
+    std::cout << "Enter Couchbase username: ";
+    std::cin >> username;
+    if (username.empty()) {
+      std::cout << "Username cannot be empty. Please enter again." << std::endl;
+      continue;
+    }
+    std::cout << "Enter Couchbase password: ";
+    std::cin >> password;
+    if (password.empty()) {
+      std::cout << "Password cannot be empty. Please enter again." << std::endl;
+      continue;
+    }
+  }
+
+  // Use high-level authentication method
+  // when connecting to capella use couchbase_ops.authenticate(username,
+  // password, FLAGS_server, true, "path/to/cert.txt");
+  brpc::CouchbaseOperations::Result auth_result =
+      couchbase_ops.authenticate(username, password, FLAGS_server, "testing");
+  if (!auth_result.success) {
+    LOG(ERROR) << "Authentication failed: " << auth_result.error_message;
+    return -1;
+  }
+
+  std::cout
+      << GREEN
+      << "Authentication successful, proceeding with Couchbase operations..."
+      << RESET << std::endl;
+
+  performOperations(couchbase_ops);
+
+  // Change bucket Selection
+  std::string bucket_name = "testing";
+  while (bucket_name.empty()) {
+    std::cout << "Enter Couchbase bucket name: ";
+    std::cin >> bucket_name;
+    if (bucket_name.empty()) {
+      std::cout << "Bucket name cannot be empty. Please enter again."
+                << std::endl;
+      continue;
+    }
+  }
+
+  // Use high-level bucket selection method
+  brpc::CouchbaseOperations::Result bucket_result =
+      couchbase_ops.selectBucket(bucket_name);
+  if (!bucket_result.success) {
+    LOG(ERROR) << "Bucket selection failed: " << bucket_result.error_message;
+    return -1;
+  } else {
+    std::cout << GREEN << "Bucket Selection Successful" << RESET << std::endl;
+  }
+  // Add operation using high-level method
+  performOperations(couchbase_ops);
+  return 0;
+}

example/couchbase_c++/multithreaded_couchbase_client.cpp

@@ -0,0 +1,375 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <brpc/couchbase.h>
+#include <bthread/bthread.h>
+#include <butil/logging.h>
+#include <butil/string_printf.h>
+
+#include <atomic>
+#include <iostream>
+#include <string>
+#include <vector>
+
+// ANSI color codes
+#define GREEN "\033[32m"
+#define RED "\033[31m"
+#define BLUE "\033[34m"
+#define YELLOW "\033[33m"
+#define CYAN "\033[36m"
+#define RESET "\033[0m"
+
+const int NUM_THREADS = 20;
+const int THREADS_PER_BUCKET = 5;
+
+// Simple global config
+struct {
+  std::string username = "Administrator";
+  std::string password = "password";
+  std::vector<std::string> bucket_names = {"t0", "t1", "t2", "t3"};
+} g_config;
+
+// Simple thread statistics
+struct ThreadStats {
+  std::atomic<int> operations_attempted{0};
+  std::atomic<int> operations_successful{0};
+  std::atomic<int> operations_failed{0};
+
+  void reset() {
+    operations_attempted = 0;
+    operations_successful = 0;
+    operations_failed = 0;
+  }
+};
+
+// Global statistics
+struct GlobalStats {
+  ThreadStats total;
+  std::vector<ThreadStats> per_thread_stats;
+
+  GlobalStats() : per_thread_stats(NUM_THREADS) {}
+
+  void aggregate_stats() {
+    total.reset();
+    for (const auto& stats : per_thread_stats) {
+      total.operations_attempted += stats.operations_attempted.load();
+      total.operations_successful += stats.operations_successful.load();
+      total.operations_failed += stats.operations_failed.load();
+    }
+  }
+} g_stats;
+
+// Simple thread arguments
+struct ThreadArgs {
+  int thread_id;
+  int bucket_id;
+  std::string bucket_name;
+  brpc::CouchbaseOperations* couchbase_ops;
+  ThreadStats* stats;
+};
+
+// Simple CRUD operations on default collection
+void perform_crud_operations_default(brpc::CouchbaseOperations& couchbase_ops,
+                                     const std::string& base_key,
+                                     ThreadStats* stats) {
+  std::string key = base_key + "_default";
+  std::string value = butil::string_printf(
+      R"({"thread_id": %d, "collection": "default"})", (int)bthread_self());
+
+  stats->operations_attempted++;
+
+  // UPSERT
+  brpc::CouchbaseOperations::Result result = couchbase_ops.upsert(key, value);
+  if (result.success) {
+    stats->operations_successful++;
+  } else {
+    stats->operations_failed++;
+    return;
+  }
+
+  stats->operations_attempted++;
+
+  // GET
+  result = couchbase_ops.get(key);
+  if (result.success) {
+    stats->operations_successful++;
+  } else {
+    stats->operations_failed++;
+    return;
+  }
+
+  stats->operations_attempted++;
+
+  // DELETE
+  result = couchbase_ops.delete_(key);
+  if (result.success) {
+    stats->operations_successful++;
+  } else {
+    stats->operations_failed++;
+  }
+}
+
+// Simple CRUD operations on col1 collection
+void perform_crud_operations_col1(brpc::CouchbaseOperations& couchbase_ops,
+                                  const std::string& base_key,
+                                  ThreadStats* stats) {
+  std::string key = base_key + "_col1";
+  std::string value = butil::string_printf(
+      R"({"thread_id": %d, "collection": "col1"})", (int)bthread_self());
+
+  stats->operations_attempted++;
+
+  // UPSERT
+  brpc::CouchbaseOperations::Result result =
+      couchbase_ops.upsert(key, value, "col1");
+  if (result.success) {
+    stats->operations_successful++;
+  } else {
+    stats->operations_failed++;
+    std::cout << "UPSERT failed: " << result.error_message << std::endl;
+    return;
+  }
+
+  stats->operations_attempted++;
+
+  // GET
+  result = couchbase_ops.get(key, "col1");
+  if (result.success) {
+    stats->operations_successful++;
+  } else {
+    stats->operations_failed++;
+    std::cout << "GET failed: " << result.error_message << std::endl;
+    return;
+  }
+
+  stats->operations_attempted++;
+
+  // DELETE
+  result = couchbase_ops.delete_(key, "col1");
+  if (result.success) {
+    stats->operations_successful++;
+  } else {
+    stats->operations_failed++;
+  }
+}
+
+// Simple thread worker function
+void* thread_worker(void* arg) {
+  ThreadArgs* args = static_cast<ThreadArgs*>(arg);
+
+  std::cout << CYAN << "Thread " << args->thread_id << " starting on bucket "
+            << args->bucket_name << RESET << std::endl;
+
+  // Create CouchbaseOperations instance for this thread
+  brpc::CouchbaseOperations couchbase_ops;
+
+  // Authentication
+  brpc::CouchbaseOperations::Result auth_result = couchbase_ops.authenticate(
+      g_config.username, g_config.password, "127.0.0.1:11210", args->bucket_name);
+  // for SSL authentication use below line instead
+  // brpc::CouchbaseOperations::Result auth_result = couchbase_ops.authenticateSSL(username, password, "127.0.0.1:11210", args->bucket_name, "/path/to/cert.txt");
+  
+  if (!auth_result.success) {
+    std::cout << RED << "Thread " << args->thread_id << ": Auth failed - "
+              << auth_result.error_message << RESET << std::endl;
+    return NULL;
+  }
+
+    // // Select bucket
+    // brpc::CouchbaseOperations::Result bucket_result =
+    //     couchbase_ops.selectBucket(args->bucket_name);
+
+    // if (!bucket_result.success) {
+    //   std::cout << RED << "Thread " << args->thread_id
+    //             << ": Bucket selection failed - " << bucket_result.error_message
+    //             << RESET << std::endl;
+    //   return NULL;
+    // }
+
+  std::cout << GREEN << "Thread " << args->thread_id << " connected to bucket "
+            << args->bucket_name << RESET << std::endl;
+
+  // Perform operations - 10 times on default collection, 10 times on col1
+  // collection
+  for (int i = 0; i < 10; ++i) {
+    std::string base_key =
+        butil::string_printf("thread_%d_op_%d", args->thread_id, i);
+
+    // CRUD operations on default collection
+    perform_crud_operations_default(couchbase_ops, base_key, args->stats);
+
+    // CRUD operations on col1 collection
+    perform_crud_operations_col1(couchbase_ops, base_key, args->stats);
+
+    // Small delay between operations
+    bthread_usleep(10000);  // 10ms
+  }
+
+  int successful = args->stats->operations_successful.load();
+  int attempted = args->stats->operations_attempted.load();
+  int failed = args->stats->operations_failed.load();
+
+  std::cout << GREEN << "Thread " << args->thread_id
+            << " completed: " << successful << "/" << attempted
+            << " operations successful, " << failed << " failed" << RESET
+            << std::endl;
+
+  return NULL;
+}
+
+void* shared_object_thread_worker(void *arg){
+    ThreadArgs* shared_args = static_cast<ThreadArgs*>(arg);
+    brpc::CouchbaseOperations* shared_couchbase_ops = shared_args->couchbase_ops;
+    // Perform operations - 10 times on default collection, 10 times on col1
+    // collection
+    for (int i = 0; i < 10; ++i) {
+        std::string base_key =
+            butil::string_printf("shared_thread_op_%d_thread_id_%d", i, shared_args->thread_id);
+        // CRUD operations on default collection
+        perform_crud_operations_default(*shared_couchbase_ops, base_key, shared_args->stats);
+        // CRUD operations on col1 collection
+        perform_crud_operations_col1(*shared_couchbase_ops, base_key, shared_args->stats);
+        // Small delay between operations
+        bthread_usleep(10000);  // 10ms
+    }
+    return NULL;
+}
+
+// Print simple statistics
+void print_stats() {
+  g_stats.aggregate_stats();
+
+  std::cout << std::endl;
+  std::cout << CYAN << "=== TEST RESULTS ===" << RESET << std::endl;
+
+  int total_attempted = g_stats.total.operations_attempted.load();
+  int total_successful = g_stats.total.operations_successful.load();
+  int total_failed = g_stats.total.operations_failed.load();
+
+  double success_rate = total_attempted > 0
+                            ? (double)total_successful / total_attempted * 100.0
+                            : 0.0;
+
+  std::cout << GREEN << "Overall Performance:" << RESET << std::endl;
+  std::cout << "  Total Operations: " << total_attempted << std::endl;
+  std::cout << "  Successful: " << total_successful << " (" << success_rate
+            << "%)" << std::endl;
+  std::cout << "  Failed: " << total_failed << std::endl;
+  std::cout << std::endl;
+
+  // Per-thread breakdown
+  std::cout << YELLOW << "Per-Thread Performance:" << RESET << std::endl;
+  for (int i = 0; i < NUM_THREADS; ++i) {
+    const auto& stats = g_stats.per_thread_stats[i];
+    int attempted = stats.operations_attempted.load();
+    int successful = stats.operations_successful.load();
+    int failed = stats.operations_failed.load();
+
+    std::cout << "  Thread " << i << ": " << attempted << " ops, " << successful
+              << " success, " << failed << " failed" << std::endl;
+  }
+  std::cout << std::endl;
+}
+
+int main(int argc, char* argv[]) {
+  std::cout << GREEN << "Starting Simple Multithreaded Couchbase Client"
+            << RESET << std::endl;
+  std::cout
+      << YELLOW
+      << "20 threads: 5 per bucket (testing0, testing1, testing2, testing3)"
+      << RESET << std::endl;
+  std::cout << BLUE
+            << "Each thread performs CRUD operations on default collection and "
+               "col1 collection"
+            << RESET << std::endl;
+
+  // Create threads and arguments
+  std::vector<bthread_t> threads(NUM_THREADS);
+  std::vector<ThreadArgs> args(NUM_THREADS);
+
+  // Assign threads to buckets
+  for (int i = 0; i < NUM_THREADS; ++i) {
+    args[i].thread_id = i;
+    args[i].bucket_id = i / THREADS_PER_BUCKET;
+    args[i].bucket_name = g_config.bucket_names[args[i].bucket_id];
+    args[i].stats = &g_stats.per_thread_stats[i];
+  }
+
+  // Print thread assignments
+  std::cout << "Thread Assignments:" << RESET << std::endl;
+  for (int i = 0; i < NUM_THREADS; ++i) {
+    std::cout << "Thread " << i << " -> Bucket: " << args[i].bucket_name
+              << std::endl;
+  }
+  std::cout << std::endl;
+
+  // Start all threads
+  for (int i = 0; i < NUM_THREADS; ++i) {
+    if (bthread_start_background(&threads[i], NULL, thread_worker, &args[i]) !=
+        0) {
+      std::cout << RED << "Failed to create thread " << i << RESET << std::endl;
+      return -1;
+    }
+  }
+
+  std::cout << GREEN << "All 20 threads started!" << RESET << std::endl;
+
+  // Wait for all threads to complete
+  std::cout << YELLOW << "Waiting for all threads to complete..." << RESET
+            << std::endl;
+  for (int i = 0; i < NUM_THREADS; ++i) {
+    bthread_join(threads[i], NULL);
+  }
+
+  std::cout << GREEN << "All threads completed!" << RESET << std::endl;
+  // create a shared CouchbaseOperations instance
+  brpc::CouchbaseOperations shared_couchbase_ops;
+  brpc::CouchbaseOperations::Result result;
+  result = shared_couchbase_ops.authenticate(
+      g_config.username, g_config.password, "127.0.0.1:11210", "t0");
+  if(result.success){
+      std::cout << GREEN << "Shared CouchbaseOperations instance authenticated successfully!" << RESET << std::endl;
+  } else {
+      std::cout << RED << "Shared CouchbaseOperations instance authentication failed: " << result.error_message << RESET << std::endl;
+      return -1;
+  }
+
+  for (int i = 0; i < NUM_THREADS; ++i) {
+    args[i].thread_id = i;
+    args[i].couchbase_ops = &shared_couchbase_ops;
+    args[i].bucket_id = 0;
+    args[i].bucket_name = "t0";
+    // args[i].stats = &g_stats.per_thread_stats[i];
+  }
+
+  for(int i = 0; i < NUM_THREADS; ++i){
+      if (bthread_start_background(&threads[i], NULL, shared_object_thread_worker, &args[i]) !=
+        0) {
+      std::cout << RED << "Failed to create shared object thread " << i << RESET << std::endl;
+      return -1;
+    }
+  }
+  for(int i = 0; i < NUM_THREADS; ++i){
+      bthread_join(threads[i], NULL);
+  }
+  std::cout << GREEN << "All shared object threads completed!" << RESET << std::endl;
+
+  // Print statistics
+  print_stats();
+
+  return 0;
+}

example/couchbase_c++/traditional_brpc_couchbase_client.cpp

@@ -0,0 +1,171 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <brpc/couchbase.h>
+
+#include <iostream>
+#include <string>
+
+// ANSI color codes for console output
+#define GREEN "\033[32m"
+#define RED "\033[31m"
+#define RESET "\033[0m"
+
+int main() {
+  // traditional bRPC Couchbase client
+  brpc::Channel channel;
+  brpc::ChannelOptions options;
+  options.protocol = brpc::PROTOCOL_COUCHBASE;
+  options.connection_type = "single";
+  options.timeout_ms = 1000;  // 1 second
+  options.max_retry = 3;
+  if (channel.Init("localhost:11210", &options) != 0) {
+    LOG(ERROR) << "Failed to initialize channel";
+    return -1;
+  }
+  brpc::Controller cntl;
+  brpc::CouchbaseOperations::CouchbaseRequest req;
+  brpc::CouchbaseOperations::CouchbaseResponse res;
+  uint64_t cas;
+  req.authenticateRequest("Administrator", "password");
+  channel.CallMethod(NULL, &cntl, &req, &res, NULL);
+  if (cntl.Failed()) {
+    LOG(ERROR) << "Unable to authenticate: Something went wrong"
+               << cntl.ErrorText();
+    return -1;
+  } else {
+    if (res.popHello(&cas) && res.popAuthenticate(&cas)) {
+      std::cout << "Traditional bRPC Couchbase Client Authentication Successful"
+                << std::endl;
+    } else {
+      std::cout << "Client Authentication Failed with status code: " << std::hex
+                << res._status_code << std::endl;
+      return -1;
+    }
+  }
+  cntl.Reset();
+  // clearing request and response
+
+  req.Clear();
+  res.Clear();
+  req.selectBucketRequest("testing");
+  channel.CallMethod(NULL, &cntl, &req, &res, NULL);
+  if (cntl.Failed()) {
+    LOG(ERROR) << "Unable to select bucket: Something went wrong"
+               << cntl.ErrorText();
+    return -1;
+  } else {
+    if (res.popSelectBucket(&cas)) {
+      std::cout
+          << "Traditional bRPC Couchbase Client Bucket Selection Successful"
+          << std::endl;
+    } else {
+      // the status code will be updated only after you do
+      // popFunctionName(param).
+      std::cout << "Traditional bRPC Couchbase Client Bucket Selection Failed "
+                   "with status code: "
+                << std::hex << res._status_code << std::endl;
+      std::cout << "Error Message: " << res.lastError() << std::endl;
+      return -1;
+    }
+  }
+  cntl.Reset();
+  // clearing request and response
+
+  req.Clear();
+  res.Clear();
+  req.addRequest(
+      "sample_key",
+      R"({"name": "John Doe", "age": 30, "email": "john@example.com"})",
+      0 /*flags*/, 0 /*exptime*/, 0 /*cas*/);
+  channel.CallMethod(NULL, &cntl, &req, &res, NULL);
+  if (cntl.Failed()) {
+    LOG(ERROR) << "Unable to add key-value: Something went wrong"
+               << cntl.ErrorText();
+    return -1;
+  } else {
+    if (res.popAdd(&cas)) {
+      std::cout
+          << "Traditional bRPC Couchbase Client Key-Value Addition Successful"
+          << std::endl;
+    } else {
+      // the status code will be updated only after you do
+      // popFunctionName(param).
+      std::cout << "Traditional bRPC Couchbase Client Key-Value Addition "
+                   "Failed with status code: "
+                << std::hex << res._status_code << std::endl;
+      std::cout << "Error Message: " << res.lastError() << std::endl;
+      return -1;
+    }
+  }
+  cntl.Reset();
+
+  // clearing request and response before doing a getRequest
+  req.Clear();
+  res.Clear();
+  req.getRequest("sample_key");
+  channel.CallMethod(NULL, &cntl, &req, &res, NULL);
+  if (cntl.Failed()) {
+    LOG(ERROR) << "Unable to get value for key: Something went wrong"
+               << cntl.ErrorText();
+    return -1;
+  } else {
+    std::string value;
+    uint32_t flags;
+    if (res.popGet(&value, &flags, &cas)) {
+      std::cout
+          << "Traditional bRPC Couchbase Client Key-Value Retrieval Successful"
+          << std::endl;
+      std::cout << "Retrieved Value: " << value << std::endl;
+    } else {
+      // note the status code will be updated only after you do
+      //  popFunctionName(param).
+      std::cout << "Traditional bRPC Couchbase Client Key-Value Retrieval "
+                   "Failed with status code: "
+                << std::hex << res._status_code << std::endl;
+      std::cout << "Error Message: " << res.lastError() << std::endl;
+      return -1;
+    }
+  }
+  cntl.Reset();
+  // clearing request and response before doing a deleteRequest
+
+  req.Clear();
+  res.Clear();
+  req.deleteRequest("sample_key");
+  channel.CallMethod(NULL, &cntl, &req, &res, NULL);
+  if (cntl.Failed()) {
+    LOG(ERROR) << "Unable to delete key-value: Something went wrong"
+               << cntl.ErrorText();
+    return -1;
+  } else {
+    if (res.popDelete()) {
+      std::cout
+          << "Traditional bRPC Couchbase Client Key-Value Deletion Successful"
+          << std::endl;
+    } else {
+      // the status code will be updated only after you do
+      // popFunctionName(param).
+      std::cout << "Traditional bRPC Couchbase Client Key-Value Deletion "
+                   "Failed with status code: "
+                << std::hex << res._status_code << std::endl;
+      std::cout << "Error Message: " << res.lastError() << std::endl;
+      return -1;
+    }
+  }
+  return 0;
+}

example/dynamic_partition_echo_c++/CMakeLists.txt

@@ -0,0 +1,118 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.16...3.28)
+project(dynamic_partition_echo_c++ C CXX)
+
+include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake)
+
+option(LINK_SO "Whether examples are linked dynamically" OFF)
+
+execute_process(
+    COMMAND bash -c "find ${PROJECT_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'"
+    OUTPUT_VARIABLE OUTPUT_PATH
+)
+
+if(OUTPUT_PATH)
+    list(PREPEND CMAKE_PREFIX_PATH ${OUTPUT_PATH})
+endif()
+
+find_package(Threads REQUIRED)
+find_package(Protobuf REQUIRED)
+protobuf_generate_cpp(PROTO_SRC PROTO_HEADER echo.proto)
+
+find_path(GPERFTOOLS_INCLUDE_DIR NAMES gperftools/heap-profiler.h)
+find_library(GPERFTOOLS_LIBRARIES NAMES tcmalloc_and_profiler)
+
+# Search for libthrift* by best effort. If it is not found and brpc is
+# compiled with thrift protocol enabled, a link error would be reported.
+find_library(THRIFT_LIB NAMES thrift)
+if (NOT THRIFT_LIB)
+    set(THRIFT_LIB "")
+endif()
+
+find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h)
+if(LINK_SO)
+    find_library(BRPC_LIB NAMES brpc)
+else()
+    find_library(BRPC_LIB NAMES libbrpc.a brpc)
+endif()
+if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB))
+    message(FATAL_ERROR "Fail to find brpc")
+endif()
+
+find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h)
+find_library(GFLAGS_LIBRARY NAMES gflags libgflags)
+if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY))
+    message(FATAL_ERROR "Fail to find gflags")
+endif()
+
+set(BRPC_EXAMPLE_ENABLE_CPU_PROFILER ON)
+
+find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h)
+find_library(LEVELDB_LIB NAMES leveldb)
+if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB))
+    message(FATAL_ERROR "Fail to find leveldb")
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT OPENSSL_ROOT_DIR)
+    set(OPENSSL_ROOT_DIR
+        "/usr/local/opt/openssl"    # Homebrew installed OpenSSL
+        )
+endif()
+
+find_package(OpenSSL REQUIRED)
+
+set(DYNAMIC_LIB
+    Threads::Threads
+    ${GFLAGS_LIBRARY}
+    ${PROTOBUF_LIBRARIES}
+    ${LEVELDB_LIB}
+    ${OPENSSL_CRYPTO_LIBRARY}
+    ${OPENSSL_SSL_LIBRARY}
+    ${THRIFT_LIB}
+    dl
+    )
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    set(DYNAMIC_LIB ${DYNAMIC_LIB}
+        pthread
+        "-framework CoreFoundation"
+        "-framework CoreGraphics"
+        "-framework CoreData"
+        "-framework CoreText"
+        "-framework Security"
+        "-framework Foundation"
+        "-Wl,-U,_MallocExtension_ReleaseFreeMemory"
+        "-Wl,-U,_ProfilerStart"
+        "-Wl,-U,_ProfilerStop"
+        "-Wl,-U,__Z13GetStackTracePPvii"
+        "-Wl,-U,_mallctl"
+        "-Wl,-U,_malloc_stats_print"
+    )
+endif()
+
+add_executable(dynamic_partition_echo_client client.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(dynamic_partition_echo_client)
+add_executable(dynamic_partition_echo_server server.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(dynamic_partition_echo_server)
+
+target_link_libraries(dynamic_partition_echo_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB} ${GPERFTOOLS_LIBRARIES})
+target_link_libraries(dynamic_partition_echo_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB} ${GPERFTOOLS_LIBRARIES})
+
+file(COPY ${PROJECT_SOURCE_DIR}/server_list
+     DESTINATION ${CMAKE_CURRENT_BINARY_DIR})

example/dynamic_partition_echo_c++/client.cpp

@@ -0,0 +1,213 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A client sending requests to server in parallel by multiple threads.
+
+#include <gflags/gflags.h>
+#include <bthread/bthread.h>
+#include <butil/logging.h>
+#include <butil/string_printf.h>
+#include <butil/time.h>
+#include <butil/macros.h>
+#include <brpc/partition_channel.h>
+#include <deque>
+#include "echo.pb.h"
+
+DEFINE_int32(thread_num, 50, "Number of threads to send requests");
+DEFINE_bool(use_bthread, false, "Use bthread to send requests");
+DEFINE_int32(attachment_size, 0, "Carry so many byte attachment along with requests");
+DEFINE_int32(request_size, 16, "Bytes of each request");
+DEFINE_string(connection_type, "", "Connection type. Available values: single, pooled, short");
+DEFINE_string(protocol, "baidu_std", "Protocol type. Defined in src/brpc/options.proto");
+DEFINE_string(server, "file://server_list", "Mapping to servers");
+DEFINE_string(load_balancer, "rr", "Name of load balancer");
+DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); 
+DEFINE_bool(dont_fail, false, "Print fatal when some call failed");
+
+std::string g_request;
+std::string g_attachment;
+pthread_mutex_t g_latency_mutex = PTHREAD_MUTEX_INITIALIZER;
+struct BAIDU_CACHELINE_ALIGNMENT SenderInfo {
+    size_t nsuccess;
+    int64_t latency_sum;
+};
+std::deque<SenderInfo> g_sender_info;
+
+static void* sender(void* arg) {
+    // Normally, you should not call a Channel directly, but instead construct
+    // a stub Service wrapping it. stub can be shared by all threads as well.
+    example::EchoService_Stub stub(static_cast<google::protobuf::RpcChannel*>(arg));
+
+    SenderInfo* info = NULL;
+    {
+        BAIDU_SCOPED_LOCK(g_latency_mutex);
+        g_sender_info.push_back(SenderInfo());
+        info = &g_sender_info.back();
+    }
+
+    int log_id = 0;
+    while (!brpc::IsAskedToQuit()) {
+        // We will receive response synchronously, safe to put variables
+        // on stack.
+        example::EchoRequest request;
+        example::EchoResponse response;
+        brpc::Controller cntl;
+
+        request.set_message(g_request);
+        cntl.set_log_id(log_id++);  // set by user
+        if (!g_attachment.empty()) {
+            // Set attachment which is wired to network directly instead of 
+            // being serialized into protobuf messages.
+            cntl.request_attachment().append(g_attachment);
+        }
+
+        // Because `done'(last parameter) is NULL, this function waits until
+        // the response comes back or error occurs(including timedout).
+        stub.Echo(&cntl, &request, &response, NULL);
+        if (!cntl.Failed()) {
+            info->latency_sum += cntl.latency_us();
+            ++info->nsuccess;
+        } else {
+            CHECK(brpc::IsAskedToQuit() || !FLAGS_dont_fail)
+                << "error=" << cntl.ErrorText() << " latency=" << cntl.latency_us();
+            // We can't connect to the server, sleep a while. Notice that this
+            // is a specific sleeping to prevent this thread from spinning too
+            // fast. You should continue the business logic in a production 
+            // server rather than sleeping.
+            bthread_usleep(50000);
+        }
+    }
+    return NULL;
+}
+
+class MyPartitionParser : public brpc::PartitionParser {
+public:
+    bool ParseFromTag(const std::string& tag, brpc::Partition* out) {
+        // "N/M" : #N partition of M partitions.
+        size_t pos = tag.find_first_of('/');
+        if (pos == std::string::npos) {
+            LOG(ERROR) << "Invalid tag=" << tag;
+            return false;
+        }
+        char* endptr = NULL;
+        out->index = strtol(tag.c_str(), &endptr, 10);
+        if (endptr != tag.data() + pos) {
+            LOG(ERROR) << "Invalid index=" << butil::StringPiece(tag.data(), pos);
+            return false;
+        }
+        out->num_partition_kinds = strtol(tag.c_str() + pos + 1, &endptr, 10);
+        if (endptr != tag.c_str() + tag.size()) {
+            LOG(ERROR) << "Invalid num=" << tag.data() + pos + 1;
+            return false;
+        }
+        return true;
+    }
+};
+
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    // A Channel represents a communication line to a Server. Notice that 
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::DynamicPartitionChannel channel;
+
+    brpc::PartitionChannelOptions options;
+    options.protocol = FLAGS_protocol;
+    options.connection_type = FLAGS_connection_type;
+    options.succeed_without_server = true;
+    options.fail_limit = 1;
+    options.timeout_ms = FLAGS_timeout_ms/*milliseconds*/;
+    options.max_retry = FLAGS_max_retry;
+
+    if (channel.Init(new MyPartitionParser(),
+                     FLAGS_server.c_str(), FLAGS_load_balancer.c_str(),
+                     &options) != 0) {
+        LOG(ERROR) << "Fail to init channel";
+        return -1;
+    }
+    if (FLAGS_attachment_size > 0) {
+        g_attachment.resize(FLAGS_attachment_size, 'a');
+    }
+    if (FLAGS_request_size <= 0) {
+        LOG(ERROR) << "Bad request_size=" << FLAGS_request_size;
+        return -1;
+    }
+    g_request.resize(FLAGS_request_size, 'r');
+
+    std::vector<bthread_t> bids;
+    std::vector<pthread_t> pids;
+    if (!FLAGS_use_bthread) {
+        pids.resize(FLAGS_thread_num);
+        for (int i = 0; i < FLAGS_thread_num; ++i) {
+            if (pthread_create(&pids[i], NULL, sender, &channel) != 0) {
+                LOG(ERROR) << "Fail to create pthread";
+                return -1;
+            }
+        }
+    } else {
+        bids.resize(FLAGS_thread_num);
+        for (int i = 0; i < FLAGS_thread_num; ++i) {
+            if (bthread_start_background(
+                    &bids[i], NULL, sender, &channel) != 0) {
+                LOG(ERROR) << "Fail to create bthread";
+                return -1;
+            }
+        }
+    }
+
+    int64_t last_counter = 0;
+    int64_t last_latency_sum = 0;
+    std::vector<size_t> last_nsuccess(FLAGS_thread_num);
+    while (!brpc::IsAskedToQuit()) {
+        sleep(1);
+        int64_t latency_sum = 0;
+        int64_t nsuccess = 0;
+        pthread_mutex_lock(&g_latency_mutex);
+        CHECK_EQ(g_sender_info.size(), (size_t)FLAGS_thread_num);
+        for (size_t i = 0; i < g_sender_info.size(); ++i) {
+            const SenderInfo& info = g_sender_info[i];
+            latency_sum += info.latency_sum;
+            nsuccess += info.nsuccess;
+            if (FLAGS_dont_fail) {
+                CHECK(info.nsuccess > last_nsuccess[i]) << "i=" << i;
+            }
+            last_nsuccess[i] = info.nsuccess;
+        }
+        pthread_mutex_unlock(&g_latency_mutex);
+
+        const int64_t avg_latency = (latency_sum - last_latency_sum) /
+            std::max(nsuccess - last_counter, (int64_t)1);
+        LOG(INFO) << "Sending EchoRequest at qps=" << nsuccess - last_counter
+                  << " latency=" << avg_latency;
+        last_counter = nsuccess;
+        last_latency_sum = latency_sum;
+    }
+
+    LOG(INFO) << "EchoClient is going to quit";
+    for (int i = 0; i < FLAGS_thread_num; ++i) {
+        if (!FLAGS_use_bthread) {
+            pthread_join(pids[i], NULL);
+        } else {
+            bthread_join(bids[i], NULL);
+        }
+    }
+
+    return 0;
+}

example/dynamic_partition_echo_c++/echo.proto

@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+syntax="proto2";
+package example;
+
+option cc_generic_services = true;
+
+message EchoRequest {
+      required string message = 1;
+};
+
+message EchoResponse {
+      required string message = 1;
+};
+
+service EchoService {
+      rpc Echo(EchoRequest) returns (EchoResponse);
+};

example/dynamic_partition_echo_c++/server.cpp

@@ -0,0 +1,173 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A server to receive EchoRequest and send back EchoResponse.
+
+#include <vector>
+#include <gflags/gflags.h>
+#include <butil/time.h>
+#include <butil/logging.h>
+#include <butil/string_printf.h>
+#include <butil/string_splitter.h>
+#include <butil/rand_util.h>
+#include <brpc/server.h>
+#include "echo.pb.h"
+
+DEFINE_bool(echo_attachment, true, "Echo attachment as well");
+DEFINE_int32(port, 8004, "TCP Port of this server");
+DEFINE_int32(idle_timeout_s, -1, "Connection will be closed if there is no "
+             "read/write operations during the last `idle_timeout_s'");
+DEFINE_int32(max_concurrency, 0, "Limit of request processing in parallel");
+DEFINE_int32(server_num, 1, "Number of servers");
+DEFINE_string(sleep_us, "", "Sleep so many microseconds before responding");
+DEFINE_bool(spin, false, "spin rather than sleep");
+DEFINE_double(exception_ratio, 0.1, "Percentage of irregular latencies");
+DEFINE_double(min_ratio, 0.2, "min_sleep / sleep_us");
+DEFINE_double(max_ratio, 10, "max_sleep / sleep_us");
+
+// Your implementation of example::EchoService
+class EchoServiceImpl : public example::EchoService {
+public:
+    EchoServiceImpl() : _index(0) {}
+    virtual ~EchoServiceImpl() {}
+    void set_index(size_t index, int64_t sleep_us) { 
+        _index = index; 
+        _sleep_us = sleep_us;
+    }
+    virtual void Echo(google::protobuf::RpcController* cntl_base,
+                      const example::EchoRequest* request,
+                      example::EchoResponse* response,
+                      google::protobuf::Closure* done) {
+        brpc::ClosureGuard done_guard(done);
+        brpc::Controller* cntl =
+            static_cast<brpc::Controller*>(cntl_base);
+        if (_sleep_us > 0) {
+            double delay = _sleep_us;
+            const double a = FLAGS_exception_ratio * 0.5;
+            if (a >= 0.0001) {
+                double x = butil::RandDouble();
+                if (x < a) {
+                    const double min_sleep_us = FLAGS_min_ratio * _sleep_us;
+                    delay = min_sleep_us + (_sleep_us - min_sleep_us) * x / a;
+                } else if (x + a > 1) {
+                    const double max_sleep_us = FLAGS_max_ratio * _sleep_us;
+                    delay = _sleep_us + (max_sleep_us - _sleep_us) * (x + a - 1) / a;
+                }
+            }
+            if (FLAGS_spin) {
+                int64_t end_time = butil::cpuwide_time_us() + (int64_t)delay;
+                while (butil::cpuwide_time_us() < end_time) {}
+            } else {
+                bthread_usleep((int64_t)delay);
+            }
+        }
+
+        // Echo request and its attachment
+        response->set_message(request->message());
+        if (FLAGS_echo_attachment) {
+            cntl->response_attachment().append(cntl->request_attachment());
+        }
+        _nreq << 1;
+    }
+
+    size_t num_requests() const { return _nreq.get_value(); }
+
+private:
+    size_t _index;
+    int64_t _sleep_us;
+    bvar::Adder<size_t> _nreq;
+};
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    if (FLAGS_server_num <= 0) {
+        LOG(ERROR) << "server_num must be positive";
+        return -1;
+    }
+
+    // We need multiple servers in this example.
+    brpc::Server* servers = new brpc::Server[FLAGS_server_num];
+    // For more options see `brpc/server.h'.
+    brpc::ServerOptions options;
+    options.idle_timeout_sec = FLAGS_idle_timeout_s;
+    options.max_concurrency = FLAGS_max_concurrency;
+
+    butil::StringSplitter sp(FLAGS_sleep_us.c_str(), ',');
+    std::vector<int64_t> sleep_list;
+    for (; sp; ++sp) {
+        sleep_list.push_back(strtoll(sp.field(), NULL, 10));
+    }
+    if (sleep_list.empty()) {
+        sleep_list.push_back(0);
+    }
+
+    // Instance of your services.
+    EchoServiceImpl* echo_service_impls = new EchoServiceImpl[FLAGS_server_num];
+    // Add the service into servers. Notice the second parameter, because the
+    // service is put on stack, we don't want server to delete it, otherwise
+    // use brpc::SERVER_OWNS_SERVICE.
+    for (int i = 0; i < FLAGS_server_num; ++i) {
+        int64_t sleep_us = sleep_list[(size_t)i < sleep_list.size() ? i : (sleep_list.size() - 1)];
+        echo_service_impls[i].set_index(i, sleep_us);
+        // will be shown on /version page
+        servers[i].set_version(butil::string_printf(
+                    "example/dynamic_partition_echo_c++[%d]", i));
+        if (servers[i].AddService(&echo_service_impls[i], 
+                                  brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
+            LOG(ERROR) << "Fail to add service";
+            return -1;
+        }
+        // Start the server.
+        int port = FLAGS_port + i;
+        if (servers[i].Start(port, &options) != 0) {
+            LOG(ERROR) << "Fail to start EchoServer";
+            return -1;
+        }
+    }
+
+    // Service logic are running in separate worker threads, for main thread,
+    // we don't have much to do, just spinning.
+    std::vector<size_t> last_num_requests(FLAGS_server_num);
+    while (!brpc::IsAskedToQuit()) {
+        sleep(1);
+
+        size_t cur_total = 0;
+        for (int i = 0; i < FLAGS_server_num; ++i) {
+            const size_t current_num_requests =
+                    echo_service_impls[i].num_requests();
+            size_t diff = current_num_requests - last_num_requests[i];
+            cur_total += diff;
+            last_num_requests[i] = current_num_requests;
+            LOG(INFO) << "S[" << i << "]=" << diff << ' ' << noflush;
+        }
+        LOG(INFO) << "[total=" << cur_total << ']';
+    }
+
+    // Don't forget to stop and join the server otherwise still-running
+    // worker threads may crash your program. 
+    for (int i = 0; i < FLAGS_server_num; ++i) {
+        servers[i].Stop(0/*not used now*/);
+    }
+    for (int i = 0; i < FLAGS_server_num; ++i) {
+        servers[i].Join();
+    }
+    delete [] servers;
+    delete [] echo_service_impls;
+    return 0;
+}

example/echo_c++/CMakeLists.txt

@@ -0,0 +1,110 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.16...3.28)
+project(echo_c++ C CXX)
+
+include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake)
+
+option(LINK_SO "Whether examples are linked dynamically" OFF)
+
+execute_process(
+    COMMAND bash -c "find ${PROJECT_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'"
+    OUTPUT_VARIABLE OUTPUT_PATH
+)
+
+if(OUTPUT_PATH)
+    list(PREPEND CMAKE_PREFIX_PATH ${OUTPUT_PATH})
+endif()
+
+find_package(Threads REQUIRED)
+find_package(Protobuf REQUIRED)
+protobuf_generate_cpp(PROTO_SRC PROTO_HEADER echo.proto)
+
+# Search for libthrift* by best effort. If it is not found and brpc is
+# compiled with thrift protocol enabled, a link error would be reported.
+find_library(THRIFT_LIB NAMES thrift)
+if (NOT THRIFT_LIB)
+    set(THRIFT_LIB "")
+endif()
+
+find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h)
+if(LINK_SO)
+    find_library(BRPC_LIB NAMES brpc)
+else()
+    find_library(BRPC_LIB NAMES libbrpc.a brpc)
+endif()
+if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB))
+    message(FATAL_ERROR "Fail to find brpc")
+endif()
+
+find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h)
+find_library(GFLAGS_LIBRARY NAMES gflags libgflags)
+if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY))
+    message(FATAL_ERROR "Fail to find gflags")
+endif()
+
+find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h)
+find_library(LEVELDB_LIB NAMES leveldb)
+if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB))
+    message(FATAL_ERROR "Fail to find leveldb")
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT OPENSSL_ROOT_DIR)
+    set(OPENSSL_ROOT_DIR
+        "/usr/local/opt/openssl"    # Homebrew installed OpenSSL
+        )
+endif()
+
+find_package(OpenSSL REQUIRED)
+
+set(DYNAMIC_LIB
+    Threads::Threads
+    ${GFLAGS_LIBRARY}
+    ${PROTOBUF_LIBRARIES}
+    ${LEVELDB_LIB}
+    ${OPENSSL_CRYPTO_LIBRARY}
+    ${OPENSSL_SSL_LIBRARY}
+    ${THRIFT_LIB}
+    dl
+    )
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    set(DYNAMIC_LIB ${DYNAMIC_LIB}
+        pthread
+        "-framework CoreFoundation"
+        "-framework CoreGraphics"
+        "-framework CoreData"
+        "-framework CoreText"
+        "-framework Security"
+        "-framework Foundation"
+        "-Wl,-U,_MallocExtension_ReleaseFreeMemory"
+        "-Wl,-U,_ProfilerStart"
+        "-Wl,-U,_ProfilerStop"
+        "-Wl,-U,__Z13GetStackTracePPvii"
+        "-Wl,-U,_mallctl"
+        "-Wl,-U,_malloc_stats_print"
+    )
+endif()
+
+add_executable(echo_client client.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(echo_client)
+add_executable(echo_server server.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(echo_server)
+
+target_link_libraries(echo_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})
+target_link_libraries(echo_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})

example/echo_c++/client.cpp

@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A client sending requests to server every 1 second.
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <butil/time.h>
+#include <brpc/channel.h>
+#include "echo.pb.h"
+
+DEFINE_string(attachment, "", "Carry this along with requests");
+DEFINE_string(protocol, "baidu_std", "Protocol type. Defined in src/brpc/options.proto");
+DEFINE_string(connection_type, "", "Connection type. Available values: single, pooled, short");
+DEFINE_string(server, "0.0.0.0:8000", "IP Address of server");
+DEFINE_string(load_balancer, "", "The algorithm for load balancing");
+DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); 
+DEFINE_int32(interval_ms, 1000, "Milliseconds between consecutive requests");
+DEFINE_bool(enable_checksum, false, "Enable checksum or not");
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+    
+    // A Channel represents a communication line to a Server. Notice that 
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::Channel channel;
+    
+    // Initialize the channel, NULL means using default options.
+    brpc::ChannelOptions options;
+    options.protocol = FLAGS_protocol;
+    options.connection_type = FLAGS_connection_type;
+    options.timeout_ms = FLAGS_timeout_ms/*milliseconds*/;
+    options.max_retry = FLAGS_max_retry;
+    if (channel.Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), &options) != 0) {
+        LOG(ERROR) << "Fail to initialize channel";
+        return -1;
+    }
+
+    // Normally, you should not call a Channel directly, but instead construct
+    // a stub Service wrapping it. stub can be shared by all threads as well.
+    example::EchoService_Stub stub(&channel);
+
+    // Send a request and wait for the response every 1 second.
+    int log_id = 0;
+    while (!brpc::IsAskedToQuit()) {
+        // We will receive response synchronously, safe to put variables
+        // on stack.
+        example::EchoRequest request;
+        example::EchoResponse response;
+        brpc::Controller cntl;
+
+        request.set_message("hello world");
+
+        cntl.set_log_id(log_id ++);  // set by user
+        // Set attachment which is wired to network directly instead of 
+        // being serialized into protobuf messages.
+        cntl.request_attachment().append(FLAGS_attachment);
+
+        // Use checksum, only support CRC32C now.
+        if (FLAGS_enable_checksum) {
+            cntl.set_request_checksum_type(brpc::CHECKSUM_TYPE_CRC32C);
+        }
+
+        // Because `done'(last parameter) is NULL, this function waits until
+        // the response comes back or error occurs(including timedout).
+        stub.Echo(&cntl, &request, &response, NULL);
+        if (!cntl.Failed()) {
+            LOG(INFO) << "Received response from " << cntl.remote_side()
+                << " to " << cntl.local_side()
+                << ": " << response.message() << " (attached="
+                << cntl.response_attachment() << ")"
+                << " latency=" << cntl.latency_us() << "us";
+        } else {
+            LOG(WARNING) << cntl.ErrorText();
+        }
+        usleep(FLAGS_interval_ms * 1000L);
+    }
+
+    LOG(INFO) << "EchoClient is going to quit";
+    return 0;
+}

example/echo_c++/echo.proto

@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+syntax="proto2";
+package example;
+
+option cc_generic_services = true;
+
+message EchoRequest {
+      required string message = 1;
+};
+
+message EchoResponse {
+      required string message = 1;
+};
+
+service EchoService {
+      rpc Echo(EchoRequest) returns (EchoResponse);
+};

example/echo_c++/server.cpp

@@ -0,0 +1,140 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A server to receive EchoRequest and send back EchoResponse.
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <brpc/server.h>
+#include <json2pb/pb_to_json.h>
+#include "echo.pb.h"
+
+DEFINE_bool(echo_attachment, true, "Echo attachment as well");
+DEFINE_int32(port, 8000, "TCP Port of this server");
+DEFINE_string(listen_addr, "", "Server listen address, may be IPV4/IPV6/UDS."
+            " If this is set, the flag port will be ignored");
+DEFINE_int32(idle_timeout_s, -1, "Connection will be closed if there is no "
+             "read/write operations during the last `idle_timeout_s'");
+DEFINE_bool(enable_checksum, false, "Enable checksum or not");
+
+// Your implementation of example::EchoService
+// Notice that implementing brpc::Describable grants the ability to put
+// additional information in /status.
+namespace example {
+class EchoServiceImpl : public EchoService {
+public:
+    EchoServiceImpl() {}
+    virtual ~EchoServiceImpl() {}
+    virtual void Echo(google::protobuf::RpcController* cntl_base,
+                      const EchoRequest* request,
+                      EchoResponse* response,
+                      google::protobuf::Closure* done) {
+        // This object helps you to call done->Run() in RAII style. If you need
+        // to process the request asynchronously, pass done_guard.release().
+        brpc::ClosureGuard done_guard(done);
+
+        brpc::Controller* cntl =
+            static_cast<brpc::Controller*>(cntl_base);
+
+        // optional: set a callback function which is called after response is sent
+        // and before cntl/req/res is destructed.
+        cntl->set_after_rpc_resp_fn(std::bind(&EchoServiceImpl::CallAfterRpc,
+            std::placeholders::_1, std::placeholders::_2, std::placeholders::_3));
+
+        // The purpose of following logs is to help you to understand
+        // how clients interact with servers more intuitively. You should 
+        // remove these logs in performance-sensitive servers.
+        LOG(INFO) << "Received request[log_id=" << cntl->log_id() 
+                  << "] from " << cntl->remote_side() 
+                  << " to " << cntl->local_side()
+                  << ": " << request->message()
+                  << " (attached=" << cntl->request_attachment() << ")";
+
+        // Fill response.
+        response->set_message(request->message());
+
+        // You can compress the response by setting Controller, but be aware
+        // that compression may be costly, evaluate before turning on.
+        // cntl->set_response_compress_type(brpc::COMPRESS_TYPE_GZIP);
+
+        if (FLAGS_echo_attachment) {
+            // Set attachment which is wired to network directly instead of
+            // being serialized into protobuf messages.
+            cntl->response_attachment().append(cntl->request_attachment());
+        }
+
+        // Use checksum, only support CRC32C now.
+        if (FLAGS_enable_checksum) {
+            cntl->set_response_checksum_type(brpc::CHECKSUM_TYPE_CRC32C);
+        }
+    }
+
+    // optional
+    static void CallAfterRpc(brpc::Controller* cntl,
+                        const google::protobuf::Message* req,
+                        const google::protobuf::Message* res) {
+        // at this time res is already sent to client, but cntl/req/res is not destructed
+        std::string req_str;
+        std::string res_str;
+        json2pb::ProtoMessageToJson(*req, &req_str, NULL);
+        json2pb::ProtoMessageToJson(*res, &res_str, NULL);
+        LOG(INFO) << "req:" << req_str
+                    << " res:" << res_str;
+    }
+};
+}  // namespace example
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    // Generally you only need one Server.
+    brpc::Server server;
+
+    // Instance of your service.
+    example::EchoServiceImpl echo_service_impl;
+
+    // Add the service into server. Notice the second parameter, because the
+    // service is put on stack, we don't want server to delete it, otherwise
+    // use brpc::SERVER_OWNS_SERVICE.
+    if (server.AddService(&echo_service_impl, 
+                          brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
+        LOG(ERROR) << "Fail to add service";
+        return -1;
+    }
+
+    butil::EndPoint point;
+    if (!FLAGS_listen_addr.empty()) {
+        if (butil::str2endpoint(FLAGS_listen_addr.c_str(), &point) < 0) {
+            LOG(ERROR) << "Invalid listen address:" << FLAGS_listen_addr;
+            return -1;
+        }
+    } else {
+        point = butil::EndPoint(butil::IP_ANY, FLAGS_port);
+    }
+    // Start the server.
+    brpc::ServerOptions options;
+    options.idle_timeout_sec = FLAGS_idle_timeout_s;
+    if (server.Start(point, &options) != 0) {
+        LOG(ERROR) << "Fail to start EchoServer";
+        return -1;
+    }
+
+    // Wait until Ctrl-C is pressed, then Stop() and Join() the server.
+    server.RunUntilAskedToQuit();
+    return 0;
+}

example/grpc_c++/CMakeLists.txt

@@ -0,0 +1,109 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.16...3.28)
+project(grpc_c++ C CXX)
+
+include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake)
+
+option(LINK_SO "Whether examples are linked dynamically" OFF)
+
+execute_process(
+    COMMAND bash -c "find ${PROJECT_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'"
+    OUTPUT_VARIABLE OUTPUT_PATH
+)
+
+if(OUTPUT_PATH)
+    list(PREPEND CMAKE_PREFIX_PATH ${OUTPUT_PATH})
+endif()
+
+find_package(Threads REQUIRED)
+find_package(Protobuf REQUIRED)
+protobuf_generate_cpp(PROTO_SRC PROTO_HEADER helloworld.proto)
+
+find_path(GPERFTOOLS_INCLUDE_DIR NAMES gperftools/heap-profiler.h)
+find_library(GPERFTOOLS_LIBRARIES NAMES tcmalloc_and_profiler)
+
+find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h)
+if(LINK_SO)
+    find_library(BRPC_LIB NAMES brpc)
+else()
+    find_library(BRPC_LIB NAMES libbrpc.a brpc)
+endif()
+if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB))
+    message(FATAL_ERROR "Fail to find brpc")
+endif()
+
+find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h)
+find_library(GFLAGS_LIBRARY NAMES gflags libgflags)
+if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY))
+    message(FATAL_ERROR "Fail to find gflags")
+endif()
+
+set(BRPC_EXAMPLE_ENABLE_CPU_PROFILER ON)
+
+find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h)
+find_library(LEVELDB_LIB NAMES leveldb)
+if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB))
+    message(FATAL_ERROR "Fail to find leveldb")
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT OPENSSL_ROOT_DIR)
+    set(OPENSSL_ROOT_DIR
+        "/usr/local/opt/openssl"    # Homebrew installed OpenSSL
+        )
+endif()
+
+find_package(OpenSSL REQUIRED)
+
+set(DYNAMIC_LIB
+    Threads::Threads
+    ${GFLAGS_LIBRARY}
+    ${PROTOBUF_LIBRARIES}
+    ${LEVELDB_LIB}
+    ${OPENSSL_CRYPTO_LIBRARY}
+    ${OPENSSL_SSL_LIBRARY}
+    ${THRIFT_LIB}
+    dl
+    )
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    set(DYNAMIC_LIB ${DYNAMIC_LIB}
+        pthread
+        "-framework CoreFoundation"
+        "-framework CoreGraphics"
+        "-framework CoreData"
+        "-framework CoreText"
+        "-framework Security"
+        "-framework Foundation"
+        "-Wl,-U,_MallocExtension_ReleaseFreeMemory"
+        "-Wl,-U,_ProfilerStart"
+        "-Wl,-U,_ProfilerStop"
+        "-Wl,-U,__Z13GetStackTracePPvii"
+        "-Wl,-U,_RegisterThriftProtocol"
+        "-Wl,-U,_mallctl"
+        "-Wl,-U,_malloc_stats_print"
+    )
+endif()
+
+add_executable(server server.cpp ${PROTO_SRC} ${PROTO_HEADER} )
+brpc_example_configure_target(server)
+add_executable(client client.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(client)
+
+target_link_libraries(server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB} ${GPERFTOOLS_LIBRARIES})
+target_link_libraries(client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB} ${GPERFTOOLS_LIBRARIES})

example/grpc_c++/client.cpp

@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A client sending requests to server every 1 second using grpc.
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <butil/time.h>
+#include <brpc/channel.h>
+#include "helloworld.pb.h"
+
+DEFINE_string(protocol, "h2:grpc", "Protocol type. Defined in src/brpc/options.proto");
+DEFINE_string(server, "0.0.0.0:50051", "IP Address of server");
+DEFINE_string(load_balancer, "", "The algorithm for load balancing");
+DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); 
+DEFINE_int32(interval_ms, 1000, "Milliseconds between consecutive requests");
+DEFINE_bool(gzip, false, "compress body using gzip");
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+    if (FLAGS_gzip) {
+        GFLAGS_NAMESPACE::SetCommandLineOption("http_body_compress_threshold", 0);
+    }
+    
+    // A Channel represents a communication line to a Server. Notice that 
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::Channel channel;
+    
+    // Initialize the channel, NULL means using default options.
+    brpc::ChannelOptions options;
+    options.protocol = FLAGS_protocol;
+    options.timeout_ms = FLAGS_timeout_ms/*milliseconds*/;
+    options.max_retry = FLAGS_max_retry;
+    if (channel.Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), &options) != 0) {
+        LOG(ERROR) << "Fail to initialize channel";
+        return -1;
+    }
+
+    // Normally, you should not call a Channel directly, but instead construct
+    // a stub Service wrapping it. stub can be shared by all threads as well.
+    helloworld::Greeter_Stub stub(&channel);
+
+    // Send a request and wait for the response every 1 second.
+    while (!brpc::IsAskedToQuit()) {
+        // We will receive response synchronously, safe to put variables
+        // on stack.
+        helloworld::HelloRequest request;
+        helloworld::HelloReply response;
+        brpc::Controller cntl;
+
+        request.set_name("grpc_req_from_brpc");
+        if (FLAGS_gzip) {
+            cntl.set_request_compress_type(brpc::COMPRESS_TYPE_GZIP);
+        }
+        // Because `done'(last parameter) is NULL, this function waits until
+        // the response comes back or error occurs(including timedout).
+        stub.SayHello(&cntl, &request, &response, NULL);
+        if (!cntl.Failed()) {
+            LOG(INFO) << "Received response from " << cntl.remote_side()
+                << " to " << cntl.local_side()
+                << ": " << response.message()
+                << " latency=" << cntl.latency_us() << "us";
+        } else {
+            LOG(WARNING) << cntl.ErrorText();
+        }
+        usleep(FLAGS_interval_ms * 1000L);
+    }
+
+    return 0;
+}

example/grpc_c++/helloworld.proto

@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+syntax = "proto2";
+
+package helloworld;
+
+option cc_generic_services = true;
+
+// The greeting service definition.
+service Greeter {
+  // Sends a greeting
+  rpc SayHello (HelloRequest) returns (HelloReply) {}
+}
+
+// The request message containing the user's name.
+message HelloRequest {
+  required string name = 1;
+}
+
+// The response message containing the greetings
+message HelloReply {
+  required string message = 1;
+}

example/grpc_c++/server.cpp

@@ -0,0 +1,77 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A server to receive HelloRequest and send back HelloReply
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <brpc/server.h>
+#include <brpc/restful.h>
+#include "helloworld.pb.h"
+
+DEFINE_int32(port, 50051, "TCP Port of this server");
+DEFINE_int32(idle_timeout_s, -1, "Connection will be closed if there is no "
+             "read/write operations during the last `idle_timeout_s'");
+DEFINE_bool(gzip, false, "compress body using gzip");
+
+class GreeterImpl : public helloworld::Greeter {
+public:
+    GreeterImpl() {}
+    virtual ~GreeterImpl() {}
+    void SayHello(google::protobuf::RpcController* cntl_base,
+                 const helloworld::HelloRequest* req,
+                 helloworld::HelloReply* res,
+                 google::protobuf::Closure* done) {
+        brpc::ClosureGuard done_guard(done);
+        brpc::Controller* cntl = static_cast<brpc::Controller*>(cntl_base);
+        if (FLAGS_gzip) {
+            cntl->set_response_compress_type(brpc::COMPRESS_TYPE_GZIP);
+        }
+        res->set_message("Hello " + req->name());
+    }
+};
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    // Generally you only need one Server.
+    brpc::Server server;
+
+    GreeterImpl http_svc;
+
+    // Add services into server. Notice the second parameter, because the
+    // service is put on stack, we don't want server to delete it, otherwise
+    // use brpc::SERVER_OWNS_SERVICE.
+    if (server.AddService(&http_svc,
+                          brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
+        LOG(ERROR) << "Fail to add http_svc";
+        return -1;
+    }
+
+    // Start the server.
+    brpc::ServerOptions options;
+    options.idle_timeout_sec = FLAGS_idle_timeout_s;
+    if (server.Start(FLAGS_port, &options) != 0) {
+        LOG(ERROR) << "Fail to start HttpServer";
+        return -1;
+    }
+
+    // Wait until Ctrl-C is pressed, then Stop() and Join() the server.
+    server.RunUntilAskedToQuit();
+    return 0;
+}

example/http_c++/CMakeLists.txt

@@ -0,0 +1,123 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.16...3.28)
+project(http_c++ C CXX)
+
+include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake)
+
+option(LINK_SO "Whether examples are linked dynamically" OFF)
+
+execute_process(
+    COMMAND bash -c "find ${PROJECT_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'"
+    OUTPUT_VARIABLE OUTPUT_PATH
+)
+
+if(OUTPUT_PATH)
+    list(PREPEND CMAKE_PREFIX_PATH ${OUTPUT_PATH})
+endif()
+
+find_package(Threads REQUIRED)
+find_package(Protobuf REQUIRED)
+protobuf_generate_cpp(PROTO_SRC PROTO_HEADER http.proto)
+
+# Search for libthrift* by best effort. If it is not found and brpc is
+# compiled with thrift protocol enabled, a link error would be reported.
+find_library(THRIFT_LIB NAMES thrift)
+if (NOT THRIFT_LIB)
+    set(THRIFT_LIB "")
+endif()
+
+find_path(GPERFTOOLS_INCLUDE_DIR NAMES gperftools/heap-profiler.h)
+find_library(GPERFTOOLS_LIBRARIES NAMES tcmalloc_and_profiler)
+
+find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h)
+if(LINK_SO)
+    find_library(BRPC_LIB NAMES brpc)
+else()
+    find_library(BRPC_LIB NAMES libbrpc.a brpc)
+endif()
+if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB))
+    message(FATAL_ERROR "Fail to find brpc")
+endif()
+
+find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h)
+find_library(GFLAGS_LIBRARY NAMES gflags libgflags)
+if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY))
+    message(FATAL_ERROR "Fail to find gflags")
+endif()
+
+set(BRPC_EXAMPLE_ENABLE_CPU_PROFILER ON)
+
+find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h)
+find_library(LEVELDB_LIB NAMES leveldb)
+if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB))
+    message(FATAL_ERROR "Fail to find leveldb")
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT OPENSSL_ROOT_DIR)
+    set(OPENSSL_ROOT_DIR
+        "/usr/local/opt/openssl"    # Homebrew installed OpenSSL
+        )
+endif()
+
+find_package(OpenSSL REQUIRED)
+
+set(DYNAMIC_LIB
+    Threads::Threads
+    ${GFLAGS_LIBRARY}
+    ${PROTOBUF_LIBRARIES}
+    ${LEVELDB_LIB}
+    ${OPENSSL_CRYPTO_LIBRARY}
+    ${OPENSSL_SSL_LIBRARY}
+    ${THRIFT_LIB}
+    dl
+    )
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    set(DYNAMIC_LIB ${DYNAMIC_LIB}
+        pthread
+        "-framework CoreFoundation"
+        "-framework CoreGraphics"
+        "-framework CoreData"
+        "-framework CoreText"
+        "-framework Security"
+        "-framework Foundation"
+        "-Wl,-U,_MallocExtension_ReleaseFreeMemory"
+        "-Wl,-U,_ProfilerStart"
+        "-Wl,-U,_ProfilerStop"
+        "-Wl,-U,__Z13GetStackTracePPvii"
+        "-Wl,-U,_mallctl"
+        "-Wl,-U,_malloc_stats_print"
+    )
+endif()
+
+add_executable(http_client http_client.cpp)
+brpc_example_configure_target(http_client)
+add_executable(http_server http_server.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(http_server)
+add_executable(benchmark_http benchmark_http.cpp)
+brpc_example_configure_target(benchmark_http)
+
+target_link_libraries(http_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB} ${GPERFTOOLS_LIBRARIES})
+target_link_libraries(http_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB} ${GPERFTOOLS_LIBRARIES})
+target_link_libraries(benchmark_http PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB} ${GPERFTOOLS_LIBRARIES})
+
+file(COPY ${PROJECT_SOURCE_DIR}/key.pem
+     DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+file(COPY ${PROJECT_SOURCE_DIR}/cert.pem
+     DESTINATION ${CMAKE_CURRENT_BINARY_DIR})

example/http_c++/benchmark_http.cpp

@@ -0,0 +1,135 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Benchmark http-server by multiple threads.
+
+#include <gflags/gflags.h>
+#include <bthread/bthread.h>
+#include <butil/logging.h>
+#include <brpc/channel.h>
+#include <brpc/server.h>
+#include <bvar/bvar.h>
+
+DEFINE_string(data, "", "POST this data to the http server");
+DEFINE_int32(thread_num, 50, "Number of threads to send requests");
+DEFINE_bool(use_bthread, false, "Use bthread to send requests");
+DEFINE_string(connection_type, "", "Connection type. Available values: single, pooled, short");
+DEFINE_string(url, "0.0.0.0:8010/HttpService/Echo", "url of server");
+DEFINE_string(load_balancer, "", "The algorithm for load balancing");
+DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); 
+DEFINE_bool(dont_fail, false, "Print fatal when some call failed");
+DEFINE_int32(dummy_port, -1, "Launch dummy server at this port");
+DEFINE_string(protocol, "http", "Client-side protocol");
+
+bvar::LatencyRecorder g_latency_recorder("client");
+
+static void* sender(void* arg) {
+    brpc::Channel* channel = static_cast<brpc::Channel*>(arg);
+
+    while (!brpc::IsAskedToQuit()) {
+        // We will receive response synchronously, safe to put variables
+        // on stack.
+        brpc::Controller cntl;
+
+        cntl.set_timeout_ms(FLAGS_timeout_ms/*milliseconds*/);
+        cntl.set_max_retry(FLAGS_max_retry);
+        cntl.http_request().uri() = FLAGS_url;
+        if (!FLAGS_data.empty()) {
+            cntl.http_request().set_method(brpc::HTTP_METHOD_POST);
+            cntl.request_attachment().append(FLAGS_data);
+        }
+
+        // Because `done'(last parameter) is NULL, this function waits until
+        // the response comes back or error occurs(including timedout).
+        channel->CallMethod(NULL, &cntl, NULL, NULL, NULL);
+        if (!cntl.Failed()) {
+            g_latency_recorder << cntl.latency_us();
+        } else {
+            CHECK(brpc::IsAskedToQuit() || !FLAGS_dont_fail)
+                << "error=" << cntl.ErrorText() << " latency=" << cntl.latency_us();
+            // We can't connect to the server, sleep a while. Notice that this
+            // is a specific sleeping to prevent this thread from spinning too
+            // fast. You should continue the business logic in a production 
+            // server rather than sleeping.
+            bthread_usleep(100000);
+        }
+    }
+    return NULL;
+}
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    // A Channel represents a communication line to a Server. Notice that 
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::Channel channel;
+    brpc::ChannelOptions options;
+    options.protocol = FLAGS_protocol;
+    options.connection_type = FLAGS_connection_type;
+    
+    // Initialize the channel, NULL means using default options. 
+    // options, see `brpc/channel.h'.
+    if (channel.Init(FLAGS_url.c_str(), FLAGS_load_balancer.c_str(), &options) != 0) {
+        LOG(ERROR) << "Fail to initialize channel";
+        return -1;
+    }
+
+    std::vector<bthread_t> bids;
+    std::vector<pthread_t> pids;
+    if (!FLAGS_use_bthread) {
+        pids.resize(FLAGS_thread_num);
+        for (int i = 0; i < FLAGS_thread_num; ++i) {
+            if (pthread_create(&pids[i], NULL, sender, &channel) != 0) {
+                LOG(ERROR) << "Fail to create pthread";
+                return -1;
+            }
+        }
+    } else {
+        bids.resize(FLAGS_thread_num);
+        for (int i = 0; i < FLAGS_thread_num; ++i) {
+            if (bthread_start_background(
+                    &bids[i], NULL, sender, &channel) != 0) {
+                LOG(ERROR) << "Fail to create bthread";
+                return -1;
+            }
+        }
+    }
+
+    if (FLAGS_dummy_port >= 0) {
+        brpc::StartDummyServerAt(FLAGS_dummy_port);
+    }
+
+    while (!brpc::IsAskedToQuit()) {
+        sleep(1);
+        LOG(INFO) << "Sending " << FLAGS_protocol << " requests at qps=" 
+                  << g_latency_recorder.qps(1)
+                  << " latency=" << g_latency_recorder.latency(1);
+    }
+
+    LOG(INFO) << "benchmark_http is going to quit";
+    for (int i = 0; i < FLAGS_thread_num; ++i) {
+        if (!FLAGS_use_bthread) {
+            pthread_join(pids[i], NULL);
+        } else {
+            bthread_join(bids[i], NULL);
+        }
+    }
+
+    return 0;
+}

example/http_c++/cert.pem

@@ -0,0 +1,26 @@
+-----BEGIN CERTIFICATE-----
+MIIEUTCCAzmgAwIBAgIBADANBgkqhkiG9w0BAQQFADB9MQswCQYDVQQGEwJDTjER
+MA8GA1UECBMIU2hhbmdoYWkxETAPBgNVBAcTCFNoYW5naGFpMQ4wDAYDVQQKEwVC
+YWlkdTEMMAoGA1UECxMDSU5GMQwwCgYDVQQDEwNTQVQxHDAaBgkqhkiG9w0BCQEW
+DXNhdEBiYWlkdS5jb20wHhcNMTUwNzE2MDMxOTUxWhcNMTgwNTA1MDMxOTUxWjB9
+MQswCQYDVQQGEwJDTjERMA8GA1UECBMIU2hhbmdoYWkxETAPBgNVBAcTCFNoYW5n
+aGFpMQ4wDAYDVQQKEwVCYWlkdTEMMAoGA1UECxMDSU5GMQwwCgYDVQQDEwNTQVQx
+HDAaBgkqhkiG9w0BCQEWDXNhdEBiYWlkdS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQCqdyAeHY39tqY1RYVbfpqZjZlJDtZb04znxjgQrX+mKmLb
+mwvXgJojlfn2Qcgp4NKYFqDFb9tU/Gbb436dRvkHyWOz0RPMspR0TTRU1NIY8wRy
+0A1LOCgLHsbRJHqktGjylejALdgsspFWyDY9bEfb4oWsnKGzJqcvIDXrPmMOOY4o
+pbA9SufSzwRZN7Yzc5jAedpaF9SK78RQXtvV0+JfCUwBsBWPKevRFFUrN7rQBYjP
+cgV/HgDuquPrqnESVSYyfEBKZba6cmNb+xzO3cB1brPTtobSXh+0o/0CtRA+2m63
+ODexxCLntgkPm42IYCJLM15xTatcfVX/3LHQ31DrAgMBAAGjgdswgdgwHQYDVR0O
+BBYEFGcd7lA//bSAoSC/NbWRx/H+O1zpMIGoBgNVHSMEgaAwgZ2AFGcd7lA//bSA
+oSC/NbWRx/H+O1zpoYGBpH8wfTELMAkGA1UEBhMCQ04xETAPBgNVBAgTCFNoYW5n
+aGFpMREwDwYDVQQHEwhTaGFuZ2hhaTEOMAwGA1UEChMFQmFpZHUxDDAKBgNVBAsT
+A0lORjEMMAoGA1UEAxMDU0FUMRwwGgYJKoZIhvcNAQkBFg1zYXRAYmFpZHUuY29t
+ggEAMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEEBQADggEBAKfoCn8SpLk3uQyT
+X+oygcRWfTeJtN3D5J69NCMJ7wB+QPfpEBPwiqMgdbp4bRJ98H7x5UQsHT+EDOT/
+9OmipomHInFY4W1ew11zNKwuENeRrnZwTcCiVLZsxZsAU41ZeI5Yq+2WdtxnePCR
+VL1/NjKOq+WoRdb2nLSNDWgYMkLRVlt32hyzryyrBbmaxUl8BxnPqUiWduMwsZUz
+HNpXkoa1xTSd+En1SHYWfMg8BOVuV0I0/fjUUG9AXVqYpuogfbjAvibVNWAmxOfo
+fOjCPCGoJC1ET3AxYkgXGwioobz0pK/13k2pV+wu7W4g+6iTfz+hwZbPsUk2a/5I
+f6vXFB0=
+-----END CERTIFICATE-----

example/http_c++/http.proto

@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+syntax="proto2";
+package example;
+
+option cc_generic_services = true;
+
+message HttpRequest {};
+message HttpResponse {};
+
+service HttpService {
+  rpc Echo(HttpRequest) returns (HttpResponse);
+  rpc EchoProtobuf(HttpRequest) returns (HttpResponse);
+};
+
+service FileService {
+  rpc default_method(HttpRequest) returns (HttpResponse);
+};
+
+service QueueService {
+  rpc start(HttpRequest) returns (HttpResponse);
+  rpc stop(HttpRequest) returns (HttpResponse);
+  rpc getstats(HttpRequest) returns (HttpResponse);
+};
+
+service HttpSSEService {
+  rpc stream(HttpRequest) returns (HttpResponse);
+};

example/http_c++/http_client.cpp

@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// - Access pb services via HTTP
+//   ./http_client http://www.foo.com:8765/EchoService/Echo -d '{"message":"hello"}'
+// - Access builtin services
+//   ./http_client http://www.foo.com:8765/vars/rpc_server*
+// - Access www.foo.com
+//   ./http_client www.foo.com
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <brpc/channel.h>
+
+DEFINE_string(d, "", "POST this data to the http server");
+DEFINE_string(load_balancer, "", "The algorithm for load balancing");
+DEFINE_int32(timeout_ms, 2000, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); 
+DEFINE_string(protocol, "http", "Client-side protocol");
+
+namespace brpc {
+DECLARE_bool(http_verbose);
+}
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    if (argc != 2) {
+        LOG(ERROR) << "Usage: ./http_client \"http(s)://www.foo.com\"";
+        return -1;
+    }
+    char* url = argv[1];
+    
+    // A Channel represents a communication line to a Server. Notice that 
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::Channel channel;
+    brpc::ChannelOptions options;
+    options.protocol = FLAGS_protocol;
+    options.timeout_ms = FLAGS_timeout_ms/*milliseconds*/;
+    options.max_retry = FLAGS_max_retry;
+
+    // Initialize the channel, NULL means using default options. 
+    // options, see `brpc/channel.h'.
+    if (channel.Init(url, FLAGS_load_balancer.c_str(), &options) != 0) {
+        LOG(ERROR) << "Fail to initialize channel";
+        return -1;
+    }
+
+    // We will receive response synchronously, safe to put variables
+    // on stack.
+    brpc::Controller cntl;
+
+    cntl.http_request().uri() = url;
+    if (!FLAGS_d.empty()) {
+        cntl.http_request().set_method(brpc::HTTP_METHOD_POST);
+        cntl.request_attachment().append(FLAGS_d);
+    }
+
+    // Because `done'(last parameter) is NULL, this function waits until
+    // the response comes back or error occurs(including timedout).
+    channel.CallMethod(NULL, &cntl, NULL, NULL, NULL);
+    if (cntl.Failed()) {
+        std::cerr << cntl.ErrorText() << std::endl;
+        return -1;
+    }
+    // If -http_verbose is on, brpc already prints the response to stderr.
+    if (!brpc::FLAGS_http_verbose) {
+        std::cout << cntl.response_attachment() << std::endl;
+    }
+    return 0;
+}

example/http_c++/http_server.cpp

@@ -0,0 +1,279 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A server to receive HttpRequest and send back HttpResponse.
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <brpc/server.h>
+#include <brpc/restful.h>
+#include <json2pb/pb_to_json.h>
+#include "http.pb.h"
+
+DEFINE_int32(port, 8010, "TCP Port of this server");
+DEFINE_int32(idle_timeout_s, -1, "Connection will be closed if there is no "
+             "read/write operations during the last `idle_timeout_s'");
+
+DEFINE_string(certificate, "cert.pem", "Certificate file path to enable SSL");
+DEFINE_string(private_key, "key.pem", "Private key file path to enable SSL");
+DEFINE_string(ciphers, "", "Cipher suite used for SSL connections");
+
+namespace example {
+
+// Service with static path.
+class HttpServiceImpl : public HttpService {
+public:
+    HttpServiceImpl() {}
+    virtual ~HttpServiceImpl() {}
+    void Echo(google::protobuf::RpcController* cntl_base,
+              const HttpRequest*,
+              HttpResponse*,
+              google::protobuf::Closure* done) {
+        // This object helps you to call done->Run() in RAII style. If you need
+        // to process the request asynchronously, pass done_guard.release().
+        brpc::ClosureGuard done_guard(done);
+
+        brpc::Controller* cntl =
+            static_cast<brpc::Controller*>(cntl_base);
+
+        // optional: set a callback function which is called after response is sent
+        // and before cntl/req/res is destructed.
+        cntl->set_after_rpc_resp_fn(std::bind(&HttpServiceImpl::CallAfterRpc,
+            std::placeholders::_1, std::placeholders::_2, std::placeholders::_3));
+
+        // Fill response.
+        cntl->http_response().set_content_type("text/plain");
+        butil::IOBufBuilder os;
+        os << "queries:";
+        for (brpc::URI::QueryIterator it = cntl->http_request().uri().QueryBegin();
+                it != cntl->http_request().uri().QueryEnd(); ++it) {
+            os << ' ' << it->first << '=' << it->second;
+        }
+        os << "\nbody: " << cntl->request_attachment() << '\n';
+        os.move_to(cntl->response_attachment());
+    }
+
+    // optional
+    static void CallAfterRpc(brpc::Controller* cntl,
+                        const google::protobuf::Message* req,
+                        const google::protobuf::Message* res) {
+        // at this time res is already sent to client, but cntl/req/res is not destructed
+        std::string req_str;
+        std::string res_str;
+        json2pb::ProtoMessageToJson(*req, &req_str, NULL);
+        json2pb::ProtoMessageToJson(*res, &res_str, NULL);
+        LOG(INFO) << "req:" << req_str
+                    << " res:" << res_str;
+    }
+};
+
+// Service with dynamic path.
+class FileServiceImpl : public FileService {
+public:
+    FileServiceImpl() {}
+    virtual ~FileServiceImpl() {}
+
+    struct Args {
+        butil::intrusive_ptr<brpc::ProgressiveAttachment> pa;
+    };
+
+    static void* SendLargeFile(void* raw_args) {
+        std::unique_ptr<Args> args(static_cast<Args*>(raw_args));
+        if (args->pa == NULL) {
+            LOG(ERROR) << "ProgressiveAttachment is NULL";
+            return NULL;
+        }
+        for (int i = 0; i < 100; ++i) {
+            char buf[16];
+            int len = snprintf(buf, sizeof(buf), "part_%d ", i);
+            args->pa->Write(buf, len);
+
+            // sleep a while to send another part.
+            bthread_usleep(10000);
+        }
+        return NULL;
+    }
+
+    void default_method(google::protobuf::RpcController* cntl_base,
+                        const HttpRequest*,
+                        HttpResponse*,
+                        google::protobuf::Closure* done) {
+        brpc::ClosureGuard done_guard(done);
+        brpc::Controller* cntl =
+            static_cast<brpc::Controller*>(cntl_base);
+        const std::string& filename = cntl->http_request().unresolved_path();
+        if (filename == "largefile") {
+            // Send the "largefile" with ProgressiveAttachment.
+            std::unique_ptr<Args> args(new Args);
+            args->pa = cntl->CreateProgressiveAttachment();
+            bthread_t th;
+            bthread_start_background(&th, NULL, SendLargeFile, args.release());
+        } else {
+            cntl->response_attachment().append("Getting file: ");
+            cntl->response_attachment().append(filename);
+        }
+    }
+};
+
+// Restful service. (The service implementation is exactly same with regular
+// services, the difference is that you need to pass a `restful_mappings'
+// when adding the service into server).
+class QueueServiceImpl : public example::QueueService {
+public:
+    QueueServiceImpl() {}
+    virtual ~QueueServiceImpl() {}
+    void start(google::protobuf::RpcController* cntl_base,
+               const HttpRequest*,
+               HttpResponse*,
+               google::protobuf::Closure* done) {
+        brpc::ClosureGuard done_guard(done);
+        brpc::Controller* cntl =
+            static_cast<brpc::Controller*>(cntl_base);
+        cntl->response_attachment().append("queue started");
+    }
+    void stop(google::protobuf::RpcController* cntl_base,
+              const HttpRequest*,
+              HttpResponse*,
+              google::protobuf::Closure* done) {
+        brpc::ClosureGuard done_guard(done);
+        brpc::Controller* cntl =
+            static_cast<brpc::Controller*>(cntl_base);
+        cntl->response_attachment().append("queue stopped");
+    }
+    void getstats(google::protobuf::RpcController* cntl_base,
+                  const HttpRequest*,
+                  HttpResponse*,
+                  google::protobuf::Closure* done) {
+        brpc::ClosureGuard done_guard(done);
+        brpc::Controller* cntl =
+            static_cast<brpc::Controller*>(cntl_base);
+        const std::string& unresolved_path = cntl->http_request().unresolved_path();
+        if (unresolved_path.empty()) {
+            cntl->response_attachment().append("Require a name after /stats");
+        } else {
+            cntl->response_attachment().append("Get stats: ");
+            cntl->response_attachment().append(unresolved_path);
+        }
+    }
+};
+
+class HttpSSEServiceImpl : public HttpSSEService {
+public:
+    HttpSSEServiceImpl() {}
+    virtual ~HttpSSEServiceImpl() {}
+
+    struct PredictJobArgs {
+        std::vector<uint32_t> input_ids;
+        butil::intrusive_ptr<brpc::ProgressiveAttachment> pa;
+    };
+
+    static void* Predict(void* raw_args) {
+        std::unique_ptr<PredictJobArgs> args(static_cast<PredictJobArgs*>(raw_args));
+        if (args->pa == NULL) {
+            LOG(ERROR) << "ProgressiveAttachment is NULL";
+            return NULL;
+        }
+        for (int i = 0; i < 100; ++i) {
+            char buf[48];
+            int len = snprintf(buf, sizeof(buf), "event: foo\ndata: Hello, world! (%d)\n\n", i);
+            args->pa->Write(buf, len);
+
+            // sleep a while to send another part.
+            bthread_usleep(10000 * 10);
+        }
+        return NULL;
+    }
+
+    void stream(google::protobuf::RpcController* cntl_base,
+                const HttpRequest*,
+                HttpResponse*,
+                google::protobuf::Closure* done) {
+        brpc::ClosureGuard done_guard(done);
+        brpc::Controller* cntl =
+            static_cast<brpc::Controller*>(cntl_base);
+
+        // Send the first SSE response
+        cntl->http_response().set_content_type("text/event-stream");
+        cntl->http_response().set_status_code(200);
+        cntl->http_response().SetHeader("Connection", "keep-alive");
+        cntl->http_response().SetHeader("Cache-Control", "no-cache");
+
+        // Send the generated words with progressiveAttachment
+        std::unique_ptr<PredictJobArgs> args(new PredictJobArgs);
+        args->pa = cntl->CreateProgressiveAttachment();
+        args->input_ids = {101, 102};
+        bthread_t th;
+        bthread_start_background(&th, NULL, Predict, args.release());
+    }
+};
+
+}  // namespace example
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    // Generally you only need one Server.
+    brpc::Server server;
+
+    example::HttpServiceImpl http_svc;
+    example::FileServiceImpl file_svc;
+    example::QueueServiceImpl queue_svc;
+    example::HttpSSEServiceImpl sse_svc;
+    
+    // Add services into server. Notice the second parameter, because the
+    // service is put on stack, we don't want server to delete it, otherwise
+    // use brpc::SERVER_OWNS_SERVICE.
+    if (server.AddService(&http_svc,
+                          brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
+        LOG(ERROR) << "Fail to add http_svc";
+        return -1;
+    }
+    if (server.AddService(&file_svc,
+                          brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
+        LOG(ERROR) << "Fail to add file_svc";
+        return -1;
+    }
+    if (server.AddService(&queue_svc,
+                          brpc::SERVER_DOESNT_OWN_SERVICE,
+                          "/v1/queue/start   => start,"
+                          "/v1/queue/stop    => stop,"
+                          "/v1/queue/stats/* => getstats") != 0) {
+        LOG(ERROR) << "Fail to add queue_svc";
+        return -1;
+    }
+    if (server.AddService(&sse_svc,
+                          brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
+        LOG(ERROR) << "Fail to add sse_svc";
+        return -1;
+    }
+
+    // Start the server.
+    brpc::ServerOptions options;
+    options.idle_timeout_sec = FLAGS_idle_timeout_s;
+    options.mutable_ssl_options()->default_cert.certificate = FLAGS_certificate;
+    options.mutable_ssl_options()->default_cert.private_key = FLAGS_private_key;
+    options.mutable_ssl_options()->ciphers = FLAGS_ciphers;
+    if (server.Start(FLAGS_port, &options) != 0) {
+        LOG(ERROR) << "Fail to start HttpServer";
+        return -1;
+    }
+
+    // Wait until Ctrl-C is pressed, then Stop() and Join() the server.
+    server.RunUntilAskedToQuit();
+    return 0;
+}

example/http_c++/key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAqncgHh2N/bamNUWFW36amY2ZSQ7WW9OM58Y4EK1/pipi25sL
+14CaI5X59kHIKeDSmBagxW/bVPxm2+N+nUb5B8ljs9ETzLKUdE00VNTSGPMEctAN
+SzgoCx7G0SR6pLRo8pXowC3YLLKRVsg2PWxH2+KFrJyhsyanLyA16z5jDjmOKKWw
+PUrn0s8EWTe2M3OYwHnaWhfUiu/EUF7b1dPiXwlMAbAVjynr0RRVKze60AWIz3IF
+fx4A7qrj66pxElUmMnxASmW2unJjW/sczt3AdW6z07aG0l4ftKP9ArUQPtputzg3
+scQi57YJD5uNiGAiSzNecU2rXH1V/9yx0N9Q6wIDAQABAoIBADN3khflnnhKzDXr
+To9IU08nRG+dbjT9U16rJ0RJze+SfpSFZHblWiSCZJzoUZHrUkofEt1pn1QyfK/J
+KPI9enTSZirlZk/4XwAaS0GNm/1yahZsIIdkZhqtaSO+GtVdrw4HGuXjMZCVPXJx
+MocrCSsnYmqyQ9P+SJ3e4Mis5mVllwDiUVlnTIamSSt16qkPdamLSJrxvI4LirQK
+9MZWNLoDFpRU1MJxQ/QzrEC3ONTq4j++AfbGzYTmDDtLeM8OSH5o72YXZ2JkaA4c
+xCzHFT+NaJYxF7esn/ctzGg50LYl8IF2UQtzOkX2l3l/OktIB1w+jGV6ONb1EWx5
+4zkkzNkCgYEA2EXj7GMsyNE3OYdMw8zrqQKUMON2CNnD+mBseGlr22/bhXtzpqK8
+uNel8WF1ezOnVvNsU8pml/W/mKUu6KQt5JfaDzen3OKjzTABVlbJxwFhPvwAeaIA
+q/tmSKyqiCgOMbR7Cq4UEwGf2A9/RII4JEC0/aipRU5srF65OYPUOJcCgYEAycco
+DFVG6jUw9w68t/X4f7NT4IYP96hSAqLUPuVz2fWwXKLWEX8JiMI+Ue3PbMz6mPcs
+4vMu364u4R3IuzrrI+PRK9iTa/pahBP6eF6ZpbY1ObI8CVLTrqUS9p22rr9lBm8V
+EZA9hwcHLYt+PWzaKcsFpbP4+AeY7nBBbL9CAM0CgYAzuJsmeB1ItUgIuQOxu7sM
+AzLfcjZTLYkBwreOIGAL7XdJN9nTmw2ZAvGLhWwsF5FIaRSaAUiBxOKaJb7PIhxb
+k7kxdHTvjT/xHS7ksAK3VewkvO18KTMR7iBq9ugdgb7LQkc+qZzhYr0QVbxw7Ndy
+TAs8sm4wxe2VV13ilFVXZwKBgDfU6ZnwBr1Llo7l/wYQA4CiSDU6IzTt2DNuhrgY
+mWPX/cLEM+OHeUXkKYZV/S0n0rd8vWjWzUOLWOFlcmOMPAAkS36MYM5h6aXeOVIR
+KwaVUkjyrnYN+xC6EHM41JGp1/RdzECd3sh8A1pw3K92bS9fQ+LD18IZqBFh8lh6
+23KJAoGAe48SwAsaGvqRO61Taww/Wf+YpGc9lnVbCvNFGScYaycPMqaRBUBmz/U3
+QQgpQY8T7JIECbA8sf78SlAZ9x93r0UQ70RekV3WzKAQHfHK8nqTjd3T0+i4aySO
+yQpYYCgE24zYO6rQgwrhzI0S4rWe7izDDlg0RmLtQh7Xw+rlkAQ=
+-----END RSA PRIVATE KEY-----

example/memcache_c++/CMakeLists.txt

@@ -0,0 +1,107 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.16...3.28)
+project(memcache_c++ C CXX)
+
+include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake)
+
+option(LINK_SO "Whether examples are linked dynamically" OFF)
+
+execute_process(
+    COMMAND bash -c "find ${PROJECT_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'"
+    OUTPUT_VARIABLE OUTPUT_PATH
+)
+
+if(OUTPUT_PATH)
+    list(PREPEND CMAKE_PREFIX_PATH ${OUTPUT_PATH})
+endif()
+
+find_package(Threads REQUIRED)
+find_package(Protobuf REQUIRED)
+protobuf_generate_cpp(PROTO_SRC PROTO_HEADER echo.proto)
+
+# Search for libthrift* by best effort. If it is not found and brpc is
+# compiled with thrift protocol enabled, a link error would be reported.
+find_library(THRIFT_LIB NAMES thrift)
+if (NOT THRIFT_LIB)
+    set(THRIFT_LIB "")
+endif()
+
+find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h)
+if(LINK_SO)
+    find_library(BRPC_LIB NAMES brpc)
+else()
+    find_library(BRPC_LIB NAMES libbrpc.a brpc)
+endif()
+if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB))
+    message(FATAL_ERROR "Fail to find brpc")
+endif()
+
+find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h)
+find_library(GFLAGS_LIBRARY NAMES gflags libgflags)
+if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY))
+    message(FATAL_ERROR "Fail to find gflags")
+endif()
+
+find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h)
+find_library(LEVELDB_LIB NAMES leveldb)
+if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB))
+    message(FATAL_ERROR "Fail to find leveldb")
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT OPENSSL_ROOT_DIR)
+    set(OPENSSL_ROOT_DIR
+        "/usr/local/opt/openssl"    # Homebrew installed OpenSSL
+        )
+endif()
+
+find_package(OpenSSL REQUIRED)
+
+set(DYNAMIC_LIB
+    Threads::Threads
+    ${GFLAGS_LIBRARY}
+    ${PROTOBUF_LIBRARIES}
+    ${LEVELDB_LIB}
+    ${OPENSSL_CRYPTO_LIBRARY}
+    ${OPENSSL_SSL_LIBRARY}
+    ${THRIFT_LIB}
+    dl
+    )
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    set(DYNAMIC_LIB ${DYNAMIC_LIB}
+        pthread
+        "-framework CoreFoundation"
+        "-framework CoreGraphics"
+        "-framework CoreData"
+        "-framework CoreText"
+        "-framework Security"
+        "-framework Foundation"
+        "-Wl,-U,_MallocExtension_ReleaseFreeMemory"
+        "-Wl,-U,_ProfilerStart"
+        "-Wl,-U,_ProfilerStop"
+        "-Wl,-U,__Z13GetStackTracePPvii"
+        "-Wl,-U,_mallctl"
+        "-Wl,-U,_malloc_stats_print"
+    )
+endif()
+
+add_executable(memcache_client client.cpp)
+brpc_example_configure_target(memcache_client)
+
+target_link_libraries(memcache_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})

example/memcache_c++/client.cpp

@@ -0,0 +1,204 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A multi-threaded client getting keys from a memcache server constantly.
+
+#include <gflags/gflags.h>
+#include <bthread/bthread.h>
+#include <butil/logging.h>
+#include <butil/string_printf.h>
+#include <brpc/channel.h>
+#include <brpc/memcache.h>
+#include <brpc/policy/couchbase_authenticator.h>
+
+DEFINE_int32(thread_num, 10, "Number of threads to send requests");
+DEFINE_bool(use_bthread, false, "Use bthread to send requests");
+DEFINE_bool(use_couchbase, false, "Use couchbase.");
+DEFINE_string(connection_type, "", "Connection type. Available values: single, pooled, short");
+DEFINE_string(server, "0.0.0.0:11211", "IP Address of server");
+DEFINE_string(bucket_name, "", "Couchbase bucktet name");
+DEFINE_string(bucket_password, "", "Couchbase bucket password");
+DEFINE_string(load_balancer, "", "The algorithm for load balancing");
+DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); 
+DEFINE_bool(dont_fail, false, "Print fatal when some call failed");
+DEFINE_int32(exptime, 0, "The to-be-got data will be expired after so many seconds");
+DEFINE_string(key, "hello", "The key to be get");
+DEFINE_string(value, "world", "The value associated with the key");
+DEFINE_int32(batch, 1, "Pipelined Operations");
+
+bvar::LatencyRecorder g_latency_recorder("client");
+bvar::Adder<int> g_error_count("client_error_count");
+butil::static_atomic<int> g_sender_count = BUTIL_STATIC_ATOMIC_INIT(0);
+
+static void* sender(void* arg) {
+    google::protobuf::RpcChannel* channel = 
+        static_cast<google::protobuf::RpcChannel*>(arg);
+    const int base_index = g_sender_count.fetch_add(1, butil::memory_order_relaxed);
+
+    std::string value;
+    std::vector<std::pair<std::string, std::string> > kvs;
+    kvs.resize(FLAGS_batch);
+    for (int i = 0; i < FLAGS_batch; ++i) {
+        kvs[i].first = butil::string_printf("%s%d", FLAGS_key.c_str(), base_index + i);
+        kvs[i].second = butil::string_printf("%s%d", FLAGS_value.c_str(), base_index + i);
+    }
+    brpc::MemcacheRequest request;
+    for (int i = 0; i < FLAGS_batch; ++i) {
+        CHECK(request.Get(kvs[i].first));
+    }
+    while (!brpc::IsAskedToQuit()) {
+        // We will receive response synchronously, safe to put variables
+        // on stack.
+        brpc::MemcacheResponse response;
+        brpc::Controller cntl;
+
+        // Because `done'(last parameter) is NULL, this function waits until
+        // the response comes back or error occurs(including timedout).
+        channel->CallMethod(NULL, &cntl, &request, &response, NULL);
+        const int64_t elp = cntl.latency_us();
+        if (!cntl.Failed()) {
+            g_latency_recorder << cntl.latency_us();
+            for (int i = 0; i < FLAGS_batch; ++i) {
+                uint32_t flags;
+                if (!response.PopGet(&value, &flags, NULL)) {
+                    LOG(INFO) << "Fail to GET the key, " << response.LastError();
+                    brpc::AskToQuit();
+                    return NULL;
+                }
+                CHECK(flags == 0xdeadbeef + base_index + i)
+                    << "flags=" << flags;
+                CHECK(kvs[i].second == value)
+                    << "base=" << base_index << " i=" << i << " value=" << value;
+            }
+        } else {
+            g_error_count << 1; 
+            CHECK(brpc::IsAskedToQuit() || !FLAGS_dont_fail)
+                << "error=" << cntl.ErrorText() << " latency=" << elp;
+            // We can't connect to the server, sleep a while. Notice that this
+            // is a specific sleeping to prevent this thread from spinning too
+            // fast. You should continue the business logic in a production 
+            // server rather than sleeping.
+            bthread_usleep(50000);
+        }
+    }
+    return NULL;
+}
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+    if (FLAGS_exptime < 0) {
+        FLAGS_exptime = 0;
+    }
+
+    // A Channel represents a communication line to a Server. Notice that 
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::Channel channel;
+    
+    // Initialize the channel, NULL means using default options. 
+    brpc::ChannelOptions options;
+    options.protocol = brpc::PROTOCOL_MEMCACHE;
+    options.connection_type = FLAGS_connection_type;
+    options.timeout_ms = FLAGS_timeout_ms/*milliseconds*/;
+    options.max_retry = FLAGS_max_retry;
+    if (FLAGS_use_couchbase && !FLAGS_bucket_name.empty()) {
+        brpc::policy::CouchbaseAuthenticator* auth =
+            new brpc::policy::CouchbaseAuthenticator(FLAGS_bucket_name,
+                                                     FLAGS_bucket_password);
+        options.auth = auth;
+    }
+
+    if (channel.Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), &options) != 0) {
+        LOG(ERROR) << "Fail to initialize channel";
+        return -1;
+    }
+
+    // Pipeline #batch * #thread_num SET requests into memcache so that we
+    // have keys to get.
+    brpc::MemcacheRequest request;
+    brpc::MemcacheResponse response;
+    brpc::Controller cntl;
+    for (int i = 0; i < FLAGS_batch * FLAGS_thread_num; ++i) {
+        if (!request.Set(butil::string_printf("%s%d", FLAGS_key.c_str(), i),
+                         butil::string_printf("%s%d", FLAGS_value.c_str(), i),
+                         0xdeadbeef + i, FLAGS_exptime, 0)) {
+            LOG(ERROR) << "Fail to SET " << i << "th request";
+            return -1;
+        }
+    }
+    channel.CallMethod(NULL, &cntl, &request, &response, NULL);
+    if (cntl.Failed()) {
+        LOG(ERROR) << "Fail to access memcache, " << cntl.ErrorText();
+        return -1;
+    }
+    for (int i = 0; i < FLAGS_batch * FLAGS_thread_num; ++i) {
+        if (!response.PopSet(NULL)) {
+            LOG(ERROR) << "Fail to SET memcache, i=" << i
+                       << ", " << response.LastError();
+            return -1;
+        }
+    }
+    if (FLAGS_exptime > 0) {
+        LOG(INFO) << "Set " << FLAGS_batch * FLAGS_thread_num 
+                  << " values, expired after " << FLAGS_exptime << " seconds";
+    } else {
+        LOG(INFO) << "Set " << FLAGS_batch * FLAGS_thread_num 
+                  << " values, never expired";
+    }
+    
+    std::vector<bthread_t> bids;
+    std::vector<pthread_t> pids;
+    if (!FLAGS_use_bthread) {
+        pids.resize(FLAGS_thread_num);
+        for (int i = 0; i < FLAGS_thread_num; ++i) {
+            if (pthread_create(&pids[i], NULL, sender, &channel) != 0) {
+                LOG(ERROR) << "Fail to create pthread";
+                return -1;
+            }
+        }
+    } else {
+        bids.resize(FLAGS_thread_num);
+        for (int i = 0; i < FLAGS_thread_num; ++i) {
+            if (bthread_start_background(
+                    &bids[i], NULL, sender, &channel) != 0) {
+                LOG(ERROR) << "Fail to create bthread";
+                return -1;
+            }
+        }
+    }
+
+    while (!brpc::IsAskedToQuit()) {
+        sleep(1);
+        LOG(INFO) << "Accessing memcache server at qps=" << g_latency_recorder.qps(1)
+                  << " latency=" << g_latency_recorder.latency(1);
+    }
+
+    LOG(INFO) << "memcache_client is going to quit";
+    for (int i = 0; i < FLAGS_thread_num; ++i) {
+        if (!FLAGS_use_bthread) {
+            pthread_join(pids[i], NULL);
+        } else {
+            bthread_join(bids[i], NULL);
+        }
+    }
+    if (options.auth) {
+        delete options.auth;
+    }
+
+    return 0;
+}

example/multi_threaded_echo_c++/CMakeLists.txt

@@ -0,0 +1,120 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.16...3.28)
+project(multi_threaded_echo_c++ C CXX)
+
+include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake)
+
+option(LINK_SO "Whether examples are linked dynamically" OFF)
+
+execute_process(
+    COMMAND bash -c "find ${PROJECT_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'"
+    OUTPUT_VARIABLE OUTPUT_PATH
+)
+
+if(OUTPUT_PATH)
+    list(PREPEND CMAKE_PREFIX_PATH ${OUTPUT_PATH})
+endif()
+
+find_package(Threads REQUIRED)
+find_package(Protobuf REQUIRED)
+protobuf_generate_cpp(PROTO_SRC PROTO_HEADER echo.proto)
+
+# Search for libthrift* by best effort. If it is not found and brpc is
+# compiled with thrift protocol enabled, a link error would be reported.
+find_library(THRIFT_LIB NAMES thrift)
+if (NOT THRIFT_LIB)
+    set(THRIFT_LIB "")
+endif()
+
+find_path(GPERFTOOLS_INCLUDE_DIR NAMES gperftools/heap-profiler.h)
+find_library(GPERFTOOLS_LIBRARIES NAMES tcmalloc_and_profiler)
+
+find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h)
+if(LINK_SO)
+    find_library(BRPC_LIB NAMES brpc)
+else()
+    find_library(BRPC_LIB NAMES libbrpc.a brpc)
+endif()
+if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB))
+    message(FATAL_ERROR "Fail to find brpc")
+endif()
+
+find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h)
+find_library(GFLAGS_LIBRARY NAMES gflags libgflags)
+if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY))
+    message(FATAL_ERROR "Fail to find gflags")
+endif()
+
+set(BRPC_EXAMPLE_ENABLE_CPU_PROFILER ON)
+
+find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h)
+find_library(LEVELDB_LIB NAMES leveldb)
+if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB))
+    message(FATAL_ERROR "Fail to find leveldb")
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT OPENSSL_ROOT_DIR)
+    set(OPENSSL_ROOT_DIR
+        "/usr/local/opt/openssl"    # Homebrew installed OpenSSL
+        )
+endif()
+
+find_package(OpenSSL REQUIRED)
+
+set(DYNAMIC_LIB
+    Threads::Threads
+    ${GFLAGS_LIBRARY}
+    ${PROTOBUF_LIBRARIES}
+    ${LEVELDB_LIB}
+    ${OPENSSL_CRYPTO_LIBRARY}
+    ${OPENSSL_SSL_LIBRARY}
+    ${THRIFT_LIB}
+    dl
+    )
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    set(DYNAMIC_LIB ${DYNAMIC_LIB}
+        pthread
+        "-framework CoreFoundation"
+        "-framework CoreGraphics"
+        "-framework CoreData"
+        "-framework CoreText"
+        "-framework Security"
+        "-framework Foundation"
+        "-Wl,-U,_MallocExtension_ReleaseFreeMemory"
+        "-Wl,-U,_ProfilerStart"
+        "-Wl,-U,_ProfilerStop"
+        "-Wl,-U,__Z13GetStackTracePPvii"
+        "-Wl,-U,_mallctl"
+        "-Wl,-U,_malloc_stats_print"
+    )
+endif()
+
+add_executable(echo_client client.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(echo_client)
+add_executable(echo_server server.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(echo_server)
+
+target_link_libraries(echo_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB} ${GPERFTOOLS_LIBRARIES})
+target_link_libraries(echo_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB} ${GPERFTOOLS_LIBRARIES})
+
+file(COPY ${PROJECT_SOURCE_DIR}/key.pem
+     DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+file(COPY ${PROJECT_SOURCE_DIR}/cert.pem
+     DESTINATION ${CMAKE_CURRENT_BINARY_DIR})

example/multi_threaded_echo_c++/cert.pem

@@ -0,0 +1,26 @@
+-----BEGIN CERTIFICATE-----
+MIIEUTCCAzmgAwIBAgIBADANBgkqhkiG9w0BAQQFADB9MQswCQYDVQQGEwJDTjER
+MA8GA1UECBMIU2hhbmdoYWkxETAPBgNVBAcTCFNoYW5naGFpMQ4wDAYDVQQKEwVC
+YWlkdTEMMAoGA1UECxMDSU5GMQwwCgYDVQQDEwNTQVQxHDAaBgkqhkiG9w0BCQEW
+DXNhdEBiYWlkdS5jb20wHhcNMTUwNzE2MDMxOTUxWhcNMTgwNTA1MDMxOTUxWjB9
+MQswCQYDVQQGEwJDTjERMA8GA1UECBMIU2hhbmdoYWkxETAPBgNVBAcTCFNoYW5n
+aGFpMQ4wDAYDVQQKEwVCYWlkdTEMMAoGA1UECxMDSU5GMQwwCgYDVQQDEwNTQVQx
+HDAaBgkqhkiG9w0BCQEWDXNhdEBiYWlkdS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQCqdyAeHY39tqY1RYVbfpqZjZlJDtZb04znxjgQrX+mKmLb
+mwvXgJojlfn2Qcgp4NKYFqDFb9tU/Gbb436dRvkHyWOz0RPMspR0TTRU1NIY8wRy
+0A1LOCgLHsbRJHqktGjylejALdgsspFWyDY9bEfb4oWsnKGzJqcvIDXrPmMOOY4o
+pbA9SufSzwRZN7Yzc5jAedpaF9SK78RQXtvV0+JfCUwBsBWPKevRFFUrN7rQBYjP
+cgV/HgDuquPrqnESVSYyfEBKZba6cmNb+xzO3cB1brPTtobSXh+0o/0CtRA+2m63
+ODexxCLntgkPm42IYCJLM15xTatcfVX/3LHQ31DrAgMBAAGjgdswgdgwHQYDVR0O
+BBYEFGcd7lA//bSAoSC/NbWRx/H+O1zpMIGoBgNVHSMEgaAwgZ2AFGcd7lA//bSA
+oSC/NbWRx/H+O1zpoYGBpH8wfTELMAkGA1UEBhMCQ04xETAPBgNVBAgTCFNoYW5n
+aGFpMREwDwYDVQQHEwhTaGFuZ2hhaTEOMAwGA1UEChMFQmFpZHUxDDAKBgNVBAsT
+A0lORjEMMAoGA1UEAxMDU0FUMRwwGgYJKoZIhvcNAQkBFg1zYXRAYmFpZHUuY29t
+ggEAMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEEBQADggEBAKfoCn8SpLk3uQyT
+X+oygcRWfTeJtN3D5J69NCMJ7wB+QPfpEBPwiqMgdbp4bRJ98H7x5UQsHT+EDOT/
+9OmipomHInFY4W1ew11zNKwuENeRrnZwTcCiVLZsxZsAU41ZeI5Yq+2WdtxnePCR
+VL1/NjKOq+WoRdb2nLSNDWgYMkLRVlt32hyzryyrBbmaxUl8BxnPqUiWduMwsZUz
+HNpXkoa1xTSd+En1SHYWfMg8BOVuV0I0/fjUUG9AXVqYpuogfbjAvibVNWAmxOfo
+fOjCPCGoJC1ET3AxYkgXGwioobz0pK/13k2pV+wu7W4g+6iTfz+hwZbPsUk2a/5I
+f6vXFB0=
+-----END CERTIFICATE-----

example/multi_threaded_echo_c++/client.cpp

@@ -0,0 +1,159 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A client sending requests to server by multiple threads.
+
+#include <gflags/gflags.h>
+#include <bthread/bthread.h>
+#include <butil/logging.h>
+#include <brpc/server.h>
+#include <brpc/channel.h>
+#include "echo.pb.h"
+#include <bvar/bvar.h>
+
+DEFINE_int32(thread_num, 50, "Number of threads to send requests");
+DEFINE_bool(use_bthread, false, "Use bthread to send requests");
+DEFINE_int32(attachment_size, 0, "Carry so many byte attachment along with requests");
+DEFINE_int32(request_size, 16, "Bytes of each request");
+DEFINE_string(protocol, "baidu_std", "Protocol type. Defined in src/brpc/options.proto");
+DEFINE_string(connection_type, "", "Connection type. Available values: single, pooled, short");
+DEFINE_string(server, "0.0.0.0:8002", "IP Address of server");
+DEFINE_string(load_balancer, "", "The algorithm for load balancing");
+DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); 
+DEFINE_bool(dont_fail, false, "Print fatal when some call failed");
+DEFINE_bool(enable_ssl, false, "Use SSL connection");
+DEFINE_int32(dummy_port, -1, "Launch dummy server at this port");
+
+std::string g_request;
+std::string g_attachment;
+
+bvar::LatencyRecorder g_latency_recorder("client");
+bvar::Adder<int> g_error_count("client_error_count");
+
+static void* sender(void* arg) {
+    // Normally, you should not call a Channel directly, but instead construct
+    // a stub Service wrapping it. stub can be shared by all threads as well.
+    example::EchoService_Stub stub(static_cast<google::protobuf::RpcChannel*>(arg));
+
+    int log_id = 0;
+    while (!brpc::IsAskedToQuit()) {
+        // We will receive response synchronously, safe to put variables
+        // on stack.
+        example::EchoRequest request;
+        example::EchoResponse response;
+        brpc::Controller cntl;
+
+        request.set_message(g_request);
+        cntl.set_log_id(log_id++);  // set by user
+        // Set attachment which is wired to network directly instead of 
+        // being serialized into protobuf messages.
+        cntl.request_attachment().append(g_attachment);
+
+        // Because `done'(last parameter) is NULL, this function waits until
+        // the response comes back or error occurs(including timedout).
+        stub.Echo(&cntl, &request, &response, NULL);
+        if (!cntl.Failed()) {
+            g_latency_recorder << cntl.latency_us();
+        } else {
+            g_error_count << 1; 
+            CHECK(brpc::IsAskedToQuit() || !FLAGS_dont_fail)
+                << "error=" << cntl.ErrorText() << " latency=" << cntl.latency_us();
+            // We can't connect to the server, sleep a while. Notice that this
+            // is a specific sleeping to prevent this thread from spinning too
+            // fast. You should continue the business logic in a production 
+            // server rather than sleeping.
+            bthread_usleep(50000);
+        }
+    }
+    return NULL;
+}
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    // A Channel represents a communication line to a Server. Notice that 
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::Channel channel;
+    
+    // Initialize the channel, NULL means using default options.
+    brpc::ChannelOptions options;
+    if (FLAGS_enable_ssl) {
+        options.mutable_ssl_options();
+    }
+    options.protocol = FLAGS_protocol;
+    options.connection_type = FLAGS_connection_type;
+    options.connect_timeout_ms = std::min(FLAGS_timeout_ms / 2, 100);
+    options.timeout_ms = FLAGS_timeout_ms;
+    options.max_retry = FLAGS_max_retry;
+    if (channel.Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), &options) != 0) {
+        LOG(ERROR) << "Fail to initialize channel";
+        return -1;
+    }
+
+    if (FLAGS_attachment_size > 0) {
+        g_attachment.resize(FLAGS_attachment_size, 'a');
+    }
+    if (FLAGS_request_size <= 0) {
+        LOG(ERROR) << "Bad request_size=" << FLAGS_request_size;
+        return -1;
+    }
+    g_request.resize(FLAGS_request_size, 'r');
+
+    if (FLAGS_dummy_port >= 0) {
+        brpc::StartDummyServerAt(FLAGS_dummy_port);
+    }
+
+    std::vector<bthread_t> bids;
+    std::vector<pthread_t> pids;
+    if (!FLAGS_use_bthread) {
+        pids.resize(FLAGS_thread_num);
+        for (int i = 0; i < FLAGS_thread_num; ++i) {
+            if (pthread_create(&pids[i], NULL, sender, &channel) != 0) {
+                LOG(ERROR) << "Fail to create pthread";
+                return -1;
+            }
+        }
+    } else {
+        bids.resize(FLAGS_thread_num);
+        for (int i = 0; i < FLAGS_thread_num; ++i) {
+            if (bthread_start_background(
+                    &bids[i], NULL, sender, &channel) != 0) {
+                LOG(ERROR) << "Fail to create bthread";
+                return -1;
+            }
+        }
+    }
+
+    while (!brpc::IsAskedToQuit()) {
+        sleep(1);
+        LOG(INFO) << "Sending EchoRequest at qps=" << g_latency_recorder.qps(1)
+                  << " latency=" << g_latency_recorder.latency(1);
+    }
+
+    LOG(INFO) << "EchoClient is going to quit";
+    for (int i = 0; i < FLAGS_thread_num; ++i) {
+        if (!FLAGS_use_bthread) {
+            pthread_join(pids[i], NULL);
+        } else {
+            bthread_join(bids[i], NULL);
+        }
+    }
+
+    return 0;
+}

example/multi_threaded_echo_c++/echo.proto

@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+syntax="proto2";
+option cc_generic_services = true;
+
+package example;
+
+message EchoRequest {
+      required string message = 1;
+};
+
+message EchoResponse {
+      required string message = 1;
+};
+
+service EchoService {
+      rpc Echo(EchoRequest) returns (EchoResponse);
+};

example/multi_threaded_echo_c++/key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAqncgHh2N/bamNUWFW36amY2ZSQ7WW9OM58Y4EK1/pipi25sL
+14CaI5X59kHIKeDSmBagxW/bVPxm2+N+nUb5B8ljs9ETzLKUdE00VNTSGPMEctAN
+SzgoCx7G0SR6pLRo8pXowC3YLLKRVsg2PWxH2+KFrJyhsyanLyA16z5jDjmOKKWw
+PUrn0s8EWTe2M3OYwHnaWhfUiu/EUF7b1dPiXwlMAbAVjynr0RRVKze60AWIz3IF
+fx4A7qrj66pxElUmMnxASmW2unJjW/sczt3AdW6z07aG0l4ftKP9ArUQPtputzg3
+scQi57YJD5uNiGAiSzNecU2rXH1V/9yx0N9Q6wIDAQABAoIBADN3khflnnhKzDXr
+To9IU08nRG+dbjT9U16rJ0RJze+SfpSFZHblWiSCZJzoUZHrUkofEt1pn1QyfK/J
+KPI9enTSZirlZk/4XwAaS0GNm/1yahZsIIdkZhqtaSO+GtVdrw4HGuXjMZCVPXJx
+MocrCSsnYmqyQ9P+SJ3e4Mis5mVllwDiUVlnTIamSSt16qkPdamLSJrxvI4LirQK
+9MZWNLoDFpRU1MJxQ/QzrEC3ONTq4j++AfbGzYTmDDtLeM8OSH5o72YXZ2JkaA4c
+xCzHFT+NaJYxF7esn/ctzGg50LYl8IF2UQtzOkX2l3l/OktIB1w+jGV6ONb1EWx5
+4zkkzNkCgYEA2EXj7GMsyNE3OYdMw8zrqQKUMON2CNnD+mBseGlr22/bhXtzpqK8
+uNel8WF1ezOnVvNsU8pml/W/mKUu6KQt5JfaDzen3OKjzTABVlbJxwFhPvwAeaIA
+q/tmSKyqiCgOMbR7Cq4UEwGf2A9/RII4JEC0/aipRU5srF65OYPUOJcCgYEAycco
+DFVG6jUw9w68t/X4f7NT4IYP96hSAqLUPuVz2fWwXKLWEX8JiMI+Ue3PbMz6mPcs
+4vMu364u4R3IuzrrI+PRK9iTa/pahBP6eF6ZpbY1ObI8CVLTrqUS9p22rr9lBm8V
+EZA9hwcHLYt+PWzaKcsFpbP4+AeY7nBBbL9CAM0CgYAzuJsmeB1ItUgIuQOxu7sM
+AzLfcjZTLYkBwreOIGAL7XdJN9nTmw2ZAvGLhWwsF5FIaRSaAUiBxOKaJb7PIhxb
+k7kxdHTvjT/xHS7ksAK3VewkvO18KTMR7iBq9ugdgb7LQkc+qZzhYr0QVbxw7Ndy
+TAs8sm4wxe2VV13ilFVXZwKBgDfU6ZnwBr1Llo7l/wYQA4CiSDU6IzTt2DNuhrgY
+mWPX/cLEM+OHeUXkKYZV/S0n0rd8vWjWzUOLWOFlcmOMPAAkS36MYM5h6aXeOVIR
+KwaVUkjyrnYN+xC6EHM41JGp1/RdzECd3sh8A1pw3K92bS9fQ+LD18IZqBFh8lh6
+23KJAoGAe48SwAsaGvqRO61Taww/Wf+YpGc9lnVbCvNFGScYaycPMqaRBUBmz/U3
+QQgpQY8T7JIECbA8sf78SlAZ9x93r0UQ70RekV3WzKAQHfHK8nqTjd3T0+i4aySO
+yQpYYCgE24zYO6rQgwrhzI0S4rWe7izDDlg0RmLtQh7Xw+rlkAQ=
+-----END RSA PRIVATE KEY-----

example/multi_threaded_echo_c++/server.cpp

@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A server to receive EchoRequest and send back EchoResponse.
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <brpc/server.h>
+#include "echo.pb.h"
+
+DEFINE_bool(echo_attachment, true, "Echo attachment as well");
+DEFINE_int32(port, 8002, "TCP Port of this server");
+DEFINE_int32(idle_timeout_s, -1, "Connection will be closed if there is no "
+             "read/write operations during the last `idle_timeout_s'");
+DEFINE_int32(max_concurrency, 0, "Limit of request processing in parallel");
+DEFINE_int32(internal_port, -1, "Only allow builtin services at this port");
+
+namespace example {
+// Your implementation of EchoService
+class EchoServiceImpl : public EchoService {
+public:
+    EchoServiceImpl() {}
+    ~EchoServiceImpl() {}
+    void Echo(google::protobuf::RpcController* cntl_base,
+              const EchoRequest* request,
+              EchoResponse* response,
+              google::protobuf::Closure* done) {
+        brpc::ClosureGuard done_guard(done);
+        brpc::Controller* cntl =
+            static_cast<brpc::Controller*>(cntl_base);
+
+        // Echo request and its attachment
+        response->set_message(request->message());
+        if (FLAGS_echo_attachment) {
+            cntl->response_attachment().append(cntl->request_attachment());
+        }
+    }
+};
+}  // namespace example
+
+DEFINE_bool(h, false, "print help information");
+
+int main(int argc, char* argv[]) {
+    std::string help_str = "dummy help infomation";
+    GFLAGS_NAMESPACE::SetUsageMessage(help_str);
+
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    if (FLAGS_h) {
+        fprintf(stderr, "%s\n%s\n%s", help_str.c_str(), help_str.c_str(), help_str.c_str());
+        return 0;
+    }
+
+    // Generally you only need one Server.
+    brpc::Server server;
+
+    // Instance of your service.
+    example::EchoServiceImpl echo_service_impl;
+
+    // Add the service into server. Notice the second parameter, because the
+    // service is put on stack, we don't want server to delete it, otherwise
+    // use brpc::SERVER_OWNS_SERVICE.
+    if (server.AddService(&echo_service_impl, 
+                          brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
+        LOG(ERROR) << "Fail to add service";
+        return -1;
+    }
+
+    // Start the server. 
+    brpc::ServerOptions options;
+    options.mutable_ssl_options()->default_cert.certificate = "cert.pem";
+    options.mutable_ssl_options()->default_cert.private_key = "key.pem";
+    options.idle_timeout_sec = FLAGS_idle_timeout_s;
+    options.max_concurrency = FLAGS_max_concurrency;
+    options.internal_port = FLAGS_internal_port;
+    if (server.Start(FLAGS_port, &options) != 0) {
+        LOG(ERROR) << "Fail to start EchoServer";
+        return -1;
+    }
+
+    // Wait until Ctrl-C is pressed, then Stop() and Join() the server.
+    server.RunUntilAskedToQuit();
+    return 0;
+}

example/multi_threaded_echo_fns_c++/CMakeLists.txt

@@ -0,0 +1,115 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.16...3.28)
+project(multi_threaded_echo_fns_c++ C CXX)
+
+include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake)
+
+option(LINK_SO "Whether examples are linked dynamically" OFF)
+
+execute_process(
+    COMMAND bash -c "find ${PROJECT_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'"
+    OUTPUT_VARIABLE OUTPUT_PATH
+)
+
+if(OUTPUT_PATH)
+    list(PREPEND CMAKE_PREFIX_PATH ${OUTPUT_PATH})
+endif()
+
+find_package(Threads REQUIRED)
+find_package(Protobuf REQUIRED)
+protobuf_generate_cpp(PROTO_SRC PROTO_HEADER echo.proto)
+
+# Search for libthrift* by best effort. If it is not found and brpc is
+# compiled with thrift protocol enabled, a link error would be reported.
+find_library(THRIFT_LIB NAMES thrift)
+if (NOT THRIFT_LIB)
+    set(THRIFT_LIB "")
+endif()
+
+find_path(GPERFTOOLS_INCLUDE_DIR NAMES gperftools/heap-profiler.h)
+find_library(GPERFTOOLS_LIBRARIES NAMES tcmalloc_and_profiler)
+
+find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h)
+if(LINK_SO)
+    find_library(BRPC_LIB NAMES brpc)
+else()
+    find_library(BRPC_LIB NAMES libbrpc.a brpc)
+endif()
+if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB))
+    message(FATAL_ERROR "Fail to find brpc")
+endif()
+
+find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h)
+find_library(GFLAGS_LIBRARY NAMES gflags libgflags)
+if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY))
+    message(FATAL_ERROR "Fail to find gflags")
+endif()
+
+set(BRPC_EXAMPLE_ENABLE_CPU_PROFILER ON)
+
+find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h)
+find_library(LEVELDB_LIB NAMES leveldb)
+if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB))
+    message(FATAL_ERROR "Fail to find leveldb")
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT OPENSSL_ROOT_DIR)
+    set(OPENSSL_ROOT_DIR
+        "/usr/local/opt/openssl"    # Homebrew installed OpenSSL
+        )
+endif()
+
+find_package(OpenSSL REQUIRED)
+
+set(DYNAMIC_LIB
+    Threads::Threads
+    ${GFLAGS_LIBRARY}
+    ${PROTOBUF_LIBRARIES}
+    ${LEVELDB_LIB}
+    ${OPENSSL_CRYPTO_LIBRARY}
+    ${OPENSSL_SSL_LIBRARY}
+    ${THRIFT_LIB}
+    dl
+    )
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    set(DYNAMIC_LIB ${DYNAMIC_LIB}
+        pthread
+        "-framework CoreFoundation"
+        "-framework CoreGraphics"
+        "-framework CoreData"
+        "-framework CoreText"
+        "-framework Security"
+        "-framework Foundation"
+        "-Wl,-U,_MallocExtension_ReleaseFreeMemory"
+        "-Wl,-U,_ProfilerStart"
+        "-Wl,-U,_ProfilerStop"
+        "-Wl,-U,__Z13GetStackTracePPvii"
+        "-Wl,-U,_mallctl"
+        "-Wl,-U,_malloc_stats_print"
+    )
+endif()
+
+add_executable(multi_threaded_echo_fns_client client.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(multi_threaded_echo_fns_client)
+add_executable(multi_threaded_echo_fns_server server.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(multi_threaded_echo_fns_server)
+
+target_link_libraries(multi_threaded_echo_fns_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB} ${GPERFTOOLS_LIBRARIES})
+target_link_libraries(multi_threaded_echo_fns_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB} ${GPERFTOOLS_LIBRARIES})

example/multi_threaded_echo_fns_c++/client.cpp

@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A client sending requests to servers(discovered by naming service) by multiple threads.
+
+#include <gflags/gflags.h>
+#include <bthread/bthread.h>
+#include <butil/logging.h>
+#include <butil/string_printf.h>
+#include <butil/time.h>
+#include <butil/macros.h>
+#include <brpc/channel.h>
+#include <brpc/server.h>
+#include <deque>
+#include "echo.pb.h"
+
+DEFINE_int32(thread_num, 50, "Number of threads to send requests");
+DEFINE_bool(use_bthread, false, "Use bthread to send requests");
+DEFINE_int32(attachment_size, 0, "Carry so many byte attachment along with requests");
+DEFINE_string(protocol, "baidu_std", "Protocol type. Defined in src/brpc/options.proto");
+DEFINE_string(connection_type, "", "Connection type. Available values: single, pooled, short");
+DEFINE_string(server, "file://server_list", "Addresses of servers");
+DEFINE_string(load_balancer, "rr", "Name of load balancer");
+DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds");
+DEFINE_int32(backup_timeout_ms, -1, "backup timeout in milliseconds");
+DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); 
+DEFINE_bool(dont_fail, false, "Print fatal when some call failed");
+DEFINE_int32(dummy_port, -1, "Launch dummy server at this port");
+
+std::string g_attachment;
+
+bvar::LatencyRecorder g_latency_recorder("client");
+bvar::Adder<int> g_error_count("client_error_count");
+butil::static_atomic<int> g_sender_count = BUTIL_STATIC_ATOMIC_INIT(0);
+
+static void* sender(void* arg) {
+    // Normally, you should not call a Channel directly, but instead construct
+    // a stub Service wrapping it. stub can be shared by all threads as well.
+    example::EchoService_Stub stub(static_cast<google::protobuf::RpcChannel*>(arg));
+
+    int log_id = 0;
+    while (!brpc::IsAskedToQuit()) {
+        // We will receive response synchronously, safe to put variables
+        // on stack.
+        example::EchoRequest request;
+        example::EchoResponse response;
+        brpc::Controller cntl;
+
+        const int thread_index = g_sender_count.fetch_add(1, butil::memory_order_relaxed);
+        const int input = ((thread_index & 0xFFF) << 20) | (log_id & 0xFFFFF);
+        request.set_value(input);
+        cntl.set_log_id(log_id ++);  // set by user
+        // Set attachment which is wired to network directly instead of 
+        // being serialized into protobuf messages.
+        cntl.request_attachment().append(g_attachment);
+
+        // Because `done'(last parameter) is NULL, this function waits until
+        // the response comes back or error occurs(including timedout).
+        stub.Echo(&cntl, &request, &response, NULL);
+        if (!cntl.Failed()) {
+            CHECK(response.value() == request.value() + 1);
+            g_latency_recorder << cntl.latency_us();
+        } else {
+            g_error_count << 1;
+            CHECK(brpc::IsAskedToQuit() || !FLAGS_dont_fail)        
+                << "input=(" << thread_index << "," << (input & 0xFFFFF)        
+                << ") error=" << cntl.ErrorText() << " latency=" << cntl.latency_us();
+            // We can't connect to the server, sleep a while. Notice that this
+            // is a specific sleeping to prevent this thread from spinning too
+            // fast. You should continue the business logic in a production 
+            // server rather than sleeping.
+            bthread_usleep(50000);
+        }
+    }
+    return NULL;
+}
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    // A Channel represents a communication line to a Server. Notice that 
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::Channel channel;
+    
+    // Initialize the channel, NULL means using default options.
+    brpc::ChannelOptions options;
+    options.backup_request_ms = FLAGS_backup_timeout_ms;
+    options.protocol = FLAGS_protocol;
+    options.connection_type = FLAGS_connection_type;
+    options.timeout_ms = FLAGS_timeout_ms/*milliseconds*/;
+    options.max_retry = FLAGS_max_retry;
+    if (channel.Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), &options) != 0) {
+        LOG(ERROR) << "Fail to initialize channel";
+        return -1;
+    }
+
+    if (FLAGS_attachment_size > 0) {
+        g_attachment.resize(FLAGS_attachment_size, 'a');
+    }
+
+    if (FLAGS_dummy_port >= 0) {
+        brpc::StartDummyServerAt(FLAGS_dummy_port);
+    }
+
+    std::vector<bthread_t> bids;
+    std::vector<pthread_t> pids;
+    if (!FLAGS_use_bthread) {
+        pids.resize(FLAGS_thread_num);
+        for (int i = 0; i < FLAGS_thread_num; ++i) {
+            if (pthread_create(&pids[i], NULL, sender, &channel) != 0) {
+                LOG(ERROR) << "Fail to create pthread";
+                return -1;
+            }
+        }
+    } else {
+        bids.resize(FLAGS_thread_num);
+        for (int i = 0; i < FLAGS_thread_num; ++i) {
+            if (bthread_start_background(
+                    &bids[i], NULL, sender, &channel) != 0) {
+                LOG(ERROR) << "Fail to create bthread";
+                return -1;
+            }
+        }
+    }
+
+    while (!brpc::IsAskedToQuit()) {
+        sleep(1);
+        LOG(INFO) << "Sending EchoRequest at qps=" << g_latency_recorder.qps(1)
+                  << " latency=" << g_latency_recorder.latency(1);
+    }
+
+    LOG(INFO) << "EchoClient is going to quit";
+    for (int i = 0; i < FLAGS_thread_num; ++i) {
+        if (!FLAGS_use_bthread) {
+            pthread_join(pids[i], NULL);
+        } else {
+            bthread_join(bids[i], NULL);
+        }
+    }
+
+    return 0;
+}

example/multi_threaded_echo_fns_c++/echo.proto

@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+syntax="proto2";
+package example;
+
+option cc_generic_services = true;
+
+message EchoRequest {
+      required int32 value = 1;
+};
+
+message EchoResponse {
+      required int32 value = 1;
+};
+
+service EchoService {
+      rpc Echo(EchoRequest) returns (EchoResponse);
+};

example/multi_threaded_echo_fns_c++/random_kill.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env sh
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+make -sj8 || exit -1
+
+function exit_func {
+    kill -- -$$
+    echo "Killed all"
+    exit 0
+}
+trap "exit_func" EXIT SIGINT SIGTERM
+
+echo > server_list
+starting_port=8004
+pids=()
+num_servers=5
+interval=2
+
+for ((i = 0; i < $num_servers; ++i)); do
+    ./echo_server -server_num 1 -port $((starting_port+i)) &
+    pids[$i]=$!
+done
+usleep 100000
+./echo_client -use_bthread -thread_num 400 -timeout_ms -1 --health_check_interval 1 -dont_fail > echo_client.log 2>&1 &
+
+echo ${pids[*]}
+
+while true; do
+    echo "Running for $interval seconds..."
+    sleep $interval
+    index=$((RANDOM%num_servers))
+    server_pid=${pids[$index]}
+    echo "Kill $server_pid(index=$index)"
+    kill -INT $server_pid
+    echo "Wait for another $interval seconds..."
+    sleep $interval
+    if grep -q "BAD\!\|^FATAL:" echo_client.log; then
+        echo "************** Found bug! ***************"
+        echo "Check echo_client.log for client logs"
+        exit 1
+    fi
+    ./echo_server -server_num 1 -port $((starting_port+index)) &
+    pids[$index]=$!
+    echo "Create echo_server=${pids[$index]} at index=$index"
+done

example/multi_threaded_echo_fns_c++/server.cpp

@@ -0,0 +1,188 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A server to receive EchoRequest and send back EchoResponse.
+
+#include <fcntl.h>  // O_RDONLY
+#include <vector>
+#include <gflags/gflags.h>
+#include <butil/time.h>
+#include <butil/logging.h>
+#include <butil/string_printf.h>
+#include <butil/string_splitter.h>
+#include <butil/rand_util.h>
+#include <brpc/server.h>
+#include "echo.pb.h"
+
+DEFINE_bool(send_attachment, false, "Carry attachment along with response");
+DEFINE_int32(port, 8004, "TCP Port of this server");
+DEFINE_int32(idle_timeout_s, -1, "Connection will be closed if there is no "
+             "read/write operations during the last `idle_timeout_s'");
+DEFINE_int32(max_concurrency, 0, "Limit of request processing in parallel");
+DEFINE_int32(server_num, 1, "Number of servers");
+DEFINE_string(sleep_us, "", "Sleep so many microseconds before responding");
+DEFINE_bool(spin, false, "spin rather than sleep");
+DEFINE_double(exception_ratio, 0.1, "Percentage of irregular latencies");
+DEFINE_double(min_ratio, 0.2, "min_sleep / sleep_us");
+DEFINE_double(max_ratio, 10, "max_sleep / sleep_us");
+
+// Your implementation of example::EchoService
+class EchoServiceImpl : public example::EchoService {
+public:
+    EchoServiceImpl() : _index(0) {}
+    virtual ~EchoServiceImpl() {}
+    void set_index(size_t index, int64_t sleep_us) { 
+        _index = index; 
+        _sleep_us = sleep_us;
+    }
+    virtual void Echo(google::protobuf::RpcController* cntl_base,
+                      const example::EchoRequest* request,
+                      example::EchoResponse* response,
+                      google::protobuf::Closure* done) {
+        brpc::ClosureGuard done_guard(done);
+        brpc::Controller* cntl =
+            static_cast<brpc::Controller*>(cntl_base);
+        if (_sleep_us > 0) {
+            double delay = _sleep_us;
+            const double a = FLAGS_exception_ratio * 0.5;
+            if (a >= 0.0001) {
+                double x = butil::RandDouble();
+                if (x < a) {
+                    const double min_sleep_us = FLAGS_min_ratio * _sleep_us;
+                    delay = min_sleep_us + (_sleep_us - min_sleep_us) * x / a;
+                } else if (x + a > 1) {
+                    const double max_sleep_us = FLAGS_max_ratio * _sleep_us;
+                    delay = _sleep_us + (max_sleep_us - _sleep_us) * (x + a - 1) / a;
+                }
+            }
+            if (FLAGS_spin) {
+                int64_t end_time = butil::cpuwide_time_us() + (int64_t)delay;
+                while (butil::cpuwide_time_us() < end_time) {}
+            } else {
+                bthread_usleep((int64_t)delay);
+            }
+        }
+
+        // Fill response.
+        response->set_value(request->value() + 1);
+        if (FLAGS_send_attachment) {
+            // Set attachment which is wired to network directly instead of
+            // being serialized into protobuf messages.
+            cntl->response_attachment().append("bar");
+        }
+        _nreq << 1;
+    }
+
+    size_t num_requests() const { return _nreq.get_value(); }
+
+private:
+    size_t _index;
+    int64_t _sleep_us;
+    bvar::Adder<size_t> _nreq;
+};
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    if (FLAGS_server_num <= 0) {
+        LOG(ERROR) << "server_num must be positive";
+        return -1;
+    }
+
+    // We need multiple servers in this example.
+    brpc::Server* servers = new brpc::Server[FLAGS_server_num];
+
+    butil::StringSplitter sp(FLAGS_sleep_us.c_str(), ',');
+    std::vector<int64_t> sleep_list;
+    for (; sp; ++sp) {
+        sleep_list.push_back(strtoll(sp.field(), NULL, 10));
+    }
+    if (sleep_list.empty()) {
+        sleep_list.push_back(0);
+    }
+
+    // Instance of your services.
+    EchoServiceImpl* echo_service_impls = new EchoServiceImpl[FLAGS_server_num];
+
+    // Add the service into servers. Notice the second parameter, because the
+    // service is put on stack, we don't want server to delete it, otherwise
+    // use brpc::SERVER_OWNS_SERVICE.
+    for (int i = 0; i < FLAGS_server_num; ++i) {
+        int64_t sleep_us = sleep_list[(size_t)i < sleep_list.size() ? i : (sleep_list.size() - 1)];
+        echo_service_impls[i].set_index(i, sleep_us);
+        servers[i].set_version(butil::string_printf(
+                    "example/multi_threaded_echo_fns_c++[%d]", i));
+        if (servers[i].AddService(&echo_service_impls[i], 
+                                  brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
+            LOG(ERROR) << "Fail to add service";
+            return -1;
+        }
+        // Start the server.
+        brpc::ServerOptions options;
+        options.idle_timeout_sec = FLAGS_idle_timeout_s;
+        options.max_concurrency = FLAGS_max_concurrency;
+        const int port = FLAGS_port + i;
+        if (servers[i].Start(port, &options) != 0) {
+            LOG(ERROR) << "Fail to start EchoServer";
+            return -1;
+        }
+
+        // Intended no truncate so that multiple servers can be added to list
+        int fd = open("./server_list", O_APPEND | O_WRONLY | O_CREAT, 0666);
+        if (fd < 0) {
+            PLOG(ERROR) << "Fail to open server_list";
+            return -1;
+        }
+        char buf[64];
+        int nw = snprintf(buf, sizeof(buf), "%s:%d\n", butil::my_ip_cstr(), port);
+        if (write(fd, buf, nw) != nw) {
+            LOG(ERROR) << "Fail to fully write int fd=" << fd;
+        }
+        close(fd);
+    }
+
+    // Service logic are running in separate worker threads, for main thread,
+    // we don't have much to do, just spinning.
+    std::vector<size_t> last_num_requests(FLAGS_server_num);
+    while (!brpc::IsAskedToQuit()) {
+        sleep(1);
+
+        size_t cur_total = 0;
+        for (int i = 0; i < FLAGS_server_num; ++i) {
+            const size_t current_num_requests =
+                    echo_service_impls[i].num_requests();
+            size_t diff = current_num_requests - last_num_requests[i];
+            cur_total += diff;
+            last_num_requests[i] = current_num_requests;
+            LOG(INFO) << "S[" << i << "]=" << diff << ' ' << noflush;
+        }
+        LOG(INFO) << "[total=" << cur_total << ']';
+    }
+
+    // Don't forget to stop and join the server otherwise still-running
+    // worker threads may crash your program. 
+    for (int i = 0; i < FLAGS_server_num; ++i) {
+        servers[i].Stop(0/*not used now*/);
+    }
+    for (int i = 0; i < FLAGS_server_num; ++i) {
+        servers[i].Join();
+    }
+    delete [] servers;
+    delete [] echo_service_impls;
+    return 0;
+}

example/mysql_c++/CMakeLists.txt

@@ -0,0 +1,148 @@
+cmake_minimum_required(VERSION 2.8.10)
+project(mysql_c++ C CXX)
+
+# Install dependencies:
+# With apt:
+# sudo apt-get install libreadline-dev 
+# sudo apt-get install ncurses-dev
+# With yum:
+# sudo yum install readline-devel
+# sudo yum install ncurses-devel
+
+option(EXAMPLE_LINK_SO "Whether examples are linked dynamically" OFF)
+
+execute_process(
+    COMMAND bash -c "find ${PROJECT_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'"
+    OUTPUT_VARIABLE OUTPUT_PATH
+)
+
+set(CMAKE_PREFIX_PATH ${OUTPUT_PATH})
+
+include(FindThreads)
+include(FindProtobuf)
+
+# Search for libthrift* by best effort. If it is not found and brpc is
+# compiled with thrift protocol enabled, a link error would be reported.
+find_library(THRIFT_LIB NAMES thrift)
+if (NOT THRIFT_LIB)
+    set(THRIFT_LIB "")
+endif()
+find_library(THRIFTNB_LIB NAMES thriftnb)
+if (NOT THRIFTNB_LIB)
+    set(THRIFTNB_LIB "")
+endif()
+
+find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h)
+if(EXAMPLE_LINK_SO)
+    find_library(BRPC_LIB NAMES brpc)
+else()
+    find_library(BRPC_LIB NAMES libbrpc.a brpc)
+endif()
+if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB))
+    message(FATAL_ERROR "Fail to find brpc")
+endif()
+include_directories(${BRPC_INCLUDE_PATH})
+
+find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h)
+find_library(GFLAGS_LIBRARY NAMES gflags libgflags)
+if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY))
+    message(FATAL_ERROR "Fail to find gflags")
+endif()
+include_directories(${GFLAGS_INCLUDE_PATH})
+
+execute_process(
+    COMMAND bash -c "grep \"namespace [_A-Za-z0-9]\\+ {\" ${GFLAGS_INCLUDE_PATH}/gflags/gflags_declare.h | head -1 | awk '{print $2}' | tr -d '\n'"
+    OUTPUT_VARIABLE GFLAGS_NS
+)
+if(${GFLAGS_NS} STREQUAL "GFLAGS_NAMESPACE")
+    execute_process(
+        COMMAND bash -c "grep \"#define GFLAGS_NAMESPACE [_A-Za-z0-9]\\+\" ${GFLAGS_INCLUDE_PATH}/gflags/gflags_declare.h | head -1 | awk '{print $3}' | tr -d '\n'"
+        OUTPUT_VARIABLE GFLAGS_NS
+    )
+endif()
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    include(CheckFunctionExists)
+    CHECK_FUNCTION_EXISTS(clock_gettime HAVE_CLOCK_GETTIME)
+    if(NOT HAVE_CLOCK_GETTIME)
+        set(DEFINE_CLOCK_GETTIME "-DNO_CLOCK_GETTIME_IN_MAC")
+    endif()
+endif()
+
+set(CMAKE_CPP_FLAGS "${DEFINE_CLOCK_GETTIME} -DGFLAGS_NS=${GFLAGS_NS}")
+set(CMAKE_CXX_FLAGS "${CMAKE_CPP_FLAGS} -DNDEBUG -O2 -D__const__= -pipe -W -Wall -Wno-unused-parameter -fPIC -fno-omit-frame-pointer")
+
+if(CMAKE_VERSION VERSION_LESS "3.1.3")
+    if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+    endif()
+    if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+    endif()
+else()
+    set(CMAKE_CXX_STANDARD 11)
+    set(CMAKE_CXX_STANDARD_REQUIRED ON)
+endif()
+
+find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h)
+find_library(LEVELDB_LIB NAMES leveldb)
+if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB))
+    message(FATAL_ERROR "Fail to find leveldb")
+endif()
+include_directories(${LEVELDB_INCLUDE_PATH})
+
+find_library(SSL_LIB NAMES ssl)
+if (NOT SSL_LIB)
+    message(FATAL_ERROR "Fail to find ssl")
+endif()
+
+find_library(CRYPTO_LIB NAMES crypto)
+if (NOT CRYPTO_LIB)
+    message(FATAL_ERROR "Fail to find crypto")
+endif()
+
+# find_path(MYSQL_INCLUDE_PATH NAMES mysql/mysql.h)
+# find_library(MYSQL_LIB NAMES mysqlclient)
+# if (NOT MYSQL_LIB)
+#     message(FATAL_ERROR "Fail to find mysqlclient")
+# endif()
+# include_directories(${MYSQL_INCLUDE_PATH})
+
+set(DYNAMIC_LIB
+    ${CMAKE_THREAD_LIBS_INIT}
+    ${GFLAGS_LIBRARY}
+    ${PROTOBUF_LIBRARIES}
+    ${LEVELDB_LIB}
+    ${SSL_LIB}
+    ${CRYPTO_LIB}
+    ${THRIFT_LIB}
+    ${THRIFTNB_LIB}
+#    ${MYSQL_LIB}
+    dl
+    )
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    set(DYNAMIC_LIB ${DYNAMIC_LIB}
+        pthread
+        "-framework CoreFoundation"
+        "-framework CoreGraphics"
+        "-framework CoreData"
+        "-framework CoreText"
+        "-framework Security"
+        "-framework Foundation"
+        "-Wl,-U,_MallocExtension_ReleaseFreeMemory"
+        "-Wl,-U,_ProfilerStart"
+        "-Wl,-U,_ProfilerStop")
+endif()
+
+add_executable(mysql_cli mysql_cli.cpp)
+add_executable(mysql_tx mysql_tx.cpp)
+add_executable(mysql_stmt mysql_stmt.cpp)
+add_executable(mysql_press mysql_press.cpp)
+# add_executable(mysqlclient_press mysqlclient_press.cpp)
+
+set(AUX_LIB readline ncurses)
+target_link_libraries(mysql_cli ${BRPC_LIB} ${DYNAMIC_LIB} ${AUX_LIB})
+target_link_libraries(mysql_tx ${BRPC_LIB} ${DYNAMIC_LIB})
+target_link_libraries(mysql_stmt ${BRPC_LIB} ${DYNAMIC_LIB})
+target_link_libraries(mysql_press ${BRPC_LIB} ${DYNAMIC_LIB})
+# target_link_libraries(mysqlclient_press ${BRPC_LIB} ${DYNAMIC_LIB})

example/mysql_c++/mysql_cli.cpp

@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// A brpc based command-line interface to talk with mysql-server
+
+#include <signal.h>
+#include <stdio.h>
+#include <readline/readline.h>
+#include <readline/history.h>
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <brpc/channel.h>
+#include "brpc/policy/mysql/mysql.h"
+#include "brpc/policy/mysql/mysql_authenticator.h"
+
+DEFINE_string(connection_type, "pooled", "Connection type. Available values: pooled, short");
+DEFINE_string(server, "127.0.0.1", "IP Address of server");
+DEFINE_int32(port, 3306, "Port of server");
+DEFINE_string(user, "brpcuser", "user name");
+DEFINE_string(password, "12345678", "password");
+DEFINE_string(schema, "brpc_test", "schema");
+DEFINE_string(params, "", "params");
+DEFINE_string(collation, "utf8mb4_general_ci", "collation");
+DEFINE_int32(timeout_ms, 5000, "RPC timeout in milliseconds");
+DEFINE_int32(connect_timeout_ms, 5000, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 0, "Max retries(not including the first RPC)");
+
+namespace brpc {
+const char* logo();
+}
+
+// Send `command' to mysql-server via `channel'
+static bool access_mysql(brpc::Channel& channel, const char* command) {
+    brpc::MysqlRequest request;
+    if (!request.Query(command)) {
+        LOG(ERROR) << "Fail to add command";
+        return false;
+    }
+    brpc::MysqlResponse response;
+    brpc::Controller cntl;
+    channel.CallMethod(NULL, &cntl, &request, &response, NULL);
+    if (!cntl.Failed()) {
+        std::cout << response << std::endl;
+    } else {
+        LOG(ERROR) << "Fail to access mysql, " << cntl.ErrorText();
+        return false;
+    }
+    return true;
+}
+
+// For freeing the memory returned by readline().
+struct Freer {
+    void operator()(char* mem) {
+        free(mem);
+    }
+};
+
+static void dummy_handler(int) {}
+
+// The getc for readline. The default getc retries reading when meeting
+// EINTR, which is not what we want.
+static bool g_canceled = false;
+static int cli_getc(FILE* stream) {
+    int c = getc(stream);
+    if (c == EOF && errno == EINTR) {
+        g_canceled = true;
+        return '\n';
+    }
+    return c;
+}
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NS::ParseCommandLineFlags(&argc, &argv, true);
+
+    // A Channel represents a communication line to a Server. Notice that
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::Channel channel;
+
+    // Initialize the channel, NULL means using default options.
+    brpc::ChannelOptions options;
+    options.protocol = brpc::PROTOCOL_MYSQL;
+    options.connection_type = FLAGS_connection_type;
+    options.timeout_ms = FLAGS_timeout_ms /*milliseconds*/;
+    options.connect_timeout_ms = FLAGS_connect_timeout_ms;
+    options.max_retry = FLAGS_max_retry;
+    options.auth = new brpc::policy::MysqlAuthenticator(
+        FLAGS_user, FLAGS_password, FLAGS_schema, FLAGS_params, FLAGS_collation);
+    if (channel.Init(FLAGS_server.c_str(), FLAGS_port, &options) != 0) {
+        LOG(ERROR) << "Fail to initialize channel";
+        return -1;
+    }
+
+    if (argc <= 1) {  // interactive mode
+        // We need this dummy signal hander to interrupt getc (and returning
+        // EINTR), SIG_IGN did not work.
+        signal(SIGINT, dummy_handler);
+
+        // Hook getc of readline.
+        rl_getc_function = cli_getc;
+
+        // Print welcome information.
+        printf("%s\n", brpc::logo());
+        printf(
+            "This command-line tool mimics the look-n-feel of official "
+            "mysql-cli, as a demostration of brpc's capability of"
+            " talking to mysql-server. The output and behavior is "
+            "not exactly same with the official one.\n\n");
+
+        for (;;) {
+            char prompt[128];
+            snprintf(prompt, sizeof(prompt), "mysql %s> ", FLAGS_server.c_str());
+            std::unique_ptr<char, Freer> command(readline(prompt));
+            if (command == NULL || *command == '\0') {
+                if (g_canceled) {
+                    // No input after the prompt and user pressed Ctrl-C,
+                    // quit the CLI.
+                    return 0;
+                }
+                // User entered an empty command by just pressing Enter.
+                continue;
+            }
+            if (g_canceled) {
+                // User entered sth. and pressed Ctrl-C, start a new prompt.
+                g_canceled = false;
+                continue;
+            }
+            // Add user's command to history so that it's browse-able by
+            // UP-key and search-able by Ctrl-R.
+            add_history(command.get());
+
+            if (!strcmp(command.get(), "help")) {
+                printf("This is a mysql CLI written in brpc.\n");
+                continue;
+            }
+            if (!strcmp(command.get(), "quit")) {
+                // Although quit is a valid mysql command, it does not make
+                // too much sense to run it in this CLI, just quit.
+                return 0;
+            }
+            access_mysql(channel, command.get());
+        }
+    } else {
+        std::string command;
+        command.reserve(argc * 16);
+        for (int i = 1; i < argc; ++i) {
+            if (i != 1) {
+                command.push_back(';');
+            }
+            command.append(argv[i]);
+        }
+        if (!access_mysql(channel, command.c_str())) {
+            return -1;
+        }
+    }
+    return 0;
+}

example/mysql_c++/mysql_go_press.go

@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"database/sql"
+	"flag"
+	"fmt"
+	_ "github.com/go-sql-driver/mysql"
+	"log"
+	"sync/atomic"
+	"time"
+)
+
+var thread_num int
+
+func init() {
+	flag.IntVar(&thread_num, "thread_num", 1, "thread number")
+}
+
+var cost int64
+var qps int64 = 1
+
+func main() {
+	flag.Parse()
+
+	db, err := sql.Open("mysql", "brpcuser:12345678@tcp(127.0.0.1:3306)/brpc_test?charset=utf8")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for i := 0; i < thread_num; i++ {
+		go func() {
+			for {
+				var (
+					id   int
+					col1 string
+					col2 string
+					col3 string
+					col4 string
+				)
+				start := time.Now()
+				rows, err := db.Query("select * from brpc_press where id = 1")
+				if err != nil {
+					log.Fatal(err)
+				}
+				for rows.Next() {
+					if err := rows.Scan(&id, &col1, &col2, &col3, &col4); err != nil {
+						log.Fatal(err)
+					}
+				}
+				atomic.AddInt64(&cost, time.Since(start).Nanoseconds())
+				atomic.AddInt64(&qps, 1)
+			}
+		}()
+	}
+
+	var q int64 = 0
+	for {
+		fmt.Println("qps =", qps-q, "latency =", cost/(qps-q)/1000)
+		q = atomic.LoadInt64(&qps)
+		atomic.StoreInt64(&cost, 0)
+		time.Sleep(1 * time.Second)
+	}
+}

example/mysql_c++/mysql_press.cpp

@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// A brpc based command-line interface to talk with mysql-server
+
+#include <sstream>
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <brpc/channel.h>
+#include "brpc/policy/mysql/mysql.h"
+#include "brpc/policy/mysql/mysql_authenticator.h"
+#include <bvar/bvar.h>
+#include <bthread/bthread.h>
+#include <brpc/server.h>
+
+DEFINE_string(connection_type, "pooled", "Connection type. Available values: pooled, short");
+DEFINE_string(server, "127.0.0.1", "IP Address of server");
+DEFINE_int32(port, 3306, "Port of server");
+DEFINE_string(user, "brpcuser", "user name");
+DEFINE_string(password, "12345678", "password");
+DEFINE_string(schema, "brpc_test", "schema");
+DEFINE_string(params, "", "params");
+DEFINE_string(collation, "utf8mb4_general_ci", "collation");
+DEFINE_string(data, "ABCDEF", "data");
+DEFINE_int32(timeout_ms, 5000, "RPC timeout in milliseconds");
+DEFINE_int32(connect_timeout_ms, 5000, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)");
+DEFINE_int32(thread_num, 50, "Number of threads to send requests");
+DEFINE_bool(use_bthread, false, "Use bthread to send requests");
+DEFINE_int32(dummy_port, -1, "port of dummy server(for monitoring)");
+DEFINE_int32(op_type, 0, "CRUD operation, 0:INSERT, 1:SELECT, 2:UPDATE");
+DEFINE_bool(dont_fail, false, "Print fatal when some call failed");
+
+bvar::LatencyRecorder g_latency_recorder("client");
+bvar::Adder<int> g_error_count("client_error_count");
+
+struct SenderArgs {
+    int base_index;
+    brpc::Channel* mysql_channel;
+};
+
+const std::string insert =
+    "insert into brpc_press(col1,col2,col3,col4) values "
+    "('"
+    "ABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCA"
+    "BCABCABCABCABCABCABCA', '" +
+    FLAGS_data +
+    "' ,1.5, "
+    "now())";
+// Send `command' to mysql-server via `channel'
+static void* sender(void* void_args) {
+    SenderArgs* args = (SenderArgs*)void_args;
+    std::stringstream command;
+    if (FLAGS_op_type == 0) {
+        command << insert;
+    } else if (FLAGS_op_type == 1) {
+        command << "select * from brpc_press where id = " << args->base_index + 1;
+    } else if (FLAGS_op_type == 2) {
+        command << "update brpc_press set col2 = '" + FLAGS_data + "' where id = "
+                << args->base_index + 1;
+    } else {
+        LOG(ERROR) << "wrong op type " << FLAGS_op_type;
+    }
+
+    brpc::MysqlRequest request;
+    if (!request.Query(command.str())) {
+        LOG(ERROR) << "Fail to execute command";
+        return NULL;
+    }
+
+    while (!brpc::IsAskedToQuit()) {
+        brpc::MysqlResponse response;
+        brpc::Controller cntl;
+        args->mysql_channel->CallMethod(NULL, &cntl, &request, &response, NULL);
+        const int64_t elp = cntl.latency_us();
+        if (!cntl.Failed()) {
+            g_latency_recorder << elp;
+            if (FLAGS_op_type == 0) {
+                CHECK_EQ(response.reply(0).is_ok(), true);
+            } else if (FLAGS_op_type == 1) {
+                CHECK_EQ(response.reply(0).row_count(), 1);
+            } else if (FLAGS_op_type == 2) {
+                CHECK_EQ(response.reply(0).is_ok(), true);
+            }
+        } else {
+            g_error_count << 1;
+            CHECK(brpc::IsAskedToQuit() || !FLAGS_dont_fail)
+                << "error=" << cntl.ErrorText() << " latency=" << elp;
+            // We can't connect to the server, sleep a while. Notice that this
+            // is a specific sleeping to prevent this thread from spinning too
+            // fast. You should continue the business logic in a production
+            // server rather than sleeping.
+            bthread_usleep(50000);
+        }
+    }
+    return NULL;
+}
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NS::ParseCommandLineFlags(&argc, &argv, true);
+
+    // A Channel represents a communication line to a Server. Notice that
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::Channel channel;
+
+    // Initialize the channel, NULL means using default options.
+    brpc::ChannelOptions options;
+    options.protocol = brpc::PROTOCOL_MYSQL;
+    options.connection_type = FLAGS_connection_type;
+    options.timeout_ms = FLAGS_timeout_ms /*milliseconds*/;
+    options.connect_timeout_ms = FLAGS_connect_timeout_ms;
+    options.max_retry = FLAGS_max_retry;
+    options.auth = new brpc::policy::MysqlAuthenticator(
+        FLAGS_user, FLAGS_password, FLAGS_schema, FLAGS_params, FLAGS_collation);
+    if (channel.Init(FLAGS_server.c_str(), FLAGS_port, &options) != 0) {
+        LOG(ERROR) << "Fail to initialize channel";
+        return -1;
+    }
+
+    // create table brpc_press
+    {
+        brpc::MysqlRequest request;
+        if (!request.Query(
+                "CREATE TABLE IF NOT EXISTS `brpc_press`(`id` INT UNSIGNED AUTO_INCREMENT, `col1` "
+                "VARCHAR(100) NOT NULL, `col2` VARCHAR(1024) NOT NULL, `col3` decimal(10,0) NOT "
+                "NULL, `col4` DATE, PRIMARY KEY ( `id` )) ENGINE=InnoDB DEFAULT CHARSET=utf8;")) {
+            LOG(ERROR) << "Fail to create table";
+            return -1;
+        }
+        brpc::MysqlResponse response;
+        brpc::Controller cntl;
+        channel.CallMethod(NULL, &cntl, &request, &response, NULL);
+        if (!cntl.Failed()) {
+            std::cout << response << std::endl;
+        } else {
+            LOG(ERROR) << "Fail to access mysql, " << cntl.ErrorText();
+            return -1;
+        }
+    }
+
+    // truncate table
+    {
+        brpc::MysqlRequest request;
+        if (!request.Query("truncate table brpc_press")) {
+            LOG(ERROR) << "Fail to truncate table";
+            return -1;
+        }
+        brpc::MysqlResponse response;
+        brpc::Controller cntl;
+        channel.CallMethod(NULL, &cntl, &request, &response, NULL);
+        if (!cntl.Failed()) {
+            std::cout << response << std::endl;
+        } else {
+            LOG(ERROR) << "Fail to access mysql, " << cntl.ErrorText();
+            return -1;
+        }
+    }
+
+    // prepare data for select, update
+    if (FLAGS_op_type != 0) {
+        for (int i = 0; i < FLAGS_thread_num; ++i) {
+            brpc::MysqlRequest request;
+            if (!request.Query(insert)) {
+                LOG(ERROR) << "Fail to execute command";
+                return -1;
+            }
+            brpc::MysqlResponse response;
+            brpc::Controller cntl;
+            channel.CallMethod(NULL, &cntl, &request, &response, NULL);
+            if (cntl.Failed()) {
+                LOG(ERROR) << cntl.ErrorText();
+                return -1;
+            }
+            if (!response.reply(0).is_ok()) {
+                LOG(ERROR) << "prepare data failed";
+                return -1;
+            }
+        }
+    }
+
+    if (FLAGS_dummy_port >= 0) {
+        brpc::StartDummyServerAt(FLAGS_dummy_port);
+    }
+
+    // test CRUD operations
+    std::vector<bthread_t> bids;
+    std::vector<pthread_t> pids;
+    bids.resize(FLAGS_thread_num);
+    pids.resize(FLAGS_thread_num);
+    std::vector<SenderArgs> args;
+    args.resize(FLAGS_thread_num);
+    for (int i = 0; i < FLAGS_thread_num; ++i) {
+        args[i].base_index = i;
+        args[i].mysql_channel = &channel;
+        if (!FLAGS_use_bthread) {
+            if (pthread_create(&pids[i], NULL, sender, &args[i]) != 0) {
+                LOG(ERROR) << "Fail to create pthread";
+                return -1;
+            }
+        } else {
+            if (bthread_start_background(&bids[i], NULL, sender, &args[i]) != 0) {
+                LOG(ERROR) << "Fail to create bthread";
+                return -1;
+            }
+        }
+    }
+
+    while (!brpc::IsAskedToQuit()) {
+        sleep(1);
+
+        LOG(INFO) << "Accessing mysql-server at qps=" << g_latency_recorder.qps(1)
+                  << " latency=" << g_latency_recorder.latency(1);
+    }
+
+    LOG(INFO) << "mysql_client is going to quit";
+    for (int i = 0; i < FLAGS_thread_num; ++i) {
+        if (!FLAGS_use_bthread) {
+            pthread_join(pids[i], NULL);
+        } else {
+            bthread_join(bids[i], NULL);
+        }
+    }
+
+    return 0;
+}

example/mysql_c++/mysql_stmt.cpp

@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// A brpc based mysql transaction example
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <bthread/bthread.h>
+#include <brpc/channel.h>
+#include "brpc/policy/mysql/mysql.h"
+#include "brpc/policy/mysql/mysql_authenticator.h"
+
+DEFINE_string(connection_type, "pooled", "Connection type. Available values: pooled, short");
+DEFINE_string(server, "127.0.0.1", "IP Address of server");
+DEFINE_int32(port, 3306, "Port of server");
+DEFINE_string(user, "brpcuser", "user name");
+DEFINE_string(password, "12345678", "password");
+DEFINE_string(schema, "brpc_test", "schema");
+DEFINE_string(params, "", "params");
+DEFINE_string(collation, "utf8mb4_general_ci", "collation");
+DEFINE_int32(timeout_ms, 5000, "RPC timeout in milliseconds");
+DEFINE_int32(connect_timeout_ms, 5000, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 0, "Max retries(not including the first RPC)");
+DEFINE_int32(thread_num, 1, "Number of threads to send requests");
+DEFINE_int32(count, 1, "Number of request to send pre thread");
+
+namespace brpc {
+const char* logo();
+}
+
+struct SenderArgs {
+    brpc::Channel* mysql_channel;
+    brpc::MysqlStatement* mysql_stmt;
+    std::vector<std::string> commands;
+};
+
+// Send `command' to mysql-server via `channel'
+static void* access_mysql(void* void_args) {
+    SenderArgs* args = (SenderArgs*)void_args;
+    brpc::Channel* channel = args->mysql_channel;
+    brpc::MysqlStatement* stmt = args->mysql_stmt;
+    const std::vector<std::string>& commands = args->commands;
+
+    for (int i = 0; i < FLAGS_count; ++i) {
+        brpc::MysqlRequest request(stmt);
+        for (size_t i = 1; i < commands.size(); i += 2) {
+            if (commands[i] == "int8") {
+                int8_t val = strtol(commands[i + 1].c_str(), NULL, 10);
+                if (!request.AddParam(val)) {
+                    LOG(ERROR) << "Fail to add int8 param";
+                    return NULL;
+                }
+            } else if (commands[i] == "uint8") {
+                uint8_t val = strtoul(commands[i + 1].c_str(), NULL, 10);
+                if (!request.AddParam(val)) {
+                    LOG(ERROR) << "Fail to add uint8 param";
+                    return NULL;
+                }
+            } else if (commands[i] == "int16") {
+                int16_t val = strtol(commands[i + 1].c_str(), NULL, 10);
+                if (!request.AddParam(val)) {
+                    LOG(ERROR) << "Fail to add uint16 param";
+                    return NULL;
+                }
+            } else if (commands[i] == "uint16") {
+                uint16_t val = strtoul(commands[i + 1].c_str(), NULL, 10);
+                if (!request.AddParam(val)) {
+                    LOG(ERROR) << "Fail to add uint16 param";
+                    return NULL;
+                }
+            } else if (commands[i] == "int32") {
+                int32_t val = strtol(commands[i + 1].c_str(), NULL, 10);
+                if (!request.AddParam(val)) {
+                    LOG(ERROR) << "Fail to add int32 param";
+                    return NULL;
+                }
+            } else if (commands[i] == "uint32") {
+                uint32_t val = strtoul(commands[i + 1].c_str(), NULL, 10);
+                if (!request.AddParam(val)) {
+                    LOG(ERROR) << "Fail to add uint32 param";
+                    return NULL;
+                }
+            } else if (commands[i] == "int64") {
+                int64_t val = strtol(commands[i + 1].c_str(), NULL, 10);
+                if (!request.AddParam(val)) {
+                    LOG(ERROR) << "Fail to add int64 param";
+                    return NULL;
+                }
+            } else if (commands[i] == "uint64") {
+                uint64_t val = strtoul(commands[i + 1].c_str(), NULL, 10);
+                if (!request.AddParam(val)) {
+                    LOG(ERROR) << "Fail to add uint64 param";
+                    return NULL;
+                }
+            } else if (commands[i] == "float") {
+                float val = strtof(commands[i + 1].c_str(), NULL);
+                if (!request.AddParam(val)) {
+                    LOG(ERROR) << "Fail to add float param";
+                    return NULL;
+                }
+            } else if (commands[i] == "double") {
+                double val = strtod(commands[i + 1].c_str(), NULL);
+                if (!request.AddParam(val)) {
+                    LOG(ERROR) << "Fail to add double param";
+                    return NULL;
+                }
+            } else if (commands[i] == "string") {
+                if (!request.AddParam(commands[i + 1])) {
+                    LOG(ERROR) << "Fail to add string param";
+                    return NULL;
+                }
+            } else {
+                LOG(ERROR) << "Wrong param type " << commands[i];
+            }
+        }
+
+        brpc::MysqlResponse response;
+        brpc::Controller cntl;
+        channel->CallMethod(NULL, &cntl, &request, &response, NULL);
+        if (cntl.Failed()) {
+            LOG(ERROR) << "Fail to access mysql, " << cntl.ErrorText();
+            return NULL;
+        }
+
+        std::cout << response << std::endl;
+    }
+
+    return NULL;
+}
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NS::ParseCommandLineFlags(&argc, &argv, true);
+
+    // A Channel represents a communication line to a Server. Notice that
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::Channel channel;
+
+    // Initialize the channel, NULL means using default options.
+    brpc::ChannelOptions options;
+    options.protocol = brpc::PROTOCOL_MYSQL;
+    options.connection_type = FLAGS_connection_type;
+    options.timeout_ms = FLAGS_timeout_ms /*milliseconds*/;
+    options.connect_timeout_ms = FLAGS_connect_timeout_ms;
+    options.max_retry = FLAGS_max_retry;
+    options.auth = new brpc::policy::MysqlAuthenticator(
+        FLAGS_user, FLAGS_password, FLAGS_schema, FLAGS_params, FLAGS_collation);
+    if (channel.Init(FLAGS_server.c_str(), FLAGS_port, &options) != 0) {
+        LOG(ERROR) << "Fail to initialize channel";
+        return -1;
+    }
+
+    if (argc <= 1) {
+        LOG(ERROR) << "No sql statement args";
+    } else {
+        std::vector<std::string> commands;
+        commands.reserve(argc * 16);
+        for (int i = 1; i < argc; ++i) {
+            commands.push_back(argv[i]);
+        }
+        auto stmt(brpc::NewMysqlStatement(channel, commands[0]));
+        if (stmt == NULL) {
+            LOG(ERROR) << "Fail to create mysql statement";
+            return -1;
+        }
+
+        std::vector<SenderArgs> args;
+        std::vector<bthread_t> bids;
+        args.resize(FLAGS_thread_num);
+        bids.resize(FLAGS_thread_num);
+
+        for (int i = 0; i < FLAGS_thread_num; ++i) {
+            args[i].mysql_channel = &channel;
+            args[i].mysql_stmt = stmt.get();
+            args[i].commands = commands;
+            if (bthread_start_background(&bids[i], NULL, access_mysql, &args[i]) != 0) {
+                LOG(ERROR) << "Fail to create bthread";
+                return -1;
+            }
+        }
+
+        for (int i = 0; i < FLAGS_thread_num; ++i) {
+            bthread_join(bids[i], NULL);
+        }
+    }
+
+    return 0;
+}
+
+/* vim: set expandtab ts=4 sw=4 sts=4 tw=100: */

example/mysql_c++/mysql_tx.cpp

@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// A brpc based mysql transaction example
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <brpc/channel.h>
+#include "brpc/policy/mysql/mysql.h"
+#include "brpc/policy/mysql/mysql_authenticator.h"
+
+DEFINE_string(connection_type, "pooled", "Connection type. Available values: pooled, short");
+DEFINE_string(server, "127.0.0.1", "IP Address of server");
+DEFINE_int32(port, 3306, "Port of server");
+DEFINE_string(user, "brpcuser", "user name");
+DEFINE_string(password, "12345678", "password");
+DEFINE_string(schema, "brpc_test", "schema");
+DEFINE_string(params, "", "params");
+DEFINE_string(collation, "utf8mb4_general_ci", "collation");
+DEFINE_int32(timeout_ms, 5000, "RPC timeout in milliseconds");
+DEFINE_int32(connect_timeout_ms, 5000, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 0, "Max retries(not including the first RPC)");
+DEFINE_bool(readonly, false, "readonly transaction");
+DEFINE_int32(isolation_level, 0, "transaction isolation level");
+
+namespace brpc {
+const char* logo();
+}
+
+// Send `command' to mysql-server via `channel'
+static bool access_mysql(brpc::Channel& channel, const std::vector<std::string>& commands) {
+    brpc::MysqlTransactionOptions options;
+    options.readonly = FLAGS_readonly;
+    options.isolation_level = brpc::MysqlIsolationLevel(FLAGS_isolation_level);
+    auto tx(brpc::NewMysqlTransaction(channel, options));
+    if (tx == NULL) {
+        LOG(ERROR) << "Fail to create transaction";
+        return false;
+    }
+
+    for (auto it = commands.begin(); it != commands.end(); ++it) {
+        brpc::MysqlRequest request(tx.get());
+        if (!request.Query(*it)) {
+            LOG(ERROR) << "Fail to add command";
+            tx->rollback();
+            return false;
+        }
+        brpc::MysqlResponse response;
+        brpc::Controller cntl;
+        channel.CallMethod(NULL, &cntl, &request, &response, NULL);
+        if (cntl.Failed()) {
+            LOG(ERROR) << "Fail to access mysql, " << cntl.ErrorText();
+            tx->rollback();
+            return false;
+        }
+        // check response
+        std::cout << response << std::endl;
+        for (size_t i = 0; i < response.reply_size(); ++i) {
+            if (response.reply(i).is_error()) {
+                tx->rollback();
+                return false;
+            }
+        }
+    }
+    tx->commit();
+    return true;
+}
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NS::ParseCommandLineFlags(&argc, &argv, true);
+
+    // A Channel represents a communication line to a Server. Notice that
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::Channel channel;
+
+    // Initialize the channel, NULL means using default options.
+    brpc::ChannelOptions options;
+    options.protocol = brpc::PROTOCOL_MYSQL;
+    options.connection_type = FLAGS_connection_type;
+    options.timeout_ms = FLAGS_timeout_ms /*milliseconds*/;
+    options.connect_timeout_ms = FLAGS_connect_timeout_ms;
+    options.max_retry = FLAGS_max_retry;
+    options.auth = new brpc::policy::MysqlAuthenticator(
+        FLAGS_user, FLAGS_password, FLAGS_schema, FLAGS_params, FLAGS_collation);
+    if (channel.Init(FLAGS_server.c_str(), FLAGS_port, &options) != 0) {
+        LOG(ERROR) << "Fail to initialize channel";
+        return -1;
+    }
+
+    if (argc <= 1) {
+        LOG(ERROR) << "No sql statement args";
+    } else {
+        std::vector<std::string> commands;
+        commands.reserve(argc * 16);
+        for (int i = 1; i < argc; ++i) {
+            commands.push_back(argv[i]);
+        }
+        if (!access_mysql(channel, commands)) {
+            return -1;
+        }
+    }
+    return 0;
+}
+
+/* vim: set expandtab ts=4 sw=4 sts=4 tw=100: */

example/mysql_c++/mysqlclient_press.cpp

@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// A brpc based command-line interface to talk with mysql-server
+
+#include <sstream>
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+extern "C" {
+#include <mysql/mysql.h>
+}
+#include <brpc/controller.h>
+#include <bvar/bvar.h>
+#include <bthread/bthread.h>
+#include <brpc/server.h>
+
+DEFINE_string(server, "127.0.0.1", "IP Address of server");
+DEFINE_int32(port, 3306, "Port of server");
+DEFINE_string(user, "brpcuser", "user name");
+DEFINE_string(password, "12345678", "password");
+DEFINE_string(schema, "brpc_test", "schema");
+DEFINE_string(params, "", "params");
+DEFINE_string(data, "ABCDEF", "data");
+DEFINE_int32(thread_num, 50, "Number of threads to send requests");
+DEFINE_bool(use_bthread, false, "Use bthread to send requests");
+DEFINE_int32(dummy_port, -1, "port of dummy server(for monitoring)");
+DEFINE_int32(op_type, 0, "CRUD operation, 0:INSERT, 1:SELECT, 3:UPDATE");
+DEFINE_bool(dont_fail, false, "Print fatal when some call failed");
+
+bvar::LatencyRecorder g_latency_recorder("client");
+bvar::Adder<int> g_error_count("client_error_count");
+
+struct SenderArgs {
+    int base_index;
+    MYSQL* mysql_conn;
+};
+
+const std::string insert =
+    "insert into mysqlclient_press(col1,col2,col3,col4) values "
+    "('"
+    "ABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCA"
+    "BCABCABCABCABCABCABCA', '" +
+    FLAGS_data +
+    "' ,1.5, "
+    "now())";
+// Send `command' to mysql-server via `channel'
+static void* sender(void* void_args) {
+    SenderArgs* args = (SenderArgs*)void_args;
+    std::stringstream command;
+    if (FLAGS_op_type == 0) {
+        command << insert;
+    } else if (FLAGS_op_type == 1) {
+        command << "select * from mysqlclient_press where id = " << args->base_index + 1;
+    } else if (FLAGS_op_type == 2) {
+        command << "update brpc_press set col2 = '" + FLAGS_data + "' where id = "
+                << args->base_index + 1;
+    } else {
+        LOG(ERROR) << "wrong op type " << FLAGS_op_type;
+    }
+
+    std::string command_str = command.str();
+
+    while (!brpc::IsAskedToQuit()) {
+        const int64_t begin_time_us = butil::cpuwide_time_us();
+        const int rc = mysql_real_query(args->mysql_conn, command_str.c_str(), command_str.size());
+        if (rc != 0) {
+            goto ERROR;
+        }
+
+        if (mysql_errno(args->mysql_conn) == 0) {
+            if (FLAGS_op_type == 0) {
+                CHECK_EQ(mysql_affected_rows(args->mysql_conn), 1);
+            } else if (FLAGS_op_type == 1) {
+                MYSQL_RES* res = mysql_store_result(args->mysql_conn);
+                if (res == NULL) {
+                    LOG(INFO) << "not found";
+                } else {
+                    CHECK_EQ(mysql_num_rows(res), 1);
+                    mysql_free_result(res);
+                }
+            } else if (FLAGS_op_type == 2) {
+            }
+            const int64_t elp = butil::cpuwide_time_us() - begin_time_us;
+            g_latency_recorder << elp;
+        } else {
+            goto ERROR;
+        }
+
+        if (false) {
+        ERROR:
+            const int64_t elp = butil::cpuwide_time_us() - begin_time_us;
+            g_error_count << 1;
+            CHECK(brpc::IsAskedToQuit() || !FLAGS_dont_fail)
+                << "error=" << mysql_error(args->mysql_conn) << " latency=" << elp;
+            // We can't connect to the server, sleep a while. Notice that this
+            // is a specific sleeping to prevent this thread from spinning too
+            // fast. You should continue the business logic in a production
+            // server rather than sleeping.
+            bthread_usleep(50000);
+        }
+    }
+    return NULL;
+}
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NS::ParseCommandLineFlags(&argc, &argv, true);
+
+    if (FLAGS_dummy_port >= 0) {
+        brpc::StartDummyServerAt(FLAGS_dummy_port);
+    }
+
+    MYSQL* conn = mysql_init(NULL);
+    if (!mysql_real_connect(conn,
+                            FLAGS_server.c_str(),
+                            FLAGS_user.c_str(),
+                            FLAGS_password.c_str(),
+                            FLAGS_schema.c_str(),
+                            FLAGS_port,
+                            NULL,
+                            0)) {
+        LOG(ERROR) << mysql_error(conn);
+        return -1;
+    }
+
+    // create table mysqlclient_press
+    {
+        const char* sql =
+            "CREATE TABLE IF NOT EXISTS `mysqlclient_press`(`id` INT UNSIGNED AUTO_INCREMENT, "
+            "`col1` "
+            "VARCHAR(100) NOT NULL, `col2` VARCHAR(1024) NOT NULL, `col3` decimal(10,0) NOT "
+            "NULL, `col4` DATE, PRIMARY KEY ( `id` )) ENGINE=InnoDB DEFAULT CHARSET=utf8;";
+        const int rc = mysql_real_query(conn, sql, strlen(sql));
+        if (rc != 0) {
+            LOG(ERROR) << "Fail to execute sql, " << mysql_error(conn);
+            return -1;
+        }
+
+        if (mysql_errno(conn) != 0) {
+            LOG(ERROR) << "Fail to store result, " << mysql_error(conn);
+            return -1;
+        }
+    }
+
+    // truncate table
+    {
+        const char* sql = "truncate table mysqlclient_press";
+        const int rc = mysql_real_query(conn, sql, strlen(sql));
+        if (rc != 0) {
+            LOG(ERROR) << "Fail to execute sql, " << mysql_error(conn);
+            return -1;
+        }
+
+        if (mysql_errno(conn) != 0) {
+            LOG(ERROR) << "Fail to store result, " << mysql_error(conn);
+            return -1;
+        }
+    }
+
+    // prepare data for select, update
+    if (FLAGS_op_type != 0) {
+        for (int i = 0; i < FLAGS_thread_num; ++i) {
+            const int rc = mysql_real_query(conn, insert.c_str(), insert.size());
+            if (rc != 0) {
+                LOG(ERROR) << "Fail to execute sql, " << mysql_error(conn);
+                return -1;
+            }
+
+            if (mysql_errno(conn) != 0) {
+                LOG(ERROR) << "Fail to store result, " << mysql_error(conn);
+                return -1;
+            }
+        }
+    }
+
+    // test CRUD operations
+    std::vector<bthread_t> bids;
+    std::vector<pthread_t> pids;
+    bids.resize(FLAGS_thread_num);
+    pids.resize(FLAGS_thread_num);
+    std::vector<SenderArgs> args;
+    args.resize(FLAGS_thread_num);
+    for (int i = 0; i < FLAGS_thread_num; ++i) {
+        MYSQL* conn = mysql_init(NULL);
+        if (!mysql_real_connect(conn,
+                                FLAGS_server.c_str(),
+                                FLAGS_user.c_str(),
+                                FLAGS_password.c_str(),
+                                FLAGS_schema.c_str(),
+                                FLAGS_port,
+                                NULL,
+                                0)) {
+            LOG(ERROR) << mysql_error(conn);
+            return -1;
+        }
+        args[i].base_index = i;
+        args[i].mysql_conn = conn;
+        if (!FLAGS_use_bthread) {
+            if (pthread_create(&pids[i], NULL, sender, &args[i]) != 0) {
+                LOG(ERROR) << "Fail to create pthread";
+                return -1;
+            }
+        } else {
+            if (bthread_start_background(&bids[i], NULL, sender, &args[i]) != 0) {
+                LOG(ERROR) << "Fail to create bthread";
+                return -1;
+            }
+        }
+    }
+
+    while (!brpc::IsAskedToQuit()) {
+        sleep(1);
+
+        LOG(INFO) << "Accessing mysql-server at qps=" << g_latency_recorder.qps(1)
+                  << " latency=" << g_latency_recorder.latency(1);
+    }
+
+    LOG(INFO) << "mysql_client is going to quit";
+    for (int i = 0; i < FLAGS_thread_num; ++i) {
+        if (!FLAGS_use_bthread) {
+            pthread_join(pids[i], NULL);
+        } else {
+            bthread_join(bids[i], NULL);
+        }
+    }
+
+    return 0;
+}

example/nshead_extension_c++/CMakeLists.txt

@@ -0,0 +1,110 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.16...3.28)
+project(nshead_extension_c++ C CXX)
+
+include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake)
+
+option(LINK_SO "Whether examples are linked dynamically" OFF)
+
+execute_process(
+    COMMAND bash -c "find ${PROJECT_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'"
+    OUTPUT_VARIABLE OUTPUT_PATH
+)
+
+if(OUTPUT_PATH)
+    list(PREPEND CMAKE_PREFIX_PATH ${OUTPUT_PATH})
+endif()
+
+find_package(Threads REQUIRED)
+find_package(Protobuf REQUIRED)
+protobuf_generate_cpp(PROTO_SRC PROTO_HEADER echo.proto)
+
+# Search for libthrift* by best effort. If it is not found and brpc is
+# compiled with thrift protocol enabled, a link error would be reported.
+find_library(THRIFT_LIB NAMES thrift)
+if (NOT THRIFT_LIB)
+    set(THRIFT_LIB "")
+endif()
+
+find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h)
+if(LINK_SO)
+    find_library(BRPC_LIB NAMES brpc)
+else()
+    find_library(BRPC_LIB NAMES libbrpc.a brpc)
+endif()
+if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB))
+    message(FATAL_ERROR "Fail to find brpc")
+endif()
+
+find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h)
+find_library(GFLAGS_LIBRARY NAMES gflags libgflags)
+if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY))
+    message(FATAL_ERROR "Fail to find gflags")
+endif()
+
+find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h)
+find_library(LEVELDB_LIB NAMES leveldb)
+if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB))
+    message(FATAL_ERROR "Fail to find leveldb")
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT OPENSSL_ROOT_DIR)
+    set(OPENSSL_ROOT_DIR
+        "/usr/local/opt/openssl"    # Homebrew installed OpenSSL
+        )
+endif()
+
+find_package(OpenSSL REQUIRED)
+
+set(DYNAMIC_LIB
+    Threads::Threads
+    ${GFLAGS_LIBRARY}
+    ${PROTOBUF_LIBRARIES}
+    ${LEVELDB_LIB}
+    ${OPENSSL_CRYPTO_LIBRARY}
+    ${OPENSSL_SSL_LIBRARY}
+    ${THRIFT_LIB}
+    dl
+    )
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    set(DYNAMIC_LIB ${DYNAMIC_LIB}
+        pthread
+        "-framework CoreFoundation"
+        "-framework CoreGraphics"
+        "-framework CoreData"
+        "-framework CoreText"
+        "-framework Security"
+        "-framework Foundation"
+        "-Wl,-U,_MallocExtension_ReleaseFreeMemory"
+        "-Wl,-U,_ProfilerStart"
+        "-Wl,-U,_ProfilerStop"
+        "-Wl,-U,__Z13GetStackTracePPvii"
+        "-Wl,-U,_mallctl"
+        "-Wl,-U,_malloc_stats_print"
+    )
+endif()
+
+add_executable(nshead_extension_client client.cpp)
+brpc_example_configure_target(nshead_extension_client)
+add_executable(nshead_extension_server server.cpp)
+brpc_example_configure_target(nshead_extension_server)
+
+target_link_libraries(nshead_extension_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})
+target_link_libraries(nshead_extension_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})

example/nshead_extension_c++/client.cpp

@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A client sending requests to server every 1 second.
+
+#include <gflags/gflags.h>
+
+#include <butil/logging.h>
+#include <butil/time.h>
+#include <butil/strings/string_piece.h>
+#include <brpc/channel.h>
+#include <brpc/nshead_message.h>
+#include <bvar/bvar.h>
+
+bvar::LatencyRecorder g_latency_recorder("client");
+
+DEFINE_string(server, "0.0.0.0:8010", "IP Address of server");
+DEFINE_string(load_balancer, "", "The algorithm for load balancing");
+DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); 
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+    
+    // A Channel represents a communication line to a Server. Notice that 
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::Channel channel;
+    
+    // Initialize the channel, NULL means using default options. 
+    brpc::ChannelOptions options;
+    options.protocol = brpc::PROTOCOL_NSHEAD;
+    options.timeout_ms = FLAGS_timeout_ms/*milliseconds*/;
+    options.max_retry = FLAGS_max_retry;
+    if (channel.Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), &options) != 0) {
+        LOG(ERROR) << "Fail to initialize channel";
+        return -1;
+    }
+
+    // Send a request and wait for the response every 1 second.
+    int log_id = 0;
+    while (!brpc::IsAskedToQuit()) {
+        brpc::NsheadMessage request;
+        brpc::NsheadMessage response;
+        brpc::Controller cntl;
+
+        // Append message to `request'
+        request.body.append("hello world");
+
+        cntl.set_log_id(log_id ++);  // set by user
+
+        // Because `done'(last parameter) is NULL, this function waits until
+        // the response comes back or error occurs(including timedout).
+        channel.CallMethod(NULL, &cntl, &request, &response, NULL);
+        if (cntl.Failed()) {
+            LOG(ERROR) << "Fail to send nshead request, " << cntl.ErrorText();
+            sleep(1); // Remove this sleep in production code.
+        } else {
+            g_latency_recorder << cntl.latency_us();
+        }
+        LOG_EVERY_SECOND(INFO)
+            << "Sending nshead requests at qps=" << g_latency_recorder.qps(1)
+            << " latency=" << g_latency_recorder.latency(1);
+    }
+
+    LOG(INFO) << "EchoClient is going to quit";
+    return 0;
+}

example/nshead_extension_c++/server.cpp

@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A server to receive EchoRequest and send back EchoResponse.
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <brpc/server.h>
+#include <brpc/nshead_service.h>
+
+DEFINE_int32(port, 8010, "TCP Port of this server");
+DEFINE_int32(idle_timeout_s, -1, "Connection will be closed if there is no "
+             "read/write operations during the last `idle_timeout_s'");
+DEFINE_int32(max_concurrency, 0, "Limit of request processing in parallel");
+
+// Adapt your own nshead-based protocol to use brpc 
+class MyNsheadProtocol : public brpc::NsheadService {
+public:
+    void ProcessNsheadRequest(const brpc::Server&,
+                              brpc::Controller* cntl,
+                              const brpc::NsheadMessage& request,
+                              brpc::NsheadMessage* response, 
+                              brpc::NsheadClosure* done) {
+        // This object helps you to call done->Run() in RAII style. If you need
+        // to process the request asynchronously, pass done_guard.release().
+        brpc::ClosureGuard done_guard(done);
+
+        if (cntl->Failed()) {
+            // NOTE: You can send back a response containing error information
+            // back to client instead of closing the connection.
+            cntl->CloseConnection("Close connection due to previous error");
+            return;
+        }
+        *response = request; // Just echo the request to client
+    }
+};
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    brpc::Server server;
+    brpc::ServerOptions options;
+    options.nshead_service = new MyNsheadProtocol;
+    options.idle_timeout_sec = FLAGS_idle_timeout_s;
+    options.max_concurrency = FLAGS_max_concurrency;
+
+    // Start the server.
+    if (server.Start(FLAGS_port, &options) != 0) {
+        LOG(ERROR) << "Fail to start EchoServer";
+        return -1;
+    }
+
+    // Wait until Ctrl-C is pressed, then Stop() and Join() the server.
+    server.RunUntilAskedToQuit();
+    return 0;
+}

example/nshead_pb_extension_c++/CMakeLists.txt

@@ -0,0 +1,110 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.16...3.28)
+project(nshead_pb_extension_c++ C CXX)
+
+include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake)
+
+option(LINK_SO "Whether examples are linked dynamically" OFF)
+
+execute_process(
+    COMMAND bash -c "find ${PROJECT_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'"
+    OUTPUT_VARIABLE OUTPUT_PATH
+)
+
+if(OUTPUT_PATH)
+    list(PREPEND CMAKE_PREFIX_PATH ${OUTPUT_PATH})
+endif()
+
+find_package(Threads REQUIRED)
+find_package(Protobuf REQUIRED)
+protobuf_generate_cpp(PROTO_SRC PROTO_HEADER echo.proto)
+
+# Search for libthrift* by best effort. If it is not found and brpc is
+# compiled with thrift protocol enabled, a link error would be reported.
+find_library(THRIFT_LIB NAMES thrift)
+if (NOT THRIFT_LIB)
+    set(THRIFT_LIB "")
+endif()
+
+find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h)
+if(LINK_SO)
+    find_library(BRPC_LIB NAMES brpc)
+else()
+    find_library(BRPC_LIB NAMES libbrpc.a brpc)
+endif()
+if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB))
+    message(FATAL_ERROR "Fail to find brpc")
+endif()
+
+find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h)
+find_library(GFLAGS_LIBRARY NAMES gflags libgflags)
+if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY))
+    message(FATAL_ERROR "Fail to find gflags")
+endif()
+
+find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h)
+find_library(LEVELDB_LIB NAMES leveldb)
+if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB))
+    message(FATAL_ERROR "Fail to find leveldb")
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT OPENSSL_ROOT_DIR)
+    set(OPENSSL_ROOT_DIR
+        "/usr/local/opt/openssl"    # Homebrew installed OpenSSL
+        )
+endif()
+
+find_package(OpenSSL REQUIRED)
+
+set(DYNAMIC_LIB
+    Threads::Threads
+    ${GFLAGS_LIBRARY}
+    ${PROTOBUF_LIBRARIES}
+    ${LEVELDB_LIB}
+    ${OPENSSL_CRYPTO_LIBRARY}
+    ${OPENSSL_SSL_LIBRARY}
+    ${THRIFT_LIB}
+    dl
+    )
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    set(DYNAMIC_LIB ${DYNAMIC_LIB}
+        pthread
+        "-framework CoreFoundation"
+        "-framework CoreGraphics"
+        "-framework CoreData"
+        "-framework CoreText"
+        "-framework Security"
+        "-framework Foundation"
+        "-Wl,-U,_MallocExtension_ReleaseFreeMemory"
+        "-Wl,-U,_ProfilerStart"
+        "-Wl,-U,_ProfilerStop"
+        "-Wl,-U,__Z13GetStackTracePPvii"
+        "-Wl,-U,_mallctl"
+        "-Wl,-U,_malloc_stats_print"
+    )
+endif()
+
+add_executable(nshead_pb_extension_client client.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(nshead_pb_extension_client)
+add_executable(nshead_pb_extension_server server.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(nshead_pb_extension_server)
+
+target_link_libraries(nshead_pb_extension_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})
+target_link_libraries(nshead_pb_extension_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB})

example/nshead_pb_extension_c++/client.cpp

@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A client sending requests to server every 1 second.
+
+#include <gflags/gflags.h>
+
+#include <butil/logging.h>
+#include <butil/time.h>
+#include <butil/strings/string_piece.h>
+#include <brpc/channel.h>
+#include <brpc/nshead_message.h>
+#include <bvar/bvar.h>
+
+bvar::LatencyRecorder g_latency_recorder("client");
+
+DEFINE_string(server, "0.0.0.0:8010", "IP Address of server");
+DEFINE_string(load_balancer, "", "The algorithm for load balancing");
+DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); 
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+    
+    // A Channel represents a communication line to a Server. Notice that 
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::Channel channel;
+    
+    // Initialize the channel, NULL means using default options. 
+    brpc::ChannelOptions options;
+    options.protocol = brpc::PROTOCOL_NSHEAD;
+    options.timeout_ms = FLAGS_timeout_ms/*milliseconds*/;
+    options.max_retry = FLAGS_max_retry;
+    if (channel.Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), &options) != 0) {
+        LOG(ERROR) << "Fail to initialize channel";
+        return -1;
+    }
+
+    // Send a request and wait for the response every 1 second.
+    int log_id = 0;
+    while (!brpc::IsAskedToQuit()) {
+        brpc::NsheadMessage request;
+        brpc::NsheadMessage response;
+        brpc::Controller cntl;
+
+        // Append message to `request'
+        request.body.append("hello world");
+
+        cntl.set_log_id(log_id ++);  // set by user
+
+        // Because `done'(last parameter) is NULL, this function waits until
+        // the response comes back or error occurs(including timedout).
+        channel.CallMethod(NULL, &cntl, &request, &response, NULL);
+        if (cntl.Failed()) {
+            LOG(ERROR) << "Fail to send nshead request, " << cntl.ErrorText();
+            sleep(1); // Remove this sleep in production code.
+        } else {
+            g_latency_recorder << cntl.latency_us();
+        }
+        LOG_EVERY_SECOND(INFO)
+            << "Sending nshead requests at qps=" << g_latency_recorder.qps(1)
+            << " latency=" << g_latency_recorder.latency(1);
+    }
+
+    LOG(INFO) << "EchoClient is going to quit";
+    return 0;
+}

example/nshead_pb_extension_c++/echo.proto

@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+syntax="proto2";
+package example;
+
+option cc_generic_services = true;
+
+message EchoRequest {
+      required string message = 1;
+};
+
+message EchoResponse {
+      required string message = 1;
+};
+
+service EchoService {
+      rpc Echo(EchoRequest) returns (EchoResponse);
+};

example/nshead_pb_extension_c++/server.cpp

@@ -0,0 +1,126 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A server to receive EchoRequest and send back EchoResponse.
+
+#include <google/protobuf/descriptor.h>
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <brpc/server.h>
+#include <brpc/nshead_pb_service_adaptor.h>
+#include "echo.pb.h"
+
+DEFINE_int32(port, 8010, "TCP Port of this server");
+DEFINE_int32(idle_timeout_s, -1, "Connection will be closed if there is no "
+             "read/write operations during the last `idle_timeout_s'");
+
+// Your implementation of example::EchoService
+namespace example {
+class EchoServiceImpl : public EchoService {
+public:
+    EchoServiceImpl() {}
+    virtual ~EchoServiceImpl() {}
+    virtual void Echo(google::protobuf::RpcController*,
+                      const EchoRequest* request,
+                      EchoResponse* response,
+                      google::protobuf::Closure* done) {
+        // This object helps you to call done->Run() in RAII style. If you need
+        // to process the request asynchronously, pass done_guard.release().
+        brpc::ClosureGuard done_guard(done);
+
+        // Echo response.
+        response->set_message(request->message());
+    }
+};
+}  // namespace example
+
+// Adapt your own nshead-based protocol to use pbrpc interface
+class MyNsheadProtocol : public brpc::NsheadPbServiceAdaptor {
+public:
+    void ParseNsheadMeta(
+        const brpc::Server&, const brpc::NsheadMessage&,
+        brpc::Controller*,
+        brpc::NsheadMeta* out_meta) const {
+        // Always use EchoService::Echo
+        const google::protobuf::ServiceDescriptor* svc =
+                example::EchoService::descriptor();
+        out_meta->set_full_method_name(svc->method(0)->full_name());
+    }
+
+    void ParseRequestFromIOBuf(const brpc::NsheadMeta&,
+                               const brpc::NsheadMessage& raw_req,
+                               brpc::Controller* cntl,
+                               google::protobuf::Message* pb_req) const {
+        // `req' MUST be EchoRequest here since we have only one RPCMethod
+        example::EchoRequest* echo_req =
+                dynamic_cast<example::EchoRequest*>(pb_req);
+        if (!echo_req) {
+            cntl->SetFailed(brpc::EREQUEST, "Fail to parse request");
+            return;
+        }
+        echo_req->set_message(raw_req.body.to_string());
+    }
+
+    void SerializeResponseToIOBuf(
+        const brpc::NsheadMeta&, brpc::Controller* cntl,
+        const google::protobuf::Message* pb_res,
+        brpc::NsheadMessage* raw_res) const {
+        if (cntl->Failed()) {
+            // Can't send failure feedback in this protocol
+            cntl->CloseConnection("Close connection due to previous error");
+            return;
+        }
+        // `res' MUST be EchoResponse here since we have only one RPCMethod
+        const example::EchoResponse* echo_res =
+                dynamic_cast<const example::EchoResponse*>(pb_res);
+        if (!echo_res) {
+            cntl->CloseConnection("Close connection due to bad response");
+            return;
+        }
+        raw_res->body.append(echo_res->message());
+    }
+};
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    brpc::Server server;
+    example::EchoServiceImpl echo_service_impl;
+
+    // Add the service into server. Notice the second parameter, because the
+    // service is put on stack, we don't want server to delete it, otherwise
+    // use brpc::SERVER_OWNS_SERVICE.
+    if (server.AddService(&echo_service_impl, 
+                          brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
+        LOG(ERROR) << "Fail to add service";
+        return -1;
+    }
+
+    // Start the server.
+    brpc::ServerOptions options;
+    options.nshead_service = new MyNsheadProtocol;  // the adaptor
+    options.idle_timeout_sec = FLAGS_idle_timeout_s;
+    if (server.Start(FLAGS_port, &options) != 0) {
+        LOG(ERROR) << "Fail to start EchoServer";
+        return -1;
+    }
+
+    // Wait until Ctrl-C is pressed, then Stop() and Join() the server.
+    server.RunUntilAskedToQuit();
+    return 0;
+}

example/parallel_echo_c++/CMakeLists.txt

@@ -0,0 +1,115 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.16...3.28)
+project(parallel_echo_c++ C CXX)
+
+include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake)
+
+option(LINK_SO "Whether examples are linked dynamically" OFF)
+
+execute_process(
+    COMMAND bash -c "find ${PROJECT_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'"
+    OUTPUT_VARIABLE OUTPUT_PATH
+)
+
+if(OUTPUT_PATH)
+    list(PREPEND CMAKE_PREFIX_PATH ${OUTPUT_PATH})
+endif()
+
+find_package(Threads REQUIRED)
+find_package(Protobuf REQUIRED)
+protobuf_generate_cpp(PROTO_SRC PROTO_HEADER echo.proto)
+
+# Search for libthrift* by best effort. If it is not found and brpc is
+# compiled with thrift protocol enabled, a link error would be reported.
+find_library(THRIFT_LIB NAMES thrift)
+if (NOT THRIFT_LIB)
+    set(THRIFT_LIB "")
+endif()
+
+find_path(GPERFTOOLS_INCLUDE_DIR NAMES gperftools/heap-profiler.h)
+find_library(GPERFTOOLS_LIBRARIES NAMES tcmalloc_and_profiler)
+
+find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h)
+if(LINK_SO)
+    find_library(BRPC_LIB NAMES brpc)
+else()
+    find_library(BRPC_LIB NAMES libbrpc.a brpc)
+endif()
+if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB))
+    message(FATAL_ERROR "Fail to find brpc")
+endif()
+
+find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h)
+find_library(GFLAGS_LIBRARY NAMES gflags libgflags)
+if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY))
+    message(FATAL_ERROR "Fail to find gflags")
+endif()
+
+set(BRPC_EXAMPLE_ENABLE_CPU_PROFILER ON)
+
+find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h)
+find_library(LEVELDB_LIB NAMES leveldb)
+if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB))
+    message(FATAL_ERROR "Fail to find leveldb")
+endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT OPENSSL_ROOT_DIR)
+    set(OPENSSL_ROOT_DIR
+        "/usr/local/opt/openssl"    # Homebrew installed OpenSSL
+        )
+endif()
+
+find_package(OpenSSL REQUIRED)
+
+set(DYNAMIC_LIB
+    Threads::Threads
+    ${GFLAGS_LIBRARY}
+    ${PROTOBUF_LIBRARIES}
+    ${LEVELDB_LIB}
+    ${OPENSSL_CRYPTO_LIBRARY}
+    ${OPENSSL_SSL_LIBRARY}
+    ${THRIFT_LIB}
+    dl
+    )
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    set(DYNAMIC_LIB ${DYNAMIC_LIB}
+        pthread
+        "-framework CoreFoundation"
+        "-framework CoreGraphics"
+        "-framework CoreData"
+        "-framework CoreText"
+        "-framework Security"
+        "-framework Foundation"
+        "-Wl,-U,_MallocExtension_ReleaseFreeMemory"
+        "-Wl,-U,_ProfilerStart"
+        "-Wl,-U,_ProfilerStop"
+        "-Wl,-U,__Z13GetStackTracePPvii"
+        "-Wl,-U,_mallctl"
+        "-Wl,-U,_malloc_stats_print"
+    )
+endif()
+
+add_executable(parallel_echo_client client.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(parallel_echo_client)
+add_executable(parallel_echo_server server.cpp ${PROTO_SRC} ${PROTO_HEADER})
+brpc_example_configure_target(parallel_echo_server)
+
+target_link_libraries(parallel_echo_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB} ${GPERFTOOLS_LIBRARIES})
+target_link_libraries(parallel_echo_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB} ${GPERFTOOLS_LIBRARIES})

example/parallel_echo_c++/client.cpp

@@ -0,0 +1,214 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A client sending requests to server in parallel by multiple threads.
+
+#include <gflags/gflags.h>
+#include <bthread/bthread.h>
+#include <butil/logging.h>
+#include <butil/string_printf.h>
+#include <butil/time.h>
+#include <butil/macros.h>
+#include <brpc/parallel_channel.h>
+#include <brpc/server.h>
+#include "echo.pb.h"
+
+DEFINE_int32(thread_num, 50, "Number of threads to send requests");
+DEFINE_int32(channel_num, 3, "Number of sub channels");
+DEFINE_bool(same_channel, false, "Add the same sub channel multiple times");
+DEFINE_bool(use_bthread, false, "Use bthread to send requests");
+DEFINE_int32(attachment_size, 0, "Carry so many byte attachment along with requests");
+DEFINE_int32(request_size, 16, "Bytes of each request");
+DEFINE_string(connection_type, "", "Connection type. Available values: single, pooled, short");
+DEFINE_string(protocol, "baidu_std", "Protocol type. Defined in src/brpc/options.proto");
+DEFINE_string(server, "0.0.0.0:8002", "IP Address of server");
+DEFINE_string(load_balancer, "", "The algorithm for load balancing");
+DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds");
+DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); 
+DEFINE_bool(dont_fail, false, "Print fatal when some call failed");
+DEFINE_int32(dummy_port, -1, "Launch dummy server at this port");
+
+std::string g_request;
+std::string g_attachment;
+bvar::LatencyRecorder g_latency_recorder("client");
+bvar::Adder<int> g_error_count("client_error_count");
+bvar::LatencyRecorder* g_sub_channel_latency = NULL;
+
+static void* sender(void* arg) {
+    // Normally, you should not call a Channel directly, but instead construct
+    // a stub Service wrapping it. stub can be shared by all threads as well.
+    example::EchoService_Stub stub(static_cast<google::protobuf::RpcChannel*>(arg));
+
+    int log_id = 0;
+    while (!brpc::IsAskedToQuit()) {
+        // We will receive response synchronously, safe to put variables
+        // on stack.
+        example::EchoRequest request;
+        example::EchoResponse response;
+        brpc::Controller cntl;
+
+        request.set_value(log_id++);
+        if (!g_attachment.empty()) {
+            // Set attachment which is wired to network directly instead of 
+            // being serialized into protobuf messages.
+            cntl.request_attachment().append(g_attachment);
+        }
+
+        // Because `done'(last parameter) is NULL, this function waits until
+        // the response comes back or error occurs(including timedout).
+        stub.Echo(&cntl, &request, &response, NULL);
+        if (!cntl.Failed()) {
+            g_latency_recorder << cntl.latency_us();
+            for (int i = 0; i < cntl.sub_count(); ++i) {
+                if (cntl.sub(i) && !cntl.sub(i)->Failed()) {
+                    g_sub_channel_latency[i] << cntl.sub(i)->latency_us();
+                }
+            }
+        } else {
+            g_error_count << 1;
+            CHECK(brpc::IsAskedToQuit() || !FLAGS_dont_fail)
+                << "error=" << cntl.ErrorText() << " latency=" << cntl.latency_us();
+            // We can't connect to the server, sleep a while. Notice that this
+            // is a specific sleeping to prevent this thread from spinning too
+            // fast. You should continue the business logic in a production 
+            // server rather than sleeping.
+            bthread_usleep(50000);
+        }
+    }
+    return NULL;
+}
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    // A Channel represents a communication line to a Server. Notice that 
+    // Channel is thread-safe and can be shared by all threads in your program.
+    brpc::ParallelChannel channel;
+    brpc::ParallelChannelOptions pchan_options;
+    pchan_options.timeout_ms = FLAGS_timeout_ms;
+    if (channel.Init(&pchan_options) != 0) {
+        LOG(ERROR) << "Fail to init ParallelChannel";
+        return -1;
+    }
+
+    brpc::ChannelOptions sub_options;
+    sub_options.protocol = FLAGS_protocol;
+    sub_options.connection_type = FLAGS_connection_type;
+    sub_options.max_retry = FLAGS_max_retry;
+    // Setting sub_options.timeout_ms does not work because timeout of sub 
+    // channels are disabled in ParallelChannel.
+
+    if (FLAGS_same_channel) {
+        // For brpc >= 1.0.155.31351, a sub channel can be added into
+        // a ParallelChannel more than once.
+        brpc::Channel* sub_channel = new brpc::Channel;
+        // Initialize the channel, NULL means using default options. 
+        // options, see `brpc/channel.h'.
+        if (sub_channel->Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), &sub_options) != 0) {
+            LOG(ERROR) << "Fail to initialize sub_channel";
+            return -1;
+        }
+        for (int i = 0; i < FLAGS_channel_num; ++i) {
+            if (channel.AddChannel(sub_channel, brpc::OWNS_CHANNEL,
+                                   NULL, NULL) != 0) {
+                LOG(ERROR) << "Fail to AddChannel, i=" << i;
+                return -1;
+            }
+        }
+    } else {
+        for (int i = 0; i < FLAGS_channel_num; ++i) {
+            brpc::Channel* sub_channel = new brpc::Channel;
+            // Initialize the channel, NULL means using default options. 
+            // options, see `brpc/channel.h'.
+            if (sub_channel->Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), &sub_options) != 0) {
+                LOG(ERROR) << "Fail to initialize sub_channel[" << i << "]";
+                return -1;
+            }
+            if (channel.AddChannel(sub_channel, brpc::OWNS_CHANNEL,
+                                   NULL, NULL) != 0) {
+                LOG(ERROR) << "Fail to AddChannel, i=" << i;
+                return -1;
+            }
+        }
+    }
+
+    // Initialize bvar for sub channel
+    g_sub_channel_latency = new bvar::LatencyRecorder[FLAGS_channel_num];
+    for (int i = 0; i < FLAGS_channel_num; ++i) {
+        std::string name;
+        butil::string_printf(&name, "client_sub_%d", i);
+        g_sub_channel_latency[i].expose(name);
+    }
+
+    if (FLAGS_attachment_size > 0) {
+        g_attachment.resize(FLAGS_attachment_size, 'a');
+    }
+    if (FLAGS_request_size <= 0) {
+        LOG(ERROR) << "Bad request_size=" << FLAGS_request_size;
+        return -1;
+    }
+    g_request.resize(FLAGS_request_size, 'r');
+
+    if (FLAGS_dummy_port >= 0) {
+        brpc::StartDummyServerAt(FLAGS_dummy_port);
+    }
+
+    std::vector<bthread_t> bids;
+    std::vector<pthread_t> pids;
+    if (!FLAGS_use_bthread) {
+        pids.resize(FLAGS_thread_num);
+        for (int i = 0; i < FLAGS_thread_num; ++i) {
+            if (pthread_create(&pids[i], NULL, sender, &channel) != 0) {
+                LOG(ERROR) << "Fail to create pthread";
+                return -1;
+            }
+        }
+    } else {
+        bids.resize(FLAGS_thread_num);
+        for (int i = 0; i < FLAGS_thread_num; ++i) {
+            if (bthread_start_background(
+                    &bids[i], NULL, sender, &channel) != 0) {
+                LOG(ERROR) << "Fail to create bthread";
+                return -1;
+            }
+        }
+    }
+
+    while (!brpc::IsAskedToQuit()) {
+        sleep(1);
+        LOG(INFO) << "Sending EchoRequest at qps=" << g_latency_recorder.qps(1)
+                  << " latency=" << g_latency_recorder.latency(1) << noflush;
+        for (int i = 0; i < FLAGS_channel_num; ++i) {
+            LOG(INFO) << " latency_" << i << "=" 
+                      << g_sub_channel_latency[i].latency(1)
+                      << noflush;
+        }
+        LOG(INFO);
+    }
+    
+    LOG(INFO) << "EchoClient is going to quit";
+    for (int i = 0; i < FLAGS_thread_num; ++i) {
+        if (!FLAGS_use_bthread) {
+            pthread_join(pids[i], NULL);
+        } else {
+            bthread_join(bids[i], NULL);
+        }
+    }
+
+    return 0;
+}

example/parallel_echo_c++/echo.proto

@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+syntax="proto2";
+package example;
+
+option cc_generic_services = true;
+
+message EchoRequest {
+      required int32 value = 1;
+};
+
+message EchoResponse {
+      required int32 value = 1;
+};
+
+service EchoService {
+      rpc Echo(EchoRequest) returns (EchoResponse);
+};

example/parallel_echo_c++/server.cpp

@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// A server to receive EchoRequest and send back EchoResponse.
+
+#include <gflags/gflags.h>
+#include <butil/logging.h>
+#include <butil/string_printf.h>
+#include <brpc/server.h>
+#include "echo.pb.h"
+
+DEFINE_bool(echo_attachment, true, "Echo attachment as well");
+DEFINE_int32(port, 8002, "TCP Port of this server");
+DEFINE_int32(idle_timeout_s, -1, "Connection will be closed if there is no "
+             "read/write operations during the last `idle_timeout_s'");
+DEFINE_int32(max_concurrency, 0, "Limit of request processing in parallel");
+
+// Your implementation of example::EchoService
+class EchoServiceImpl : public example::EchoService {
+public:
+    EchoServiceImpl() {}
+    ~EchoServiceImpl() {}
+    void Echo(google::protobuf::RpcController* cntl_base,
+              const example::EchoRequest* request,
+              example::EchoResponse* response,
+              google::protobuf::Closure* done) {
+        brpc::ClosureGuard done_guard(done);
+        brpc::Controller* cntl =
+            static_cast<brpc::Controller*>(cntl_base);
+
+        // Echo request and its attachment
+        response->set_value(request->value());
+        if (FLAGS_echo_attachment) {
+            cntl->response_attachment().append(cntl->request_attachment());
+        }
+    }
+};
+
+int main(int argc, char* argv[]) {
+    // Parse gflags. We recommend you to use gflags as well.
+    GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+    // Generally you only need one Server.
+    brpc::Server server;
+
+    // Instance of your service.
+    EchoServiceImpl echo_service_impl;
+
+    // Add the service into server. Notice the second parameter, because the
+    // service is put on stack, we don't want server to delete it, otherwise
+    // use brpc::SERVER_OWNS_SERVICE.
+    if (server.AddService(&echo_service_impl, 
+                          brpc::SERVER_DOESNT_OWN_SERVICE) != 0) {
+        LOG(ERROR) << "Fail to add service";
+        return -1;
+    }
+
+    // Start the server. 
+    brpc::ServerOptions options;
+    options.idle_timeout_sec = FLAGS_idle_timeout_s;
+    options.max_concurrency = FLAGS_max_concurrency;
+    if (server.Start(FLAGS_port, &options) != 0) {
+        LOG(ERROR) << "Fail to start EchoServer";
+        return -1;
+    }
+
+    // Wait until Ctrl-C is pressed, then Stop() and Join() the server.
+    server.RunUntilAskedToQuit();
+    return 0;
+}