From 84156084a4c5228e1d2fe21e068fff330bbc40d1 Mon Sep 17 00:00:00 2001
From: Zhouxiang Zhan <zhouxzhan@apache.org>
Date: Sun, 8 Oct 2023 11:13:25 +0800
Subject: [PATCH 1/7] [ISSUE #7321] Refector NettyRemotingAbstract with unify
 future implementation (#7322)

* Refector NettyRemotingAbstract

* Add invoke with future method

* Deprecate InvokeCallback#operationComplete

* Add operationSuccess and operationException for InvokeCallback

* fix unit test

* fix unit test

* Keep InvokeCallback#operationComplete

* Optimize invokeAsyncImpl operationComplete

* Add unit test for NettyRemotingClient

* fix checkstyle
---
 .../rocketmq/broker/out/BrokerOuterAPI.java   | 147 +++++----
 .../rocketmq/client/impl/MQAdminImpl.java     |  71 ++--
 .../rocketmq/client/impl/MQClientAPIImpl.java | 239 +++++++-------
 .../client/impl/mqclient/MQClientAPIExt.java  | 309 ++++++++----------
 .../client/impl/MQClientAPIImplTest.java      |  12 +-
 .../remoting/RemotingProtocolServer.java      |  22 +-
 .../service/mqclient/MQClientAPIExtTest.java  |  97 +++---
 .../rocketmq/remoting/InvokeCallback.java     |  15 +
 .../rocketmq/remoting/RemotingClient.java     |  27 +-
 .../remoting/netty/NettyRemotingAbstract.java | 123 ++++---
 .../remoting/netty/NettyRemotingClient.java   |  33 +-
 .../remoting/netty/ResponseFuture.java        |  15 +
 .../rocketmq/remoting/rpc/RpcClientImpl.java  |  29 +-
 .../rocketmq/remoting/RemotingServerTest.java |  22 +-
 .../rocketmq/remoting/netty/MockChannel.java  |  21 +-
 .../remoting/netty/MockChannelPromise.java    | 191 +++++++++++
 .../netty/NettyRemotingAbstractTest.java      |  54 ++-
 .../netty/NettyRemotingClientTest.java        | 185 ++++++++++-
 18 files changed, 1029 insertions(+), 583 deletions(-)
 rename client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java => remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java (57%)
 create mode 100644 remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java

diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
index 9dfb8127d..6fde48dd9 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
@@ -73,6 +73,7 @@ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
 import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;
 import org.apache.rocketmq.remoting.netty.NettyClientConfig;
 import org.apache.rocketmq.remoting.netty.NettyRemotingClient;
+import org.apache.rocketmq.remoting.netty.ResponseFuture;
 import org.apache.rocketmq.remoting.protocol.BrokerSyncInfo;
 import org.apache.rocketmq.remoting.protocol.DataVersion;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
@@ -107,6 +108,8 @@ import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2;
 import org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoResponseHeader;
@@ -124,8 +127,6 @@ import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerReques
 import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerResponseHeader;
 import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterTopicRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.namesrv.UnRegisterBrokerRequestHeader;
-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader;
-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;
 import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
 import org.apache.rocketmq.remoting.protocol.namesrv.RegisterBrokerResult;
 import org.apache.rocketmq.remoting.protocol.route.BrokerData;
@@ -151,7 +152,6 @@ public class BrokerOuterAPI {
     private final RpcClient rpcClient;
     private String nameSrvAddr = null;
 
-
     public BrokerOuterAPI(final NettyClientConfig nettyClientConfig) {
         this(nettyClientConfig, new DynamicalExtFieldRPCHook(), new ClientMetadata());
     }
@@ -459,7 +459,7 @@ public class BrokerOuterAPI {
      * @param filterServerList
      * @param oneway
      * @param timeoutMills
-     * @param compressed default false
+     * @param compressed         default false
      * @return
      */
     public List<RegisterBrokerResult> registerBrokerAll(
@@ -643,7 +643,6 @@ public class BrokerOuterAPI {
         queueDatas.add(queueData);
         final byte[] topicRouteBody = topicRouteData.encode();
 
-
         List<String> nameServerAddressList = this.remotingClient.getNameServerAddressList();
         final CountDownLatch countDownLatch = new CountDownLatch(nameServerAddressList.size());
         for (final String namesrvAddr : nameServerAddressList) {
@@ -910,25 +909,33 @@ public class BrokerOuterAPI {
         RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null);
 
         request.setBody(requestBody.encode());
-        this.remotingClient.invokeAsync(addr, request, timeoutMillis, responseFuture -> {
-            if (callback == null) {
-                return;
+        this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {
+            @Override
+            public void operationComplete(ResponseFuture responseFuture) {
+
             }
 
-            try {
-                RemotingCommand response = responseFuture.getResponseCommand();
-                if (response != null) {
-                    if (response.getCode() == ResponseCode.SUCCESS) {
-                        LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(),
-                            LockBatchResponseBody.class);
-                        Set<MessageQueue> messageQueues = responseBody.getLockOKMQSet();
-                        callback.onSuccess(messageQueues);
-                    } else {
-                        callback.onException(new MQBrokerException(response.getCode(), response.getRemark()));
-                    }
+            @Override
+            public void operationSucceed(RemotingCommand response) {
+                if (callback == null) {
+                    return;
                 }
-            } catch (Throwable ignored) {
+                if (response.getCode() == ResponseCode.SUCCESS) {
+                    LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(),
+                        LockBatchResponseBody.class);
+                    Set<MessageQueue> messageQueues = responseBody.getLockOKMQSet();
+                    callback.onSuccess(messageQueues);
+                } else {
+                    callback.onException(new MQBrokerException(response.getCode(), response.getRemark()));
+                }
+            }
 
+            @Override
+            public void operationFail(Throwable throwable) {
+                if (callback == null) {
+                    return;
+                }
+                callback.onException(throwable);
             }
         });
     }
@@ -942,22 +949,30 @@ public class BrokerOuterAPI {
 
         request.setBody(requestBody.encode());
 
-        this.remotingClient.invokeAsync(addr, request, timeoutMillis, responseFuture -> {
-            if (callback == null) {
-                return;
+        this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {
+            @Override
+            public void operationComplete(ResponseFuture responseFuture) {
+
             }
 
-            try {
-                RemotingCommand response = responseFuture.getResponseCommand();
-                if (response != null) {
-                    if (response.getCode() == ResponseCode.SUCCESS) {
-                        callback.onSuccess();
-                    } else {
-                        callback.onException(new MQBrokerException(response.getCode(), response.getRemark()));
-                    }
+            @Override
+            public void operationSucceed(RemotingCommand response) {
+                if (callback == null) {
+                    return;
                 }
-            } catch (Throwable ignored) {
+                if (response.getCode() == ResponseCode.SUCCESS) {
+                    callback.onSuccess();
+                } else {
+                    callback.onException(new MQBrokerException(response.getCode(), response.getRemark()));
+                }
+            }
 
+            @Override
+            public void operationFail(Throwable throwable) {
+                if (callback == null) {
+                    return;
+                }
+                callback.onException(throwable);
             }
         });
     }
@@ -983,21 +998,27 @@ public class BrokerOuterAPI {
         CompletableFuture<SendResult> cf = new CompletableFuture<>();
         final String msgId = msg.getMsgId();
         try {
-            this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
-                RemotingCommand response = responseFuture.getResponseCommand();
-                if (null != response) {
-                    SendResult sendResult = null;
+            this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() {
+                @Override
+                public void operationComplete(ResponseFuture responseFuture) {
+
+                }
+
+                @Override
+                public void operationSucceed(RemotingCommand response) {
                     try {
-                        sendResult = this.processSendResponse(brokerName, msg, response);
+                        SendResult sendResult = processSendResponse(brokerName, msg, response);
                         cf.complete(sendResult);
                     } catch (MQBrokerException | RemotingCommandException e) {
                         LOGGER.error("processSendResponse in sendMessageToSpecificBrokerAsync failed, msgId=" + msgId, e);
                         cf.completeExceptionally(e);
                     }
-                } else {
-                    cf.complete(null);
                 }
 
+                @Override
+                public void operationFail(Throwable throwable) {
+                    cf.completeExceptionally(throwable);
+                }
             });
         } catch (Throwable t) {
             LOGGER.error("invokeAsync failed in sendMessageToSpecificBrokerAsync, msgId=" + msgId, t);
@@ -1057,7 +1078,7 @@ public class BrokerOuterAPI {
         }
         if (sendStatus != null) {
             SendMessageResponseHeader responseHeader =
-                    (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class);
+                (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class);
 
             //If namespace not null , reset Topic without namespace.
             String topic = msg.getTopic();
@@ -1073,8 +1094,8 @@ public class BrokerOuterAPI {
                 uniqMsgId = sb.toString();
             }
             SendResult sendResult = new SendResult(sendStatus,
-                    uniqMsgId,
-                    responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset());
+                uniqMsgId,
+                responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset());
             sendResult.setTransactionId(responseHeader.getTransactionId());
             String regionId = response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION);
             String traceOn = response.getExtFields().get(MessageConst.PROPERTY_TRACE_SWITCH);
@@ -1218,8 +1239,9 @@ public class BrokerOuterAPI {
     /**
      * Broker try to elect itself as a master in broker set
      */
-    public Pair<ElectMasterResponseHeader, Set<Long>> brokerElect(String controllerAddress, String clusterName, String brokerName,
-                                                 Long brokerId) throws Exception {
+    public Pair<ElectMasterResponseHeader, Set<Long>> brokerElect(String controllerAddress, String clusterName,
+        String brokerName,
+        Long brokerId) throws Exception {
 
         final ElectMasterRequestHeader requestHeader = ElectMasterRequestHeader.ofBrokerTrigger(clusterName, brokerName, brokerId);
         RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_ELECT_MASTER, requestHeader);
@@ -1237,7 +1259,8 @@ public class BrokerOuterAPI {
         throw new MQBrokerException(response.getCode(), response.getRemark());
     }
 
-    public GetNextBrokerIdResponseHeader getNextBrokerId(final String clusterName, final String brokerName, final String controllerAddress) throws Exception {
+    public GetNextBrokerIdResponseHeader getNextBrokerId(final String clusterName, final String brokerName,
+        final String controllerAddress) throws Exception {
         final GetNextBrokerIdRequestHeader requestHeader = new GetNextBrokerIdRequestHeader(clusterName, brokerName);
         final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_GET_NEXT_BROKER_ID, requestHeader);
         final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000);
@@ -1248,7 +1271,8 @@ public class BrokerOuterAPI {
         throw new MQBrokerException(response.getCode(), response.getRemark());
     }
 
-    public ApplyBrokerIdResponseHeader applyBrokerId(final String clusterName, final String brokerName, final Long brokerId, final String registerCheckCode, final String controllerAddress) throws Exception {
+    public ApplyBrokerIdResponseHeader applyBrokerId(final String clusterName, final String brokerName,
+        final Long brokerId, final String registerCheckCode, final String controllerAddress) throws Exception {
         final ApplyBrokerIdRequestHeader requestHeader = new ApplyBrokerIdRequestHeader(clusterName, brokerName, brokerId, registerCheckCode);
         final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_APPLY_BROKER_ID, requestHeader);
         final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000);
@@ -1259,7 +1283,9 @@ public class BrokerOuterAPI {
         throw new MQBrokerException(response.getCode(), response.getRemark());
     }
 
-    public Pair<RegisterBrokerToControllerResponseHeader, Set<Long>> registerBrokerToController(final String clusterName, final String brokerName, final Long brokerId, final String brokerAddress, final String controllerAddress) throws Exception {
+    public Pair<RegisterBrokerToControllerResponseHeader, Set<Long>> registerBrokerToController(
+        final String clusterName, final String brokerName, final Long brokerId, final String brokerAddress,
+        final String controllerAddress) throws Exception {
         final RegisterBrokerToControllerRequestHeader requestHeader = new RegisterBrokerToControllerRequestHeader(clusterName, brokerName, brokerId, brokerAddress);
         final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_REGISTER_BROKER, requestHeader);
         final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000);
@@ -1355,16 +1381,25 @@ public class BrokerOuterAPI {
 
         RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, requestHeader);
         CompletableFuture<PullResult> pullResultFuture = new CompletableFuture<>();
-        this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
-            if (responseFuture.getCause() != null) {
-                pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>()));
-                return;
+        this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() {
+            @Override
+            public void operationComplete(ResponseFuture responseFuture) {
+
             }
-            try {
-                PullResultExt pullResultExt = this.processPullResponse(responseFuture.getResponseCommand(), brokerAddr);
-                this.processPullResult(pullResultExt, brokerName, queueId);
-                pullResultFuture.complete(pullResultExt);
-            } catch (Exception e) {
+
+            @Override
+            public void operationSucceed(RemotingCommand response) {
+                try {
+                    PullResultExt pullResultExt = processPullResponse(response, brokerAddr);
+                    processPullResult(pullResultExt, brokerName, queueId);
+                    pullResultFuture.complete(pullResultExt);
+                } catch (Exception e) {
+                    pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>()));
+                }
+            }
+
+            @Override
+            public void operationFail(Throwable throwable) {
                 pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>()));
             }
         });
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java
index 1ef3a9483..83835bd3d 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java
@@ -44,6 +44,8 @@ import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageId;
 import org.apache.rocketmq.common.message.MessageQueue;
 import org.apache.rocketmq.common.utils.NetworkUtil;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 import org.apache.rocketmq.remoting.InvokeCallback;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.exception.RemotingException;
@@ -55,8 +57,6 @@ import org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader;
 import org.apache.rocketmq.remoting.protocol.header.QueryMessageResponseHeader;
 import org.apache.rocketmq.remoting.protocol.route.BrokerData;
 import org.apache.rocketmq.remoting.protocol.route.TopicRouteData;
-import org.apache.rocketmq.logging.org.slf4j.Logger;
-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
 
 public class MQAdminImpl {
 
@@ -357,44 +357,51 @@ public class MQAdminImpl {
                             new InvokeCallback() {
                                 @Override
                                 public void operationComplete(ResponseFuture responseFuture) {
+
+                                }
+
+                                @Override
+                                public void operationSucceed(RemotingCommand response) {
                                     try {
-                                        RemotingCommand response = responseFuture.getResponseCommand();
-                                        if (response != null) {
-                                            switch (response.getCode()) {
-                                                case ResponseCode.SUCCESS: {
-                                                    QueryMessageResponseHeader responseHeader = null;
-                                                    try {
-                                                        responseHeader =
-                                                            (QueryMessageResponseHeader) response
-                                                                .decodeCommandCustomHeader(QueryMessageResponseHeader.class);
-                                                    } catch (RemotingCommandException e) {
-                                                        log.error("decodeCommandCustomHeader exception", e);
-                                                        return;
-                                                    }
-
-                                                    List<MessageExt> wrappers =
-                                                        MessageDecoder.decodes(ByteBuffer.wrap(response.getBody()), true);
-
-                                                    QueryResult qr = new QueryResult(responseHeader.getIndexLastUpdateTimestamp(), wrappers);
-                                                    try {
-                                                        lock.writeLock().lock();
-                                                        queryResultList.add(qr);
-                                                    } finally {
-                                                        lock.writeLock().unlock();
-                                                    }
-                                                    break;
+                                        switch (response.getCode()) {
+                                            case ResponseCode.SUCCESS: {
+                                                QueryMessageResponseHeader responseHeader = null;
+                                                try {
+                                                    responseHeader =
+                                                        (QueryMessageResponseHeader) response
+                                                            .decodeCommandCustomHeader(QueryMessageResponseHeader.class);
+                                                } catch (RemotingCommandException e) {
+                                                    log.error("decodeCommandCustomHeader exception", e);
+                                                    return;
                                                 }
-                                                default:
-                                                    log.warn("getResponseCommand failed, {} {}", response.getCode(), response.getRemark());
-                                                    break;
+
+                                                List<MessageExt> wrappers =
+                                                    MessageDecoder.decodes(ByteBuffer.wrap(response.getBody()), true);
+
+                                                QueryResult qr = new QueryResult(responseHeader.getIndexLastUpdateTimestamp(), wrappers);
+                                                try {
+                                                    lock.writeLock().lock();
+                                                    queryResultList.add(qr);
+                                                } finally {
+                                                    lock.writeLock().unlock();
+                                                }
+                                                break;
                                             }
-                                        } else {
-                                            log.warn("getResponseCommand return null");
+                                            default:
+                                                log.warn("getResponseCommand failed, {} {}", response.getCode(), response.getRemark());
+                                                break;
                                         }
+
                                     } finally {
                                         countDownLatch.countDown();
                                     }
                                 }
+
+                                @Override
+                                public void operationFail(Throwable throwable) {
+                                    log.error("queryMessage error, requestHeader={}", requestHeader);
+                                    countDownLatch.countDown();
+                                }
                             }, isUniqKey);
                     } catch (Exception e) {
                         log.warn("queryMessage exception", e);
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
index 3201a493f..2407e5737 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
@@ -33,7 +33,6 @@ import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.client.ClientConfig;
 import org.apache.rocketmq.client.Validators;
-import org.apache.rocketmq.client.common.ClientErrorCode;
 import org.apache.rocketmq.client.consumer.AckCallback;
 import org.apache.rocketmq.client.consumer.AckResult;
 import org.apache.rocketmq.client.consumer.AckStatus;
@@ -653,10 +652,13 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
             this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {
                 @Override
                 public void operationComplete(ResponseFuture responseFuture) {
-                    long cost = System.currentTimeMillis() - beginStartTime;
-                    RemotingCommand response = responseFuture.getResponseCommand();
-                    if (null == sendCallback && response != null) {
 
+                }
+
+                @Override
+                public void operationSucceed(RemotingCommand response) {
+                    long cost = System.currentTimeMillis() - beginStartTime;
+                    if (null == sendCallback) {
                         try {
                             SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr);
                             if (context != null && sendResult != null) {
@@ -666,46 +668,47 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
                         } catch (Throwable e) {
                         }
 
-                        producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false, true);
+                        producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, false, true);
                         return;
                     }
 
-                    if (response != null) {
+                    try {
+                        SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr);
+                        assert sendResult != null;
+                        if (context != null) {
+                            context.setSendResult(sendResult);
+                            context.getProducer().executeSendMessageHookAfter(context);
+                        }
+
                         try {
-                            SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr);
-                            assert sendResult != null;
-                            if (context != null) {
-                                context.setSendResult(sendResult);
-                                context.getProducer().executeSendMessageHookAfter(context);
-                            }
+                            sendCallback.onSuccess(sendResult);
+                        } catch (Throwable e) {
+                        }
 
-                            try {
-                                sendCallback.onSuccess(sendResult);
-                            } catch (Throwable e) {
-                            }
+                        producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, false, true);
+                    } catch (Exception e) {
+                        producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, true, true);
+                        onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
+                            retryTimesWhenSendFailed, times, e, context, false, producer);
+                    }
+                }
 
-                            producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false, true);
-                        } catch (Exception e) {
-                            producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true, true);
-                            onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
-                                retryTimesWhenSendFailed, times, e, context, false, producer);
-                        }
+                @Override
+                public void operationFail(Throwable throwable) {
+                    producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, true, true);
+                    long cost = System.currentTimeMillis() - beginStartTime;
+                    if (throwable instanceof RemotingSendRequestException) {
+                        MQClientException ex = new MQClientException("send request failed", throwable);
+                        onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
+                            retryTimesWhenSendFailed, times, ex, context, true, producer);
+                    } else if (throwable instanceof RemotingTimeoutException) {
+                        MQClientException ex = new MQClientException("wait response timeout, cost=" + cost, throwable);
+                        onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
+                            retryTimesWhenSendFailed, times, ex, context, true, producer);
                     } else {
-                        producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true, true);
-                        if (!responseFuture.isSendRequestOK()) {
-                            MQClientException ex = new MQClientException("send request failed", responseFuture.getCause());
-                            onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
-                                retryTimesWhenSendFailed, times, ex, context, true, producer);
-                        } else if (responseFuture.isTimeout()) {
-                            MQClientException ex = new MQClientException("wait response timeout " + responseFuture.getTimeoutMillis() + "ms",
-                                responseFuture.getCause());
-                            onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
-                                retryTimesWhenSendFailed, times, ex, context, true, producer);
-                        } else {
-                            MQClientException ex = new MQClientException("unknow reseaon", responseFuture.getCause());
-                            onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
-                                retryTimesWhenSendFailed, times, ex, context, true, producer);
-                        }
+                        MQClientException ex = new MQClientException("unknow reseaon", throwable);
+                        onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
+                            retryTimesWhenSendFailed, times, ex, context, true, producer);
                     }
                 }
             });
@@ -857,30 +860,25 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
         final long timeoutMillis, final PopCallback popCallback
     ) throws RemotingException, InterruptedException {
         final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.POP_MESSAGE, requestHeader);
-        this.remotingClient.invokeAsync(addr, request, timeoutMillis, new BaseInvokeCallback(MQClientAPIImpl.this) {
+        this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {
             @Override
-            public void onComplete(ResponseFuture responseFuture) {
-                RemotingCommand response = responseFuture.getResponseCommand();
-                if (response != null) {
-                    try {
-                        PopResult
-                            popResult = MQClientAPIImpl.this.processPopResponse(brokerName, response, requestHeader.getTopic(), requestHeader);
-                        assert popResult != null;
-                        popCallback.onSuccess(popResult);
-                    } catch (Exception e) {
-                        popCallback.onException(e);
-                    }
-                } else {
-                    if (!responseFuture.isSendRequestOK()) {
-                        popCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause()));
-                    } else if (responseFuture.isTimeout()) {
-                        popCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request,
-                            responseFuture.getCause()));
-                    } else {
-                        popCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause()));
-                    }
+            public void operationComplete(ResponseFuture responseFuture) {
+
+            }
+
+            @Override
+            public void operationSucceed(RemotingCommand response) {
+                try {
+                    PopResult popResult = MQClientAPIImpl.this.processPopResponse(brokerName, response, requestHeader.getTopic(), requestHeader);
+                    popCallback.onSuccess(popResult);
+                } catch (Exception e) {
+                    popCallback.onException(e);
                 }
             }
+            @Override
+            public void operationFail(Throwable throwable) {
+                popCallback.onException(throwable);
+            }
         });
     }
 
@@ -959,34 +957,26 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
                 request.setBody(requestBody.encode());
             }
         }
-        this.remotingClient.invokeAsync(addr, request, timeOut, new BaseInvokeCallback(MQClientAPIImpl.this) {
+        this.remotingClient.invokeAsync(addr, request, timeOut, new InvokeCallback() {
+            @Override
+            public void operationComplete(ResponseFuture responseFuture) {
+
+            }
 
             @Override
-            public void onComplete(ResponseFuture responseFuture) {
-                RemotingCommand response = responseFuture.getResponseCommand();
-                if (response != null) {
-                    try {
-                        AckResult ackResult = new AckResult();
-                        if (ResponseCode.SUCCESS == response.getCode()) {
-                            ackResult.setStatus(AckStatus.OK);
-                        } else {
-                            ackResult.setStatus(AckStatus.NO_EXIST);
-                        }
-                        ackCallback.onSuccess(ackResult);
-                    } catch (Exception e) {
-                        ackCallback.onException(e);
-                    }
+            public void operationSucceed(RemotingCommand response) {
+                AckResult ackResult = new AckResult();
+                if (ResponseCode.SUCCESS == response.getCode()) {
+                    ackResult.setStatus(AckStatus.OK);
                 } else {
-                    if (!responseFuture.isSendRequestOK()) {
-                        ackCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause()));
-                    } else if (responseFuture.isTimeout()) {
-                        ackCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request,
-                            responseFuture.getCause()));
-                    } else {
-                        ackCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeOut + ". Request: " + request, responseFuture.getCause()));
-                    }
+                    ackResult.setStatus(AckStatus.NO_EXIST);
                 }
+                ackCallback.onSuccess(ackResult);
+            }
 
+            @Override
+            public void operationFail(Throwable throwable) {
+                ackCallback.onException(throwable);
             }
         });
     }
@@ -999,39 +989,37 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
         final AckCallback ackCallback
     ) throws RemotingException, MQBrokerException, InterruptedException {
         final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader);
-        this.remotingClient.invokeAsync(addr, request, timeoutMillis, new BaseInvokeCallback(MQClientAPIImpl.this) {
+        this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {
             @Override
-            public void onComplete(ResponseFuture responseFuture) {
-                RemotingCommand response = responseFuture.getResponseCommand();
-                if (response != null) {
-                    try {
-                        ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.decodeCommandCustomHeader(ChangeInvisibleTimeResponseHeader.class);
-                        AckResult ackResult = new AckResult();
-                        if (ResponseCode.SUCCESS == response.getCode()) {
-                            ackResult.setStatus(AckStatus.OK);
-                            ackResult.setPopTime(responseHeader.getPopTime());
-                            ackResult.setExtraInfo(ExtraInfoUtil
-                                .buildExtraInfo(requestHeader.getOffset(), responseHeader.getPopTime(), responseHeader.getInvisibleTime(),
-                                    responseHeader.getReviveQid(), requestHeader.getTopic(), brokerName, requestHeader.getQueueId()) + MessageConst.KEY_SEPARATOR
-                                + requestHeader.getOffset());
-                        } else {
-                            ackResult.setStatus(AckStatus.NO_EXIST);
-                        }
-                        ackCallback.onSuccess(ackResult);
-                    } catch (Exception e) {
-                        ackCallback.onException(e);
-                    }
-                } else {
-                    if (!responseFuture.isSendRequestOK()) {
-                        ackCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause()));
-                    } else if (responseFuture.isTimeout()) {
-                        ackCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request,
-                            responseFuture.getCause()));
+            public void operationComplete(ResponseFuture responseFuture) {
+
+            }
+
+            @Override
+            public void operationSucceed(RemotingCommand response) {
+                try {
+                    ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.decodeCommandCustomHeader(ChangeInvisibleTimeResponseHeader.class);
+                    AckResult ackResult = new AckResult();
+                    if (ResponseCode.SUCCESS == response.getCode()) {
+                        ackResult.setStatus(AckStatus.OK);
+                        ackResult.setPopTime(responseHeader.getPopTime());
+                        ackResult.setExtraInfo(ExtraInfoUtil
+                            .buildExtraInfo(requestHeader.getOffset(), responseHeader.getPopTime(), responseHeader.getInvisibleTime(),
+                                responseHeader.getReviveQid(), requestHeader.getTopic(), brokerName, requestHeader.getQueueId()) + MessageConst.KEY_SEPARATOR
+                            + requestHeader.getOffset());
                     } else {
-                        ackCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause()));
+                        ackResult.setStatus(AckStatus.NO_EXIST);
                     }
+                    ackCallback.onSuccess(ackResult);
+                } catch (Exception e) {
+                    ackCallback.onException(e);
                 }
             }
+
+            @Override
+            public void operationFail(Throwable throwable) {
+                ackCallback.onException(throwable);
+            }
         });
     }
 
@@ -1044,26 +1032,23 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
         this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {
             @Override
             public void operationComplete(ResponseFuture responseFuture) {
-                RemotingCommand response = responseFuture.getResponseCommand();
-                if (response != null) {
-                    try {
-                        PullResult pullResult = MQClientAPIImpl.this.processPullResponse(response, addr);
-                        assert pullResult != null;
-                        pullCallback.onSuccess(pullResult);
-                    } catch (Exception e) {
-                        pullCallback.onException(e);
-                    }
-                } else {
-                    if (!responseFuture.isSendRequestOK()) {
-                        pullCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause()));
-                    } else if (responseFuture.isTimeout()) {
-                        pullCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request,
-                            responseFuture.getCause()));
-                    } else {
-                        pullCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause()));
-                    }
+
+            }
+
+            @Override
+            public void operationSucceed(RemotingCommand response) {
+                try {
+                    PullResult pullResult = MQClientAPIImpl.this.processPullResponse(response, addr);
+                    pullCallback.onSuccess(pullResult);
+                } catch (Exception e) {
+                    pullCallback.onException(e);
                 }
             }
+
+            @Override
+            public void operationFail(Throwable throwable) {
+                pullCallback.onException(throwable);
+            }
         });
     }
 
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java
index d7c8ef8d9..f3102e175 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java
@@ -30,7 +30,6 @@ import org.apache.rocketmq.client.consumer.PullCallback;
 import org.apache.rocketmq.client.consumer.PullResult;
 import org.apache.rocketmq.client.consumer.PullStatus;
 import org.apache.rocketmq.client.exception.MQBrokerException;
-import org.apache.rocketmq.client.exception.MQClientException;
 import org.apache.rocketmq.client.exception.OffsetNotFoundException;
 import org.apache.rocketmq.client.impl.ClientRemotingProcessor;
 import org.apache.rocketmq.client.impl.CommunicationMode;
@@ -47,6 +46,7 @@ import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageQueue;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.remoting.InvokeCallback;
 import org.apache.rocketmq.remoting.RPCHook;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.netty.NettyClientConfig;
@@ -106,19 +106,6 @@ public class MQClientAPIExt extends MQClientAPIImpl {
         return false;
     }
 
-    protected static MQClientException processNullResponseErr(ResponseFuture responseFuture) {
-        MQClientException ex;
-        if (!responseFuture.isSendRequestOK()) {
-            ex = new MQClientException("send request failed", responseFuture.getCause());
-        } else if (responseFuture.isTimeout()) {
-            ex = new MQClientException("wait response timeout " + responseFuture.getTimeoutMillis() + "ms",
-                responseFuture.getCause());
-        } else {
-            ex = new MQClientException("unknown reason", responseFuture.getCause());
-        }
-        return ex;
-    }
-
     public CompletableFuture<Void> sendHeartbeatOneway(
         String brokerAddr,
         HeartbeatData heartbeatData,
@@ -146,24 +133,15 @@ public class MQClientAPIExt extends MQClientAPIImpl {
         request.setLanguage(clientConfig.getLanguage());
         request.setBody(heartbeatData.encode());
 
-        CompletableFuture<Integer> future = new CompletableFuture<>();
-        try {
-            this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
-                RemotingCommand response = responseFuture.getResponseCommand();
-                if (response != null) {
-                    if (ResponseCode.SUCCESS == response.getCode()) {
-                        future.complete(response.getVersion());
-                    } else {
-                        future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr));
-                    }
-                } else {
-                    future.completeExceptionally(processNullResponseErr(responseFuture));
-                }
-            });
-        } catch (Throwable t) {
-            future.completeExceptionally(t);
-        }
-        return future;
+        return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {
+            CompletableFuture<Integer> future0 = new CompletableFuture<>();
+            if (ResponseCode.SUCCESS == response.getCode()) {
+                future0.complete(response.getVersion());
+            } else {
+                future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr));
+            }
+            return future0;
+        });
     }
 
     public CompletableFuture<SendResult> sendMessageAsync(
@@ -177,24 +155,15 @@ public class MQClientAPIExt extends MQClientAPIImpl {
         RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, requestHeaderV2);
         request.setBody(msg.getBody());
 
-        CompletableFuture<SendResult> future = new CompletableFuture<>();
-        try {
-            this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
-                RemotingCommand response = responseFuture.getResponseCommand();
-                if (response != null) {
-                    try {
-                        future.complete(this.processSendResponse(brokerName, msg, response, brokerAddr));
-                    } catch (Exception e) {
-                        future.completeExceptionally(e);
-                    }
-                } else {
-                    future.completeExceptionally(processNullResponseErr(responseFuture));
-                }
-            });
-        } catch (Throwable t) {
-            future.completeExceptionally(t);
-        }
-        return future;
+        return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {
+            CompletableFuture<SendResult> future0 = new CompletableFuture<>();
+            try {
+                future0.complete(this.processSendResponse(brokerName, msg, response, brokerAddr));
+            } catch (Exception e) {
+                future0.completeExceptionally(e);
+            }
+            return future0;
+        });
     }
 
     public CompletableFuture<SendResult> sendMessageAsync(
@@ -216,17 +185,14 @@ public class MQClientAPIExt extends MQClientAPIImpl {
             msgBatch.setBody(body);
 
             request.setBody(body);
-            this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
-                RemotingCommand response = responseFuture.getResponseCommand();
-                if (response != null) {
-                    try {
-                        future.complete(this.processSendResponse(brokerName, msgBatch, response, brokerAddr));
-                    } catch (Exception e) {
-                        future.completeExceptionally(e);
-                    }
-                } else {
-                    future.completeExceptionally(processNullResponseErr(responseFuture));
+            return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {
+                CompletableFuture<SendResult> future0 = new CompletableFuture<>();
+                try {
+                    future0.complete(processSendResponse(brokerName, msgBatch, response, brokerAddr));
+                } catch (Exception e) {
+                    future0.completeExceptionally(e);
                 }
+                return future0;
             });
         } catch (Throwable t) {
             future.completeExceptionally(t);
@@ -240,21 +206,7 @@ public class MQClientAPIExt extends MQClientAPIImpl {
         long timeoutMillis
     ) {
         RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, requestHeader);
-
-        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
-        try {
-            this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
-                RemotingCommand response = responseFuture.getResponseCommand();
-                if (response != null) {
-                    future.complete(response);
-                } else {
-                    future.completeExceptionally(processNullResponseErr(responseFuture));
-                }
-            });
-        } catch (Throwable t) {
-            future.completeExceptionally(t);
-        }
-        return future;
+        return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis);
     }
 
     public CompletableFuture<PopResult> popMessageAsync(
@@ -402,38 +354,31 @@ public class MQClientAPIExt extends MQClientAPIImpl {
         QueryConsumerOffsetRequestHeader requestHeader,
         long timeoutMillis
     ) {
-        CompletableFuture<Long> future = new CompletableFuture<>();
-        try {
-            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader);
-            this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
-                RemotingCommand response = responseFuture.getResponseCommand();
-                if (response != null) {
-                    switch (response.getCode()) {
-                        case ResponseCode.SUCCESS: {
-                            try {
-                                QueryConsumerOffsetResponseHeader responseHeader =
-                                    (QueryConsumerOffsetResponseHeader) response.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class);
-                                future.complete(responseHeader.getOffset());
-                            } catch (RemotingCommandException e) {
-                                future.completeExceptionally(e);
-                            }
-                            break;
-                        }
-                        case ResponseCode.QUERY_NOT_FOUND: {
-                            future.completeExceptionally(new OffsetNotFoundException(response.getCode(), response.getRemark(), brokerAddr));
-                            break;
-                        }
-                        default:
-                            break;
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader);
+        return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {
+            CompletableFuture<Long> future0 = new CompletableFuture<>();
+            switch (response.getCode()) {
+                case ResponseCode.SUCCESS: {
+                    try {
+                        QueryConsumerOffsetResponseHeader responseHeader =
+                            (QueryConsumerOffsetResponseHeader) response.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class);
+                        future0.complete(responseHeader.getOffset());
+                    } catch (RemotingCommandException e) {
+                        future0.completeExceptionally(e);
                     }
-                } else {
-                    future.completeExceptionally(processNullResponseErr(responseFuture));
+                    break;
                 }
-            });
-        } catch (Throwable t) {
-            future.completeExceptionally(t);
-        }
-        return future;
+                case ResponseCode.QUERY_NOT_FOUND: {
+                    future0.completeExceptionally(new OffsetNotFoundException(response.getCode(), response.getRemark(), brokerAddr));
+                    break;
+                }
+                default: {
+                    future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));
+                    break;
+                }
+            }
+            return future0;
+        });
     }
 
     public CompletableFuture<Void> updateConsumerOffsetOneWay(
@@ -461,9 +406,14 @@ public class MQClientAPIExt extends MQClientAPIImpl {
 
         CompletableFuture<List<String>> future = new CompletableFuture<>();
         try {
-            this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
-                RemotingCommand response = responseFuture.getResponseCommand();
-                if (response != null) {
+            this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() {
+                @Override
+                public void operationComplete(ResponseFuture responseFuture) {
+
+                }
+
+                @Override
+                public void operationSucceed(RemotingCommand response) {
                     switch (response.getCode()) {
                         case ResponseCode.SUCCESS: {
                             if (response.getBody() != null) {
@@ -485,8 +435,11 @@ public class MQClientAPIExt extends MQClientAPIImpl {
                             break;
                     }
                     future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));
-                } else {
-                    future.completeExceptionally(processNullResponseErr(responseFuture));
+                }
+
+                @Override
+                public void operationFail(Throwable throwable) {
+                    future.completeExceptionally(throwable);
                 }
             });
         } catch (Throwable t) {
@@ -501,9 +454,14 @@ public class MQClientAPIExt extends MQClientAPIImpl {
 
         CompletableFuture<Long> future = new CompletableFuture<>();
         try {
-            this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
-                RemotingCommand response = responseFuture.getResponseCommand();
-                if (response != null) {
+            this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() {
+                @Override
+                public void operationComplete(ResponseFuture responseFuture) {
+
+                }
+
+                @Override
+                public void operationSucceed(RemotingCommand response) {
                     if (ResponseCode.SUCCESS == response.getCode()) {
                         try {
                             GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class);
@@ -513,8 +471,11 @@ public class MQClientAPIExt extends MQClientAPIImpl {
                         }
                     }
                     future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));
-                } else {
-                    future.completeExceptionally(processNullResponseErr(responseFuture));
+                }
+
+                @Override
+                public void operationFail(Throwable throwable) {
+                    future.completeExceptionally(throwable);
                 }
             });
         } catch (Throwable t) {
@@ -529,9 +490,14 @@ public class MQClientAPIExt extends MQClientAPIImpl {
 
         CompletableFuture<Long> future = new CompletableFuture<>();
         try {
-            this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
-                RemotingCommand response = responseFuture.getResponseCommand();
-                if (response != null) {
+            this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() {
+                @Override
+                public void operationComplete(ResponseFuture responseFuture) {
+
+                }
+
+                @Override
+                public void operationSucceed(RemotingCommand response) {
                     if (ResponseCode.SUCCESS == response.getCode()) {
                         try {
                             GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class);
@@ -541,8 +507,11 @@ public class MQClientAPIExt extends MQClientAPIImpl {
                         }
                     }
                     future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));
-                } else {
-                    future.completeExceptionally(processNullResponseErr(responseFuture));
+                }
+
+                @Override
+                public void operationFail(Throwable throwable) {
+                    future.completeExceptionally(throwable);
                 }
             });
         } catch (Throwable t) {
@@ -555,57 +524,41 @@ public class MQClientAPIExt extends MQClientAPIImpl {
         long timeoutMillis) {
         RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, requestHeader);
 
-        CompletableFuture<Long> future = new CompletableFuture<>();
-        try {
-            this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
-                RemotingCommand response = responseFuture.getResponseCommand();
-                if (response != null) {
-                    if (response.getCode() == ResponseCode.SUCCESS) {
-                        try {
-                            SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.decodeCommandCustomHeader(SearchOffsetResponseHeader.class);
-                            future.complete(responseHeader.getOffset());
-                        } catch (Throwable t) {
-                            future.completeExceptionally(t);
-                        }
-                    }
-                    future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));
-                } else {
-                    future.completeExceptionally(processNullResponseErr(responseFuture));
+        return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {
+            CompletableFuture<Long> future0 = new CompletableFuture<>();
+            if (response.getCode() == ResponseCode.SUCCESS) {
+                try {
+                    SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.decodeCommandCustomHeader(SearchOffsetResponseHeader.class);
+                    future0.complete(responseHeader.getOffset());
+                } catch (Throwable t) {
+                    future0.completeExceptionally(t);
                 }
-            });
-        } catch (Throwable t) {
-            future.completeExceptionally(t);
-        }
-        return future;
+            } else {
+                future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));
+            }
+            return future0;
+        });
     }
 
     public CompletableFuture<Set<MessageQueue>> lockBatchMQWithFuture(String brokerAddr,
         LockBatchRequestBody requestBody, long timeoutMillis) {
-        CompletableFuture<Set<MessageQueue>> future = new CompletableFuture<>();
         RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null);
         request.setBody(requestBody.encode());
-        try {
-            this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> {
-                RemotingCommand response = responseFuture.getResponseCommand();
-                if (response != null) {
-                    if (response.getCode() == ResponseCode.SUCCESS) {
-                        try {
-                            LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), LockBatchResponseBody.class);
-                            Set<MessageQueue> messageQueues = responseBody.getLockOKMQSet();
-                            future.complete(messageQueues);
-                        } catch (Throwable t) {
-                            future.completeExceptionally(t);
-                        }
-                    }
-                    future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));
-                } else {
-                    future.completeExceptionally(processNullResponseErr(responseFuture));
+        return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {
+            CompletableFuture<Set<MessageQueue>> future0 = new CompletableFuture<>();
+            if (response.getCode() == ResponseCode.SUCCESS) {
+                try {
+                    LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), LockBatchResponseBody.class);
+                    Set<MessageQueue> messageQueues = responseBody.getLockOKMQSet();
+                    future0.complete(messageQueues);
+                } catch (Throwable t) {
+                    future0.completeExceptionally(t);
                 }
-            });
-        } catch (Exception e) {
-            future.completeExceptionally(e);
-        }
-        return future;
+            } else {
+                future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));
+            }
+            return future0;
+        });
     }
 
     public CompletableFuture<Void> unlockBatchMQOneway(String brokerAddr,
@@ -624,25 +577,21 @@ public class MQClientAPIExt extends MQClientAPIImpl {
 
     public CompletableFuture<Boolean> notification(String brokerAddr, NotificationRequestHeader requestHeader,
         long timeoutMillis) {
-        CompletableFuture<Boolean> future = new CompletableFuture<>();
         RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.NOTIFICATION, requestHeader);
-        try {
-            this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenAccept(response -> {
-                if (response.getCode() == ResponseCode.SUCCESS) {
-                    try {
-                        NotificationResponseHeader responseHeader = (NotificationResponseHeader) response.decodeCommandCustomHeader(NotificationResponseHeader.class);
-                        future.complete(responseHeader.isHasMsg());
-                    } catch (Throwable t) {
-                        future.completeExceptionally(t);
-                    }
-                } else {
-                    future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));
+        return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> {
+            CompletableFuture<Boolean> future0 = new CompletableFuture<>();
+            if (response.getCode() == ResponseCode.SUCCESS) {
+                try {
+                    NotificationResponseHeader responseHeader = (NotificationResponseHeader) response.decodeCommandCustomHeader(NotificationResponseHeader.class);
+                    future0.complete(responseHeader.isHasMsg());
+                } catch (Throwable t) {
+                    future0.completeExceptionally(t);
                 }
-            });
-        } catch (Throwable t) {
-            future.completeExceptionally(t);
-        }
-        return future;
+            } else {
+                future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark()));
+            }
+            return future0;
+        });
     }
 
     public CompletableFuture<RemotingCommand> invoke(String brokerAddr, RemotingCommand request, long timeoutMillis) {
diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java
index d13f2cfe4..c152d38ea 100644
--- a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java
+++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java
@@ -212,7 +212,7 @@ public class MQClientAPIImplTest {
                 RemotingCommand request = mock.getArgument(1);
                 ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
                 responseFuture.setResponseCommand(createSendMessageSuccessResponse(request));
-                callback.operationComplete(responseFuture);
+                callback.operationSucceed(responseFuture.getResponseCommand());
                 return null;
             }
         }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
@@ -386,7 +386,7 @@ public class MQClientAPIImplTest {
                 RemotingCommand request = mock.getArgument(1);
                 ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
                 responseFuture.setResponseCommand(createSendMessageSuccessResponse(request));
-                callback.operationComplete(responseFuture);
+                callback.operationSucceed(responseFuture.getResponseCommand());
                 return null;
             }
         }).when(remotingClient).invokeAsync(Matchers.anyString(), Matchers.any(RemotingCommand.class), Matchers.anyLong(), Matchers.any(InvokeCallback.class));
@@ -472,7 +472,7 @@ public class MQClientAPIImplTest {
                 message.putUserProperty("key", "value");
                 response.setBody(MessageDecoder.encode(message, false));
                 responseFuture.setResponseCommand(response);
-                callback.operationComplete(responseFuture);
+                callback.operationSucceed(responseFuture.getResponseCommand());
                 return null;
             }
         }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
@@ -543,7 +543,7 @@ public class MQClientAPIImplTest {
                 message.getProperties().put(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, String.valueOf(0));
                 response.setBody(MessageDecoder.encode(message, false));
                 responseFuture.setResponseCommand(response);
-                callback.operationComplete(responseFuture);
+                callback.operationSucceed(responseFuture.getResponseCommand());
                 return null;
             }
         }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
@@ -585,7 +585,7 @@ public class MQClientAPIImplTest {
                 response.setOpaque(request.getOpaque());
                 response.setCode(ResponseCode.SUCCESS);
                 responseFuture.setResponseCommand(response);
-                callback.operationComplete(responseFuture);
+                callback.operationSucceed(responseFuture.getResponseCommand());
                 return null;
             }
         }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
@@ -622,7 +622,7 @@ public class MQClientAPIImplTest {
                 responseHeader.setPopTime(System.currentTimeMillis());
                 responseHeader.setInvisibleTime(10 * 1000L);
                 responseFuture.setResponseCommand(response);
-                callback.operationComplete(responseFuture);
+                callback.operationSucceed(responseFuture.getResponseCommand());
                 return null;
             }
         }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
index fe07090d5..3227d1e1c 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java
@@ -26,7 +26,6 @@ import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import org.apache.rocketmq.acl.AccessValidator;
-import org.apache.rocketmq.client.exception.MQClientException;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.future.FutureTaskExt;
 import org.apache.rocketmq.common.thread.ThreadPoolMonitor;
@@ -51,10 +50,12 @@ import org.apache.rocketmq.proxy.remoting.channel.RemotingChannelManager;
 import org.apache.rocketmq.proxy.remoting.pipeline.AuthenticationPipeline;
 import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline;
 import org.apache.rocketmq.remoting.ChannelEventListener;
+import org.apache.rocketmq.remoting.InvokeCallback;
 import org.apache.rocketmq.remoting.RemotingServer;
 import org.apache.rocketmq.remoting.netty.NettyRemotingServer;
 import org.apache.rocketmq.remoting.netty.NettyServerConfig;
 import org.apache.rocketmq.remoting.netty.RequestTask;
+import org.apache.rocketmq.remoting.netty.ResponseFuture;
 import org.apache.rocketmq.remoting.netty.TlsSystemConfig;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.remoting.protocol.RequestCode;
@@ -239,12 +240,21 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu
         long timeoutMillis) {
         CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
         try {
-            this.defaultRemotingServer.invokeAsync(channel, request, timeoutMillis, responseFuture -> {
-                if (responseFuture.getResponseCommand() == null) {
-                    future.completeExceptionally(new MQClientException("response is null after send request to client", responseFuture.getCause()));
-                    return;
+            this.defaultRemotingServer.invokeAsync(channel, request, timeoutMillis, new InvokeCallback() {
+                @Override
+                public void operationComplete(ResponseFuture responseFuture) {
+
+                }
+
+                @Override
+                public void operationSucceed(RemotingCommand response) {
+                    future.complete(response);
+                }
+
+                @Override
+                public void operationFail(Throwable throwable) {
+                    future.completeExceptionally(throwable);
                 }
-                future.complete(responseFuture.getResponseCommand());
             });
         } catch (Throwable t) {
             future.completeExceptionally(t);
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java
index 3f3a4ae40..e2d05b0f5 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
@@ -85,6 +86,7 @@ import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
 
 @RunWith(MockitoJUnitRunner.class)
 public class MQClientAPIExtTest {
@@ -109,13 +111,9 @@ public class MQClientAPIExtTest {
 
     @Test
     public void testSendHeartbeatAsync() throws Exception {
-        doAnswer((Answer<Void>) mock -> {
-            InvokeCallback invokeCallback = mock.getArgument(3);
-            ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null);
-            responseFuture.putResponse(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, ""));
-            invokeCallback.operationComplete(responseFuture);
-            return null;
-        }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any());
+        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
+        future.complete(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, ""));
+        doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());
 
         assertNotNull(mqClientAPI.sendHeartbeatAsync(BROKER_ADDR, new HeartbeatData(), TIMEOUT).get());
     }
@@ -123,20 +121,16 @@ public class MQClientAPIExtTest {
     @Test
     public void testSendMessageAsync() throws Exception {
         AtomicReference<String> msgIdRef = new AtomicReference<>();
-        doAnswer((Answer<Void>) mock -> {
-            InvokeCallback invokeCallback = mock.getArgument(3);
-            ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null);
-            RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);
-            SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader();
-            sendMessageResponseHeader.setMsgId(msgIdRef.get());
-            sendMessageResponseHeader.setQueueId(0);
-            sendMessageResponseHeader.setQueueOffset(1L);
-            response.setCode(ResponseCode.SUCCESS);
-            response.makeCustomHeaderToNet();
-            responseFuture.putResponse(response);
-            invokeCallback.operationComplete(responseFuture);
-            return null;
-        }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any());
+        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
+        RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);
+        SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader();
+        sendMessageResponseHeader.setMsgId(msgIdRef.get());
+        sendMessageResponseHeader.setQueueId(0);
+        sendMessageResponseHeader.setQueueOffset(1L);
+        response.setCode(ResponseCode.SUCCESS);
+        response.makeCustomHeaderToNet();
+        future.complete(response);
+        doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());
 
         MessageExt messageExt = createMessage();
         msgIdRef.set(MessageClientIDSetter.getUniqID(messageExt));
@@ -150,20 +144,16 @@ public class MQClientAPIExtTest {
 
     @Test
     public void testSendMessageListAsync() throws Exception {
-        doAnswer((Answer<Void>) mock -> {
-            InvokeCallback invokeCallback = mock.getArgument(3);
-            ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null);
-            RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);
-            SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader();
-            sendMessageResponseHeader.setMsgId("");
-            sendMessageResponseHeader.setQueueId(0);
-            sendMessageResponseHeader.setQueueOffset(1L);
-            response.setCode(ResponseCode.SUCCESS);
-            response.makeCustomHeaderToNet();
-            responseFuture.putResponse(response);
-            invokeCallback.operationComplete(responseFuture);
-            return null;
-        }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any());
+        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
+        RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);
+        SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader();
+        sendMessageResponseHeader.setMsgId("");
+        sendMessageResponseHeader.setQueueId(0);
+        sendMessageResponseHeader.setQueueOffset(1L);
+        response.setCode(ResponseCode.SUCCESS);
+        response.makeCustomHeaderToNet();
+        future.complete(response);
+        doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());
 
         List<MessageExt> messageExtList = new ArrayList<>();
         StringBuilder sb = new StringBuilder();
@@ -182,13 +172,9 @@ public class MQClientAPIExtTest {
 
     @Test
     public void testSendMessageBackAsync() throws Exception {
-        doAnswer((Answer<Void>) mock -> {
-            InvokeCallback invokeCallback = mock.getArgument(3);
-            ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null);
-            responseFuture.putResponse(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, ""));
-            invokeCallback.operationComplete(responseFuture);
-            return null;
-        }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any());
+        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
+        future.complete(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, ""));
+        doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());
 
         RemotingCommand remotingCommand = mqClientAPI.sendMessageBackAsync(BROKER_ADDR, new ConsumerSendMsgBackRequestHeader(), TIMEOUT)
             .get();
@@ -285,7 +271,7 @@ public class MQClientAPIExtTest {
             body.setConsumerIdList(clientIds);
             response.setBody(body.encode());
             responseFuture.putResponse(response);
-            invokeCallback.operationComplete(responseFuture);
+            invokeCallback.operationSucceed(responseFuture.getResponseCommand());
             return null;
         }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any());
 
@@ -302,7 +288,7 @@ public class MQClientAPIExtTest {
             response.setCode(ResponseCode.SYSTEM_ERROR);
             response.makeCustomHeaderToNet();
             responseFuture.putResponse(response);
-            invokeCallback.operationComplete(responseFuture);
+            invokeCallback.operationSucceed(responseFuture.getResponseCommand());
             return null;
         }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any());
 
@@ -322,7 +308,7 @@ public class MQClientAPIExtTest {
             response.setCode(ResponseCode.SUCCESS);
             response.makeCustomHeaderToNet();
             responseFuture.putResponse(response);
-            invokeCallback.operationComplete(responseFuture);
+            invokeCallback.operationSucceed(responseFuture.getResponseCommand());
             return null;
         }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any());
 
@@ -335,18 +321,15 @@ public class MQClientAPIExtTest {
     @Test
     public void testSearchOffsetAsync() throws Exception {
         long offset = ThreadLocalRandom.current().nextLong();
-        doAnswer((Answer<Void>) mock -> {
-            InvokeCallback invokeCallback = mock.getArgument(3);
-            ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null);
-            RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class);
-            SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader();
-            responseHeader.setOffset(offset);
-            response.setCode(ResponseCode.SUCCESS);
-            response.makeCustomHeaderToNet();
-            responseFuture.putResponse(response);
-            invokeCallback.operationComplete(responseFuture);
-            return null;
-        }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any());
+        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
+        RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class);
+        SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader();
+        responseHeader.setOffset(offset);
+        response.setCode(ResponseCode.SUCCESS);
+        response.makeCustomHeaderToNet();
+        future.complete(response);
+
+        doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());
 
         SearchOffsetRequestHeader requestHeader = new SearchOffsetRequestHeader();
         requestHeader.setTopic(TOPIC);
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java b/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java
index ce78fa923..6be491745 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java
@@ -17,7 +17,22 @@
 package org.apache.rocketmq.remoting;
 
 import org.apache.rocketmq.remoting.netty.ResponseFuture;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 
 public interface InvokeCallback {
+    /**
+     * This method is expected to be invoked after {@link #operationSucceed(RemotingCommand)}
+     * or {@link #operationFail(Throwable)}
+     *
+     * @param responseFuture the returned object contains response or exception
+     */
     void operationComplete(final ResponseFuture responseFuture);
+
+    default void operationSucceed(final RemotingCommand response) {
+
+    }
+
+    default void operationFail(final Throwable throwable) {
+
+    }
 }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java
index ff0b3df95..c8389eedb 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java
@@ -20,11 +20,11 @@ import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
 import org.apache.rocketmq.remoting.exception.RemotingConnectException;
-import org.apache.rocketmq.remoting.exception.RemotingException;
 import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
 import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
 import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;
 import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
+import org.apache.rocketmq.remoting.netty.ResponseFuture;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 
 public interface RemotingClient extends RemotingService {
@@ -51,18 +51,21 @@ public interface RemotingClient extends RemotingService {
         final long timeoutMillis) {
         CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
         try {
-            invokeAsync(addr, request, timeoutMillis, responseFuture -> {
-                RemotingCommand response = responseFuture.getResponseCommand();
-                if (response != null) {
+            invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {
+
+                @Override
+                public void operationComplete(ResponseFuture responseFuture) {
+
+                }
+
+                @Override
+                public void operationSucceed(RemotingCommand response) {
                     future.complete(response);
-                } else {
-                    if (!responseFuture.isSendRequestOK()) {
-                        future.completeExceptionally(new RemotingSendRequestException(addr, responseFuture.getCause()));
-                    } else if (responseFuture.isTimeout()) {
-                        future.completeExceptionally(new RemotingTimeoutException(addr, timeoutMillis, responseFuture.getCause()));
-                    } else {
-                        future.completeExceptionally(new RemotingException(request.toString(), responseFuture.getCause()));
-                    }
+                }
+
+                @Override
+                public void operationFail(Throwable throwable) {
+                    future.completeExceptionally(throwable);
                 }
             });
         } catch (Throwable t) {
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
index fce2de267..12e66f913 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
@@ -23,20 +23,23 @@ import io.netty.handler.ssl.SslContext;
 import io.netty.handler.ssl.SslHandler;
 import io.netty.util.concurrent.Future;
 import io.opentelemetry.api.common.AttributesBuilder;
-import java.net.SocketAddress;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map.Entry;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import javax.annotation.Nullable;
 import org.apache.rocketmq.common.AbortProcessException;
@@ -125,7 +128,7 @@ public abstract class NettyRemotingAbstract {
      * Constructor, specifying capacity of one-way and asynchronous semaphores.
      *
      * @param permitsOneway Number of permits for one-way requests.
-     * @param permitsAsync Number of permits for asynchronous requests.
+     * @param permitsAsync  Number of permits for asynchronous requests.
      */
     public NettyRemotingAbstract(final int permitsOneway, final int permitsAsync) {
         this.semaphoreOneway = new Semaphore(permitsOneway, true);
@@ -367,8 +370,7 @@ public abstract class NettyRemotingAbstract {
                 responseFuture.release();
             }
         } else {
-            log.warn("receive response, but not matched any request, " + RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
-            log.warn(cmd.toString());
+            log.warn("receive response, cmd={}, but not matched any request, address={}", cmd, RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
         }
     }
 
@@ -467,57 +469,68 @@ public abstract class NettyRemotingAbstract {
     public RemotingCommand invokeSyncImpl(final Channel channel, final RemotingCommand request,
         final long timeoutMillis)
         throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException {
-        //get the request id
-        final int opaque = request.getOpaque();
-
         try {
-            final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis, null, null);
-            this.responseTable.put(opaque, responseFuture);
-            final SocketAddress addr = channel.remoteAddress();
-            channel.writeAndFlush(request).addListener((ChannelFutureListener) f -> {
-                if (f.isSuccess()) {
-                    responseFuture.setSendRequestOK(true);
-                    return;
-                }
-
-                responseFuture.setSendRequestOK(false);
-                responseTable.remove(opaque);
-                responseFuture.setCause(f.cause());
-                responseFuture.putResponse(null);
-                log.warn("Failed to write a request command to {}, caused by underlying I/O operation failure", addr);
-            });
+            return invokeImpl(channel, request, timeoutMillis).thenApply(ResponseFuture::getResponseCommand)
+                .get(timeoutMillis, TimeUnit.MILLISECONDS);
+        } catch (ExecutionException e) {
+            throw new RemotingSendRequestException(channel.remoteAddress().toString(), e.getCause());
+        } catch (TimeoutException e) {
+            throw new RemotingTimeoutException(channel.remoteAddress().toString(), timeoutMillis, e.getCause());
+        }
+    }
 
-            RemotingCommand responseCommand = responseFuture.waitResponse(timeoutMillis);
-            if (null == responseCommand) {
-                if (responseFuture.isSendRequestOK()) {
-                    throw new RemotingTimeoutException(RemotingHelper.parseSocketAddressAddr(addr), timeoutMillis,
-                        responseFuture.getCause());
-                } else {
-                    throw new RemotingSendRequestException(RemotingHelper.parseSocketAddressAddr(addr), responseFuture.getCause());
-                }
+    public CompletableFuture<ResponseFuture> invokeImpl(final Channel channel, final RemotingCommand request,
+        final long timeoutMillis) {
+        String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel);
+        doBeforeRpcHooks(channelRemoteAddr, request);
+        return invoke0(channel, request, timeoutMillis).whenComplete((v, t) -> {
+            if (t == null) {
+                doAfterRpcHooks(channelRemoteAddr, request, v.getResponseCommand());
             }
-
-            return responseCommand;
-        } finally {
-            this.responseTable.remove(opaque);
-        }
+        });
     }
 
-    public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis,
-        final InvokeCallback invokeCallback)
-        throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException {
+    protected CompletableFuture<ResponseFuture> invoke0(final Channel channel, final RemotingCommand request,
+        final long timeoutMillis) {
+        CompletableFuture<ResponseFuture> future = new CompletableFuture<>();
         long beginStartTime = System.currentTimeMillis();
         final int opaque = request.getOpaque();
-        boolean acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS);
+
+        boolean acquired;
+        try {
+            acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS);
+        } catch (Throwable t) {
+            future.completeExceptionally(t);
+            return future;
+        }
         if (acquired) {
             final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreAsync);
             long costTime = System.currentTimeMillis() - beginStartTime;
             if (timeoutMillis < costTime) {
                 once.release();
-                throw new RemotingTimeoutException("invokeAsyncImpl call timeout");
+                future.completeExceptionally(new RemotingTimeoutException("invokeAsyncImpl call timeout"));
+                return future;
             }
 
-            final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis - costTime, invokeCallback, once);
+            AtomicReference<ResponseFuture> responseFutureReference = new AtomicReference<>();
+            final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, request, timeoutMillis - costTime,
+                new InvokeCallback() {
+                    @Override
+                    public void operationComplete(ResponseFuture responseFuture) {
+
+                    }
+
+                    @Override
+                    public void operationSucceed(RemotingCommand response) {
+                        future.complete(responseFutureReference.get());
+                    }
+
+                    @Override
+                    public void operationFail(Throwable throwable) {
+                        future.completeExceptionally(throwable);
+                    }
+                }, once);
+            responseFutureReference.set(responseFuture);
             this.responseTable.put(opaque, responseFuture);
             try {
                 channel.writeAndFlush(request).addListener((ChannelFutureListener) f -> {
@@ -528,15 +541,17 @@ public abstract class NettyRemotingAbstract {
                     requestFail(opaque);
                     log.warn("send a request command to channel <{}> failed.", RemotingHelper.parseChannelRemoteAddr(channel));
                 });
+                return future;
             } catch (Exception e) {
                 responseTable.remove(opaque);
                 responseFuture.release();
                 log.warn("send a request command to channel <" + RemotingHelper.parseChannelRemoteAddr(channel) + "> Exception", e);
-                throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e);
+                future.completeExceptionally(new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e));
+                return future;
             }
         } else {
             if (timeoutMillis <= 0) {
-                throw new RemotingTooMuchRequestException("invokeAsyncImpl invoke too fast");
+                future.completeExceptionally(new RemotingTooMuchRequestException("invokeAsyncImpl invoke too fast"));
             } else {
                 String info =
                     String.format("invokeAsyncImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreAsyncValue: %d",
@@ -545,11 +560,31 @@ public abstract class NettyRemotingAbstract {
                         this.semaphoreAsync.availablePermits()
                     );
                 log.warn(info);
-                throw new RemotingTimeoutException(info);
+                future.completeExceptionally(new RemotingTimeoutException(info));
             }
+            return future;
         }
     }
 
+    public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis,
+        final InvokeCallback invokeCallback) {
+        invokeImpl(channel, request, timeoutMillis)
+            .whenComplete((v, t) -> {
+                if (t == null) {
+                    invokeCallback.operationComplete(v);
+                } else {
+                    ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, timeoutMillis, null, null);
+                    responseFuture.setCause(t);
+                    invokeCallback.operationComplete(responseFuture);
+                }
+            })
+            .thenAccept(responseFuture -> invokeCallback.operationSucceed(responseFuture.getResponseCommand()))
+            .exceptionally(t -> {
+                invokeCallback.operationFail(t);
+                return null;
+            });
+    }
+
     private void requestFail(final int opaque) {
         ResponseFuture responseFuture = responseTable.remove(opaque);
         if (responseFuture != null) {
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
index 64621dd6c..d784351a5 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
@@ -527,15 +527,13 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
         if (channel != null && channel.isActive()) {
             long left = timeoutMillis;
             try {
-                doBeforeRpcHooks(channelRemoteAddr, request);
                 long costTime = System.currentTimeMillis() - beginStartTime;
                 left -= costTime;
                 if (left <= 0) {
                     throw new RemotingTimeoutException("invokeSync call the addr[" + channelRemoteAddr + "] timeout");
                 }
                 RemotingCommand response = this.invokeSyncImpl(channel, request, left);
-                doAfterRpcHooks(channelRemoteAddr, request, response);
-                this.updateChannelLastResponseTime(addr);
+                updateChannelLastResponseTime(addr);
                 return response;
             } catch (RemotingSendRequestException e) {
                 LOGGER.warn("invokeSync: send request exception, so close the channel[{}]", channelRemoteAddr);
@@ -727,18 +725,11 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
         final Channel channel = this.getAndCreateChannel(addr);
         String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel);
         if (channel != null && channel.isActive()) {
-            try {
-                doBeforeRpcHooks(channelRemoteAddr, request);
-                long costTime = System.currentTimeMillis() - beginStartTime;
-                if (timeoutMillis < costTime) {
-                    throw new RemotingTooMuchRequestException("invokeAsync call the addr[" + channelRemoteAddr + "] timeout");
-                }
-                this.invokeAsyncImpl(channel, request, timeoutMillis - costTime, new InvokeCallbackWrapper(invokeCallback, addr));
-            } catch (RemotingSendRequestException e) {
-                LOGGER.warn("invokeAsync: send request exception, so close the channel[{}]", channelRemoteAddr);
-                this.closeChannel(addr, channel);
-                throw e;
+            long costTime = System.currentTimeMillis() - beginStartTime;
+            if (timeoutMillis < costTime) {
+                throw new RemotingTooMuchRequestException("invokeAsync call the addr[" + channelRemoteAddr + "] timeout");
             }
+            this.invokeAsyncImpl(channel, request, timeoutMillis - costTime, new InvokeCallbackWrapper(invokeCallback, addr));
         } else {
             this.closeChannel(addr, channel);
             throw new RemotingConnectException(addr);
@@ -931,11 +922,19 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
 
         @Override
         public void operationComplete(ResponseFuture responseFuture) {
-            if (responseFuture != null && responseFuture.isSendRequestOK() && responseFuture.getResponseCommand() != null) {
-                NettyRemotingClient.this.updateChannelLastResponseTime(addr);
-            }
             this.invokeCallback.operationComplete(responseFuture);
         }
+
+        @Override
+        public void operationSucceed(RemotingCommand response) {
+            updateChannelLastResponseTime(addr);
+            this.invokeCallback.operationSucceed(response);
+        }
+
+        @Override
+        public void operationFail(final Throwable throwable) {
+            this.invokeCallback.operationFail(throwable);
+        }
     }
 
     class NettyClientHandler extends SimpleChannelInboundHandler<RemotingCommand> {
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java
index 19f705d74..0882818fe 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java
@@ -22,6 +22,9 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.rocketmq.remoting.InvokeCallback;
 import org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
+import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 
 public class ResponseFuture {
@@ -59,6 +62,18 @@ public class ResponseFuture {
     public void executeInvokeCallback() {
         if (invokeCallback != null) {
             if (this.executeCallbackOnlyOnce.compareAndSet(false, true)) {
+                RemotingCommand response = getResponseCommand();
+                if (response != null) {
+                    invokeCallback.operationSucceed(response);
+                } else {
+                    if (!isSendRequestOK()) {
+                        invokeCallback.operationFail(new RemotingSendRequestException(channel.remoteAddress().toString(), getCause()));
+                    } else if (isTimeout()) {
+                        invokeCallback.operationFail(new RemotingTimeoutException(channel.remoteAddress().toString(), getTimeoutMillis(), getCause()));
+                    } else {
+                        invokeCallback.operationFail(new RemotingException(getRequestCommand().toString(), getCause()));
+                    }
+                }
                 invokeCallback.operationComplete(this);
             }
         }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java b/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java
index 133e0ed31..5328e8845 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java
@@ -160,31 +160,38 @@ public class RpcClientImpl implements RpcClient {
         InvokeCallback callback = new InvokeCallback() {
             @Override
             public void operationComplete(ResponseFuture responseFuture) {
-                RemotingCommand responseCommand = responseFuture.getResponseCommand();
-                if (responseCommand == null) {
-                    processFailedResponse(addr, requestCommand, responseFuture, rpcResponsePromise);
-                    return;
-                }
+
+            }
+
+            @Override
+            public void operationSucceed(RemotingCommand response) {
                 try {
-                    switch (responseCommand.getCode()) {
+                    switch (response.getCode()) {
                         case ResponseCode.SUCCESS:
                         case ResponseCode.PULL_NOT_FOUND:
                         case ResponseCode.PULL_RETRY_IMMEDIATELY:
                         case ResponseCode.PULL_OFFSET_MOVED:
                             PullMessageResponseHeader responseHeader =
-                                    (PullMessageResponseHeader) responseCommand.decodeCommandCustomHeader(PullMessageResponseHeader.class);
-                            rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody()));
+                                (PullMessageResponseHeader) response.decodeCommandCustomHeader(PullMessageResponseHeader.class);
+                            rpcResponsePromise.setSuccess(new RpcResponse(response.getCode(), responseHeader, response.getBody()));
                         default:
-                            RpcResponse rpcResponse = new RpcResponse(new RpcException(responseCommand.getCode(), "unexpected remote response code"));
+                            RpcResponse rpcResponse = new RpcResponse(new RpcException(response.getCode(), "unexpected remote response code"));
                             rpcResponsePromise.setSuccess(rpcResponse);
 
                     }
                 } catch (Exception e) {
-                    String errorMessage = "process failed. addr: " + addr + ", timeoutMillis: " + responseFuture.getTimeoutMillis() + ". Request: " + requestCommand;
-                    RpcResponse  rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, e));
+                    String errorMessage = "process failed. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + requestCommand;
+                    RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, e));
                     rpcResponsePromise.setSuccess(rpcResponse);
                 }
             }
+
+            @Override
+            public void operationFail(Throwable throwable) {
+                String errorMessage = "process failed. addr: " + addr + ". Request: " + requestCommand;
+                RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, throwable));
+                rpcResponsePromise.setSuccess(rpcResponse);
+            }
         };
 
         this.remotingClient.invokeAsync(addr, requestCommand, timeoutMillis, callback);
diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java
index 90072960b..d0da0eb2e 100644
--- a/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java
+++ b/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java
@@ -26,12 +26,12 @@ import org.apache.rocketmq.remoting.exception.RemotingConnectException;
 import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
 import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
 import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;
-import org.apache.rocketmq.remoting.netty.ResponseFuture;
-import org.apache.rocketmq.remoting.netty.NettyServerConfig;
 import org.apache.rocketmq.remoting.netty.NettyClientConfig;
-import org.apache.rocketmq.remoting.netty.NettyRemotingServer;
 import org.apache.rocketmq.remoting.netty.NettyRemotingClient;
+import org.apache.rocketmq.remoting.netty.NettyRemotingServer;
 import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
+import org.apache.rocketmq.remoting.netty.NettyServerConfig;
+import org.apache.rocketmq.remoting.netty.ResponseFuture;
 import org.apache.rocketmq.remoting.protocol.LanguageCode;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.junit.AfterClass;
@@ -40,7 +40,6 @@ import org.junit.Test;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
 
 public class RemotingServerTest {
     private static RemotingServer remotingServer;
@@ -122,10 +121,19 @@ public class RemotingServerTest {
         remotingClient.invokeAsync("localhost:" + remotingServer.localListenPort(), request, 1000 * 3, new InvokeCallback() {
             @Override
             public void operationComplete(ResponseFuture responseFuture) {
+
+            }
+
+            @Override
+            public void operationSucceed(RemotingCommand response) {
                 latch.countDown();
-                assertTrue(responseFuture != null);
-                assertThat(responseFuture.getResponseCommand().getLanguage()).isEqualTo(LanguageCode.JAVA);
-                assertThat(responseFuture.getResponseCommand().getExtFields()).hasSize(2);
+                assertThat(response.getLanguage()).isEqualTo(LanguageCode.JAVA);
+                assertThat(response.getExtFields()).hasSize(2);
+            }
+
+            @Override
+            public void operationFail(Throwable throwable) {
+
             }
         });
         latch.await();
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java
similarity index 57%
rename from client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java
rename to remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java
index 80188832e..8ddcdf35d 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java
+++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java
@@ -15,23 +15,14 @@
  * limitations under the License.
  */
 
-package org.apache.rocketmq.client.impl;
+package org.apache.rocketmq.remoting.netty;
 
-import org.apache.rocketmq.remoting.InvokeCallback;
-import org.apache.rocketmq.remoting.netty.ResponseFuture;
-
-public abstract class BaseInvokeCallback implements InvokeCallback {
-    private final MQClientAPIImpl mqClientAPI;
-
-    public BaseInvokeCallback(MQClientAPIImpl mqClientAPI) {
-        this.mqClientAPI = mqClientAPI;
-    }
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.local.LocalChannel;
 
+public class MockChannel extends LocalChannel {
     @Override
-    public void operationComplete(final ResponseFuture responseFuture) {
-        mqClientAPI.execRpcHooksAfterRequest(responseFuture);
-        onComplete(responseFuture);
+    public ChannelFuture writeAndFlush(Object msg) {
+        return new MockChannelPromise(MockChannel.this);
     }
-
-    public abstract void onComplete(final ResponseFuture responseFuture);
 }
diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java
new file mode 100644
index 000000000..9c3a35487
--- /dev/null
+++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.remoting.netty;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelPromise;
+import io.netty.util.concurrent.Future;
+import io.netty.util.concurrent.GenericFutureListener;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import org.jetbrains.annotations.NotNull;
+
+public class MockChannelPromise implements ChannelPromise {
+    protected Channel channel;
+
+    public MockChannelPromise(Channel channel) {
+        this.channel = channel;
+    }
+
+    @Override
+    public Channel channel() {
+        return channel;
+    }
+
+    @Override
+    public ChannelPromise setSuccess(Void result) {
+        return this;
+    }
+
+    @Override
+    public ChannelPromise setSuccess() {
+        return this;
+    }
+
+    @Override
+    public boolean trySuccess() {
+        return false;
+    }
+
+    @Override
+    public ChannelPromise setFailure(Throwable cause) {
+        return this;
+    }
+
+    @Override
+    public ChannelPromise addListener(GenericFutureListener<? extends Future<? super Void>> listener) {
+        return this;
+    }
+
+    @Override
+    public ChannelPromise addListeners(GenericFutureListener<? extends Future<? super Void>>... listeners) {
+        return this;
+    }
+
+    @Override
+    public ChannelPromise removeListener(GenericFutureListener<? extends Future<? super Void>> listener) {
+        return this;
+    }
+
+    @Override
+    public ChannelPromise removeListeners(GenericFutureListener<? extends Future<? super Void>>... listeners) {
+        return this;
+    }
+
+    @Override
+    public ChannelPromise sync() throws InterruptedException {
+        return this;
+    }
+
+    @Override
+    public ChannelPromise syncUninterruptibly() {
+        return this;
+    }
+
+    @Override
+    public ChannelPromise await() throws InterruptedException {
+        return this;
+    }
+
+    @Override
+    public ChannelPromise awaitUninterruptibly() {
+        return this;
+    }
+
+    @Override
+    public ChannelPromise unvoid() {
+        return this;
+    }
+
+    @Override
+    public boolean isVoid() {
+        return false;
+    }
+
+    @Override
+    public boolean trySuccess(Void result) {
+        return false;
+    }
+
+    @Override
+    public boolean tryFailure(Throwable cause) {
+        return false;
+    }
+
+    @Override
+    public boolean setUncancellable() {
+        return false;
+    }
+
+    @Override
+    public boolean isSuccess() {
+        return false;
+    }
+
+    @Override
+    public boolean isCancellable() {
+        return false;
+    }
+
+    @Override
+    public Throwable cause() {
+        return null;
+    }
+
+    @Override
+    public boolean await(long timeout, TimeUnit unit) throws InterruptedException {
+        return false;
+    }
+
+    @Override
+    public boolean await(long timeoutMillis) throws InterruptedException {
+        return false;
+    }
+
+    @Override
+    public boolean awaitUninterruptibly(long timeout, TimeUnit unit) {
+        return false;
+    }
+
+    @Override
+    public boolean awaitUninterruptibly(long timeoutMillis) {
+        return false;
+    }
+
+    @Override
+    public Void getNow() {
+        return null;
+    }
+
+    @Override
+    public boolean cancel(boolean mayInterruptIfRunning) {
+        return false;
+    }
+
+    @Override
+    public boolean isCancelled() {
+        return false;
+    }
+
+    @Override
+    public boolean isDone() {
+        return false;
+    }
+
+    @Override
+    public Void get() throws InterruptedException, ExecutionException {
+        return null;
+    }
+
+    @Override
+    public Void get(long timeout,
+        @NotNull java.util.concurrent.TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
+        return null;
+    }
+}
diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java
index 8381c132b..dbbea86ea 100644
--- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java
+++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java
@@ -39,9 +39,19 @@ public class NettyRemotingAbstractTest {
         final Semaphore semaphore = new Semaphore(0);
         ResponseFuture responseFuture = new ResponseFuture(null, 1, 3000, new InvokeCallback() {
             @Override
-            public void operationComplete(final ResponseFuture responseFuture) {
+            public void operationComplete(ResponseFuture responseFuture) {
+
+            }
+
+            @Override
+            public void operationSucceed(RemotingCommand response) {
                 assertThat(semaphore.availablePermits()).isEqualTo(0);
             }
+
+            @Override
+            public void operationFail(Throwable throwable) {
+
+            }
         }, new SemaphoreReleaseOnlyOnce(semaphore));
 
         remotingAbstract.responseTable.putIfAbsent(1, responseFuture);
@@ -75,9 +85,19 @@ public class NettyRemotingAbstractTest {
         final Semaphore semaphore = new Semaphore(0);
         ResponseFuture responseFuture = new ResponseFuture(null, 1, 3000, new InvokeCallback() {
             @Override
-            public void operationComplete(final ResponseFuture responseFuture) {
+            public void operationComplete(ResponseFuture responseFuture) {
+
+            }
+
+            @Override
+            public void operationSucceed(RemotingCommand response) {
                 assertThat(semaphore.availablePermits()).isEqualTo(0);
             }
+
+            @Override
+            public void operationFail(Throwable throwable) {
+
+            }
         }, new SemaphoreReleaseOnlyOnce(semaphore));
 
         remotingAbstract.responseTable.putIfAbsent(1, responseFuture);
@@ -98,7 +118,18 @@ public class NettyRemotingAbstractTest {
         // mock timeout
         ResponseFuture responseFuture = new ResponseFuture(null, dummyId, -1000, new InvokeCallback() {
             @Override
-            public void operationComplete(final ResponseFuture responseFuture) {
+            public void operationComplete(ResponseFuture responseFuture) {
+
+            }
+
+            @Override
+            public void operationSucceed(RemotingCommand response) {
+
+            }
+
+            @Override
+            public void operationFail(Throwable throwable) {
+
             }
         }, null);
         remotingAbstract.responseTable.putIfAbsent(dummyId, responseFuture);
@@ -111,7 +142,22 @@ public class NettyRemotingAbstractTest {
         final Semaphore semaphore = new Semaphore(0);
         RemotingCommand request = RemotingCommand.createRequestCommand(1, null);
         ResponseFuture responseFuture = new ResponseFuture(null, 1, request, 3000,
-            responseFuture1 -> assertThat(semaphore.availablePermits()).isEqualTo(0), new SemaphoreReleaseOnlyOnce(semaphore));
+            new InvokeCallback() {
+                @Override
+                public void operationComplete(ResponseFuture responseFuture) {
+
+                }
+
+                @Override
+                public void operationSucceed(RemotingCommand response) {
+                    assertThat(semaphore.availablePermits()).isEqualTo(0);
+                }
+
+                @Override
+                public void operationFail(Throwable throwable) {
+
+                }
+            }, new SemaphoreReleaseOnlyOnce(semaphore));
 
         remotingAbstract.responseTable.putIfAbsent(1, responseFuture);
         RemotingCommand response = RemotingCommand.createResponseCommand(0, "Foo");
diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java
index 8fabbb21d..e72e7bd53 100644
--- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java
+++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java
@@ -16,10 +16,17 @@
  */
 package org.apache.rocketmq.remoting.netty;
 
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.local.LocalChannel;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.Semaphore;
 import org.apache.rocketmq.remoting.InvokeCallback;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce;
 import org.apache.rocketmq.remoting.exception.RemotingConnectException;
 import org.apache.rocketmq.remoting.exception.RemotingException;
 import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
@@ -29,23 +36,33 @@ import org.apache.rocketmq.remoting.protocol.RequestCode;
 import org.apache.rocketmq.remoting.protocol.ResponseCode;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.mockito.Mock;
 import org.mockito.Spy;
 import org.mockito.junit.MockitoJUnitRunner;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.assertj.core.api.AssertionsForClassTypes.catchThrowable;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 
 @RunWith(MockitoJUnitRunner.class)
 public class NettyRemotingClientTest {
     @Spy
     private NettyRemotingClient remotingClient = new NettyRemotingClient(new NettyClientConfig());
+    @Mock
+    private RPCHook rpcHookMock;
 
     @Test
-    public void testSetCallbackExecutor() throws NoSuchFieldException, IllegalAccessException {        
+    public void testSetCallbackExecutor() {
         ExecutorService customized = Executors.newCachedThreadPool();
         remotingClient.setCallbackExecutor(customized);
         assertThat(remotingClient.getCallbackExecutor()).isEqualTo(customized);
@@ -61,7 +78,7 @@ public class NettyRemotingClientTest {
             InvokeCallback callback = invocation.getArgument(3);
             ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
             responseFuture.setResponseCommand(response);
-            callback.operationComplete(responseFuture);
+            callback.operationSucceed(responseFuture.getResponseCommand());
             return null;
         }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
 
@@ -78,9 +95,7 @@ public class NettyRemotingClientTest {
         response.setCode(ResponseCode.SUCCESS);
         doAnswer(invocation -> {
             InvokeCallback callback = invocation.getArgument(3);
-            ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
-            responseFuture.setSendRequestOK(false);
-            callback.operationComplete(responseFuture);
+            callback.operationFail(new RemotingSendRequestException(null));
             return null;
         }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
 
@@ -97,8 +112,7 @@ public class NettyRemotingClientTest {
         response.setCode(ResponseCode.SUCCESS);
         doAnswer(invocation -> {
             InvokeCallback callback = invocation.getArgument(3);
-            ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), -1L, null, null);
-            callback.operationComplete(responseFuture);
+            callback.operationFail(new RemotingTimeoutException(""));
             return null;
         }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
 
@@ -115,8 +129,7 @@ public class NettyRemotingClientTest {
         response.setCode(ResponseCode.SUCCESS);
         doAnswer(invocation -> {
             InvokeCallback callback = invocation.getArgument(3);
-            ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
-            callback.operationComplete(responseFuture);
+            callback.operationFail(new RemotingException(null));
             return null;
         }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
 
@@ -134,4 +147,158 @@ public class NettyRemotingClientTest {
             assertThat(e.getMessage()).contains(addr);
         }
     }
+
+    @Test
+    public void testInvoke0() throws ExecutionException, InterruptedException {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
+        RemotingCommand response = RemotingCommand.createResponseCommand(null);
+        response.setCode(ResponseCode.SUCCESS);
+        Channel channel = new MockChannel() {
+            @Override
+            public ChannelFuture writeAndFlush(Object msg) {
+                ResponseFuture responseFuture = remotingClient.responseTable.get(request.getOpaque());
+                responseFuture.setResponseCommand(response);
+                responseFuture.executeInvokeCallback();
+                return super.writeAndFlush(msg);
+            }
+        };
+        CompletableFuture<ResponseFuture> future = remotingClient.invoke0(channel, request, 1000L);
+        assertThat(future.get().getResponseCommand()).isEqualTo(response);
+    }
+
+    @Test
+    public void testInvoke0WithException() {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
+        RemotingCommand response = RemotingCommand.createResponseCommand(null);
+        response.setCode(ResponseCode.SUCCESS);
+        Channel channel = new MockChannel() {
+            @Override
+            public ChannelFuture writeAndFlush(Object msg) {
+                ResponseFuture responseFuture = remotingClient.responseTable.get(request.getOpaque());
+                responseFuture.executeInvokeCallback();
+                return super.writeAndFlush(msg);
+            }
+        };
+        CompletableFuture<ResponseFuture> future = remotingClient.invoke0(channel, request, 1000L);
+        assertThatThrownBy(future::get).getCause().isInstanceOf(RemotingException.class);
+    }
+
+    @Test
+    public void testInvokeSync() throws RemotingSendRequestException, RemotingTimeoutException, InterruptedException {
+        remotingClient.registerRPCHook(rpcHookMock);
+
+        Channel channel = new LocalChannel();
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
+        RemotingCommand response = RemotingCommand.createResponseCommand(null);
+        response.setCode(ResponseCode.SUCCESS);
+        ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() {
+            @Override
+            public void operationComplete(ResponseFuture responseFuture) {
+
+            }
+        }, new SemaphoreReleaseOnlyOnce(new Semaphore(1)));
+        responseFuture.setResponseCommand(response);
+        CompletableFuture<ResponseFuture> future = new CompletableFuture<>();
+        future.complete(responseFuture);
+
+        doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong());
+        RemotingCommand actual = remotingClient.invokeSyncImpl(channel, request, 1000);
+        assertThat(actual).isEqualTo(response);
+
+        verify(rpcHookMock).doBeforeRequest(anyString(), eq(request));
+        verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response));
+    }
+
+    @Test
+    public void testInvokeAsync() {
+        remotingClient.registerRPCHook(rpcHookMock);
+        Channel channel = new LocalChannel();
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
+        RemotingCommand response = RemotingCommand.createResponseCommand(null);
+        response.setCode(ResponseCode.SUCCESS);
+        ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() {
+            @Override
+            public void operationComplete(ResponseFuture responseFuture) {
+
+            }
+        }, new SemaphoreReleaseOnlyOnce(new Semaphore(1)));
+        responseFuture.setResponseCommand(response);
+        CompletableFuture<ResponseFuture> future = new CompletableFuture<>();
+        future.complete(responseFuture);
+
+        doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong());
+
+        InvokeCallback callback = mock(InvokeCallback.class);
+        remotingClient.invokeAsyncImpl(channel, request, 1000, callback);
+        verify(callback, times(1)).operationSucceed(eq(response));
+        verify(callback, times(1)).operationComplete(eq(responseFuture));
+        verify(callback, never()).operationFail(any());
+
+        verify(rpcHookMock).doBeforeRequest(anyString(), eq(request));
+        verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response));
+    }
+
+    @Test
+    public void testInvokeAsyncFail() {
+        remotingClient.registerRPCHook(rpcHookMock);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
+
+        Channel channel = new LocalChannel();
+        CompletableFuture<ResponseFuture> future = new CompletableFuture<>();
+        future.completeExceptionally(new RemotingException(null));
+
+        doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong());
+
+        InvokeCallback callback = mock(InvokeCallback.class);
+        remotingClient.invokeAsyncImpl(channel, request, 1000, callback);
+        verify(callback, never()).operationSucceed(any());
+        verify(callback, times(1)).operationComplete(any());
+        verify(callback, times(1)).operationFail(any());
+
+        verify(rpcHookMock).doBeforeRequest(anyString(), eq(request));
+        verify(rpcHookMock, never()).doAfterResponse(anyString(), eq(request), any());
+    }
+
+    @Test
+    public void testInvokeImpl() throws ExecutionException, InterruptedException {
+        remotingClient.registerRPCHook(rpcHookMock);
+        Channel channel = new LocalChannel();
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
+        RemotingCommand response = RemotingCommand.createResponseCommand(null);
+        response.setCode(ResponseCode.SUCCESS);
+        ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() {
+            @Override
+            public void operationComplete(ResponseFuture responseFuture) {
+
+            }
+        }, new SemaphoreReleaseOnlyOnce(new Semaphore(1)));
+        responseFuture.setResponseCommand(response);
+        CompletableFuture<ResponseFuture> future = new CompletableFuture<>();
+        future.complete(responseFuture);
+
+        doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong());
+
+        CompletableFuture<ResponseFuture> future0 = remotingClient.invokeImpl(channel, request, 1000);
+        assertThat(future0.get()).isEqualTo(responseFuture);
+
+        verify(rpcHookMock).doBeforeRequest(anyString(), eq(request));
+        verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response));
+    }
+
+    @Test
+    public void testInvokeImplFail() {
+        remotingClient.registerRPCHook(rpcHookMock);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
+
+        Channel channel = new LocalChannel();
+        CompletableFuture<ResponseFuture> future = new CompletableFuture<>();
+        future.completeExceptionally(new RemotingException(null));
+
+        doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong());
+
+        assertThatThrownBy(() -> remotingClient.invokeImpl(channel, request, 1000).get()).getCause().isInstanceOf(RemotingException.class);
+
+        verify(rpcHookMock).doBeforeRequest(anyString(), eq(request));
+        verify(rpcHookMock, never()).doAfterResponse(anyString(), eq(request), any());
+    }
 }
-- 
2.32.0.windows.2


From b9ffe0f9576f68b8a37cf3e2f68051658ae5a9a2 Mon Sep 17 00:00:00 2001
From: Zhouxiang Zhan <zhouxzhan@apache.org>
Date: Sun, 8 Oct 2023 16:33:44 +0800
Subject: [PATCH 2/7] [ISSUE #7296] Add ChannelEventListener for
 MQClientAPIImpl (#7324)

* Add ChannelEventListener for MQClientAPIImpl

* add heartbeat when channel connect

* remove log

* Add enableHeartbeatChannelEventListener for ClientConfig
---
 .../apache/rocketmq/client/ClientConfig.java  | 55 ++++++++++++++-----
 .../rocketmq/client/impl/MQClientAPIImpl.java |  9 ++-
 .../client/impl/factory/MQClientInstance.java | 35 +++++++++++-
 .../remoting/netty/NettyRemotingClient.java   |  2 +
 4 files changed, 85 insertions(+), 16 deletions(-)

diff --git a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java
index bb0fe3522..f9843cc02 100644
--- a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java
+++ b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java
@@ -94,6 +94,8 @@ public class ClientConfig {
     private boolean sendLatencyEnable = Boolean.parseBoolean(System.getProperty(SEND_LATENCY_ENABLE, "false"));
     private boolean startDetectorEnable = Boolean.parseBoolean(System.getProperty(START_DETECTOR_ENABLE, "false"));
 
+    private boolean enableHeartbeatChannelEventListener = true;
+
     public String buildMQClientId() {
         StringBuilder sb = new StringBuilder();
         sb.append(this.getClientIP());
@@ -201,6 +203,7 @@ public class ClientConfig {
         this.useHeartbeatV2 = cc.useHeartbeatV2;
         this.startDetectorEnable = cc.startDetectorEnable;
         this.sendLatencyEnable = cc.sendLatencyEnable;
+        this.enableHeartbeatChannelEventListener = cc.enableHeartbeatChannelEventListener;
         this.detectInterval = cc.detectInterval;
         this.detectTimeout = cc.detectTimeout;
     }
@@ -228,6 +231,7 @@ public class ClientConfig {
         cc.enableStreamRequestType = enableStreamRequestType;
         cc.useHeartbeatV2 = useHeartbeatV2;
         cc.startDetectorEnable = startDetectorEnable;
+        cc.enableHeartbeatChannelEventListener = enableHeartbeatChannelEventListener;
         cc.sendLatencyEnable = sendLatencyEnable;
         cc.detectInterval = detectInterval;
         cc.detectTimeout = detectTimeout;
@@ -418,6 +422,14 @@ public class ClientConfig {
         this.startDetectorEnable = startDetectorEnable;
     }
 
+    public boolean isEnableHeartbeatChannelEventListener() {
+        return enableHeartbeatChannelEventListener;
+    }
+
+    public void setEnableHeartbeatChannelEventListener(boolean enableHeartbeatChannelEventListener) {
+        this.enableHeartbeatChannelEventListener = enableHeartbeatChannelEventListener;
+    }
+
     public int getDetectTimeout() {
         return this.detectTimeout;
     }
@@ -444,19 +456,34 @@ public class ClientConfig {
 
     @Override
     public String toString() {
-        return "ClientConfig [namesrvAddr=" + namesrvAddr
-            + ", clientIP=" + clientIP + ", instanceName=" + instanceName
-            + ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads
-            + ", pollNameServerInterval=" + pollNameServerInterval
-            + ", heartbeatBrokerInterval=" + heartbeatBrokerInterval
-            + ", persistConsumerOffsetInterval=" + persistConsumerOffsetInterval
-            + ", pullTimeDelayMillsWhenException=" + pullTimeDelayMillsWhenException
-            + ", unitMode=" + unitMode + ", unitName=" + unitName
-            + ", vipChannelEnabled=" + vipChannelEnabled + ", useTLS=" + useTLS
-            + ", socksProxyConfig=" + socksProxyConfig + ", language=" + language.name()
-            + ", namespace=" + namespace + ", mqClientApiTimeout=" + mqClientApiTimeout
-            + ", decodeReadBody=" + decodeReadBody + ", decodeDecompressBody=" + decodeDecompressBody
-            + ", sendLatencyEnable=" + sendLatencyEnable + ", startDetectorEnable=" + startDetectorEnable
-            + ", enableStreamRequestType=" + enableStreamRequestType + ", useHeartbeatV2=" + useHeartbeatV2 + "]";
+        return "ClientConfig{" +
+            "namesrvAddr='" + namesrvAddr + '\'' +
+            ", clientIP='" + clientIP + '\'' +
+            ", instanceName='" + instanceName + '\'' +
+            ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads +
+            ", namespace='" + namespace + '\'' +
+            ", namespaceInitialized=" + namespaceInitialized +
+            ", accessChannel=" + accessChannel +
+            ", pollNameServerInterval=" + pollNameServerInterval +
+            ", heartbeatBrokerInterval=" + heartbeatBrokerInterval +
+            ", persistConsumerOffsetInterval=" + persistConsumerOffsetInterval +
+            ", pullTimeDelayMillsWhenException=" + pullTimeDelayMillsWhenException +
+            ", unitMode=" + unitMode +
+            ", unitName='" + unitName + '\'' +
+            ", decodeReadBody=" + decodeReadBody +
+            ", decodeDecompressBody=" + decodeDecompressBody +
+            ", vipChannelEnabled=" + vipChannelEnabled +
+            ", useHeartbeatV2=" + useHeartbeatV2 +
+            ", useTLS=" + useTLS +
+            ", socksProxyConfig='" + socksProxyConfig + '\'' +
+            ", mqClientApiTimeout=" + mqClientApiTimeout +
+            ", detectTimeout=" + detectTimeout +
+            ", detectInterval=" + detectInterval +
+            ", language=" + language +
+            ", enableStreamRequestType=" + enableStreamRequestType +
+            ", sendLatencyEnable=" + sendLatencyEnable +
+            ", startDetectorEnable=" + startDetectorEnable +
+            ", enableHeartbeatChannelEventListener=" + enableHeartbeatChannelEventListener +
+            '}';
     }
 }
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
index 2407e5737..e152be811 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
@@ -79,6 +79,7 @@ import org.apache.rocketmq.common.sysflag.PullSysFlag;
 import org.apache.rocketmq.common.topic.TopicValidator;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.remoting.ChannelEventListener;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.InvokeCallback;
 import org.apache.rocketmq.remoting.RPCHook;
@@ -246,10 +247,16 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
     public MQClientAPIImpl(final NettyClientConfig nettyClientConfig,
         final ClientRemotingProcessor clientRemotingProcessor,
         RPCHook rpcHook, final ClientConfig clientConfig) {
+        this(nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig, null);
+    }
+
+    public MQClientAPIImpl(final NettyClientConfig nettyClientConfig,
+        final ClientRemotingProcessor clientRemotingProcessor,
+        RPCHook rpcHook, final ClientConfig clientConfig, final ChannelEventListener channelEventListener) {
         this.clientConfig = clientConfig;
         topAddressing = new DefaultTopAddressing(MixAll.getWSAddr(), clientConfig.getUnitName());
         topAddressing.registerChangeCallBack(this);
-        this.remotingClient = new NettyRemotingClient(nettyClientConfig, null);
+        this.remotingClient = new NettyRemotingClient(nettyClientConfig, channelEventListener);
         this.clientRemotingProcessor = clientRemotingProcessor;
 
         // Inject stream rpc hook first to make reserve field signature
diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
index 9484b26f8..09534a176 100644
--- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
+++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
@@ -16,6 +16,7 @@
  */
 package org.apache.rocketmq.client.impl.factory;
 
+import io.netty.channel.Channel;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -65,6 +66,7 @@ import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageQueue;
 import org.apache.rocketmq.common.message.MessageQueueAssignment;
 import org.apache.rocketmq.common.topic.TopicValidator;
+import org.apache.rocketmq.remoting.ChannelEventListener;
 import org.apache.rocketmq.remoting.RPCHook;
 import org.apache.rocketmq.remoting.common.HeartbeatV2Result;
 import org.apache.rocketmq.remoting.exception.RemotingException;
@@ -151,7 +153,38 @@ public class MQClientInstance {
         this.nettyClientConfig.setUseTLS(clientConfig.isUseTLS());
         this.nettyClientConfig.setSocksProxyConfig(clientConfig.getSocksProxyConfig());
         ClientRemotingProcessor clientRemotingProcessor = new ClientRemotingProcessor(this);
-        this.mQClientAPIImpl = new MQClientAPIImpl(this.nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig);
+        ChannelEventListener channelEventListener;
+        if (clientConfig.isEnableHeartbeatChannelEventListener()) {
+            channelEventListener = new ChannelEventListener() {
+                private final ConcurrentMap<String, HashMap<Long, String>> brokerAddrTable = MQClientInstance.this.brokerAddrTable;
+                @Override
+                public void onChannelConnect(String remoteAddr, Channel channel) {
+                    for (Map.Entry<String, HashMap<Long, String>> addressEntry : brokerAddrTable.entrySet()) {
+                        for (String address : addressEntry.getValue().values()) {
+                            if (address.equals(remoteAddr)) {
+                                sendHeartbeatToAllBrokerWithLockV2(false);
+                                break;
+                            }
+                        }
+                    }
+                }
+
+                @Override
+                public void onChannelClose(String remoteAddr, Channel channel) {
+                }
+
+                @Override
+                public void onChannelException(String remoteAddr, Channel channel) {
+                }
+
+                @Override
+                public void onChannelIdle(String remoteAddr, Channel channel) {
+                }
+            };
+        } else {
+            channelEventListener = null;
+        }
+        this.mQClientAPIImpl = new MQClientAPIImpl(this.nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig, channelEventListener);
 
         if (this.clientConfig.getNamesrvAddr() != null) {
             this.mQClientAPIImpl.updateNameServerAddressList(this.clientConfig.getNamesrvAddr());
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
index d784351a5..8631d0447 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
@@ -229,6 +229,8 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
             handler.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
         }
 
+        nettyEventExecutor.start();
+
         TimerTask timerTaskScanResponseTable = new TimerTask() {
             @Override
             public void run(Timeout timeout) {
-- 
2.32.0.windows.2


From 3808387e1389278edbe4ef023d200ecb3015622b Mon Sep 17 00:00:00 2001
From: lk <xdkxlk@outlook.com>
Date: Mon, 9 Oct 2023 16:07:56 +0800
Subject: [PATCH 3/7] [ISSUE #7429] clean channel map when CLIENT_UNREGISTER in
 proxy

---
 .../service/sysmessage/HeartbeatSyncer.java   | 31 ++++++---
 .../sysmessage/HeartbeatSyncerTest.java       | 68 +++++++++++++++++++
 2 files changed, 88 insertions(+), 11 deletions(-)

diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
index f70c06b8f..fee3ea87d 100644
--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
+++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java
@@ -18,6 +18,7 @@
 package org.apache.rocketmq.proxy.service.sysmessage;
 
 import com.alibaba.fastjson.JSON;
+import io.netty.channel.Channel;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Map;
@@ -73,16 +74,8 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer {
         );
         this.consumerManager.appendConsumerIdsChangeListener(new ConsumerIdsChangeListener() {
             @Override
-            public void handle(ConsumerGroupEvent event, String s, Object... args) {
-                if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) {
-                    if (args == null || args.length < 1) {
-                        return;
-                    }
-                    if (args[0] instanceof ClientChannelInfo) {
-                        ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0];
-                        remoteChannelMap.remove(clientChannelInfo.getChannel().id().asLongText());
-                    }
-                }
+            public void handle(ConsumerGroupEvent event, String group, Object... args) {
+                processConsumerGroupEvent(event, group, args);
             }
 
             @Override
@@ -98,6 +91,18 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer {
         super.shutdown();
     }
 
+    protected void processConsumerGroupEvent(ConsumerGroupEvent event, String group, Object... args) {
+        if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) {
+            if (args == null || args.length < 1) {
+                return;
+            }
+            if (args[0] instanceof ClientChannelInfo) {
+                ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0];
+                remoteChannelMap.remove(buildKey(group, clientChannelInfo.getChannel()));
+            }
+        }
+    }
+
     public void onConsumerRegister(String consumerGroup, ClientChannelInfo clientChannelInfo,
         ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,
         Set<SubscriptionData> subList) {
@@ -189,7 +194,7 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer {
                 }
 
                 RemoteChannel decodedChannel = RemoteChannel.decode(data.getChannelData());
-                RemoteChannel channel = remoteChannelMap.computeIfAbsent(data.getGroup() + "@" + decodedChannel.id().asLongText(), key -> decodedChannel);
+                RemoteChannel channel = remoteChannelMap.computeIfAbsent(buildKey(data.getGroup(), decodedChannel), key -> decodedChannel);
                 channel.setExtendAttribute(decodedChannel.getChannelExtendAttribute());
                 ClientChannelInfo clientChannelInfo = new ClientChannelInfo(
                     channel,
@@ -228,4 +233,8 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer {
         // use local address, remoting port and grpc port to build unique local proxy Id
         return proxyConfig.getLocalServeAddr() + "%" + proxyConfig.getRemotingListenPort() + "%" + proxyConfig.getGrpcServerPort();
     }
+
+    private static String buildKey(String group, Channel channel) {
+        return group + "@" + channel.id().asLongText();
+    }
 }
diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
index 43fba3d03..9a2c5e343 100644
--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
+++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java
@@ -27,6 +27,7 @@ import com.google.common.collect.Sets;
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelId;
 import java.time.Duration;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -35,6 +36,7 @@ import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.rocketmq.broker.client.ClientChannelInfo;
+import org.apache.rocketmq.broker.client.ConsumerGroupEvent;
 import org.apache.rocketmq.broker.client.ConsumerManager;
 import org.apache.rocketmq.client.impl.mqclient.MQClientAPIExt;
 import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory;
@@ -320,6 +322,72 @@ public class HeartbeatSyncerTest extends InitConfigTest {
         }
     }
 
+    @Test
+    public void testProcessConsumerGroupEventForRemoting() {
+        String consumerGroup = "consumerGroup";
+        Channel channel = createMockChannel();
+        RemotingProxyOutClient remotingProxyOutClient = mock(RemotingProxyOutClient.class);
+        RemotingChannel remotingChannel = new RemotingChannel(remotingProxyOutClient, proxyRelayService, channel, clientId, Collections.emptySet());
+        ClientChannelInfo clientChannelInfo = new ClientChannelInfo(
+            remotingChannel,
+            clientId,
+            LanguageCode.JAVA,
+            4
+        );
+
+        testProcessConsumerGroupEvent(consumerGroup, clientChannelInfo);
+    }
+
+    @Test
+    public void testProcessConsumerGroupEventForGrpcV2() {
+        String consumerGroup = "consumerGroup";
+        GrpcClientSettingsManager grpcClientSettingsManager = mock(GrpcClientSettingsManager.class);
+        GrpcChannelManager grpcChannelManager = mock(GrpcChannelManager.class);
+        GrpcClientChannel grpcClientChannel = new GrpcClientChannel(
+            proxyRelayService, grpcClientSettingsManager, grpcChannelManager,
+            ProxyContext.create().setRemoteAddress(remoteAddress).setLocalAddress(localAddress),
+            clientId);
+        ClientChannelInfo clientChannelInfo = new ClientChannelInfo(
+            grpcClientChannel,
+            clientId,
+            LanguageCode.JAVA,
+            5
+        );
+
+        testProcessConsumerGroupEvent(consumerGroup, clientChannelInfo);
+    }
+
+    private void testProcessConsumerGroupEvent(String consumerGroup, ClientChannelInfo clientChannelInfo) {
+        HeartbeatSyncer heartbeatSyncer = new HeartbeatSyncer(topicRouteService, adminService, consumerManager, mqClientAPIFactory, null);
+        SendResult okSendResult = new SendResult();
+        okSendResult.setSendStatus(SendStatus.SEND_OK);
+
+        ArgumentCaptor<Message> messageArgumentCaptor = ArgumentCaptor.forClass(Message.class);
+        doReturn(CompletableFuture.completedFuture(okSendResult)).when(this.mqClientAPIExt)
+            .sendMessageAsync(anyString(), anyString(), messageArgumentCaptor.capture(), any(), anyLong());
+
+        heartbeatSyncer.onConsumerRegister(
+            consumerGroup,
+            clientChannelInfo,
+            ConsumeType.CONSUME_PASSIVELY,
+            MessageModel.CLUSTERING,
+            ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET,
+            Collections.emptySet()
+        );
+        await().atMost(Duration.ofSeconds(3)).until(() -> messageArgumentCaptor.getAllValues().size() == 1);
+
+        // change local serve addr, to simulate other proxy receive messages
+        heartbeatSyncer.localProxyId = RandomStringUtils.randomAlphabetic(10);
+        ArgumentCaptor<ClientChannelInfo> channelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class);
+        doReturn(true).when(consumerManager).registerConsumer(anyString(), channelInfoArgumentCaptor.capture(), any(), any(), any(), any(), anyBoolean());
+
+        heartbeatSyncer.consumeMessage(convertFromMessage(messageArgumentCaptor.getAllValues()), null);
+        assertEquals(1, heartbeatSyncer.remoteChannelMap.size());
+
+        heartbeatSyncer.processConsumerGroupEvent(ConsumerGroupEvent.CLIENT_UNREGISTER, consumerGroup, channelInfoArgumentCaptor.getValue());
+        assertTrue(heartbeatSyncer.remoteChannelMap.isEmpty());
+    }
+
     private MessageExt convertFromMessage(Message message) {
         MessageExt messageExt = new MessageExt();
         messageExt.setTopic(message.getTopic());
-- 
2.32.0.windows.2


From 0027a1486d4f2d6f7dce3010751167e883783945 Mon Sep 17 00:00:00 2001
From: redlsz <szliu0927@gmail.com>
Date: Mon, 9 Oct 2023 16:52:10 +0800
Subject: [PATCH 4/7] [ISSUE #7412] Fix pop revive message error when reput
 checkpoint

---
 .../org/apache/rocketmq/broker/processor/PopReviveService.java   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java
index 93167db37..d5174d3d1 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java
@@ -595,6 +595,7 @@ public class PopReviveService extends ServiceThread {
         newCk.setCId(oldCK.getCId());
         newCk.setTopic(oldCK.getTopic());
         newCk.setQueueId(oldCK.getQueueId());
+        newCk.setBrokerName(oldCK.getBrokerName());
         newCk.addDiff(0);
         MessageExtBrokerInner ckMsg = brokerController.getPopMessageProcessor().buildCkMsg(newCk, queueId);
         brokerController.getMessageStore().putMessage(ckMsg);
-- 
2.32.0.windows.2


From b18e564addbcff50165a5e1d9d4ab7db789d901b Mon Sep 17 00:00:00 2001
From: rongtong <jinrongtong5@163.com>
Date: Mon, 9 Oct 2023 21:43:01 +0800
Subject: [PATCH 5/7] [ISSUE #7431] Fix flaky test of
 DLedgerControllerTest#testBrokerLifecycleListener (#7432)

* Fix flaky test of DLedgerControllerTest#testBrokerLifecycleListener
---
 .../impl/DLedgerControllerTest.java           | 26 ++++++++++++-------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java b/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java
index 595a5cb65..d6e5449c5 100644
--- a/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java
+++ b/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java
@@ -63,7 +63,8 @@ public class DLedgerControllerTest {
     private List<String> baseDirs;
     private List<DLedgerController> controllers;
 
-    public DLedgerController launchController(final String group, final String peers, final String selfId, final boolean isEnableElectUncleanMaster) {
+    public DLedgerController launchController(final String group, final String peers, final String selfId,
+        final boolean isEnableElectUncleanMaster) {
         String tmpdir = System.getProperty("java.io.tmpdir");
         final String path = (StringUtils.endsWith(tmpdir, File.separator) ? tmpdir : tmpdir + File.separator) + group + File.separator + selfId;
         baseDirs.add(path);
@@ -121,11 +122,11 @@ public class DLedgerControllerTest {
         final RegisterBrokerToControllerRequestHeader registerBrokerToControllerRequestHeader = new RegisterBrokerToControllerRequestHeader(clusterName, brokerName, nextBrokerId, brokerAddress);
         RemotingCommand remotingCommand2 = leader.registerBroker(registerBrokerToControllerRequestHeader).get(2, TimeUnit.SECONDS);
 
-
         assertEquals(ResponseCode.SUCCESS, remotingCommand2.getCode());
     }
 
-    public void brokerTryElectMaster(Controller leader, String clusterName, String brokerName, String brokerAddress, Long brokerId,
+    public void brokerTryElectMaster(Controller leader, String clusterName, String brokerName, String brokerAddress,
+        Long brokerId,
         boolean exceptSuccess) throws Exception {
         final ElectMasterRequestHeader electMasterRequestHeader = ElectMasterRequestHeader.ofBrokerTrigger(clusterName, brokerName, brokerId);
         RemotingCommand command = leader.electMaster(electMasterRequestHeader).get(2, TimeUnit.SECONDS);
@@ -186,9 +187,9 @@ public class DLedgerControllerTest {
         registerNewBroker(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L);
         registerNewBroker(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L);
         // try elect
-        brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L,true);
-        brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L,  false);
-        brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L,false);
+        brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L, true);
+        brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L, false);
+        brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L, false);
         final RemotingCommand getInfoResponse = leader.getReplicaInfo(new GetReplicaInfoRequestHeader(DEFAULT_BROKER_NAME)).get(10, TimeUnit.SECONDS);
         final GetReplicaInfoResponseHeader replicaInfo = (GetReplicaInfoResponseHeader) getInfoResponse.readCustomHeader();
         assertEquals(1, replicaInfo.getMasterEpoch().intValue());
@@ -239,6 +240,8 @@ public class DLedgerControllerTest {
     @Test
     public void testBrokerLifecycleListener() throws Exception {
         final DLedgerController leader = mockMetaData(false);
+
+        assertTrue(leader.isLeaderState());
         // Mock that master broker has been inactive, and try to elect a new master from sync-state-set
         // But we shut down two controller, so the ElectMasterEvent will be appended to DLedger failed.
         // So the statemachine still keep the stale master's information
@@ -247,15 +250,20 @@ public class DLedgerControllerTest {
             dLedgerController.shutdown();
             controllers.remove(dLedgerController);
         }
+
         final ElectMasterRequestHeader request = ElectMasterRequestHeader.ofControllerTrigger(DEFAULT_BROKER_NAME);
         setBrokerElectPolicy(leader, 1L);
         Exception exception = null;
+        RemotingCommand remotingCommand = null;
         try {
-            leader.electMaster(request).get(5, TimeUnit.SECONDS);
+            remotingCommand = leader.electMaster(request).get(5, TimeUnit.SECONDS);
         } catch (Exception e) {
             exception = e;
         }
-        assertNotNull(exception);
+
+        assertTrue(exception != null ||
+            remotingCommand != null && remotingCommand.getCode() == ResponseCode.CONTROLLER_NOT_LEADER);
+
         // Shut down leader controller
         leader.shutdown();
         controllers.remove(leader);
@@ -272,7 +280,7 @@ public class DLedgerControllerTest {
         setBrokerAlivePredicate(newLeader, 1L);
         // Check if the statemachine is stale
         final RemotingCommand resp = newLeader.getReplicaInfo(new GetReplicaInfoRequestHeader(DEFAULT_BROKER_NAME)).
-                get(10, TimeUnit.SECONDS);
+            get(10, TimeUnit.SECONDS);
         final GetReplicaInfoResponseHeader replicaInfo = (GetReplicaInfoResponseHeader) resp.readCustomHeader();
         assertEquals(1, replicaInfo.getMasterBrokerId().longValue());
         assertEquals(1, replicaInfo.getMasterEpoch().intValue());
-- 
2.32.0.windows.2


From 38d3d5d95d371ac89f7d491a4c8719b4a22c60e1 Mon Sep 17 00:00:00 2001
From: mxsm <ljbmxsm@gmail.com>
Date: Tue, 10 Oct 2023 09:37:04 +0800
Subject: [PATCH 6/7] [ISSUE #7433]Update the version in the README.md document
 to 5.1.4 (#7434)

---
 README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 56d253ce1..5aaa2ba73 100644
--- a/README.md
+++ b/README.md
@@ -49,21 +49,21 @@ $ java -version
 java version "1.8.0_121"
 ```
 
-For Windows users, click [here](https://dist.apache.org/repos/dist/release/rocketmq/5.1.3/rocketmq-all-5.1.3-bin-release.zip) to download the 5.1.3 RocketMQ binary release,
+For Windows users, click [here](https://dist.apache.org/repos/dist/release/rocketmq/5.1.4/rocketmq-all-5.1.4-bin-release.zip) to download the 5.1.4 RocketMQ binary release,
 unpack it to your local disk, such as `D:\rocketmq`.
 For macOS and Linux users, execute following commands:
 
 ```shell
 # Download release from the Apache mirror
-$ wget https://dist.apache.org/repos/dist/release/rocketmq/5.1.3/rocketmq-all-5.1.3-bin-release.zip
+$ wget https://dist.apache.org/repos/dist/release/rocketmq/5.1.4/rocketmq-all-5.1.4-bin-release.zip
 
 # Unpack the release
-$ unzip rocketmq-all-5.1.3-bin-release.zip
+$ unzip rocketmq-all-5.1.4-bin-release.zip
 ```
 
 Prepare a terminal and change to the extracted `bin` directory:
 ```shell
-$ cd rocketmq-all-5.1.3-bin-release/bin
+$ cd rocketmq-all-5.1.4-bin-release/bin
 ```
 
 **1) Start NameServer**
-- 
2.32.0.windows.2


From 4acb43ecee03e429d036e3ff4c28bd402d1b30c7 Mon Sep 17 00:00:00 2001
From: Zhouxiang Zhan <zhouxzhan@apache.org>
Date: Tue, 10 Oct 2023 13:54:01 +0800
Subject: [PATCH 7/7] [ISSUE #7330] Add goaway and reconnection mechanism
 (#7331)

* Add shutdown wait for NettyRemotingServer

* Add goaway and reconnection mechanism

* Add client version check

* Add enableTransparentRetry for NettyClientConfig

* Add enableReconnectForGoAway for NettyClientConfig

* fix unit test

* fix client version check
---
 .../remoting/netty/NettyClientConfig.java     |  30 ++++
 .../remoting/netty/NettyRemotingAbstract.java |  15 ++
 .../remoting/netty/NettyRemotingClient.java   | 153 ++++++++++++++++--
 .../remoting/netty/NettyRemotingServer.java   |  31 ++--
 .../remoting/netty/NettyServerConfig.java     |  19 +++
 .../remoting/protocol/ResponseCode.java       |   2 +
 .../netty/NettyRemotingClientTest.java        |  39 ++---
 7 files changed, 239 insertions(+), 50 deletions(-)

diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java
index b2e7df754..c28288786 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java
@@ -53,6 +53,12 @@ public class NettyClientConfig {
     private boolean disableCallbackExecutor = false;
     private boolean disableNettyWorkerGroup = false;
 
+    private long maxReconnectIntervalTimeSeconds = 60;
+
+    private boolean enableReconnectForGoAway = true;
+
+    private boolean enableTransparentRetry = true;
+
     public boolean isClientCloseSocketIfTimeout() {
         return clientCloseSocketIfTimeout;
     }
@@ -181,6 +187,30 @@ public class NettyClientConfig {
         this.disableNettyWorkerGroup = disableNettyWorkerGroup;
     }
 
+    public long getMaxReconnectIntervalTimeSeconds() {
+        return maxReconnectIntervalTimeSeconds;
+    }
+
+    public void setMaxReconnectIntervalTimeSeconds(long maxReconnectIntervalTimeSeconds) {
+        this.maxReconnectIntervalTimeSeconds = maxReconnectIntervalTimeSeconds;
+    }
+
+    public boolean isEnableReconnectForGoAway() {
+        return enableReconnectForGoAway;
+    }
+
+    public void setEnableReconnectForGoAway(boolean enableReconnectForGoAway) {
+        this.enableReconnectForGoAway = enableReconnectForGoAway;
+    }
+
+    public boolean isEnableTransparentRetry() {
+        return enableTransparentRetry;
+    }
+
+    public void setEnableTransparentRetry(boolean enableTransparentRetry) {
+        this.enableTransparentRetry = enableTransparentRetry;
+    }
+
     public String getSocksProxyConfig() {
         return socksProxyConfig;
     }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
index 12e66f913..07ace28ea 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
@@ -40,9 +40,11 @@ import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
 import javax.annotation.Nullable;
 import org.apache.rocketmq.common.AbortProcessException;
+import org.apache.rocketmq.common.MQVersion;
 import org.apache.rocketmq.common.Pair;
 import org.apache.rocketmq.common.ServiceThread;
 import org.apache.rocketmq.common.UtilAll;
@@ -60,6 +62,7 @@ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;
 import org.apache.rocketmq.remoting.metrics.RemotingMetricsManager;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
 
 import static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_IS_LONG_POLLING;
 import static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_REQUEST_CODE;
@@ -120,6 +123,8 @@ public abstract class NettyRemotingAbstract {
      */
     protected List<RPCHook> rpcHooks = new ArrayList<>();
 
+    protected AtomicBoolean isShuttingDown = new AtomicBoolean(false);
+
     static {
         NettyLogger.initNettyLogger();
     }
@@ -264,6 +269,16 @@ public abstract class NettyRemotingAbstract {
 
         Runnable run = buildProcessRequestHandler(ctx, cmd, pair, opaque);
 
+        if (isShuttingDown.get()) {
+            if (cmd.getVersion() > MQVersion.Version.V5_1_4.ordinal()) {
+                final RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.GO_AWAY,
+                    "please go away");
+                response.setOpaque(opaque);
+                writeResponse(ctx.channel(), cmd, response);
+                return;
+            }
+        }
+
         if (pair.getObject1().rejectRequest()) {
             final RemotingCommand response = RemotingCommand.createResponseCommand(RemotingSysResponseCode.SYSTEM_BUSY,
                 "[REJECTREQUEST]system busy, start flow control for a while");
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
index 8631d0447..4bc51bd83 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
@@ -18,6 +18,7 @@ package org.apache.rocketmq.remoting.netty;
 
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.TypeReference;
+import com.google.common.base.Stopwatch;
 import io.netty.bootstrap.Bootstrap;
 import io.netty.buffer.PooledByteBufAllocator;
 import io.netty.channel.Channel;
@@ -48,6 +49,7 @@ import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.SocketAddress;
 import java.security.cert.CertificateException;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -57,6 +59,7 @@ import java.util.Map;
 import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
@@ -66,6 +69,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.common.Pair;
 import org.apache.rocketmq.common.ThreadFactoryImpl;
@@ -82,6 +86,7 @@ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
 import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
 import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
 import org.apache.rocketmq.remoting.proxy.SocksProxyConfig;
 
 public class NettyRemotingClient extends NettyRemotingAbstract implements RemotingClient {
@@ -97,6 +102,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
     private final Map<String /* cidr */, SocksProxyConfig /* proxy */> proxyMap = new HashMap<>();
     private final ConcurrentHashMap<String /* cidr */, Bootstrap> bootstrapMap = new ConcurrentHashMap<>();
     private final ConcurrentMap<String /* addr */, ChannelWrapper> channelTables = new ConcurrentHashMap<>();
+    private final ConcurrentMap<Channel, ChannelWrapper> channelWrapperTables = new ConcurrentHashMap<>();
 
     private final HashedWheelTimer timer = new HashedWheelTimer(r -> new Thread(r, "ClientHouseKeepingService"));
 
@@ -356,9 +362,10 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
             this.timer.stop();
 
             for (String addr : this.channelTables.keySet()) {
-                this.closeChannel(addr, this.channelTables.get(addr).getChannel());
+                this.channelTables.get(addr).close();
             }
 
+            this.channelWrapperTables.clear();
             this.channelTables.clear();
 
             this.eventLoopGroupWorker.shutdownGracefully();
@@ -416,7 +423,10 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
                     }
 
                     if (removeItemFromTable) {
-                        this.channelTables.remove(addrRemote);
+                        ChannelWrapper channelWrapper = this.channelWrapperTables.remove(channel);
+                        if (channelWrapper != null && channelWrapper.tryClose(channel)) {
+                            this.channelTables.remove(addrRemote);
+                        }
                         LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote);
                     }
 
@@ -463,7 +473,10 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
                     }
 
                     if (removeItemFromTable) {
-                        this.channelTables.remove(addrRemote);
+                        ChannelWrapper channelWrapper = this.channelWrapperTables.remove(channel);
+                        if (channelWrapper != null && channelWrapper.tryClose(channel)) {
+                            this.channelTables.remove(addrRemote);
+                        }
                         LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote);
                         RemotingHelper.closeChannel(channel);
                     }
@@ -511,7 +524,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
                         if (addr.contains(namesrvAddr)) {
                             ChannelWrapper channelWrapper = this.channelTables.get(addr);
                             if (channelWrapper != null) {
-                                closeChannel(channelWrapper.getChannel());
+                                channelWrapper.close();
                             }
                         }
                     }
@@ -689,8 +702,9 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
                     ChannelFuture channelFuture = fetchBootstrap(addr)
                         .connect(hostAndPort[0], Integer.parseInt(hostAndPort[1]));
                     LOGGER.info("createChannel: begin to connect remote host[{}] asynchronously", addr);
-                    cw = new ChannelWrapper(channelFuture);
+                    cw = new ChannelWrapper(addr, channelFuture);
                     this.channelTables.put(addr, cw);
+                    this.channelWrapperTables.put(channelFuture.channel(), cw);
                 }
             } catch (Exception e) {
                 LOGGER.error("createChannel: create channel exception", e);
@@ -758,6 +772,64 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
         }
     }
 
+    @Override
+    public CompletableFuture<RemotingCommand> invoke(String addr, RemotingCommand request,
+        long timeoutMillis) {
+        CompletableFuture<RemotingCommand> future = new CompletableFuture<>();
+        try {
+            final Channel channel = this.getAndCreateChannel(addr);
+            if (channel != null && channel.isActive()) {
+                return invokeImpl(channel, request, timeoutMillis).whenComplete((v, t) -> {
+                    if (t == null) {
+                        updateChannelLastResponseTime(addr);
+                    }
+                }).thenApply(ResponseFuture::getResponseCommand);
+            } else {
+                this.closeChannel(addr, channel);
+                future.completeExceptionally(new RemotingConnectException(addr));
+            }
+        } catch (Throwable t) {
+            future.completeExceptionally(t);
+        }
+        return future;
+    }
+
+    @Override
+    public CompletableFuture<ResponseFuture> invokeImpl(final Channel channel, final RemotingCommand request,
+        final long timeoutMillis) {
+        Stopwatch stopwatch = Stopwatch.createStarted();
+        return super.invokeImpl(channel, request, timeoutMillis).thenCompose(responseFuture -> {
+            RemotingCommand response = responseFuture.getResponseCommand();
+            if (response.getCode() == ResponseCode.GO_AWAY) {
+                if (nettyClientConfig.isEnableReconnectForGoAway()) {
+                    ChannelWrapper channelWrapper = channelWrapperTables.computeIfPresent(channel, (channel0, channelWrapper0) -> {
+                        try {
+                            if (channelWrapper0.reconnect()) {
+                                LOGGER.info("Receive go away from channel {}, recreate the channel", channel0);
+                                channelWrapperTables.put(channelWrapper0.getChannel(), channelWrapper0);
+                            }
+                        } catch (Throwable t) {
+                            LOGGER.error("Channel {} reconnect error", channelWrapper0, t);
+                        }
+                        return channelWrapper0;
+                    });
+                    if (channelWrapper != null) {
+                        if (nettyClientConfig.isEnableTransparentRetry()) {
+                            long duration = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+                            stopwatch.stop();
+                            RemotingCommand retryRequest = RemotingCommand.createRequestCommand(request.getCode(), request.readCustomHeader());
+                            Channel retryChannel = channelWrapper.getChannel();
+                            if (channel != retryChannel) {
+                                return super.invokeImpl(retryChannel, retryRequest, timeoutMillis - duration);
+                            }
+                        }
+                    }
+                }
+            }
+            return CompletableFuture.completedFuture(responseFuture);
+        });
+    }
+
     @Override
     public void registerProcessor(int requestCode, NettyRequestProcessor processor, ExecutorService executor) {
         ExecutorService executorThis = executor;
@@ -877,30 +949,41 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
         }
     }
 
-    static class ChannelWrapper {
-        private final ChannelFuture channelFuture;
+    class ChannelWrapper {
+        private final ReentrantReadWriteLock lock;
+        private ChannelFuture channelFuture;
         // only affected by sync or async request, oneway is not included.
+        private ChannelFuture channelToClose;
         private long lastResponseTime;
+        private volatile long lastReconnectTimestamp = 0L;
+        private final String channelAddress;
 
-        public ChannelWrapper(ChannelFuture channelFuture) {
+        public ChannelWrapper(String address, ChannelFuture channelFuture) {
+            this.lock = new ReentrantReadWriteLock();
             this.channelFuture = channelFuture;
             this.lastResponseTime = System.currentTimeMillis();
+            this.channelAddress = address;
         }
 
         public boolean isOK() {
-            return this.channelFuture.channel() != null && this.channelFuture.channel().isActive();
+            return getChannel() != null && getChannel().isActive();
         }
 
         public boolean isWritable() {
-            return this.channelFuture.channel().isWritable();
+            return getChannel().isWritable();
         }
 
         private Channel getChannel() {
-            return this.channelFuture.channel();
+            return getChannelFuture().channel();
         }
 
         public ChannelFuture getChannelFuture() {
-            return channelFuture;
+            lock.readLock().lock();
+            try {
+                return this.channelFuture;
+            } finally {
+                lock.readLock().unlock();
+            }
         }
 
         public long getLastResponseTime() {
@@ -910,6 +993,52 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti
         public void updateLastResponseTime() {
             this.lastResponseTime = System.currentTimeMillis();
         }
+
+        public boolean reconnect() {
+            if (lock.writeLock().tryLock()) {
+                try {
+                    if (lastReconnectTimestamp == 0L || System.currentTimeMillis() - lastReconnectTimestamp > Duration.ofSeconds(nettyClientConfig.getMaxReconnectIntervalTimeSeconds()).toMillis()) {
+                        channelToClose = channelFuture;
+                        String[] hostAndPort = getHostAndPort(channelAddress);
+                        channelFuture = fetchBootstrap(channelAddress)
+                            .connect(hostAndPort[0], Integer.parseInt(hostAndPort[1]));
+                        lastReconnectTimestamp = System.currentTimeMillis();
+                        return true;
+                    }
+                } finally {
+                    lock.writeLock().unlock();
+                }
+            }
+            return false;
+        }
+
+        public boolean tryClose(Channel channel) {
+            try {
+                lock.readLock().lock();
+                if (channelFuture != null) {
+                    if (channelFuture.channel().equals(channel)) {
+                        return true;
+                    }
+                }
+            } finally {
+                lock.readLock().unlock();
+            }
+            return false;
+        }
+
+        public void close() {
+            try {
+                lock.writeLock().lock();
+                if (channelFuture != null) {
+                    closeChannel(channelFuture.channel());
+                }
+                if (channelToClose != null) {
+                    closeChannel(channelToClose.channel());
+                }
+            } finally {
+                lock.writeLock().unlock();
+            }
+        }
     }
 
     class InvokeCallbackWrapper implements InvokeCallback {
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
index aa0d46542..735d36168 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java
@@ -53,6 +53,19 @@ import io.netty.util.HashedWheelTimer;
 import io.netty.util.Timeout;
 import io.netty.util.TimerTask;
 import io.netty.util.concurrent.DefaultEventExecutorGroup;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.security.cert.CertificateException;
+import java.time.Duration;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.common.Pair;
@@ -74,19 +87,6 @@ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
 import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.security.cert.CertificateException;
-import java.util.List;
-import java.util.NoSuchElementException;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
 @SuppressWarnings("NullableProblems")
 public class NettyRemotingServer extends NettyRemotingAbstract implements RemotingServer {
     private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME);
@@ -305,6 +305,10 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
     @Override
     public void shutdown() {
         try {
+            if (nettyServerConfig.isEnableShutdownGracefully() && isShuttingDown.compareAndSet(false, true)) {
+                Thread.sleep(Duration.ofSeconds(nettyServerConfig.getShutdownWaitTimeSeconds()).toMillis());
+            }
+
             this.timer.stop();
 
             this.eventLoopGroupBoss.shutdownGracefully();
@@ -736,6 +740,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti
 
         @Override
         public void shutdown() {
+            isShuttingDown.set(true);
             if (this.serverChannel != null) {
                 try {
                     this.serverChannel.close().await(5, TimeUnit.SECONDS);
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java
index 59ef2c84f..756661f62 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java
@@ -38,6 +38,9 @@ public class NettyServerConfig implements Cloneable {
     private int serverSocketBacklog = NettySystemConfig.socketBacklog;
     private boolean serverPooledByteBufAllocatorEnable = true;
 
+    private boolean enableShutdownGracefully = false;
+    private int shutdownWaitTimeSeconds = 30;
+
     /**
      * make install
      *
@@ -171,4 +174,20 @@ public class NettyServerConfig implements Cloneable {
     public void setWriteBufferHighWaterMark(int writeBufferHighWaterMark) {
         this.writeBufferHighWaterMark = writeBufferHighWaterMark;
     }
+
+    public boolean isEnableShutdownGracefully() {
+        return enableShutdownGracefully;
+    }
+
+    public void setEnableShutdownGracefully(boolean enableShutdownGracefully) {
+        this.enableShutdownGracefully = enableShutdownGracefully;
+    }
+
+    public int getShutdownWaitTimeSeconds() {
+        return shutdownWaitTimeSeconds;
+    }
+
+    public void setShutdownWaitTimeSeconds(int shutdownWaitTimeSeconds) {
+        this.shutdownWaitTimeSeconds = shutdownWaitTimeSeconds;
+    }
 }
diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java
index e81dadf2e..be945c48f 100644
--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java
+++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java
@@ -99,6 +99,8 @@ public class ResponseCode extends RemotingSysResponseCode {
     public static final int RPC_SEND_TO_CHANNEL_FAILED = -1004;
     public static final int RPC_TIME_OUT = -1006;
 
+    public static final int GO_AWAY = 1500;
+
     /**
      * Controller response code
      */
diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java
index e72e7bd53..1cc6b4f46 100644
--- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java
+++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java
@@ -47,7 +47,6 @@ import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
@@ -74,13 +73,11 @@ public class NettyRemotingClientTest {
 
         RemotingCommand response = RemotingCommand.createResponseCommand(null);
         response.setCode(ResponseCode.SUCCESS);
-        doAnswer(invocation -> {
-            InvokeCallback callback = invocation.getArgument(3);
-            ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
-            responseFuture.setResponseCommand(response);
-            callback.operationSucceed(responseFuture.getResponseCommand());
-            return null;
-        }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
+        ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null);
+        responseFuture.setResponseCommand(response);
+        CompletableFuture<RemotingCommand> future0 = new CompletableFuture<>();
+        future0.complete(responseFuture.getResponseCommand());
+        doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());
 
         CompletableFuture<RemotingCommand> future = remotingClient.invoke("0.0.0.0", request, 1000);
         RemotingCommand actual = future.get();
@@ -93,11 +90,9 @@ public class NettyRemotingClientTest {
 
         RemotingCommand response = RemotingCommand.createResponseCommand(null);
         response.setCode(ResponseCode.SUCCESS);
-        doAnswer(invocation -> {
-            InvokeCallback callback = invocation.getArgument(3);
-            callback.operationFail(new RemotingSendRequestException(null));
-            return null;
-        }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
+        CompletableFuture<RemotingCommand> future0 = new CompletableFuture<>();
+        future0.completeExceptionally(new RemotingSendRequestException(null));
+        doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());
 
         CompletableFuture<RemotingCommand> future = remotingClient.invoke("0.0.0.0", request, 1000);
         Throwable thrown = catchThrowable(future::get);
@@ -110,11 +105,9 @@ public class NettyRemotingClientTest {
 
         RemotingCommand response = RemotingCommand.createResponseCommand(null);
         response.setCode(ResponseCode.SUCCESS);
-        doAnswer(invocation -> {
-            InvokeCallback callback = invocation.getArgument(3);
-            callback.operationFail(new RemotingTimeoutException(""));
-            return null;
-        }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
+        CompletableFuture<RemotingCommand> future0 = new CompletableFuture<>();
+        future0.completeExceptionally(new RemotingTimeoutException(""));
+        doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());
 
         CompletableFuture<RemotingCommand> future = remotingClient.invoke("0.0.0.0", request, 1000);
         Throwable thrown = catchThrowable(future::get);
@@ -125,13 +118,9 @@ public class NettyRemotingClientTest {
     public void testRemotingException() throws Exception {
         RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null);
 
-        RemotingCommand response = RemotingCommand.createResponseCommand(null);
-        response.setCode(ResponseCode.SUCCESS);
-        doAnswer(invocation -> {
-            InvokeCallback callback = invocation.getArgument(3);
-            callback.operationFail(new RemotingException(null));
-            return null;
-        }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class));
+        CompletableFuture<RemotingCommand> future0 = new CompletableFuture<>();
+        future0.completeExceptionally(new RemotingException(""));
+        doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong());
 
         CompletableFuture<RemotingCommand> future = remotingClient.invoke("0.0.0.0", request, 1000);
         Throwable thrown = catchThrowable(future::get);
-- 
2.32.0.windows.2

