RocketMQ的消息发送方式主要含syncSend()同步发送、asyncSend()异步发送、sendOneWay()三种方式，sendOneWay()也是异步发送，区别在于不需等待Broker返回确认，
所以可能会存在信息丢失的状况，但吞吐量更高，具体需根据业务情况选用。
一个队列只会被消费组内的一个消费者消费，即如果topic相同，但是有多个consumerGroup，可能有A、B两个,那么此时一个队列会被A、B共同消费。只不过只会被A和B中各自的一个服务器消费。
详情见 https://codetd.com/article/12138580#3_54 中的第3点。
当实例化RocketMQTemplate过程中因为实现了InitializingBean接口，会执行afterPropertiesSet()方法
this.producer.start();之后执行
public void start() throws MQClientException {
this.defaultMQProducerImpl.start();
。。。。。。。。。。。。
}之后执行
public void start(boolean startFactory) throws MQClientException {
.....................
this.mQClientFactory.start();
.....................
}之后执行
MQClientInstance
public void start() throws MQClientException {
//各种线程的start
//NettyRemotingClient实现Netty客户器端功能，接受数据包，在客户器端处理后发送给服务端。
this.mQClientAPIImpl.start();
//获取nameserver的地址 默认每3秒更新Topic规则 清除下线的borken并且发送心跳到broken 持久化消费者进度 适配执行消费请求的线程池的核心线程书大小
this.startScheduledTask();
//启动消息拉取服务，循环拉取阻塞队列pullRequestQueue
this.pullMessageService.start();
//负载均衡服务开始
this.rebalanceService.start();
this.defaultMQProducer.getDefaultMQProducerImpl().start(false);
this.log.info("the client factory [{}] start OK", this.clientId);
this.serviceState = ServiceState.RUNNING;
}
this.pullMessageService.start()解析1 public void run() {
2 this.log.info(this.getServiceName() + " service started");
3
4 while(!this.isStopped()) {
5 try {
//PullRequest是一个阻塞队列
6 PullRequest pullRequest = (PullRequest)this.pullRequestQueue.take();
7 this.pullMessage(pullRequest);
8 } catch (InterruptedException var2) {
9 } catch (Exception var3) {
10 this.log.error("Pull Message Service Run Method exception", var3);
11 }
12 }
13
14 this.log.info(this.getServiceName() + " service end");
15 }之后会执行拉取消息
1 public void pullMessage(final PullRequest pullRequest) {
//校验队列是否被丢弃
2 final ProcessQueue processQueue = pullRequest.getProcessQueue();
3 if (processQueue.isDropped()) {
4 log.info("the pull request[{}] is dropped.", pullRequest.toString());
5 return;
6 }
7
8 pullRequest.getProcessQueue().setLastPullTimestamp(System.currentTimeMillis());
9 //校验消费端的情况，状态是否正常，师傅暂停
10 try {
11 this.makeSureStateOK();
12 } catch (MQClientException e) {
13 log.warn("pullMessage exception, consumer state not ok", e);
14 this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException);
15 return;
16 }
17
18 if (this.isPause()) {
19 log.warn("consumer was paused, execute pull request later. instanceName={}, group={}", this.defaultMQPushConsumer.getInstanceName(), this.defaultMQPushConsumer.getConsumerGroup());
20 this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_SUSPEND);
21 return;
22 }
23 //获取该队列中的消息数量
24 long cachedMessageCount = processQueue.getMsgCount().get();
//获取该队列中的消息大小（M）
25 long cachedMessageSizeInMiB = processQueue.getMsgSize().get() / (1024 * 1024);
26 //如果该队列中的消息数量>每个队列中最大的消息数量(默认1000) 这里做这些判断主要是进行流控，即rocketmq的削峰原理就在这里
27 if (cachedMessageCount > this.defaultMQPushConsumer.getPullThresholdForQueue()) {
//等待50ms再次去拉取，从这里可以看出实际上拉取消息是实时的，没有时间间隔的，
28 this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL);
//该队列的流控次数每达到1000次则打印日志
29 if ((queueFlowControlTimes++ % 1000) == 0) {
30 log.warn(
31 "the cached message count exceeds the threshold {}, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, pullRequest={}, flowControlTimes={}",
32 this.defaultMQPushConsumer.getPullThresholdForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, pullRequest, queueFlowControlTimes);
33 }
34 return;
35 }
36 //如果该队列中的消息大小（M）超过100M(默认)，则执行流控，实现同上
37 if (cachedMessageSizeInMiB > this.defaultMQPushConsumer.getPullThresholdSizeForQueue()) {
38 this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL);
39 if ((queueFlowControlTimes++ % 1000) == 0) {
40 log.warn(
41 "the cached message size exceeds the threshold {} MiB, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, pullRequest={}, flowControlTimes={}",
42 this.defaultMQPushConsumer.getPullThresholdSizeForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, pullRequest, queueFlowControlTimes);
43 }
44 return;
45 }
46 //如果不是顺序消费,即并发消费
47 if (!this.consumeOrderly) {
//获取单队列并行消费允许的最大跨度 关键代码：这里用了读写锁保证并发 this.msgTreeMap.lastKey() - this.msgTreeMap.firstKey();
/* RocketMQ是以consumer group+queue为单位是管理消费进度的，以一个consumer offset标记这个这个消费组在这条queue上的消费进度。
如果某已存在的消费组出现了新消费实例的时候，依靠这个组的消费进度，就可以判断第一次是从哪里开始拉取的。
每次消息成功后，本地的消费进度会被更新，然后由定时器定时同步到broker，以此持久化消费进度。
mq消费端做幂等原因：
由于消费进度只是记录了一个下标，就可能出现拉取了100条消息如 2101-2200的消息，后面99条都消费结束了，只有2101消费一直没有结束的情况。
在这种情况下，RocketMQ为了保证消息肯定被消费成功，消费进度职能维持在2101，直到2101也消费结束了，本地的消费进度才会一下子更新到2200。
在这种设计下，就有消费大量重复的风险。如2101在还没有消费完成的时候消费实例突然退出（机器断电，或者被kill）。
这条queue的消费进度还是维持在2101，当queue重新分配给新的实例的时候，新的实例从broker上拿到的消费进度还是维持在2101，
这时候就会又从2101开始消费，2102-2200这批消息实际上已经被消费过还是会投递一次。
*/
//单队列并行消费允许的最大跨度>2000(默认) 比如有1000条消息，offset=2这条消息一直没有消费完，但是其他都消费完了，但是会实时拉取消息，当多次拉取条消息
// 直到offset=2003了，此时offset=2的这条还没有消费完成，mq就会认为有阻塞，会实行流控
48 if (processQueue.getMaxSpan() > this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan()) {
49 this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL);
50 if ((queueMaxSpanFlowControlTimes++ % 1000) == 0) {
51 log.warn(
52 "the queue's messages, span too long, so do flow control, minOffset={}, maxOffset={}, maxSpan={}, pullRequest={}, flowControlTimes={}",
//本次拉取的总消息数量中最小的offset 和最大的offset
53 processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), processQueue.getMaxSpan(),
54 pullRequest, queueMaxSpanFlowControlTimes);
55 }
56 return;
57 }
58 } else {
//顺序消费
59 if (processQueue.isLocked()) {
60 if (!pullRequest.isLockedFirst()) {
61 final long offset = this.rebalanceImpl.computePullFromWhere(pullRequest.getMessageQueue());
62 boolean brokerBusy = offset < pullRequest.getNextOffset();
63 log.info("the first time to pull message, so fix offset from broker. pullRequest: {} NewOffset: {} brokerBusy: {}",
64 pullRequest, offset, brokerBusy);
65 if (brokerBusy) {
66 log.info("[NOTIFYME]the first time to pull message, but pull request offset larger than broker consume offset. pullRequest: {} NewOffset: {}",
67 pullRequest, offset);
68 }
69
70 pullRequest.setLockedFirst(true);
71 pullRequest.setNextOffset(offset);
72 }
73 } else {
74 this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException);
75 log.info("pull message later because not locked in broker, {}", pullRequest);
76 return;
77 }
78 }
79 //获取该队列的topic、consumerGroup、tag信息
80 final SubscriptionData subscriptionData = this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic());
81 if (null == subscriptionData) {
82 this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException);
83 log.warn("find the consumer's subscription failed, {}", pullRequest);
84 return;
85 }
86
87 final long beginTimestamp = System.currentTimeMillis();
88 //回调函数， 在这个方法被调用，里面的逻辑执行到onSuccess()时才会真正执行到这里 this.pullAPIWrapper.pullKernelImpl(。。。，pullCallback)
//真正调用的地方：private void pullMessageAsync(....) -> pullCallback.onSuccess(pullResult); pullCallback.onException(e);
//下面会对这个回调函数进行具体解析
89 PullCallback pullCallback = new PullCallback() {
90 。。。。。。。。。。。。。。。。。。。。。191 };
192
193 boolean commitOffsetEnable = false;
194 long commitOffsetValue = 0L;
//如果是集群模式，此处读取方式是ReadOffsetType.READ_FROM_MEMORY 表示从内存中获取消费进度 也有从broken中获取进度
195 if (MessageModel.CLUSTERING == this.defaultMQPushConsumer.getMessageModel()) {
196 commitOffsetValue = this.offsetStore.readOffset(pullRequest.getMessageQueue(), ReadOffsetType.READ_FROM_MEMORY);
197 if (commitOffsetValue > 0) {
198 commitOffsetEnable = true;
199 }
200 }
201
202 String subExpression = null;
203 boolean classFilter = false;
//再次获取该队列的topic、consumerGroup、tag信息，之前已经获取了，不明白这里为啥要再次获取一次？？
204 SubscriptionData sd = this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic());
205 if (sd != null) {
206 if (this.defaultMQPushConsumer.isPostSubscriptionWhenPull() && !sd.isClassFilterMode()) {
//就是tag的值
207 subExpression = sd.getSubString();
208 }
209
210 classFilter = sd.isClassFilterMode();
211 }
212
213 int sysFlag = PullSysFlag.buildSysFlag(
214 commitOffsetEnable, // commitOffset
215 true, // suspend
216 subExpression != null, // subscription
217 classFilter // class filter
218 );
//拉取服务的真正实现入口：kernel：内核
219 try {
220 this.pullAPIWrapper.pullKernelImpl(
221 pullRequest.getMessageQueue(),
222 subExpression,
223 subscriptionData.getExpressionType(),
224 subscriptionData.getSubVersion(),
225 pullRequest.getNextOffset(),
226 this.defaultMQPushConsumer.getPullBatchSize(),
227 sysFlag,
228 commitOffsetValue,
229 BROKER_SUSPEND_MAX_TIME_MILLIS,
230 CONSUMER_TIMEOUT_MILLIS_WHEN_SUSPEND,
231 CommunicationMode.ASYNC,
232 pullCallback
233 );
234 } catch (Exception e) {
235 log.error("pullKernelImpl exception", e);
236 this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException);
237 }
238 }
之后拉取消息的核心实现：
1 public PullResult pullKernelImpl(
2 final MessageQueue mq,
3 final String subExpression,
4 final String expressionType,
5 final long subVersion,
6 final long offset,
7 final int maxNums,
8 final int sysFlag,
9 final long commitOffset,
10 final long brokerSuspendMaxTimeMillis,
11 final long timeoutMillis,
12 final CommunicationMode communicationMode,
13 final PullCallback pullCallback
14 ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
//获取broker的地址
15 FindBrokerResult findBrokerResult =
16 this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(),
17 this.recalculatePullFromWhichNode(mq), false);
18 if (null == findBrokerResult) {
19 this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());
20 findBrokerResult =
21 this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(),
22 this.recalculatePullFromWhichNode(mq), false);
23 }
24
25 if (findBrokerResult != null) {
26 {
27 // check version
28 if (!ExpressionType.isTagType(expressionType)
29 && findBrokerResult.getBrokerVersion() < MQVersion.Version.V4_1_0_SNAPSHOT.ordinal()) {
30 throw new MQClientException("The broker[" + mq.getBrokerName() + ", "
31 + findBrokerResult.getBrokerVersion() + "] does not upgrade to support for filter message by " + expressionType, null);
32 }
33 }
34 int sysFlagInner = sysFlag;
35 //如果是备用机器？？
36 if (findBrokerResult.isSlave()) {
37 sysFlagInner = PullSysFlag.clearCommitOffsetFlag(sysFlagInner);
38 }
39 //通过netty拉取信息，这里组装请求信息
40 PullMessageRequestHeader requestHeader = new PullMessageRequestHeader();
41 requestHeader.setConsumerGroup(this.consumerGroup);
42 requestHeader.setTopic(mq.getTopic());
43 requestHeader.setQueueId(mq.getQueueId());
44 requestHeader.setQueueOffset(offset);
45 requestHeader.setMaxMsgNums(maxNums);//一次拉取的最大数据条数
46 requestHeader.setSysFlag(sysFlagInner);
47 requestHeader.setCommitOffset(commitOffset);
48 requestHeader.setSuspendTimeoutMillis(brokerSuspendMaxTimeMillis);
49 requestHeader.setSubscription(subExpression);
50 requestHeader.setSubVersion(subVersion);
51 requestHeader.setExpressionType(expressionType);
52
53 String brokerAddr = findBrokerResult.getBrokerAddr();
54 if (PullSysFlag.hasClassFilterFlag(sysFlagInner)) {
55 brokerAddr = computPullFromWhichFilterServer(mq.getTopic(), brokerAddr);
56 }
57
58 PullResult pullResult = this.mQClientFactory.getMQClientAPIImpl().pullMessage(
59 brokerAddr,
60 requestHeader,
61 timeoutMillis,
62 communicationMode,
63 pullCallback);
64
65 return pullResult;
66 }
67
68 throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
69 }异步拉取：
1 private void pullMessageAsync(
2 final String addr,
3 final RemotingCommand request,
4 final long timeoutMillis,
5 final PullCallback pullCallback
6 ) throws RemotingException, InterruptedException {
//异步通过netty进行异步调用
7 this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() {
//拉取消息回调后会执行operationComplete()
8 @Override
9 public void operationComplete(ResponseFuture responseFuture) {
10 RemotingCommand response = responseFuture.getResponseCommand();
11 if (response != null) {
12 try {
//获取拉取的原始结果，这个pullResult里面就有拉取的数据
13 PullResult pullResult = MQClientAPIImpl.this.processPullResponse(response);
14 assert pullResult != null;
//这里才会真正执行到回调函数，具体解析详见下面
15 pullCallback.onSuccess(pullResult);
16 } catch (Exception e) {
17 pullCallback.onException(e);
18 }
19 } else {
20 if (!responseFuture.isSendRequestOK()) {
21 pullCallback.onException(new MQClientException("send request failed to " + addr + ". Request: " + request, responseFuture.getCause()));
22 } else if (responseFuture.isTimeout()) {
23 pullCallback.onException(new MQClientException("wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request,
24 responseFuture.getCause()));
25 } else {
26 pullCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause()));
27 }
28 }
29 }
30 });
31 }
拉取完成后执行回调函数：
1 PullCallback pullCallback = new PullCallback() {
2 @Override
3 public void onSuccess(PullResult pullResult) {
4 if (pullResult != null) {
//详细解析见下面
5 pullResult = DefaultMQPushConsumerImpl.this.pullAPIWrapper.processPullResult(pullRequest.getMessageQueue(), pullResult,
6 subscriptionData);
7
8 switch (pullResult.getPullStatus()) {
9 case FOUND:
10 long prevRequestOffset = pullRequest.getNextOffset();
11 pullRequest.setNextOffset(pullResult.getNextBeginOffset());
12 long pullRT = System.currentTimeMillis() - beginTimestamp;
13 DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullRT(pullRequest.getConsumerGroup(),
14 pullRequest.getMessageQueue().getTopic(), pullRT);
15
16 long firstMsgOffset = Long.MAX_VALUE;
17 if (pullResult.getMsgFoundList() == null || pullResult.getMsgFoundList().isEmpty()) {
18 DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
19 } else {
20 firstMsgOffset = pullResult.getMsgFoundList().get(0).getQueueOffset();
21
22 DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullTPS(pullRequest.getConsumerGroup(),
23 pullRequest.getMessageQueue().getTopic(), pullResult.getMsgFoundList().size());
24 //判断是否能发送给消费者，并发没有用到这个参数
25 boolean dispatchToConsume = processQueue.putMessage(pullResult.getMsgFoundList());
26 DefaultMQPushConsumerImpl.this.consumeMessageService.submitConsumeRequest(
27 pullResult.getMsgFoundList(),
28 processQueue,
29 pullRequest.getMessageQueue(),
30 dispatchToConsume);
31
32 if (DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval() > 0) {
33 DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest,
34 DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval());
35 } else {
36 DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
37 }
38 }
39
40 if (pullResult.getNextBeginOffset() < prevRequestOffset
41 || firstMsgOffset < prevRequestOffset) {
42 log.warn(
43 "[BUG] pull message result maybe data wrong, nextBeginOffset: {} firstMsgOffset: {} prevRequestOffset: {}",
44 pullResult.getNextBeginOffset(),
45 firstMsgOffset,
46 prevRequestOffset);
47 }
48
49 break;
50 case NO_NEW_MSG:
51 pullRequest.setNextOffset(pullResult.getNextBeginOffset());
52
53 DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest);
54
55 DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
56 break;
57 case NO_MATCHED_MSG:
58 pullRequest.setNextOffset(pullResult.getNextBeginOffset());
59
60 DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest);
61
62 DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
63 break;
64 case OFFSET_ILLEGAL:
65 log.warn("the pull request offset illegal, {} {}",
66 pullRequest.toString(), pullResult.toString());
67 pullRequest.setNextOffset(pullResult.getNextBeginOffset());
68
69 pullRequest.getProcessQueue().setDropped(true);
70 DefaultMQPushConsumerImpl.this.executeTaskLater(new Runnable() {
71
72 @Override
73 public void run() {
74 try {
75 DefaultMQPushConsumerImpl.this.offsetStore.updateOffset(pullRequest.getMessageQueue(),
76 pullRequest.getNextOffset(), false);
77
78 DefaultMQPushConsumerImpl.this.offsetStore.persist(pullRequest.getMessageQueue());
79
80 DefaultMQPushConsumerImpl.this.rebalanceImpl.removeProcessQueue(pullRequest.getMessageQueue());
81
82 log.warn("fix the pull request offset, {}", pullRequest);
83 } catch (Throwable e) {
84 log.error("executeTaskLater Exception", e);
85 }
86 }
87 }, 10000);
88 break;
89 default:
90 break;
91 }
92 }
93 }
94
95 @Override
96 public void onException(Throwable e) {
97 if (!pullRequest.getMessageQueue().getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
98 log.warn("execute the pull request exception", e);
99 }
100
101 DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException);
102 }
103 };
解析：public PullResult processPullResult(。。。。)主要是讲原始消息解析成 MessageExt1 public PullResult processPullResult(final MessageQueue mq, final PullResult pullResult,
2 final SubscriptionData subscriptionData) {
3 PullResultExt pullResultExt = (PullResultExt) pullResult;
4 //更新mq(topic、队列id、broker)和brokerId的对应关系
5 this.updatePullFromWhichNode(mq, pullResultExt.getSuggestWhichBrokerId());
//如果拉取到了数据
6 if (PullStatus.FOUND == pullResult.getPullStatus()) {
//解析拉取的数据
7 ByteBuffer byteBuffer = ByteBuffer.wrap(pullResultExt.getMessageBinary());
8 List<MessageExt> msgList = MessageDecoder.decodes(byteBuffer);
9
10 List<MessageExt> msgListFilterAgain = msgList;
11 if (!subscriptionData.getTagsSet().isEmpty() && !subscriptionData.isClassFilterMode()) {
12 msgListFilterAgain = new ArrayList<MessageExt>(msgList.size());
13 for (MessageExt msg : msgList) {
14 if (msg.getTags() != null) {
//再次过滤 tagsSet是什么？？
15 if (subscriptionData.getTagsSet().contains(msg.getTags())) {
16 msgListFilterAgain.add(msg);
17 }
18 }
19 }
20 }
21 //钩子，目前没有实现，可扩展
22 if (this.hasHook()) {
23 FilterMessageContext filterMessageContext = new FilterMessageContext();
24 filterMessageContext.setUnitMode(unitMode);
25 filterMessageContext.setMsgList(msgListFilterAgain);
26 this.executeHook(filterMessageContext);
27 }
28 //MessageExt中属性塞入一些值
29 for (MessageExt msg : msgListFilterAgain) {
30 String traFlag = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED);
31 if (Boolean.parseBoolean(traFlag)) {
32 msg.setTransactionId(msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));
33 }
34 MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MIN_OFFSET,
35 Long.toString(pullResult.getMinOffset()));
36 MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MAX_OFFSET,
37 Long.toString(pullResult.getMaxOffset()));
38 msg.setBrokerName(mq.getBrokerName());
39 }
40
41 pullResultExt.setMsgFoundList(msgListFilterAgain);
42 }
43
44 pullResultExt.setMessageBinary(null);
45
46 return pullResult;
47 }
解析：DefaultMQPushConsumerImpl.this.consumeMessageService.submitConsumeRequest(.....)1 public void submitConsumeRequest(
2 final List<MessageExt> msgs,
3 final ProcessQueue processQueue,
4 final MessageQueue messageQueue,
5 final boolean dispatchToConsume) {
//消费者一次可以消费的最大数量
6 final int consumeBatchSize = this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize();
7 if (msgs.size() <= consumeBatchSize) {
8 ConsumeRequest consumeRequest = new ConsumeRequest(msgs, processQueue, messageQueue);
9 try {
10 this.consumeExecutor.submit(consumeRequest);
11 } catch (RejectedExecutionException e) {
12 this.submitConsumeRequestLater(consumeRequest);
13 }
14 } else {
//如果拉取的总的消息数量msgs.size有8笔，consumeBatchSize=2,则分4次去消费
15 for (int total = 0; total < msgs.size(); ) {
16 List<MessageExt> msgThis = new ArrayList<MessageExt>(consumeBatchSize);
17 for (int i = 0; i < consumeBatchSize; i++, total++) {
18 if (total < msgs.size()) {
19 msgThis.add(msgs.get(total));
20 } else {
21 break;
22 }
23 }
24 //通过线程池执行ConsumeRequest的run()
25 ConsumeRequest consumeRequest = new ConsumeRequest(msgThis, processQueue, messageQueue);
26 try {
27 this.consumeExecutor.submit(consumeRequest);
28 } catch (RejectedExecutionException e) {
29 for (; total < msgs.size(); total++) {
30 msgThis.add(msgs.get(total));
31 }
32
33 this.submitConsumeRequestLater(consumeRequest);
34 }
35 }
36 }
37 }
解析：ConsumeRequest的run() 这里面会调用listen.consumeMessage(....) 这个方法就是实际上我们业务代码中拿到消费信息的方法1 public void run() {
//如果该队列被丢弃，直接返回
2 if (this.processQueue.isDropped()) {
3 ConsumeMessageConcurrentlyService.log.info("the message queue not be able to consume, because it's dropped. group={} {}", ConsumeMessageConcurrentlyService.this.consumerGroup, this.messageQueue);
4 } else {
5 int exeSize = this.msgs.size();
6 MessageListenerConcurrently listener = ConsumeMessageConcurrentlyService.this.messageListener;
7 ConsumeConcurrentlyContext context = new ConsumeConcurrentlyContext(this.messageQueue);
8 ConsumeConcurrentlyStatus status = null;
9 ConsumeMessageContext consumeMessageContext = null;
//作用？？
10 if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) {
11 consumeMessageContext = new ConsumeMessageContext();
12 consumeMessageContext.setConsumerGroup(ConsumeMessageConcurrentlyService.this.defaultMQPushConsumer.getConsumerGroup());
13 consumeMessageContext.setProps(new HashMap());
14 consumeMessageContext.setMq(this.messageQueue);
15 consumeMessageContext.setMsgList(this.msgs);
16 consumeMessageContext.setSuccess(false);
17 ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.executeHoefore(consumeMessageContext);
18 }
19
20 long beginTimestamp = System.currentTimeMillis();
21 boolean hasException = false;
22 ConsumeReturnType returnType = ConsumeReturnType.SUCCESS;
23
24 try {
25 ConsumeMessageConcurrentlyService.this.resetRetryTopic(this.msgs);
26 if (this.msgs != null && !this.msgs.isEmpty()) {
27 Iterator var10 = this.msgs.iterator();
28
29 while(var10.hasNext()) {
30 MessageExt msg = (MessageExt)var10.next();
31 MessageAccessor.setConsumeStartTimeStamp(msg, String.valueOf(System.currentTimeMillis()));
32 }
33 }
34
35 if (exeSize == 1) {
36 MQContextHelper.readChainContextFrom((Message)this.msgs.get(0), true);
37 }
38 //这个listener的实际对象由DefaultMQPushConsumer传入，所以这里会调用到DefaultMQPushConsumer中的接口实现方法中
39 status = listener.consumeMessage(Collections.unmodifiableList(tis.msgs), context);
40 } catch (Throwable var12) {
41 ConsumeMessageConcurrentlyService.log.warn("consumeMessage exception: {} Group: {} Msgs: {} MQ: {}", new Object[]{RemotingHelper.exceptionSimpleDesc(var12), ConsumeMessageConcurrentlyService.this.consumerGroup, this.msgs, this.messageQueue});
42 hasException = true;
43 }
44
45 long consumeRT = System.currentTimeMillis() - beginTimestamp;
46 if (null == status) {
47 if (hasException) {
48 returnType = ConsumeReturnType.EXCEPTION;
49 } else {
50 returnType = ConsumeReturnType.RETURNNULL;
51 }
52 } else if (consumeRT >= ConsumeMessageConcurrentlyService.this.defaultMQPushConsumer.getConsumeTimeout() * 60L * 1000L) {
53 returnType = ConsumeReturnType.TIME_OUT;
54 } else if (ConsumeConcurrentlyStatus.RECONSUME_LATER == status) {
55 returnType = ConsumeReturnType.FAILED;
56 } else if (ConsumeConcurrentlyStatus.CONSUME_SUCCESS == status) {
57 returnType = ConsumeReturnType.SUCCESS;
58 }
59
60 if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) {
61 consumeMessageContext.getProps().put("ConsumeContextType", returnType.name());
62 }
63
64 if (null == status) {
65 ConsumeMessageConcurrentlyService.log.warn("consumeMessage return null, Group: {} Msgs: {} MQ: {}", new Object[]{ConsumeMessageConcurrentlyService.this.consumerGroup, this.msgs, this.messageQueue});
66 status = ConsumeConcurrentlyStatus.RECONSUME_LATER;
67 }
68
69 if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) {
70 consumeMessageContext.setStatus(status.toString());
71 consumeMessageContext.setSuccess(ConsumeConcurrentlyStatus.CONSUME_SUCCESS == status);
72 ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.executeHookAfter(consumeMessageContext);
73 }
74
75 ConsumeMessageConcurrentlyService.this.getConsumerStatsManager().incConsumeRT(ConsumeMessageConcurrentlyService.this.consumerGroup, this.messageQueue.getTopic(), consumeRT);
76 if (!this.processQueue.isDropped()) {
//处理消费结果 详见下面解析
77 ConsumeMessageConcurrentlyService.this.processConsumeResult(status, context, this);
78 } else {
79 ConsumeMessageConcurrentlyService.log.warn("processQueue is dropped without process consume result. messageQueue={}, msgs={}", this.messageQueue, this.msgs);
80 }
81
82 }
83
84 }
解析：processConsumeResult(....) 处理消费结果 这个方法里面会对消费失败的进行消息重试 （生产发送失败也可以重试，但只限于同步发送的情况下）
在确定是否需要重试的时候，进一步处理哪些消息需要重试，也就是哪些消息会发送回broker1 public void processConsumeResult(
2 final ConsumeConcurrentlyStatus status,
3 final ConsumeConcurrentlyContext context,
4 final ConsumeRequest consumeRequest
5 ) {
//默认是int类型能存储的最大值
6 int ackIndex = context.getAckIndex();
7 //判断拉取的消息数量是否为空
8 if (consumeRequest.getMsgs().isEmpty())
9 return;
10
11 switch (status) {
12 case CONSUME_SUCCESS:
13 if (ackIndex >= consumeRequest.getMsgs().size()) {
14 ackIndex = consumeRequest.getMsgs().size() - 1;
15 }
16 int ok = ackIndex + 1;
17 int failed = consumeRequest.getMsgs().size() - ok;
//记录消费成功或者失败的吞吐量？？ TPS
18 this.getConsumerStatsManager().incConsumeOKTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), ok);
19 this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), failed);
20 break;
21 case RECONSUME_LATER:
22 ackIndex = -1;
23 this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(),
24 consumeRequest.getMsgs().size());
25 break;
26 default:
27 break;
28 }
29
30 switch (this.defaultMQPushConsumer.getMessageModel()) {
31 case BROADCASTING:
32 for (int i = ackIndex + 1; i < consumeRequest.getMsgs().size(); i++) {
33 MessageExt msg = consumeRequest.getMsgs().get(i);
34 log.warn("BROADCASTING, the message consume failed, drop it, {}", msg.toString());
35 }
36 break;
//一般使用集群模式
37 case CLUSTERING:
38 List<MessageExt> msgBackFailed = new ArrayList<MessageExt>(consumeRequest.getMsgs().size());
//正常情况下ackIndex会在上面的代码中被赋值为consumeRequest.getMsgs().size()
//所以这里的for循环就可以=for(int i=3;i<3;i++)这种情况了，根本就不会进行循环，也就不会进行消息重试了
39 for (int i = ackIndex + 1; i < consumeRequest.getMsgs().size(); i++) {
40 MessageExt msg = consumeRequest.getMsgs().get(i);
//直接重新发送消息到broker 这个方法的解析流程具体可以看：
//如果msgs有一条消息要重试，则msgs中的所有消息都要重试
41 boolean result = this.sendMessageBack(msg, context);
42 if (!result) {
43 msg.setReconsumeTimes(msg.getReconsumeTimes() + 1);
44 msgBackFailed.add(msg);
45 }
46 }
47
48 if (!msgBackFailed.isEmpty()) {
49 consumeRequest.getMsgs().removeAll(msgBackFailed);
50 //如果发送消息失败，则直接再次消费
51 this.submitConsumeRequestLater(msgBackFailed, consumeRequest.getProcessQueue(), consumeRequest.getMessageQueue());
52 }
53 break;
54 default:
55 break;
56 }
// 这篇文章讲了很多下面代码的实现
//removeMessage解析见如下
58 long offset = consumeRequest.getProcessQueue().removeMessage(consumeRequest.getMsgs());
//更新offset,之后就从此offset开始消费 ，这里也是会导致重复消费的地方，详见上面链接中的文章
59 if (offset >= 0 && !consumeRequest.getProcessQueue().isDropped()) {
60 this.defaultMQPushConsumerImpl.getOffsetStore().updateOffset(consumeRequest.getMessageQueue(), offset, true);
61 }
62 }
解析removeMessage
1 public long removeMessage(final List<MessageExt> msgs) {
2 long result = -1;
3 final long now = System.currentTimeMillis();
4 try {
5 this.lockTreeMap.writeLock().lockInterruptibly();
6 this.lastConsumeTimestamp = now;
7 try {
//本次从一个队列中拉取的消息总数，不只是这一次消费的数量
8 if (!msgTreeMap.isEmpty()) {
//默认下次从该队列中最大的一个offset中再+1进行 比如本次拉取的队列中的消息的offset是1000，则下次默认从1001开始
9 result = this.queueOffsetMax + 1;
10 int removedCnt = 0;
11 for (MessageExt msg : msgs) {
//已经消费完成的就从msgTreeMap中删除掉
12 MessageExt prev = msgTreeMap.remove(msg.getQueueOffset());
13 if (prev != null) {
14 removedCnt--;
15 msgSize.addAndGet(0 - msg.getBody().length);
16 }
17 }
18 msgCount.addAndGet(removedCnt);
19 //如果本次从队列中一共拉取了10条，但是最大消费数量为2 ，那么此时的msgTreeMap=8
20 if (!msgTreeMap.isEmpty()) {
//获取剩余未消费的消息集合中的最小的offset,可能拉取了100条 201-300都消费完成了，但是200还没有，
//那么此时的result=200，如果此时断电了，那么消费的进度就维持在了200，就会造成重复消费。
21 result = msgTreeMap.firstKey();
22 }
23 }
24 } finally {
25 this.lockTreeMap.writeLock().unlock();
26 }
27 } catch (Throwable t) {
28 log.error("removeMessage exception", t);
29 }
30
31 return result;
32 }msgTreeMap
本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。
