前言
回顾 Consumer 消息拉取消费机制 我们了解了 Consumer 如何启动并拉取消息消费的机制。PullMessageService 负责对消息队列进行消息拉取，从远程服务器拉取消息后将消息存入 ProcessQueue 消息队列处理队列中，然后调用 ConsumeMessageService#submitConsumeRequest 方法进行消息消费，使用线程池来消费消息，确保了消息拉取与消费的解耦。
但是对于拉取到的消息怎么处理，我们带着以下问题来进行分析：
1、Consume如何获取并维护消费进度和确保消息消费不丢失？
一、从何处开始消费？
Consumer 通过调用：
consumer.start();在启动的时候，会加载消费进度，如下所示：
org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl#start 代码片段
if (this.defaultMQPushConsumer.getOffsetStore() != null) {
this.offsetStore = this.defaultMQPushConsumer.getOffsetStore();
} else {
switch (this.defaultMQPushConsumer.getMessageModel()) {
case BROADCASTING:
this.offsetStore = new LocalFileOffsetStore(this.mQClientFactory, this.defaultMQPushConsumer.getConsumerGroup());
break;
case CLUSTERING:
this.offsetStore = new RemoteBrokerOffsetStore(this.mQClientFactory, this.defaultMQPushConsumer.getConsumerGroup());
break;
default:
break;
}
this.defaultMQPushConsumer.setOffsetStore(this.offsetStore);
}
this.offsetStore.load(); //加载本地消费进度OffsetStore提供了操作消费进度的方法，例如：加载消费进度，读取消费进度，更新消费进度等等。
在集群消费模式下，消费进度并没有持久化在 Consumer 端，而是保存在了远程 Broker 端，例如上面使用的是 RemoteBrokerOffsetStore 类：
public class RemoteBrokerOffsetStore implements OffsetStore {
private final static InternalLogger log = ClientLogger.getLog();
private final MQClientInstance mQClientFactory;
private final String groupName;
private ConcurrentMap<MessageQueue, AtomicLong> offsetTable =
new ConcurrentHashMap<MessageQueue, AtomicLong>();
//...
}因为消费进度保存在 Broker端，用于加载本地消费进度的load方法，在RemoteBrokerOffsetStore中是空的，不做任何事，真正读取消费进度是通过readOffset方法实现的。
了解 OffetStore 类的作用后，我们看看它实在何时被调用上。
二、消息队列负载与重新分布
PullRequest 何时被创建加入拉取队列中？集群中多个消费者如何负载消费队列？如何重新分布？我们在 Consumer 消息拉取消费机制 该篇中有所提到。
一个 Topic 有多个消费队列，而同一组group下面也可能有多个 Consumer订阅了这个 Topic，于是这些队列需要按照一定策略分配给同一个组下面的 Consumer 消费，例如下面的平均分配：
在上图中，TOPIC_A 有5个队列，有2个 Consumer 订阅了，如果按照平均分配的话，那就是 Consumer1 消费其中3个队列，Consumer2 消费其中2个队列。这个就是负载均衡。
一个 Consumer 分配到了几个消息队列，就会相应的创建几个消息处理队列（ProcessQueue，消费消息时会用到），并且此时会生成一个拉取消息的请求（PullRequest，请求消息时会用到），这个请求不是真正的发往broker端的获取消息的请求，而是保存在一个阻塞队列里面，然后由专门的拉取消息的服务线程读取它并组装获取消息请求，发送给broker端（这么做当然是为了获得异步的好处）。
它由 RebalanceService 服务线程来实现，每隔 20s 最终调用RebalanceImpl#doRebalance 方法。
org.apache.rocketmq.client.impl.consumer.RebalanceImpl#updateProcessQueueTableInRebalance 代码片段
private boolean updateProcessQueueTableInRebalance(final String topic, final Set<MessageQueue> mqSet,
final boolean isOrder) {
boolean changed = false;
List<PullRequest> pullRequestList = new ArrayList<PullRequest>();
for (MessageQueue mq : mqSet) {
if (!this.processQueueTable.containsKey(mq)) {
if (isOrder && !this.lock(mq)) {
log.warn("doRebalance, {}, add a new mq failed, {}, because lock failed", consumerGroup, mq);
continue;
}
this.removeDirtyOffset(mq);
ProcessQueue pq = new ProcessQueue();
//计算拉取的消费进度
long nextOffset = this.computePullFromWhere(mq);
if (nextOffset >= 0) {
ProcessQueue pre = this.processQueueTable.putIfAbsent(mq, pq);
if (pre != null) {
log.info("doRebalance, {}, mq already exists, {}", consumerGroup, mq);
} else {
log.info("doRebalance, {}, add a new mq, {}", consumerGroup, mq);
PullRequest pullRequest = new PullRequest();
pullRequest.setConsumerGroup(consumerGroup);
pullRequest.setNextOffset(nextOffset);
pullRequest.setMessageQueue(mq);
pullRequest.setProcessQueue(pq);
pullRequestList.add(pullRequest);
changed = true;
}
} else {
log.warn("doRebalance, {}, add new mq failed, {}", consumerGroup, mq);
}
}
}
//分派到 PullMessageService 服务线程的请求队列中，等待异步拉取
this.dispatchPullRequest(pullRequestList);
return changed;
}三、消费拉取消费并维护消费进度
在上述 Consumer 负载均衡分配了 消息队列之后，PullMessageService 服务线程即可从 Broker 拉取消息，最终调用以下方法。
org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl#pullMessage 代码片段
public void pullMessage(final PullRequest pullRequest) {
//忽略非核心代码...
try {
this.pullAPIWrapper.pullKernelImpl(
pullRequest.getMessageQueue(), //待拉取消息的消费队列
subExpression,
subscriptionData.getExpressionType(),
subscriptionData.getSubVersion(),
pullRequest.getNextOffset(), // 下个消息进度
this.defaultMQPushConsumer.getPullBatchSize(),
sysFlag,
commitOffsetValue,
BROKER_SUSPEND_MAX_TIME_MILLIS,
CONSUMER_TIMEOUT_MILLIS_WHEN_SUSPEND,
CommunicationMode.ASYNC,
pullCallback //拉取后回调
);
} catch (Exception e) {
this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException);
}
}PullCallback pullCallback = new PullCallback() {
@Override
public void onSuccess(PullResult pullResult) {
if (pullResult != null) {
//反序列化消息列表
pullResult = DefaultMQPushConsumerImpl.this.pullAPIWrapper.processPullResult(pullRequest.getMessageQueue(), pullResult, subscriptionData);
switch (pullResult.getPullStatus()) {
case FOUND:
long prevRequestOffset = pullRequest.getNextOffset();
//设置下次拉取的进度
pullRequest.setNextOffset(pullResult.getNextBeginOffset());
long firstMsgOffset = Long.MAX_VALUE;
if (pullResult.getMsgFoundList() == null || pullResult.getMsgFoundList().isEmpty()) {
//拉取无消息，重新放回拉取请求队列中
DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
} else {
firstMsgOffset = pullResult.getMsgFoundList().get(0).getQueueOffset();
//将消息丢入消费服务线程中进行消费
boolean dispatchToConsume = processQueue.putMessage(pullResult.getMsgFoundList());
DefaultMQPushConsumerImpl.this.consumeMessageService.submitConsumeRequest(
pullResult.getMsgFoundList(),
processQueue,
pullRequest.getMessageQueue(),
dispatchToConsume);
}
break;
}
}
}
};
org.apache.rocketmq.client.impl.consumer.ConsumeMessageConcurrentlyService#submitConsumeRequest 代码片段
public void submitConsumeRequest(final List<MessageExt> msgs, final ProcessQueue processQueue, final MessageQueue messageQueue, final boolean dispatchToConsume) {
//忽略非核心代码...
//根据配置，对消息列表进行分割，默认1
final int consumeBatchSize = this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize();
if (msgs.size() <= consumeBatchSize) {
ConsumeRequest consumeRequest = new ConsumeRequest(msgs, processQueue, messageQueue);
try {
this.consumeExecutor.submit(consumeRequest);
} catch (RejectedExecutionException e) {
this.submitConsumeRequestLater(consumeRequest);
}
} else {
for (int total = 0; total < msgs.size(); ) {
List<MessageExt> msgThis = new ArrayList<MessageExt>(consumeBatchSize);
for (int i = 0; i < consumeBatchSize; i++, total++) {
if (total < msgs.size()) {
msgThis.add(msgs.get(total));
} else {
break;
}
}
//封装成消费请求，丢入消费线程池中处理（异步）
ConsumeRequest consumeRequest = new ConsumeRequest(msgThis, processQueue, messageQueue);
try {
this.consumeExecutor.submit(consumeRequest);
} catch (RejectedExecutionException e) {
for (; total < msgs.size(); total++) {
msgThis.add(msgs.get(total));
}
this.submitConsumeRequestLater(consumeRequest);
}
}
}
} org.apache.rocketmq.client.impl.consumer.ConsumeMessageConcurrentlyService.ConsumeRequest 代码片段
class ConsumeRequest implements Runnable {
//忽略非核心代码...
private final List<MessageExt> msgs;
private final ProcessQueue processQueue;
private final MessageQueue messageQueue;
@Override
public void run() {
if (this.processQueue.isDropped()) {
return;
}
MessageListenerConcurrently listener = ConsumeMessageConcurrentlyService.this.messageListener;
ConsumeConcurrentlyContext context = new ConsumeConcurrentlyContext(messageQueue);
ConsumeConcurrentlyStatus status = null;
defaultMQPushConsumerImpl.resetRetryAndNamespace(msgs, defaultMQPushConsumer.getConsumerGroup());
long beginTimestamp = System.currentTimeMillis();
boolean hasException = false;
ConsumeReturnType returnType = ConsumeReturnType.SUCCESS;
try {
if (msgs != null && !msgs.isEmpty()) {
for (MessageExt msg : msgs) {
MessageAccessor.setConsumeStartTimeStamp(msg, String.valueOf(System.currentTimeMillis()));
}
}
//丢入消息监听器处理
status = listener.consumeMessage(Collections.unmodifiableList(msgs), context);
} catch (Throwable e) {
log.warn("consumeMessage exception: {} Group: {} Msgs: {} MQ: {}",
RemotingHelper.exceptionSimpleDesc(e),
ConsumeMessageConcurrentlyService.this.consumerGroup,
msgs,
messageQueue);
hasException = true;
}
if (!processQueue.isDropped()) {
//消息消费后，处理队列未挂起，处理消费结果
ConsumeMessageConcurrentlyService.this.processConsumeResult(status, context, this);
}
}
} org.apache.rocketmq.client.impl.consumer.ConsumeMessageConcurrentlyService#processConsumeResult 代码片段
public void processConsumeResult(final ConsumeConcurrentlyStatus status, final ConsumeConcurrentlyContext context, final ConsumeRequest consumeRequest) {
int ackIndex = context.getAckIndex();
if (consumeRequest.getMsgs().isEmpty())
return;
switch (status) {
case CONSUME_SUCCESS:
//消费成功，ack 等于 消息条数
if (ackIndex >= consumeRequest.getMsgs().size()) {
ackIndex = consumeRequest.getMsgs().size() - 1;
}
int ok = ackIndex + 1;
int failed = consumeRequest.getMsgs().size() - ok;
break;
case RECONSUME_LATER:
ackIndex = -1;
break;
}
switch (this.defaultMQPushConsumer.getMessageModel()) {
case CLUSTERING:
List<MessageExt> msgBackFailed = new ArrayList<MessageExt>(consumeRequest.getMsgs().size());
//若消费不成功，重新消费，ack 小于 消息条数
for (int i = ackIndex + 1; i < consumeRequest.getMsgs().size(); i++) {
MessageExt msg = consumeRequest.getMsgs().get(i);
//把消息重新发回 Broker，延迟消费
boolean result = this.sendMessageBack(msg, context);
if (!result) {
msg.setReconsumeTimes(msg.getReconsumeTimes() + 1);
msgBackFailed.add(msg);
}
}
//发回失败（Broker宕机等），重新丢入消费请求池中
if (!msgBackFailed.isEmpty()) {
consumeRequest.getMsgs().removeAll(msgBackFailed);
this.submitConsumeRequestLater(msgBackFailed, consumeRequest.getProcessQueue(), consumeRequest.getMessageQueue());
}
break;
}
long offset = consumeRequest.getProcessQueue().removeMessage(consumeRequest.getMsgs());
if (offset >= 0 && !consumeRequest.getProcessQueue().isDropped()) {
//更新消费进度（在内存中），集群模式下，同步更新到 Broker
this.defaultMQPushConsumerImpl.getOffsetStore().updateOffset(consumeRequest.getMessageQueue(), offset, true);
}
}在 Consumer 启动 MQClientInstance 期间，开启定时任务每隔 5s，调用 Instance#persistAllConsumerOffset 持久化所有消费进度。
org.apache.rocketmq.client.impl.factory.MQClientInstance#persistAllConsumerOffset 代码片段
private void persistAllConsumerOffset() {
//遍历消费者列表
Iterator<Entry<String, MQConsumerInner>> it = this.consumerTable.entrySet().iterator();
while (it.hasNext()) {
Entry<String, MQConsumerInner> entry = it.next();
MQConsumerInner impl = entry.getValue();
impl.persistConsumerOffset();
}
} org.apache.rocketmq.client.consumer.store.RemoteBrokerOffsetStore#persistAll 代码片段
public void persistAll(Set<MessageQueue> mqs) {
if (null == mqs || mqs.isEmpty())
return;
final HashSet<MessageQueue> unusedMQ = new HashSet<MessageQueue>();
for (Map.Entry<MessageQueue, AtomicLong> entry : this.offsetTable.entrySet()) {
MessageQueue mq = entry.getKey();
AtomicLong offset = entry.getValue();
if (offset != null) {
if (mqs.contains(mq)) {
try {
//更新消费进度 到 Broker
this.updateConsumeOffsetToBroker(mq, offset.get());
} catch (Exception e) {
log.error("updateConsumeOffsetToBroker exception, " + mq.toString(), e);
}
} else {
unusedMQ.add(mq);
}
}
}
}
public void updateConsumeOffsetToBroker(MessageQueue mq, long offset, boolean isOneway) throws RemotingException,
MQBrokerException, InterruptedException, MQClientException {
//从本地获取 Broker 地址
FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName());
if (null == findBrokerResult) {
//不存在，从 NameSrv 获取
this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());
findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName());
}
if (findBrokerResult != null) {
UpdateConsumerOffsetRequestHeader requestHeader = new UpdateConsumerOffsetRequestHeader();
requestHeader.setTopic(mq.getTopic());
requestHeader.setConsumerGroup(this.groupName);
requestHeader.setQueueId(mq.getQueueId());
requestHeader.setCommitOffset(offset); //设置要更新的消费进度
if (isOneway) {
this.mQClientFactory.getMQClientAPIImpl().updateConsumerOffsetOneway(
findBrokerResult.getBrokerAddr(), requestHeader, 1000 * 5);
} else {
this.mQClientFactory.getMQClientAPIImpl().updateConsumerOffset(
findBrokerResult.getBrokerAddr(), requestHeader, 1000 * 5);
}
} else {
throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
}
}
本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。
