springcloudstream kafka 3 springcloudstream kafka 3.1

转载

mob64ca1415bcee 2024-04-15 11:32:26

文章标签 kafka spring cloud java spring jar 文章分类 架构后端开发

一、项目下载

二、配置Pom文件

<dependencies>
        <!-- 增加了 Controller 方便测试 -->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <!-- 核心jar -->
        <dependency>
            <groupId>org.springframework.cloud</groupId>
            <artifactId>spring-cloud-stream-binder-kafka</artifactId>
            <version>3.0.1.RELEASE</version>
        </dependency>

        <!-- 单元测试 -->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
            <scope>test</scope>
        </dependency>
    </dependencies>

三、配置application.yml

server:
  port: 2021
  servlet:
    context-path: /
  tomcat:
    uri-encoding: UTF-8

spring:
  cloud:
    stream:
      kafka:
        binder:
          brokers: 127.0.0.1:9092
          # auto-create-topics: true
      bindings:
        channel-name-1:
          destination: topic-name
          content-type: application/json
          group: oms-group-1
        channel-name-2:
          destination: topic-name
          content-type: application/json
          group: oms-group-2

四、生产者代码

package com.example.demo.producer;

import org.springframework.cloud.stream.annotation.Output;
import org.springframework.messaging.MessageChannel;

/**
 * @author liuxx
 * @date 2021/12/9 16:48
 * @desc 订单管理信道接口
 */
public interface OmsSource {

    String CHANNEL_NAME_1 = "channel-name-1";

    String CHANNEL_NAME_2 = "channel-name-2";

    @Output(OmsSource.CHANNEL_NAME_1)
    MessageChannel outputChannel1();

    @Output(OmsSource.CHANNEL_NAME_2)
    MessageChannel outputChannel2();

}

package com.example.demo.producer;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.cloud.stream.annotation.EnableBinding;
import org.springframework.integration.support.MessageBuilder;
import org.springframework.stereotype.Component;

import javax.annotation.Resource;

/**
 * @author liuxx
 * @date 2021/12/9 16:58
 * @desc 发送订单相关消息
 */
@Component
@EnableBinding(OmsSource.class)
public class OmsProducer {

    @Resource
    private OmsSource omsSource;

    private Logger logger = LoggerFactory.getLogger(OmsProducer.class);

    public void outputChannel1(String msg) {
        boolean successful = omsSource.outputChannel1().send(MessageBuilder.withPayload(msg).build());
        logger.info("给发送OMS的Channel-1消息 {}", successful ? "成功" : "失败");
    }

    public void outputChannel2(String msg) {
        boolean successful = omsSource.outputChannel1().send(MessageBuilder.withPayload(msg).build());
        logger.info("给发送OMS的Channel-2消息 {}", successful ? "成功" : "失败");
    }

}

五、消费者代码

package com.example.demo.consumer;

import org.springframework.cloud.stream.annotation.Input;
import org.springframework.messaging.SubscribableChannel;

/**
 * @author liuxx
 * @date 2021/12/9 16:45
 * @desc 订单管理信道接口
 */
public interface OmsSink {

    String CHANNEL_NAME_1 = "channel-name-1";

    String CHANNEL_NAME_2 = "channel-name-2";

    @Input(OmsSink.CHANNEL_NAME_1)
    SubscribableChannel inputChannel1();

    @Input(OmsSink.CHANNEL_NAME_2)
    SubscribableChannel inputChannel2();

}

package com.example.demo.consumer;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.cloud.stream.annotation.EnableBinding;
import org.springframework.cloud.stream.annotation.StreamListener;
import org.springframework.stereotype.Component;

/**
 * @author liuxx
 * @date 2021/12/9 16:59
 * @desc 消费订单相关消息
 */
@Component
@EnableBinding(OmsSink.class)
public class OmsConsumer {

    private Logger logger = LoggerFactory.getLogger(OmsConsumer.class);

    @StreamListener(OmsSink.CHANNEL_NAME_1)
    public void inputChannel1(String message) {
        logger.info("开始消费信道 1 消息:{}", message);
    }

    @StreamListener(OmsSink.CHANNEL_NAME_2)
    public void inputChannel2(String message) {
        logger.info("开始消费信道 2 消息:{}", message);
    }

}

六、消息测试

package com.example.demo.controller;

import com.example.demo.producer.OmsProducer;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import javax.annotation.Resource;

/**
 * @author liuxx
 * @date 2021/12/9 16:58
 * @desc 测试kafka消息controller
 */
@RestController
@RequestMapping("/kafka")
public class KafkaMessageController {

    @Resource
    private OmsProducer omsProducer;


    @GetMapping("/send/1/{msg}")
    public String sendChannel1(@PathVariable String msg) {
        omsProducer.outputChannel1(msg);
        return msg;
    }

    @GetMapping("/send/2/{msg}")
    public String sendChannel2(@PathVariable String msg) {
        omsProducer.outputChannel1(msg);
        return msg;
    }


}

七、测试

http://localhost:2021/kafka/send/1/{"key":"value"}

结果：

springcloudstream kafka 3 springcloudstream kafka 3.1_jar

说明：这里使用了两个channel做测试。是因为可能存在一条消息在同一个项目里存在多个业务需要消费，但是特别需要注意的是配置文件中 group 不能一样

八、问题

1.同一个项目中同一个Topic，配置多个消费信道时，错误配置（Group配置相同）：

Exception thrown while starting consumer:

org.springframework.cloud.stream.binder.BinderException: Exception thrown while starting consumer: 
	at org.springframework.cloud.stream.binder.AbstractMessageChannelBinder.doBindConsumer(AbstractMessageChannelBinder.java:471) ~[spring-cloud-stream-3.0.1.RELEASE.jar:3.0.1.RELEASE]
	at org.springframework.cloud.stream.binder.AbstractMessageChannelBinder.doBindConsumer(AbstractMessageChannelBinder.java:90) ~[spring-cloud-stream-3.0.1.RELEASE.jar:3.0.1.RELEASE]
	at org.springframework.cloud.stream.binder.AbstractBinder.bindConsumer(AbstractBinder.java:143) ~[spring-cloud-stream-3.0.1.RELEASE.jar:3.0.1.RELEASE]
	at org.springframework.cloud.stream.binding.BindingService.lambda$rescheduleConsumerBinding$0(BindingService.java:194) ~[spring-cloud-stream-3.0.1.RELEASE.jar:3.0.1.RELEASE]
	at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:54) ~[spring-context-5.2.13.RELEASE.jar:5.2.13.RELEASE]
	at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) ~[na:na]
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[na:na]
	at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[na:na]
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[na:na]
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[na:na]
	at java.base/java.lang.Thread.run(Thread.java:834) ~[na:na]
Caused by: org.springframework.beans.factory.support.BeanDefinitionOverrideException: Invalid bean definition with name 'topic-name.oms-group-1.errors.recoverer' defined in null: Cannot register bean definition [Root bean: class [org.springframework.integration.handler.advice.ErrorMessageSendingRecoverer]; scope=; abstract=false; lazyInit=null; autowireMode=0; dependencyCheck=0; autowireCandidate=true; primary=false; factoryBeanName=null; factoryMethodName=null; initMethodName=null; destroyMethodName=null] for bean 'topic-name.oms-group-1.errors.recoverer': There is already [Root bean: class [org.springframework.integration.handler.advice.ErrorMessageSendingRecoverer]; scope=; abstract=false; lazyInit=null; autowireMode=0; dependencyCheck=0; autowireCandidate=true; primary=false; factoryBeanName=null; factoryMethodName=null; initMethodName=null; destroyMethodName=null] bound.
	at org.springframework.beans.factory.support.DefaultListableBeanFactory.registerBeanDefinition(DefaultListableBeanFactory.java:945) ~[spring-beans-5.2.13.RELEASE.jar:5.2.13.RELEASE]
	at org.springframework.context.support.GenericApplicationContext.registerBeanDefinition(GenericApplicationContext.java:323) ~[spring-context-5.2.13.RELEASE.jar:5.2.13.RELEASE]
	at org.springframework.context.support.GenericApplicationContext.registerBean(GenericApplicationContext.java:471) ~[spring-context-5.2.13.RELEASE.jar:5.2.13.RELEASE]
	at org.springframework.cloud.stream.binder.AbstractMessageChannelBinder.registerErrorInfrastructure(AbstractMessageChannelBinder.java:691) ~[spring-cloud-stream-3.0.1.RELEASE.jar:3.0.1.RELEASE]
	at org.springframework.cloud.stream.binder.AbstractMessageChannelBinder.registerErrorInfrastructure(AbstractMessageChannelBinder.java:643) ~[spring-cloud-stream-3.0.1.RELEASE.jar:3.0.1.RELEASE]
	at org.springframework.cloud.stream.binder.kafka.KafkaMessageChannelBinder.createConsumerEndpoint(KafkaMessageChannelBinder.java:642) ~[spring-cloud-stream-binder-kafka-3.0.1.RELEASE.jar:3.0.1.RELEASE]
	at org.springframework.cloud.stream.binder.kafka.KafkaMessageChannelBinder.createConsumerEndpoint(KafkaMessageChannelBinder.java:147) ~[spring-cloud-stream-binder-kafka-3.0.1.RELEASE.jar:3.0.1.RELEASE]
	at org.springframework.cloud.stream.binder.AbstractMessageChannelBinder.doBindConsumer(AbstractMessageChannelBinder.java:417) ~[spring-cloud-stream-3.0.1.RELEASE.jar:3.0.1.RELEASE]
	... 10 common frames omitted

2.注意分开Source和Sink

亲身经历：我们真实使用的时候，使用一个类定义了生产者和消费者，导致其他开发者乱用了。（消费时未配置@Input，未申明对应的Bean，直接使用了@StreamListener消费对应Topic，导致其他项目的不同消费者Group的消费者消费不到消息）

这里我们用上面的Demo举例：

springcloudstream kafka 3 springcloudstream kafka 3.1_spring_02

接着重复测试：

http://localhost:2021/kafka/send/1/{"key":"value"}

结果：只有错误配置的channel收到了消息，另一个消费者没有收到

springcloudstream kafka 3 springcloudstream kafka 3.1_spring cloud_03

所以我们实际使用时，最好分开定义

3.Kafka Rebalance

Error while processing: ConsumerRecord(xxxx…)
org.springframework.kafka.listener.ListenerExecutionFailedException: Listener failed; nested exception is org.apache.kafka.clients.consumer.CommitFailedException: Commit cannot be completed since the group has already rebalanced and assigned the partitions to another member. This means that the time between subsequent calls to poll() was longer than the configured max.poll.interval.ms, which typically implies that the poll loop is spending too much time message processing. You can address this either by increasing max.poll.interval.ms or by reducing the maximum size of batches returned in poll() with max.poll.records.
其实我们可以翻译这个报错即可知：

消费者心跳超时，导致 rebalance。
消费者处理时间过长，导致 rebalance。

可以快速修改配置重启，但是最重要的还是确认为什么消费很慢

spring:
  kafka:
    consumer:
      max-poll-records: 100    -- 单次批量拉取100条数据(默认500)
      heartbeat-interval-ms: 4000 -- 心跳间隔4秒(默认3s)
      properties:
        max:
          poll:
            interval:
              ms: 900000 -- 每次消费的处理时间 15分钟(默认5分钟)
        session:
          timeout:
            ms: 12300 -- 心跳超时时间(建议间隔时间的3倍)+ 时延（100ms）
        request:
          timeout:
            ms: 300000  -- 发送消息时的超时时间

4.Dispatcher has no subscribers for channel ‘XXXX’

申明了SubscribableChannel但是没有写相应的@StreamListener

这里我们还是使用上面的Demo举例：

springcloudstream kafka 3 springcloudstream kafka 3.1_java_04

2021-12-09 18:34:28.434 ERROR 12764 --- [container-0-C-1] essageListenerContainer$ListenerConsumer : Error handler threw an exception

org.springframework.kafka.KafkaException: Seek to current after exception; nested exception is org.springframework.kafka.listener.ListenerExecutionFailedException: Listener failed; nested exception is org.springframework.messaging.MessageDeliveryException: Dispatcher has no subscribers for channel 'application.channel-name-1'.; nested exception is org.springframework.integration.MessageDispatchingException: Dispatcher has no subscribers, failedMessage=GenericMessage [payload=byte[15], headers={kafka_offset=3, scst_nativeHeadersPresent=true, kafka_consumer=org.apache.kafka.clients.consumer.KafkaConsumer@7edd54c, deliveryAttempt=3, kafka_timestampType=CREATE_TIME, kafka_receivedPartitionId=0, contentType=application/json, kafka_receivedTopic=topic-name, kafka_receivedTimestamp=1639046065363, kafka_groupId=oms-group-1}], failedMessage=GenericMessage [payload=byte[15], headers={kafka_offset=3, scst_nativeHeadersPresent=true, kafka_consumer=org.apache.kafka.clients.consumer.KafkaConsumer@7edd54c, deliveryAttempt=3, kafka_timestampType=CREATE_TIME, kafka_receivedPartitionId=0, contentType=application/json, kafka_receivedTopic=topic-name, kafka_receivedTimestamp=1639046065363, kafka_groupId=oms-group-1}]
	at org.springframework.kafka.listener.SeekUtils.seekOrRecover(SeekUtils.java:157) ~[spring-kafka-2.5.11.RELEASE.jar:2.5.11.RELEASE]
	at org.springframework.kafka.listener.SeekToCurrentErrorHandler.handle(SeekToCurrentErrorHandler.java:113) ~[spring-kafka-2.5.11.RELEASE.jar:2.5.11.RELEASE]
	at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.invokeErrorHandler(KafkaMessageListenerContainer.java:2072) ~[spring-kafka-2.5.11.RELEASE.jar:2.5.11.RELEASE]
	at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.doInvokeRecordListener(KafkaMessageListenerContainer.java:1971) ~[spring-kafka-2.5.11.RELEASE.jar:2.5.11.RELEASE]
	at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.doInvokeWithRecords(KafkaMessageListenerContainer.java:1898) ~[spring-kafka-2.5.11.RELEASE.jar:2.5.11.RELEASE]
	at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.invokeRecordListener(KafkaMessageListenerContainer.java:1786) ~[spring-kafka-2.5.11.RELEASE.jar:2.5.11.RELEASE]
	at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.invokeListener(KafkaMessageListenerContainer.java:1505) ~[spring-kafka-2.5.11.RELEASE.jar:2.5.11.RELEASE]
	at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.pollAndInvoke(KafkaMessageListenerContainer.java:1164) ~[spring-kafka-2.5.11.RELEASE.jar:2.5.11.RELEASE]
	at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.run(KafkaMessageListenerContainer.java:1059) ~[spring-kafka-2.5.11.RELEASE.jar:2.5.11.RELEASE]
	at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) ~[na:na]
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[na:na]
	at java.base/java.lang.Thread.run(Thread.java:834) ~[na:na]
Caused by: org.springframework.kafka.listener.ListenerExecutionFailedException: Listener failed; nested exception is org.springframework.messaging.MessageDeliveryException: Dispatcher has no subscribers for channel 'application.channel-name-1'.; nested exception is org.springframework.integration.MessageDispatchingException: Dispatcher has no subscribers, failedMessage=GenericMessage [payload=byte[15], headers={kafka_offset=3, scst_nativeHeadersPresent=true, kafka_consumer=org.apache.kafka.clients.consumer.KafkaConsumer@7edd54c, deliveryAttempt=3, kafka_timestampType=CREATE_TIME, kafka_receivedPartitionId=0, contentType=application/json, kafka_receivedTopic=topic-name, kafka_receivedTimestamp=1639046065363, kafka_groupId=oms-group-1}], failedMessage=GenericMessage [payload=byte[15], headers={kafka_offset=3, scst_nativeHeadersPresent=true, kafka_consumer=org.apache.kafka.clients.consumer.KafkaConsumer@7edd54c, deliveryAttempt=3, kafka_timestampType=CREATE_TIME, kafka_receivedPartitionId=0, contentType=application/json, kafka_receivedTopic=topic-name, kafka_receivedTimestamp=1639046065363, kafka_groupId=oms-group-1}]
	at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.decorateException(KafkaMessageListenerContainer.java:2087) ~[spring-kafka-2.5.11.RELEASE.jar:2.5.11.RELEASE]
	... 10 common frames omitted
Caused by: org.springframework.messaging.MessageDeliveryException: Dispatcher has no subscribers for channel 'application.channel-name-1'.; nested exception is org.springframework.integration.MessageDispatchingException: Dispatcher has no subscribers, failedMessage=GenericMessage [payload=byte[15], headers={kafka_offset=3, scst_nativeHeadersPresent=true, kafka_consumer=org.apache.kafka.clients.consumer.KafkaConsumer@7edd54c, deliveryAttempt=3, kafka_timestampType=CREATE_TIME, kafka_receivedPartitionId=0, contentType=application/json, kafka_receivedTopic=topic-name, kafka_receivedTimestamp=1639046065363, kafka_groupId=oms-group-1}]
	at org.springframework.integration.channel.AbstractSubscribableChannel.doSend(AbstractSubscribableChannel.java:76) ~[spring-integration-core-5.3.6.RELEASE.jar:5.3.6.RELEASE]
	at org.springframework.integration.channel.AbstractMessageChannel.send(AbstractMessageChannel.java:570) ~[spring-integration-core-5.3.6.RELEASE.jar:5.3.6.RELEASE]
	at org.springframework.integration.channel.AbstractMessageChannel.send(AbstractMessageChannel.java:520) ~[spring-integration-core-5.3.6.RELEASE.jar:5.3.6.RELEASE]
	at org.springframework.messaging.core.GenericMessagingTemplate.doSend(GenericMessagingTemplate.java:187) ~[spring-messaging-5.2.13.RELEASE.jar:5.2.13.RELEASE]
	at org.springframework.messaging.core.GenericMessagingTemplate.doSend(GenericMessagingTemplate.java:166) ~[spring-messaging-5.2.13.RELEASE.jar:5.2.13.RELEASE]
	at org.springframework.messaging.core.GenericMessagingTemplate.doSend(GenericMessagingTemplate.java:47) ~[spring-messaging-5.2.13.RELEASE.jar:5.2.13.RELEASE]
	at org.springframework.messaging.core.AbstractMessageSendingTemplate.send(AbstractMessageSendingTemplate.java:109) ~[spring-messaging-5.2.13.RELEASE.jar:5.2.13.RELEASE]
	at org.springframework.integration.endpoint.MessageProducerSupport.sendMessage(MessageProducerSupport.java:208) ~[spring-integration-core-5.3.6.RELEASE.jar:5.3.6.RELEASE]
	at org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter.sendMessageIfAny(KafkaMessageDrivenChannelAdapter.java:384) ~[spring-integration-kafka-3.2.1.RELEASE.jar:3.2.1.RELEASE]
	at org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter.access$300(KafkaMessageDrivenChannelAdapter.java:75) ~[spring-integration-kafka-3.2.1.RELEASE.jar:3.2.1.RELEASE]
	at org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter$IntegrationRecordMessageListener.onMessage(KafkaMessageDrivenChannelAdapter.java:443) ~[spring-integration-kafka-3.2.1.RELEASE.jar:3.2.1.RELEASE]
	at org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter$IntegrationRecordMessageListener.onMessage(KafkaMessageDrivenChannelAdapter.java:417) ~[spring-integration-kafka-3.2.1.RELEASE.jar:3.2.1.RELEASE]
	at org.springframework.kafka.listener.adapter.RetryingMessageListenerAdapter.lambda$onMessage$0(RetryingMessageListenerAdapter.java:120) ~[spring-kafka-2.5.11.RELEASE.jar:2.5.11.RELEASE]
	at org.springframework.retry.support.RetryTemplate.doExecute(RetryTemplate.java:287) ~[spring-retry-1.2.5.RELEASE.jar:na]
	at org.springframework.retry.support.RetryTemplate.execute(RetryTemplate.java:211) ~[spring-retry-1.2.5.RELEASE.jar:na]
	at org.springframework.kafka.listener.adapter.RetryingMessageListenerAdapter.onMessage(RetryingMessageListenerAdapter.java:114) ~[spring-kafka-2.5.11.RELEASE.jar:2.5.11.RELEASE]
	at org.springframework.kafka.listener.adapter.RetryingMessageListenerAdapter.onMessage(RetryingMessageListenerAdapter.java:40) ~[spring-kafka-2.5.11.RELEASE.jar:2.5.11.RELEASE]
	at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.doInvokeOnMessage(KafkaMessageListenerContainer.java:2039) ~[spring-kafka-2.5.11.RELEASE.jar:2.5.11.RELEASE]
	at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.invokeOnMessage(KafkaMessageListenerContainer.java:2021) ~[spring-kafka-2.5.11.RELEASE.jar:2.5.11.RELEASE]
	at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.doInvokeRecordListener(KafkaMessageListenerContainer.java:1958) ~[spring-kafka-2.5.11.RELEASE.jar:2.5.11.RELEASE]
	... 8 common frames omitted
Caused by: org.springframework.integration.MessageDispatchingException: Dispatcher has no subscribers
	at org.springframework.integration.dispatcher.UnicastingDispatcher.doDispatch(UnicastingDispatcher.java:139) ~[spring-integration-core-5.3.6.RELEASE.jar:5.3.6.RELEASE]
	at org.springframework.integration.dispatcher.UnicastingDispatcher.dispatch(UnicastingDispatcher.java:106) ~[spring-integration-core-5.3.6.RELEASE.jar:5.3.6.RELEASE]
	at org.springframework.integration.channel.AbstractSubscribableChannel.doSend(AbstractSubscribableChannel.java:72) ~[spring-integration-core-5.3.6.RELEASE.jar:5.3.6.RELEASE]
	... 27 common frames omitted

写代码一定要仔细，注意检查
当然这个错误不一定是这个原因，每个人遇到的问题不尽相同，还是自己分析吧~

本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。