flink job 运行设置
1、pom文件依赖
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.exam.iot</groupId>
<artifactId>flink-job</artifactId>
<version>0.0.1</version>
<name>flink-job</name>
<packaging>jar</packaging>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<logback.version>1.2.3</logback.version>
<log4j-over-slf4j.version>1.7.30</log4j-over-slf4j.version>
<java.version>1.8</java.version>
<mysql.version>8.0.28</mysql.version>
<jackson.version>2.12.5</jackson.version>
<flink.version>1.9.3</flink.version>
<dubbo.version>2.7.14</dubbo.version>
<guava.version>31.1-jre</guava.version>
<influxdb-client.version>2.22</influxdb-client.version>
</properties>
<dependencies>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
<version>${logback.version}</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-access</artifactId>
<version>${logback.version}</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>${logback.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
<version>${log4j-over-slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
</dependency>
<!-- redis依赖 -->
<dependency>
<groupId>org.redisson</groupId>
<artifactId>redisson</artifactId>
<version>3.16.8</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.79</version>
</dependency>
<!-- MySQL的JDBC驱动包 -->
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>${mysql.version}</version>
</dependency>
<dependency>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
<version>4.0.3</version>
</dependency>
<!-- jackson:用于json和Object转换 -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
</dependency>
<!--flink依赖-->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients_2.11</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_2.11</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka_2.11</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-runtime-web_2.11</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-avro</artifactId>
<version>${flink.version}</version>
</dependency>
<!-- 提供 LoadingCache -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<!--JSON解析依赖-->
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.9.0</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.22</version>
<optional>true</optional>
</dependency>
<!-- 提供过期移除map的集合ExpiringMap、LoadingCache -->
<dependency>
<groupId>net.jodah</groupId>
<artifactId>expiringmap</artifactId>
<version>0.5.10</version>
</dependency>
<!-- influxdb -->
<dependency>
<groupId>org.influxdb</groupId>
<artifactId>influxdb-java</artifactId>
<version>${influxdb-client.version}</version>
</dependency>
<!-- tdEngine -->
<dependency>
<groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId>
<version>2.0.37</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-pool2</artifactId>
<version>2.11.1</version>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>5.7.22</version>
</dependency>
</dependencies>
<build>
<finalName>${project.name}</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<!--打jar包-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.3.0</version>
<configuration>
<archive>
<manifest>
<mainClass>com.exam.job.JobApplication</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.4</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<artifactSet>
<excludes>
<exclude>com.google.code.findbugs:jsr305</exclude>
<exclude>org.slf4j:*</exclude>
<exclude>log4j:*</exclude>
</excludes>
</artifactSet>
<filters>
<filter>
<!-- Do not copy the signatures in the META-INF folder.
Otherwise, this might cause SecurityExceptions when using the JAR. -->
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
<transformers combine.children="append">
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>com.exam.iot.JobApplication</mainClass>
</transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer">
</transformer>
</transformers>
<relocations>
<relocation>
<pattern>org.codehaus.plexus.util</pattern>
<shadedPattern>org.shaded.plexus.util</shadedPattern>
<excludes>
<exclude>org.codehaus.plexus.util.xml.Xpp3Dom</exclude>
<exclude>org.codehaus.plexus.util.xml.pull.*</exclude>
</excludes>
</relocation>
</relocations>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
<resources>
<resource>
<directory>src/main/java</directory>
<includes>
<include>**/*.yml</include>
<include>**/*.properties</include>
<include>**/*.xml</include>
</includes>
<filtering>false</filtering>
</resource>
<resource>
<directory>src/main/resources</directory>
<includes>
<include>**/*.yml</include>
<include>**/*.properties</include>
<include>**/*.xml</include>
</includes>
<filtering>false</filtering>
</resource>
</resources>
</build>
</project>2、在src/main/resources创建logback.xml文件
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<property name="LOGS" value="./logs/" />
<appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
<layout class="ch.qos.logback.classic.PatternLayout">
<!-- (1) -->
<pattern>%black(%d{ISO8601}) %highlight(%-5level) [%blue(%t)] %yellow(%C{1.}): %msg%n%throwable</pattern>
<!-- (2) -->
<pattern>%date{yyyy-MM-dd HH:mm:ss} %highlight(%-5level) (%file:%line\)- %m%n</pattern>
<!-- 控制台也要使用utf-8，不要使用gbk -->
<charset>UTF-8</charset>
</layout>
</appender>
<appender name="RollingFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${LOGS}/electric-job.log</file>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<Pattern>%d %p %C{1.} [%t] %m%n</Pattern>
</encoder>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- rollover daily and when the file reaches 10 MegaBytes -->
<fileNamePattern>${LOGS}/archived/job-%d{yyyy-MM-dd}.%i.log
</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>100MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
</rollingPolicy>
</appender>
<!-- RollingFileAppender:滚动记录文件，先将日志记录到指定文件，当符合某个条件时，将日志记录到其他文件 -->
<!-- 1.先按日期存日志，日期变了，将前一天的日志文件名重命名为xxx%日期%索引，新的日志仍然是sys.log -->
<!-- 2.如果日期没有变化，但是当前日志文件的大小超过1kb时，对当前日志进行分割 重名名 -->
<appender name="syslog" class="ch.qos.logback.core.rolling.RollingFileAppender">
<!--<File>${catalina.base}/mylog/sys.log</File>-->
<File>./logs/%d/plug-flink-job.log</File>
<!-- rollingPolicy:当发生滚动时，决定 RollingFileAppender 的行为，涉及文件移动和重命名。 -->
<!-- TimeBasedRollingPolicy： 最常用的滚动策略，它根据时间来制定滚动策略，既负责滚动也负责出发滚动 -->
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- 活动文件的名字会根据fileNamePattern的值，每隔一段时间改变一次 -->
<!-- 文件名：mylog/sys.2017-12-05.0.log -->
<!--<fileNamePattern>${catalina.base}/mylog/sys.%d.%i.log</fileNamePattern>-->
<fileNamePattern>./logs/%d/plug-flink-job.%d.%i.log</fileNamePattern>
<!-- 每产生一个日志文件，该日志文件的保存期限为30天 -->
<maxHistory>30</maxHistory>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<!-- maxFileSize:这是活动文件的大小，默认值是10MB,本篇设置为1KB，只是为了演示 -->
<maxFileSize>128MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
</rollingPolicy>
<encoder>
<!-- pattern节点，用来设置日志的输入格式 -->
<pattern>
%d %p (%file:%line\)- %m%n
</pattern>
<!-- 记录日志的编码 -->
<charset>UTF-8</charset> <!-- 此处设置字符集 -->
</encoder>
</appender>
<!-- LOG everything at INFO level -->
<root level="INFO">
<appender-ref ref="RollingFile" />
<appender-ref ref="Console" />
</root>
<!-- LOG "com.exam.job*" at TRACE level -->
<logger name="com.exam.iot" level="INFO" additivity="false">
<appender-ref ref="RollingFile" />
<appender-ref ref="Console" />
</logger>
</configuration>3、job示例
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.util.Collector;
import java.util.Properties;
public class FlinkKafkaWordCount {
public static void main(String[] args) throws Exception {
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.enableCheckpointing(5000);
env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
Properties properties = new Properties();
properties.setProperty("bootstrap.servers", "10.170.224.246:9092,10.170.224.246:9093,10.170.224.246:9094");
properties.setProperty("group.id", "word-count-group");
FlinkKafkaConsumer<String> consumer = new FlinkKafkaConsumer<>("wc_topic_post", new SimpleStringSchema(), properties);
DataStream<WordWithCount> counts = env.addSource(consumer)
.flatMap(new FlatMapFunction<String, WordWithCount>() {
@Override
public void flatMap(String value, Collector<WordWithCount> collector) throws Exception {
for (String word : value.split("\t")) {
collector.collect(new WordWithCount(word, 1L));
}
}
})
.keyBy("word")
.timeWindow(Time.seconds(5))
.reduce(new ReduceFunction<WordWithCount>() {
@Override
public WordWithCount reduce(WordWithCount a, WordWithCount b) throws Exception {
return new WordWithCount(a.word, a.count + b.count);
}
});
counts.print().setParallelism(1);
env.execute("flink-kafka-wordcount");
}
public static class WordWithCount {
private String word;
private long count;
public WordWithCount() {
}
public WordWithCount(String word, long count) {
this.word = word;
this.count = count;
}
public String getWord() {
return word;
}
public void setWord(String word) {
this.word = word;
}
public long getCount() {
return count;
}
public void setCount(long count) {
this.count = count;
}
@Override
public String toString() {
return word + " : " + count;
}
}
}4、设置checkpoint
// 并行度优先级:算子>env>client>配置文件
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.enableCheckpointing(10 * 60 * 1000, CheckpointingMode.EXACTLY_ONCE);
// set checkpoint
CheckpointConfig checkpointConfig = env.getCheckpointConfig();
checkpointConfig.setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
checkpointConfig.setCheckpointTimeout(60 * 1000);
checkpointConfig.setMinPauseBetweenCheckpoints((60 * 1000));
checkpointConfig.setMaxConcurrentCheckpoints(1);
checkpointConfig.setTolerableCheckpointFailureNumber(3);
// checkpointConfig.setPreferCheckpointForRecovery(true);
// checkpointConfig.enableUnalignedCheckpoints(); // 非对齐的 Checkpoint
// 可以配置 Checkpoint 定期持久化到从外部存储中。使用这种方式不会在任务正常停止的过程中清理 Checkpoint 数据，而是会一直保存在外部存储中，另外我们也可以通过从外部 Checkpoint 中对任务进行恢复：
checkpointConfig.enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
// 设置重启策略
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 60 * 1000));5、job设置
try {
env.addSource(AppConfig.getConsumer())
.setParallelism(8)
.map(new RichMapFunction<String, DeviceClientInfo>() {
@Override
public DeviceClientInfo map(String jsonLine) {
try {
log.info("jsonLine={}", jsonLine);
String line = jsonLine.replaceAll("socket_abb.", "");
return JSONObject.parseObject(line, DeviceClientInfo.class);
} catch (Exception e) {
log.error("job.line.map报错: {}", e.getMessage());
}
return null;
}
})
.assignTimestampsAndWatermarks(new AssignerWithPeriodicWatermarks<DeviceClientInfo>() {
// 观察到的最大时间戳
private long currentMaxTs = Long.MIN_VALUE;
@Nullable
@Override
public Watermark getCurrentWatermark() {
// 生成具有30秒容忍度的水位线
// 定义30秒的容忍间隔时间，即允许数据的最大乱序时间
long maxOutofOrderness = 30 * 1000;
return new Watermark(currentMaxTs - maxOutofOrderness);
}
@Override
public long extractTimestamp(DeviceClientInfo deviceClientInfo, long previousElementTimestamp) {
//获取当前记录的时间戳
long currentTs = deviceClientInfo.getTimestamp();
// 更新最大的时间戳
currentMaxTs = Math.max(currentMaxTs, currentTs);
// 返回记录的时间戳
return currentTs;
}
})
.filter(new RichFilterFunction<DeviceClientInfo>() {
@Override
public boolean filter(DeviceClientInfo deviceClientInfo) {
return deviceClientInfo != null
&& StringUtils.isNotEmpty(deviceClientInfo.getDeviceId())
&& deviceClientInfo.getPayload() != null
&& deviceClientInfo.getPayload().getProperties() != null;
}
})
.map(new RichMapFunction<DeviceClientInfo, Electric>() {
final int recordFixedTime = 15;
@Override
public Electric map(DeviceClientInfo deviceClientInfo) {
try {
return parseElectric(recordFixedTime, deviceClientInfo);
} catch (Exception e) {
log.error("job.dci.map报错: ", e);
}
return null;
}
}).filter(new RichFilterFunction<Electric>() {
@Override
public boolean filter(Electric electric) {
return electric != null;
}
})
.keyBy(Electric::getDeviceId)
.timeWindow(Time.minutes(1))
.reduce(new ReduceFunction<Electric>() {
@Override
public Electric reduce(Electric preE, Electric curE) {
log.info("***** Electric curE={}", JSONObject.toJSONString(curE));
return curE;
}
})
.addSink(new SinkMysqlWithOne())
.name("mysqlSink");
} catch (Exception e) {
e.printStackTrace();
}
env.execute("degree-sink");6、在src/main/resources目录下创建log4j.properties文件
log4j.rootLogger=INFO,logfile,stdout
#log4j.logger.org.springframework.web.servlet=INFO,db
#log4j.logger.org.springframework.beans.factory.xml=INFO
#log4j.logger.com.exam.job=INFO,db
#log4j.appender.stdout=org.apache.log4j.ConsoleAppender
#log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
#log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH\:mm\:ss} %p [%c] %X{remoteAddr} %X{remotePort} %X{remoteHost} %X{remoteUser} operator\:[\u59D3\u540D\:%X{userName} \u5DE5\u53F7\:%X{userId}] message\:<%m>%n
#write log into file
log4j.appender.logfile=org.apache.log4j.DailyRollingFileAppender
log4j.appender.logfile.Threshold=warn
log4j.appender.logfile.File=./logs/job.log
log4j.appender.logfile.DatePattern=.yyyy-MM-dd
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=[app]-[%d{yyyy-MM-dd HH:mm:ss}] %X{remoteAddr} %X{remotePort} %m %n
#display in console
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Threshold=info
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=[app]-[%d{yyyy-MM-dd HH:mm:ss}] %X{remoteAddr} %X{remotePort} %m %n7、滚动窗口示例
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.util.Collector;
public class WindowTest {
public static void main(String[] args) {
try {
// 创建流处理的执行环境
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
//2.使用StreamExecutionEnvironment创建DataStream
//Source(可以有多个Source)
//Socket 监听本地端口8888
// 接收一个socket文本流
DataStreamSource<String> lines = env.socketTextStream("localhost", 8888);
// Transformation(s) 对数据进行转换处理统计，先分词，再按照word进行分组，最后进行聚合统计
DataStream<Tuple2<String, Integer>> windowCount = lines.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
public void flatMap(String line, Collector<Tuple2<String, Integer>> collector) throws Exception {
String[] words = line.split(" ");
for (String word : words) {
//将每个单词与 1 组合，形成一个元组
Tuple2<String, Integer> tp = Tuple2.of(word, 1);
//将组成的Tuple放入到 Collector 集合，并输出
collector.collect(tp);
}
}
});
// 1. 滚动窗口（Tumbling Windows）使用例子
//进行分组聚合(keyBy：将key相同的分到一个组中) //定义一个1分钟的翻滚窗口,每分钟统计一次
DataStream<Tuple2<String, Integer>> windowStream = windowCount.keyBy(0)
.timeWindow(Time.minutes(1))
.sum(1);
// 调用Sink （Sink必须调用）
windowStream.print("windows: ").setParallelism(1);
//timePoint+=30;
//启动(这个异常不建议try...catch... 捕获,因为它会抛给上层flink,flink根据异常来做相应的重启策略等处理)
env.execute("StreamWordCount");
}catch (Exception e) {
e.printStackTrace();
}
}
}8、kafka消费配置
static FlinkKafkaConsumer<String> getConsumer() {
Properties props = new Properties();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA_BROKER_LIST);
props.put(ConsumerConfig.GROUP_ID_CONFIG, KAFKA_GROUP_ID);
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true);
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "3000");
// start rebanlance conf
props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "10000");
props.put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, "3000");
props.put(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, "5000");
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");
// consumer ability
props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, "1000");
props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, "2000");
// idle time
props.put(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG, "3000");
props.put(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG, "60000");
props.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, "52428800");
props.put(ConsumerConfig.SOCKET_CONNECTION_SETUP_TIMEOUT_MAX_MS_CONFIG, "30000");
FlinkKafkaConsumer<String> consumer = new FlinkKafkaConsumer<>(KAFKA_TOPIC, new SimpleStringSchema(), props);
consumer.assignTimestampsAndWatermarks(new AscendingTimestampExtractor<String>() {
@Override
public long extractAscendingTimestamp(String s) {
return System.currentTimeMillis();
}
});
return consumer;
}
本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。
