Flink learning

Time:2021-9-8

Flink build

Environment: centos7 docker
1.docker pull flink
2. Create the file docker-compose.yml as follows:

version: "2.1"
services:
  jobmanager:
    image: flink:1.9.2-scala_2.12
    expose:
      - "6123"
    ports:
      - "8081:8081"
    command: jobmanager
    environment:
      - JOB_MANAGER_RPC_ADDRESS=jobmanager
  taskmanager:
    image: flink:1.9.2-scala_2.12
    expose:
      - "6121"
      - "6122"
    depends_on:
      - jobmanager
    command: taskmanager
    links:
      - "jobmanager:jobmanager"
    environment:
      - JOB_MANAGER_RPC_ADDRESS=jobmanager

3. Execute the command docker compose up – D in the file directory
4. Host IP + 8081 to check whether the console displays
Flink learning

Flynk project construction

1. Maven creates a flick project

$ mvn archetype:generate                               
      -DarchetypeGroupId=org.apache.flink              
      -DarchetypeArtifactId=flink-quickstart-java      
      -DarchetypeVersion=1.13.1

2. Add rabbitmq dependency to pom.xml file as follows

<dependencies>
        <!-- Apache Flink dependencies -->
        <!-- These dependencies are provided, because they should not be packaged into the JAR file. -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>${flink.version}</version>
<!--            <scope>provided</scope>-->
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
<!--            <scope>provided</scope>-->
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
<!--            <scope>provided</scope>-->
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-rabbitmq_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <!-- Add connector dependencies here. They must be in the default scope (compile). -->

        <!-- Example:

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        -->

        <!-- Add logging framework, to produce console output when running in the IDE. -->
        <!-- These dependencies are excluded from the application JAR by default. -->
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-slf4j-impl</artifactId>
            <version>${log4j.version}</version>
<!--            <scope>runtime</scope>-->
        </dependency>
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-api</artifactId>
            <version>${log4j.version}</version>
<!--            <scope>runtime</scope>-->
        </dependency>
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-core</artifactId>
            <version>${log4j.version}</version>
<!--            <scope>runtime</scope>-->
        </dependency>
    </dependencies>

3.WordCountJob

public class WordCountJob {
    public static void main(String[] args) throws Exception {
        //1. Get the running environment of Flink
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        RMQConnectionConfig connectionConfig = new RMQConnectionConfig.Builder()
                .setHost("1.117.78.150")
                .setPort(5672)
                .setUserName("guest")
                .setPassword("guest")
                .setVirtualHost("/")
                .build();
        //2. Connect socket to obtain input data (data source)
        //2.1. Rabbitmq connection configuration, 2. Rabbitmq queue name, consumed queue name
        DataStream<String> dataStreamSource = env.addSource(new RMQSource<String>(connectionConfig,
                "test.flink",true, new SimpleStringSchema()));
        dataStreamSource.print();   // Output source information


        //3. Data conversion
        //Mapfunction: the first parameter is the type of data you receive
        //Mapfunction: the second parameter is the type of returned data
        DataStream<String> MessageVo = dataStreamSource.map(
                (MapFunction<String, String>) s -> s
        );
        MessageVo.addSink(new MySink());
        //5. This line of code must be implemented, otherwise the program will not be executed
        env.execute("liu er gou");

//        SpringApplication.run(PaymentFlinkApplication.class, args);
    }
}

4.MySink

public class MySink extends RichSinkFunction<String> {

    private static final Logger log = LoggerFactory.getLogger(MySink.class);


    @Override
    public void invoke(String value, Context context) throws Exception {
        log.info(value);
    }

    @Override
    public void close() throws Exception {
        log.info("MySink close");
    }
}

5. Maven install get the jar package
Flink learning
6. Open the Flink console to upload the jar package and run the wordcountjob class. No error indicates that the operation is successful. At the same time, the console will have a corresponding job
Flink learning
Flink learning
7. The rabbit queue sends messages to check whether the program is successful. The Flink console will display
Flink learning