Apache Flink 1.11 流式写入 S3
创始人
2024-09-04 00:32:50
0

在Apache Flink 1.11中,可以使用S3FileSystem作为输出源将数据流写入Amazon S3。下面是一个示例代码:

import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.fs.Path;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.fs.bucketing.BucketingSink;
import org.apache.flink.streaming.connectors.fs.bucketing.DateTimeBucketer;
import org.apache.flink.streaming.connectors.fs.bucketing.StringWriter;
import org.apache.flink.streaming.connectors.fs.bucketing.bucketassigners.DateTimeBucketAssigner;
import org.apache.flink.streaming.connectors.fs.bucketing.bucketassigners.SimpleVersionedStringSerializer;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.BucketingFileWriter;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.DateTimeBucketer;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.Writer;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.buckets.Bucket;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.buckets.BucketFactory;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.CheckpointRollingPolicy;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.DefaultRollingPolicy;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.RollingPolicy;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.partitioner.BasePathPartitioner;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.partitioner.BucketPartitions;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.partitioner.Partitioner;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.partitioner.PathPartitioner;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.partitioner.PrefixPartitioner;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.partitioner.SubtaskIndexPartitioner;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.partitioner.TaskIdPartitioner;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.partitioner.field.FieldExtractor;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.partitioner.field.TimestampExtractor;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.partitioner.field.ValueExtractor;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.partitioner.field.VelocityExtractor;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.trigger.OnCheckpointRollingPolicy;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.trigger.OnProcessingTimeRollingPolicy;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.trigger.RollingPolicyTrigger;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.trigger.RollingPolicyTriggers;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.trigger.OnCheckpointRollingPolicy;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.trigger.OnProcessingTimeRollingPolicy;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.trigger.RollingPolicyTrigger;
import org.apache.flink.streaming.connectors.fs.bucketing.writer.rollingpolicies.trigger.RollingPolicyTriggers;

import java.util.HashMap;
import java.util.Map;

public class S3WriterExample {

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        
        // 设置S3文件系统
        Configuration config = new Configuration();
        config.setString("s3.access.key", "YOUR_ACCESS_KEY");
        config.setString("s3.secret.key", "YOUR_SECRET_KEY");
        config.setString("s3.endpoint", "YOUR_ENDPOINT");
        config.setString("s3.path.style.access", "true");
        config.setString("s3.signer.type", "S3SignerType");
        config.setBoolean("s3.use.experimental.fallback.signer.config", true);
        config.setString("s3.region", "us-west-2");

        FileSystem.initialize(config);

        // 创建一个数据流
        DataStream stream = env.fromElements("data1", "data2", "data3");

        // 创建一个BucketingSink将数据流写入S3
        BucketingSink bucketingSink = new BucketingSink<>("s3://your-bucket/path");
        bucketingSink.setBucketer(new DateTimeBucketer<>("yyyy-MM-dd--HHmm"));
        bucketingSink.setWriter(new StringWriter<>());
        bucketingSink.setBatchSize(1024 *

相关内容

热门资讯

透视工具!wepoker辅助器... 透视工具!wepoker辅助器下载,pokemmo脚本辅助器下载“必备开挂透视挂辅助工具”1、pok...
每日必看教程!游戏茶苑辅助器,... 您好,游戏茶苑辅助器这款游戏可以开挂的,确实是有挂的,需要了解加微【485275054】很多玩家在这...
辅助透视!wepoker辅助器... 辅助透视!wepoker辅助器最新版本更新内容,wepoker透视版下载“关于开挂透视挂辅助神器”1...
总算了解!欢聚水鱼辅助视频,微... 总算了解!欢聚水鱼辅助视频,微信小程序边锋辅助,扑克教程(存在有开挂);亲,有的,ai轻松简单,又可...
透视苹果版!有人wepoker... 透视苹果版!有人wepoker,约局吧德州可以透视“科普开挂透视挂辅助app”;约局吧德州可以透视辅...
重大科普!四川途游小程序辅助破... 重大科普!四川途游小程序辅助破解版,微乐广西麻辣辅助器,科技教程(真的是有开挂);1、点击下载安装,...
透视黑科技!wepoker辅助... 透视黑科技!wepoker辅助真的假的,newpoker可以安装脚本“教你开挂透视挂辅助软件”new...
技术分享!兴动互娱辅助工具,随... 技术分享!兴动互娱辅助工具,随意玩辅助器视频透视挂,wpk教程(是有开挂);1、完成随意玩辅助器视频...
辅助透视!wepoker辅助器... 辅助透视!wepoker辅助器,约局吧可以看有挂“揭幕开挂透视挂辅助教程”1、金币登录送、破产送、升...
实测必看!潮友会鱼虾蟹看穿神器... 实测必看!潮友会鱼虾蟹看穿神器,微信途游有辅助,微扑克教程(真的有开挂);亲真的是有正版授权,小编(...