/// <summary> /// 添加DLI转储任务 /// </summary> /// <param name="streamName"></param> /// <returns></returns> public static ResponseResult CreateTransferTaskWithDLI(string streamName) { var dic = new DISIngestionClient(); var request = new AddTransferTasksRequest { DestinationType = "DLI", DLIDestinationDescriptor = new DLIDestinationDescriptorEntity { //转储任务的名称 TaskName = "task_1111", //IAM委托名称 AgencyName = "all", //存储该通道数据的DLI数据库名称 DLIDatabaseName = "dli", //存储该通道数据的DLI表名称 DLITableName = "test1", //偏移量 ConsumerStrategy = "LATEST", //桶名称 OBSBucketPath = "obs-shawn", //自定义文件 FilePrefix = "1012", //数据导入OBS时间,取值范围:30~900 DeliverTimeInterval = 30, //重试时间 RetryDuration = 10 } }; ResponseResult response = dic.CreateTransferTask(streamName, request); Console.WriteLine(response); return(response); }
/// <summary> /// 添加OBS转储服务 /// </summary> /// <param name="streamName">通道名称</param> /// <returns></returns> public static ResponseResult CreateTransferTaskWithOBS(string streamName) { var dic = new DISIngestionClient(); var request = new AddTransferTasksRequest { DestinationType = "OBS", ObsDestinationDescriptor = new ObsDestinationDescriptorEntity { //转储任务的名称 TaskName = "task_1234", //通过DestinationFileType可以设置转储文件格式为text、parquet和carbon DestinationFileType = "text", //偏移量 ConsumerStrategy = "LATEST", //IAM委托名称 AgencyName = "all", //桶名称 ObsBucketPath = "obs-shawn", //自定义文件 FilePrefix = "1012", //数据导入OBS时间,取值范围:30~900 DeliverTimeInterval = 30, //目录层次结构 PartitionFormat = "yyyy", //分隔符 RecordDelimiter = "\n" } }; //创建parquet和carbon的转储任务,必须先创建一个具有源数据Schema的通道,否则无法创建转储任务 //转储的目标文件格式为parquet,如果需要自定义OBS的时间目录,则必选下面代码片段 //自定义时间目录,如果不设置,则注释该代码片段 if (request.ObsDestinationDescriptor.DestinationFileType.Equals("parquet")) { //根据源数据的时间戳和已配置的"partition_format"生成对应的转储时间目录 request.ObsDestinationDescriptor.ProcessingSchema = new ProcessingSchema(); //创建通道时源数据 Schema,Json格式的数据类型的{"key":"value"}转为源数据 Schema格式{"type":"record","name":"RecordName","fields":[{"name":"key","type":"string","doc":"Type inferred from '\"value\"'"}]} //TimestampName的值就是Json格式的键值 request.ObsDestinationDescriptor.ProcessingSchema.TimestampName = "key"; //源数据时间戳的类型有String、Timestamp request.ObsDestinationDescriptor.ProcessingSchema.TimestampType = "String"; //源数据时间戳的类型为String时必选,用于根据时间戳格式生成OBS的时间目录。取值范围有下面的几种 // yyyy/MM/dd HH:mm:ss // MM/dd/yyyy HH:mm:ss // dd/MM/yyyy HH:mm:ss // yyyy-MM-dd HH:mm:ss // MM-dd-yyyy HH:mm:ss // dd-MM-yyyy HH:mm:ss request.ObsDestinationDescriptor.ProcessingSchema.TimestampFormat = "yyyy/MM/dd HH:mm:ss"; } ResponseResult response = dic.CreateTransferTask(streamName, request); Console.WriteLine(response); return(response); }