/// <summary>
        /// 添加DLI转储任务
        /// </summary>
        /// <param name="streamName"></param>
        /// <returns></returns>
        public static ResponseResult CreateTransferTaskWithDLI(string streamName)
        {
            var dic     = new DISIngestionClient();
            var request = new AddTransferTasksRequest
            {
                DestinationType          = "DLI",
                DLIDestinationDescriptor = new DLIDestinationDescriptorEntity
                {
                    //转储任务的名称
                    TaskName = "task_1111",
                    //IAM委托名称
                    AgencyName = "all",
                    //存储该通道数据的DLI数据库名称
                    DLIDatabaseName = "dli",
                    //存储该通道数据的DLI表名称
                    DLITableName = "test1",
                    //偏移量
                    ConsumerStrategy = "LATEST",
                    //桶名称
                    OBSBucketPath = "obs-shawn",
                    //自定义文件
                    FilePrefix = "1012",
                    //数据导入OBS时间,取值范围:30~900
                    DeliverTimeInterval = 30,
                    //重试时间
                    RetryDuration = 10
                }
            };

            ResponseResult response = dic.CreateTransferTask(streamName, request);

            Console.WriteLine(response);
            return(response);
        }
        /// <summary>
        /// 添加OBS转储服务
        /// </summary>
        /// <param name="streamName">通道名称</param>
        /// <returns></returns>
        public static ResponseResult CreateTransferTaskWithOBS(string streamName)
        {
            var dic     = new DISIngestionClient();
            var request = new AddTransferTasksRequest
            {
                DestinationType          = "OBS",
                ObsDestinationDescriptor = new ObsDestinationDescriptorEntity
                {
                    //转储任务的名称
                    TaskName = "task_1234",
                    //通过DestinationFileType可以设置转储文件格式为text、parquet和carbon
                    DestinationFileType = "text",
                    //偏移量
                    ConsumerStrategy = "LATEST",
                    //IAM委托名称
                    AgencyName = "all",
                    //桶名称
                    ObsBucketPath = "obs-shawn",
                    //自定义文件
                    FilePrefix = "1012",
                    //数据导入OBS时间,取值范围:30~900
                    DeliverTimeInterval = 30,
                    //目录层次结构
                    PartitionFormat = "yyyy",
                    //分隔符
                    RecordDelimiter = "\n"
                }
            };

            //创建parquet和carbon的转储任务,必须先创建一个具有源数据Schema的通道,否则无法创建转储任务

            //转储的目标文件格式为parquet,如果需要自定义OBS的时间目录,则必选下面代码片段
            //自定义时间目录,如果不设置,则注释该代码片段
            if (request.ObsDestinationDescriptor.DestinationFileType.Equals("parquet"))
            {
                //根据源数据的时间戳和已配置的"partition_format"生成对应的转储时间目录
                request.ObsDestinationDescriptor.ProcessingSchema = new ProcessingSchema();
                //创建通道时源数据 Schema,Json格式的数据类型的{"key":"value"}转为源数据 Schema格式{"type":"record","name":"RecordName","fields":[{"name":"key","type":"string","doc":"Type inferred from '\"value\"'"}]}
                //TimestampName的值就是Json格式的键值
                request.ObsDestinationDescriptor.ProcessingSchema.TimestampName = "key";
                //源数据时间戳的类型有String、Timestamp
                request.ObsDestinationDescriptor.ProcessingSchema.TimestampType = "String";
                //源数据时间戳的类型为String时必选,用于根据时间戳格式生成OBS的时间目录。取值范围有下面的几种
                // yyyy/MM/dd HH:mm:ss
                // MM/dd/yyyy HH:mm:ss
                // dd/MM/yyyy HH:mm:ss
                // yyyy-MM-dd HH:mm:ss
                // MM-dd-yyyy HH:mm:ss
                // dd-MM-yyyy HH:mm:ss
                request.ObsDestinationDescriptor.ProcessingSchema.TimestampFormat = "yyyy/MM/dd HH:mm:ss";
            }

            ResponseResult response = dic.CreateTransferTask(streamName, request);

            Console.WriteLine(response);
            return(response);
        }
Exemplo n.º 3
0
        public ResponseResult CreateTransferTask(string streamName, AddTransferTasksRequest addTransferTasksRequest)
        {
            ObsWebServiceRequest obsWebServiceRequest = new DISWebServiceRequest();
            IRequest             requestobs           = new DISDefaultRequest(obsWebServiceRequest, Constants.SERVICENAME)
            {
                HttpMethod = HttpMethodName.POST.ToString()
            };

            var resourcePath = ResourcePathBuilder.Standard()
                               .WithProjectId(_disConfig.GetProjectId())
                               .WithResource(new StreamResource(null, streamName))
                               .WithResource(new TransferTaskResource(null))
                               .Build();

            requestobs.ResourcePath = resourcePath;
            var results = this.Request <ResponseResult>(addTransferTasksRequest, requestobs);

            return(results);
        }