/// <summary>
/// 添加DLI转储任务
/// </summary>
/// <param name="streamName"></param>
/// <returns></returns>
public static ResponseResult CreateTransferTaskWithDLI(string streamName)
{
var dic = new DISIngestionClient();
var request = new AddTransferTasksRequest
{
DestinationType = "DLI",
DLIDestinationDescriptor = new DLIDestinationDescriptorEntity
{
//转储任务的名称
TaskName = "task_1111",
//IAM委托名称
AgencyName = "all",
//存储该通道数据的DLI数据库名称
DLIDatabaseName = "dli",
//存储该通道数据的DLI表名称
DLITableName = "test1",
//偏移量
ConsumerStrategy = "LATEST",
//桶名称
OBSBucketPath = "obs-shawn",
//自定义文件
FilePrefix = "1012",
//数据导入OBS时间,取值范围：30～900
DeliverTimeInterval = 30,
//重试时间
RetryDuration = 10
}
};
ResponseResult response = dic.CreateTransferTask(streamName, request);
Console.WriteLine(response);
return(response);
}
/// <summary>
/// 添加OBS转储服务
/// </summary>
/// <param name="streamName">通道名称</param>
/// <returns></returns>
public static ResponseResult CreateTransferTaskWithOBS(string streamName)
{
var dic = new DISIngestionClient();
var request = new AddTransferTasksRequest
{
DestinationType = "OBS",
ObsDestinationDescriptor = new ObsDestinationDescriptorEntity
{
//转储任务的名称
TaskName = "task_1234",
//通过DestinationFileType可以设置转储文件格式为text、parquet和carbon
DestinationFileType = "text",
//偏移量
ConsumerStrategy = "LATEST",
//IAM委托名称
AgencyName = "all",
//桶名称
ObsBucketPath = "obs-shawn",
//自定义文件
FilePrefix = "1012",
//数据导入OBS时间,取值范围：30～900
DeliverTimeInterval = 30,
//目录层次结构
PartitionFormat = "yyyy",
//分隔符
RecordDelimiter = "\n"
}
};
//创建parquet和carbon的转储任务，必须先创建一个具有源数据Schema的通道，否则无法创建转储任务
//转储的目标文件格式为parquet，如果需要自定义OBS的时间目录，则必选下面代码片段
//自定义时间目录，如果不设置，则注释该代码片段
if (request.ObsDestinationDescriptor.DestinationFileType.Equals("parquet"))
{
//根据源数据的时间戳和已配置的"partition_format"生成对应的转储时间目录
request.ObsDestinationDescriptor.ProcessingSchema = new ProcessingSchema();
//创建通道时源数据 Schema，Json格式的数据类型的{"key":"value"}转为源数据 Schema格式{"type":"record","name":"RecordName","fields":[{"name":"key","type":"string","doc":"Type inferred from '\"value\"'"}]}
//TimestampName的值就是Json格式的键值
request.ObsDestinationDescriptor.ProcessingSchema.TimestampName = "key";
//源数据时间戳的类型有String、Timestamp
request.ObsDestinationDescriptor.ProcessingSchema.TimestampType = "String";
//源数据时间戳的类型为String时必选，用于根据时间戳格式生成OBS的时间目录。取值范围有下面的几种
// yyyy/MM/dd HH:mm:ss
// MM/dd/yyyy HH:mm:ss
// dd/MM/yyyy HH:mm:ss
// yyyy-MM-dd HH:mm:ss
// MM-dd-yyyy HH:mm:ss
// dd-MM-yyyy HH:mm:ss
request.ObsDestinationDescriptor.ProcessingSchema.TimestampFormat = "yyyy/MM/dd HH:mm:ss";
}
ResponseResult response = dic.CreateTransferTask(streamName, request);
Console.WriteLine(response);
return(response);
}
public ResponseResult CreateTransferTask(string streamName, AddTransferTasksRequest addTransferTasksRequest)
{
ObsWebServiceRequest obsWebServiceRequest = new DISWebServiceRequest();
IRequest requestobs = new DISDefaultRequest(obsWebServiceRequest, Constants.SERVICENAME)
{
HttpMethod = HttpMethodName.POST.ToString()
};
var resourcePath = ResourcePathBuilder.Standard()
.WithProjectId(_disConfig.GetProjectId())
.WithResource(new StreamResource(null, streamName))
.WithResource(new TransferTaskResource(null))
.Build();
requestobs.ResourcePath = resourcePath;
var results = this.Request <ResponseResult>(addTransferTasksRequest, requestobs);
return(results);
}
