protected PutRecordsResult InnerPutRecordsSupportingCache(PutRecordsRequest putRecordsParam, PutRecordMethod putRecordMethod)
{
if (_disConfig.IsDataCacheEnabled())
{
// 开启本地缓存
PutRecordsResult putRecordsResult = null;
try
{
putRecordsResult = InnerPutRecordsWithRetry(putRecordsParam, putRecordMethod);
}
catch (Exception e)
{
string errorMsg = e.InnerException.Message;
int statusCode = int.Parse(errorMsg.Split('\n')[0]);
// 如果不是可以重试的异常 或者 已达到重试次数，则直接抛出异常
if (Utils.Utils.IsCacheData(statusCode))
{
// 网络异常 全部记录上传失败
logger.Info("Local data cache is enabled, try to put failed records to local.");
CacheUtils.PutToCache(putRecordsParam, _disConfig); // 写入本地缓存
}
throw e;
}
try
{
// 部分记录上传失败
if (putRecordsResult.FailedRecordCount > 0)
{
// 过滤出上传失败的记录
List <PutRecordsResultEntry> putRecordsResultEntries = putRecordsResult.Records;
List <PutRecordsRequestEntry> failedPutRecordsRequestEntries = new List <PutRecordsRequestEntry>();
int index = 0;
foreach (PutRecordsResultEntry putRecordsResultEntry in putRecordsResultEntries)
{
if (!String.IsNullOrEmpty(putRecordsResultEntry.ErrorCode))
{
failedPutRecordsRequestEntries.Add(putRecordsParam.Records[index]);
}
index++;
}
putRecordsParam.Records = failedPutRecordsRequestEntries;
logger.Info("Local data cache is enabled, try to put failed records to local.");
CacheUtils.PutToCache(putRecordsParam, _disConfig); // 写入本地缓存
}
}
catch (Exception e)
{
throw e;
}
return(putRecordsResult);
}
else
{
return(InnerPutRecordsWithRetry(putRecordsParam, putRecordMethod));
}
}
/// <summary>
/// 上传数据
/// </summary>
/// <param name="streamName">通道名称</param>
/// <param name="shardId">分区ID</param>
public static PutRecordsResult RunProducerDemo(string streamName, string shardId)
{
var dic = new DISIngestionClient();
PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
putRecordsRequest.StreamName = streamName;
var putRecordsRequestEntries = new List <PutRecordsRequestEntry>();
for (int i = 0; i < 3; i++)
{
string a = shardId + i;
var putRecordsRequestEntry = new PutRecordsRequestEntry
{
//需要上传的数据
Data = Encoding.UTF8.GetBytes(a),
//用于明确数据需要写入分区的哈希值，此哈希值将覆盖“PartitionKey”的哈希值
//ExplicitHashKey = "0",
//如果传了PartitionId参数，则优先使用PartitionId参数。如果PartitionId没有传，则使用PartitionKey
//PartitionKey = new Random().Next().ToString(),
//分区的唯一标识符
PartitionId = shardId,
};
putRecordsRequestEntries.Add(putRecordsRequestEntry);
}
putRecordsRequest.Records = putRecordsRequestEntries;
PutRecordsResult response = dic.PutRecords(putRecordsRequest);
foreach (var item in response.Records)
{
Console.WriteLine(item);
}
return(response);
}
private async Task <int> UploadFile(string streamName, string file, string fileName)
{
int failedRecordCount = 0;
//最多上传128M的文件，这里先设置为最多上传500k
int splitFileSize = 500 * 1024;
FileInfo fileInfo = new FileInfo(file);
//分几次传
int steps = (int)Math.Ceiling((decimal)fileInfo.Length / splitFileSize);
string deliverDataId = Guid.NewGuid().ToString("N");
using (FileStream fs = new FileStream(file, FileMode.Open, FileAccess.Read))
{
using (BinaryReader br = new BinaryReader(fs))
{
for (int i = 0; i < steps; i++)
{
byte[] input = br.ReadBytes(splitFileSize);
{
PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
putRecordsRequest.StreamName = streamName;
var putRecordsRequestEntries = new List <PutRecordsRequestEntry>();
var putRecordsRequestEntry = new PutRecordsRequestEntry
{
Data = input,
ExtenedInfo = new PutRecordsRequestEntryExtendedInfo()
{
FileName = fileName,
DeliverDataId = deliverDataId,
SequenceNumber = i,
EndFlag = i == steps - 1 ? true : false,
}
};
putRecordsRequestEntries.Add(putRecordsRequestEntry);
putRecordsRequest.Records = putRecordsRequestEntries;
PutRecordsResult response = await this.UploadFile(putRecordsRequest);
failedRecordCount += response.FailedRecordCount;
//Thread.Sleep(500);
foreach (var item in response.Records)
{
Console.WriteLine("异步" + item);
}
}
}
}
}
return(failedRecordCount);
}
/// <summary>
/// 同步方式上传文件
/// </summary>
/// <param name="streamName">通道名称</param>
/// <param name="file">文件名</param>
public static int UploadFileDemo(string streamName, string file)
{
int failedRecordCount = 0;
var dic = new DISIngestionClient();
//最多上传128M的文件，这里先设置为最多上传500k
int splitFileSize = 500 * 1024;
string deliverDataId = Guid.NewGuid().ToString("N");
FileInfo fileInfo = new FileInfo(file);
//分几次传
int steps = (int)Math.Ceiling((decimal)fileInfo.Length / splitFileSize);
using (FileStream fs = new FileStream(file, FileMode.Open, FileAccess.Read))
{
using (BinaryReader br = new BinaryReader(fs))
{
for (int i = 0; i < steps; i++)
{
byte[] input = br.ReadBytes(splitFileSize);
{
PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
putRecordsRequest.StreamName = streamName;
var putRecordsRequestEntries = new List <PutRecordsRequestEntry>();
var putRecordsRequestEntry = new PutRecordsRequestEntry
{
Data = input,
ExtenedInfo = new PutRecordsRequestEntryExtendedInfo()
{
FileName = @"thisisshawnfolder/2018/07/04/14/56/Penguins2.jpg",
DeliverDataId = deliverDataId,
SequenceNumber = i,
EndFlag = i == steps - 1 ? true : false,
}
};
putRecordsRequestEntries.Add(putRecordsRequestEntry);
putRecordsRequest.Records = putRecordsRequestEntries;
PutRecordsResult response = dic.PutFileRecords(putRecordsRequest);
failedRecordCount += response.FailedRecordCount;
//Thread.Sleep(500);
foreach (var item in response.Records)
{
Console.WriteLine("同步" + item);
}
}
}
}
}
return(failedRecordCount);
}
protected PutRecordsResult InnerPutRecordsWithRetry(PutRecordsRequest putRecordsParam, PutRecordMethod putRecordMethod)
{
//数据上传的结果集
PutRecordsResult putRecordsResult = null;
//用该数组来汇总每次请求后的结果
PutRecordsResultEntry[] putRecordsResultEntryList = null;
//记录每次请求失败的下标位置
int[] retryIndex = null;
//每次需要上传的请求数据
PutRecordsRequest retryPutRecordsRequest = putRecordsParam;
int retryCount = -1;
int currentFailed = 0;
ExponentialBackOff backOff = null;
try
{
do
{
retryCount++;
if (retryCount > 0)
{
// 等待一段时间再发起重试
if (backOff == null)
{
Monitor.Enter(objlock);
logger.Info("Put records retry lock.");
backOff = new ExponentialBackOff(ExponentialBackOff.DEFAULT_INITIAL_INTERVAL,
ExponentialBackOff.DEFAULT_MULTIPLIER, _disConfig.GetBackOffMaxIntervalMs(),
ExponentialBackOff.DEFAULT_MAX_ELAPSED_TIME);
}
if (putRecordsResult != null && currentFailed != putRecordsResult.Records.Count)
{
// 部分失败则重置退避时间
backOff.resetCurrentInterval();
}
long sleepMs = backOff.getNextBackOff();
if (retryPutRecordsRequest.Records.Count > 0)
{
logger.DebugFormat(
"Put {0} records but {1} failed, will re-try after backoff {2} ms, current retry count is {3}.",
putRecordsResult != null ? putRecordsResult.Records.Count
: putRecordsParam.Records.Count,
currentFailed,
sleepMs,
retryCount);
}
backOff.backOff(sleepMs);
}
try
{
putRecordsResult = putRecordMethod(retryPutRecordsRequest);
}
catch (Exception t)
{
if (putRecordsResultEntryList != null)
{
logger.Error(t.Message, t);
break;
}
throw t;
}
if (putRecordsResult != null)
{
currentFailed = putRecordsResult.FailedRecordCount;
if (putRecordsResultEntryList == null && currentFailed == 0 || _disConfig.GetRecordsRetries() == 0)
{
// 第一次发送全部成功或者不需要重试，则直接返回结果
return(putRecordsResult);
}
if (putRecordsResultEntryList == null)
{
// 存在发送失败的情况，需要重试，则使用数组来汇总每次请求后的结果。
putRecordsResultEntryList = new PutRecordsResultEntry[putRecordsParam.Records.Count];
}
// 需要重试发送数据的原始下标
List <int> retryIndexTemp = new List <int>(currentFailed);
if (currentFailed > 0)
{
// 初始化重试发送的数据请求
retryPutRecordsRequest = new PutRecordsRequest();
retryPutRecordsRequest.StreamName = putRecordsParam.StreamName;
retryPutRecordsRequest.Records = new List <PutRecordsRequestEntry>(currentFailed);
}
// 对每条结果分析，更新结果数据
for (int i = 0; i < putRecordsResult.Records.Count; i++)
{
// 获取重试数据在原始数据中的下标位置
int originalIndex = retryIndex == null ? i : retryIndex[i];
PutRecordsResultEntry putRecordsResultEntry = putRecordsResult.Records[i];
// 对所有异常进行重试 && "DIS.4303".equals(putRecordsResultEntry.getErrorCode())
if (!string.IsNullOrEmpty(putRecordsResultEntry.ErrorCode))
{
retryIndexTemp.Add(originalIndex);
retryPutRecordsRequest.Records.Add(putRecordsParam.Records[originalIndex]);
}
putRecordsResultEntryList[originalIndex] = putRecordsResultEntry;
}
retryIndex = retryIndexTemp.Count > 0 ? retryIndexTemp.ToArray()
: new int[0];
}
} while ((retryIndex == null || retryIndex.Length > 0) && retryCount < _disConfig.GetRecordsRetries());
}
finally
{
if (retryCount > 0)
{
Monitor.Exit(objlock);
logger.Info("Put records retry unlock.");
}
}
putRecordsResult = new PutRecordsResult();
if (retryIndex == null)
{
// 不可能存在此情况，完全没有发送出去会直接抛出异常
putRecordsResult.FailedRecordCount = putRecordsParam.Records.Count;
}
else
{
putRecordsResult.FailedRecordCount = retryIndex.Length;
putRecordsResult.Records = new List <PutRecordsResultEntry>(putRecordsResultEntryList);
}
return(putRecordsResult);
}
