public void Setup()
{
var resource = new AccountResource();
var persistentModel = new AccountEntity();
var context = new PutContext <AccountResource, AccountEntity>(resource, new ValidationState())
{
PersistentModel = persistentModel
};
var eTagProvider = Stub <IETagProvider>();
eTagProvider.Stub(x => x.GetETag(null))
.IgnoreArguments()
.Throw(new Exception("Some Fun Exception"));
StubRepository <AccountEntity> repository = New.StubRepository <AccountEntity>()
.ResourceIsNeverCreatedOrModified;
var step = new PersistEntityModel <PutContext <AccountResource, AccountEntity>, PutResult, AccountResource, AccountEntity>(
repository,
eTagProvider);
step.ExecuteAsync(context, _putResult, CancellationToken.None).WaitSafely();
}
// HTTP통신의 완료 콜백(비동기, 다른 스레드 가능성). HTTP통신이 성공해도 Kinesis전송도 성공한 것은 아니다.
private void OnPutRequestCompletedImpl(String error, String uploadResult, PutContext putContext)
{
Boolean success = false;
try
{
if (String.IsNullOrEmpty(error))
{
PostUploadDataCompleted(uploadResult, putContext);
success = true;
}
else
{
_errorCounter.RaiseError(error);
}
}
catch (Exception exception)
{
_errorCounter.RaiseError($"{exception.Message}\n{exception.InnerException?.Message}");
}
finally
{
if (success == false)
{
// 전송은 성공했지만 PostUploadDataCompleted 에서 예외가 발생하면 로그를 다시 보낼 수도 있다.
// 발생 가능성이 낮고 완벽한 보장도 어렵기 때문에 허용한다(조회에서 같은 로그인지 구별할 수 있다).
RetryPutLog(putContext.PutLog, putContext.RetryCount);
}
}
}
private void PostUploadDataCompleted(String result, PutContext putContext)
{
if (0 < putContext.RetryCount)
{
SendSlackRetrySuccess(putContext.RetryCount, putContext.PutLog.RawLogs.Length);
}
var response = JsonConvert.DeserializeObject <ResponsePutRecords>(result);
if (response == null)
{
_errorCounter.RaiseError($"invalid put kinesis response!\nresult: {result}");
return;
}
// <성공, 실패> 로 분할한다.
var successAndFail = SplitSuccessAndFailPutLog(response, putContext);
PutLog successPutLog = successAndFail.Item1;
PutLog failPutLog = successAndFail.Item2;
if (successPutLog != null)
{
_errorCounter.ResetSerialError();
_completePutNotifier.Push(successAndFail.Item1.RawLogs, CompletePutNoticeResultType.Success);
}
if (failPutLog != null)
{
RetryPutLog(successAndFail.Item2, putContext.RetryCount);
if (_config.UseThroughputControl == 1)
{
_throughputController.EnableThrottling(true);
}
_reporter.Info($"Partial Fail PutKinesis Fail/Total(`{response.FailedRecordCount.ToString()}/{response.Records.Count.ToString()}`)");
DebugLog.Log($"Partial Fail PutKinesis (`{response.FailedRecordCount.ToString()}/{response.Records.Count.ToString()}`)", "klogger:putter");
}
else
{
if (_config.UseThroughputControl == 1)
{
_throughputController.EnableThrottling(false);
}
}
UpdateWatcher(response);
DebugLog.Log($"KinesisResponse: RecordCount:{response.Records.Count.ToString()}, FailCount({response.FailedRecordCount.ToString()})", "klogger:putter");
}
public void Setup()
{
var resource = new AccountResource();
var persistentModel = new AccountEntity();
var context = new PutContext <AccountResource, AccountEntity>(resource, new ValidationState())
{
PersistentModel = persistentModel
};
var eTagProvider = Stub <IETagProvider>();
StubRepository <AccountEntity> repository = New.StubRepository <AccountEntity>()
.OnUpsertThrow(new Exception());
var step = new PersistEntityModel <PutContext <AccountResource, AccountEntity>, PutResult, AccountResource, AccountEntity>(
repository,
eTagProvider);
step.ExecuteAsync(context, _putResult, CancellationToken.None).WaitSafely();
}
private void PutInternal(PutContext putContext)
{
try
{
Byte[] post = _kinesisPutAPI.CreatePost(putContext.PutLog.EncodedLogs);
_kinesisPutAPI.Put(post, putContext);
if (_config.UseThroughputControl == 1)
{
_throughputController.UseCapacity(putContext.PutLog.TotalEncodedLogByte, putContext.PutLog.EncodedLogs.Length);
}
}
catch (Exception exception)
{
_errorCounter.RaiseError($"{exception.Message}\n{exception.InnerException?.Message}");
// 여기서 예외는 재시도해도 소용없기 때문에 로그를 바로 버린다.
DropLog(putContext.PutLog, putContext.RetryCount);
}
}
public async Task <PutResult> ProcessAsync(
PutContext <TResourceModel, TEntityModel> context,
CancellationToken cancellationToken)
{
return(await _pipeline.ProcessAsync(context, cancellationToken));
}
// <성공, 실패> 로 분할.
private Tuple <PutLog, PutLog> SplitSuccessAndFailPutLog(ResponsePutRecords response, PutContext putContext)
{
// 모두 성공한 경우.
if (response.FailedRecordCount <= 0)
{
return(new Tuple <PutLog, PutLog>(putContext.PutLog, null));
}
// 모두 실패한 경우.
if (response.Records.Count == response.FailedRecordCount)
{
return(new Tuple <PutLog, PutLog>(null, putContext.PutLog));
}
Int32 totalCount = response.Records.Count;
Int32 successCount = totalCount - response.FailedRecordCount;
Int32 failCount = response.FailedRecordCount;
var successRawLogs = new List <ILog>(successCount);
var successEncodedLogs = new List <Byte[]>(successCount);
Int32 successTotalEncodedLogByte = 0;
var failRawLogs = new List <ILog>(failCount);
var failEncodedLogs = new List <Byte[]>(failCount);
Int32 failTotalEncodedLogByte = 0;
for (Int32 i = 0; i < totalCount; ++i)
{
ILog rawLogs = putContext.PutLog.RawLogs[i];
Byte[] encodedLog = putContext.PutLog.EncodedLogs[i];
// 중요: Kinesis의 응답이 보낸 순서와 일치한다(Kinesis가 보낸 순서와 응답을 맞춰서 준다).
Record record = response.Records[i];
if (String.IsNullOrEmpty(record.ErrorCode))
{
successRawLogs.Add(rawLogs);
successEncodedLogs.Add(encodedLog);
successTotalEncodedLogByte += encodedLog.Length;
}
else
{
failRawLogs.Add(rawLogs);
failEncodedLogs.Add(encodedLog);
failTotalEncodedLogByte += encodedLog.Length;
}
}
var successPutLog = new PutLog(successRawLogs.ToArray(), successEncodedLogs.ToArray(), successTotalEncodedLogByte);
var failPutLog = new PutLog(failRawLogs.ToArray(), failEncodedLogs.ToArray(), failTotalEncodedLogByte);
return(new Tuple <PutLog, PutLog>(successPutLog, failPutLog));
}
