// 이전 틱에 보내지 못했던 로그를 분할해서라도 보낸다.
private void PutFragmentRemainPutLog()
{
if (_remainPutLogs.Count <= 0)
{
return;
}
var nextRemainPutLogs = new List <PutLog>();
foreach (PutLog putLog in _remainPutLogs)
{
Tuple <PutLog, PutLog> usableAndNotUsable = SplitUsableAndNotUsablePutLog(putLog);
PutLog usablePutLog = usableAndNotUsable.Item1;
PutLog notUsablePutLog = usableAndNotUsable.Item2;
if (0 < usablePutLog.TotalEncodedLogByte)
{
DebugLog.Log($"{nameof(ThroughputControl)} FragmentPut: {usablePutLog.TotalEncodedLogByte}, {usablePutLog.EncodedLogs.Length}", "klogger:throughput-control");
Put(usablePutLog);
}
if (0 < notUsablePutLog.TotalEncodedLogByte)
{
// 이번에도 보내지 못한 로그는 다음에 다시 시도한다.
nextRemainPutLogs.Add(notUsablePutLog);
}
}
_remainPutLogs = nextRemainPutLogs;
}
// 로그를 분할하지 않고 최대한 보낸다.
private void PutNoFragment()
{
Int32 loopCount = _putLogs.Count;
for (Int32 i = 0; i < loopCount; ++i)
{
PutLog putLog = _putLogs.Pop();
if (putLog == null)
{
DebugLog.Log($"{nameof(PutNoFragment)} PutLog is null(Error).", "klogger:throughput-control");
return; // 외부 스레드에서 빼내지 않는한 이럴 수 없음.
}
if (CheckThroughputCondition() &&
CheckRemainCapacity(putLog.TotalEncodedLogByte, putLog.EncodedLogs.Length))
{
DebugLog.Log($"{nameof(ThroughputControl)} No FragmentPut: {putLog.TotalEncodedLogByte}, {putLog.EncodedLogs.Length}", "klogger:throughput-control");
Put(putLog);
}
else
{
// 다음 틱에 분할해서라도 보내게 된다.
AddRemainPutLog(putLog);
}
}
}
private Boolean CheckPutLog(PutLog putLog)
{
if (putLog == null)
{
_reporter.Warn($"{nameof(putLog)} is null!");
return(false);
}
if (putLog.RawLogs == null)
{
_reporter.Warn($"{nameof(putLog.RawLogs)} is null!");
return(false);
}
if (putLog.EncodedLogs == null)
{
_reporter.Warn($"{nameof(putLog.EncodedLogs)} is null!");
return(false);
}
if (putLog.TotalEncodedLogByte <= 0)
{
_reporter.Warn($"{nameof(putLog.TotalEncodedLogByte)}({putLog.TotalEncodedLogByte}) <= 0");
return(false);
}
if (putLog.RawLogs.Length != putLog.EncodedLogs.Length)
{
_reporter.Warn($"{nameof(putLog.RawLogs.Length)}({putLog.RawLogs.Length}) != {nameof(putLog.EncodedLogs.Length)}({putLog.EncodedLogs.Length})");
return(false);
}
return(true);
}
private void SleepAndPut(PutLog putLog, Int32 retryCount)
{
Thread.Sleep(Math.Min(MAX_RETRY_RETRY_SLEEP_MS, RETRY_SLEEP_MS * retryCount));
Put(putLog, retryCount);
DebugLog.Log($"SleepAndPut. RetryCount: {retryCount.ToString()}", "klogger:putter");
}
internal void Put(PutLog putLog, Int32 retryCount = 0)
{
if (CheckPutLog(putLog) == false)
{
return;
}
PutInternal(new PutContext(putLog, retryCount));
}
public Bean NewestValue()
{
PutLog log = (PutLog)Current.GetLog(ObjectId);
if (null != log)
{
return(log.Value);
}
return(OriginRecord.Value);
}
private void RetryPutLog(PutLog putLog, Int32 retryCount)
{
if (_config.MaxRetrySendCount <= retryCount)
{
DropLog(putLog, retryCount);
return;
}
// 대부분 ProvisionedThroughputExceededException 오류이므로 기다렸다 재시도한다.
ThreadPool.QueueUserWorkItem(_ => SleepAndPut(putLog, retryCount + 1));
}
// 스레드로컬에 복사한 상태이므로 락 바깥에서 호출할 수 있다.
private void ProcessThreadLocal()
{
DebugLog.Log($"ProcessThreadLocal: {_threadLocalLogs.Value.Count.ToString()}", "klogger:tick");
PutLog putLog = ConvertThreadLocalLogsToPutLogs();
PutKinesis(putLog);
UpdateWatcherCounter();
ClearAllThreadLocal();
}
// <성공, 실패> 로 분할.
private Tuple <PutLog, PutLog> SplitSuccessAndFailPutLog(ResponsePutRecords response, PutContext putContext)
{
// 모두 성공한 경우.
if (response.FailedRecordCount <= 0)
{
return(new Tuple <PutLog, PutLog>(putContext.PutLog, null));
}
// 모두 실패한 경우.
if (response.Records.Count == response.FailedRecordCount)
{
return(new Tuple <PutLog, PutLog>(null, putContext.PutLog));
}
Int32 totalCount = response.Records.Count;
Int32 successCount = totalCount - response.FailedRecordCount;
Int32 failCount = response.FailedRecordCount;
var successRawLogs = new List <ILog>(successCount);
var successEncodedLogs = new List <Byte[]>(successCount);
Int32 successTotalEncodedLogByte = 0;
var failRawLogs = new List <ILog>(failCount);
var failEncodedLogs = new List <Byte[]>(failCount);
Int32 failTotalEncodedLogByte = 0;
for (Int32 i = 0; i < totalCount; ++i)
{
ILog rawLogs = putContext.PutLog.RawLogs[i];
Byte[] encodedLog = putContext.PutLog.EncodedLogs[i];
// 중요: Kinesis의 응답이 보낸 순서와 일치한다(Kinesis가 보낸 순서와 응답을 맞춰서 준다).
Record record = response.Records[i];
if (String.IsNullOrEmpty(record.ErrorCode))
{
successRawLogs.Add(rawLogs);
successEncodedLogs.Add(encodedLog);
successTotalEncodedLogByte += encodedLog.Length;
}
else
{
failRawLogs.Add(rawLogs);
failEncodedLogs.Add(encodedLog);
failTotalEncodedLogByte += encodedLog.Length;
}
}
var successPutLog = new PutLog(successRawLogs.ToArray(), successEncodedLogs.ToArray(), successTotalEncodedLogByte);
var failPutLog = new PutLog(failRawLogs.ToArray(), failEncodedLogs.ToArray(), failTotalEncodedLogByte);
return(new Tuple <PutLog, PutLog>(successPutLog, failPutLog));
}
private void PostUploadDataCompleted(String result, PutContext putContext)
{
if (0 < putContext.RetryCount)
{
SendSlackRetrySuccess(putContext.RetryCount, putContext.PutLog.RawLogs.Length);
}
var response = JsonConvert.DeserializeObject <ResponsePutRecords>(result);
if (response == null)
{
_errorCounter.RaiseError($"invalid put kinesis response!\nresult: {result}");
return;
}
// <성공, 실패> 로 분할한다.
var successAndFail = SplitSuccessAndFailPutLog(response, putContext);
PutLog successPutLog = successAndFail.Item1;
PutLog failPutLog = successAndFail.Item2;
if (successPutLog != null)
{
_errorCounter.ResetSerialError();
_completePutNotifier.Push(successAndFail.Item1.RawLogs, CompletePutNoticeResultType.Success);
}
if (failPutLog != null)
{
RetryPutLog(successAndFail.Item2, putContext.RetryCount);
if (_config.UseThroughputControl == 1)
{
_throughputController.EnableThrottling(true);
}
_reporter.Info($"Partial Fail PutKinesis Fail/Total(`{response.FailedRecordCount.ToString()}/{response.Records.Count.ToString()}`)");
DebugLog.Log($"Partial Fail PutKinesis (`{response.FailedRecordCount.ToString()}/{response.Records.Count.ToString()}`)", "klogger:putter");
}
else
{
if (_config.UseThroughputControl == 1)
{
_throughputController.EnableThrottling(false);
}
}
UpdateWatcher(response);
DebugLog.Log($"KinesisResponse: RecordCount:{response.Records.Count.ToString()}, FailCount({response.FailedRecordCount.ToString()})", "klogger:putter");
}
private void DropLog(PutLog putLog, Int32 retryCount)
{
Boolean success = _completePutNotifier.Push(putLog.RawLogs, CompletePutNoticeResultType.FailRetry);
if (success == false)
{
_reporter.Error($"Fail CompletePutNotifier.Push ({putLog.RawLogs.Length})");
}
_watcher.DropLog(putLog.RawLogs.Length);
_reporter.Error($"Drop Log: {putLog.RawLogs.Length}, RetryCount: {retryCount}");
DebugLog.Log($"Drop Log({putLog.EncodedLogs.Length}, {putLog.TotalEncodedLogByte}).", "klogger:putter");
}
// 보낼 수 있는 로그와 보낼 수 없는 로그로 나눈다.
private Tuple <PutLog, PutLog> SplitUsableAndNotUsablePutLog(PutLog putLog)
{
var usableRawLogs = new List <ILog>();
var usableEncodedLogs = new List <Byte[]>();
Int32 usableTotalEncodedLogByte = 0;
var notUsableRawLogs = new List <ILog>();
var notUsableEncodedLogs = new List <Byte[]>();
Int32 notUsableTotalEncodedLogByte = 0;
Int64 useByteCapacity = 0;
Int32 useRecordCapacity = 0;
for (Int32 i = 0; i < putLog.EncodedLogs.Length; ++i)
{
ILog rawLog = putLog.RawLogs[i];
Byte[] encodedLog = putLog.EncodedLogs[i];
if (IsValidLogSize(encodedLog.Length) == false)
{
continue; // 로그를 버린다.
}
if (CheckThroughputCondition() &&
CheckRemainCapacity(useByteCapacity + encodedLog.Length, useRecordCapacity + 1) &&
CheckMaxKinesisBatchSize(usableRawLogs.Count))
{
useByteCapacity += encodedLog.Length;
++useRecordCapacity;
usableRawLogs.Add(rawLog);
usableEncodedLogs.Add(encodedLog);
usableTotalEncodedLogByte += encodedLog.Length;
}
else
{
notUsableRawLogs.Add(rawLog);
notUsableEncodedLogs.Add(encodedLog);
notUsableTotalEncodedLogByte += encodedLog.Length;
}
}
var usablePutLog = new PutLog(usableRawLogs.ToArray(), usableEncodedLogs.ToArray(), usableTotalEncodedLogByte);
var notUsablePutLog = new PutLog(notUsableRawLogs.ToArray(), notUsableEncodedLogs.ToArray(), notUsableTotalEncodedLogByte);
return(new Tuple <PutLog, PutLog>(usablePutLog, notUsablePutLog));
}
private void Flush()
{
while (true)
{
PutLog putLog = _putLogs.Pop();
if (putLog == null)
{
break;
}
Put(putLog);
}
foreach (PutLog reservedPutLog in _remainPutLogs)
{
Put(reservedPutLog);
}
_remainPutLogs.Clear();
}
private void PutKinesis(PutLog putLog)
{
if (putLog == null || putLog.RawLogs.Length <= 0 || putLog.EncodedLogs.Length <= 0)
{
return;
}
if (State == StateType.Stopping)
{
Putter.Put(putLog);
return;
}
if (Config.UseThroughputControl == 1)
{
ThroughputController.Push(putLog);
}
else
{
Putter.Put(putLog);
}
}
internal void Push(PutLog putLog)
{
_putLogs.Push(putLog);
DebugLog.Log($"{nameof(ThroughputControl)} Push: {putLog.TotalEncodedLogByte}, {putLog.EncodedLogs.Length}", "klogger:throughput-control");
}
private void AddRemainPutLog(PutLog putLog)
{
_remainPutLogs.Add(putLog);
}
private void Put(PutLog putLog)
{
_putter.Put(putLog);
}
