/// <summary>
/// Emit a batch of log events, running to completion asynchronously.
/// </summary>
/// <param name="events">The events to be logged to Kinesis</param>
protected override void EmitBatch(IEnumerable<LogEvent> events)
{
var request = new PutRecordsRequest
{
StreamName = _state.Options.StreamName
};
foreach (var logEvent in events)
{
var json = new StringWriter();
_state.Formatter.Format(logEvent, json);
var bytes = Encoding.UTF8.GetBytes(json.ToString());
var entry = new PutRecordsRequestEntry
{
PartitionKey = Guid.NewGuid().ToString(),
Data = new MemoryStream(bytes),
};
request.Records.Add(entry);
}
_state.KinesisClient.PutRecords(request);
}
private async Task InternalPublish(IEnumerable <T> dataList, CancellationToken cancellationToken = default)
{
if (dataList == null)
{
throw new ArgumentNullException(nameof(dataList));
}
var dataStreams = dataList.ToList();
if (dataStreams.Count() > 500)
{
throw new InvalidEventLimitException();
}
Logger.Info("Kinesis Publisher Started");
var request = new Amazon.Kinesis.Model.PutRecordsRequest()
{
StreamName = BusName,
Records = new List <PutRecordsRequestEntry>()
};
foreach (var data in dataStreams)
{
if (data.TraceId.IsNullOrEmpty())
{
data.TraceId = base.GetTraceId();
}
var dataAsBytes = Encoding.UTF8.GetBytes(System.Text.Json.JsonSerializer.Serialize <object>(data));
await using var ms = new MemoryStream(dataAsBytes);
request.Records.Add(new PutRecordsRequestEntry()
{
Data = ms,
PartitionKey = data.Partition
});
}
Logger.Info("Publishing Data for Bus @BusName", BusName);
var policy = base.CreateDefaultRetryAsyncPolicy();
var results = await policy.ExecuteAsync(async() => await KinesisClient.PutRecordsAsync(request, cancellationToken));
if (results.FailedRecordCount == 0)
{
Logger.Info("All data published to Bus @BusName", BusName);
return;
}
var errorRecords = results.Records.Where(r => r.ErrorCode != null);
foreach (var error in errorRecords)
{
Logger.Error("Error publishing message. Error: @ErrorCode, ErrorMessage: @ErrorMessage ", error.ErrorCode, error.ErrorMessage);
}
}
public object Execute(ExecutorContext context)
{
var cmdletContext = context as CmdletContext;
// create request
var request = new Amazon.Kinesis.Model.PutRecordsRequest();
if (cmdletContext.Record != null)
{
request.Records = cmdletContext.Record;
}
if (cmdletContext.StreamName != null)
{
request.StreamName = cmdletContext.StreamName;
}
CmdletOutput output;
// issue call
var client = Client ?? CreateClient(_CurrentCredentials, _RegionEndpoint);
try
{
var response = CallAWSServiceOperation(client, request);
object pipelineOutput = null;
pipelineOutput = cmdletContext.Select(response, this);
output = new CmdletOutput
{
PipelineOutput = pipelineOutput,
ServiceResponse = response
};
}
catch (Exception e)
{
output = new CmdletOutput {
ErrorResponse = e
};
}
return(output);
}
/// <summary>
/// Initiates the asynchronous execution of the PutRecords operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the PutRecords operation.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
public Task<PutRecordsResponse> PutRecordsAsync(PutRecordsRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = new PutRecordsRequestMarshaller();
var unmarshaller = PutRecordsResponseUnmarshaller.Instance;
return InvokeAsync<PutRecordsRequest,PutRecordsResponse>(request, marshaller,
unmarshaller, cancellationToken);
}
internal PutRecordsResponse PutRecords(PutRecordsRequest request)
{
var marshaller = new PutRecordsRequestMarshaller();
var unmarshaller = PutRecordsResponseUnmarshaller.Instance;
return Invoke<PutRecordsRequest,PutRecordsResponse>(request, marshaller, unmarshaller);
}
/// <summary>
/// Initiates the asynchronous execution of the PutRecords operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the PutRecords operation on AmazonKinesisClient.</param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
///
/// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndPutRecords
/// operation.</returns>
public IAsyncResult BeginPutRecords(PutRecordsRequest request, AsyncCallback callback, object state)
{
var marshaller = new PutRecordsRequestMarshaller();
var unmarshaller = PutRecordsResponseUnmarshaller.Instance;
return BeginInvoke<PutRecordsRequest>(request, marshaller, unmarshaller,
callback, state);
}
/// <summary>
/// Initiates the asynchronous execution of the PutRecords operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the PutRecords operation on AmazonKinesisClient.</param>
/// <param name="callback">An Action delegate that is invoked when the operation completes.</param>
/// <param name="options">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
public void PutRecordsAsync(PutRecordsRequest request, AmazonServiceCallback<PutRecordsRequest, PutRecordsResponse> callback, AsyncOptions options = null)
{
options = options == null?new AsyncOptions():options;
var marshaller = new PutRecordsRequestMarshaller();
var unmarshaller = PutRecordsResponseUnmarshaller.Instance;
Action<AmazonWebServiceRequest, AmazonWebServiceResponse, Exception, AsyncOptions> callbackHelper = null;
if(callback !=null )
callbackHelper = (AmazonWebServiceRequest req, AmazonWebServiceResponse res, Exception ex, AsyncOptions ao) => {
AmazonServiceResult<PutRecordsRequest,PutRecordsResponse> responseObject
= new AmazonServiceResult<PutRecordsRequest,PutRecordsResponse>((PutRecordsRequest)req, (PutRecordsResponse)res, ex , ao.State);
callback(responseObject);
};
BeginInvoke<PutRecordsRequest>(request, marshaller, unmarshaller, options, callbackHelper);
}
private Amazon.Kinesis.Model.PutRecordsResponse CallAWSServiceOperation(IAmazonKinesis client, Amazon.Kinesis.Model.PutRecordsRequest request)
{
Utils.Common.WriteVerboseEndpointMessage(this, client.Config, "Amazon Kinesis", "PutRecords");
try
{
#if DESKTOP
return(client.PutRecords(request));
#elif CORECLR
return(client.PutRecordsAsync(request).GetAwaiter().GetResult());
#else
#error "Unknown build edition"
#endif
}
catch (AmazonServiceException exc)
{
var webException = exc.InnerException as System.Net.WebException;
if (webException != null)
{
throw new Exception(Utils.Common.FormatNameResolutionFailureMessage(client.Config, webException.Message), webException);
}
throw;
}
}