private static void SendXml(string eventXml)
{
string counterName = "XML Events Sent";
if (sendInBatches)
{
eventXmlList.EventXmlListItems.Add(new EventXmlItem
{
EventXml = eventXml,
UploadServer = Environment.MachineName,
UploadDateTime = DateTime.UtcNow
});
if (eventXmlList.EventXmlListItems.Count == batchItemCount || DateTime.UtcNow - lastSendDateTime >= MaxLingerTimeSpan)
{
PerformanceCounterManagement(batchItemCount, PerfCounterAction.Increment, counterName);
SendEventXmlList(eventXmlList).ConfigureAwait(false);
eventXmlList = new EventXmlList();
PerformanceCounterManagement(batchItemCount, PerfCounterAction.Decrement, counterName);
lastSendDateTime = DateTime.UtcNow;
}
}
else
{
SendEventXml(eventXml).ConfigureAwait(false);
}
}
private static async Task SendEventXmlList(EventXmlList eventXmlItems)
{
bool uploadSuccessful;
try
{
// Get a random server name from the configured list
string receiverServer = "localhost";
string receiverPort = "5000";
Stopwatch sendStopWatch = Stopwatch.StartNew();
eventXmlItems.UploadServer = Environment.MachineName;
eventXmlItems.UploadDateTime = DateTime.UtcNow;
// Build the client data for the file metrics
try
{
string uri = $"http://{receiverServer}:{receiverPort}/api/EventXmlList";
var parameters = JsonConvert.SerializeObject(eventXmlItems);
if (CompressionType == CompressionType.Compressed)
{
var req = GZipWebClient.GetWebRequest(new Uri(uri));
req.Method = "POST";
req.ContentType = "application/json";
req.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
byte[] bytes = Encoding.ASCII.GetBytes(parameters);
req.ContentLength = bytes.Length;
using (var os = req.GetRequestStream())
{
os.Write(bytes, 0, bytes.Length);
os.Close();
}
using (var str = req.GetResponse().GetResponseStream())
using (var sr = new StreamReader(str))
{
string streamResult = await sr.ReadToEndAsync();
var returnEventXmlList = JsonConvert.DeserializeObject <EventXmlList>(streamResult);
PrintToConsole(
$"Processed {returnEventXmlList.EventXmlListItems.Count} record(s) in {(returnEventXmlList.ProcessingDateTime - returnEventXmlList.UploadDateTime).TotalMilliseconds:N2} milliseconds.");
}
}
else
{
var req = WebRequest.Create(uri);
req.Method = "POST";
req.ContentType = "application/json";
byte[] bytes = Encoding.ASCII.GetBytes(parameters);
req.ContentLength = bytes.Length;
using (var os = req.GetRequestStream())
{
os.Write(bytes, 0, bytes.Length);
os.Close();
}
var stream = req.GetResponse().GetResponseStream();
if (stream != null)
{
using (stream)
using (var sr = new StreamReader(stream))
{
string streamResult = sr.ReadToEnd().Trim();
var returnEventXmlList = JsonConvert.DeserializeObject <EventXmlList>(streamResult);
PrintToConsole(
$"Processed {returnEventXmlList.EventXmlListItems.Count} record(s) in {(returnEventXmlList.ProcessingDateTime - returnEventXmlList.UploadDateTime).TotalMilliseconds:N2} milliseconds.");
}
}
}
}
catch (Exception e)
{
uploadSuccessful = false;
}
sendStopWatch.Stop();
Stopwatch conversionStopwatch = Stopwatch.StartNew();
List <LogRecordSentinel> listLogRecordCdocs =
eventXmlList.EventXmlListItems.Select(d => new PipelineCostConversion().ToLogRecordCdoc(d.EventXml, ConfigHelperObject.ServiceName)).ToList();
conversionStopwatch.Stop();
Stopwatch dataloadStopwatch = Stopwatch.StartNew();
ConfigHelperObject.LoadDataToKusto("EventData", listLogRecordCdocs);
dataloadStopwatch.Stop();
List <SentinelCostMetric> metricList = new List <SentinelCostMetric>
{
new SentinelCostMetric
{
MachineName = Environment.MachineName,
ServiceName = ConfigHelperObject.ServiceName,
OccurenceUtc = DateTime.UtcNow,
PackageGuid = eventXmlItems.EventXmlListItems[0].PackageGuid,
PackageId = eventXmlItems.EventXmlListItems[0].PackageId,
EventType = Enum.GetName(typeof(EventType), EventType.Send),
MetricData = new Dictionary <string, object>()
{
{ "SendType", Enum.GetName(typeof(DataType), dataType) },
{ "Send", ConfigHelperObject.GetStopWatchDictionary(sendStopWatch) },
{ "Conversion", ConfigHelperObject.GetStopWatchDictionary(conversionStopwatch) },
{ "DataLoad", ConfigHelperObject.GetStopWatchDictionary(dataloadStopwatch) }
}
}
};
ConfigHelperObject.LoadMetricsToKusto("Metrics", metricList);
}
catch (Exception ex)
{
ExceptionHandler(ex);
}
}
public async Task <ActionResult <EventXmlList> > PostEventXmlItem(EventXmlList eventXmlList)
{
string serviceName = PipelineCostCommon.GetServiceName();
Stopwatch processingStopwatch = Stopwatch.StartNew();
Stopwatch conversionStopwatch = Stopwatch.StartNew();
List <LogRecordSentinel> listLogRecordCdocs =
eventXmlList.EventXmlListItems.Select(d => new PipelineCostConversion().ToLogRecordCdoc(d.EventXml, configHelper.ServiceName)).ToList();
conversionStopwatch.Stop();
// Update each record with data
foreach (EventXmlItem xmlItem in eventXmlList.EventXmlListItems)
{
eventsProcessed++;
xmlItem.ProcessingDateTime = DateTime.UtcNow;
xmlItem.ProcessingServer = Environment.MachineName;
}
PerformanceCounter xmlEventsReceived = new PerformanceCounter("WECEvents", "XML Events Received");
xmlEventsReceived.ReadOnly = false;
xmlEventsReceived.IncrementBy(eventsProcessed);
//_context.EventxmlItems.AddRange(eventxmlList.EventxmlListItems);
//await _context.SaveChangesAsync();
processingStopwatch.Stop();
Stopwatch dataloadStopwatch = Stopwatch.StartNew();
configHelper.LoadDataToKusto("EventData", listLogRecordCdocs);
dataloadStopwatch.Stop();
PerformanceCounter xmlParsingEfficiency = new PerformanceCounter("WECEvents", "XML Parsing Efficiency");
xmlParsingEfficiency.ReadOnly = false;
xmlParsingEfficiency.RawValue = conversionStopwatch.Elapsed.Milliseconds;
// Kusto upload metric in MS
PerformanceCounter xmlUploadEfficiency = new PerformanceCounter("WECEvents", "XML Upload Efficiency");
xmlUploadEfficiency.ReadOnly = false;
xmlUploadEfficiency.RawValue = dataloadStopwatch.Elapsed.Milliseconds;
// Decrement Event Counter
xmlEventsReceived.IncrementBy(-1 * eventsProcessed);
// Set return values
eventXmlList.ProcessingDateTime = DateTime.UtcNow;
eventXmlList.ProcessingServer = Environment.MachineName;
List <SentinelCostMetric> metricList = new List <SentinelCostMetric>
{
new SentinelCostMetric
{
MachineName = Environment.MachineName,
ServiceName = configHelper.ServiceName,
OccurenceUtc = DateTime.UtcNow,
PackageGuid = eventXmlList.EventXmlListItems[0].PackageGuid,
PackageId = eventXmlList.EventXmlListItems[0].PackageId,
EventType = Enum.GetName(typeof(EventType), EventType.Receive),
MetricData = new Dictionary <string, object>()
{
{ "DataType", Enum.GetName(typeof(DataType), dataType) },
{ "ProcessTime", configHelper.GetStopWatchDictionary(processingStopwatch) },
{ "DataLoadTime", configHelper.GetStopWatchDictionary(dataloadStopwatch) },
}
}
};
configHelper.LoadMetricsToKusto("Metrics", metricList);
return(CreatedAtAction("GetEventXmlItem", new EventXmlItem {
PackageId = eventXmlList.PackageId
}, JsonConvert.SerializeObject(eventXmlList)));
}
