/// <summary>
/// Emit a batch of log events, running to completion synchronously.
/// </summary>
/// <param name="events">The events to emit.</param>
protected override void EmitBatch(IEnumerable <LogEvent> events)
{
var batchedEvents = new List <EventData>();
var batchPartitionKey = Guid.NewGuid().ToString();
// Possible optimizations for the below:
// 1. Reuse a StringWriter object for the whole batch, or possibly across batches.
// 2. Reuse byte[] buffers instead of reallocating every time.
foreach (var logEvent in events)
{
byte[] body;
using (var render = new StringWriter())
{
_formatter.Format(logEvent, render);
body = Encoding.UTF8.GetBytes(render.ToString());
}
var eventHubData = new EventData(body)
{
PartitionKey = batchPartitionKey
};
_eventDataAction?.Invoke(eventHubData, logEvent);
batchedEvents.Add(eventHubData);
}
using (var transaction = new TransactionScope(TransactionScopeOption.Suppress))
{
_eventHubClient.SendBatch(batchedEvents);
}
}
private void SendBatchAsOneChunk(IEnumerable <EventData> batchedEvents)
{
using (new TransactionScope(TransactionScopeOption.Suppress))
{
_eventHubClient.SendBatch(batchedEvents);
}
}
private void SendDeviceEventStream(EventHubClient eventHubClient)
{
var allEvents = new List<EventData>();
for (int i = 0; i < DeviceSendingDetails.NumberOfDevices; i++)
{
string deviceName = "device" + i;
var rand = new Random();
// set up the modifier to enable
float modifier = 1.0F;
if (DeviceSendingDetails.FailureConditions.Any(device => device.FailedDeviceId == i))
{
var deviceDetails = DeviceSendingDetails.FailureConditions.First(device => device.FailedDeviceId == i);
modifier = modifier += deviceDetails.FailedDeviceGradient;
}
var deviceValue = rand.Next((int) ((DeviceSendingDetails.TemperatureMin * modifier)*100),
(int) ((DeviceSendingDetails.TemperatureMax * modifier)*100));
var deviceData = new DeviceData()
{
deviceid = deviceName,
temperature = (deviceValue/100F),
timestamp = DateTime.UtcNow
};
var jsonDeviceDetail = JsonConvert.SerializeObject(deviceData);
var encodedPayload = Encoding.UTF8.GetBytes(jsonDeviceDetail);
var eventData = new EventData(encodedPayload)
{
PartitionKey = "devices"
};
allEvents.Add(eventData);
}
eventHubClient.SendBatch(allEvents);
}
private void SendDeviceEventStream(EventHubClient eventHubClient)
{
var allEvents = new List <EventData>();
for (int i = 0; i < DeviceSendingDetails.NumberOfDevices; i++)
{
string deviceName = "device" + i;
var rand = new Random();
// set up the modifier to enable
float modifier = 1.0F;
if (DeviceSendingDetails.FailureConditions.Any(device => device.FailedDeviceId == i))
{
var deviceDetails = DeviceSendingDetails.FailureConditions.First(device => device.FailedDeviceId == i);
modifier = modifier += deviceDetails.FailedDeviceGradient;
}
var deviceValue = rand.Next((int)((DeviceSendingDetails.TemperatureMin * modifier) * 100),
(int)((DeviceSendingDetails.TemperatureMax * modifier) * 100));
var deviceData = new DeviceData()
{
deviceid = deviceName,
temperature = (deviceValue / 100F),
timestamp = DateTime.UtcNow
};
var jsonDeviceDetail = JsonConvert.SerializeObject(deviceData);
var encodedPayload = Encoding.UTF8.GetBytes(jsonDeviceDetail);
var eventData = new EventData(encodedPayload)
{
PartitionKey = "devices"
};
allEvents.Add(eventData);
}
eventHubClient.SendBatch(allEvents);
}
void FlushEventHubBuffer()
{
if (sendBuffer.Count > 0)
{
eventHubClient.SendBatch(sendBuffer);
numEventsSent += sendBuffer.Count;
sendBuffer.Clear();
bufferedSizeInBytes = 0;
}
}
private void SendBatchAsOneChunk(IEnumerable <EventData> batchedEvents)
{
#if NET45
using (new TransactionScope(TransactionScopeOption.Suppress))
{
_eventHubClient.SendBatch(batchedEvents);
}
#else
_eventHubClient.SendAsync(batchedEvents).Wait();
#endif
}
/// <summary>
/// Asynchronously sends a batch of event data to the same partition.
/// All the event data in the batch need to have the same value in the Partitionkey property.
/// If the batch size is greater than the maximum batch size,
/// the method partitions the original batch into multiple batches,
/// each smaller in size than the maximum batch size.
/// </summary>
/// <param name="eventHubClient">The current EventHubClient object.</param>
/// <param name="eventDataEnumerable">An IEnumerable object containing event data instances.</param>
/// <param name="trace">true to cause a message to be written; otherwise, false.</param>
public static void SendPartitionedBatch(this EventHubClient eventHubClient, IEnumerable <EventData> eventDataEnumerable, bool trace = false)
{
var eventDataList = eventDataEnumerable as IList <EventData> ?? eventDataEnumerable.ToList();
if (eventDataEnumerable == null || !eventDataList.Any())
{
throw new ArgumentNullException(EventDataListCannotBeNullOrEmpty);
}
var batchList = new List <EventData>();
long batchSize = 0;
foreach (var eventData in eventDataList)
{
if ((batchSize + eventData.SerializedSizeInBytes) > Constants.MaxBathSizeInBytes)
{
// Send current batch
eventHubClient.SendBatch(batchList);
Trace.WriteLineIf(trace, string.Format(SendPartitionedBatchFormat, batchSize, batchList.Count));
// Initialize a new batch
batchList = new List <EventData> {
eventData
};
batchSize = eventData.SerializedSizeInBytes;
}
else
{
// Add the EventData to the current batch
batchList.Add(eventData);
batchSize += eventData.SerializedSizeInBytes;
}
}
// The final batch is sent outside of the loop
eventHubClient.SendBatch(batchList);
Trace.WriteLineIf(trace, string.Format(SendPartitionedBatchFormat, batchSize, batchList.Count));
}
private bool Send(string partitionKey, AsyncLogEventInfo[] logEvents)
{
if (_messsagingFactory == null)
{
_messsagingFactory = MessagingFactory.CreateFromConnectionString(CloudConfigurationManager.GetSetting(EventHubConnectionStringKey));
}
if (_eventHubClient == null)
{
_eventHubClient = _messsagingFactory.CreateEventHubClient(CloudConfigurationManager.GetSetting(EventHubNameKey));
}
var payload = FormPayload(logEvents.Select(e => e.LogEvent), partitionKey);
_eventHubClient.SendBatch(payload);
return(true);
}
/// <summary>
/// Emit a batch of log events, running to completion synchronously.
/// </summary>
/// <param name="events">The events to emit.</param>
protected override void EmitBatch(IEnumerable <LogEvent> events)
{
var batchedEvents = new List <EventData>();
foreach (var logEvent in events)
{
var render = new StringWriter();
_formatter.Format(logEvent, render);
var eventHubData = new EventData(Encoding.UTF8.GetBytes(render.ToString()))
{
PartitionKey = _partitionKey
};
eventHubData.Properties.Add(
"Type", "SerilogEvent_" + DateTime.Now.ToLongTimeString());
batchedEvents.Add(eventHubData);
}
_eventHubClient.SendBatch(batchedEvents);
}
public void Publish <T>(IEnumerable <T> myEvents)
{
if (myEvents == null)
{
throw new ArgumentNullException(nameof(myEvents));
}
var events = new List <EventData>();
foreach (var myEvent in myEvents)
{
// 1. Serialize each event
var serializedEvent = JsonConvert.SerializeObject(myEvent);
// 2. Convert serialized event to bytes
var eventBytes = Encoding.UTF8.GetBytes(serializedEvent);
// 3. Wrap event bytes in EventData instance
events.Add(new EventData(eventBytes));
}
// 4. Publish the events
_eventHubClient.SendBatch(events);
}
public void Send(IEnumerable <EventData> events)
{
_eventHubClient.SendBatch(events);
}
public static void SendBatchToEventHub(List <EventData> batch, EventHubClient eventHubClient)
{
eventHubClient.SendBatch(batch);
}
static void Main(string[] args)
{
Assembly execAssembly = Assembly.GetCallingAssembly();
AssemblyName assemblyName = execAssembly.GetName();
int traceLevel = 3;
int event_count = 0;
int total_event_count = 0;
int updowncount = 0;
List <EventData> e = new List <EventData>();
//EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(connectionString);
EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(CloudConfigurationManager.GetSetting("EventHubConnectionString"));
// Parse the connection string and return a reference to the storage account.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
// Create the blob client.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Retrieve reference to a previously created container.
CloudBlobContainer container = blobClient.GetContainerReference("syslogs");
// Retrieve reference to a previously created container for output logs
CloudBlobContainer containerlogs = blobClient.GetContainerReference("logs");
// Create the logs container if it does not already exist
containerlogs.CreateIfNotExists();
CloudAppendBlob logout = containerlogs.GetAppendBlobReference("ideapoceh04out");
if (!logout.Exists())
{
logout.CreateOrReplace();
}
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Beginning execution {2}", DateTime.UtcNow, assemblyName, Environment.NewLine));
if (traceLevel == 4)
{
// Loop over items within the container and output the length and URI.
foreach (IListBlobItem item in container.ListBlobs(null, false))
{
if (item.GetType() == typeof(CloudBlockBlob))
{
CloudBlockBlob blob = (CloudBlockBlob)item;
//Console.WriteLine("Block blob of length {0}: {1}", blob.Properties.Length, blob.Uri);
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Block blob of length {2}: {3} {4}", DateTime.UtcNow, assemblyName, blob.Properties.Length, blob.Uri, Environment.NewLine));
}
else if (item.GetType() == typeof(CloudPageBlob))
{
CloudPageBlob pageBlob = (CloudPageBlob)item;
//Console.WriteLine("Page blob of length {0}: {1}", pageBlob.Properties.Length, pageBlob.Uri);
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Page blob of length {2}: {3} {4}", DateTime.UtcNow, assemblyName, pageBlob.Properties.Length, pageBlob.Uri, Environment.NewLine));
}
else if (item.GetType() == typeof(CloudBlobDirectory))
{
CloudBlobDirectory directory = (CloudBlobDirectory)item;
//Console.WriteLine("Directory: {0}", directory.Uri);
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t Directory {2}: {3}", DateTime.UtcNow, assemblyName, directory.Uri, Environment.NewLine));
}
else if (item.GetType() == typeof(CloudAppendBlob))
{
CloudAppendBlob appendBlob = (CloudAppendBlob)item;
//Console.WriteLine("Append blob of length: {0}: {1}", appendBlob.Properties.Length, appendBlob.Uri);
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Append blob of length {2}: {3} {4}", DateTime.UtcNow, assemblyName, appendBlob.Properties.Length, appendBlob.Uri, Environment.NewLine));
}
}
}
//Console.ReadLine();
// Retrieve reference to a blob named "myblob.txt"
CloudBlockBlob blockBlob2 = container.GetBlockBlobReference("mls_syslog");
string line;
//string json_eh_message;
DateTime startTime = DateTime.UtcNow;
//using (var memoryStream = new MemoryStream())
using (var stream = blockBlob2.OpenRead())
{
using (StreamReader reader = new StreamReader(stream))
{
// while ( !reader.EndOfStream)
while ((line = reader.ReadLine()) != null)
{
// Console.WriteLine(reader.ReadLine());
//Console.WriteLine(line);
if (traceLevel == 4)
{
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : {2} {3}", DateTime.UtcNow, assemblyName, line, Environment.NewLine));
}
//Console.ReadLine();
string[] items = line.Split('\t');
syslog sl = new syslog();
sl.time = items[0];
sl.facility = items[1];
sl.address = items[2];
sl.message = items[3];
bool testupdown = sl.message.StartsWith("%LINK-3-UPDOWN:");
/* Console.WriteLine("Found UPDOWN? {0}", testupdown); */
if (testupdown)
{
updowncount++;
string[] messageItems = sl.message.Split(' ');
string msgInterface = messageItems[2];
/* Console.WriteLine("Interface is {0}", msgInterface); */
sl.netInterface = msgInterface;
}
// string json_eh_message = JsonConvert.SerializeObject(sl);
json_eh_message = JsonConvert.SerializeObject(sl);
if (event_count > 500)
{
try
{
//eventHubClient.SendBatchAsync(e);
eventHubClient.SendBatch(e);
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Submitting a batch of {2} events to EventHub {3}", DateTime.UtcNow, assemblyName, event_count, Environment.NewLine));
//Console.WriteLine("Submitting batch of {0} events", event_count);
}
catch (Exception exception)
{
//Console.ForegroundColor = ConsoleColor.Red;
//Console.WriteLine("{0} > Exception: {1}", DateTime.Now, exception.Message);
//Console.ResetColor();
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Exception raised: {2} {3}", DateTime.UtcNow, assemblyName, exception.Message, Environment.NewLine));
}
total_event_count = total_event_count + event_count;
event_count = 0;
//Console.WriteLine("Total events submitted: {0}", total_event_count);
e.Clear();
e.Add(new EventData(Encoding.UTF8.GetBytes(json_eh_message)));
event_count++;
//Console.WriteLine(json_eh_message);
}
else
{
e.Add(new EventData(Encoding.UTF8.GetBytes(json_eh_message)));
event_count++;
//Console.WriteLine(json_eh_message);
}
} // EOF -- submit final batch of remaining events
try
{
//eventHubClient.SendBatchAsync(e);
eventHubClient.SendBatch(e);
//Console.WriteLine("Submitting final batch of {0}", event_count);
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Submitting final batch of {2} events to EventHub {3}", DateTime.UtcNow, assemblyName, event_count, Environment.NewLine));
}
catch (Exception exception)
{
//Console.ForegroundColor = ConsoleColor.Red;
//Console.WriteLine("{0} > Exception: {1}", DateTime.Now, exception.Message);
//Console.ResetColor();
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Exception occurred while submitting batch to EventHub: {2} {3}", DateTime.UtcNow, assemblyName, exception.Message, Environment.NewLine));
}
total_event_count += event_count;
}
}
DateTime endTime = DateTime.UtcNow;
var ts = new TimeSpan(startTime.Ticks - endTime.Ticks);
double delta = Math.Abs(ts.TotalSeconds);
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : {2} link up/down events processed {3}", DateTime.UtcNow, assemblyName, updowncount, Environment.NewLine));
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : {2} total events processed {3}", DateTime.UtcNow, assemblyName, total_event_count, Environment.NewLine));
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : {2} events per second {3}", DateTime.UtcNow, assemblyName, total_event_count / delta, Environment.NewLine));
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Ending execution {2}", DateTime.UtcNow, assemblyName, Environment.NewLine));
//Console.WriteLine("Port flap event count: {0}", updowncount);
//Console.WriteLine("Total events: {0}", total_event_count);
//Console.ReadLine();
//using (var memoryStream2 = new BufferedStream(memoryStream))
/* {
* blockBlob2.DownloadToStream(memoryStream);
* //while ((text = memoryStream2.Read)
* text = System.Text.Encoding.UTF8.GetString(memoryStream.ToArray());
* text = memoryStream.
* Console.WriteLine(text);
* Console.ReadLine();
* } */
}
protected virtual void Publish()
{
Task task = new Task(() =>
{
List <T> itemsToLog = new List <T>();
try
{
if (Interlocked.CompareExchange(ref _onPublishExecuted, 1, 0) > 0)
{
return;
}
//if (IsPublishing())
// return;
_timer.Stop();
T item;
//Batch*********************************
EventData[] arrEventData = Array.ConvertAll(base.ToArray(), myelement => new EventData(Encoding.UTF8.GetBytes(myelement.ToString())));
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
int a = arrEventData.GetLength(0);
_eventHubClient.SendBatch(arrEventData);
//_eventHubClient.SendBatchAsync(arrEventData);
stopWatch.Stop();
TimeSpan ts = stopWatch.Elapsed;
int millisecondSpent = ts.Milliseconds;
Console.WriteLine(string.Format("Total milliseconds to deliver in EH {0} - Seconds {1}", millisecondSpent, (millisecondSpent / 1000)), ConsoleColor.Red);
//Batch*********************************
//Single message***********************
//while (TryDequeue(out item))
//{
// itemsToLog.Add(item);
// EventData data = new EventData(Encoding.UTF8.GetBytes(item.ToString()));
// //_eventHubClient.Send(data);
// _eventHubClient.SendAsync(data);
//}
//Single message***********************
}
catch (ThreadAbortException tex)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("General lev 1 Exception {0}", tex);
}
catch (Exception ex)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("General lev 2 Exception {0}", ex);
}
finally
{
Interlocked.Decrement(ref _onPublishExecuted);
OnPublish(itemsToLog);
CompletePublishing();
}
});
task.Start();
}
