private void _internalAdd(WorkloadEvent evt)
{
if (evt is ExecutionWorkloadEvent)
{
_internalAdd((ExecutionWorkloadEvent)evt);
}
if (evt is ErrorWorkloadEvent)
{
_internalAdd((ErrorWorkloadEvent)evt);
}
if (evt is CounterWorkloadEvent)
{
_internalAdd((CounterWorkloadEvent)evt);
}
if (evt is WaitStatsWorkloadEvent)
{
_internalAdd((WaitStatsWorkloadEvent)evt);
}
}
private void ProcessQueue()
{
while (!stopped)
{
lock (_internalQueue)
{
CloseInterval();
if (_internalQueue.Count == 0)
{
Thread.Sleep(10);
continue;
}
WorkloadEvent data = _internalQueue.Dequeue();
_internalAdd(data);
}
}
}
private void InsertCounterEvent(WorkloadEvent evnt)
{
CounterWorkloadEvent evt = (CounterWorkloadEvent)evnt;
events_cmd.Parameters.AddWithValue("$event_sequence", event_sequence++);
events_cmd.Parameters.AddWithValue("$event_type", evt.Type);
events_cmd.Parameters.AddWithValue("$start_time", evt.StartTime);
events_cmd.Parameters.AddWithValue("$client_app_name", null);
events_cmd.Parameters.AddWithValue("$client_host_name", null);
events_cmd.Parameters.AddWithValue("$database_name", null);
events_cmd.Parameters.AddWithValue("$server_principal_name", null);
events_cmd.Parameters.AddWithValue("$session_id", null);
events_cmd.Parameters.AddWithValue("$sql_text", null);
events_cmd.Parameters.AddWithValue("$cpu", null);
events_cmd.Parameters.AddWithValue("$duration", null);
events_cmd.Parameters.AddWithValue("$reads", null);
events_cmd.Parameters.AddWithValue("$writes", null);
events_cmd.ExecuteNonQuery();
SQLiteTransaction tran = conn.BeginTransaction();
try
{
foreach (var dr in evt.Counters)
{
counters_cmd.Parameters.AddWithValue("$event_sequence", event_sequence);
counters_cmd.Parameters.AddWithValue("$name", dr.Key.ToString());
counters_cmd.Parameters.AddWithValue("$value", dr.Value);
counters_cmd.ExecuteNonQuery();
}
tran.Commit();
}
catch (Exception)
{
tran.Rollback();
throw;
}
}
private void InsertExecutionEvent(WorkloadEvent evnt)
{
ExecutionWorkloadEvent evt = (ExecutionWorkloadEvent)evnt;
events_cmd.Parameters.AddWithValue("$event_sequence", event_sequence++);
events_cmd.Parameters.AddWithValue("$event_type", evt.Type);
events_cmd.Parameters.AddWithValue("$start_time", evt.StartTime);
events_cmd.Parameters.AddWithValue("$client_app_name", evt.ApplicationName);
events_cmd.Parameters.AddWithValue("$client_host_name", evt.HostName);
events_cmd.Parameters.AddWithValue("$database_name", evt.DatabaseName);
events_cmd.Parameters.AddWithValue("$server_principal_name", evt.LoginName);
events_cmd.Parameters.AddWithValue("$session_id", evt.SPID);
events_cmd.Parameters.AddWithValue("$sql_text", evt.Text);
events_cmd.Parameters.AddWithValue("$cpu", evt.CPU);
events_cmd.Parameters.AddWithValue("$duration", evt.Duration);
events_cmd.Parameters.AddWithValue("$reads", evt.Reads);
events_cmd.Parameters.AddWithValue("$writes", evt.Writes);
events_cmd.ExecuteNonQuery();
}
public override sealed void Consume(WorkloadEvent evt)
{
if (evt == null)
{
return;
}
// Ensure that the buffer does not get too big
while (Buffer.Count >= BufferSize)
{
spin.SpinOnce();
}
// If the buffer has room, enqueue the event
Buffer.Enqueue(evt);
if (BufferReader == null)
{
BufferReader = Task.Factory.StartNew(() => ProcessBuffer());
}
}
public override WorkloadEvent Read()
{
if (!started)
{
Task t = Task.Factory.StartNew(ReadEventsFromFile);
started = true;
}
WorkloadEvent result = null;
while (!Events.TryDequeue(out result))
{
if (stopped)
{
return(null);
}
Thread.Sleep(5);
}
return(result);
}
protected void ProcessBuffer()
{
while (!stopped)
{
WorkloadEvent evt = null;
while (!Buffer.TryDequeue(out evt))
{
if (stopped)
{
return;
}
spin.SpinOnce();
}
if (evt == null)
{
continue;
}
ConsumeBuffered(evt);
}
}
private void InsertEvent(WorkloadEvent evnt)
{
try
{
if ((evnt is ExecutionWorkloadEvent))
{
InsertExecutionEvent(evnt);
}
if ((evnt is CounterWorkloadEvent))
{
InsertCounterEvent(evnt);
}
if ((evnt is WaitStatsWorkloadEvent))
{
InsertWaitEvent(evnt);
}
_rowsInserted++;
if ((_rowsInserted % CACHE_SIZE == 0) || forceFlush)
{
if (forceFlush)
{
forceFlush = false;
}
logger.Info($"{_rowsInserted} events saved");
}
}
catch (Exception e)
{
if (stopped)
{
return;
}
logger.Error(e, "Unable to write to the destination file");
throw;
}
}
public abstract void Write(WorkloadEvent evt);
public override WorkloadEvent Read()
{
WorkloadEvent result = null;
long commandOffset = 0;
// first I need to return the event that
// contains the total number of events in the file
// once this is done I can start sending the actual events
if (!totalEventsMessageSent)
{
totalEventsMessageSent = true;
return(totalEventsMessage);
}
// process actual events from the file
try
{
if (reader == null)
{
return(null);
}
bool validEventFound = false;
do
{
if (!reader.Read())
{
stopped = true;
return(null);
}
result = ReadEvent(reader);
// Handle replay sleep for synchronization mode
// The sleep cannot happen here, but it has to
// happen later in the replay workflow, because
// it would only delay the insertion in the queue
// and it would not separate the events during the replay
if (result is ExecutionWorkloadEvent)
{
ExecutionWorkloadEvent execEvent = result as ExecutionWorkloadEvent;
if (SynchronizationMode)
{
if (startTime != DateTime.MinValue)
{
commandOffset = (long)((result.StartTime - startTime).TotalMilliseconds);
if (commandOffset > 0)
{
execEvent.ReplayOffset = commandOffset;
}
}
else
{
startTime = execEvent.StartTime;
}
}
else
{
// Leave it at 0. The replay consumer will interpret this
// as "do not wait for the requested offset" and will replay
// the event without waiting
execEvent.ReplayOffset = 0;
}
}
// Filter events
if (result is ExecutionWorkloadEvent)
{
validEventFound = Filter.Evaluate(result);
}
else
{
validEventFound = true;
}
}while (!validEventFound);
}
catch (Exception e)
{
if (stopped)
{
return(null);
}
DateTime?eventDate = null;
if (result != null)
{
eventDate = result.StartTime;
}
logger.Error(e);
logger.Error($"Unable to read next event. Current event date: {eventDate}");
throw;
}
return(result);
}
public override void ConsumeBuffered(WorkloadEvent evnt)
{
if (evnt is MessageWorkloadEvent)
{
MessageWorkloadEvent msgEvent = evnt as MessageWorkloadEvent;
if (msgEvent.MsgType == MessageWorkloadEvent.MessageType.TotalEvents)
{
try
{
totalEventCount = (long)msgEvent.Value;
}
catch (Exception e)
{
logger.Error($"Unable to set the total number of events: {e.Message}");
}
}
}
if (!(evnt is ExecutionWorkloadEvent))
{
return;
}
if (evnt.Type != WorkloadEvent.EventType.RPCCompleted && evnt.Type != WorkloadEvent.EventType.BatchCompleted)
{
return;
}
eventCount++;
if ((eventCount > 0) && (eventCount % WorkerStatsCommandCount == 0))
{
string percentInfo = (totalEventCount > 0) ? "( " + ((eventCount * 100) / totalEventCount).ToString() + "% )" : "";
logger.Info($"{eventCount} events queued for replay {percentInfo}");
}
if (startTime == DateTime.MinValue)
{
startTime = DateTime.Now;
}
ExecutionWorkloadEvent evt = (ExecutionWorkloadEvent)evnt;
ReplayCommand command = new ReplayCommand()
{
CommandText = evt.Text,
Database = evt.DatabaseName,
ApplicationName = evt.ApplicationName,
ReplayOffset = evt.ReplayOffset,
EventSequence = evt.EventSequence
};
int session_id = -1;
session_id = (int)evt.SPID;
ReplayWorker rw = null;
if (ReplayWorkers.TryGetValue(session_id, out rw))
{
// Ensure that the buffer does not get too big
while (rw.QueueLength >= (BufferSize * .9))
{
spin.SpinOnce();
}
rw.AppendCommand(command);
}
else
{
rw = new ReplayWorker()
{
ConnectionInfo = this.ConnectionInfo,
ReplayIntervalSeconds = 0,
StopOnError = false,
Name = session_id.ToString(),
DisplayWorkerStats = this.DisplayWorkerStats,
ConsumeResults = this.ConsumeResults,
QueryTimeoutSeconds = this.QueryTimeoutSeconds,
WorkerStatsCommandCount = this.WorkerStatsCommandCount,
MimicApplicationName = this.MimicApplicationName,
DatabaseMap = this.DatabaseMap,
StartTime = startTime,
FailRetryCount = this.FailRetryCount,
TimeoutRetryCount = this.TimeoutRetryCount
};
ReplayWorkers.TryAdd(session_id, rw);
rw.AppendCommand(command);
logger.Info($"Worker [{session_id}] - Starting");
}
if (runner == null)
{
runner = new Thread(new ThreadStart(
delegate
{
try
{
RunWorkers();
}
catch (Exception e)
{
try { logger.Error(e, "Unhandled exception in ReplayConsumer.RunWorkers"); }
catch { Console.WriteLine(e.Message); }
}
}
));
runner.IsBackground = true;
runner.Start();
}
if (sweeper == null)
{
sweeper = new Thread(new ThreadStart(
delegate
{
try
{
RunSweeper();
}
catch (Exception e)
{
try { logger.Error(e, "Unhandled exception in TraceManager.RunSweeper"); }
catch { Console.WriteLine(e.Message); }
}
}
));
sweeper.IsBackground = true;
sweeper.Start();
}
}
public abstract void Consume(WorkloadEvent evt);
public void TestEnqueueDequeueFixedList()
{
int[] numbers =
{
+9772
, -9479
, +70255
, -49216
, +18796
, -39641
, +60528
, -60690
, +78808
, -49406
, +42422
, -72132
, +65861
, -34935
, +55297
, -10699
, +96237
, -72432
, +55697
, -85962
, +18370
, -72056
, +97085
, -50146
, +43353
, -53808
, +28408
, -76107
, +51235
, -50290
, +67421
, -9696
, +65303
, -45014
, +53121
, -50691
, +68663
, -54973
, +34989
, -66099
, +15014
, -53872
, +97248
, -38096
, +705
, -23998
, +13872
, -42048
, +77390
, -71767
, +86413
, -6260
, +61030
, -51330
, +14412
, -37716
, +16394
, -20109
, +5862
, -64988
, +67733
, -84421
, +23954
, -3518
, +81985
, -32726
, +14828
, -20847
, +81813
, -4605
, +42036
, -41263
, +37442
, -89598
, +70947
, -64497
, +74808
, -58988
, +49441
, -19355
, -166474
};
using (BinarySerializedBufferedEventQueue queue = new BinarySerializedBufferedEventQueue())
{
queue.BufferSize = 10000;
int total = 0;
for (int i = 0; i < numbers.Length; i++)
{
if (i == 28)
{
Debug.WriteLine("Uh oh");
}
int num = numbers[i];
if (num > 0)
{
int initialCount = queue.Count;
for (int j = 0; j < num; j++)
{
queue.Enqueue(new ExecutionWorkloadEvent()
{
Text = $"SELECT {j} FROM sometable WHERE somecolumn = someValue ORDER BY someOtherColumn"
});
if (i == 28)
{
Console.WriteLine($" {j}: should be {initialCount + j + 1} | is {queue.Count}");
if (initialCount + j + 1 == 8854)
{
Console.WriteLine($"Aaaaah!");
}
}
}
}
else
{
int initialCount = queue.Count;
num = num * -1;
WorkloadEvent evnt = null;
for (int k = 0; k < num; k++)
{
queue.TryDequeue(out evnt);
//Console.WriteLine($" {k}: should be {initialCount - k} | is {queue.Count}");
}
num = num * -1;
}
total += num;
Assert.AreEqual(queue.Count, total);
}
}
}
public void TestEnqueueRandomDequeueAll()
{
using (BinarySerializedBufferedEventQueue queue = new BinarySerializedBufferedEventQueue())
{
queue.BufferSize = 10000;
Random r = new Random();
Stopwatch watch = new Stopwatch();
for (int j = 0; j < 10; j++)
{
watch.Reset();
watch.Start();
int numElements = (int)(r.NextDouble() * 100000);
for (int i = 0; i < numElements; i++)
{
queue.Enqueue(new ExecutionWorkloadEvent()
{
Text = $"SELECT {i} FROM sometable WHERE somecolumn = someValue ORDER BY someOtherColumn"
});
}
watch.Stop();
Console.WriteLine($"Enqueue {numElements} elements elapsed: {watch.Elapsed}");
int queueLen = queue.Count;
numElements = (int)(r.NextDouble() * 100000);
while (numElements > queueLen)
{
numElements -= 500;
}
watch.Reset();
watch.Start();
for (int i = 0; i < numElements; i++)
{
WorkloadEvent evt = null;
queue.TryDequeue(out evt);
//Console.WriteLine(((ExecutionWorkloadEvent)evt).Text);
}
watch.Stop();
Console.WriteLine($"Dequeue {numElements} elements elapsed: {watch.Elapsed}");
}
watch.Reset();
watch.Start();
int len = queue.Count;
WorkloadEvent evnt = null;
while (queue.TryDequeue(out evnt))
{
;
}
watch.Stop();
Console.WriteLine($"Dequeue all {len} remaining elements elapsed: {watch.Elapsed}");
Assert.AreEqual(queue.Count, 0);
}
}
public override WorkloadEvent Read()
{
WorkloadEvent result = null;
double msSleep = 0;
try
{
if (reader == null)
{
return(null);
}
bool validEventFound = false;
do
{
if (!reader.Read())
{
stopped = true;
return(null);
}
result = ReadEvent(reader);
// Handle replay sleep for synchronization mode
// The sleep cannot happen here, but it has to
// happen later in the replay workflow, because
// it would only delay the insertion in the queue
// and it would not separate the events during the replay
if (result is ExecutionWorkloadEvent)
{
ExecutionWorkloadEvent execEvent = result as ExecutionWorkloadEvent;
if (SynchronizationMode)
{
DateTime lastEndTime = GetLastEventEndTime((int)execEvent.SPID);
if (lastEndTime != DateTime.MinValue)
{
msSleep = (result.StartTime - lastEndTime).TotalMilliseconds;
if (msSleep > 0)
{
if (msSleep > Int32.MaxValue)
{
msSleep = Int32.MaxValue;
}
Thread.Sleep(Convert.ToInt32(msSleep));
}
}
SetLastEventEndTime((int)execEvent.SPID, execEvent.StartTime.AddMilliseconds((double)execEvent.Duration / 1000));
}
else
{
execEvent.ReplaySleepMilliseconds = 0;
}
}
// Filter events
if (result is ExecutionWorkloadEvent)
{
validEventFound = Filter.Evaluate(result);
}
else
{
validEventFound = true;
}
}while (!validEventFound);
}
catch (Exception e)
{
if (stopped)
{
return(null);
}
DateTime?eventDate = null;
if (result != null)
{
eventDate = result.StartTime;
}
logger.Error(e);
logger.Error($"Unable to read next event. Current event date: {eventDate}");
throw;
}
return(result);
}
public override void Write(WorkloadEvent evt)
{
consumer.Consume(evt);
}
public override WorkloadEvent Read()
{
WorkloadEvent result = null;
double msSleep = 0;
try
{
if (reader == null)
{
return(null);
}
bool validEventFound = false;
do
{
if (!reader.Read())
{
stopped = true;
return(null);
}
result = ReadEvent(reader);
if (SynchronizationMode)
{
if (previousDate != DateTime.MinValue)
{
msSleep = (result.StartTime - previousDate).TotalMilliseconds;
if (msSleep > 0)
{
if (msSleep > Int32.MaxValue)
{
msSleep = Int32.MaxValue;
}
Thread.Sleep(Convert.ToInt32(msSleep));
}
}
}
previousDate = result.StartTime;
// Filter events
if (result is ExecutionWorkloadEvent)
{
validEventFound = Filter.Evaluate(result);
}
else
{
validEventFound = true;
}
}while (!validEventFound);
}
catch (Exception e)
{
if (stopped)
{
return(null);
}
DateTime?eventDate = null;
if (result != null)
{
eventDate = result.StartTime;
}
logger.Error(e);
logger.Error($"Unable to read next event. Current event date: {eventDate} Last event date: {previousDate} Requested sleep: {msSleep}");
throw;
}
return(result);
}
public override void ConsumeBuffered(WorkloadEvent evnt)
{
if (!(evnt is ExecutionWorkloadEvent))
{
return;
}
if (evnt.Type != WorkloadEvent.EventType.RPCCompleted && evnt.Type != WorkloadEvent.EventType.BatchCompleted)
{
return;
}
ExecutionWorkloadEvent evt = (ExecutionWorkloadEvent)evnt;
ReplayCommand command = new ReplayCommand()
{
CommandText = evt.Text,
Database = evt.DatabaseName,
ApplicationName = evt.ApplicationName
};
int session_id = -1;
session_id = (int)evt.SPID;
ReplayWorker rw = null;
if (ReplayWorkers.TryGetValue(session_id, out rw))
{
rw.AppendCommand(command);
}
else
{
rw = new ReplayWorker()
{
ConnectionInfo = this.ConnectionInfo,
ReplayIntervalSeconds = 0,
StopOnError = false,
Name = session_id.ToString()
};
ReplayWorkers.TryAdd(session_id, rw);
rw.AppendCommand(command);
logger.Info(String.Format("Worker [{0}] - Starting", session_id));
}
if (runner == null)
{
runner = new Thread(new ThreadStart(
delegate
{
try
{
RunWorkers();
}
catch (Exception e)
{
try { logger.Error(e, "Unhandled exception in TraceManager.RunWorkers"); }
catch { Console.WriteLine(e.Message); }
}
}
));
runner.IsBackground = true;
runner.Start();
}
if (sweeper == null)
{
sweeper = new Thread(new ThreadStart(
delegate
{
try
{
RunSweeper();
}
catch (Exception e)
{
try { logger.Error(e, "Unhandled exception in TraceManager.RunSweeper"); }
catch { Console.WriteLine(e.Message); }
}
}
));
sweeper.IsBackground = true;
sweeper.Start();
}
}
public override void ConsumeBuffered(WorkloadEvent evnt)
{
if (!(evnt is ExecutionWorkloadEvent))
{
return;
}
if (evnt.Type != WorkloadEvent.EventType.RPCCompleted && evnt.Type != WorkloadEvent.EventType.BatchCompleted)
{
return;
}
eventCount++;
if ((eventCount > 0) && (eventCount % WorkerStatsCommandCount == 0))
{
logger.Info($"{eventCount} events queued for replay");
}
ExecutionWorkloadEvent evt = (ExecutionWorkloadEvent)evnt;
ReplayCommand command = new ReplayCommand()
{
CommandText = evt.Text,
Database = evt.DatabaseName,
ApplicationName = evt.ApplicationName,
BeforeSleepMilliseconds = evt.ReplaySleepMilliseconds
};
int session_id = -1;
session_id = (int)evt.SPID;
ReplayWorker rw = null;
if (ReplayWorkers.TryGetValue(session_id, out rw))
{
// Ensure that the buffer does not get too big
while (rw.QueueLength >= (BufferSize * .9))
{
spin.SpinOnce();
}
rw.AppendCommand(command);
}
else
{
rw = new ReplayWorker()
{
ConnectionInfo = this.ConnectionInfo,
ReplayIntervalSeconds = 0,
StopOnError = false,
Name = session_id.ToString(),
DisplayWorkerStats = this.DisplayWorkerStats,
ConsumeResults = this.ConsumeResults,
QueryTimeoutSeconds = this.QueryTimeoutSeconds,
WorkerStatsCommandCount = this.WorkerStatsCommandCount,
MimicApplicationName = this.MimicApplicationName
};
ReplayWorkers.TryAdd(session_id, rw);
rw.AppendCommand(command);
logger.Info($"Worker [{session_id}] - Starting");
}
if (runner == null)
{
runner = new Thread(new ThreadStart(
delegate
{
try
{
RunWorkers();
}
catch (Exception e)
{
try { logger.Error(e, "Unhandled exception in TraceManager.RunWorkers"); }
catch { Console.WriteLine(e.Message); }
}
}
));
runner.IsBackground = true;
runner.Start();
}
if (sweeper == null)
{
sweeper = new Thread(new ThreadStart(
delegate
{
try
{
RunSweeper();
}
catch (Exception e)
{
try { logger.Error(e, "Unhandled exception in TraceManager.RunSweeper"); }
catch { Console.WriteLine(e.Message); }
}
}
));
sweeper.IsBackground = true;
sweeper.Start();
}
}
private void ProcessQueue()
{
DateTime lastDump = DateTime.Now;
while (!stopped)
{
// Write collected data to the destination database
TimeSpan duration = DateTime.Now - lastDump;
if (duration.TotalMinutes >= Interval)
{
try
{
int numRetries = 0;
while (numRetries <= MAX_WRITE_RETRIES)
{
try
{
WriteToServer();
numRetries = MAX_WRITE_RETRIES + 1;
}
catch (Exception ex)
{
logger.Warn("Unable to write workload analysis.");
logger.Warn(ex.Message);
if (numRetries == MAX_WRITE_RETRIES)
{
throw;
}
}
numRetries++;
}
}
catch (Exception e)
{
try
{
logger.Error(e, "Unable to write workload analysis info to the destination database.");
logger.Error(e.StackTrace);
}
catch
{
Console.WriteLine(String.Format("Unable to write to the database: {0}.", e.Message));
}
}
finally
{
lastDump = DateTime.Now;
}
}
if (_internalQueue.Count == 0)
{
Thread.Sleep(10);
continue;
}
lock (_internalQueue)
{
WorkloadEvent data = _internalQueue.Dequeue();
_internalAdd(data);
}
}
}
public abstract void ConsumeBuffered(WorkloadEvent evt);
