private void _internalAdd(WorkloadEvent evt) { if (evt is ExecutionWorkloadEvent) { _internalAdd((ExecutionWorkloadEvent)evt); } if (evt is ErrorWorkloadEvent) { _internalAdd((ErrorWorkloadEvent)evt); } if (evt is CounterWorkloadEvent) { _internalAdd((CounterWorkloadEvent)evt); } if (evt is WaitStatsWorkloadEvent) { _internalAdd((WaitStatsWorkloadEvent)evt); } }
private void ProcessQueue() { while (!stopped) { lock (_internalQueue) { CloseInterval(); if (_internalQueue.Count == 0) { Thread.Sleep(10); continue; } WorkloadEvent data = _internalQueue.Dequeue(); _internalAdd(data); } } }
private void InsertCounterEvent(WorkloadEvent evnt) { CounterWorkloadEvent evt = (CounterWorkloadEvent)evnt; events_cmd.Parameters.AddWithValue("$event_sequence", event_sequence++); events_cmd.Parameters.AddWithValue("$event_type", evt.Type); events_cmd.Parameters.AddWithValue("$start_time", evt.StartTime); events_cmd.Parameters.AddWithValue("$client_app_name", null); events_cmd.Parameters.AddWithValue("$client_host_name", null); events_cmd.Parameters.AddWithValue("$database_name", null); events_cmd.Parameters.AddWithValue("$server_principal_name", null); events_cmd.Parameters.AddWithValue("$session_id", null); events_cmd.Parameters.AddWithValue("$sql_text", null); events_cmd.Parameters.AddWithValue("$cpu", null); events_cmd.Parameters.AddWithValue("$duration", null); events_cmd.Parameters.AddWithValue("$reads", null); events_cmd.Parameters.AddWithValue("$writes", null); events_cmd.ExecuteNonQuery(); SQLiteTransaction tran = conn.BeginTransaction(); try { foreach (var dr in evt.Counters) { counters_cmd.Parameters.AddWithValue("$event_sequence", event_sequence); counters_cmd.Parameters.AddWithValue("$name", dr.Key.ToString()); counters_cmd.Parameters.AddWithValue("$value", dr.Value); counters_cmd.ExecuteNonQuery(); } tran.Commit(); } catch (Exception) { tran.Rollback(); throw; } }
private void InsertExecutionEvent(WorkloadEvent evnt) { ExecutionWorkloadEvent evt = (ExecutionWorkloadEvent)evnt; events_cmd.Parameters.AddWithValue("$event_sequence", event_sequence++); events_cmd.Parameters.AddWithValue("$event_type", evt.Type); events_cmd.Parameters.AddWithValue("$start_time", evt.StartTime); events_cmd.Parameters.AddWithValue("$client_app_name", evt.ApplicationName); events_cmd.Parameters.AddWithValue("$client_host_name", evt.HostName); events_cmd.Parameters.AddWithValue("$database_name", evt.DatabaseName); events_cmd.Parameters.AddWithValue("$server_principal_name", evt.LoginName); events_cmd.Parameters.AddWithValue("$session_id", evt.SPID); events_cmd.Parameters.AddWithValue("$sql_text", evt.Text); events_cmd.Parameters.AddWithValue("$cpu", evt.CPU); events_cmd.Parameters.AddWithValue("$duration", evt.Duration); events_cmd.Parameters.AddWithValue("$reads", evt.Reads); events_cmd.Parameters.AddWithValue("$writes", evt.Writes); events_cmd.ExecuteNonQuery(); }
public override sealed void Consume(WorkloadEvent evt) { if (evt == null) { return; } // Ensure that the buffer does not get too big while (Buffer.Count >= BufferSize) { spin.SpinOnce(); } // If the buffer has room, enqueue the event Buffer.Enqueue(evt); if (BufferReader == null) { BufferReader = Task.Factory.StartNew(() => ProcessBuffer()); } }
public override WorkloadEvent Read() { if (!started) { Task t = Task.Factory.StartNew(ReadEventsFromFile); started = true; } WorkloadEvent result = null; while (!Events.TryDequeue(out result)) { if (stopped) { return(null); } Thread.Sleep(5); } return(result); }
protected void ProcessBuffer() { while (!stopped) { WorkloadEvent evt = null; while (!Buffer.TryDequeue(out evt)) { if (stopped) { return; } spin.SpinOnce(); } if (evt == null) { continue; } ConsumeBuffered(evt); } }
private void InsertEvent(WorkloadEvent evnt) { try { if ((evnt is ExecutionWorkloadEvent)) { InsertExecutionEvent(evnt); } if ((evnt is CounterWorkloadEvent)) { InsertCounterEvent(evnt); } if ((evnt is WaitStatsWorkloadEvent)) { InsertWaitEvent(evnt); } _rowsInserted++; if ((_rowsInserted % CACHE_SIZE == 0) || forceFlush) { if (forceFlush) { forceFlush = false; } logger.Info($"{_rowsInserted} events saved"); } } catch (Exception e) { if (stopped) { return; } logger.Error(e, "Unable to write to the destination file"); throw; } }
public abstract void Write(WorkloadEvent evt);
public override WorkloadEvent Read() { WorkloadEvent result = null; long commandOffset = 0; // first I need to return the event that // contains the total number of events in the file // once this is done I can start sending the actual events if (!totalEventsMessageSent) { totalEventsMessageSent = true; return(totalEventsMessage); } // process actual events from the file try { if (reader == null) { return(null); } bool validEventFound = false; do { if (!reader.Read()) { stopped = true; return(null); } result = ReadEvent(reader); // Handle replay sleep for synchronization mode // The sleep cannot happen here, but it has to // happen later in the replay workflow, because // it would only delay the insertion in the queue // and it would not separate the events during the replay if (result is ExecutionWorkloadEvent) { ExecutionWorkloadEvent execEvent = result as ExecutionWorkloadEvent; if (SynchronizationMode) { if (startTime != DateTime.MinValue) { commandOffset = (long)((result.StartTime - startTime).TotalMilliseconds); if (commandOffset > 0) { execEvent.ReplayOffset = commandOffset; } } else { startTime = execEvent.StartTime; } } else { // Leave it at 0. The replay consumer will interpret this // as "do not wait for the requested offset" and will replay // the event without waiting execEvent.ReplayOffset = 0; } } // Filter events if (result is ExecutionWorkloadEvent) { validEventFound = Filter.Evaluate(result); } else { validEventFound = true; } }while (!validEventFound); } catch (Exception e) { if (stopped) { return(null); } DateTime?eventDate = null; if (result != null) { eventDate = result.StartTime; } logger.Error(e); logger.Error($"Unable to read next event. Current event date: {eventDate}"); throw; } return(result); }
public override void ConsumeBuffered(WorkloadEvent evnt) { if (evnt is MessageWorkloadEvent) { MessageWorkloadEvent msgEvent = evnt as MessageWorkloadEvent; if (msgEvent.MsgType == MessageWorkloadEvent.MessageType.TotalEvents) { try { totalEventCount = (long)msgEvent.Value; } catch (Exception e) { logger.Error($"Unable to set the total number of events: {e.Message}"); } } } if (!(evnt is ExecutionWorkloadEvent)) { return; } if (evnt.Type != WorkloadEvent.EventType.RPCCompleted && evnt.Type != WorkloadEvent.EventType.BatchCompleted) { return; } eventCount++; if ((eventCount > 0) && (eventCount % WorkerStatsCommandCount == 0)) { string percentInfo = (totalEventCount > 0) ? "( " + ((eventCount * 100) / totalEventCount).ToString() + "% )" : ""; logger.Info($"{eventCount} events queued for replay {percentInfo}"); } if (startTime == DateTime.MinValue) { startTime = DateTime.Now; } ExecutionWorkloadEvent evt = (ExecutionWorkloadEvent)evnt; ReplayCommand command = new ReplayCommand() { CommandText = evt.Text, Database = evt.DatabaseName, ApplicationName = evt.ApplicationName, ReplayOffset = evt.ReplayOffset, EventSequence = evt.EventSequence }; int session_id = -1; session_id = (int)evt.SPID; ReplayWorker rw = null; if (ReplayWorkers.TryGetValue(session_id, out rw)) { // Ensure that the buffer does not get too big while (rw.QueueLength >= (BufferSize * .9)) { spin.SpinOnce(); } rw.AppendCommand(command); } else { rw = new ReplayWorker() { ConnectionInfo = this.ConnectionInfo, ReplayIntervalSeconds = 0, StopOnError = false, Name = session_id.ToString(), DisplayWorkerStats = this.DisplayWorkerStats, ConsumeResults = this.ConsumeResults, QueryTimeoutSeconds = this.QueryTimeoutSeconds, WorkerStatsCommandCount = this.WorkerStatsCommandCount, MimicApplicationName = this.MimicApplicationName, DatabaseMap = this.DatabaseMap, StartTime = startTime, FailRetryCount = this.FailRetryCount, TimeoutRetryCount = this.TimeoutRetryCount }; ReplayWorkers.TryAdd(session_id, rw); rw.AppendCommand(command); logger.Info($"Worker [{session_id}] - Starting"); } if (runner == null) { runner = new Thread(new ThreadStart( delegate { try { RunWorkers(); } catch (Exception e) { try { logger.Error(e, "Unhandled exception in ReplayConsumer.RunWorkers"); } catch { Console.WriteLine(e.Message); } } } )); runner.IsBackground = true; runner.Start(); } if (sweeper == null) { sweeper = new Thread(new ThreadStart( delegate { try { RunSweeper(); } catch (Exception e) { try { logger.Error(e, "Unhandled exception in TraceManager.RunSweeper"); } catch { Console.WriteLine(e.Message); } } } )); sweeper.IsBackground = true; sweeper.Start(); } }
public abstract void Consume(WorkloadEvent evt);
public void TestEnqueueDequeueFixedList() { int[] numbers = { +9772 , -9479 , +70255 , -49216 , +18796 , -39641 , +60528 , -60690 , +78808 , -49406 , +42422 , -72132 , +65861 , -34935 , +55297 , -10699 , +96237 , -72432 , +55697 , -85962 , +18370 , -72056 , +97085 , -50146 , +43353 , -53808 , +28408 , -76107 , +51235 , -50290 , +67421 , -9696 , +65303 , -45014 , +53121 , -50691 , +68663 , -54973 , +34989 , -66099 , +15014 , -53872 , +97248 , -38096 , +705 , -23998 , +13872 , -42048 , +77390 , -71767 , +86413 , -6260 , +61030 , -51330 , +14412 , -37716 , +16394 , -20109 , +5862 , -64988 , +67733 , -84421 , +23954 , -3518 , +81985 , -32726 , +14828 , -20847 , +81813 , -4605 , +42036 , -41263 , +37442 , -89598 , +70947 , -64497 , +74808 , -58988 , +49441 , -19355 , -166474 }; using (BinarySerializedBufferedEventQueue queue = new BinarySerializedBufferedEventQueue()) { queue.BufferSize = 10000; int total = 0; for (int i = 0; i < numbers.Length; i++) { if (i == 28) { Debug.WriteLine("Uh oh"); } int num = numbers[i]; if (num > 0) { int initialCount = queue.Count; for (int j = 0; j < num; j++) { queue.Enqueue(new ExecutionWorkloadEvent() { Text = $"SELECT {j} FROM sometable WHERE somecolumn = someValue ORDER BY someOtherColumn" }); if (i == 28) { Console.WriteLine($" {j}: should be {initialCount + j + 1} | is {queue.Count}"); if (initialCount + j + 1 == 8854) { Console.WriteLine($"Aaaaah!"); } } } } else { int initialCount = queue.Count; num = num * -1; WorkloadEvent evnt = null; for (int k = 0; k < num; k++) { queue.TryDequeue(out evnt); //Console.WriteLine($" {k}: should be {initialCount - k} | is {queue.Count}"); } num = num * -1; } total += num; Assert.AreEqual(queue.Count, total); } } }
public void TestEnqueueRandomDequeueAll() { using (BinarySerializedBufferedEventQueue queue = new BinarySerializedBufferedEventQueue()) { queue.BufferSize = 10000; Random r = new Random(); Stopwatch watch = new Stopwatch(); for (int j = 0; j < 10; j++) { watch.Reset(); watch.Start(); int numElements = (int)(r.NextDouble() * 100000); for (int i = 0; i < numElements; i++) { queue.Enqueue(new ExecutionWorkloadEvent() { Text = $"SELECT {i} FROM sometable WHERE somecolumn = someValue ORDER BY someOtherColumn" }); } watch.Stop(); Console.WriteLine($"Enqueue {numElements} elements elapsed: {watch.Elapsed}"); int queueLen = queue.Count; numElements = (int)(r.NextDouble() * 100000); while (numElements > queueLen) { numElements -= 500; } watch.Reset(); watch.Start(); for (int i = 0; i < numElements; i++) { WorkloadEvent evt = null; queue.TryDequeue(out evt); //Console.WriteLine(((ExecutionWorkloadEvent)evt).Text); } watch.Stop(); Console.WriteLine($"Dequeue {numElements} elements elapsed: {watch.Elapsed}"); } watch.Reset(); watch.Start(); int len = queue.Count; WorkloadEvent evnt = null; while (queue.TryDequeue(out evnt)) { ; } watch.Stop(); Console.WriteLine($"Dequeue all {len} remaining elements elapsed: {watch.Elapsed}"); Assert.AreEqual(queue.Count, 0); } }
public override WorkloadEvent Read() { WorkloadEvent result = null; double msSleep = 0; try { if (reader == null) { return(null); } bool validEventFound = false; do { if (!reader.Read()) { stopped = true; return(null); } result = ReadEvent(reader); // Handle replay sleep for synchronization mode // The sleep cannot happen here, but it has to // happen later in the replay workflow, because // it would only delay the insertion in the queue // and it would not separate the events during the replay if (result is ExecutionWorkloadEvent) { ExecutionWorkloadEvent execEvent = result as ExecutionWorkloadEvent; if (SynchronizationMode) { DateTime lastEndTime = GetLastEventEndTime((int)execEvent.SPID); if (lastEndTime != DateTime.MinValue) { msSleep = (result.StartTime - lastEndTime).TotalMilliseconds; if (msSleep > 0) { if (msSleep > Int32.MaxValue) { msSleep = Int32.MaxValue; } Thread.Sleep(Convert.ToInt32(msSleep)); } } SetLastEventEndTime((int)execEvent.SPID, execEvent.StartTime.AddMilliseconds((double)execEvent.Duration / 1000)); } else { execEvent.ReplaySleepMilliseconds = 0; } } // Filter events if (result is ExecutionWorkloadEvent) { validEventFound = Filter.Evaluate(result); } else { validEventFound = true; } }while (!validEventFound); } catch (Exception e) { if (stopped) { return(null); } DateTime?eventDate = null; if (result != null) { eventDate = result.StartTime; } logger.Error(e); logger.Error($"Unable to read next event. Current event date: {eventDate}"); throw; } return(result); }
public override void Write(WorkloadEvent evt) { consumer.Consume(evt); }
public override WorkloadEvent Read() { WorkloadEvent result = null; double msSleep = 0; try { if (reader == null) { return(null); } bool validEventFound = false; do { if (!reader.Read()) { stopped = true; return(null); } result = ReadEvent(reader); if (SynchronizationMode) { if (previousDate != DateTime.MinValue) { msSleep = (result.StartTime - previousDate).TotalMilliseconds; if (msSleep > 0) { if (msSleep > Int32.MaxValue) { msSleep = Int32.MaxValue; } Thread.Sleep(Convert.ToInt32(msSleep)); } } } previousDate = result.StartTime; // Filter events if (result is ExecutionWorkloadEvent) { validEventFound = Filter.Evaluate(result); } else { validEventFound = true; } }while (!validEventFound); } catch (Exception e) { if (stopped) { return(null); } DateTime?eventDate = null; if (result != null) { eventDate = result.StartTime; } logger.Error(e); logger.Error($"Unable to read next event. Current event date: {eventDate} Last event date: {previousDate} Requested sleep: {msSleep}"); throw; } return(result); }
public override void ConsumeBuffered(WorkloadEvent evnt) { if (!(evnt is ExecutionWorkloadEvent)) { return; } if (evnt.Type != WorkloadEvent.EventType.RPCCompleted && evnt.Type != WorkloadEvent.EventType.BatchCompleted) { return; } ExecutionWorkloadEvent evt = (ExecutionWorkloadEvent)evnt; ReplayCommand command = new ReplayCommand() { CommandText = evt.Text, Database = evt.DatabaseName, ApplicationName = evt.ApplicationName }; int session_id = -1; session_id = (int)evt.SPID; ReplayWorker rw = null; if (ReplayWorkers.TryGetValue(session_id, out rw)) { rw.AppendCommand(command); } else { rw = new ReplayWorker() { ConnectionInfo = this.ConnectionInfo, ReplayIntervalSeconds = 0, StopOnError = false, Name = session_id.ToString() }; ReplayWorkers.TryAdd(session_id, rw); rw.AppendCommand(command); logger.Info(String.Format("Worker [{0}] - Starting", session_id)); } if (runner == null) { runner = new Thread(new ThreadStart( delegate { try { RunWorkers(); } catch (Exception e) { try { logger.Error(e, "Unhandled exception in TraceManager.RunWorkers"); } catch { Console.WriteLine(e.Message); } } } )); runner.IsBackground = true; runner.Start(); } if (sweeper == null) { sweeper = new Thread(new ThreadStart( delegate { try { RunSweeper(); } catch (Exception e) { try { logger.Error(e, "Unhandled exception in TraceManager.RunSweeper"); } catch { Console.WriteLine(e.Message); } } } )); sweeper.IsBackground = true; sweeper.Start(); } }
public override void ConsumeBuffered(WorkloadEvent evnt) { if (!(evnt is ExecutionWorkloadEvent)) { return; } if (evnt.Type != WorkloadEvent.EventType.RPCCompleted && evnt.Type != WorkloadEvent.EventType.BatchCompleted) { return; } eventCount++; if ((eventCount > 0) && (eventCount % WorkerStatsCommandCount == 0)) { logger.Info($"{eventCount} events queued for replay"); } ExecutionWorkloadEvent evt = (ExecutionWorkloadEvent)evnt; ReplayCommand command = new ReplayCommand() { CommandText = evt.Text, Database = evt.DatabaseName, ApplicationName = evt.ApplicationName, BeforeSleepMilliseconds = evt.ReplaySleepMilliseconds }; int session_id = -1; session_id = (int)evt.SPID; ReplayWorker rw = null; if (ReplayWorkers.TryGetValue(session_id, out rw)) { // Ensure that the buffer does not get too big while (rw.QueueLength >= (BufferSize * .9)) { spin.SpinOnce(); } rw.AppendCommand(command); } else { rw = new ReplayWorker() { ConnectionInfo = this.ConnectionInfo, ReplayIntervalSeconds = 0, StopOnError = false, Name = session_id.ToString(), DisplayWorkerStats = this.DisplayWorkerStats, ConsumeResults = this.ConsumeResults, QueryTimeoutSeconds = this.QueryTimeoutSeconds, WorkerStatsCommandCount = this.WorkerStatsCommandCount, MimicApplicationName = this.MimicApplicationName }; ReplayWorkers.TryAdd(session_id, rw); rw.AppendCommand(command); logger.Info($"Worker [{session_id}] - Starting"); } if (runner == null) { runner = new Thread(new ThreadStart( delegate { try { RunWorkers(); } catch (Exception e) { try { logger.Error(e, "Unhandled exception in TraceManager.RunWorkers"); } catch { Console.WriteLine(e.Message); } } } )); runner.IsBackground = true; runner.Start(); } if (sweeper == null) { sweeper = new Thread(new ThreadStart( delegate { try { RunSweeper(); } catch (Exception e) { try { logger.Error(e, "Unhandled exception in TraceManager.RunSweeper"); } catch { Console.WriteLine(e.Message); } } } )); sweeper.IsBackground = true; sweeper.Start(); } }
private void ProcessQueue() { DateTime lastDump = DateTime.Now; while (!stopped) { // Write collected data to the destination database TimeSpan duration = DateTime.Now - lastDump; if (duration.TotalMinutes >= Interval) { try { int numRetries = 0; while (numRetries <= MAX_WRITE_RETRIES) { try { WriteToServer(); numRetries = MAX_WRITE_RETRIES + 1; } catch (Exception ex) { logger.Warn("Unable to write workload analysis."); logger.Warn(ex.Message); if (numRetries == MAX_WRITE_RETRIES) { throw; } } numRetries++; } } catch (Exception e) { try { logger.Error(e, "Unable to write workload analysis info to the destination database."); logger.Error(e.StackTrace); } catch { Console.WriteLine(String.Format("Unable to write to the database: {0}.", e.Message)); } } finally { lastDump = DateTime.Now; } } if (_internalQueue.Count == 0) { Thread.Sleep(10); continue; } lock (_internalQueue) { WorkloadEvent data = _internalQueue.Dequeue(); _internalAdd(data); } } }
public abstract void ConsumeBuffered(WorkloadEvent evt);