/// <summary>
/// Sets the path of the log files.
/// </summary>
/// <param name="logDirectory"></param>
public void SetPath(string logDirectory)
{
FilePath.ValidatePathName(logDirectory);
lock (m_syncRoot)
{
m_path = logDirectory;
m_flushTask.Start(1000);
}
}
/// <summary>
/// Provides a thread safe way to enqueue points.
/// While points are streaming all other writes are blocked. Therefore,
/// this point stream should be high speed.
/// </summary>
/// <param name="stream"></param>
public void Enqueue(TreeStream <HistorianKey, HistorianValue> stream)
{
lock (m_syncWrite)
{
PointData data = default(PointData);
while (data.Load(stream))
{
m_blocks.Enqueue(data);
}
}
m_worker.Start();
}
public void CanRunAfterError()
{
var times = 0;
var errors = 0;
ScheduledTask s = null;
s = new ScheduledTask(() =>
{
times++;
if (times == 2)
{
throw new Exception("error");
}
if (times == 4)
{
s.Stop();
}
})
{
StartInterval = 10, AfterSuccessInterval = 10, AfterErrorInterval = 10, ProceedOnError = true
};
s.OnError += _ => errors++;
s.Start();
Thread.Sleep(1000);
Assert.AreEqual(4, times);
Assert.AreEqual(1, errors);
}
/// <summary>
/// Invalidates the current routing table.
/// </summary>
private void RecalculateRoutingTable()
{
RoutingTablesValid = false;
//Wait some time before recalculating the routing tables. This will be done automatically
//if a message is routed before the table is recalculated.
m_calculateRoutingTable.Start(10);
}
/// <summary>
/// Queues the supplied file as a file that needs to be deleted.
/// MUST be called from a synchronized context.
/// </summary>
/// <param name="file"></param>
void AddFileToDelete(SortedTreeTable <TKey, TValue> file)
{
if (file.BaseFile.IsMemoryFile)
{
AddFileToDispose(file);
return;
}
if (!InternalIsFileBeingUsed(file))
{
file.BaseFile.Delete();
return;
}
m_listLog.AddFileToDelete(file.ArchiveId);
m_filesToDelete.Add(file);
m_processRemovals.Start(1000);
}
private void CollectionRunning(object sender, EventArgs <ScheduledTaskRunningReason> e)
{
m_collection.Start(1000);
m_countHistory.Enqueue(m_queue.Count);
if (m_countHistory.Count >= 60)
{
int objectsCreated = Interlocked.Exchange(ref m_objectsCreated, 0);
//if there were ever more than the target items in the queue over the past 60 seconds
//remove some items.
//However, don't remove items if the pool ever got to 0 and had objects that had to be created.
int min = m_countHistory.Min();
m_countHistory.Clear();
if (objectsCreated > 250)
{
Log.Publish(MessageLevel.Error, MessageFlags.PerformanceIssue | MessageFlags.UsageIssue, "Items Created since last collection cycle.", (objectsCreated).ToString());
}
else if (objectsCreated > 50)
{
Log.Publish(MessageLevel.Warning, "Items Created since last collection cycle.", (objectsCreated).ToString());
}
else if (objectsCreated > 0)
{
Log.Publish(MessageLevel.Info, "Items Created since last collection cycle.", (objectsCreated).ToString());
}
if (min > m_targetCount && objectsCreated == 0)
{
int itemsRemoved = (min - m_targetCount);
if (itemsRemoved > 250)
{
Log.Publish(MessageLevel.Error, MessageFlags.PerformanceIssue | MessageFlags.UsageIssue, "Removing items", itemsRemoved.ToString());
}
else if (itemsRemoved > 50)
{
Log.Publish(MessageLevel.Warning, "Removing items", itemsRemoved.ToString());
}
else if (itemsRemoved > 0)
{
Log.Publish(MessageLevel.Info, "Removing items", itemsRemoved.ToString());
}
}
while (min > m_targetCount && objectsCreated == 0)
{
T item;
if (m_queue.TryDequeue(out item))
{
(item as IDisposable)?.Dispose();
}
else
{
return;
}
min--;
}
}
}
public void TestMethod1()
{
m_task = new ScheduledTask(ThreadingMode.DedicatedBackground, ThreadPriority.Highest);
m_task.Running += task_Running;
m_task.Start(10);
Thread.Sleep(1000);
m_task.Dispose();
System.Console.WriteLine("Disposed");
}
/// <summary>
/// Attempts to connect to data input source.
/// </summary>
protected override void AttemptConnection()
{
m_nextPublicationTime = ToPublicationTime(DateTime.UtcNow.Ticks);
m_nextPublicationTimeWithLatency = m_nextPublicationTime + (long)(m_latency.Next() * TimeSpan.TicksPerMillisecond);
if ((object)m_timer == null)
{
m_timer = new ScheduledTask(ThreadingMode.ThreadPool);
m_timer.Running += m_timer_Running;
}
if ((object)m_statusUpdate == null)
{
m_statusUpdate = new ScheduledTask(ThreadingMode.ThreadPool);
m_statusUpdate.Running += m_statusUpdate_Running;
}
m_timer.Start();
m_statusUpdate.Start(10000);
}
public void RoutingComplete()
{
if (MeasurementsToRoute.Count > 0)
{
m_pendingMeasurements.Enqueue(MeasurementsToRoute);
MeasurementsToRoute = new List <IMeasurement>();
m_task.Start();
}
}
void task_Running(object sender, EventArgs <ScheduledTaskRunningReason> e)
{
if (e.Argument == ScheduledTaskRunningReason.Disposing)
{
return;
}
m_task.Start(10);
m_count++;
System.Console.WriteLine(m_count);
}
void TestTimed(ThreadingMode mode)
{
const int Count = 1000000000;
Stopwatch sw = new Stopwatch();
m_doWorkCount = 0;
using (ScheduledTask work = new ScheduledTask(mode))
{
work.Running += work_DoWork;
sw.Start();
for (int x = 0; x < 1000; x++)
{
work.Start(1);
work.Start();
}
sw.Stop();
}
m_doWorkCount = 0;
sw.Reset();
using (ScheduledTask work = new ScheduledTask(mode))
{
work.Running += work_DoWork;
sw.Start();
for (int x = 0; x < Count; x++)
{
work.Start(1000);
work.Start();
}
sw.Stop();
}
Console.WriteLine(mode.ToString());
Console.WriteLine(" Fire Event Count: " + m_doWorkCount.ToString());
Console.WriteLine(" Fire Event Rate: " + (m_doWorkCount / sw.Elapsed.TotalSeconds / 1000000).ToString("0.00"));
Console.WriteLine(" Total Calls Time: " + sw.Elapsed.TotalMilliseconds.ToString("0.0") + "ms");
Console.WriteLine(" Total Calls Rate: " + (Count / sw.Elapsed.TotalSeconds / 1000000).ToString("0.00"));
Console.WriteLine();
}
public void Start()
{
lock (this)
{
//Check if the last queue has processed, if not, then quit.
if (m_time.HasValue)
{
return;
}
m_time = ShortTime.Now;
if (m_delay == 0)
{
m_task.Start();
}
else
{
m_task.Start(m_delay);
}
}
}
public void ScheduledTask_Test()
{
// create and start the task
var task = new ScheduledTask("Task", ct => Thread.Sleep(500), 100);
task.Changed += (s, e) => ReportAction(e);
task.Start();
// allow the task time to be scheduled and run
Thread.Sleep(5000);
// stop the task
task.Stop();
// wait for completion (only in unit tests)
task.Wait();
}
/// <summary>
/// Triggers a rollover if the provided transaction id has not yet been triggered.
/// This method does not block
/// </summary>
/// <param name="transactionId">the transaction id to execute the commit on.</param>
public void Commit(long transactionId)
{
lock (m_syncRoot)
{
if (m_stopped)
{
if (m_disposed)
{
Log.Publish(MessageLevel.Warning, "Disposed Object", "A call to Commit() occured after this class disposed");
return;
}
else
{
Log.Publish(MessageLevel.Warning, "Writer Stopped", "A call to Commit() occured after this class was stopped");
return;
}
}
if (transactionId > m_currentTransactionIdRollingOver)
{
m_rolloverTask.Start();
}
}
}
public void ScheduledTask_Test()
{
// create and start the task
var task = new ScheduledTask("Task", ct => Thread.Sleep(500), 100);
task.Changed += (s, e) => ReportAction(e);
task.Start();
// allow the task time to be scheduled and run
Thread.Sleep(5000);
// stop the task
task.Stop();
// wait for completion (only in unit tests)
task.Wait();
}
public void Test()
{
using (ScheduledTask work = new ScheduledTask(ThreadingMode.DedicatedForeground))
{
work.Running += work_DoWork;
work.Running += work_CleanupWork;
work.Start();
}
double x = 1;
while (x > 3)
{
x--;
}
}
void BlastStartMethod(object obj)
{
try
{
ScheduledTask task = (ScheduledTask)obj;
const int Count = 100000000;
for (int x = 0; x < Count; x++)
{
task.Start();
}
}
catch (Exception)
{
}
}
/// <summary>
/// Creates a new Resource Queue.
/// </summary>
/// <param name="instance">A delegate that will return the necessary queue.</param>
/// <param name="targetCount">the ideal number of objects that are always pending on the queue.</param>
public DynamicObjectPool(Func <T> instance, int targetCount)
{
if (instance == null)
{
throw new ArgumentNullException(nameof(instance));
}
m_targetCount = targetCount;
m_countHistory = new Queue <int>(100);
m_instanceObject = instance;
m_queue = new ConcurrentQueue <T>();
m_collection = new ScheduledTask();
m_collection.Running += CollectionRunning;
m_collection.Start(1000);
}
/// <summary>
/// Creates a stage writer.
/// </summary>
/// <param name="settings">the settings for this stage</param>
/// <param name="archiveList">the archive list</param>
/// <param name="rolloverLog">the rollover log</param>
public CombineFiles(CombineFilesSettings settings, ArchiveList <TKey, TValue> archiveList, RolloverLog rolloverLog)
: base(MessageClass.Framework)
{
m_settings = settings.CloneReadonly();
m_settings.Validate();
m_archiveList = archiveList;
m_createNextStageFile = new SimplifiedArchiveInitializer <TKey, TValue>(settings.ArchiveSettings);
m_rolloverLog = rolloverLog;
m_rolloverComplete = new ManualResetEvent(false);
m_syncRoot = new object();
m_rolloverTask = new ScheduledTask(ThreadingMode.DedicatedForeground, ThreadPriority.BelowNormal);
m_rolloverTask.Running += OnExecute;
m_rolloverTask.UnhandledException += OnException;
m_rolloverTask.Start(m_settings.ExecuteTimer);
}
public void Test()
{
using (ScheduledTask work = new ScheduledTask(ThreadingMode.DedicatedForeground))
{
work.Running += work_DoWork;
work.Running += work_CleanupWork;
work.Start();
}
double x = 1;
while (x > 3)
{
x--;
}
}
void TestConcurrent(ThreadingMode mode)
{
int workCount;
const int Count = 100000000;
Stopwatch sw = new Stopwatch();
m_doWorkCount = 0;
using (ScheduledTask work = new ScheduledTask(mode))
{
work.Running += work_DoWork;
sw.Start();
for (int x = 0; x < 1000; x++)
{
work.Start();
}
sw.Stop();
}
m_doWorkCount = 0;
sw.Reset();
using (ScheduledTask work = new ScheduledTask(mode))
{
work.Running += work_DoWork;
sw.Start();
ThreadPool.QueueUserWorkItem(BlastStartMethod, work);
ThreadPool.QueueUserWorkItem(BlastStartMethod, work);
for (int x = 0; x < Count; x++)
{
work.Start();
}
workCount = m_doWorkCount;
sw.Stop();
Thread.Sleep(100);
}
Console.WriteLine(mode.ToString());
Console.WriteLine(" Fire Event Count: " + workCount.ToString());
Console.WriteLine(" Fire Event Rate: " + (workCount / sw.Elapsed.TotalSeconds / 1000000).ToString("0.00"));
Console.WriteLine(" Total Calls Time: " + sw.Elapsed.TotalMilliseconds.ToString("0.0") + "ms");
Console.WriteLine(" Total Calls Rate: " + (Count / sw.Elapsed.TotalSeconds / 1000000).ToString("0.00"));
Console.WriteLine();
}
/// <summary>
/// Creates a <see cref="RouteMappingHighLatencyLowCpu"/>
/// </summary>
public RouteMappingHighLatencyLowCpu()
{
m_lastStatusUpdate = ShortTime.Now;
m_maxPendingMeasurements = 1000;
m_routeLatency = OptimizationOptions.RoutingLatency;
m_batchSize = OptimizationOptions.RoutingBatchSize;
m_inboundQueue = new ConcurrentQueue <List <IMeasurement> >();
m_task = new ScheduledTask(ThreadingMode.DedicatedBackground, ThreadPriority.AboveNormal);
m_task.Running += m_task_Running;
m_task.UnhandledException += m_task_UnhandledException;
m_task.Disposing += m_task_Disposing;
m_task.Start(m_routeLatency);
m_onStatusMessage = x => { };
m_onProcessException = x => { };
m_globalCache = new GlobalCache(new Dictionary <IAdapter, Consumer>(), 0);
RouteCount = m_globalCache.GlobalSignalLookup.Count(x => x != null);
}
public void CanRunRegular()
{
var times = 0;
ScheduledTask s = null;
s = new ScheduledTask(() => {
Console.WriteLine("CanRunRegular " + times);
times++;
if (times == 3)
{
s.Stop();
}
})
{
StartInterval = 10, AfterSuccessInterval = 10
};
s.Start();
Thread.Sleep(1000);
Assert.AreEqual(3, times);
}
public void ScheduledTask_WithCancellation_Test()
{
// create and start the task
var task = new ScheduledTask("Task", ct => Thread.Sleep(500), 100);
task.Changed += (s, e) => ReportAction(e);
task.Start();
// schedule a cancellation in 1 to 5 seconds
var cancelTask = Task.Factory.StartNew(() =>
{
var random = new Random();
Thread.Sleep(random.Next(1000, 5000));
task.Cancel();
});
// allow the task time to be scheduled and run
Thread.Sleep(5000);
// stop the task
task.Stop();
// wait for completion (only in unit tests)
task.Wait();
// wait for completion of cancellation task
cancelTask.Wait();
}
/// <summary>
/// Creates a new Resource Queue.
/// </summary>
/// <param name="instance">A delegate that will return the necessary queue.</param>
/// <param name="lifetimeInSeconds">The age of the object before it is expired.</param>
public ExpireObjectPool(Func <T> instance, int lifetimeInSeconds)
{
if (instance == null)
{
throw new ArgumentNullException(nameof(instance));
}
m_syncRoot = new object();
m_lifeCycleInSeconds = lifetimeInSeconds;
m_instanceObject = instance;
m_queue = new Queue <T>();
m_allocations = 0;
m_enqueueCount = 0;
m_dequeueCount = 0;
m_maxQueueSize = 0;
m_minQueueSize = 0;
m_collection = new ScheduledTask();
m_collection.Running += CollectionRunning;
m_collection.Start(1000 * m_lifeCycleInSeconds);
}
/// <summary>
/// Creates a <see cref="RouteMappingHighLatencyLowCpu"/>
/// </summary>
/// <param name="routeLatency">The desired wait latency. Must be between 1 and 500ms inclusive</param>
public RouteMappingHighLatencyLowCpu(int routeLatency)
{
if (routeLatency < 1 || routeLatency > 500)
{
throw new ArgumentOutOfRangeException("routeLatency", "Must be between 1 and 500ms");
}
m_maxPendingMeasurements = 1000;
m_routeLatency = routeLatency;
m_list = new ConcurrentQueue <IMeasurement[]>();
m_task = new ScheduledTask(ThreadingMode.DedicatedBackground, ThreadPriority.AboveNormal);
m_task.Running += m_task_Running;
m_task.UnhandledException += m_task_UnhandledException;
m_task.Disposing += m_task_Disposing;
m_task.Start(m_routeLatency);
m_onStatusMessage = x => { };
m_onProcessException = x => { };
m_producerLookup = new Dictionary <IAdapter, LocalCache>();
m_globalCache = new GlobalCache(new Dictionary <IAdapter, Consumer>(), 0);
m_injectMeasurementsLocalCache = new LocalCache(this, null);
}
public void ScheduledTask_WithCancellation_Test()
{
// create and start the task
var task = new ScheduledTask("Task", ct => Thread.Sleep(500), 100);
task.Changed += (s, e) => ReportAction(e);
task.Start();
// schedule a cancellation in 1 to 5 seconds
var cancelTask = Task.Factory.StartNew(() =>
{
var random = new Random();
Thread.Sleep(random.Next(1000, 5000));
task.Cancel();
});
// allow the task time to be scheduled and run
Thread.Sleep(5000);
// stop the task
task.Stop();
// wait for completion (only in unit tests)
task.Wait();
// wait for completion of cancellation task
cancelTask.Wait();
}
private void AsyncDoWork(object sender, EventArgs eventArgs)
{
if (BeforeExecuteQuery != null)
{
BeforeExecuteQuery(this, EventArgs.Empty);
}
DateTime startTime;
DateTime stopTime;
DateTime currentTime;
object token;
List <MetadataBase> activeSignals;
lock (m_syncRoot)
{
token = m_requestToken;
m_requestToken = null;
if (Mode == ExecutionMode.Manual)
{
startTime = m_lowerBounds;
stopTime = m_upperBounds;
currentTime = m_focusedDate;
activeSignals = m_activeSignals;
}
else
{
m_playback.GetTimes(out startTime, out stopTime);
currentTime = stopTime;
activeSignals = m_activeSignals;
}
}
IDictionary <Guid, SignalDataBase> results = m_query.GetQueryResult(startTime, stopTime, 0, activeSignals);
QueryResultsEventArgs queryResults = new QueryResultsEventArgs(results, token, startTime, stopTime);
if (AfterQuery != null)
{
AfterQuery(this, EventArgs.Empty);
}
if (NewQueryResults != null)
{
NewQueryResults.ParallelRunAndWait(this, queryResults);
}
if (SynchronousNewQueryResults != null || ParallelWithControlLockNewQueryResults != null)
{
m_syncEvent.RaiseEvent(new QueryResultsEventArgs(results, token, startTime, stopTime));
}
lock (m_syncRoot)
{
if (Mode == ExecutionMode.Automatic)
{
m_async.Start(m_refreshInterval.Milliseconds);
}
}
if (AfterExecuteQuery != null)
{
AfterExecuteQuery(this, EventArgs.Empty);
}
}
private void OnExecute(object sender, EventArgs <ScheduledTaskRunningReason> e)
{
//The worker can be disposed either via the Stop() method or
//the Dispose() method. If via the dispose method, then
//don't do any cleanup.
if (m_disposed && e.Argument == ScheduledTaskRunningReason.Disposing)
{
return;
}
//go ahead and schedule the next rollover since nothing
//will happen until this function exits anyway.
//if the task is disposing, the following line does nothing.
m_rolloverTask.Start(m_settings.ExecuteTimer);
lock (m_syncRoot)
{
if (m_disposed)
{
return;
}
using (ArchiveListSnapshot <TKey, TValue> resource = m_archiveList.CreateNewClientResources())
{
resource.UpdateSnapshot();
List <ArchiveTableSummary <TKey, TValue> > list = new List <ArchiveTableSummary <TKey, TValue> >();
List <Guid> listIds = new List <Guid>();
for (int x = 0; x < resource.Tables.Length; x++)
{
ArchiveTableSummary <TKey, TValue> table = resource.Tables[x];
if (table.SortedTreeTable.BaseFile.Snapshot.Header.Flags.Contains(m_settings.MatchFlag) && table.SortedTreeTable.BaseFile.Snapshot.Header.Flags.Contains(FileFlags.IntermediateFile))
{
list.Add(table);
listIds.Add(table.FileId);
}
else
{
resource.Tables[x] = null;
}
}
bool shouldRollover = list.Count >= m_settings.CombineOnFileCount;
long size = 0;
for (int x = 0; x < list.Count; x++)
{
size += list[x].SortedTreeTable.BaseFile.ArchiveSize;
if (size > m_settings.CombineOnFileSize)
{
if (x != list.Count - 1)//If not the last entry
{
list.RemoveRange(x + 1, list.Count - x - 1);
}
break;
}
}
if (size > m_settings.CombineOnFileSize)
{
shouldRollover = true;
}
if (shouldRollover)
{
TKey startKey = new TKey();
TKey endKey = new TKey();
startKey.SetMax();
endKey.SetMin();
foreach (Guid fileId in listIds)
{
ArchiveTableSummary <TKey, TValue> table = resource.TryGetFile(fileId);
if (table is null)
{
throw new Exception("File not found");
}
if (!table.IsEmpty)
{
if (startKey.IsGreaterThan(table.FirstKey))
{
table.FirstKey.CopyTo(startKey);
}
if (endKey.IsLessThan(table.LastKey))
{
table.LastKey.CopyTo(endKey);
}
}
}
RolloverLogFile logFile = null;
Action <Guid> createLog = (x) =>
{
logFile = m_rolloverLog.Create(listIds, x);
};
using (UnionReader <TKey, TValue> reader = new UnionReader <TKey, TValue>(list))
{
SortedTreeTable <TKey, TValue> dest = m_createNextStageFile.CreateArchiveFile(startKey, endKey, size, reader, createLog);
resource.Dispose();
using (ArchiveListEditor <TKey, TValue> edit = m_archiveList.AcquireEditLock())
{
//Add the newly created file.
edit.Add(dest);
foreach (ArchiveTableSummary <TKey, TValue> table in list)
{
edit.TryRemoveAndDelete(table.FileId);
}
}
}
if (logFile != null)
{
logFile.Delete();
}
}
resource.Dispose();
}
m_rolloverComplete.Set();
}
}
private void CollectionRunning(object sender, EventArgs <ScheduledTaskRunningReason> e)
{
m_collection.Start(1000 * m_lifeCycleInSeconds);
StringBuilder sb = new StringBuilder(1000);
int allocations = 0;
lock (m_syncRoot)
{
//Determines how many items in this list have expired based on time.
int itemsProcessedPerCollectionCycle = Math.Max(m_enqueueCount, m_dequeueCount);
int error = Math.Max(4, itemsProcessedPerCollectionCycle >> 2); //Allow 25% error.
int expiredItemsCount = m_queue.Count - itemsProcessedPerCollectionCycle - error;
//Determine how many items in this list are never accessed.
//The minimum size of the queue during this window, minus 25% of the range of the queue.
int listSizeExcess = m_minQueueSize - ((m_maxQueueSize - m_minQueueSize) >> 2);
int retireItems = Math.Max(listSizeExcess, expiredItemsCount);
//Don't shrink the list based on the number of allocations.
for (int x = m_allocations; x < retireItems; x++)
{
(m_queue.Dequeue() as IDisposable)?.Dispose();
}
sb.Append("Allocations: ");
sb.Append(m_allocations);
sb.Append(" Enqueue Count: ");
sb.Append(m_enqueueCount);
sb.Append(" Dequeue Count: ");
sb.Append(m_dequeueCount);
sb.Append(" Max Queue Size: ");
sb.Append(m_maxQueueSize);
sb.Append(" Min Queue Size: ");
sb.Append(m_minQueueSize);
sb.Append(" Expired Items Count: ");
sb.Append(expiredItemsCount);
sb.Append(" List Size Excess: ");
sb.Append(listSizeExcess);
allocations = m_allocations;
m_allocations = 0;
m_enqueueCount = 0;
m_dequeueCount = 0;
m_maxQueueSize = m_queue.Count;
m_minQueueSize = m_queue.Count;
}
if (allocations > 250)
{
m_log.Publish(MessageLevel.Error, MessageFlags.PerformanceIssue | MessageFlags.UsageIssue, "Object Pool is likely not optimized", sb.ToString());
}
else if (allocations > 50)
{
m_log.Publish(MessageLevel.Warning, "Items Created since last collection cycle.", sb.ToString());
}
else if (allocations > 0)
{
m_log.Publish(MessageLevel.Info, "Items Created since last collection cycle.", sb.ToString());
}
}
void m_task_Running(object sender, EventArgs <ScheduledTaskRunningReason> e)
{
if (e.Argument == ScheduledTaskRunningReason.Disposing)
{
return;
}
m_task.Start(m_routeLatency);
m_routeOperations++;
if (m_lastStatusUpdate.ElapsedSeconds() > 15)
{
m_lastStatusUpdate = ShortTime.Now;
Log.Publish(MessageLevel.Info, MessageFlags.None, "Routing Update",
string.Format("Route Operations: {0}, Input Frames: {1}, Input Measurements: {2}, Output Measurements: {3}",
m_routeOperations, m_measurementsRoutedInputFrames,
m_measurementsRoutedInputMeasurements,
m_measurementsRoutedOutput));
}
var map = m_globalCache;
try
{
int measurementsRouted = 0;
List <IMeasurement> measurements;
while (m_inboundQueue.TryDequeue(out measurements))
{
measurementsRouted += measurements.Count;
Interlocked.Add(ref m_pendingMeasurements, -measurements.Count);
//For loops are faster than ForEach for List<T>
//Process Optimized Consumers
for (int x = 0; x < map.RoutingPassthroughAdapters.Count; x++)
{
map.RoutingPassthroughAdapters[x].ProcessMeasurementList(measurements);
}
//Process Broadcast Consumers
for (int x = 0; x < map.BroadcastConsumers.Count; x++)
{
m_measurementsRoutedOutput += measurements.Count;
map.BroadcastConsumers[x].MeasurementsToRoute.AddRange(measurements);
}
m_measurementsRoutedInputFrames++;
m_measurementsRoutedInputMeasurements += measurements.Count;
for (int x = 0; x < measurements.Count; x++)
{
var measurement = measurements[x];
List <Consumer> consumers = map.GlobalSignalLookup[measurement.Key.RuntimeID];
if (consumers != null)
{
for (int i = 0; i < consumers.Count; i++)
{
m_measurementsRoutedOutput++;
consumers[i].MeasurementsToRoute.Add(measurement);
}
}
}
//If any adapter has too many measurements on their batch
//Route all adapter's measurements
if (measurementsRouted > m_batchSize)
{
measurementsRouted = 0;
foreach (var consumer in map.NormalDestinationAdapters)
{
measurementsRouted = Math.Max(measurementsRouted, consumer.MeasurementsToRoute.Count);
if (consumer.MeasurementsToRoute.Count > m_batchSize)
{
foreach (var c2 in map.NormalDestinationAdapters)
{
c2.RoutingComplete();
}
measurementsRouted = 0;
break;
}
}
}
}
}
finally
{
foreach (var consumer in map.NormalDestinationAdapters)
{
consumer.RoutingComplete();
}
}
}
void m_task_Running(object sender, EventArgs <ScheduledTaskRunningReason> e)
{
if (e.Argument == ScheduledTaskRunningReason.Disposing)
{
return;
}
m_task.Start(m_routeLatency);
m_routeOperations++;
if (m_routeOperations % 1000 == 0)
{
m_onStatusMessage(string.Format(
"Route Operations: {0}, Input Frames: {1}, Input Measurements: {2}, Output Measurements: {3}",
m_routeOperations, m_measurementsRoutedInputFrames,
m_measurementsRoutedInputMeasurements,
m_measurementsRoutedOutput));
}
var map = m_globalCache;
try
{
IMeasurement[] measurements;
while (m_list.TryDequeue(out measurements))
{
Interlocked.Add(ref m_pendingMeasurements, -measurements.Length);
m_measurementsRoutedInputFrames++;
m_measurementsRoutedInputMeasurements += measurements.Length;
foreach (var measurement in measurements)
{
List <Consumer> consumers;
if (!map.GlobalSignalLookup.TryGetValue(measurement.ID, out consumers))
{
consumers = map.BroadcastConsumers;
}
// Add this measurement to the producers' list
for (int index = 0; index < consumers.Count; index++)
{
var consumer = consumers[index];
m_measurementsRoutedOutput++;
consumer.MeasurementsToRoute.Add(measurement);
}
}
//Limit routing to no more than 1000 measurements per sub-route.
foreach (var consumer in map.GlobalDestinationList)
{
if (consumer.MeasurementsToRoute.Count > 1000)
{
foreach (var c2 in map.GlobalDestinationLookup.Values)
{
c2.RoutingComplete();
}
break;
}
}
}
}
finally
{
foreach (var consumer in map.GlobalDestinationList)
{
consumer.RoutingComplete();
}
}
}
/// <summary>
/// Appends this data to this stage. Also queues up for deletion if necessary.
/// </summary>
/// <param name="args">arguments handed to this class from either the
/// PrestageWriter or another StageWriter of a previous generation</param>
/// <remarks>
/// This method must be called in a single threaded manner.
/// </remarks>
public void AppendData(PrebufferRolloverArgs <TKey, TValue> args)
{
if (m_stopped)
{
Log.Publish(MessageLevel.Info, "No new points can be added. Point queue has been stopped. Data in rollover will be lost");
return;
}
if (m_disposed)
{
Log.Publish(MessageLevel.Info, "First stage writer has been disposed. Data in rollover will be lost");
return;
}
SortedTreeFile file = SortedTreeFile.CreateInMemory(4096);
SortedTreeTable <TKey, TValue> table = file.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod);
using (SortedTreeTableEditor <TKey, TValue> edit = table.BeginEdit())
{
edit.AddPoints(args.Stream);
edit.Commit();
}
bool shouldWait = false;
//If there is data to write then write it to the current archive.
lock (m_syncRoot)
{
if (m_stopped)
{
Log.Publish(MessageLevel.Info, "No new points can be added. Point queue has been stopped. Data in rollover will be lost");
table.Dispose();
return;
}
if (m_disposed)
{
Log.Publish(MessageLevel.Info, "First stage writer has been disposed. Data in rollover will be lost");
table.Dispose();
return;
}
using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock())
{
edit.Add(table);
}
m_pendingTables1.Add(table);
if (m_pendingTables1.Count == 10)
{
using (UnionTreeStream <TKey, TValue> reader = new UnionTreeStream <TKey, TValue>(m_pendingTables1.Select(x => new ArchiveTreeStreamWrapper <TKey, TValue>(x)), true))
{
SortedTreeFile file1 = SortedTreeFile.CreateInMemory(4096);
SortedTreeTable <TKey, TValue> table1 = file1.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod);
using (SortedTreeTableEditor <TKey, TValue> edit = table1.BeginEdit())
{
edit.AddPoints(reader);
edit.Commit();
}
using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock())
{
//Add the newly created file.
edit.Add(table1);
foreach (SortedTreeTable <TKey, TValue> table2 in m_pendingTables1)
{
edit.TryRemoveAndDelete(table2.ArchiveId);
}
}
m_pendingTables2.Add(table1);
m_pendingTables1.Clear();
}
}
if (m_pendingTables2.Count == 10)
{
using (UnionTreeStream <TKey, TValue> reader = new UnionTreeStream <TKey, TValue>(m_pendingTables2.Select(x => new ArchiveTreeStreamWrapper <TKey, TValue>(x)), true))
{
SortedTreeFile file1 = SortedTreeFile.CreateInMemory(4096);
SortedTreeTable <TKey, TValue> table1 = file1.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod);
using (SortedTreeTableEditor <TKey, TValue> edit = table1.BeginEdit())
{
edit.AddPoints(reader);
edit.Commit();
}
using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock())
{
//Add the newly created file.
edit.Add(table1);
foreach (SortedTreeTable <TKey, TValue> table2 in m_pendingTables2)
{
edit.TryRemoveAndDelete(table2.ArchiveId);
}
}
m_pendingTables3.Add(table1);
m_pendingTables2.Clear();
}
}
m_lastCommitedSequenceNumber.Value = args.TransactionId;
long currentSizeMb = (m_pendingTables1.Sum(x => x.BaseFile.ArchiveSize) + m_pendingTables2.Sum(x => x.BaseFile.ArchiveSize)) >> 20;
if (currentSizeMb > m_settings.MaximumAllowedMb)
{
shouldWait = true;
m_rolloverTask.Start();
m_rolloverComplete.Reset();
}
else if (currentSizeMb > m_settings.RolloverSizeMb)
{
m_rolloverTask.Start();
}
else
{
m_rolloverTask.Start(m_settings.RolloverInterval);
}
}
if (SequenceNumberCommitted != null)
{
SequenceNumberCommitted(args.TransactionId);
}
if (shouldWait)
{
Log.Publish(MessageLevel.NA, MessageFlags.PerformanceIssue, "Queue is full", "Rollover task is taking a long time. A long pause on the inputs is about to occur.");
m_rolloverComplete.WaitOne();
}
}