/// <summary>
/// Creates a new instance of the <see cref="FileProcessor"/> class.
/// </summary>
public FileProcessor()
{
m_filter = DefaultFilter;
m_folderExclusion = DefaultFolderExclusion;
m_trackChanges = DefaultTrackChanges;
m_internalBufferSize = DefaultInternalBufferSize;
m_enumerationStrategy = DefaultEnumerationStrategy;
m_trackedDirectoriesLock = new object();
m_trackedDirectories = new List <TrackedDirectory>();
m_enumerationOperation = new TaskSynchronizedOperation(EnumerateWatchDirectoriesAsync, OnError);
m_threadScheduler = new LogicalThreadScheduler(2);
m_threadScheduler.UnhandledException += (sender, args) => OnError(args.Argument);
m_processingThread = m_threadScheduler.CreateThread();
m_watcherThread = m_threadScheduler.CreateThread();
m_sequentialEnumerationThread = m_threadScheduler.CreateThread();
m_fileWatchTimer = new Timer(15000);
m_fileWatchTimer.Elapsed += FileWatchTimer_Elapsed;
m_requeueTokenSource = new ManagedCancellationTokenSource();
m_touchedFiles = new Dictionary <string, DateTime>(StringComparer.OrdinalIgnoreCase);
}
// Forces enumeration of directories currently being watched.
private async Task EnumerateWatchDirectoriesAsync()
{
List <TrackedDirectory> GetTrackedDirectories()
{
lock (m_trackedDirectoriesLock)
return(new List <TrackedDirectory>(m_trackedDirectories));
}
List <TrackedDirectory> trackedDirectories = GetTrackedDirectories();
FileEnumerationStrategy enumerationStrategy = m_enumerationStrategy;
bool sequential = enumerationStrategy == FileEnumerationStrategy.Sequential;
Func <LogicalThread> getEnumerationThread = sequential
? () => m_sequentialEnumerationThread
: () => m_threadScheduler.CreateThread();
async Task EnumerateTrackedDirectoryAsync(TrackedDirectory trackedDirectory)
{
try
{
LogicalThread enumerationThread = getEnumerationThread();
await enumerationThread.Join();
await trackedDirectory.EnumerateAsync(enumerationStrategy);
}
catch (Exception ex)
{
OnError(ex);
}
}
IEnumerable <Task> enumerationTasks = trackedDirectories
.Select(EnumerateTrackedDirectoryAsync);
async Task ForEach(IEnumerable <Task> tasks)
{
foreach (Task task in tasks)
{
await task;
}
}
Func <IEnumerable <Task>, Task> whenAll =
sequential ? ForEach : Task.WhenAll;
await whenAll(enumerationTasks);
}
private async Task EnumerateDirectoryAsync(DirectoryInfo directory, FileEnumerationStrategy fileEnumerationStrategy, CancellationToken cancellationToken)
{
bool parallelSubdirectories =
fileEnumerationStrategy == FileEnumerationStrategy.ParallelSubdirectories;
LogicalThread thread = LogicalThread.CurrentThread;
if (parallelSubdirectories)
{
LogicalThreadScheduler scheduler = m_fileProcessor.m_threadScheduler;
thread = scheduler.CreateThread();
await thread.Join();
}
async Task ForEach(IEnumerable <Task> tasks)
{
foreach (Task task in tasks)
{
if (cancellationToken.IsCancelled)
{
return;
}
await task;
await thread.Yield();
}
}
LogicalThread processingThread = m_fileProcessor.m_processingThread;
string activePath = null;
async Task VisitSubdirectoryAsync(DirectoryInfo subdirectory)
{
activePath = subdirectory.FullName;
if (cancellationToken.IsCancelled)
{
return;
}
if (m_fileProcessor.MatchesFolderExclusion(subdirectory.FullName))
{
return;
}
await EnumerateDirectoryAsync(subdirectory, fileEnumerationStrategy, cancellationToken);
}
async Task <Task> VisitFileAsync(FileInfo file)
{
activePath = file.FullName;
if (cancellationToken.IsCancelled)
{
return(Task.CompletedTask);
}
async Task InvokeOnProcessingThread(Action action)
{
await processingThread.Join();
action();
}
DateTime lastWriteTime = file.LastWriteTimeUtc;
void Process() => m_fileProcessor.TouchAndProcess(file.FullName, lastWriteTime, false);
void Skip() => m_fileProcessor.TouchAndSkip(file.FullName, lastWriteTime);
if (!m_fileProcessor.MatchesFilter(file.FullName))
{
return(InvokeOnProcessingThread(Skip));
}
Task processTask = InvokeOnProcessingThread(Process);
await thread.Yield();
return(processTask);
}
async Task EnumerateSubdirectoriesAsync()
{
Func <IEnumerable <Task>, Task> whenAll =
parallelSubdirectories ? Task.WhenAll : ForEach;
IEnumerable <Task> subdirectoryTasks = FilePath
.EnumerateDirectories(directory, "*", SearchOption.TopDirectoryOnly, m_fileProcessor.OnError)
.Select(VisitSubdirectoryAsync);
await whenAll(subdirectoryTasks);
}
async Task EnumerateFilesAsync()
{
IEnumerable <Task <Task> > fileTasks = FilePath
.EnumerateFiles(directory, "*", SearchOption.TopDirectoryOnly)
.Select(VisitFileAsync);
List <Task> processTasks = new List <Task>();
foreach (Task <Task> fileTask in fileTasks)
{
processTasks.Add(await fileTask);
}
await Task.WhenAll(processTasks);
}
EventHandler <EventArgs <List <string> > > handler = (sender, args) =>
{
if (!(activePath is null))
{
args.Argument.Add(activePath);
}
};
try
{
ActivelyVisitedPathsRequested += handler;
if (parallelSubdirectories)
{
Task subdirectoriesTask = EnumerateSubdirectoriesAsync();
await EnumerateFilesAsync();
ActivelyVisitedPathsRequested -= handler;
await subdirectoriesTask;
}
else
{
await EnumerateFilesAsync();
await EnumerateSubdirectoriesAsync();
}
}
finally
{
ActivelyVisitedPathsRequested -= handler;
}
}
public void ProcessPQWebReport(DateTime?date = null, int?meterID = null)
{
DateTime processDate = date ?? DateTime.Now.AddDays(-1);
DateTime startDay = processDate.Date;
DateTime endDay = startDay.AddDays(1).AddTicks(-1);
LogicalThreadScheduler logicalThreadScheduler = new LogicalThreadScheduler();
logicalThreadScheduler.UnhandledException += (sender, args) => Log.Error(args.Argument.Message, args.Argument);
using (DataContext dataContext = new DataContext("systemSettings"))
{
logicalThreadScheduler.MaxThreadCount = dataContext.Connection.ExecuteScalar <int?>("SELECT Value FROM Setting WHERE Name = 'MaxThreadCount'") ?? 1;
string historianServer = dataContext.Connection.ExecuteScalar <string>("SELECT Value FROM Setting WHERE Name = 'Historian.Server'") ?? "127.0.0.1";
string historianInstance = dataContext.Connection.ExecuteScalar <string>("SELECT Value FROM Setting WHERE Name = 'Historian.Instance'") ?? "XDA";
IEnumerable <Meter> meters;
if (meterID == null)
{
meters = dataContext.Table <Meter>().QueryRecordsWhere("ID IN (SELECT MeterID FROM MeterDataQualitySummary)");
}
else
{
meters = dataContext.Table <Meter>().QueryRecordsWhere("ID = {0}", meterID);
}
IEnumerable <PQMeasurement> pQMeasurements = dataContext.Table <PQMeasurement>().QueryRecords();
int returnVal = Interlocked.CompareExchange(ref m_runningCount, meters.Count() * pQMeasurements.Count(), 0);
if (returnVal != 0)
{
Log.Error("PQ Treding Web Report aggregtaion process may already be running. Aborting....");
return;
}
foreach (Meter meter in meters)
{
foreach (PQMeasurement measurement in pQMeasurements)
{
try
{
logicalThreadScheduler.CreateThread().Push(() => {
try
{
ProcessMeasurement(meter, measurement, historianServer, historianInstance, startDay, endDay);
}
finally {
Interlocked.Decrement(ref m_runningCount);
if (RunningCount == 0)
{
Log.Info("PQ Trend Web Report has completed aggregating data.");
}
}
});
}
catch (Exception ex)
{
Log.Error(ex.Message, ex);
}
}
}
}
}