/// <summary> /// Creates a new instance of the <see cref="FileProcessor"/> class. /// </summary> public FileProcessor() { m_filter = DefaultFilter; m_folderExclusion = DefaultFolderExclusion; m_trackChanges = DefaultTrackChanges; m_internalBufferSize = DefaultInternalBufferSize; m_enumerationStrategy = DefaultEnumerationStrategy; m_trackedDirectoriesLock = new object(); m_trackedDirectories = new List <TrackedDirectory>(); m_enumerationOperation = new TaskSynchronizedOperation(EnumerateWatchDirectoriesAsync, OnError); m_threadScheduler = new LogicalThreadScheduler(2); m_threadScheduler.UnhandledException += (sender, args) => OnError(args.Argument); m_processingThread = m_threadScheduler.CreateThread(); m_watcherThread = m_threadScheduler.CreateThread(); m_sequentialEnumerationThread = m_threadScheduler.CreateThread(); m_fileWatchTimer = new Timer(15000); m_fileWatchTimer.Elapsed += FileWatchTimer_Elapsed; m_requeueTokenSource = new ManagedCancellationTokenSource(); m_touchedFiles = new Dictionary <string, DateTime>(StringComparer.OrdinalIgnoreCase); }
// Forces enumeration of directories currently being watched. private async Task EnumerateWatchDirectoriesAsync() { List <TrackedDirectory> GetTrackedDirectories() { lock (m_trackedDirectoriesLock) return(new List <TrackedDirectory>(m_trackedDirectories)); } List <TrackedDirectory> trackedDirectories = GetTrackedDirectories(); FileEnumerationStrategy enumerationStrategy = m_enumerationStrategy; bool sequential = enumerationStrategy == FileEnumerationStrategy.Sequential; Func <LogicalThread> getEnumerationThread = sequential ? () => m_sequentialEnumerationThread : () => m_threadScheduler.CreateThread(); async Task EnumerateTrackedDirectoryAsync(TrackedDirectory trackedDirectory) { try { LogicalThread enumerationThread = getEnumerationThread(); await enumerationThread.Join(); await trackedDirectory.EnumerateAsync(enumerationStrategy); } catch (Exception ex) { OnError(ex); } } IEnumerable <Task> enumerationTasks = trackedDirectories .Select(EnumerateTrackedDirectoryAsync); async Task ForEach(IEnumerable <Task> tasks) { foreach (Task task in tasks) { await task; } } Func <IEnumerable <Task>, Task> whenAll = sequential ? ForEach : Task.WhenAll; await whenAll(enumerationTasks); }
private async Task EnumerateDirectoryAsync(DirectoryInfo directory, FileEnumerationStrategy fileEnumerationStrategy, CancellationToken cancellationToken) { bool parallelSubdirectories = fileEnumerationStrategy == FileEnumerationStrategy.ParallelSubdirectories; LogicalThread thread = LogicalThread.CurrentThread; if (parallelSubdirectories) { LogicalThreadScheduler scheduler = m_fileProcessor.m_threadScheduler; thread = scheduler.CreateThread(); await thread.Join(); } async Task ForEach(IEnumerable <Task> tasks) { foreach (Task task in tasks) { if (cancellationToken.IsCancelled) { return; } await task; await thread.Yield(); } } LogicalThread processingThread = m_fileProcessor.m_processingThread; string activePath = null; async Task VisitSubdirectoryAsync(DirectoryInfo subdirectory) { activePath = subdirectory.FullName; if (cancellationToken.IsCancelled) { return; } if (m_fileProcessor.MatchesFolderExclusion(subdirectory.FullName)) { return; } await EnumerateDirectoryAsync(subdirectory, fileEnumerationStrategy, cancellationToken); } async Task <Task> VisitFileAsync(FileInfo file) { activePath = file.FullName; if (cancellationToken.IsCancelled) { return(Task.CompletedTask); } async Task InvokeOnProcessingThread(Action action) { await processingThread.Join(); action(); } DateTime lastWriteTime = file.LastWriteTimeUtc; void Process() => m_fileProcessor.TouchAndProcess(file.FullName, lastWriteTime, false); void Skip() => m_fileProcessor.TouchAndSkip(file.FullName, lastWriteTime); if (!m_fileProcessor.MatchesFilter(file.FullName)) { return(InvokeOnProcessingThread(Skip)); } Task processTask = InvokeOnProcessingThread(Process); await thread.Yield(); return(processTask); } async Task EnumerateSubdirectoriesAsync() { Func <IEnumerable <Task>, Task> whenAll = parallelSubdirectories ? Task.WhenAll : ForEach; IEnumerable <Task> subdirectoryTasks = FilePath .EnumerateDirectories(directory, "*", SearchOption.TopDirectoryOnly, m_fileProcessor.OnError) .Select(VisitSubdirectoryAsync); await whenAll(subdirectoryTasks); } async Task EnumerateFilesAsync() { IEnumerable <Task <Task> > fileTasks = FilePath .EnumerateFiles(directory, "*", SearchOption.TopDirectoryOnly) .Select(VisitFileAsync); List <Task> processTasks = new List <Task>(); foreach (Task <Task> fileTask in fileTasks) { processTasks.Add(await fileTask); } await Task.WhenAll(processTasks); } EventHandler <EventArgs <List <string> > > handler = (sender, args) => { if (!(activePath is null)) { args.Argument.Add(activePath); } }; try { ActivelyVisitedPathsRequested += handler; if (parallelSubdirectories) { Task subdirectoriesTask = EnumerateSubdirectoriesAsync(); await EnumerateFilesAsync(); ActivelyVisitedPathsRequested -= handler; await subdirectoriesTask; } else { await EnumerateFilesAsync(); await EnumerateSubdirectoriesAsync(); } } finally { ActivelyVisitedPathsRequested -= handler; } }
public void ProcessPQWebReport(DateTime?date = null, int?meterID = null) { DateTime processDate = date ?? DateTime.Now.AddDays(-1); DateTime startDay = processDate.Date; DateTime endDay = startDay.AddDays(1).AddTicks(-1); LogicalThreadScheduler logicalThreadScheduler = new LogicalThreadScheduler(); logicalThreadScheduler.UnhandledException += (sender, args) => Log.Error(args.Argument.Message, args.Argument); using (DataContext dataContext = new DataContext("systemSettings")) { logicalThreadScheduler.MaxThreadCount = dataContext.Connection.ExecuteScalar <int?>("SELECT Value FROM Setting WHERE Name = 'MaxThreadCount'") ?? 1; string historianServer = dataContext.Connection.ExecuteScalar <string>("SELECT Value FROM Setting WHERE Name = 'Historian.Server'") ?? "127.0.0.1"; string historianInstance = dataContext.Connection.ExecuteScalar <string>("SELECT Value FROM Setting WHERE Name = 'Historian.Instance'") ?? "XDA"; IEnumerable <Meter> meters; if (meterID == null) { meters = dataContext.Table <Meter>().QueryRecordsWhere("ID IN (SELECT MeterID FROM MeterDataQualitySummary)"); } else { meters = dataContext.Table <Meter>().QueryRecordsWhere("ID = {0}", meterID); } IEnumerable <PQMeasurement> pQMeasurements = dataContext.Table <PQMeasurement>().QueryRecords(); int returnVal = Interlocked.CompareExchange(ref m_runningCount, meters.Count() * pQMeasurements.Count(), 0); if (returnVal != 0) { Log.Error("PQ Treding Web Report aggregtaion process may already be running. Aborting...."); return; } foreach (Meter meter in meters) { foreach (PQMeasurement measurement in pQMeasurements) { try { logicalThreadScheduler.CreateThread().Push(() => { try { ProcessMeasurement(meter, measurement, historianServer, historianInstance, startDay, endDay); } finally { Interlocked.Decrement(ref m_runningCount); if (RunningCount == 0) { Log.Info("PQ Trend Web Report has completed aggregating data."); } } }); } catch (Exception ex) { Log.Error(ex.Message, ex); } } } } }