public async Task EnumerateAsync(FileEnumerationStrategy fileEnumerationStrategy)
            {
                if (m_disposed)
                {
                    return;
                }

                CancellationToken cancellationToken = new CancellationToken();

                Interlocked.Exchange(ref m_cancellationToken, cancellationToken);
                DirectoryInfo directory = new DirectoryInfo(Path);

                await EnumerateDirectoryAsync(directory, fileEnumerationStrategy, cancellationToken);
            }
        // Forces enumeration of directories currently being watched.
        private async Task EnumerateWatchDirectoriesAsync()
        {
            List <TrackedDirectory> GetTrackedDirectories()
            {
                lock (m_trackedDirectoriesLock)
                    return(new List <TrackedDirectory>(m_trackedDirectories));
            }

            List <TrackedDirectory> trackedDirectories  = GetTrackedDirectories();
            FileEnumerationStrategy enumerationStrategy = m_enumerationStrategy;
            bool sequential = enumerationStrategy == FileEnumerationStrategy.Sequential;

            Func <LogicalThread> getEnumerationThread = sequential
                ? () => m_sequentialEnumerationThread
                : () => m_threadScheduler.CreateThread();

            async Task EnumerateTrackedDirectoryAsync(TrackedDirectory trackedDirectory)
            {
                try
                {
                    LogicalThread enumerationThread = getEnumerationThread();
                    await enumerationThread.Join();

                    await trackedDirectory.EnumerateAsync(enumerationStrategy);
                }
                catch (Exception ex)
                {
                    OnError(ex);
                }
            }

            IEnumerable <Task> enumerationTasks = trackedDirectories
                                                  .Select(EnumerateTrackedDirectoryAsync);

            async Task ForEach(IEnumerable <Task> tasks)
            {
                foreach (Task task in tasks)
                {
                    await task;
                }
            }

            Func <IEnumerable <Task>, Task> whenAll =
                sequential ? ForEach : Task.WhenAll;

            await whenAll(enumerationTasks);
        }
        /// <summary>
        /// Creates a new instance of the <see cref="FileProcessor"/> class.
        /// </summary>
        public FileProcessor()
        {
            m_filter              = DefaultFilter;
            m_folderExclusion     = DefaultFolderExclusion;
            m_trackChanges        = DefaultTrackChanges;
            m_internalBufferSize  = DefaultInternalBufferSize;
            m_enumerationStrategy = DefaultEnumerationStrategy;

            m_trackedDirectoriesLock = new object();
            m_trackedDirectories     = new List <TrackedDirectory>();
            m_enumerationOperation   = new TaskSynchronizedOperation(EnumerateWatchDirectoriesAsync, OnError);

            m_threadScheduler = new LogicalThreadScheduler(2);
            m_threadScheduler.UnhandledException += (sender, args) => OnError(args.Argument);
            m_processingThread            = m_threadScheduler.CreateThread();
            m_watcherThread               = m_threadScheduler.CreateThread();
            m_sequentialEnumerationThread = m_threadScheduler.CreateThread();
            m_fileWatchTimer              = new Timer(15000);
            m_fileWatchTimer.Elapsed     += FileWatchTimer_Elapsed;
            m_requeueTokenSource          = new ManagedCancellationTokenSource();

            m_touchedFiles = new Dictionary <string, DateTime>(StringComparer.OrdinalIgnoreCase);
        }
            private async Task EnumerateDirectoryAsync(DirectoryInfo directory, FileEnumerationStrategy fileEnumerationStrategy, CancellationToken cancellationToken)
            {
                bool parallelSubdirectories =
                    fileEnumerationStrategy == FileEnumerationStrategy.ParallelSubdirectories;

                LogicalThread thread = LogicalThread.CurrentThread;

                if (parallelSubdirectories)
                {
                    LogicalThreadScheduler scheduler = m_fileProcessor.m_threadScheduler;
                    thread = scheduler.CreateThread();
                    await thread.Join();
                }

                async Task ForEach(IEnumerable <Task> tasks)
                {
                    foreach (Task task in tasks)
                    {
                        if (cancellationToken.IsCancelled)
                        {
                            return;
                        }

                        await task;
                        await thread.Yield();
                    }
                }

                LogicalThread processingThread = m_fileProcessor.m_processingThread;
                string        activePath       = null;

                async Task VisitSubdirectoryAsync(DirectoryInfo subdirectory)
                {
                    activePath = subdirectory.FullName;

                    if (cancellationToken.IsCancelled)
                    {
                        return;
                    }

                    if (m_fileProcessor.MatchesFolderExclusion(subdirectory.FullName))
                    {
                        return;
                    }

                    await EnumerateDirectoryAsync(subdirectory, fileEnumerationStrategy, cancellationToken);
                }

                async Task <Task> VisitFileAsync(FileInfo file)
                {
                    activePath = file.FullName;

                    if (cancellationToken.IsCancelled)
                    {
                        return(Task.CompletedTask);
                    }

                    async Task InvokeOnProcessingThread(Action action)
                    {
                        await processingThread.Join();

                        action();
                    }

                    DateTime lastWriteTime = file.LastWriteTimeUtc;

                    void Process() => m_fileProcessor.TouchAndProcess(file.FullName, lastWriteTime, false);
                    void Skip() => m_fileProcessor.TouchAndSkip(file.FullName, lastWriteTime);

                    if (!m_fileProcessor.MatchesFilter(file.FullName))
                    {
                        return(InvokeOnProcessingThread(Skip));
                    }

                    Task processTask = InvokeOnProcessingThread(Process);
                    await thread.Yield();

                    return(processTask);
                }

                async Task EnumerateSubdirectoriesAsync()
                {
                    Func <IEnumerable <Task>, Task> whenAll =
                        parallelSubdirectories ? Task.WhenAll : ForEach;

                    IEnumerable <Task> subdirectoryTasks = FilePath
                                                           .EnumerateDirectories(directory, "*", SearchOption.TopDirectoryOnly, m_fileProcessor.OnError)
                                                           .Select(VisitSubdirectoryAsync);

                    await whenAll(subdirectoryTasks);
                }

                async Task EnumerateFilesAsync()
                {
                    IEnumerable <Task <Task> > fileTasks = FilePath
                                                           .EnumerateFiles(directory, "*", SearchOption.TopDirectoryOnly)
                                                           .Select(VisitFileAsync);

                    List <Task> processTasks = new List <Task>();

                    foreach (Task <Task> fileTask in fileTasks)
                    {
                        processTasks.Add(await fileTask);
                    }

                    await Task.WhenAll(processTasks);
                }

                EventHandler <EventArgs <List <string> > > handler = (sender, args) =>
                {
                    if (!(activePath is null))
                    {
                        args.Argument.Add(activePath);
                    }
                };

                try
                {
                    ActivelyVisitedPathsRequested += handler;

                    if (parallelSubdirectories)
                    {
                        Task subdirectoriesTask = EnumerateSubdirectoriesAsync();
                        await EnumerateFilesAsync();

                        ActivelyVisitedPathsRequested -= handler;
                        await subdirectoriesTask;
                    }
                    else
                    {
                        await EnumerateFilesAsync();
                        await EnumerateSubdirectoriesAsync();
                    }
                }
                finally
                {
                    ActivelyVisitedPathsRequested -= handler;
                }
            }
Exemple #5
0
        /// <summary>
        /// Creates a new instance of the <see cref="FileProcessor"/> class.
        /// </summary>
        /// <param name="processorID">Identifies the file processor so that it can locate its processed file cache.</param>
        public FileProcessor(Guid processorID)
        {
            m_processorID = processorID;

            m_filter = DefaultFilter;
            m_filterMethod = filePath => true;
            m_trackChanges = DefaultTrackChanges;
            m_cachePath = DefaultCachePath;
            m_internalBufferSize = DefaultInternalBufferSize;
            m_maxFragmentation = DefaultMaxFragmentation;
            m_enumerationStrategy = DefaultEnumerationStrategy;

            m_fileWatchersLock = new object();
            m_fileWatchers = new List<SafeFileWatcher>();
            m_threadScheduler = new LogicalThreadScheduler();
            m_threadScheduler.UnhandledException += (sender, args) => OnError(args.Argument);
            m_processingThread = m_threadScheduler.CreateThread(2);
            m_watcherThread = m_threadScheduler.CreateThread();
            m_fileWatchTimer = new Timer(15000);
            m_fileWatchTimer.Elapsed += FileWatchTimer_Elapsed;
            m_waitObject = new ManualResetEvent(false);

            m_touchedFiles = new Dictionary<string, DateTime>(StringComparer.OrdinalIgnoreCase);
            m_processedFiles = new FileBackedHashSet<string>(Path.Combine(m_cachePath, m_processorID.ToString()), StringComparer.OrdinalIgnoreCase);

            // Create the enumerator last since we are passing
            // a reference to 'this' into its constructor
            m_enumerator = new FileEnumerator(this);
        }