public override IEnumerable <string> Process() { PrepareBeforeProcessing(); var result = new ConcurrentBag <string>(); var taskProcessors = GetTaskProcessors(); if (ParallelMode && taskProcessors.Count > 1) { var exceptions = new ConcurrentQueue <Exception>(); int totalCount = taskProcessors.Count; Progress.SetRange(0, totalCount); var finishedProcessors = new ConcurrentList <IParallelTaskProcessor>(); var curProcessors = new ConcurrentList <IParallelTaskProcessor>(); Parallel.ForEach(taskProcessors, Option, (processor, loopState) => { curProcessors.Add(processor); Progress.SetMessage("Processing {0}, finished {1} / {2}", curProcessors.Count, finishedProcessors.Count, totalCount); processor.LoopState = loopState; processor.Progress = Progress; try { var curResult = processor.Process(); foreach (var f in curResult) { result.Add(f); } curProcessors.Remove(processor); finishedProcessors.Add(processor); Progress.SetPosition(finishedProcessors.Count); Progress.SetMessage("Processing {0}, finished {1} / {2}", curProcessors.Count, finishedProcessors.Count, totalCount); } catch (Exception e) { exceptions.Enqueue(e); loopState.Stop(); } GC.Collect(); }); if (Progress.IsCancellationPending()) { throw new UserTerminatedException(); } if (exceptions.Count > 0) { if (exceptions.Count == 1) { throw exceptions.First(); } else { StringBuilder sb = new StringBuilder(); foreach (var ex in exceptions) { sb.AppendLine(ex.ToString()); } throw new Exception(sb.ToString()); } } } else { for (int i = 0; i < taskProcessors.Count; i++) { if (Progress.IsCancellationPending()) { throw new UserTerminatedException(); } string rootMsg = MyConvert.Format("{0} / {1}", i + 1, taskProcessors.Count); Progress.SetMessage(1, rootMsg); var processor = taskProcessors[i]; processor.Progress = Progress; var curResult = processor.Process(); foreach (var f in curResult) { result.Add(f); } } } DoAfterProcessing(result); return(result); }
public override IEnumerable <string> Process(string aPath) { Progress.SetMessage("Preparing ... "); PrepareBeforeProcessing(aPath); var result = new ConcurrentBag <string>(); Progress.SetMessage("Start processing ... "); DateTime start = DateTime.Now; int totalCount = sourceFiles.Count; Progress.SetRange(0, totalCount); var exceptions = new ConcurrentQueue <Exception>(); if (ParallelMode && sourceFiles.Count > 1) { var curFiles = new ConcurrentList <string>(); var finishedFiles = new ConcurrentList <string>(); var curProcessors = new ConcurrentList <IParallelTaskFileProcessor>(); Parallel.ForEach(sourceFiles, Option, (sourceFile, loopState) => { curFiles.Add(sourceFile); IParallelTaskFileProcessor processor = GetTaskProcessor(aPath, sourceFile); if (processor == null) { curFiles.Remove(sourceFile); finishedFiles.Add(sourceFile); return; } processor.LoopState = loopState; processor.Progress = Progress; curProcessors.Add(processor); try { var curResult = processor.Process(sourceFile); foreach (var f in curResult) { result.Add(f); } curFiles.Remove(sourceFile); finishedFiles.Add(sourceFile); curProcessors.Remove(processor); if (!Progress.IsConsole()) { Progress.SetPosition(finishedFiles.Count); } DateTime end = DateTime.Now; var cost = end - start; var expectEnd = end.AddSeconds(cost.TotalSeconds / finishedFiles.Count * (totalCount - finishedFiles.Count)); Progress.SetMessage("Processed {0}, {1} processing, {2} / {3} finished, expect to finish at {4}", Path.GetFileName(sourceFile), curProcessors.Count, finishedFiles.Count, totalCount, expectEnd); } catch (Exception e) { curProcessors.Remove(processor); exceptions.Enqueue(new Exception("Failed to process " + sourceFile + " : " + e.Message, e)); if (StopAtException) { loopState.Stop(); } else { Progress.SetMessage("FAIL at {0} : {1}", Path.GetFileName(sourceFile), e.Message); } } GC.Collect(); }); if (Progress.IsCancellationPending()) { throw new UserTerminatedException(); } } else { for (int i = 0; i < sourceFiles.Count; i++) { if (Progress.IsCancellationPending()) { throw new UserTerminatedException(); } IParallelTaskFileProcessor processor = GetTaskProcessor(aPath, sourceFiles[i]); try { var curResult = processor.Process(sourceFiles[i]); foreach (var f in curResult) { result.Add(f); } DateTime end = DateTime.Now; var cost = end - start; var expectEnd = end.AddSeconds(cost.TotalSeconds / (i + 1) * (totalCount - i - 1)); Progress.SetMessage("Processed {0}, {1} / {2} finished, expect to be end at {3}", Path.GetFileName(sourceFiles[i]), i + 1, totalCount, expectEnd); if (!Progress.IsConsole()) { Progress.SetPosition(i + 1); } } catch (Exception e) { var errorMsg = "Failed to process " + sourceFiles[i] + " : " + e.Message; exceptions.Enqueue(new Exception(errorMsg, e)); if (StopAtException) { break; } else { Progress.SetMessage("FAIL at {0} : {1}", Path.GetFileName(sourceFiles[i]), e.Message); } } } } if (exceptions.Count > 0) { if (!StopAtException) { using (var sw = new StreamWriter(Path.Combine(aPath, "failed.filelist"))) { foreach (var ex in exceptions) { sw.WriteLine(ex.ToString()); } } } if (exceptions.Count == 1) { throw exceptions.First(); } else { StringBuilder sb = new StringBuilder(); foreach (var ex in exceptions) { sb.AppendLine(ex.ToString()); } throw new Exception(sb.ToString()); } } DateTime totalEnd = DateTime.Now; var totalCost = totalEnd - start; Progress.SetMessage("Processed finished. Start at {0}, end at {1}, total time cost:{2}", start, totalEnd, MiscUtils.GetTimeCost(totalCost)); DoAfterProcessing(aPath, result); if (result.Count > 100) { return(new[] { "More than 100 files saved in " + aPath + "." }); } else { return(result); } }