public override IEnumerable <string> Process()
{
PrepareBeforeProcessing();
var result = new ConcurrentBag <string>();
var taskProcessors = GetTaskProcessors();
if (ParallelMode && taskProcessors.Count > 1)
{
var exceptions = new ConcurrentQueue <Exception>();
int totalCount = taskProcessors.Count;
Progress.SetRange(0, totalCount);
var finishedProcessors = new ConcurrentList <IParallelTaskProcessor>();
var curProcessors = new ConcurrentList <IParallelTaskProcessor>();
Parallel.ForEach(taskProcessors, Option, (processor, loopState) =>
{
curProcessors.Add(processor);
Progress.SetMessage("Processing {0}, finished {1} / {2}", curProcessors.Count, finishedProcessors.Count, totalCount);
processor.LoopState = loopState;
processor.Progress = Progress;
try
{
var curResult = processor.Process();
foreach (var f in curResult)
{
result.Add(f);
}
curProcessors.Remove(processor);
finishedProcessors.Add(processor);
Progress.SetPosition(finishedProcessors.Count);
Progress.SetMessage("Processing {0}, finished {1} / {2}", curProcessors.Count, finishedProcessors.Count, totalCount);
}
catch (Exception e)
{
exceptions.Enqueue(e);
loopState.Stop();
}
GC.Collect();
});
if (Progress.IsCancellationPending())
{
throw new UserTerminatedException();
}
if (exceptions.Count > 0)
{
if (exceptions.Count == 1)
{
throw exceptions.First();
}
else
{
StringBuilder sb = new StringBuilder();
foreach (var ex in exceptions)
{
sb.AppendLine(ex.ToString());
}
throw new Exception(sb.ToString());
}
}
}
else
{
for (int i = 0; i < taskProcessors.Count; i++)
{
if (Progress.IsCancellationPending())
{
throw new UserTerminatedException();
}
string rootMsg = MyConvert.Format("{0} / {1}", i + 1, taskProcessors.Count);
Progress.SetMessage(1, rootMsg);
var processor = taskProcessors[i];
processor.Progress = Progress;
var curResult = processor.Process();
foreach (var f in curResult)
{
result.Add(f);
}
}
}
DoAfterProcessing(result);
return(result);
}
public override IEnumerable <string> Process(string aPath)
{
Progress.SetMessage("Preparing ... ");
PrepareBeforeProcessing(aPath);
var result = new ConcurrentBag <string>();
Progress.SetMessage("Start processing ... ");
DateTime start = DateTime.Now;
int totalCount = sourceFiles.Count;
Progress.SetRange(0, totalCount);
var exceptions = new ConcurrentQueue <Exception>();
if (ParallelMode && sourceFiles.Count > 1)
{
var curFiles = new ConcurrentList <string>();
var finishedFiles = new ConcurrentList <string>();
var curProcessors = new ConcurrentList <IParallelTaskFileProcessor>();
Parallel.ForEach(sourceFiles, Option, (sourceFile, loopState) =>
{
curFiles.Add(sourceFile);
IParallelTaskFileProcessor processor = GetTaskProcessor(aPath, sourceFile);
if (processor == null)
{
curFiles.Remove(sourceFile);
finishedFiles.Add(sourceFile);
return;
}
processor.LoopState = loopState;
processor.Progress = Progress;
curProcessors.Add(processor);
try
{
var curResult = processor.Process(sourceFile);
foreach (var f in curResult)
{
result.Add(f);
}
curFiles.Remove(sourceFile);
finishedFiles.Add(sourceFile);
curProcessors.Remove(processor);
if (!Progress.IsConsole())
{
Progress.SetPosition(finishedFiles.Count);
}
DateTime end = DateTime.Now;
var cost = end - start;
var expectEnd = end.AddSeconds(cost.TotalSeconds / finishedFiles.Count * (totalCount - finishedFiles.Count));
Progress.SetMessage("Processed {0}, {1} processing, {2} / {3} finished, expect to finish at {4}", Path.GetFileName(sourceFile), curProcessors.Count, finishedFiles.Count, totalCount, expectEnd);
}
catch (Exception e)
{
curProcessors.Remove(processor);
exceptions.Enqueue(new Exception("Failed to process " + sourceFile + " : " + e.Message, e));
if (StopAtException)
{
loopState.Stop();
}
else
{
Progress.SetMessage("FAIL at {0} : {1}", Path.GetFileName(sourceFile), e.Message);
}
}
GC.Collect();
});
if (Progress.IsCancellationPending())
{
throw new UserTerminatedException();
}
}
else
{
for (int i = 0; i < sourceFiles.Count; i++)
{
if (Progress.IsCancellationPending())
{
throw new UserTerminatedException();
}
IParallelTaskFileProcessor processor = GetTaskProcessor(aPath, sourceFiles[i]);
try
{
var curResult = processor.Process(sourceFiles[i]);
foreach (var f in curResult)
{
result.Add(f);
}
DateTime end = DateTime.Now;
var cost = end - start;
var expectEnd = end.AddSeconds(cost.TotalSeconds / (i + 1) * (totalCount - i - 1));
Progress.SetMessage("Processed {0}, {1} / {2} finished, expect to be end at {3}", Path.GetFileName(sourceFiles[i]), i + 1, totalCount, expectEnd);
if (!Progress.IsConsole())
{
Progress.SetPosition(i + 1);
}
}
catch (Exception e)
{
var errorMsg = "Failed to process " + sourceFiles[i] + " : " + e.Message;
exceptions.Enqueue(new Exception(errorMsg, e));
if (StopAtException)
{
break;
}
else
{
Progress.SetMessage("FAIL at {0} : {1}", Path.GetFileName(sourceFiles[i]), e.Message);
}
}
}
}
if (exceptions.Count > 0)
{
if (!StopAtException)
{
using (var sw = new StreamWriter(Path.Combine(aPath, "failed.filelist")))
{
foreach (var ex in exceptions)
{
sw.WriteLine(ex.ToString());
}
}
}
if (exceptions.Count == 1)
{
throw exceptions.First();
}
else
{
StringBuilder sb = new StringBuilder();
foreach (var ex in exceptions)
{
sb.AppendLine(ex.ToString());
}
throw new Exception(sb.ToString());
}
}
DateTime totalEnd = DateTime.Now;
var totalCost = totalEnd - start;
Progress.SetMessage("Processed finished. Start at {0}, end at {1}, total time cost:{2}", start, totalEnd, MiscUtils.GetTimeCost(totalCost));
DoAfterProcessing(aPath, result);
if (result.Count > 100)
{
return(new[] { "More than 100 files saved in " + aPath + "." });
}
else
{
return(result);
}
}
