public void PartialSave()
{
foreach (var task in Tasks)
{
if (task == null || !task.IsRunning || task.Output == null)
{
continue;
}
task.PartialSaveInProgress = true;
OutputPlugin outp = task.Output.GetPlugin() as OutputPlugin;
if (sw != null)
{
outPutstartTime = sw.ElapsedMilliseconds;
}
var status = new StatusDelegate(outp_StatusUpdate);
outp.StatusUpdate += status;
outp.Save(task.ReduceResult);
task.Output.Location = Guid.NewGuid() + ".db3";
outp.StatusUpdate -= status;
task.ReduceResult.Clear(); // clear memory after saving
GC.Collect();
task.PartialSaveInProgress = false;
}
}
public IDictionary Start()
{
sw = Stopwatch.StartNew();
completed = false;
ThreadPool.QueueUserWorkItem(new WaitCallback(WatchDog));
//execute all tasks
for (int i = 0; i < Tasks.Count; i++)
{
var task = Tasks[i];
task.SetStopWatch(sw);
task.StatusUpdate += new StatusDelegate(outp_StatusUpdate);
// process the result of previous MapReduce tasks
if (task.Input == null)
{
var inputTask = Tasks[i - 1];
for (int j = 0; j < i - 1; j++)
{
var mergeTask = Tasks[j];
MergeDictionaries(inputTask, mergeTask.ReduceResult);
}
ExecuteStep(task, inputTask.ReduceResult);
}
else
{
ExecuteStep(task);
}
// if not parallel execution, wait for complete and merge with previous results, then save
if (!task.Parallel)
{
task.WaitForComplete();
if (task.Output != null && task.Output.PluginType != null)
{
for (int j = 0; j < i; j++)
{
var prevTask = Tasks[j];
MergeDictionaries(task, prevTask.ReduceResult);
}
OutputPlugin outp = task.Output.GetPlugin() as OutputPlugin;
if (sw != null)
{
outPutstartTime = sw.ElapsedMilliseconds;
}
outp.StatusUpdate += new StatusDelegate(outp_StatusUpdate);
outp.Save(task.ReduceResult);
task.ReduceResult = null; // clear memory after saving
}
}
}
// wait for all pending tasks
WaitForComplete();
var lastTask = Tasks.Last();
int mergeCount = 0;
// if there were parallel tasks, merge all of them then save
for (int j = 0; j < Tasks.Count - 1; j++)
{
var prevTask = Tasks[j];
if (!prevTask.Parallel)
{
continue;
}
mergeCount++;
MergeDictionaries(lastTask, prevTask.ReduceResult);
}
if (mergeCount > 0 && lastTask.Output != null)
{
OutputPlugin outp = lastTask.Output.GetPlugin() as OutputPlugin;
outp.Save(lastTask.ReduceResult);
}
return(lastTask.ReduceResult);
}
