/// <summary>Retrieves the aggregated result of a map/reduce job.</summary>
/// <typeparam name="T">The type of the result.</typeparam>
/// <param name="jobName">The name of the job.</param>
/// <returns>The aggregated result.</returns>
/// <exception cref="InvalidOperationException">If the is not yet complete.</exception>
/// <exception cref="ArgumentException">If <paramref name="jobName"/> refers to an inexistent job.</exception>
public T GetAggregatedResult <T>(string jobName)
{
var config = GetJobConfig(jobName);
if (!config.HasValue)
{
throw new ArgumentException("Unknown job", "jobName");
}
var counter = new BlobCounter(_blobStorage, BlobCounterName.Create(jobName));
int completedBlobsets = (int)counter.GetValue();
if (completedBlobsets < config.Value.BlobSetCount)
{
throw new InvalidOperationException("Job is not complete (there still are blobsets to process)");
}
Type mapOut = Type.GetType(config.Value.TMapOutType);
var blobName = AggregatedBlobName.Create(jobName);
string ignored;
var aggregatedResult = _blobStorage.GetBlob(blobName.ContainerName, blobName.ToString(), mapOut, out ignored);
if (!aggregatedResult.HasValue)
{
throw new InvalidOperationException("Job is not complete (reduced items must still be aggregated)");
}
return((T)aggregatedResult.Value);
}
/// <summary>Deletes all the data related to a job, regardless of the job status.</summary>
/// <param name="jobName">The name of the job.</param>
/// <remarks>Messages enqueued cannot be deleted but they cause no harm.</remarks>
public void DeleteJobData(string jobName)
{
_blobStorage.DeleteBlobIfExist(MapReduceConfigurationName.Create(jobName));
_blobStorage.DeleteAllBlobs(InputBlobName.GetPrefix(jobName));
_blobStorage.DeleteAllBlobs(ReducedBlobName.GetPrefix(jobName));
_blobStorage.DeleteBlobIfExist(AggregatedBlobName.Create(jobName));
_blobStorage.DeleteBlobIfExist(BlobCounterName.Create(jobName));
}
/// <summary>Performs the aggregate operation on a blobset.</summary>
/// <param name="jobName">The name of the job.</param>
public void PerformAggregate(string jobName)
{
// 1. Load config
// 2. Do aggregation
// 3. Store result
// 4. Delete reduced data
// 1. Load config
var config = GetJobConfig(jobName).Value;
var reducedBlobPrefix = ReducedBlobName.GetPrefix(jobName);
var aggregateResults = new List <object>();
Type mapOut = Type.GetType(config.TMapOutType);
// 2. Load reduced items and do aggregation
string ignored;
foreach (var blobName in _blobStorage.ListBlobNames(reducedBlobPrefix))
{
var blob = _blobStorage.GetBlob(blobName.ContainerName, blobName.ToString(), mapOut, out ignored);
if (!blob.HasValue)
{
continue;
}
aggregateResults.Add(blob.Value);
}
IMapReduceFunctions mapReduceFunctions = GetMapReduceFunctions(config.MapReduceFunctionsImplementor);
while (aggregateResults.Count > 1)
{
object item1 = aggregateResults[0];
object item2 = aggregateResults[1];
aggregateResults.RemoveAt(0);
aggregateResults.RemoveAt(0);
object aggregResult = InvokeAsDelegate(mapReduceFunctions.GetReducer(), item1, item2);
aggregateResults.Add(aggregResult);
}
// 3. Store aggregated result
var aggregatedBlobName = AggregatedBlobName.Create(jobName);
_blobStorage.PutBlob(aggregatedBlobName.ContainerName, aggregatedBlobName.ToString(), aggregateResults[0], mapOut, false, out ignored);
// 4. Delete reduced data
_blobStorage.DeleteAllBlobs(reducedBlobPrefix);
}
