/// <summary>Deletes all the data related to a job, regardless of the job status.</summary> /// <param name="jobName">The name of the job.</param> /// <remarks>Messages enqueued cannot be deleted but they cause no harm.</remarks> public void DeleteJobData(string jobName) { _blobStorage.DeleteBlobIfExist(MapReduceConfigurationName.Create(jobName)); _blobStorage.DeleteAllBlobs(InputBlobName.GetPrefix(jobName)); _blobStorage.DeleteAllBlobs(ReducedBlobName.GetPrefix(jobName)); _blobStorage.DeleteBlobIfExist(AggregatedBlobName.Create(jobName)); _blobStorage.DeleteBlobIfExist(BlobCounterName.Create(jobName)); }
/// <summary>Performs the aggregate operation on a blobset.</summary> /// <param name="jobName">The name of the job.</param> public void PerformAggregate(string jobName) { // 1. Load config // 2. Do aggregation // 3. Store result // 4. Delete reduced data // 1. Load config var config = GetJobConfig(jobName).Value; var reducedBlobPrefix = ReducedBlobName.GetPrefix(jobName); var aggregateResults = new List <object>(); Type mapOut = Type.GetType(config.TMapOutType); // 2. Load reduced items and do aggregation string ignored; foreach (var blobName in _blobStorage.ListBlobNames(reducedBlobPrefix)) { var blob = _blobStorage.GetBlob(blobName.ContainerName, blobName.ToString(), mapOut, out ignored); if (!blob.HasValue) { continue; } aggregateResults.Add(blob.Value); } IMapReduceFunctions mapReduceFunctions = GetMapReduceFunctions(config.MapReduceFunctionsImplementor); while (aggregateResults.Count > 1) { object item1 = aggregateResults[0]; object item2 = aggregateResults[1]; aggregateResults.RemoveAt(0); aggregateResults.RemoveAt(0); object aggregResult = InvokeAsDelegate(mapReduceFunctions.GetReducer(), item1, item2); aggregateResults.Add(aggregResult); } // 3. Store aggregated result var aggregatedBlobName = AggregatedBlobName.Create(jobName); _blobStorage.PutBlob(aggregatedBlobName.ContainerName, aggregatedBlobName.ToString(), aggregateResults[0], mapOut, false, out ignored); // 4. Delete reduced data _blobStorage.DeleteAllBlobs(reducedBlobPrefix); }