/// <summary>Generates the blob sets that are required to run cloud-based map/reduce operations.</summary>
/// <param name="jobName">The name of the job (should be unique).</param>
/// <param name="items">The items that must be processed (at least two).</param>
/// <param name="functions">The map/reduce/aggregate functions (aggregate is optional).</param>
/// <param name="workerCount">The number of workers to use.</param>
/// <param name="mapIn">The type of the map input.</param>
/// <param name="mapOut">The type of the map output.</param>
/// <remarks>This method should be called from <see cref="T:MapReduceJob"/>.</remarks>
public void GenerateBlobSets(string jobName, IList <object> items, IMapReduceFunctions functions, int workerCount, Type mapIn, Type mapOut)
{
// Note: items is IList and not IEnumerable because the number of items must be known up-front
// 1. Store config
// 2. Put blobs and queue job messages
// 3. Put messages in the work queue
int itemCount = items.Count;
// Note: each blobset should contain at least two elements
int blobSetCount = Math.Min(workerCount, itemCount);
float blobsPerSet = (float)itemCount / (float)blobSetCount;
string ignored;
// 1. Store configuration
var configBlobName = MapReduceConfigurationName.Create(jobName);
var config = new MapReduceConfiguration()
{
TMapInType = mapIn.AssemblyQualifiedName,
TMapOutType = mapOut.AssemblyQualifiedName,
MapReduceFunctionsImplementor = functions.GetType().AssemblyQualifiedName,
BlobSetCount = blobSetCount
};
_blobStorage.PutBlob(configBlobName.ContainerName, configBlobName.ToString(),
config, typeof(MapReduceConfiguration), false, out ignored);
// 2.1. Allocate blobsets
var allNames = new InputBlobName[blobSetCount][];
int processedBlobs = 0;
for (int currSet = 0; currSet < blobSetCount; currSet++)
{
// Last blobset might be smaller
int thisSetSize = currSet != blobSetCount - 1 ? (int)Math.Ceiling(blobsPerSet) : itemCount - processedBlobs;
allNames[currSet] = new InputBlobName[thisSetSize];
processedBlobs += thisSetSize;
}
if (processedBlobs != itemCount)
{
throw new InvalidOperationException("Processed Blobs are less than the number of items");
}
// 2.2. Store input data (separate cycle for clarity)
processedBlobs = 0;
for (int currSet = 0; currSet < blobSetCount; currSet++)
{
for (int i = 0; i < allNames[currSet].Length; i++)
{
// BlobSet and Blob IDs start from zero
allNames[currSet][i] = InputBlobName.Create(jobName, currSet, i);
var item = items[processedBlobs];
_blobStorage.PutBlob(allNames[currSet][i].ContainerName, allNames[currSet][i].ToString(), item, mapIn, false, out ignored);
processedBlobs++;
}
_queueStorage.Put(JobsQueueName, new JobMessage()
{
Type = MessageType.BlobSetToProcess, JobName = jobName, BlobSetId = currSet
});
}
}
