public void GetValueIncrement()
{
var provider = GlobalSetup.Container.Resolve <IBlobStorageProvider>();
provider.CreateContainerIfNotExist(ContainerName);
var counter = new BlobCounter(provider, ContainerName, BlobName);
var val = (int)counter.GetValue();
if (0 != val)
{
counter.Delete();
}
counter.Increment(10);
val = (int)counter.GetValue();
Assert.AreEqual(10, val, "#A00");
var val2 = counter.Increment(-5);
val = (int)counter.GetValue();
Assert.AreEqual(5, val, "#A01");
Assert.AreEqual(val, val2, "#A02");
var flag1 = counter.Delete();
var flag2 = counter.Delete();
Assert.IsTrue(flag1, "#A03");
Assert.IsFalse(flag2, "#A04");
}
public void GetValueIncrement()
{
var provider = GlobalSetup.Container.Resolve<IBlobStorageProvider>();
provider.CreateContainer(ContainerName);
var counter = new BlobCounter(provider, ContainerName, BlobName);
var val = (int)counter.GetValue();
if (0 != val) counter.Delete();
counter.Increment(10);
val = (int) counter.GetValue();
Assert.AreEqual(10, val, "#A00");
var val2 = counter.Increment(-5);
val = (int)counter.GetValue();
Assert.AreEqual(5, val, "#A01");
Assert.AreEqual(val, val2, "#A02");
var flag1 = counter.Delete();
var flag2 = counter.Delete();
Assert.IsTrue(flag1, "#A03");
Assert.IsFalse(flag2, "#A04");
}
public void IncrementMultiThread()
{
var provider = GlobalSetup.Container.Resolve <IBlobStorageProvider>();
provider.CreateContainerIfNotExist(ContainerName);
//creating thread parameters
var counter = new BlobCounter(provider, ContainerName, "SomeBlobName");
counter.Reset(0);
var random = new Random();
const int threadsCount = 4;
var increments = Range.Array(threadsCount).Select(e => Range.Array(5).Select(i => random.Next(20)).ToArray()).ToArray();
var localSums = increments.AsParallel().Select(
e =>
{
var c = new BlobCounter(provider, ContainerName, "SomeBlobName");
foreach (var increment in e)
{
c.Increment(increment);
}
return(e.Sum());
}).ToList();
Assert.AreEqual(increments.Sum(i => i.Sum()), localSums.Sum(), "Broken invariant.");
Assert.AreEqual(localSums.Sum(), counter.GetValue(), "Values should be equal, BlobCounter supposed to be thread-safe");
}
protected override void Start(SquareMessage message)
{
if (message.IsStart)
{
var counterName = TemporaryBlobName <decimal> .GetNew(message.Expiration);
var counter = new BlobCounter(Blobs, counterName);
counter.Reset(BlobCounter.Aleph);
var blobNames = Blobs.ListBlobNames(message.ContainerName).ToList();
foreach (var blobName in blobNames)
{
Put(new SquareMessage
{
BlobName = blobName,
ContainerName = message.ContainerName,
IsStart = false,
BlobCounter = counterName
});
}
// dealing with rare race condition
if (0m >= counter.Increment(-BlobCounter.Aleph + blobNames.Count))
{
Finish(counter);
}
}
else
{
var value = Blobs.GetBlob <double>(message.ContainerName, message.BlobName).Value;
Blobs.PutBlob(message.ContainerName, message.BlobName, value * value);
var counter = new BlobCounter(Blobs, message.BlobCounter);
if (0m >= counter.Increment(-1))
{
Finish(counter);
}
}
}
public void IncrementMultiThread()
{
var provider = GlobalSetup.Container.Resolve<IBlobStorageProvider>();
provider.CreateContainer(ContainerName);
//creating thread parameters
var count = new BlobCounter(provider, ContainerName, "SomeBlobName");
count.Reset(0);
var random = new Random();
const int threadsCount = 4;
var increments = Range.Array(threadsCount).Convert(e => Range.Array(5).Convert(i => random.Next(20) - 10));
var localSums = increments.SelectInParallel(e =>
{
var counter = new BlobCounter(provider, ContainerName, "SomeBlobName");
foreach (var increment in e)
{
counter.Increment(increment);
}
return e.Sum();
}, threadsCount);
Assert.AreEqual(localSums.Sum(), count.GetValue(), "values should be equal, BlobCounter supposed to be thread-safe");
}
/// <summary>Performs map/reduce operations on a blobset.</summary>
/// <param name="jobName">The name of the job.</param>
/// <param name="blobSetId">The blobset ID.</param>
/// <remarks>This method should be called from <see cref="T:MapReduceService"/>.</remarks>
public void PerformMapReduce(string jobName, int blobSetId)
{
// 1. Load config
// 2. For all blobs in blobset, do map (output N)
// 3. For all mapped items, do reduce (output 1)
// 4. Store reduce result
// 5. Update counter
// 6. If aggregator != null && blobsets are all processed --> enqueue aggregation message
// 7. Delete blobset
// 1. Load config
var config = GetJobConfig(jobName).Value;
var blobsetPrefix = InputBlobName.GetPrefix(jobName, blobSetId);
var mapResults = new List <object>();
var mapReduceFunctions = GetMapReduceFunctions(config.MapReduceFunctionsImplementor);
var mapIn = Type.GetType(config.TMapInType);
var mapOut = Type.GetType(config.TMapOutType);
// 2. Do map for all blobs in the blobset
string ignored;
foreach (var blobName in _blobStorage.ListBlobNames(blobsetPrefix))
{
var inputBlob = _blobStorage.GetBlob(blobName.ContainerName, blobName.ToString(), mapIn, out ignored);
if (!inputBlob.HasValue)
{
continue;
}
object mapResult = InvokeAsDelegate(mapReduceFunctions.GetMapper(), inputBlob.Value);
mapResults.Add(mapResult);
}
// 3. Do reduce for all mapped results
while (mapResults.Count > 1)
{
object item1 = mapResults[0];
object item2 = mapResults[1];
mapResults.RemoveAt(0);
mapResults.RemoveAt(0);
object reduceResult = InvokeAsDelegate(mapReduceFunctions.GetReducer(), item1, item2);
mapResults.Add(reduceResult);
}
// 4. Store reduced result
var reducedBlobName = ReducedBlobName.Create(jobName, blobSetId);
_blobStorage.PutBlob(reducedBlobName.ContainerName, reducedBlobName.ToString(), mapResults[0], mapOut, false, out ignored);
// 5. Update counter
var counterName = BlobCounterName.Create(jobName);
var counter = new BlobCounter(_blobStorage, counterName);
var totalCompletedBlobSets = (int)counter.Increment(1);
// 6. Queue aggregation if appropriate
if (totalCompletedBlobSets == config.BlobSetCount)
{
_queueStorage.Put(JobsQueueName,
new JobMessage {
JobName = jobName, BlobSetId = null, Type = MessageType.ReducedDataToAggregate
});
}
// 7. Delete blobset's blobs
_blobStorage.DeleteAllBlobs(blobsetPrefix);
}