public void RunJobMapValueReduce()
{
MapReduceJob<int, string, string, int, string, int> job = new MapReduceJob<int, string, string, int, string, int>(Map, Reduce);
var items = new string[] { "a", "word", "is", "a", "word" };
foreach (var item in items)
job.MapValue(item);
var result = job.Reduce();
Assert.IsNotNull(result);
Assert.AreEqual(3, result.Count());
var pair1 = result.FirstOrDefault(p => p.Key == "word");
Assert.IsNotNull(pair1);
Assert.AreEqual(2, pair1.Value);
var pair2 = result.FirstOrDefault(p => p.Key == "a");
Assert.IsNotNull(pair2);
Assert.AreEqual(2, pair2.Value);
var pair3 = result.FirstOrDefault(p => p.Key == "is");
Assert.IsNotNull(pair3);
Assert.AreEqual(1, pair3.Value);
}
public void RunJobTwoMapsAndReduce()
{
MapReduceJob<int, string, string, int, string, int> job = new MapReduceJob<int, string, string, int, string, int>(Map, Reduce);
var items1 = new string[] { "a", "word", "is" };
var pairs1 = items1.Select((item, k) => new Pair<int, string>() { Key = k, Value = item });
var items2 = new string[] { "a", "word" };
var pairs2 = items2.Select((item, k) => new Pair<int, string>() { Key = k, Value = item });
job.Map(pairs1);
job.Map(pairs2);
var result = job.Reduce();
Assert.IsNotNull(result);
Assert.IsNotNull(result);
Assert.AreEqual(3, result.Count());
var pair1 = result.FirstOrDefault(p => p.Key == "word");
Assert.IsNotNull(pair1);
Assert.AreEqual(2, pair1.Value);
var pair2 = result.FirstOrDefault(p => p.Key == "a");
Assert.IsNotNull(pair2);
Assert.AreEqual(2, pair2.Value);
var pair3 = result.FirstOrDefault(p => p.Key == "is");
Assert.IsNotNull(pair3);
Assert.AreEqual(1, pair3.Value);
}
private void _btnStart_Click(object sender, EventArgs e)
{
_btnStart.Enabled = false;
_btnBrowse.Enabled = false;
_prgProgress.Style = ProgressBarStyle.Marquee;
_currentHistogram = null;
_pnlHistogram.Refresh();
_mapReduceJob = new MapReduceJob<byte[], Histogram>(
Setup.Container.Resolve<IBlobStorageProvider>(),
Setup.Container.Resolve<IQueueStorageProvider>());
// Do this asynchronously because it requires a few seconds
ThreadPool.QueueUserWorkItem(new WaitCallback((s) =>
{
using(var input = (Bitmap)Bitmap.FromFile(_currentFileName))
{
var slices = Helpers.SliceBitmapAsPng(input, 14);
// Queue slices
_mapReduceJob.PushItems(new HistogramMapReduceFunctions(), slices, 4);
//_currentHistogram = Helpers.ComputeHistogram(input);
//_pnlHistogram.Refresh();
}
BeginInvoke(new Action(() => _timer.Start()));
}));
}
private void _btnStart_Click(object sender, EventArgs e)
{
_btnStart.Enabled = false;
_btnBrowse.Enabled = false;
_prgProgress.Style = ProgressBarStyle.Marquee;
_currentHistogram = null;
_pnlHistogram.Refresh();
var storage = CloudStorage
.ForAzureConnectionString(Properties.Settings.Default.DataConnectionString)
.BuildStorageProviders();
_mapReduceJob = new MapReduceJob<byte[], Histogram>(storage.BlobStorage, storage.QueueStorage);
// Do this asynchronously because it requires a few seconds
ThreadPool.QueueUserWorkItem(s =>
{
using(var input = (Bitmap)Bitmap.FromFile(_currentFileName))
{
var slices = Helpers.SliceBitmapAsPng(input, 14);
// Queue slices
_mapReduceJob.PushItems(new HistogramMapReduceFunctions(), slices, 4);
//_currentHistogram = Helpers.ComputeHistogram(input);
//_pnlHistogram.Refresh();
}
BeginInvoke(new Action(() => _timer.Start()));
});
}
public void ShouldCountWordsContainedInDataSource()
{
IDataSource dataSource = new WordCountDataSource();
MapReduceJob<Int32> job = new MapReduceJob<Int32>(new WordCounter());
job.Run(dataSource);
Assert.AreEqual(job.GetFinalResults("tom"), 2);
Assert.AreEqual(job.GetFinalResults("kim"), 2);
Assert.AreEqual(job.GetFinalResults("ian"), 2);
Assert.AreEqual(job.GetFinalResults("nancy"), 1);
Assert.AreEqual(job.GetFinalResults("bob"), 1);
}
public Emitter(MapReduceJob job)
{
_reducer = job;
}
public void Consume(MapReduceJob product)
{
Emit(new Emitter(product));
}