static void Main(string[] args)
{
Tuple<string, int>[] tuples = new Tuple<string, int>[] {
new Tuple<string, int>("a", 3),
new Tuple<string, int>("b", 2),
new Tuple<string, int>("b", 5) };
MapReduce<string, int> letters = new MapReduce<string, int>(tuples);
IEnumerable<Tuple<string, int>> newmap;
letters.Map<string, int>((input) =>
{
return new Tuple<string, int>[] { new Tuple<string, int>(input.Item1, input.Item2 * input.Item2) };
}, out newmap);
IEnumerable<Tuple<string, int>> reduction = letters.Reduce<string, int>(newmap, (key, values) =>
{
int total = 0;
foreach (var item in values)
{
total += item;
}
return total;
});
foreach (var item in reduction)
{
Console.WriteLine("{0} = {1}", item.Item1, item.Item2);
}
Console.WriteLine("Press enter to exit.");
Console.ReadLine();
}
public static void Emit(string key, string value)
{
// Find out which reducer to assign this key to
int reducerIndex = (int)MapReduce.partitioner(key, MapReduce.numPartitions);
bucketLock.WaitOne();
var bucket = reducers.ElementAt(reducerIndex).Find(x => x.key == key);
// If we are able to find a bucket that already has our key, just add our value to that bucket
if (bucket != null)
{
bucket.values.Add(value);
}
else
{
// Since we could not find an existing key bucket for us to drop the value into, initialize a new bucket for that reducer to work on with our value.
reducers.ElementAt(reducerIndex).Add(new Bucket()
{
key = key, values = new List <string>()
{
value
}
});
}
bucketLock.ReleaseMutex();
}
/// <summary>
/// Executes the map reduce with any options
/// </summary>
/// <typeparam retval="X">The return type</typeparam>
/// <param retval="options">The options</param>
/// <returns></returns>
public IEnumerable <X> MapReduce <X>(MapReduceOptions <T> options)
{
var mr = new MapReduce(_db);
var response = mr.Execute(options);
var collection = response.GetCollection <X>();
return(collection.Find().ToList());
}
static void Mapper(string fileName)
{
string text = File.ReadAllText(fileName);
foreach (var word in text.Split(' ', '\t', '\n', '\r', '\0'))
{
MapReduce.Emit(word, "1");
}
}
public void test_columnsSum()
{
Dictionary <string, int[, ]> input_mat = new Dictionary <string, int[, ]>
{
["firstmat"] = new int[, ] {
{ 1, 2, 3 }, { 5, 9, 8 }
},
["secondMat"] = new int[, ] {
{ 8, 6, 3 }, { 10, 5, 3 }
}
};
#region Map function
IList <KeyValuePair <int, int> > Map_mat(string key, int[,] value)
{
IList <KeyValuePair <int, int> > res = new List <KeyValuePair <int, int> >();
for (int j = 0; j < value.GetLength(1); j++)
{
int sum = 0;
for (int i = 0; i < value.GetLength(0); i++)
{
sum += value[i, j];
}
res.Add(new KeyValuePair <int, int>(j, sum));
}
return(res);
}
#endregion
#region Reduce function
KeyValuePair <int, int> Reduce_mat(int key, IEnumerable <int> values)
{
int sum = 0;
foreach (int value in values)
{
sum += value;
}
return(new KeyValuePair <int, int>(key, sum));
}
#endregion
MapReduce <string, int[, ], int, int, int, int> mapreduce_mat = new MapReduce <string, int[, ], int, int, int, int>(Map_mat, Reduce_mat);
IEnumerable <KeyValuePair <int, int> > res_mat = mapreduce_mat.get_result(input_mat);
Dictionary <int, int> theorycal_res = new Dictionary <int, int> {
[0] = 24, [1] = 22, [2] = 17
};
Assert.IsTrue(Enumerable.SequenceEqual(theorycal_res, res_mat));
}
public void test_wordCounting()
{
Dictionary <string, string> input_Text = new Dictionary <string, string>
{
["firstDocument"] = "chien chat",
["secondDocument"] = "chat licorne"
};
#region Reduce function
KeyValuePair <string, int> Reduce_Text(string key, IEnumerable <int> values)
{
int sum = 0;
foreach (int value in values)
{
sum += value;
}
return(new KeyValuePair <string, int>(key, sum));
}
#endregion
#region Map function
IList <KeyValuePair <string, int> > Map_Text(string key, string value)
{
List <KeyValuePair <string, int> > result = new List <KeyValuePair <string, int> >();
foreach (var word in value.Split(' '))
{
result.Add(new KeyValuePair <string, int>(word, 1));
}
return(result);
}
#endregion
MapReduce <string, string, string, int, string, int> mapreduce_text = new MapReduce <string, string, string, int, string, int>(Map_Text, Reduce_Text);
IEnumerable <KeyValuePair <string, int> > res_text = mapreduce_text.get_result(input_Text);
Dictionary <string, int> theorycal_res = new Dictionary <string, int> {
["chien"] = 1, ["chat"] = 2, ["licorne"] = 1
};
Assert.IsTrue(Enumerable.SequenceEqual(theorycal_res, res_text));
}
//Helper for using map reduce in mongo
public T MapReduce <T>(string map, string reduce)
{
T result = default(T);
MapReduce mr = Db.CreateMapReduce();
MapReduceResponse response =
mr.Execute(new MapReduceOptions(typeof(T).Name)
{
Map = map,
Reduce = reduce
});
IMongoCollection <MapReduceResult <T> > coll = response.GetCollection <MapReduceResult <T> >();
MapReduceResult <T> r = coll.Find().FirstOrDefault();
result = r.Value;
return(result);
}
public static int Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine($"Usage: {AppDomain.CurrentDomain.FriendlyName} <filename>");
return(-1);
}
DataReader dr;
try {
dr = new DataReader(args[0]);
}
catch (FileNotFoundException) {
Console.WriteLine($"{args[0]}: File not found");
return(-2);
}
Console.WriteLine($"Done reading {args[0]}, {dr.Count} records read.");
var mr = new MapReduce <double>();
foreach (double data in dr)
{
mr.Add(data);
}
mr.Map(i => { return((i + i) / 2.0); });
var rtask = mr.ReduceAsync((accum, cur) => {
return((accum + cur) / Math.Sqrt(accum + cur));
});
ulong iters = 0;
while (rtask.IsCompleted == false)
{
iters++;
}
rtask.Wait();
Console.WriteLine($"Reduce returned value: {rtask.Result:F3}. Reduce took {iters} iterations.");
return(0);
}
public static void Problem2()
{
#region Word counting Example
Console.WriteLine("\n---------------- Word Counting ----------------");
Dictionary <string, string> input_Text = new Dictionary <string, string> {
["firstDocument"] = "hello hello je suis pierrick et je suis là pour vous faire rigoler",
["secondDocument"] = "je suis encore là !"
};
KeyValuePair <string, int> Reduce_Text(string key, IEnumerable <int> values)
{
int sum = 0;
foreach (int value in values)
{
sum += value;
}
return(new KeyValuePair <string, int>(key, sum));
}
IList <KeyValuePair <string, int> > Map_Text(string key, string value)
{
List <KeyValuePair <string, int> > result = new List <KeyValuePair <string, int> >();
foreach (var word in value.Split(' '))
{
result.Add(new KeyValuePair <string, int>(word, 1));
}
return(result);
}
MapReduce <string, string, string, int, string, int> mapreduce_text = new MapReduce <string, string, string, int, string, int>(Map_Text, Reduce_Text);
IEnumerable <KeyValuePair <string, int> > res_text = mapreduce_text.get_result(input_Text);
foreach (KeyValuePair <string, int> pair in res_text)
{
Console.WriteLine(pair.Key + ": " + pair.Value);
}
#endregion
#region Columns sum example
Console.WriteLine("\n---------------- Columns sum ----------------");
Dictionary <string, int[, ]> input_mat = new Dictionary <string, int[, ]>
{
["firstmat"] = new int[, ] {
{ 1, 2, 3 }, { 5, 9, 8 }
},
["secondMat"] = new int[, ] {
{ 8, 6, 3 }, { 10, 5, 3 }
}
};
IList <KeyValuePair <int, int> > Map_mat(string key, int[,] value)
{
IList <KeyValuePair <int, int> > res = new List <KeyValuePair <int, int> >();
for (int j = 0; j < value.GetLength(1); j++)
{
int sum = 0;
for (int i = 0; i < value.GetLength(0); i++)
{
sum += value[i, j];
}
res.Add(new KeyValuePair <int, int>(j, sum));
}
return(res);
}
KeyValuePair <int, int> Reduce_mat(int key, IEnumerable <int> values)
{
int sum = 0;
foreach (int value in values)
{
sum += value;
}
return(new KeyValuePair <int, int>(key, sum));
}
MapReduce <string, int[, ], int, int, int, int> mapreduce_mat = new MapReduce <string, int[, ], int, int, int, int>(Map_mat, Reduce_mat);
IEnumerable <KeyValuePair <int, int> > res_mat = mapreduce_mat.get_result(input_mat);
foreach (KeyValuePair <int, int> pair in res_mat)
{
Console.WriteLine("Column n°" + (pair.Key + 1) + ": " + pair.Value);
}
#endregion
}
/// <summary>
/// Initializes a new instance of the <see cref="MongoMapReduceException"/> class.
/// </summary>
/// <param name="exception">The exception.</param>
/// <param name="mapReduce">The map reduce.</param>
public MongoMapReduceException(MongoCommandException exception, MapReduce mapReduce)
:base(exception.Message,exception.Error, exception.Command) {
MapReduceResult = new MapReduce.MapReduceResult(exception.Error);
}
static void Main(string[] args)
{
Console.WriteLine("Word Count");
MapReduce.Run(args, Mapper, Reduce, 3, 1, MapReduce.DefaultHashPartitioner);
}
public Doc MapReduce(MapReduce mr)
{
throw new NotImplementedException();
}
public MapReduceCallInProc(string name, MapReduce map, MapReduce reduce)
: base(name)
{
this.map = map;
this.reduce = reduce;
}
public MapReduceCallInProc(string name, MapReduce mr)
: base(name)
{
this.map = mr;
this.reduce = mr;
}