public IList <BsonDocument> MapReduce(string _map, string _reduce, string collName, string fileter, string outputResult, int timeOutSeconds = 60, bool isDel = false)
{
IList <BsonDocument> result = new List <BsonDocument>();
var map = new BsonJavaScript(_map);
var reduce = new BsonJavaScript(_reduce);
//const key = this.Date.getFullYear();
if (isDel && !string.IsNullOrEmpty(outputResult))
{
//删除结果集合
mDB.DropCollection(outputResult);
}
var coll = mDB.GetCollection <BsonDocument>(collName);
//var options = new MapReduceOptions<BsonDocument, BsonDocument>();//what should be TResult?
//options.OutputOptions = MapReduceOutputOptions.Inline;
//查询条件
FilterDefinition <BsonDocument> _filter = null;
if (!string.IsNullOrEmpty(fileter))
{
_filter = fileter;
}
//输出结果集合
var output = MapReduceOutputOptions.Reduce(outputResult, nonAtomic: true);
if (string.IsNullOrEmpty(outputResult))
{
output = MapReduceOutputOptions.Inline;
}
MapReduceOptions <BsonDocument, BsonDocument> options = new MapReduceOptions <BsonDocument, BsonDocument>
{
Filter = _filter,
MaxTime = TimeSpan.FromSeconds(timeOutSeconds),
OutputOptions = output,
Verbose = true,
};
try
{
result = coll.MapReduce(map, reduce, options).ToList();
}
catch (Exception ex)
{
Console.WriteLine($"Exception occurred {ex.Message}");
}
return(result);
}
public async Task DemoMapReduceAsync(IMongoCollection <ClubMember> collection)
{
//The match function has to be written in JavaScript
//the map method outputs a key value pair for each document
//in this case the key is the Lastname property value and the value is the Age property value
var map = new BsonJavaScript(@"function()
{
//Associate each LastName property with the Age value
emit(this.Lastname,this.Age);
}");
//The MapReduce method uses the output from the Map method
// to produce a list of ages for each unique LastName value.
// Then each key and its associated list of values is presented to the Reduce method in turn.
var reduce = new BsonJavaScript(@"function(lastName,ages)
{
return Array.sum(ages);
}");
//The Reduce method returns the Lastname as the key and the sum of the ages as the value
//The beauty of this technique is that data can be processed in batches
//The output of one batch is combined with that of another and fed back through the Reducer
//This is repeated until the output is reduced to the number of unique keys.
// The results are output to a new collection named ResultsCollection on the server.
// This saves on the use of computer memory and enables the results to be queried effectively by using indexes.
var options = new MapReduceOptions <ClubMember, BsonDocument>
{
//Replace (the default) : the content of existing collection is dropped, and the new output go into it
// Merge The existing collection is kept.When a result with the same Key(the _id) exists, the value in the collection is replaced with it.
//Reduce : The existing collection is kept.When a result with the same Key(the _id) exists,
//MongoDB runs the reduce function against the two of them, and stores the result.
OutputOptions = MapReduceOutputOptions.Reduce("ResultsCollection")
};
var resultAsBsonDocumentList = await collection.MapReduce(map, reduce, options).ToListAsync();
Console.WriteLine("The total age for every member of each family is ....");
var reduction =
resultAsBsonDocumentList.Select(
doc => new { family = doc["_id"].AsString, age = (int)doc["value"].AsDouble });
foreach (var anon in reduction)
{
Console.WriteLine("{0} Family Total Age {1}", anon.family, anon.age);
}
}
//public static void test()
//{
// var coll = mDB.GetCollection<ProtocolByPub>("publog");
// string mapFunction = @"function(){
// emit(this.cusid, this.price);
// };";
// string reduceFunction = @"function(cusid, prices){
// var total = 0;
// total = Array.sum(prices);
// return { sum: total };
// };";
// FilterDefinition<BsonDocument> filter = "{ x : { $regex : /ABC/ } }";
// var options = new MapReduceOptions<BsonDocument, ProtocolByPub>();//what should be TResult?
// options.Filter = filter;
// options.OutputOptions = MapReduceOutputOptions.Inline;
// var cusid_prices_results = coll.MapReduce<ProtocolByPub>(mapFunction, reduceFunction);
//}
//this.Date.getFullYear()
// private const string MapJs = @"function mapF() {
// const key = this.Date.getFullYear();
// const valuePerYear = { total: 1};
// emit(key, valuePerYear);
//}; ";
// private const string ReduceJS = @"function reduceF(year, values) {
// var sum = 0;
// values.forEach(function(v) {
// sum += v.total;
// });
// return {total: NumberInt(sum)};
//}";
public void test1()
{
string MapJs = @"function mapF() {
const key = this.firstName;
const ages = this.age;
emit(key, ages);
}; ";
string ReduceJS = @"function reduceF(key, ages) {
var sum = 0;
ages.forEach(function(v) {
sum += v.age;
});
return NumberInt(sum);
}";
//string mongoConnectionString = "my-connection-string";
IMongoCollection <BsonDocument> collection = mDB.GetCollection <BsonDocument>("test2");
BsonJavaScript map = new BsonJavaScript(MapJs);
BsonJavaScript reduce = new BsonJavaScript(ReduceJS);
FilterDefinitionBuilder <BsonDocument> filterBuilder = new FilterDefinitionBuilder <BsonDocument>();
var f = filterBuilder.Eq("age", 12);
// FilterDefinition<BsonDocument> filter = new BsonDocument{
// {"_id",12}
//};
MapReduceOptions <BsonDocument, BsonDocument> options = new MapReduceOptions <BsonDocument, BsonDocument>
{
Filter = f,
MaxTime = TimeSpan.FromMinutes(1),
OutputOptions = MapReduceOutputOptions.Reduce("Result", nonAtomic: true),
Verbose = true
};
try
{
var result = collection.MapReduce(map, reduce, options).ToList();
}
catch (Exception ex)
{
Console.WriteLine($"Exception occurred {ex.Message}");
}
}
static void Main(string[] args)
{
var client = new MongoDB.Driver.MongoClient("mongodb://localhost");
var database = client.GetDatabase("MapReduce");
if (!database.ListCollections().Any())
{
SeedOrders(database);
SeedCustomers(database);
}
var map = GetFunction("CustomerMapFunction");
var reduce = GetFunction("CustomerReduceFunction");
var options = new MapReduceOptions <Customer, BsonDocument>();
options.Filter = Builders <Customer> .Filter.Eq(o => o.CustomerId, 5000);
options.Verbose = false;
options.OutputOptions = MapReduceOutputOptions.Replace("Out");
database.GetCollection <Customer>(CUSTOMER_COLLECTION).MapReduce <BsonDocument>(new BsonJavaScript(map), new BsonJavaScript(reduce), options);
var map2 = GetFunction("OrderMapFunction");
var reduce2 = GetFunction("OrderReduceFunction");
var options2 = new MapReduceOptions <Order, BsonDocument>();
options2.Filter = Builders <Order> .Filter.Eq(o => o.CustomerId, 5000);
options2.Verbose = false;
options2.OutputOptions = MapReduceOutputOptions.Reduce("Out");
var res = database.GetCollection <Order>(ORDER_COLLECTION).MapReduce <BsonDocument>(new BsonJavaScript(map2), new BsonJavaScript(reduce2), options2);
var output = database.GetCollection <BsonDocument>("Out");
foreach (var doc in output.Find(b => true).ToList())
{
Console.WriteLine();
Console.WriteLine(doc.ToJson());
}
Console.Read();
}
public object mapReduce(object map, object reduce, object options)
{
var mapReduceOptions = new MapReduceOptions <BsonDocument, BsonDocument>();
object sort = options.GetProperty("sort");
if (sort != null)
{
mapReduceOptions.Sort = new BsonDocumentSortDefinition <BsonDocument>(sort.ToBsonDocument());
}
object query = options.GetProperty("query");
if (query != null)
{
mapReduceOptions.Filter = new BsonDocumentFilterDefinition <BsonDocument>(query.ToBsonDocument());
}
object scope = options.GetProperty("scope");
if (scope != null)
{
mapReduceOptions.Scope = scope.ToBsonDocument();
}
object limit = options.GetProperty("limit");
if (limit != null)
{
mapReduceOptions.Limit = limit.ToIntOrDefault();
}
object verbose = options.GetProperty("verbose");
if (limit != null)
{
mapReduceOptions.Verbose = verbose.ToBoolOrDefault();
}
object jsMode = options.GetProperty("jsMode");
if (jsMode != null)
{
mapReduceOptions.JavaScriptMode = jsMode.ToBoolOrDefault();
}
object bypassDocumentValidation = options.GetProperty("bypassDocumentValidation");
if (bypassDocumentValidation != null)
{
mapReduceOptions.BypassDocumentValidation = bypassDocumentValidation.ToBoolOrDefault();
}
object finalize = options.GetProperty("finalize");
if (finalize != null)
{
mapReduceOptions.Finalize = new BsonJavaScript(finalize.ToStringOrDefault());
}
object outCollection = options.GetProperty("out");
if (outCollection is string)
{
mapReduceOptions.OutputOptions = MapReduceOutputOptions.Replace(outCollection.ToStringOrDefault());
}
if (outCollection is IDictionary <string, object> )
{
var outDicionary = (IDictionary <string, object>)outCollection;
object inline = outDicionary.GetValueOrDefault2("inline");
if (inline != null)
{
mapReduceOptions.OutputOptions = MapReduceOutputOptions.Inline;
}
else
{
string replaceAction = outDicionary.GetValueOrDefault2("replace").ToStringOrDefault();
string mergeAction = outDicionary.GetValueOrDefault2("merge").ToStringOrDefault();
string reduceAction = outDicionary.GetValueOrDefault2("reduce").ToStringOrDefault();
string db = outDicionary.GetValueOrDefault2("db").ToStringOrDefault();
bool? nonAtomic = outDicionary.GetValueOrDefault2("nonAtomic").ToBoolOrDefault();
bool? sharded = outDicionary.GetValueOrDefault2("sharded").ToBoolOrDefault();
if (replaceAction != null)
{
mapReduceOptions.OutputOptions = MapReduceOutputOptions.Replace(replaceAction, db, sharded);
}
if (mergeAction != null)
{
mapReduceOptions.OutputOptions = MapReduceOutputOptions.Merge(mergeAction, db, sharded, nonAtomic);
}
if (reduceAction != null)
{
mapReduceOptions.OutputOptions = MapReduceOutputOptions.Reduce(reduceAction, db, sharded, nonAtomic);
}
}
}
return(new MongoEvaluatedCursor <BsonDocument>(
_collection.MapReduce <BsonDocument>(
new BsonJavaScript(map.ToStringOrDefault()),
new BsonJavaScript(reduce.ToStringOrDefault()),
mapReduceOptions)
));
}
public async void GroupByAccountFromCommand(string companyName)
{
try
{
ConsoleLogger.LogInformation(
string.Concat("Calculation Test: Mapreduce on Sharded Cluster: ",
companyName, " Started."));
const string reduceCollectionName = "calculationresults";
var accountMapScript = Helpers.GetScriptContent(@"SampleMongoScripts\MapReduce\accountMap.mongojs");
var accountFactorMapScript =
Helpers.GetScriptContent(@"SampleMongoScripts\MapReduce\accountFactorMap.mongojs");
var reduceScript = Helpers.GetScriptContent(@"SampleMongoScripts\MapReduce\reduce.mongojs");
var finalizeScript = Helpers.GetScriptContent(@"SampleMongoScripts\MapReduce\finalize.mongojs");
var mapReduceAggregationScript =
File.ReadAllText(Path.Combine(Directory.GetCurrentDirectory(),
@"SampleMongoScripts\MapReduce\aggregation.mongojs"))
.Replace("[[TargetCollection]]", reduceCollectionName)
.Replace("[[CompanyName]]", companyName);
var timer = Stopwatch.StartNew();
Task.Run(async() =>
{
await MongoDataContext.Repository <Account>()
.GetCollection()
.MapReduceAsync(accountMapScript, reduceScript,
new MapReduceOptions <Account, BsonDocument>
{
Finalize = finalizeScript,
OutputOptions =
MapReduceOutputOptions.Reduce(reduceCollectionName,
MongoDataContext.Database.DatabaseNamespace.DatabaseName)
});
await MongoDataContext.Repository <AccountFactor>()
.GetCollection()
.MapReduceAsync(accountFactorMapScript, reduceScript,
new MapReduceOptions <AccountFactor, BsonDocument>
{
Finalize = finalizeScript,
OutputOptions =
MapReduceOutputOptions.Reduce(reduceCollectionName,
MongoDataContext.Database.DatabaseNamespace.DatabaseName)
});
ConsoleLogger.LogWarning(string.Concat("Mapreduce finished. New collection created: ",
reduceCollectionName));
}).Wait();
ConsoleLogger.LogWarning(string.Concat("Document count for ", reduceCollectionName, ": ",
MongoDataContext.Database.GetCollection <BsonDocument>(reduceCollectionName)
.AsQueryable()
.Count()
.ToString("N0")));
var result =
await
MongoDataContext.Database.GetCommandResultAsync <MapReduceCalculationResult>(
mapReduceAggregationScript);
timer.Stop();
Console.WriteLine(string.Concat(Environment.NewLine, "### MapreduceOnShardedCluster >> ",
companyName, " ###"));
ConsoleTable.From(result).Write(Format.Alternative);
var timespan = timer.Elapsed;
ConsoleLogger.LogInformation(
string.Concat("Calculation Test: Mapreduce on Sharded Cluster: ",
companyName, " Finished. Duration: ",
string.Format("{0:00}:{1:00}:{2:00}", timespan.Minutes, timespan.Seconds,
timespan.Milliseconds / 10)));
_testCounter++;
ConsoleLogger.LogWarning(string.Concat("Test(s) left: ", _testCounter, "/", _testCount));
}
catch (Exception ex)
{
ConsoleLogger.LogCritical(ex.GetAllMessages());
}
}
public virtual IEnumerable <TResult> MapReduce <TEntity, TResult>(MapReduceOptionsProxy <TEntity, TResult> options) where TEntity : CollectionEntityBase, new()
{
if (string.IsNullOrEmpty(options.Map))
{
throw new Exception("map is must");
}
if (string.IsNullOrEmpty(options.Reduce))
{
throw new Exception("reduce is must");
}
var o = new MapReduceOptions <TEntity, TResult>();
if (options.OutputEnum.Equals(MapReduceOutputOptionsEnum.Inline))
{
o.OutputOptions = MapReduceOutputOptions.Inline;
}
else
{
if (string.IsNullOrEmpty(options.DatabaseName) || string.IsNullOrEmpty(options.CollectionName))
{
throw new Exception("DatabaseName and CollectionName is must");
}
if (options.OutputEnum.Equals(MapReduceOutputOptionsEnum.Merge))
{
o.OutputOptions = MapReduceOutputOptions.Reduce(options.CollectionName, options.DatabaseName);
}
else if (options.OutputEnum.Equals(MapReduceOutputOptionsEnum.Reduce))
{
o.OutputOptions = MapReduceOutputOptions.Reduce(options.CollectionName, options.DatabaseName);
}
else if (options.OutputEnum.Equals(MapReduceOutputOptionsEnum.Replace))
{
o.OutputOptions = MapReduceOutputOptions.Replace(options.CollectionName, options.DatabaseName);
}
}
o.BypassDocumentValidation = options.BypassDocumentValidation;
if (!string.IsNullOrEmpty(options.Filter))
{
o.Filter = options.Filter;
}
if (!string.IsNullOrEmpty(options.Finalize))
{
o.Finalize = options.Finalize;
}
o.JavaScriptMode = options.JavaScriptMode;
o.Limit = options.Limit;
o.MaxTime = options.MaxTime;
if (!string.IsNullOrEmpty(options.Sort))
{
o.Sort = options.Sort;
}
o.Verbose = options.Verbose;
var r = this.CurrentCollection <TEntity>().MapReduce <TResult>(new MongoDB.Bson.BsonJavaScript(options.Map), new MongoDB.Bson.BsonJavaScript(options.Reduce), o);
if (options.OutputEnum.Equals(MapReduceOutputOptionsEnum.Inline))
{
return(r.Current);
}
else
{
return(null);
}
}
