public IList <BsonDocument> MapReduce(string _map, string _reduce, string collName, string fileter, string outputResult, int timeOutSeconds = 60, bool isDel = false) { IList <BsonDocument> result = new List <BsonDocument>(); var map = new BsonJavaScript(_map); var reduce = new BsonJavaScript(_reduce); //const key = this.Date.getFullYear(); if (isDel && !string.IsNullOrEmpty(outputResult)) { //删除结果集合 mDB.DropCollection(outputResult); } var coll = mDB.GetCollection <BsonDocument>(collName); //var options = new MapReduceOptions<BsonDocument, BsonDocument>();//what should be TResult? //options.OutputOptions = MapReduceOutputOptions.Inline; //查询条件 FilterDefinition <BsonDocument> _filter = null; if (!string.IsNullOrEmpty(fileter)) { _filter = fileter; } //输出结果集合 var output = MapReduceOutputOptions.Reduce(outputResult, nonAtomic: true); if (string.IsNullOrEmpty(outputResult)) { output = MapReduceOutputOptions.Inline; } MapReduceOptions <BsonDocument, BsonDocument> options = new MapReduceOptions <BsonDocument, BsonDocument> { Filter = _filter, MaxTime = TimeSpan.FromSeconds(timeOutSeconds), OutputOptions = output, Verbose = true, }; try { result = coll.MapReduce(map, reduce, options).ToList(); } catch (Exception ex) { Console.WriteLine($"Exception occurred {ex.Message}"); } return(result); }
public async Task DemoMapReduceAsync(IMongoCollection <ClubMember> collection) { //The match function has to be written in JavaScript //the map method outputs a key value pair for each document //in this case the key is the Lastname property value and the value is the Age property value var map = new BsonJavaScript(@"function() { //Associate each LastName property with the Age value emit(this.Lastname,this.Age); }"); //The MapReduce method uses the output from the Map method // to produce a list of ages for each unique LastName value. // Then each key and its associated list of values is presented to the Reduce method in turn. var reduce = new BsonJavaScript(@"function(lastName,ages) { return Array.sum(ages); }"); //The Reduce method returns the Lastname as the key and the sum of the ages as the value //The beauty of this technique is that data can be processed in batches //The output of one batch is combined with that of another and fed back through the Reducer //This is repeated until the output is reduced to the number of unique keys. // The results are output to a new collection named ResultsCollection on the server. // This saves on the use of computer memory and enables the results to be queried effectively by using indexes. var options = new MapReduceOptions <ClubMember, BsonDocument> { //Replace (the default) : the content of existing collection is dropped, and the new output go into it // Merge The existing collection is kept.When a result with the same Key(the _id) exists, the value in the collection is replaced with it. //Reduce : The existing collection is kept.When a result with the same Key(the _id) exists, //MongoDB runs the reduce function against the two of them, and stores the result. OutputOptions = MapReduceOutputOptions.Reduce("ResultsCollection") }; var resultAsBsonDocumentList = await collection.MapReduce(map, reduce, options).ToListAsync(); Console.WriteLine("The total age for every member of each family is ...."); var reduction = resultAsBsonDocumentList.Select( doc => new { family = doc["_id"].AsString, age = (int)doc["value"].AsDouble }); foreach (var anon in reduction) { Console.WriteLine("{0} Family Total Age {1}", anon.family, anon.age); } }
//public static void test() //{ // var coll = mDB.GetCollection<ProtocolByPub>("publog"); // string mapFunction = @"function(){ // emit(this.cusid, this.price); // };"; // string reduceFunction = @"function(cusid, prices){ // var total = 0; // total = Array.sum(prices); // return { sum: total }; // };"; // FilterDefinition<BsonDocument> filter = "{ x : { $regex : /ABC/ } }"; // var options = new MapReduceOptions<BsonDocument, ProtocolByPub>();//what should be TResult? // options.Filter = filter; // options.OutputOptions = MapReduceOutputOptions.Inline; // var cusid_prices_results = coll.MapReduce<ProtocolByPub>(mapFunction, reduceFunction); //} //this.Date.getFullYear() // private const string MapJs = @"function mapF() { // const key = this.Date.getFullYear(); // const valuePerYear = { total: 1}; // emit(key, valuePerYear); //}; "; // private const string ReduceJS = @"function reduceF(year, values) { // var sum = 0; // values.forEach(function(v) { // sum += v.total; // }); // return {total: NumberInt(sum)}; //}"; public void test1() { string MapJs = @"function mapF() { const key = this.firstName; const ages = this.age; emit(key, ages); }; "; string ReduceJS = @"function reduceF(key, ages) { var sum = 0; ages.forEach(function(v) { sum += v.age; }); return NumberInt(sum); }"; //string mongoConnectionString = "my-connection-string"; IMongoCollection <BsonDocument> collection = mDB.GetCollection <BsonDocument>("test2"); BsonJavaScript map = new BsonJavaScript(MapJs); BsonJavaScript reduce = new BsonJavaScript(ReduceJS); FilterDefinitionBuilder <BsonDocument> filterBuilder = new FilterDefinitionBuilder <BsonDocument>(); var f = filterBuilder.Eq("age", 12); // FilterDefinition<BsonDocument> filter = new BsonDocument{ // {"_id",12} //}; MapReduceOptions <BsonDocument, BsonDocument> options = new MapReduceOptions <BsonDocument, BsonDocument> { Filter = f, MaxTime = TimeSpan.FromMinutes(1), OutputOptions = MapReduceOutputOptions.Reduce("Result", nonAtomic: true), Verbose = true }; try { var result = collection.MapReduce(map, reduce, options).ToList(); } catch (Exception ex) { Console.WriteLine($"Exception occurred {ex.Message}"); } }
static void Main(string[] args) { var client = new MongoDB.Driver.MongoClient("mongodb://localhost"); var database = client.GetDatabase("MapReduce"); if (!database.ListCollections().Any()) { SeedOrders(database); SeedCustomers(database); } var map = GetFunction("CustomerMapFunction"); var reduce = GetFunction("CustomerReduceFunction"); var options = new MapReduceOptions <Customer, BsonDocument>(); options.Filter = Builders <Customer> .Filter.Eq(o => o.CustomerId, 5000); options.Verbose = false; options.OutputOptions = MapReduceOutputOptions.Replace("Out"); database.GetCollection <Customer>(CUSTOMER_COLLECTION).MapReduce <BsonDocument>(new BsonJavaScript(map), new BsonJavaScript(reduce), options); var map2 = GetFunction("OrderMapFunction"); var reduce2 = GetFunction("OrderReduceFunction"); var options2 = new MapReduceOptions <Order, BsonDocument>(); options2.Filter = Builders <Order> .Filter.Eq(o => o.CustomerId, 5000); options2.Verbose = false; options2.OutputOptions = MapReduceOutputOptions.Reduce("Out"); var res = database.GetCollection <Order>(ORDER_COLLECTION).MapReduce <BsonDocument>(new BsonJavaScript(map2), new BsonJavaScript(reduce2), options2); var output = database.GetCollection <BsonDocument>("Out"); foreach (var doc in output.Find(b => true).ToList()) { Console.WriteLine(); Console.WriteLine(doc.ToJson()); } Console.Read(); }
public object mapReduce(object map, object reduce, object options) { var mapReduceOptions = new MapReduceOptions <BsonDocument, BsonDocument>(); object sort = options.GetProperty("sort"); if (sort != null) { mapReduceOptions.Sort = new BsonDocumentSortDefinition <BsonDocument>(sort.ToBsonDocument()); } object query = options.GetProperty("query"); if (query != null) { mapReduceOptions.Filter = new BsonDocumentFilterDefinition <BsonDocument>(query.ToBsonDocument()); } object scope = options.GetProperty("scope"); if (scope != null) { mapReduceOptions.Scope = scope.ToBsonDocument(); } object limit = options.GetProperty("limit"); if (limit != null) { mapReduceOptions.Limit = limit.ToIntOrDefault(); } object verbose = options.GetProperty("verbose"); if (limit != null) { mapReduceOptions.Verbose = verbose.ToBoolOrDefault(); } object jsMode = options.GetProperty("jsMode"); if (jsMode != null) { mapReduceOptions.JavaScriptMode = jsMode.ToBoolOrDefault(); } object bypassDocumentValidation = options.GetProperty("bypassDocumentValidation"); if (bypassDocumentValidation != null) { mapReduceOptions.BypassDocumentValidation = bypassDocumentValidation.ToBoolOrDefault(); } object finalize = options.GetProperty("finalize"); if (finalize != null) { mapReduceOptions.Finalize = new BsonJavaScript(finalize.ToStringOrDefault()); } object outCollection = options.GetProperty("out"); if (outCollection is string) { mapReduceOptions.OutputOptions = MapReduceOutputOptions.Replace(outCollection.ToStringOrDefault()); } if (outCollection is IDictionary <string, object> ) { var outDicionary = (IDictionary <string, object>)outCollection; object inline = outDicionary.GetValueOrDefault2("inline"); if (inline != null) { mapReduceOptions.OutputOptions = MapReduceOutputOptions.Inline; } else { string replaceAction = outDicionary.GetValueOrDefault2("replace").ToStringOrDefault(); string mergeAction = outDicionary.GetValueOrDefault2("merge").ToStringOrDefault(); string reduceAction = outDicionary.GetValueOrDefault2("reduce").ToStringOrDefault(); string db = outDicionary.GetValueOrDefault2("db").ToStringOrDefault(); bool? nonAtomic = outDicionary.GetValueOrDefault2("nonAtomic").ToBoolOrDefault(); bool? sharded = outDicionary.GetValueOrDefault2("sharded").ToBoolOrDefault(); if (replaceAction != null) { mapReduceOptions.OutputOptions = MapReduceOutputOptions.Replace(replaceAction, db, sharded); } if (mergeAction != null) { mapReduceOptions.OutputOptions = MapReduceOutputOptions.Merge(mergeAction, db, sharded, nonAtomic); } if (reduceAction != null) { mapReduceOptions.OutputOptions = MapReduceOutputOptions.Reduce(reduceAction, db, sharded, nonAtomic); } } } return(new MongoEvaluatedCursor <BsonDocument>( _collection.MapReduce <BsonDocument>( new BsonJavaScript(map.ToStringOrDefault()), new BsonJavaScript(reduce.ToStringOrDefault()), mapReduceOptions) )); }
public async void GroupByAccountFromCommand(string companyName) { try { ConsoleLogger.LogInformation( string.Concat("Calculation Test: Mapreduce on Sharded Cluster: ", companyName, " Started.")); const string reduceCollectionName = "calculationresults"; var accountMapScript = Helpers.GetScriptContent(@"SampleMongoScripts\MapReduce\accountMap.mongojs"); var accountFactorMapScript = Helpers.GetScriptContent(@"SampleMongoScripts\MapReduce\accountFactorMap.mongojs"); var reduceScript = Helpers.GetScriptContent(@"SampleMongoScripts\MapReduce\reduce.mongojs"); var finalizeScript = Helpers.GetScriptContent(@"SampleMongoScripts\MapReduce\finalize.mongojs"); var mapReduceAggregationScript = File.ReadAllText(Path.Combine(Directory.GetCurrentDirectory(), @"SampleMongoScripts\MapReduce\aggregation.mongojs")) .Replace("[[TargetCollection]]", reduceCollectionName) .Replace("[[CompanyName]]", companyName); var timer = Stopwatch.StartNew(); Task.Run(async() => { await MongoDataContext.Repository <Account>() .GetCollection() .MapReduceAsync(accountMapScript, reduceScript, new MapReduceOptions <Account, BsonDocument> { Finalize = finalizeScript, OutputOptions = MapReduceOutputOptions.Reduce(reduceCollectionName, MongoDataContext.Database.DatabaseNamespace.DatabaseName) }); await MongoDataContext.Repository <AccountFactor>() .GetCollection() .MapReduceAsync(accountFactorMapScript, reduceScript, new MapReduceOptions <AccountFactor, BsonDocument> { Finalize = finalizeScript, OutputOptions = MapReduceOutputOptions.Reduce(reduceCollectionName, MongoDataContext.Database.DatabaseNamespace.DatabaseName) }); ConsoleLogger.LogWarning(string.Concat("Mapreduce finished. New collection created: ", reduceCollectionName)); }).Wait(); ConsoleLogger.LogWarning(string.Concat("Document count for ", reduceCollectionName, ": ", MongoDataContext.Database.GetCollection <BsonDocument>(reduceCollectionName) .AsQueryable() .Count() .ToString("N0"))); var result = await MongoDataContext.Database.GetCommandResultAsync <MapReduceCalculationResult>( mapReduceAggregationScript); timer.Stop(); Console.WriteLine(string.Concat(Environment.NewLine, "### MapreduceOnShardedCluster >> ", companyName, " ###")); ConsoleTable.From(result).Write(Format.Alternative); var timespan = timer.Elapsed; ConsoleLogger.LogInformation( string.Concat("Calculation Test: Mapreduce on Sharded Cluster: ", companyName, " Finished. Duration: ", string.Format("{0:00}:{1:00}:{2:00}", timespan.Minutes, timespan.Seconds, timespan.Milliseconds / 10))); _testCounter++; ConsoleLogger.LogWarning(string.Concat("Test(s) left: ", _testCounter, "/", _testCount)); } catch (Exception ex) { ConsoleLogger.LogCritical(ex.GetAllMessages()); } }
public virtual IEnumerable <TResult> MapReduce <TEntity, TResult>(MapReduceOptionsProxy <TEntity, TResult> options) where TEntity : CollectionEntityBase, new() { if (string.IsNullOrEmpty(options.Map)) { throw new Exception("map is must"); } if (string.IsNullOrEmpty(options.Reduce)) { throw new Exception("reduce is must"); } var o = new MapReduceOptions <TEntity, TResult>(); if (options.OutputEnum.Equals(MapReduceOutputOptionsEnum.Inline)) { o.OutputOptions = MapReduceOutputOptions.Inline; } else { if (string.IsNullOrEmpty(options.DatabaseName) || string.IsNullOrEmpty(options.CollectionName)) { throw new Exception("DatabaseName and CollectionName is must"); } if (options.OutputEnum.Equals(MapReduceOutputOptionsEnum.Merge)) { o.OutputOptions = MapReduceOutputOptions.Reduce(options.CollectionName, options.DatabaseName); } else if (options.OutputEnum.Equals(MapReduceOutputOptionsEnum.Reduce)) { o.OutputOptions = MapReduceOutputOptions.Reduce(options.CollectionName, options.DatabaseName); } else if (options.OutputEnum.Equals(MapReduceOutputOptionsEnum.Replace)) { o.OutputOptions = MapReduceOutputOptions.Replace(options.CollectionName, options.DatabaseName); } } o.BypassDocumentValidation = options.BypassDocumentValidation; if (!string.IsNullOrEmpty(options.Filter)) { o.Filter = options.Filter; } if (!string.IsNullOrEmpty(options.Finalize)) { o.Finalize = options.Finalize; } o.JavaScriptMode = options.JavaScriptMode; o.Limit = options.Limit; o.MaxTime = options.MaxTime; if (!string.IsNullOrEmpty(options.Sort)) { o.Sort = options.Sort; } o.Verbose = options.Verbose; var r = this.CurrentCollection <TEntity>().MapReduce <TResult>(new MongoDB.Bson.BsonJavaScript(options.Map), new MongoDB.Bson.BsonJavaScript(options.Reduce), o); if (options.OutputEnum.Equals(MapReduceOutputOptionsEnum.Inline)) { return(r.Current); } else { return(null); } }