public async Task<int> MapReduceToDocumentid(MongoCollection collection) { string map = @" function() { var auditItem = this; emit(auditItem.DocumentId, { count: 1 }); }"; string reduce = @" function(key, values) { var result = {count: 0, Key:key }; values.forEach(function(value){ result.count += value.count; }); return result; }"; int count = 0; try { var options = new MapReduceOptionsBuilder(); //options.SetFinalize(finalize); options.SetOutput(MapReduceOutput.Inline); var results = collection.MapReduce(map, reduce, options); var l = new Dictionary<Guid, MapReduceResult>(); foreach (var result in results.GetResults()) { BsonValue rVal = result["value"]; MapReduceResult d = BsonSerializer.Deserialize<MapReduceResult>(rVal.ToJson()); if (await KeyExists(d.Key)) { log.InfoFormat("Map reduce duplicate for document id {0} ", d.Key); continue; } count++; Console.WriteLine("Adding No [{0}] Key {1} ", count, d.Key.ToString()); l.Add(d.Key, d); var o = await AddToLocalDB(d); if (count % 100 == 0) { log.InfoFormat("{0} document keys processed", count); } } } catch (Exception ex) { log.Error(ex); } log.InfoFormat("{0} document keys saved to local cache "); return count; }
public void DoMapReduce() { string map = @" function() { var logEntry = this; emit(logEntry.Country, { count: 1, size: logEntry.Size }); }"; string reduce = @" function(key, values) { var result = {count: 0, size: 0 }; values.forEach(function(value){ result.count += value.count; result.size += value.size; }); return result; }"; string finalize = @" function(key, value){ value.averageSize = value.size / value.count; return value; }"; var collection = database.GetCollection(CollectionName); var options = new MapReduceOptionsBuilder(); options.SetFinalize(finalize); options.SetOutput(MapReduceOutput.Inline); var results = collection.MapReduce(map, reduce, options); foreach (var result in results.GetResults()) { Console.WriteLine(result.ToJson()); } }
public ActionResult MethodInfo(string machineName, string serviceName, string methodName) { IList<MethodDetailModel> methodDetailModelList = new List<MethodDetailModel>(); ViewBag.MachineName = machineName; ViewBag.ServiceName = serviceName; ViewBag.MethodName = methodName; MongoServer server = MongoServer.Create("mongodb://appsit01"); var database = server.GetDatabase("logs"); using (server.RequestStart(database)) { var servicePerformances = database.GetCollection("service_performance"); var map = @" function() { var me = this; var key = {}; key.day = me.createdDate.getDate(); key.month = me.createdDate.getMonth(); key.year = me.createdDate.getFullYear(); if(me.machineName === ':machineName'){ if(me.class === ':serviceName'){ if(me.method === ':methodName'){ emit(key, { count: 1, totalElapsedMilliseconds: me.elapsedMilliseconds }); } } } }"; map = map.Replace(":machineName", machineName); map = map.Replace(":serviceName", serviceName); map = map.Replace(":methodName", methodName); var reduce = @" function(key, values) { var result = { count: 0, totalElapsedMilliseconds: 0 }; values.forEach(function(value){ result.count += value.count; result.totalElapsedMilliseconds += value.totalElapsedMilliseconds; }); return result; }"; var finalize = @" function(key, value){ value.averageElapsedMilliseconds = value.totalElapsedMilliseconds / value.count; return value; }"; var options = new MapReduceOptionsBuilder(); options.SetFinalize(finalize); options.SetOutput(MapReduceOutput.Inline); var results = servicePerformances.MapReduce(map, reduce, options); foreach (var result in results.GetResults()) { var doc = result.ToBsonDocument(); var id = doc["_id"].AsBsonDocument; var value = doc["value"].AsBsonDocument; var count = Convert.ToInt32(value["count"]); var averageElapsedMilliseconds = Convert.ToDouble(value["averageElapsedMilliseconds"]); var totalElapsedMilliseconds = Convert.ToInt32(value["totalElapsedMilliseconds"]); var day = Convert.ToInt32(id["day"]); var month = Convert.ToInt32(id["month"]); var year = Convert.ToInt32(id["year"]); string toDay = string.Format("{0}/{1}/{2}", month, day, year); MethodDetailModel methodDetailModel = new MethodDetailModel { Date = new DateTime(year, month, day), MachineName = machineName, ServiceName = serviceName, MethodName = methodName, TotalUsage = count, TotalElapsedMilliseconds = totalElapsedMilliseconds, AverageElapsedMilliseconds = averageElapsedMilliseconds, }; methodDetailModelList.Add(methodDetailModel); } } methodDetailModelList = methodDetailModelList.OrderByDescending(m => m.Date).ToList(); return View(methodDetailModelList); }
// This should be a service action. public Dictionary<string, double> generateRatings() { var finalResults = new Dictionary<string, double>(); // TODO: perform incremental mapReduce // Join the messsages and users tables on the username string mapMessages = @" function() { emit(this.Author.username, { blockedBy : 0, Likes : this.Likes}); }"; string mapUsers = @" function() { emit(this.Username, { blockedBy : 0, Likes : 0}); for (var i = 0; i < this.BlockedUsers.length; i++){ var blockedUser = this.BlockedUsers[i]; emit(blockedUser.Username, { blockedBy : 1, Likes : 0 }); } }"; string reduce = @" function(key, values){ var result = { blockedBy : 0, Likes : 0 }; values.forEach(function(value) { result.blockedBy += value.blockedBy; result.Likes += value.Likes; }); return result; }"; string finalize = @" function(key, reducedValue){ var blocked = 1; if(reducedValue.blockedBy != 0){ blocked = parseInt(reducedValue.blockedBy); } var rating = parseInt(reducedValue.Likes) / (blocked * blocked); reducedValue.rating = parseInt(rating); return reducedValue; }"; var options = new MapReduceOptionsBuilder(); options.SetOutput(new MapReduceOutput { CollectionName = "ratings", Mode = MapReduceOutputMode.Reduce }); messages.MapReduce(mapMessages, reduce, options); options.SetFinalize(finalize); users.MapReduce(mapUsers, reduce, options); // Since sorting requires a pre build index // when using mapReduce we issue another query. var results = ratings.FindAllAs<BsonDocument>().SetSortOrder(SortBy.Descending("value.rating")); foreach ( var result in results ) { var values = result["value"]; var rating = values["rating"].AsDouble; var username = result["_id"].AsString; finalResults.Add(username, rating); } return finalResults; }
public void GetDetailOfMethod() { using (mServer.RequestStart(mLogs)) { var servicePerformances = mLogs.GetCollection("service_performance"); var map = new BsonJavaScript(@" function() { var me = this; var key = {}; key.day = me.createdDate.getDate(); key.month = me.createdDate.getMonth(); key.year = me.createdDate.getFullYear(); if(me.machineName === 'APPSIT01'){ if(me.class === 'DeviceService'){ if(me.method === 'GetDeployedGeoFenceConf'){ emit(key, { count: 1, totalElapsedMilliseconds: me.elapsedMilliseconds }); } } } }"); var reduce = new BsonJavaScript(@" function(key, values) { var result = { count: 0, totalElapsedMilliseconds: 0.0 }; values.forEach(function(value){ result.count += value.count; result.totalElapsedMilliseconds += value.totalElapsedMilliseconds; }); return result; }"); var finalize = new BsonJavaScript(@" function(key, value){ value.averageElapsedMilliseconds = value.totalElapsedMilliseconds / value.count; return value; }"); var options = new MapReduceOptionsBuilder(); options.SetFinalize(finalize); options.SetOutput(MapReduceOutput.Inline); var results = servicePerformances.MapReduce(map, reduce, options); foreach (var result in results.GetResults()) { var doc = result.ToBsonDocument(); var id = doc["_id"].AsBsonDocument; var value = doc["value"].AsBsonDocument; var count = Convert.ToInt32(value["count"]); var averageElapsedMilliseconds = Convert.ToDouble(value["averageElapsedMilliseconds"]); var totalElapsedMilliseconds = Convert.ToInt32(value["totalElapsedMilliseconds"]); var day = Convert.ToInt32(id["day"]); var month = Convert.ToInt32(id["month"]); var year = Convert.ToInt32(id["year"]); string toDay = string.Format("{0}/{1}/{2}", day, month, year); Console.WriteLine( string.Format("{0} - {1} - {2} - {3}", toDay, count, totalElapsedMilliseconds, averageElapsedMilliseconds)); } var query = Query.And( Query.EQ("machineName", "APPSIT01"), Query.EQ("class", "DeviceService"), Query.EQ("method", "GetDeviceByImei") ); var all = servicePerformances.Find(query).AsEnumerable(); int totalEMs = 0; foreach (var result in all) { var doc = result.ToBsonDocument(); var elapsedMilliseconds = Convert.ToInt32(doc[4].RawValue); totalEMs += elapsedMilliseconds; } Console.WriteLine(totalEMs); } }
public void GetPerformance() { using (mServer.RequestStart(mLogs)) { var servicePerformances = mLogs.GetCollection("service_performance"); var map = new BsonJavaScript(@" function() { var key = {}; key.class = this.class; key.method = this.method; key.machineName = this.machineName; emit(key, { count: 1 }); }"); var reduce = new BsonJavaScript(@" function(key, values) { var result = { count: 0 }; values.forEach(function(value){ result.count += value.count; }); return result }"); var finalize = new BsonJavaScript(@" function(key, value){ return value; }"); var options = new MapReduceOptionsBuilder(); options.SetFinalize(finalize); options.SetOutput(MapReduceOutput.Inline); var results = servicePerformances.MapReduce(map, reduce, options); int totalCal = 0; foreach (var result in results.GetResults()) { var doc = result.ToBsonDocument(); var id = doc["_id"].AsBsonDocument; var value = doc["value"].AsBsonDocument; totalCal += Convert.ToInt32(value["count"].AsDouble); string className = id["class"].AsString; string method = id["method"].AsString; string machineName = id["machineName"].AsString; } var all = servicePerformances.FindAll().ToList().Count; Assert.AreEqual(all, totalCal); } }
public String DataAnalytics() { string maps = @" function() { var Roo = this; var a=Roo.preTime; if(!a) { a='unknown'; } emit(a, { count: 1 }); }"; string reduces = @" function(key, values) { var result = {count: 0}; values.forEach(function(value){ result.count += value.count; }); return result; }"; string finalizes = @" function(key, value){ value.average = value.count / value.count; return value; }"; var option = new MapReduceOptionsBuilder(); var posts = blog.GetCollection<Roo>("Recipe"); // var que = Query<Roo>.All(h => h.prepTime, ""); //options.SetQuery(que); option.SetFinalize(finalizes); option.SetOutput(MapReduceOutput.Inline); var result = posts.MapReduce(maps, reduces, option); String sc = "["; foreach (var r in result.GetResults()) { sc += r.ToJson()+","; } return sc+"]"; }
public void TestReduceDateAndMethod() { var connectionString = "mongodb://*****:*****@" function() { var date = new Date( this.timestamp.getFullYear(), this.timestamp.getMonth(), this.timestamp.getDay() ); var out = { methods: {}, userNames: {} } var msg = JSON.parse(this.message); out.methods[msg.method] = 1; out.userNames[this.userName] = 1; emit(date, out); }"); var reduce = new BsonJavaScript(@" function(key, values) { var out = { methods: {}, userNames: {} } values.forEach(function(value) { for( var method in value.methods ){ if( out.methods[method] ){ out.methods[method] += value[method]; } else{ out.methods[method] = value[method]; } }; for( var userName in value.userNames ){ if( out.userNames[userName] ){ out.userNames[userName] += value[userName]; } else{ out.methods[userName] = value[userName]; } } }); return out; }"); var finalizef = new BsonJavaScript(@" function(key, value) { return value; }"); var options = new MapReduceOptionsBuilder(); options.SetOutput(MapReduceOutput.Inline); options.SetFinalize(finalizef); var mapRet = collection.MapReduce(map, reduce, options); var ret = mapRet.GetResults().ToList(); int count = 0; ret.ForEach(r => { Console.Write("[#]"); foreach (var item2 in r) { if (item2.Name.Equals("value")) { var obj = JsonConvert.DeserializeObject<JToken>(item2.Value.ToString()); var v = obj["count"].Value<int>(); count += v; Console.Write("{0}:{1} ", item2.Name.ToString(), item2.Value); } } Console.WriteLine(""); }); Console.WriteLine("Total: {0}", count); }
public void TestReduceMachineNameAndDateOri() { //var connectionString = "mongodb://*****:*****@" function() { var ts = this._id.getTimestamp(); var month = ts.getUTCFullYear() + '-' + (ts.getUTCMonth() + 1); var stats = {}; var aDay = month + '-' + ts.getUTCDate(); stats[aDay] = 1; var strArr = this.machineName; emit(strArr, stats); }"); var reduce = new BsonJavaScript(@" function(key, values) { function merge(out, stat) { for (var aDay in stat) { if( aDay != 'total' && aDay != 'totalDay') { out.total += stat[aDay]; if (!stat.hasOwnProperty(aDay)) { continue; } out[aDay] = (out[aDay] || 0) + stat[aDay]; } } } var out = {total: 0, totalDay:0}; for (var i=0; i < values.length; i++) { merge(out, values[i]); } return out; }"); var finalizef = new BsonJavaScript(@" function(key, value) { value.average = value.total / 2; return value; }"); var options = new MapReduceOptionsBuilder(); options.SetOutput(MapReduceOutput.Inline); //if (FinalizeExists(name)) //{ options.SetFinalize(finalizef); //} var mapRet = collection.MapReduce(map, reduce, options); var ret = mapRet.GetResults().ToList(); int count = 0; ret.ForEach(r => { Console.Write("[#]"); foreach (var item2 in r) { Console.WriteLine("{0}:{1} ", item2.Name.ToString(), item2.Value); } Console.WriteLine(""); }); Console.WriteLine("Total: {0}", count); }
public void TestReduceMachineNameAndDate() { var connectionString = "mongodb://*****:*****@" function() { var ts = this._id.getTimestamp(); var month = ts.getUTCFullYear() + '-' + (ts.getUTCMonth() + 1); var stats = {}; var field = month + '-' + ts.getUTCDate(); stats[field] = 1; var name = this.machineName; var date = field; emit({name:name,date:date}, {count: 1, date: field}); }"); var reduce = new BsonJavaScript(@" function(key, values) { var sum = 0; var date = ''; values.forEach(function(doc) { sum += doc.count; date = doc.date; }); return {date: date, count: sum}; }"); var finalizef = new BsonJavaScript(@" function(key, value) { value.average = value.count / 2; return value; }"); var options = new MapReduceOptionsBuilder(); options.SetOutput(MapReduceOutput.Inline); options.SetFinalize(finalizef); //var query = new QueryDocument("x", "$lt:3"); //var query2 = Query.LT("a", 3); var mapRet = collection.MapReduce( map, reduce, options); var ret = mapRet.GetResults().ToList(); int count = 0; ret.ForEach(r => { Console.Write("[#]"); foreach (var item2 in r) { if (item2.Name.Equals("value")) { var obj = JsonConvert.DeserializeObject<JToken>(item2.Value.ToString()); var v = obj["count"].Value<int>(); count += v; Console.Write("{0}:{1} ", item2.Name.ToString(), item2.Value); } } Console.WriteLine(""); }); Console.WriteLine("Total: {0}", count); }
public static void test() { var connectionString = "mongodb://localhost/?safe=true"; // TODO: make this configurable var mongoUrlBuilder = new MongoUrlBuilder(connectionString); var serverSettings = mongoUrlBuilder.ToServerSettings(); if (!serverSettings.SafeMode.Enabled) { serverSettings.SafeMode = SafeMode.True; } var mongo = MongoServer.Create(serverSettings); var db = mongo["csharpdriverunittests"]; var collection = db["movies"]; var movies = new List<Movie> { new Movie { Title="The Perfect Developer", Category="SciFi", Minutes=118 }, new Movie { Title="Lost In Frankfurt am Main", Category="Horror", Minutes=122 }, new Movie { Title="The Infinite Standup", Category="Horror", Minutes=341 } }; collection.InsertBatch(movies); string map = @" function() { var movie = this; emit(movie.Category, { count: 1, totalMinutes: movie.Minutes }); }"; string reduce = @" function(key, values) { var result = {count: 0, totalMinutes: 0 }; values.forEach(function(value){ result.count += value.count; result.totalMinutes += value.totalMinutes; }); return result; }"; string finalize = @" function(key, value){ value.average = value.totalMinutes / value.count; return value; }"; //var linq = from p in collection[""] // select t; var options = new MapReduceOptionsBuilder(); options.SetFinalize(finalize); options.SetOutput(MapReduceOutput.Inline); var results = collection.MapReduce(map, reduce, options); foreach (var result in results.GetResults()) { Console.WriteLine(result.ElementAt(0).Name); } }
protected override void BeginProcessing() { var mc = TargetCollection.Collection as MongoCollection; if (mc == null) ThrowNotImplementedForFiles("MapReduce"); var options = new MapReduceOptionsBuilder(); options.SetJSMode(JSMode); if (Function.Length == 3) options.SetFinalize(new BsonJavaScript(Function[2])); if (_Query != null) options.SetQuery(_Query); if (_SortBy != null) options.SetSortOrder(_SortBy); if (First > 0) options.SetLimit(First); if (Scope != null) options.SetScope(new ScopeDocument(Scope)); if (!string.IsNullOrEmpty(OutCollection) && OutMode == MapReduceOutputMode.Inline) OutMode = MapReduceOutputMode.Replace; var output = new MapReduceOutput(); output.Mode = OutMode; output.DatabaseName = OutDatabase; output.CollectionName = OutCollection; options.SetOutput(output); var result = mc.MapReduce(new BsonJavaScript(Function[0]), new BsonJavaScript(Function[1]), options); if (ResultVariable != null) SessionState.PSVariable.Set(ResultVariable, result); if (OutMode != MapReduceOutputMode.Inline) return; var documentAs = _ParameterAs ?? new ParameterAs(null); //_131018_160000 foreach (var it in result.GetInlineResultsAs(documentAs.Type)) WriteObject(it); }
public static void test() { var connectionString = "mongodb://localhost/?safe=true"; // TODO: make this configurable var mongoUrlBuilder = new MongoUrlBuilder(connectionString); var serverSettings = mongoUrlBuilder.ToServerSettings(); if (!serverSettings.SafeMode.Enabled) { serverSettings.SafeMode = SafeMode.True; } var mongo = MongoServer.Create(serverSettings); var db = mongo["guangzhou_gb"]; var collection = db["Gi_Get2x_Multi"]; string map = @" function() { var abc = this; emit(abc.URI_Main, { count: 1, totalMinutes: 0,totalCount:0 }); } }" ; string reduce = @" function(key, values) { var result = {count: 0, totalMinutes: 0,totalCount:0 }; values.forEach(function(value){ result.count += value.count; result.totalMinutes += value.totalMinutes ; }); return result; }" ; //db.RunCommand var abbsum = db["Gi_Get2x_Multi"].FindAll().Count(); string cmd = @" db.Gi_Get2x_Multi.aggregate([ { $group: {_id:null, total:{$sum:'$Get2x'} } } ]) "; //CommandResult cr = new CommandResult(); var comd = db.RunCommandAs(typeof(CommandResult), cmd); string finalize = @" function(key, value){ value.average =value.count / " + comd.Response.AsBsonArray[0].ToString()+ @"; value.totalCount=" + abbsum.ToString() + @" return value; }" ; // //var query=QueryBuilde IMongoSortBy abc = SortBy.Descending("value.count"); IMongoQuery query=Query.EQ("value.count",1); //MapReduceOutput outname= // var abc = "{ \"sort\" : { \"value.count\" : 1} }"; //MapReduceOutput out=new Console.WriteLine(abc.ToString()); var options = new MapReduceOptionsBuilder(); //options.SetScope( //options.SetQuery(query);// // options.SetSortOrder(abc); //options.SetLimit(2); //options.SetKeepTemp(true); options.SetFinalize(finalize); options.SetOutput("tempppp"); var results = collection.MapReduce(map, reduce, options); Console.WriteLine(results.Command); //results.Command.ToString(); // var resultss = results.GetResults().OrderByDescending(e => e.Elements.Select["count"]).Take(100); var resultss = db["tempppp"].FindAll().SetSortOrder(abc).SetLimit(10); //var resultss = results.GetResults().OrderByDescending(p => "value.count").Take(10); foreach (var result in resultss ) { Console.WriteLine(result); } }
public void MapReduce() { const string ConnectionString = "mongodb://*****:*****@" function() { var movie = this; emit(movie.Category, { count: 1, totalMinutes: movie.Minutes }); }"; string reduce = @" function(key, values) { var result = {count: 0, totalMinutes: 0 }; values.forEach(function(value){ result.count += value.count; result.totalMinutes += value.totalMinutes; }); return result; }"; string finalize = @" function(key, value){ value.average = value.totalMinutes / value.count; return value; }"; var options = new MapReduceOptionsBuilder(); options.SetFinalize(finalize); options.SetOutput(MapReduceOutput.Inline); var results = mov.MapReduce(map, reduce, options); foreach (var result in results.GetResults()) { Response.Write(result.ToJson()); } string maps = @" function() { var Roo = this; var a=Roo.preTime; if(!a) { a='unknown'; } emit(a, { count: 1 }); }"; string reduces = @" function(key, values) { var result = {count: 0}; values.forEach(function(value){ result.count += value.count; }); return result; }"; string finalizes = @" function(key, value){ value.average = value.count / value.count; return value; }"; try { var option = new MapReduceOptionsBuilder(); var posts=blog.GetCollection<Roo>("Recipe"); // var que = Query<Roo>.All(h => h.prepTime, ""); //options.SetQuery(que); option.SetFinalize(finalizes); option.SetOutput(MapReduceOutput.Inline); var result = posts.MapReduce(maps, reduces,option); List<MongoMap> mn=new List<MongoMap>(); foreach (var r in result.GetResults()) { //Response.Write(r.ToJson()+ "</br>"); mn.Add(JsonConvert.DeserializeObject<MongoMap>(r.ToJson())); } } catch (MongoException ex) { Response.Write(ex.StackTrace + ex.Message + ex.Source); } }
/// <summary> /// Returns the table data /// </summary> /// <param name="strTableName"></param> /// <returns></returns> public DataTable GetData(String strDataBaseName, String strTableName, String strFilter, out Int32 intMatches) { //Declarations DataTable dtData = new DataTable(); MongoCollection<NetworkData> colData = getServer().GetDatabase(strDataBaseName).GetCollection<NetworkData>(strTableName); //Initialize intMatches = 0; #region Map Reduce definition var map = @"function Map() { //Declaration var searchFactor = '###'; var hasMatch = false; var parameter; //Get the Request Parameters var requestParameters = this.RequestParameters; //Check whether its not null if(requestParameters != null){ //Check whether its not in Array format <TEMP> //if(requestParameters instanceof Array){ //parameter = true; for(var key in requestParameters){ if(requestParameters.hasOwnProperty(key)){ //Get the parameter parameter = requestParameters[key]; //Check whether the parameter has a match if(parameter.indexOf(searchFactor) != -1){ hasMatch = true; break; } } } //} } //Emit the results emit(hasMatch, { Count: 1, Matches: [{Key:key, Parameter:parameter, FoundOn: this}] }); }"; //Set the search factor map = map.Replace("###", strFilter); var reduce = @"function Reduce(key, values) { //Define the result structure var result = { Count:0, Matches:[] }; //If there is a match if(key){ values.forEach(function(val){ result.Count += val.Count; //Push all the matches val.Matches.forEach(function(match){ result.Matches.push(match); }); }); } return result; }"; #endregion MapReduceOptionsBuilder options = new MapReduceOptionsBuilder(); options.SetOutput(MapReduceOutput.Inline); //Do Map reduce MapReduceResult mapReducedResults = colData.MapReduce(map, reduce, options); //Serialize the results IEnumerable<Result> results = mapReducedResults.GetResultsAs<Result>(); //Change it to a list List<Result> lstResults = results.ToList<Result>(); //Get the result which had a match Result result = lstResults.Find(resultToFind => { return (resultToFind._id); }); //Set the columns dtData.Columns.Add("ID", typeof(String)); dtData.Columns.Add("HostName", typeof(String)); dtData.Columns.Add("SentAs", typeof(String)); dtData.Columns.Add("SentAt", typeof(String)); try { //Set the number of matches from the result intMatches = result.value.Count; //Create the data for the viewer foreach (Match match in result.value.Matches) { //If there is a result if (result._id) { DataRow drRow = dtData.NewRow(); drRow[0] = match.FoundOn.Id; drRow[1] = match.FoundOn.HostName; drRow[2] = match.Key + ":" + match.Parameter; drRow[3] = match.FoundOn.RequestedAt; //Add the row dtData.Rows.Add(drRow); } } } catch (Exception ex) { Console.WriteLine(ex.Message); } return dtData; }