/// <summary> /// mapReduce得到结果 /// </summary> /// <param name="collectionName"></param> /// <param name="query"></param> /// <param name="finalizeJs"></param> /// <param name="mapJs"></param> /// <param name="reduceJs"></param> /// <param name="limit"></param> /// <param name="sortBy"></param> /// <returns></returns> public MapReduceResult MapReduceToResult(string collectionName, IMongoQuery query, string finalizeJs, string mapJs, string reduceJs, int limit, IMongoSortBy sortBy) { MongoCollection collection = GetMongoCollection(collectionName); MapReduceOptionsBuilder builder = new MapReduceOptionsBuilder(); builder.SetFinalize(finalizeJs); if (limit > 0) { builder.SetLimit(limit); } if (sortBy != null) { builder.SetSortOrder(sortBy); } MapReduceOutput output = new MapReduceOutput(); output.Mode = MapReduceOutputMode.Inline; builder.SetOutput(output); if (query == null || query == MongoDB.Driver.Builders.Query.Null) { return(collection.MapReduce(mapJs, reduceJs, builder)); } else { return(collection.MapReduce(query, mapJs, reduceJs, builder)); } }
private object ExecuteMapReduce(MongoQueryObject queryObject) { var options = new MapReduceOptionsBuilder(); options.SetQuery(new QueryDocument(queryObject.Query)); options.SetFinalize(new BsonJavaScript(queryObject.FinalizerFunction)); options.SetLimit(queryObject.NumberToLimit); if (queryObject.Sort != null) { options.SetSortOrder(new SortByDocument(queryObject.Sort)); } if (queryObject.NumberToSkip != 0) { throw new InvalidQueryException("MapReduce queries do not support Skips."); } var mapReduce = _collection.MapReduce( new BsonJavaScript(queryObject.MapFunction), new BsonJavaScript(queryObject.ReduceFunction), options); var executor = GetExecutor(typeof(BsonDocument), queryObject.Projector, queryObject.Aggregator, true); return(executor.Compile().DynamicInvoke(queryObject.Collection.Database[mapReduce.CollectionName].FindAll())); }
public void GetPerformance() { using (mServer.RequestStart(mLogs)) { var servicePerformances = mLogs.GetCollection("service_performance"); var map = new BsonJavaScript(@" function() { var key = {}; key.class = this.class; key.method = this.method; key.machineName = this.machineName; emit(key, { count: 1 }); }"); var reduce = new BsonJavaScript(@" function(key, values) { var result = { count: 0 }; values.forEach(function(value){ result.count += value.count; }); return result }"); var finalize = new BsonJavaScript(@" function(key, value){ return value; }"); var options = new MapReduceOptionsBuilder(); options.SetFinalize(finalize); options.SetOutput(MapReduceOutput.Inline); var results = servicePerformances.MapReduce(map, reduce, options); int totalCal = 0; foreach (var result in results.GetResults()) { var doc = result.ToBsonDocument(); var id = doc["_id"].AsBsonDocument; var value = doc["value"].AsBsonDocument; totalCal += Convert.ToInt32(value["count"].AsDouble); string className = id["class"].AsString; string method = id["method"].AsString; string machineName = id["machineName"].AsString; } var all = servicePerformances.FindAll().ToList().Count; Assert.AreEqual(all, totalCal); } }
private void MapReduce() { var map = string.Format(@" function() {{ var queuedTrack = this; var requestCount = (queuedTrack.User.toLowerCase().indexOf('{0}') == 0) ? 0 : 1; var autoPlayCount = (requestCount==1) ? 0 : 1; emit(queuedTrack.Track.Link, {{ requestCount: requestCount, vetoCount: queuedTrack.Vetoes.length, likeCount: queuedTrack.Likes.length, autoPlayCount: autoPlayCount,lastPlayed: Date.parse(queuedTrack.StartedPlayingDateTime),excluded: queuedTrack.Excluded, track: queuedTrack.Track}}); }}", "autoplay"); const string reduce = @" function(key, values) { var result = {requestCount: 0, vetoCount: 0, likeCount: 0, autoPlayCount: 0, lastPlayed: 0, excluded: false, track: null }; values.forEach(function(value){ result.requestCount += value.requestCount; result.vetoCount += value.vetoCount; result.likeCount += value.likeCount; result.autoPlayCount += value.autoPlayCount; if(result.excluded == false) result.excluded = value.excluded; //waiting until we find at least one play if (value.lastPlayed > result.lastPlayed) result.lastPlayed = value.lastPlayed; if (!result.track) result.track = value.track; }); return result; }"; string finalize = string.Format(@" function(key, value){{ var score = (value.likeCount - value.vetoCount)+1; if (score <= 0) score = 0.001; score = Math.round((score * (Math.random()/{0}))*1000); return {{IsExcluded: value.excluded, Score: score, MillisecondsSinceLastPlay: Date.now()-value.lastPlayed, Track: value.track}}; }}", 3); var options = new MapReduceOptionsBuilder(); options.SetFinalize(finalize); options.SetOutput(MapReduceOutput.Replace(DataCollectionName)); if (queuedTrackDataService.DataCollection.Exists()) { queuedTrackDataService.DataCollection.MapReduce(map, reduce, options).GetResults(); } }
public void DoMapReduce() { string map = @" function() { var logEntry = this; emit(logEntry.Country, { count: 1, size: logEntry.Size }); }"; string reduce = @" function(key, values) { var result = {count: 0, size: 0 }; values.forEach(function(value){ result.count += value.count; result.size += value.size; }); return result; }"; string finalize = @" function(key, value){ value.averageSize = value.size / value.count; return value; }"; var collection = database.GetCollection(CollectionName); var options = new MapReduceOptionsBuilder(); options.SetFinalize(finalize); options.SetOutput(MapReduceOutput.Inline); var results = collection.MapReduce(map, reduce, options); foreach (var result in results.GetResults()) { Console.WriteLine(result.ToJson()); } }
public ActionResult MethodInfo(string machineName, string serviceName, string methodName) { IList <MethodDetailModel> methodDetailModelList = new List <MethodDetailModel>(); ViewBag.MachineName = machineName; ViewBag.ServiceName = serviceName; ViewBag.MethodName = methodName; MongoServer server = MongoServer.Create("mongodb://appsit01"); var database = server.GetDatabase("logs"); using (server.RequestStart(database)) { var servicePerformances = database.GetCollection("service_performance"); var map = @" function() { var me = this; var key = {}; key.day = me.createdDate.getDate(); key.month = me.createdDate.getMonth(); key.year = me.createdDate.getFullYear(); if(me.machineName === ':machineName'){ if(me.class === ':serviceName'){ if(me.method === ':methodName'){ emit(key, { count: 1, totalElapsedMilliseconds: me.elapsedMilliseconds }); } } } }"; map = map.Replace(":machineName", machineName); map = map.Replace(":serviceName", serviceName); map = map.Replace(":methodName", methodName); var reduce = @" function(key, values) { var result = { count: 0, totalElapsedMilliseconds: 0 }; values.forEach(function(value){ result.count += value.count; result.totalElapsedMilliseconds += value.totalElapsedMilliseconds; }); return result; }"; var finalize = @" function(key, value){ value.averageElapsedMilliseconds = value.totalElapsedMilliseconds / value.count; return value; }"; var options = new MapReduceOptionsBuilder(); options.SetFinalize(finalize); options.SetOutput(MapReduceOutput.Inline); var results = servicePerformances.MapReduce(map, reduce, options); foreach (var result in results.GetResults()) { var doc = result.ToBsonDocument(); var id = doc["_id"].AsBsonDocument; var value = doc["value"].AsBsonDocument; var count = Convert.ToInt32(value["count"]); var averageElapsedMilliseconds = Convert.ToDouble(value["averageElapsedMilliseconds"]); var totalElapsedMilliseconds = Convert.ToInt32(value["totalElapsedMilliseconds"]); var day = Convert.ToInt32(id["day"]); var month = Convert.ToInt32(id["month"]); var year = Convert.ToInt32(id["year"]); string toDay = string.Format("{0}/{1}/{2}", month, day, year); MethodDetailModel methodDetailModel = new MethodDetailModel { Date = new DateTime(year, month, day), MachineName = machineName, ServiceName = serviceName, MethodName = methodName, TotalUsage = count, TotalElapsedMilliseconds = totalElapsedMilliseconds, AverageElapsedMilliseconds = averageElapsedMilliseconds, }; methodDetailModelList.Add(methodDetailModel); } } methodDetailModelList = methodDetailModelList.OrderByDescending(m => m.Date).ToList(); return(View(methodDetailModelList)); }
/// <summary> /// Returns the table data /// </summary> /// <param name="strTableName"></param> /// <returns></returns> public DataTable GetData(String strDataBaseName, String strTableName, String strFilter, out Int32 intMatches) { //Declarations DataTable dtData = new DataTable(); MongoCollection <NetworkData> colData = getServer().GetDatabase(strDataBaseName).GetCollection <NetworkData>(strTableName); //Initialize intMatches = 0; #region Map Reduce definition var map = @"function Map() { //Declaration var searchFactor = '###'; var hasMatch = false; var parameter; //Get the Request Parameters var requestParameters = this.RequestParameters; //Check whether its not null if(requestParameters != null){ //Check whether its not in Array format <TEMP> //if(requestParameters instanceof Array){ //parameter = true; for(var key in requestParameters){ if(requestParameters.hasOwnProperty(key)){ //Get the parameter parameter = requestParameters[key]; //Check whether the parameter has a match if(parameter.indexOf(searchFactor) != -1){ hasMatch = true; break; } } } //} } //Emit the results emit(hasMatch, { Count: 1, Matches: [{Key:key, Parameter:parameter, FoundOn: this}] }); }"; //Set the search factor map = map.Replace("###", strFilter); var reduce = @"function Reduce(key, values) { //Define the result structure var result = { Count:0, Matches:[] }; //If there is a match if(key){ values.forEach(function(val){ result.Count += val.Count; //Push all the matches val.Matches.forEach(function(match){ result.Matches.push(match); }); }); } return result; }"; #endregion MapReduceOptionsBuilder options = new MapReduceOptionsBuilder(); options.SetOutput(MapReduceOutput.Inline); //Do Map reduce MapReduceResult mapReducedResults = colData.MapReduce(map, reduce, options); //Serialize the results IEnumerable <Result> results = mapReducedResults.GetResultsAs <Result>(); //Change it to a list List <Result> lstResults = results.ToList <Result>(); //Get the result which had a match Result result = lstResults.Find(resultToFind => { return(resultToFind._id); }); //Set the columns dtData.Columns.Add("ID", typeof(String)); dtData.Columns.Add("HostName", typeof(String)); dtData.Columns.Add("SentAs", typeof(String)); dtData.Columns.Add("SentAt", typeof(String)); try { //Set the number of matches from the result intMatches = result.value.Count; //Create the data for the viewer foreach (Match match in result.value.Matches) { //If there is a result if (result._id) { DataRow drRow = dtData.NewRow(); drRow[0] = match.FoundOn.Id; drRow[1] = match.FoundOn.HostName; drRow[2] = match.Key + ":" + match.Parameter; drRow[3] = match.FoundOn.RequestedAt; //Add the row dtData.Rows.Add(drRow); } } } catch (Exception ex) { Console.WriteLine(ex.Message); } return(dtData); }
public void TestReduceMachineNameAndDateOri() { //var connectionString = "mongodb://*****:*****@" function() { var ts = this._id.getTimestamp(); var month = ts.getUTCFullYear() + '-' + (ts.getUTCMonth() + 1); var stats = {}; var aDay = month + '-' + ts.getUTCDate(); stats[aDay] = 1; var strArr = this.machineName; emit(strArr, stats); }"); var reduce = new BsonJavaScript(@" function(key, values) { function merge(out, stat) { for (var aDay in stat) { if( aDay != 'total' && aDay != 'totalDay') { out.total += stat[aDay]; if (!stat.hasOwnProperty(aDay)) { continue; } out[aDay] = (out[aDay] || 0) + stat[aDay]; } } } var out = {total: 0, totalDay:0}; for (var i=0; i < values.length; i++) { merge(out, values[i]); } return out; }"); var finalizef = new BsonJavaScript(@" function(key, value) { value.average = value.total / 2; return value; }"); var options = new MapReduceOptionsBuilder(); options.SetOutput(MapReduceOutput.Inline); //if (FinalizeExists(name)) //{ options.SetFinalize(finalizef); //} var mapRet = collection.MapReduce(map, reduce, options); var ret = mapRet.GetResults().ToList(); int count = 0; ret.ForEach(r => { Console.Write("[#]"); foreach (var item2 in r) { Console.WriteLine("{0}:{1} ", item2.Name.ToString(), item2.Value); } Console.WriteLine(""); }); Console.WriteLine("Total: {0}", count); }
public void TestReduceDateAndMethod() { var connectionString = "mongodb://*****:*****@" function() { var date = new Date( this.timestamp.getFullYear(), this.timestamp.getMonth(), this.timestamp.getDay() ); var out = { methods: {}, userNames: {} } var msg = JSON.parse(this.message); out.methods[msg.method] = 1; out.userNames[this.userName] = 1; emit(date, out); }"); var reduce = new BsonJavaScript(@" function(key, values) { var out = { methods: {}, userNames: {} } values.forEach(function(value) { for( var method in value.methods ){ if( out.methods[method] ){ out.methods[method] += value[method]; } else{ out.methods[method] = value[method]; } }; for( var userName in value.userNames ){ if( out.userNames[userName] ){ out.userNames[userName] += value[userName]; } else{ out.methods[userName] = value[userName]; } } }); return out; }"); var finalizef = new BsonJavaScript(@" function(key, value) { return value; }"); var options = new MapReduceOptionsBuilder(); options.SetOutput(MapReduceOutput.Inline); options.SetFinalize(finalizef); var mapRet = collection.MapReduce(map, reduce, options); var ret = mapRet.GetResults().ToList(); int count = 0; ret.ForEach(r => { Console.Write("[#]"); foreach (var item2 in r) { if (item2.Name.Equals("value")) { var obj = JsonConvert.DeserializeObject <JToken>(item2.Value.ToString()); var v = obj["count"].Value <int>(); count += v; Console.Write("{0}:{1} ", item2.Name.ToString(), item2.Value); } } Console.WriteLine(""); }); Console.WriteLine("Total: {0}", count); }
public void TestReduceMachineNameAndDate() { var connectionString = "mongodb://*****:*****@" function() { var ts = this._id.getTimestamp(); var month = ts.getUTCFullYear() + '-' + (ts.getUTCMonth() + 1); var stats = {}; var field = month + '-' + ts.getUTCDate(); stats[field] = 1; var name = this.machineName; var date = field; emit({name:name,date:date}, {count: 1, date: field}); }"); var reduce = new BsonJavaScript(@" function(key, values) { var sum = 0; var date = ''; values.forEach(function(doc) { sum += doc.count; date = doc.date; }); return {date: date, count: sum}; }"); var finalizef = new BsonJavaScript(@" function(key, value) { value.average = value.count / 2; return value; }"); var options = new MapReduceOptionsBuilder(); options.SetOutput(MapReduceOutput.Inline); options.SetFinalize(finalizef); //var query = new QueryDocument("x", "$lt:3"); //var query2 = Query.LT("a", 3); var mapRet = collection.MapReduce(map, reduce, options); var ret = mapRet.GetResults().ToList(); int count = 0; ret.ForEach(r => { Console.Write("[#]"); foreach (var item2 in r) { if (item2.Name.Equals("value")) { var obj = JsonConvert.DeserializeObject <JToken>(item2.Value.ToString()); var v = obj["count"].Value <int>(); count += v; Console.Write("{0}:{1} ", item2.Name.ToString(), item2.Value); } } Console.WriteLine(""); }); Console.WriteLine("Total: {0}", count); }
public String Mapreduced(Dictionary <string, string> fields, List <string> movement = null, List <string> objs = null, List <string> locations = null, List <string> users = null, Int64 start = 0, Int64 end = 0) { var orderfield = "folio"; try { List <string> cols = new List <string>(); foreach (var x in fields) { cols.Add(x.Key); } cols.Add("CreatedTimeStamp"); string[] arrayfields = cols.ToArray(); string[] arraymov = movement.ToArray(); string[] arrayobjs = objs.ToArray(); string[] arraylocations = locations.ToArray(); string[] arrayusers = users.ToArray(); BsonArray bsonarraymov = new BsonArray(); BsonArray bsonarrayobjs = new BsonArray(); BsonArray bsonarrayloc = new BsonArray(); BsonArray bsonarrayusers = new BsonArray(); for (int i = 0; i < arraymov.Length; i++) { bsonarraymov.Add(arraymov[i]); } for (int i = 0; i < arrayobjs.Length; i++) { bsonarrayobjs.Add(arrayobjs[i]); } for (int i = 0; i < arraylocations.Length; i++) { bsonarrayloc.Add(arraylocations[i]); } for (int i = 0; i < arrayusers.Length; i++) { bsonarrayusers.Add(arrayusers[i]); } List <BsonDocument> documents = new List <BsonDocument>(); var query = Query.And(Query.In("movement", bsonarraymov), Query.In("object", bsonarrayobjs), Query.In("location", bsonarrayloc), Query.In("Creator", bsonarrayusers), Query.GTE("CreatedTimeStamp", start), Query.LTE("CreatedTimeStamp", end)); /*Realiza la conexion */ MongoCollection Demand = conection.getCollection("Demand"); MongoCollection Movements = conection1.getCollection("MovementProfiles"); MongoCollection Locations = conection2.getCollection("Locations"); var Movementsfunction = @"function() { var key = this._id.valueOf(); var value={movement:this.name}; emit(key,value); }"; var Locationsfunction = @"function() { var key = this._id.valueOf(); var value={location:this.name}; emit(key,value); }"; var Demandfunction = @"function() { var key = this.movement; var value = {id:this._id.valueOf(),folio:this.folio,object:this.object,total:this.total,status:this.status,movementFields:this.movementFields,CreatedDate:this.CreatedDate,CreatedTimeStamp:this.CreatedTimeStamp,Creator:this.Creator,system_status:this.system_status}; emit(key,value); }"; var Demandfunction2 = @"function() { var key = this.location; var value = {id:this._id.valueOf(),folio:this.folio,object:this.object,total:this.total,status:this.status,movementFields:this.movementFields,CreatedDate:this.CreatedDate,CreatedTimeStamp:this.CreatedTimeStamp,Creator:this.Creator,system_status:this.system_status}; emit(key,value); }"; var reducefunction = @"function (key, values) { var reduced = {id:'',folio:'',movement:'',location:'',object:'',total:'',status:'',movementFields:'',CreatedDate:'',CreatedTimeStamp:'',Creator:'',system_status:''}; var index=0; values.forEach(function(value){ if(value.movement != null && reduced.movement ==''){ reduced.movement = value.movement; } if(value.location != null && reduced.location == ''){ reduced.location = value.location; } if(value.id != null){ reduced.id= value.id; reduced.folio = value.folio; reduced.object = value.object; reduced.total = value.total; reduced.status = value.status; reduced.movementFields = value.movementFields; reduced.CreatedDate = value.CreatedDate; reduced.CreatedTimeStamp = value.CreatedTimeStamp; reduced.Creator = value.Creator; reduced.system_status = value.system_status; } }); return reduced; }"; /*Establece las propiedades a seguir del mapreduce */ MapReduceOptionsBuilder options = new MapReduceOptionsBuilder(); options.SetOutput(MapReduceOutput.Inline); options.SetOutput(MapReduceOutput.Reduce("result", true)); /* Realiza los mapreduce sobre el mismo resultado */ MapReduceResult result = Movements.MapReduce(Movementsfunction, reducefunction, options); // result = Locations.MapReduce(Locationsfunction, reducefunction, options); result = Demand.MapReduce(Demandfunction, reducefunction, options); // result = Demand.MapReduce(Demandfunction2, reducefunction, options); /* resultado en json */ string results = result.GetResults().ToJson(); return(results); } catch (Exception ex) { return(null); } }
public void GetDetailOfMethod() { using (mServer.RequestStart(mLogs)) { var servicePerformances = mLogs.GetCollection("service_performance"); var map = new BsonJavaScript(@" function() { var me = this; var key = {}; key.day = me.createdDate.getDate(); key.month = me.createdDate.getMonth(); key.year = me.createdDate.getFullYear(); if(me.machineName === 'APPSIT01'){ if(me.class === 'DeviceService'){ if(me.method === 'GetDeployedGeoFenceConf'){ emit(key, { count: 1, totalElapsedMilliseconds: me.elapsedMilliseconds }); } } } }"); var reduce = new BsonJavaScript(@" function(key, values) { var result = { count: 0, totalElapsedMilliseconds: 0.0 }; values.forEach(function(value){ result.count += value.count; result.totalElapsedMilliseconds += value.totalElapsedMilliseconds; }); return result; }"); var finalize = new BsonJavaScript(@" function(key, value){ value.averageElapsedMilliseconds = value.totalElapsedMilliseconds / value.count; return value; }"); var options = new MapReduceOptionsBuilder(); options.SetFinalize(finalize); options.SetOutput(MapReduceOutput.Inline); var results = servicePerformances.MapReduce(map, reduce, options); foreach (var result in results.GetResults()) { var doc = result.ToBsonDocument(); var id = doc["_id"].AsBsonDocument; var value = doc["value"].AsBsonDocument; var count = Convert.ToInt32(value["count"]); var averageElapsedMilliseconds = Convert.ToDouble(value["averageElapsedMilliseconds"]); var totalElapsedMilliseconds = Convert.ToInt32(value["totalElapsedMilliseconds"]); var day = Convert.ToInt32(id["day"]); var month = Convert.ToInt32(id["month"]); var year = Convert.ToInt32(id["year"]); string toDay = string.Format("{0}/{1}/{2}", day, month, year); Console.WriteLine( string.Format("{0} - {1} - {2} - {3}", toDay, count, totalElapsedMilliseconds, averageElapsedMilliseconds)); } var query = Query.And( Query.EQ("machineName", "APPSIT01"), Query.EQ("class", "DeviceService"), Query.EQ("method", "GetDeviceByImei") ); var all = servicePerformances.Find(query).AsEnumerable(); int totalEMs = 0; foreach (var result in all) { var doc = result.ToBsonDocument(); var elapsedMilliseconds = Convert.ToInt32(doc[4].RawValue); totalEMs += elapsedMilliseconds; } Console.WriteLine(totalEMs); } }