/// <summary>
/// mapReduce得到结果
/// </summary>
/// <param name="collectionName"></param>
/// <param name="query"></param>
/// <param name="finalizeJs"></param>
/// <param name="mapJs"></param>
/// <param name="reduceJs"></param>
/// <param name="limit"></param>
/// <param name="sortBy"></param>
/// <returns></returns>
public MapReduceResult MapReduceToResult(string collectionName, IMongoQuery query, string finalizeJs, string mapJs, string reduceJs, int limit, IMongoSortBy sortBy)
{
MongoCollection collection = GetMongoCollection(collectionName);
MapReduceOptionsBuilder builder = new MapReduceOptionsBuilder();
builder.SetFinalize(finalizeJs);
if (limit > 0)
{
builder.SetLimit(limit);
}
if (sortBy != null)
{
builder.SetSortOrder(sortBy);
}
MapReduceOutput output = new MapReduceOutput();
output.Mode = MapReduceOutputMode.Inline;
builder.SetOutput(output);
if (query == null || query == MongoDB.Driver.Builders.Query.Null)
{
return(collection.MapReduce(mapJs, reduceJs, builder));
}
else
{
return(collection.MapReduce(query, mapJs, reduceJs, builder));
}
}
private object ExecuteMapReduce(MongoQueryObject queryObject)
{
var options = new MapReduceOptionsBuilder();
options.SetQuery(new QueryDocument(queryObject.Query));
options.SetFinalize(new BsonJavaScript(queryObject.FinalizerFunction));
options.SetLimit(queryObject.NumberToLimit);
if (queryObject.Sort != null)
{
options.SetSortOrder(new SortByDocument(queryObject.Sort));
}
if (queryObject.NumberToSkip != 0)
{
throw new InvalidQueryException("MapReduce queries do not support Skips.");
}
var mapReduce = _collection.MapReduce(
new BsonJavaScript(queryObject.MapFunction),
new BsonJavaScript(queryObject.ReduceFunction),
options);
var executor = GetExecutor(typeof(BsonDocument), queryObject.Projector, queryObject.Aggregator, true);
return(executor.Compile().DynamicInvoke(queryObject.Collection.Database[mapReduce.CollectionName].FindAll()));
}
public void GetPerformance()
{
using (mServer.RequestStart(mLogs))
{
var servicePerformances = mLogs.GetCollection("service_performance");
var map = new BsonJavaScript(@"
function() {
var key = {};
key.class = this.class;
key.method = this.method;
key.machineName = this.machineName;
emit(key, { count: 1 });
}");
var reduce = new BsonJavaScript(@"
function(key, values) {
var result = { count: 0 };
values.forEach(function(value){
result.count += value.count;
});
return result
}");
var finalize = new BsonJavaScript(@"
function(key, value){
return value;
}");
var options = new MapReduceOptionsBuilder();
options.SetFinalize(finalize);
options.SetOutput(MapReduceOutput.Inline);
var results = servicePerformances.MapReduce(map, reduce, options);
int totalCal = 0;
foreach (var result in results.GetResults())
{
var doc = result.ToBsonDocument();
var id = doc["_id"].AsBsonDocument;
var value = doc["value"].AsBsonDocument;
totalCal += Convert.ToInt32(value["count"].AsDouble);
string className = id["class"].AsString;
string method = id["method"].AsString;
string machineName = id["machineName"].AsString;
}
var all = servicePerformances.FindAll().ToList().Count;
Assert.AreEqual(all, totalCal);
}
}
private void MapReduce()
{
var map = string.Format(@"
function() {{
var queuedTrack = this;
var requestCount = (queuedTrack.User.toLowerCase().indexOf('{0}') == 0) ? 0 : 1;
var autoPlayCount = (requestCount==1) ? 0 : 1;
emit(queuedTrack.Track.Link, {{ requestCount: requestCount, vetoCount: queuedTrack.Vetoes.length, likeCount: queuedTrack.Likes.length, autoPlayCount: autoPlayCount,lastPlayed: Date.parse(queuedTrack.StartedPlayingDateTime),excluded: queuedTrack.Excluded, track: queuedTrack.Track}});
}}", "autoplay");
const string reduce = @"
function(key, values) {
var result = {requestCount: 0, vetoCount: 0, likeCount: 0, autoPlayCount: 0, lastPlayed: 0, excluded: false, track: null };
values.forEach(function(value){
result.requestCount += value.requestCount;
result.vetoCount += value.vetoCount;
result.likeCount += value.likeCount;
result.autoPlayCount += value.autoPlayCount;
if(result.excluded == false) result.excluded = value.excluded; //waiting until we find at least one play
if (value.lastPlayed > result.lastPlayed) result.lastPlayed = value.lastPlayed;
if (!result.track) result.track = value.track;
});
return result;
}";
string finalize = string.Format(@"
function(key, value){{
var score = (value.likeCount - value.vetoCount)+1;
if (score <= 0) score = 0.001;
score = Math.round((score * (Math.random()/{0}))*1000);
return {{IsExcluded: value.excluded, Score: score, MillisecondsSinceLastPlay: Date.now()-value.lastPlayed, Track: value.track}};
}}", 3);
var options = new MapReduceOptionsBuilder();
options.SetFinalize(finalize);
options.SetOutput(MapReduceOutput.Replace(DataCollectionName));
if (queuedTrackDataService.DataCollection.Exists())
{
queuedTrackDataService.DataCollection.MapReduce(map, reduce, options).GetResults();
}
}
public void DoMapReduce()
{
string map = @"
function() {
var logEntry = this;
emit(logEntry.Country, { count: 1, size: logEntry.Size });
}";
string reduce = @"
function(key, values) {
var result = {count: 0, size: 0 };
values.forEach(function(value){
result.count += value.count;
result.size += value.size;
});
return result;
}";
string finalize = @"
function(key, value){
value.averageSize = value.size / value.count;
return value;
}";
var collection = database.GetCollection(CollectionName);
var options = new MapReduceOptionsBuilder();
options.SetFinalize(finalize);
options.SetOutput(MapReduceOutput.Inline);
var results = collection.MapReduce(map, reduce, options);
foreach (var result in results.GetResults())
{
Console.WriteLine(result.ToJson());
}
}
public ActionResult MethodInfo(string machineName, string serviceName, string methodName)
{
IList <MethodDetailModel> methodDetailModelList = new List <MethodDetailModel>();
ViewBag.MachineName = machineName;
ViewBag.ServiceName = serviceName;
ViewBag.MethodName = methodName;
MongoServer server = MongoServer.Create("mongodb://appsit01");
var database = server.GetDatabase("logs");
using (server.RequestStart(database))
{
var servicePerformances = database.GetCollection("service_performance");
var map = @"
function() {
var me = this;
var key = {};
key.day = me.createdDate.getDate();
key.month = me.createdDate.getMonth();
key.year = me.createdDate.getFullYear();
if(me.machineName === ':machineName'){
if(me.class === ':serviceName'){
if(me.method === ':methodName'){
emit(key, { count: 1, totalElapsedMilliseconds: me.elapsedMilliseconds });
}
}
}
}";
map = map.Replace(":machineName", machineName);
map = map.Replace(":serviceName", serviceName);
map = map.Replace(":methodName", methodName);
var reduce = @"
function(key, values) {
var result = { count: 0, totalElapsedMilliseconds: 0 };
values.forEach(function(value){
result.count += value.count;
result.totalElapsedMilliseconds += value.totalElapsedMilliseconds;
});
return result;
}";
var finalize = @"
function(key, value){
value.averageElapsedMilliseconds = value.totalElapsedMilliseconds / value.count;
return value;
}";
var options = new MapReduceOptionsBuilder();
options.SetFinalize(finalize);
options.SetOutput(MapReduceOutput.Inline);
var results = servicePerformances.MapReduce(map, reduce, options);
foreach (var result in results.GetResults())
{
var doc = result.ToBsonDocument();
var id = doc["_id"].AsBsonDocument;
var value = doc["value"].AsBsonDocument;
var count = Convert.ToInt32(value["count"]);
var averageElapsedMilliseconds = Convert.ToDouble(value["averageElapsedMilliseconds"]);
var totalElapsedMilliseconds = Convert.ToInt32(value["totalElapsedMilliseconds"]);
var day = Convert.ToInt32(id["day"]);
var month = Convert.ToInt32(id["month"]);
var year = Convert.ToInt32(id["year"]);
string toDay = string.Format("{0}/{1}/{2}", month, day, year);
MethodDetailModel methodDetailModel = new MethodDetailModel
{
Date = new DateTime(year, month, day),
MachineName = machineName,
ServiceName = serviceName,
MethodName = methodName,
TotalUsage = count,
TotalElapsedMilliseconds = totalElapsedMilliseconds,
AverageElapsedMilliseconds = averageElapsedMilliseconds,
};
methodDetailModelList.Add(methodDetailModel);
}
}
methodDetailModelList = methodDetailModelList.OrderByDescending(m => m.Date).ToList();
return(View(methodDetailModelList));
}
/// <summary>
/// Returns the table data
/// </summary>
/// <param name="strTableName"></param>
/// <returns></returns>
public DataTable GetData(String strDataBaseName, String strTableName, String strFilter, out Int32 intMatches)
{
//Declarations
DataTable dtData = new DataTable();
MongoCollection <NetworkData> colData = getServer().GetDatabase(strDataBaseName).GetCollection <NetworkData>(strTableName);
//Initialize
intMatches = 0;
#region Map Reduce definition
var map =
@"function Map() {
//Declaration
var searchFactor = '###';
var hasMatch = false;
var parameter;
//Get the Request Parameters
var requestParameters = this.RequestParameters;
//Check whether its not null
if(requestParameters != null){
//Check whether its not in Array format <TEMP>
//if(requestParameters instanceof Array){
//parameter = true;
for(var key in requestParameters){
if(requestParameters.hasOwnProperty(key)){
//Get the parameter
parameter = requestParameters[key];
//Check whether the parameter has a match
if(parameter.indexOf(searchFactor) != -1){
hasMatch = true;
break;
}
}
}
//}
}
//Emit the results
emit(hasMatch, { Count: 1, Matches: [{Key:key, Parameter:parameter, FoundOn: this}] });
}";
//Set the search factor
map = map.Replace("###", strFilter);
var reduce =
@"function Reduce(key, values) {
//Define the result structure
var result = { Count:0, Matches:[] };
//If there is a match
if(key){
values.forEach(function(val){
result.Count += val.Count;
//Push all the matches
val.Matches.forEach(function(match){
result.Matches.push(match);
});
});
}
return result;
}";
#endregion
MapReduceOptionsBuilder options = new MapReduceOptionsBuilder();
options.SetOutput(MapReduceOutput.Inline);
//Do Map reduce
MapReduceResult mapReducedResults = colData.MapReduce(map, reduce, options);
//Serialize the results
IEnumerable <Result> results = mapReducedResults.GetResultsAs <Result>();
//Change it to a list
List <Result> lstResults = results.ToList <Result>();
//Get the result which had a match
Result result = lstResults.Find(resultToFind => { return(resultToFind._id); });
//Set the columns
dtData.Columns.Add("ID", typeof(String));
dtData.Columns.Add("HostName", typeof(String));
dtData.Columns.Add("SentAs", typeof(String));
dtData.Columns.Add("SentAt", typeof(String));
try
{
//Set the number of matches from the result
intMatches = result.value.Count;
//Create the data for the viewer
foreach (Match match in result.value.Matches)
{
//If there is a result
if (result._id)
{
DataRow drRow = dtData.NewRow();
drRow[0] = match.FoundOn.Id;
drRow[1] = match.FoundOn.HostName;
drRow[2] = match.Key + ":" + match.Parameter;
drRow[3] = match.FoundOn.RequestedAt;
//Add the row
dtData.Rows.Add(drRow);
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
return(dtData);
}
public void TestReduceMachineNameAndDateOri()
{
//var connectionString = "mongodb://*****:*****@"
function() {
var ts = this._id.getTimestamp();
var month = ts.getUTCFullYear() + '-' + (ts.getUTCMonth() + 1);
var stats = {};
var aDay = month + '-' + ts.getUTCDate();
stats[aDay] = 1;
var strArr = this.machineName;
emit(strArr, stats);
}");
var reduce = new BsonJavaScript(@"
function(key, values) {
function merge(out, stat) {
for (var aDay in stat) {
if( aDay != 'total' && aDay != 'totalDay')
{
out.total += stat[aDay];
if (!stat.hasOwnProperty(aDay)) {
continue;
}
out[aDay] = (out[aDay] || 0) + stat[aDay];
}
}
}
var out = {total: 0, totalDay:0};
for (var i=0; i < values.length; i++) {
merge(out, values[i]);
}
return out;
}");
var finalizef = new BsonJavaScript(@"
function(key, value)
{
value.average = value.total / 2;
return value;
}");
var options = new MapReduceOptionsBuilder();
options.SetOutput(MapReduceOutput.Inline);
//if (FinalizeExists(name))
//{
options.SetFinalize(finalizef);
//}
var mapRet = collection.MapReduce(map, reduce, options);
var ret = mapRet.GetResults().ToList();
int count = 0;
ret.ForEach(r =>
{
Console.Write("[#]");
foreach (var item2 in r)
{
Console.WriteLine("{0}:{1} ", item2.Name.ToString(), item2.Value);
}
Console.WriteLine("");
});
Console.WriteLine("Total: {0}", count);
}
public void TestReduceDateAndMethod()
{
var connectionString = "mongodb://*****:*****@"
function() {
var date = new Date( this.timestamp.getFullYear(),
this.timestamp.getMonth(),
this.timestamp.getDay() );
var out = { methods: {}, userNames: {} }
var msg = JSON.parse(this.message);
out.methods[msg.method] = 1;
out.userNames[this.userName] = 1;
emit(date, out);
}");
var reduce = new BsonJavaScript(@"
function(key, values) {
var out = { methods: {}, userNames: {} }
values.forEach(function(value) {
for( var method in value.methods ){
if( out.methods[method] ){
out.methods[method] += value[method];
}
else{
out.methods[method] = value[method];
}
};
for( var userName in value.userNames ){
if( out.userNames[userName] ){
out.userNames[userName] += value[userName];
}
else{
out.methods[userName] = value[userName];
}
}
});
return out;
}");
var finalizef = new BsonJavaScript(@"
function(key, value)
{
return value;
}");
var options = new MapReduceOptionsBuilder();
options.SetOutput(MapReduceOutput.Inline);
options.SetFinalize(finalizef);
var mapRet = collection.MapReduce(map, reduce, options);
var ret = mapRet.GetResults().ToList();
int count = 0;
ret.ForEach(r =>
{
Console.Write("[#]");
foreach (var item2 in r)
{
if (item2.Name.Equals("value"))
{
var obj = JsonConvert.DeserializeObject <JToken>(item2.Value.ToString());
var v = obj["count"].Value <int>();
count += v;
Console.Write("{0}:{1} ", item2.Name.ToString(), item2.Value);
}
}
Console.WriteLine("");
});
Console.WriteLine("Total: {0}", count);
}
public void TestReduceMachineNameAndDate()
{
var connectionString = "mongodb://*****:*****@"
function() {
var ts = this._id.getTimestamp();
var month = ts.getUTCFullYear() + '-' + (ts.getUTCMonth() + 1);
var stats = {};
var field = month + '-' + ts.getUTCDate();
stats[field] = 1;
var name = this.machineName;
var date = field;
emit({name:name,date:date}, {count: 1, date: field});
}");
var reduce = new BsonJavaScript(@"
function(key, values) {
var sum = 0;
var date = '';
values.forEach(function(doc) {
sum += doc.count;
date = doc.date;
});
return {date: date, count: sum};
}");
var finalizef = new BsonJavaScript(@"
function(key, value)
{
value.average = value.count / 2;
return value;
}");
var options = new MapReduceOptionsBuilder();
options.SetOutput(MapReduceOutput.Inline);
options.SetFinalize(finalizef);
//var query = new QueryDocument("x", "$lt:3");
//var query2 = Query.LT("a", 3);
var mapRet = collection.MapReduce(map, reduce, options);
var ret = mapRet.GetResults().ToList();
int count = 0;
ret.ForEach(r =>
{
Console.Write("[#]");
foreach (var item2 in r)
{
if (item2.Name.Equals("value"))
{
var obj = JsonConvert.DeserializeObject <JToken>(item2.Value.ToString());
var v = obj["count"].Value <int>();
count += v;
Console.Write("{0}:{1} ", item2.Name.ToString(), item2.Value);
}
}
Console.WriteLine("");
});
Console.WriteLine("Total: {0}", count);
}
public String Mapreduced(Dictionary <string, string> fields, List <string> movement = null, List <string> objs = null, List <string> locations = null, List <string> users = null, Int64 start = 0, Int64 end = 0)
{
var orderfield = "folio";
try
{
List <string> cols = new List <string>();
foreach (var x in fields)
{
cols.Add(x.Key);
}
cols.Add("CreatedTimeStamp");
string[] arrayfields = cols.ToArray();
string[] arraymov = movement.ToArray();
string[] arrayobjs = objs.ToArray();
string[] arraylocations = locations.ToArray();
string[] arrayusers = users.ToArray();
BsonArray bsonarraymov = new BsonArray();
BsonArray bsonarrayobjs = new BsonArray();
BsonArray bsonarrayloc = new BsonArray();
BsonArray bsonarrayusers = new BsonArray();
for (int i = 0; i < arraymov.Length; i++)
{
bsonarraymov.Add(arraymov[i]);
}
for (int i = 0; i < arrayobjs.Length; i++)
{
bsonarrayobjs.Add(arrayobjs[i]);
}
for (int i = 0; i < arraylocations.Length; i++)
{
bsonarrayloc.Add(arraylocations[i]);
}
for (int i = 0; i < arrayusers.Length; i++)
{
bsonarrayusers.Add(arrayusers[i]);
}
List <BsonDocument> documents = new List <BsonDocument>();
var query = Query.And(Query.In("movement", bsonarraymov), Query.In("object", bsonarrayobjs), Query.In("location", bsonarrayloc), Query.In("Creator", bsonarrayusers), Query.GTE("CreatedTimeStamp", start), Query.LTE("CreatedTimeStamp", end));
/*Realiza la conexion */
MongoCollection Demand = conection.getCollection("Demand");
MongoCollection Movements = conection1.getCollection("MovementProfiles");
MongoCollection Locations = conection2.getCollection("Locations");
var Movementsfunction = @"function() {
var key = this._id.valueOf();
var value={movement:this.name};
emit(key,value);
}";
var Locationsfunction = @"function() {
var key = this._id.valueOf();
var value={location:this.name};
emit(key,value);
}";
var Demandfunction = @"function() {
var key = this.movement;
var value = {id:this._id.valueOf(),folio:this.folio,object:this.object,total:this.total,status:this.status,movementFields:this.movementFields,CreatedDate:this.CreatedDate,CreatedTimeStamp:this.CreatedTimeStamp,Creator:this.Creator,system_status:this.system_status};
emit(key,value);
}";
var Demandfunction2 = @"function() {
var key = this.location;
var value = {id:this._id.valueOf(),folio:this.folio,object:this.object,total:this.total,status:this.status,movementFields:this.movementFields,CreatedDate:this.CreatedDate,CreatedTimeStamp:this.CreatedTimeStamp,Creator:this.Creator,system_status:this.system_status};
emit(key,value);
}";
var reducefunction = @"function (key, values) {
var reduced = {id:'',folio:'',movement:'',location:'',object:'',total:'',status:'',movementFields:'',CreatedDate:'',CreatedTimeStamp:'',Creator:'',system_status:''};
var index=0;
values.forEach(function(value){
if(value.movement != null && reduced.movement ==''){
reduced.movement = value.movement;
}
if(value.location != null && reduced.location == ''){
reduced.location = value.location;
}
if(value.id != null){
reduced.id= value.id;
reduced.folio = value.folio;
reduced.object = value.object;
reduced.total = value.total;
reduced.status = value.status;
reduced.movementFields = value.movementFields;
reduced.CreatedDate = value.CreatedDate;
reduced.CreatedTimeStamp = value.CreatedTimeStamp;
reduced.Creator = value.Creator;
reduced.system_status = value.system_status;
}
});
return reduced;
}";
/*Establece las propiedades a seguir del mapreduce */
MapReduceOptionsBuilder options = new MapReduceOptionsBuilder();
options.SetOutput(MapReduceOutput.Inline);
options.SetOutput(MapReduceOutput.Reduce("result", true));
/* Realiza los mapreduce sobre el mismo resultado */
MapReduceResult result = Movements.MapReduce(Movementsfunction, reducefunction, options);
// result = Locations.MapReduce(Locationsfunction, reducefunction, options);
result = Demand.MapReduce(Demandfunction, reducefunction, options);
// result = Demand.MapReduce(Demandfunction2, reducefunction, options);
/* resultado en json */
string results = result.GetResults().ToJson();
return(results);
}
catch (Exception ex)
{
return(null);
}
}
public void GetDetailOfMethod()
{
using (mServer.RequestStart(mLogs))
{
var servicePerformances = mLogs.GetCollection("service_performance");
var map = new BsonJavaScript(@"
function() {
var me = this;
var key = {};
key.day = me.createdDate.getDate();
key.month = me.createdDate.getMonth();
key.year = me.createdDate.getFullYear();
if(me.machineName === 'APPSIT01'){
if(me.class === 'DeviceService'){
if(me.method === 'GetDeployedGeoFenceConf'){
emit(key, { count: 1, totalElapsedMilliseconds: me.elapsedMilliseconds });
}
}
}
}");
var reduce = new BsonJavaScript(@"
function(key, values) {
var result = { count: 0, totalElapsedMilliseconds: 0.0 };
values.forEach(function(value){
result.count += value.count;
result.totalElapsedMilliseconds += value.totalElapsedMilliseconds;
});
return result;
}");
var finalize = new BsonJavaScript(@"
function(key, value){
value.averageElapsedMilliseconds = value.totalElapsedMilliseconds / value.count;
return value;
}");
var options = new MapReduceOptionsBuilder();
options.SetFinalize(finalize);
options.SetOutput(MapReduceOutput.Inline);
var results = servicePerformances.MapReduce(map, reduce, options);
foreach (var result in results.GetResults())
{
var doc = result.ToBsonDocument();
var id = doc["_id"].AsBsonDocument;
var value = doc["value"].AsBsonDocument;
var count = Convert.ToInt32(value["count"]);
var averageElapsedMilliseconds = Convert.ToDouble(value["averageElapsedMilliseconds"]);
var totalElapsedMilliseconds = Convert.ToInt32(value["totalElapsedMilliseconds"]);
var day = Convert.ToInt32(id["day"]);
var month = Convert.ToInt32(id["month"]);
var year = Convert.ToInt32(id["year"]);
string toDay = string.Format("{0}/{1}/{2}", day, month, year);
Console.WriteLine(
string.Format("{0} - {1} - {2} - {3}",
toDay, count, totalElapsedMilliseconds, averageElapsedMilliseconds));
}
var query = Query.And(
Query.EQ("machineName", "APPSIT01"),
Query.EQ("class", "DeviceService"),
Query.EQ("method", "GetDeviceByImei")
);
var all = servicePerformances.Find(query).AsEnumerable();
int totalEMs = 0;
foreach (var result in all)
{
var doc = result.ToBsonDocument();
var elapsedMilliseconds = Convert.ToInt32(doc[4].RawValue);
totalEMs += elapsedMilliseconds;
}
Console.WriteLine(totalEMs);
}
}
