static void Main(string[] args)
{
AssemblyBuilder _assemblyBuilder = AppDomain.CurrentDomain.DefineDynamicAssembly(new AssemblyName("MyBuilder"), AssemblyBuilderAccess.Run);
ModuleBuilder _moduleBuilder = _assemblyBuilder.DefineDynamicModule("MyModule");
var typeBuilder = _moduleBuilder.DefineType("lulz43", TypeAttributes.Class | TypeAttributes.Public);
var fieldBuilder = typeBuilder.DefineField("zomglist", typeof(List <long>), FieldAttributes.Public);
var cctor = typeBuilder.DefineConstructor(MethodAttributes.Public, CallingConventions.Standard | CallingConventions.HasThis, new Type[0]);
var ilGenerator = cctor.GetILGenerator();
MakeCtor(new FieldBuilder[] { fieldBuilder }, ilGenerator);
var type32 = typeBuilder.CreateType();
//var lolzfda = Activator.CreateInstance(type32);
var param = Expression.Parameter(typeof(Mention), "x");
var tags = Expression.MakeMemberAccess(param, typeof(Mention).GetField("Tags"));
var ds = Expression.MakeMemberAccess(param, typeof(Mention).GetField("Datasources"));
var a = new List <int> {
1, 2
};
var b = new List <int> {
3, 4
};
var c = new List <int> {
5, 6
};
var joinedlistsexpressions = UnionEnumerableExpressions(new Expression[] { tags, ds, tags, ds });
var joinedlists = ToArrayCollection(UnionEnumerables(new object[] { a, b, c }), typeof(int));
//var lol = Convert.ChangeType("5", typeof(string));
var listlol = DataHelper.GetMentions(10000);
//var sqlexpr = new EvoSQLExpression("SELECT Type, Color, Count() FROM Cats GROUP BY Color, Fur");
//Console.WriteLine(sqlexpr.Tree.ToString());
var tzs = TimeZoneInfo.GetSystemTimeZones();
var query22 = "SELECT Type FROM (SELECT Type GROUP BY Type)";
var query23 = "SELECT Sentiment FROM (SELECT Sentiment, Id FROM Mentions)";
var query24 = @"
SELECT SUM({fn CONVERT(1, SQL_BIGINT)}) AS ""sum_Number_of_Records_qk""
,
{
fn TIMESTAMPADD
(
SQL_TSI_MONTH,
CAST
(
{fn TRUNCATE((3 * (CAST({fn TRUNCATE({fn QUARTER(""Mentions"".""Date"")},0)} AS INTEGER) - 1)),0)} AS INTEGER),
{fn TIMESTAMPADD(SQL_TSI_DAY,CAST({fn TRUNCATE(-({fn DAYOFYEAR(""Mentions"".""Date"")} - 1),0)} AS INTEGER), CAST(""Mentions"".""Date"" AS DATE))}
)
} AS ""tqr_Date_qk""
FROM ""Bermuda"".""Mentions"" ""Mentions""
GROUP BY ""tqr_Date_qk""";
var query24mini = @" SELECT
SUM({fn CONVERT(1, SQL_BIGINT)}) AS ""sum_Number_of_Records_qk"",
{ fn TIMESTAMPADD( SQL_TSI_MONTH, CAST({fn TRUNCATE(3,3)} as sql_bigint), Date ) } AS ""tqr_Date_qk""
FROM ""Bermuda"".""Mentions"" ""Mentions""
GROUP BY ""tqr_Date_qk""";
var reduceexpr1 = EvoQLBuilder.GetReduceExpression("SELECT Average(NulNum), * GROUP BY IsDisabled, Interval(OccurredOn, \"Year\", 0)", typeof(MentionTest2), true);
var reduceexpr11 = EvoQLBuilder.GetReduceExpression("SELECT Count() WHERE IsDisabled NulNum>5 Tags:5", typeof(MentionTest2), true);
var sellol52 = EvoQLBuilder.GetReduceExpression("SELECT Count() as Value, * GROUP BY TOP 5 Tags as Id VIA Count(), Interval(Date,Day,-5) as Id2", typeof(Mention), true);
//var whereexpr2 = EvoQLBuilder.GetWhereExpression("GET Activity WHERE Whateverz:50 DONCAER:OK jkdsadasjd DATE>\"2012/06/04 16:19:58\"", typeof(Mention));
var sellol3 = EvoQLBuilder.GetReduceExpression(query24, typeof(Mention), true);
var sellol4 = EvoQLBuilder.GetMergeExpression(query24, typeof(Mention), null);
//var sellol5 = EvoQLBuilder.GetReduceExpression("SELECT SUM({fn CONVERT(CONVERT(Sentiment, SQL_BIGINT), SQL_BIGINT)}) \"sum_Number_of_Records_qk\" FROM Nulz", typeof(Mention), true);
var sellol7 = EvoQLBuilder.GetReduceExpression("SELECT -Sentiment+Influence as Ten", typeof(Mention), true);
var sellol6 = EvoQLBuilder.GetReduceExpression("SELECT Sentiment+Influence as Ten, -Interval(Date, Day,-5) as DLOL , -Sentiment as neg, cat cat, Sentiment as S FROM Lulzcol Lulzforsur", typeof(Mention), true);
var sellol = EvoQLBuilder.GetReduceExpression("SELECT \"Mentions\".\"Name\", Sentiment, {fn DatePart(Date,Hour,-5)} as Hour FROM Whatever as Lol", typeof(Mention), true);
var kdsadsa = EvoQLBuilder.GetCollections(null, "SELECT Sentiment FROM Mentions metn", null, null);
//var whereexpr = EvoQLBuilder.GetWhereExpression("Themes:lol NOT:(lol or hi or bye) (Tag:((@52 OR @75) AND NOT:(@53)) ) Tags:5 DATE:-13.551235..-12.551235 (Sentiment:(-100..100 OR -100..100) ) ", typeof(Mention));
var mergeexpr = EvoQLBuilder.GetMergeExpression("SELECT Average(Sentiment) as C GROUP BY Interval(Date, Day, -5) as IntervalTheKilla FROM sigh", typeof(Mention), null);
var zomg = "SELECT 5+5 as lol FROM Mentions";
var red = EvoQLBuilder.GetReduceExpression(zomg, typeof(Mention), true);
var red2 = EvoQLBuilder.GetMergeExpression(zomg, typeof(Mention), null);
var cols = EvoQLBuilder.GetCollections("GET Mention DATE:\"2012/04/09 04:00:00\"..\"2012/04/10 04:00:00\" ( ( (Tag:@2 )))", "SELECT Sum(Followers) AS Count, * FROM Catz", null, null);
//var reduceexpr = EvoQLBuilder.GetReduceExpression("SELECT Count(*) as Value, Average(Sentiment) as Retard GROUP BY TOP 5 Type VIA Count(), OccurredOn as X INTERVAL Day", typeof(MentionTest));
//var reducestring = "SELECT Count() AS Value GROUP BY TOP 10 Tags VIA Count() AS Id, Datepart(OccurredOn, QuarterHour, -5) AS Id2";
var reducestring = "SELECT Sentiment FROM (SELECT Sentiment, Id FROM Mentions)";
var reduceexpr = EvoQLBuilder.GetReduceExpression(reducestring, typeof(MentionTest));
//var mergeexpr = EvoQLBuilder.GetMergeExpression("SELECT Average(Sentiment), Count() as Lulz GROUP BY Tags, OccurredOn INTERVAL Day WHERE Name:no", typeof(Mention), null);
var pagingexpr = EvoQLBuilder.GetPagingExpression("ORDERED BY Date LIMIT 5,25", typeof(Mention));
//var reduceexpr = EvoQLBuilder.GetReduceExpression("CHART Count BY TOP 11 Tags VIA Sentiment OVER Date INTERVAL Day WHERE cats TAG:@1");
//var reduceexpr = ReduceExpressionGeneration.MakeExpression
//(
// null,
// "Count",
// new ReduceDimension[]
// {
// new ReduceDimension{ GroupBy = "Type" },
// new ReduceDimension{ GroupBy = "OccurredOn", Interval = IntervalTypes.Day }
// }
//);
var mytype = LinqRuntimeTypeBuilder.GetDynamicType(new Dictionary <string, Type> {
{ "Count", typeof(long) }, { "Value", typeof(double) }, { "Zomg", typeof(string) }
});
var ctor = Activator.CreateInstance(mytype, false);
var type = LinqRuntimeTypeBuilder.GetTypeKey(listlol.GetType().GetGenericArguments().FirstOrDefault());
var json = LinqRuntimeTypeBuilder.SerializeObject(listlol.Take(5).ToList());
var res = LinqRuntimeTypeBuilder.DeserializeJson(json, type, true);
var type2 = LinqRuntimeTypeBuilder.GetTypeKey(ReduceExpressionGeneration.GetTypeOfEnumerable(res.GetType()));
var type_consistent = type == type2;
//var listlol = DataHelper.GetPremadeMentions();
var dsadsadas = listlol.GroupBy(x => x).Select(g => g.FirstOrDefault());
//Console.WriteLine(reduceexpr);
Console.WriteLine(mergeexpr);
//int serverCount = 20;
//int bucketCount = 131;
//ServerId == BucketId % ServerCount
//var parititions = Enumerable.Range(0, bucketCount).Split(serverCount);
//foreach (var s in parititions){ foreach (var b in s) Console.Write(b + "\t"); Console.WriteLine();}
//Console.WriteLine();
//parititions = Enumerable.Range(0, bucketCount).Split(serverCount + 3);
//foreach (var s in parititions) { foreach (var b in s) Console.Write(b + "\t"); Console.WriteLine(); }
//########################################################################
//## TEST REDUCE EXPRESSION
//########################################################################
//var poitsad = listlol.AsParallel().GroupBy(m => m.Type).Select(g => new EnumMetadata<IGrouping<string, MentionTest>>() { Enum = g }).Select(gmd => new { Type = gmd.Enum.Key, Type_Hash = (long)(gmd.Enum.Key.GetHashCode()), _Count = gmd.Enum.LongCount() }).ToArray();
var compileMethod = reduceexpr.GetType().GetMethods().FirstOrDefault(x => x.Name == "Compile" && x.GetParameters().Length == 0);
var mapreduceFunc = compileMethod.Invoke(reduceexpr, new object[0]);
var mapReduceFuncInvoke = mapreduceFunc.GetType().GetMethod("Invoke");
var pointenum = mapReduceFuncInvoke.Invoke(mapreduceFunc, new object[] { listlol });
var genericToArrayInfos = typeof(Enumerable).GetMethods().Where(x => x.Name == "ToArray" && x.IsGenericMethod && x.GetParameters().Length == 1);
var genericToArrayInfo = genericToArrayInfos.FirstOrDefault();
var enumType = pointenum.GetType();
var pointType = enumType.GetGenericArguments().Last();
var toArrayInfo = genericToArrayInfo.MakeGenericMethod(pointType);
Stopwatch sw2 = new Stopwatch();
sw2.Start();
var data = toArrayInfo.Invoke(null, new object[] { pointenum });
sw2.Stop();
var jsonz = new JavaScriptSerializer().Serialize(pointenum);
Console.WriteLine(sw2.Elapsed);
//var filter = EvoQLBuilder.GetFilterLambda("(b and a) or 23");
//var filtered = listlol.Where(filter.Compile());
//var result = filtered.ToArray();
Console.ReadKey();
//########################################################################
//## BENCHMARK
//########################################################################
#region playing with expressions
//var expr = EvoQLBuilder.GetReduceExpression("CHART Count BY Type OVER Date WHERE cat");
/*
* var validTags = new long[] { 1, 2 };
*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> reduce = collection =>
* from mention in collection
* from tag in mention.Tags
* from datasource in mention.Datasources
* group mention by new MentionGroup { Id = tag, Id2 = datasource } into g
* select new Datapoint { Id = g.Key.Id, Id2 = g.Key.Id2, Timestamp = g.Key.Timestamp };
*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Tag>, IEnumerable<Datapoint>>> reduce2 = (mentions, tags) =>
* from mention in mentions
* join tag in tags on mention.Id equals tag.MentionId
* group mention by new MentionGroup { Timestamp = mention.OccurredOnTicks, Id = tag.TagId } into g
* select new Datapoint { Id = g.Key.Id, Timestamp = g.Key.Timestamp };
*
* var dayTicks = TimeSpan.FromDays(1).Ticks;
*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> reduce34 = mentions =>
* from mention in mentions
* group mention by mention.OccurredOnTicks - mention.OccurredOnTicks % dayTicks into g
* select new Datapoint { Timestamp = g.Key };
*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> reduce33 = mentions =>
* from mention in mentions
* group mention by mention.OccurredOn.Date.Ticks into g
* select new Datapoint { Timestamp = g.Key };
*
*
*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> reduce3 = mentions =>
* from mention in mentions
* from tag in mention.Tags
* where validTags.Contains(tag)
* group mention by new MentionGroup { Timestamp = mention.OccurredOnTicks, Id = tag } into g
* select new Datapoint { Id = g.Key.Id, Timestamp = g.Key.Timestamp };
*
*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> mapreduce3 = collection =>
* collection
* .SelectMany(m => m.Tags, (m, t) => new MentionMetadata2 { Mention = m, Id = t })
* .SelectMany(md => md.Mention.Datasources, (md, ds) => new MentionMetadata2 { Child = md, Id = ds })
* .GroupBy(md => new MentionGroup { Timestamp = md.Child.Mention.OccurredOnTicks, Id = md.Child.Id, Id2 = md.Id })
* .Select(x => new Datapoint { Timestamp = x.Key.Timestamp, Id = x.Key.Id, Id2 = x.Key.Id2 });
*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> mapreduce4 = collection =>
* collection
* .SelectMany(m => m.Tags, (m, t) => new MentionMetadata2 { Mention = m, Id = t })
* .GroupBy(mt => mt.Id)
* .OrderByDescending(g => g.Count())
* .Take(5)
* .SelectMany(x => x)
* .GroupBy(md => new MentionGroup { Timestamp = md.Child.Mention.OccurredOnTicks, Id = md.Child.Id, Id2 = md.Id })
* .Select(x => new Datapoint { Timestamp = x.Key.Timestamp, Id = x.Key.Id, Id2 = x.Key.Id2 });
*
*
*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> mapreduce5 = collection =>
* collection
* .SelectMany(m => m.Tags, (m, t) => new MentionMetadata { Item = m, Id = t })
* .GroupBy(md => md.Id)
* .Select(g => new { Group = g, Value = g.Count() })
* .OrderByDescending(md => md.Value)
* .Take(5)
* .Select(x => new Datapoint());
*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> mapreduce9 = collection =>
* collection
* .AsParallel()
* .SelectMany(m => m.Themes, (m, t) => new MentionMetadata { Item = m, Id = t })
*
* .GroupBy(md => md.Id, md => md.Item)
* .Select(g => new GroupMetadata { Group = g, Value = g.Count() })
* .OrderByDescending(g => g.Value)
* .Take(5)
*
* .SelectMany
* (
* gmd =>
*
* gmd.Group.GroupBy(md => md.OccurredOnTicks)
* .Select(g => new GroupMetadata { Group = g })
* .Select(gmd2 => new Datapoint { Id = gmd.Group.Key, Id2 = gmd2.Group.Key, CountValue = gmd.Value })
* );
*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> mapreduce6 = collection =>
*
* collection
* .AsParallel()
* .SelectMany(m => m.Themes, (m, t) => new MentionMetadata { Item = m, Id = t })
* .GroupBy(md => md.Id, md => md.Item)
* .Select(g => new GroupMetadata { Group = g, Value = g.Count() })
* .OrderByDescending(g => g.Value)
* .Take(5)
*
* .SelectMany
* (
* gmd =>
*
* gmd.Group
* .SelectMany(m => m.Tags, (m, t) => new MentionMetadata { Item = m, Id = t })
* .GroupBy(md => md.Id, md => md.Item)
* .Select(g => new GroupMetadata { Group = g, Value = gmd.Value })
* .OrderByDescending(g => g.Value)
* .Take(2)
*
* .Select(gmd2 => new Datapoint { Id = gmd.Group.Key, Id2 = gmd2.Group.Key, CountValue = gmd2.Value })
*
*
* );
*/
//ExpressionSerializer serializer = new ExpressionSerializer();
//var xml = serializer.Serialize(mapreduce6);
//Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> dsajkdsa = collection => collection.SelectMany(m => m.Ngrams, (m, t) => new MentionMetadata() { Mention = m, Id = t }).GroupBy(md => new MentionGroup() { Timestamp = md.Mention.OccurredOnTicks, Id2 = md.Id }).Select(x => new Datapoint() { Timestamp = x.Key.Timestamp, Id2 = x.Key.Id2 });
/*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> mapreduce11 = collection =>
* collection
* .SelectMany(m => m.Themes, (m, t) => new MentionMetadata { Item = m, Id = t })
* .GroupBy(md => new MentionGroup { Timestamp = md.Item.OccurredOnTicks, Id2 = md.Id })
* .Select(x => new Datapoint { Timestamp = x.Key.Timestamp, Id2 = x.Key.Id2 });*/
#endregion
var exprs = new Expression <Func <IEnumerable <Mention>, IEnumerable <Datapoint> > >[] { };
var sw = new Stopwatch();
foreach (var expr in exprs)
{
Console.WriteLine(expr);
var func = expr.Compile();
sw.Restart();
for (int i = 0; i < 100; i++)
{
//var dsadqw = func(listlol).ToArray();
}
sw.Stop();
Console.WriteLine(sw.Elapsed);
}
Console.ReadKey();
/*
* Console.WriteLine("WAITING");
*
*
* //http://localhost:13866/Connection.External/ExternalService.svc
*
* var client = GetServiceClient();
*
*
* Random rng = new Random();
*
* while (true)
* {
* char randomChar = (char)(rng.Next(0, 10) + '0');
*
* //ParameterExpression param = Expression.Parameter(typeof(ThriftMention));
*
* object[] parameters = new object[] { randomChar };
*
* Expression<Func<ThriftMention, object[], bool>> query = (x, p) => x.Description.Contains((char)p[0]);
*
* Expression<Func<IEnumerable<ThriftMention>, IEnumerable<ThriftDatapoint>>> mapreduce = x => x.SelectMany(y => y.Tags).GroupBy(y => y).Select(y => new ThriftDatapoint { Count = y.Count(), Value = y.Count(), EntityId = y.Key });
*
* Expression<Func<IEnumerable<ThriftDatapoint>, double>> merge = x => x.Sum(y => y.Value);
*
* var domain = "evoapp";
* var minDate = DateTime.Today.AddDays(-600);
* var maxDate = DateTime.Today.AddDays(1);
*
* Stopwatch sw = new Stopwatch();
* sw.Start();
* var datapoints = client.GetDatapointList(domain, query, mapreduce, merge, minDate, maxDate, parameters);
* sw.Stop();
*
* Console.WriteLine("QUERY: " + randomChar);
* Console.WriteLine(sw.Elapsed.ToString());
*
* Console.WriteLine(string.Join(" ", datapoints.Select(x => "tag" + x.EntityId + ": " + x.Value + " (" + x.Count + ")")));
* Console.WriteLine(string.Empty);
*
* Thread.Sleep(1000);
* }
*/
}
public BermudaResult GetData(string domain, string query, string mapreduce, string merge, string paging, int remdepth, string command, string cursor, string paging2)
{
var args = ParseCommand(command);
bool noCache = args.Contains("-nocache");
bool makeCursor = cursor == MakeCursorToken;
bool useCursor = !makeCursor && !string.IsNullOrWhiteSpace(cursor);
DateTime minDate = DateTime.MinValue;
DateTime maxDate = DateTime.MaxValue;
if (remdepth > 0)
{
//map
var queryHash = cursor ?? GetQueryHash(domain, query, mapreduce, merge, paging, null);
//reduce
BermudaResult cachedDatapoints;
if (!noCache && CachedData.TryGetValue(queryHash, out cachedDatapoints) && (DateTime.Now.Ticks - cachedDatapoints.CreatedOn) < CacheLifetime)
{
#if DEBUG
if (CacheTraceMessageLevel < 3)
{
Trace.WriteLine("returned CACHED BLOBS DATAPOINTS results FOR ENTIRE BLOB SET [REMDEPTH:" + remdepth + "]");
}
#endif
if (useCursor)
{
var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(cachedDatapoints.DataType);
return(GetCursorData(paging2, cachedDatapoints, dataType));
}
else
{
return(new BermudaResult {
DataType = cachedDatapoints.DataType, Data = cachedDatapoints.Data, Metadata = new BermudaNodeStatistic {
Notes = "Cache_Hit_1"
}, CacheKey = cachedDatapoints.CacheKey
});
}
}
else
{
if (useCursor)
{
throw new Exception("Cursor " + cursor + " not found");
}
//var assignments = PartitionBlobs(domain, blobInterfaces, minDate, maxDate, false, true);
var reducers = HostEnvironment.Instance.GetAvailablePeerConnections();
if (!reducers.Any())
{
throw new Exception("Specified dataset not loaded: " + domain);
}
ConcurrentDictionary <PeerInfo, BermudaResult> results = new ConcurrentDictionary <PeerInfo, BermudaResult>();
Stopwatch sw = new Stopwatch();
sw.Start();
List <Task> tasks = new List <Task>();
foreach (var reducer in reducers)
{
Task t = new Task((peerObj) =>
{
var peerInfo = peerObj as PeerInfo;
var initiated = DateTime.Now;
var subqueryHash = GetQueryHash(domain, query, mapreduce, merge, paging, peerInfo.ToString());
Stopwatch sw3 = new Stopwatch();
sw3.Start();
//see if the cache contains a matching result and return it if it's not outdated
BermudaResult cachedDatapoints2;
if (!noCache && CachedData.TryGetValue(subqueryHash, out cachedDatapoints2) && (DateTime.Now.Ticks - cachedDatapoints2.CreatedOn) < CacheLifetime)
{
if (CacheTraceMessageLevel < 2)
{
Trace.WriteLine("returned CACHED BLOB DATAPOINT results FOR BLOB SUBSET [REMDEPTH:" + remdepth + "]");
}
BermudaResult res = null;
if (useCursor)
{
var dataType2 = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(cachedDatapoints2.DataType);
res = GetCursorData(paging2, cachedDatapoints2, dataType2);
}
else
{
res = new BermudaResult {
DataType = cachedDatapoints2.DataType, Data = cachedDatapoints2.Data, Metadata = new BermudaNodeStatistic {
Notes = "Cache_Hit_2"
}
};
}
results[peerInfo] = res;
}
else
{
try
{
Stopwatch sw2 = new Stopwatch();
sw2.Start();
BermudaResult subresult = null;
if (peerInfo.Equals(Endpoint))
{
subresult = GetData(domain, query, mapreduce, merge, paging, remdepth - 1, command, cursor, paging2);
}
else
{
using (var client = HostEnvironment.GetServiceClient(peerInfo))
{
subresult = client.GetData(domain, query, mapreduce, merge, paging, remdepth - 1, command, cursor, paging2);
}
//subresult = GetDataFromPeer(domain, query, mapreduce, merge, minDate, maxDate, remdepth - 1, command, assignment.PeerEndpoint.Endpoint);
}
sw2.Stop();
subresult.CreatedOn = DateTime.Now.Ticks;
subresult.Metadata.Initiated = initiated;
subresult.Metadata.Completed = DateTime.Now;
subresult.Metadata.OperationTime = sw2.Elapsed;
results[peerInfo] = CachedData[subqueryHash] = subresult;
}
catch (Exception ex)
{
results[peerInfo] = new BermudaResult {
Error = "[Failed Node] " + ex
};
}
}
}, reducer, TaskCreationOptions.LongRunning);
tasks.Add(t);
t.Start();
}
Task.WaitAll(tasks.ToArray());
sw.Stop();
#if DEBUG
Trace.WriteLine("Join Time:" + sw.Elapsed);
#endif
if (results.Any(x => x.Value.Error != null))
{
throw new BermudaException("Some nodes failed:\r\n" + string.Join("\r\n", results.Select(x => x.Value.Error)));
}
if (results.All(x => x.Value.Data == null))
{
return new BermudaResult {
Metadata = new BermudaNodeStatistic {
Notes = "No Data"
}
}
}
;
//if all results are not the same time throw an error
if (results.GroupBy(x => x.Value.DataType).Count() > 1)
{
throw new BermudaException("Subresults must all return the same type");
}
var dataTypeDescriptor = results.Select(x => x.Value.DataType).FirstOrDefault(x => x != null);
if (dataTypeDescriptor == null)
{
return new BermudaResult {
Error = "Could not determine the merge type, none of the nodes provided type info"
}
}
;
//use the passed combine espression to make multiple datapoint sets into one
var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(dataTypeDescriptor);
//allItems = results.Values.SelectMany(x => x.DataObject)
var totalJson = "[" + string.Join(",", results.Values.Where(x => !string.IsNullOrWhiteSpace(x.Data)).Select(x => x.Data.Trim('[', ']')).Where(x => !string.IsNullOrWhiteSpace(x))) + "]";
var allItems = LinqRuntimeTypeBuilder.DeserializeJson(totalJson, dataTypeDescriptor, true);
//var aaa = new JavaScriptSerializer().Deserialize<Datapoint[]>(totalJson);
//var ggc = aaa.GroupBy(x => new { x.Id, x.Id2 }).Count();
//InvokeSelectManyViaReflectionTheKilla(results.Values.Select(x => x.DataObject), dataType);
var mergeFunc = GetMergeFunc(merge, mapreduce, dataType, dataType);
if (mergeFunc != null)
{
//var dataType = "kdsajkdsa";
var mergeInvokeMethod = mergeFunc.GetType().GetMethod("Invoke");
allItems = mergeInvokeMethod.Invoke(mergeFunc, new object[] { allItems }); // MergeDatapoints(results.Values.Where(x => x.Data != null).SelectMany(x => x.Data), mergeFunc);
}
var pagingFunc = GetPagingFunc(paging, dataType);
if (pagingFunc != null)
{
var pagingInvokeMethod = pagingFunc.GetType().GetMethod("Invoke");
allItems = pagingInvokeMethod.Invoke(pagingFunc, new object[] { allItems });
}
//figure out the metadata
var finalMetadata = new BermudaNodeStatistic {
Notes = "Merged Datapoints in " + sw.Elapsed, NodeId = HostEnvironment.Instance.CurrentInstanceId, ChildNodes = results.Values.Select(x => x.Metadata).ToArray()
};
var arraylol = ToArrayCollection(allItems, dataType);
var json = JsonConvert.SerializeObject(arraylol);
//var json = JsonConvert.SerializeObject(allItems);
var originalData = makeCursor ? arraylol : null;
var finalResult = new BermudaResult {
DataType = dataTypeDescriptor, OriginalData = originalData, Data = json, CreatedOn = DateTime.Now.Ticks, Metadata = finalMetadata, CacheKey = queryHash
};
CachedData[queryHash] = finalResult;
return(finalResult);
}
}
else
{
ConcurrentDictionary <string, BermudaResult> results = new ConcurrentDictionary <string, BermudaResult>();
BermudaNodeStatistic stats = new BermudaNodeStatistic();
var bucketInterfaces = HostEnvironment.Instance.GetBucketInterfacesForDomain(domain);
if (!bucketInterfaces.Any())
{
throw new BermudaException("Data not loaded for: " + domain);
}
if (bucketInterfaces.Count() > 1)
{
throw new BermudaException("Multiple buckets not supported by BermudaMapReduce");
}
var queryHash = GetQueryHash(domain, query, mapreduce, merge, paging, Endpoint.ToString());
BermudaResult cachedDatapoints;
if (!noCache && CachedData.TryGetValue(queryHash, out cachedDatapoints) && (DateTime.Now.Ticks - cachedDatapoints.CreatedOn) < CacheLifetime)
{
if (CacheTraceMessageLevel < 2)
{
Trace.WriteLine("returned CACHED BLOB SET DATAPOINT results [REMDEPTH:" + remdepth + "]");
}
if (useCursor)
{
var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(cachedDatapoints.DataType);
return(GetCursorData(paging2, cachedDatapoints, dataType));
}
else
{
return(new BermudaResult {
DataType = cachedDatapoints.DataType, Data = cachedDatapoints.Data, Metadata = new BermudaNodeStatistic {
Notes = "Cache_Hit_3"
}, CacheKey = queryHash
});
}
}
else
{
//Chad: short circuiting to test WCF response time in Azure
//return new DatapointResult() { Datapoints = new List<Datapoint>(), CreatedOn = DateTime.Now.Ticks, Metadata = new BermudaNodeStatistic() };
//IEnumerable<Datapoint> datapoints = null;
object datapoints = null;
Stopwatch sw = new Stopwatch();
sw.Start();
Type itemType = null;
Type resultType = null;
string json = null;
foreach (var bucketInterface in bucketInterfaces)
{
var bucketKey = GetQueryHash(domain, query, mapreduce, merge, paging, Endpoint.ToString());
//see if the cache contains a matching result and return it if it's not outdated
BermudaResult cachedDatapoints2;
if (!noCache && CachedData.TryGetValue(bucketKey, out cachedDatapoints2) && (DateTime.Now.Ticks - cachedDatapoints2.CreatedOn) < CacheLifetime)
{
if (CacheTraceMessageLevel < 1)
{
Trace.WriteLine("returned CACHED BLOB DATAPOINT results [REMDEPTH:" + remdepth + "]");
}
if (useCursor)
{
if (cachedDatapoints2.OriginalData == null)
{
throw new Exception("Cursor " + cursor + " contains null data");
}
var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(cachedDatapoints2.DataType);
results[bucketInterface.Name] = GetCursorData(paging2, cachedDatapoints2, dataType);
}
else
{
results[bucketInterface.Name] = new BermudaResult {
DataType = cachedDatapoints2.DataType, Data = cachedDatapoints2.Data, Metadata = new BermudaNodeStatistic {
Notes = "Cache_Hit_4"
}
};
json = cachedDatapoints2.Data;
}
}
else
{
//get mentions
var collections = GetCollections(query, mapreduce);
if (collections.Count() > 1)
{
throw new BermudaException("More than one collection specified: " + string.Join(",", collections));
}
var table = collections.FirstOrDefault();
var tableName = table == null ? null : table.Source;
var raw = bucketInterface.GetData(tableName);
//var rawType = raw.GetType();
//itemType = ReduceExpressionGeneration.GetTypeOfEnumerable(rawType);
itemType = bucketInterface.GetDataType(tableName);
var mapreduceFunc = GetMapReduceFunc(mapreduce, itemType, out resultType);
var queryFunc = GetFilterFunc(query, itemType);
var pagingFunc = GetPagingFunc(paging, resultType);
var minDateTicks = minDate.Ticks;
var maxDateTicks = maxDate.Ticks;
object subresult = raw;
//queryFunc == null ?
// raw.AsParallel() :
//minDate == DateTime.MinValue && maxDate == DateTime.MaxValue ?
// raw.AsParallel().Where(x => queryFunc) :
// raw.AsParallel().Where(x => x.OccurredOnTicks >= minDateTicks && x.OccurredOnTicks <= maxDateTicks && queryFunc(x, parameters));
if (json == null)
{
if (queryFunc != null)
{
var queryFuncInvoke = queryFunc.GetType().GetMethod("Invoke");
subresult = queryFuncInvoke.Invoke(queryFunc, new object[] { subresult });
}
//reduce them using the passed expression
if (mapreduceFunc != null)
{
var mapReduceFuncInvoke = mapreduceFunc.GetType().GetMethod("Invoke");
subresult = mapReduceFuncInvoke.Invoke(mapreduceFunc, new object[] { subresult });
}
if (pagingFunc != null)
{
var pagingInvokeMethod = pagingFunc.GetType().GetMethod("Invoke");
subresult = pagingInvokeMethod.Invoke(pagingFunc, new object[] { subresult });
}
datapoints = subresult;
}
//format a metada string
if (!args.Contains("-nocount"))
{
stats.TotalItems = bucketInterface.GetCount(tableName);
//stats.FilteredItems = filtered.Count();
//stats.ReducedItems = subresult.Count();
}
//cache the result
//results[blobInterface.Name] = new DatapointResult { Datapoints = subresult, CreatedOn = DateTime.UtcNow.Ticks, Metadata = stats.Serialize() };
//CachedDatapoints[blobKey] = new DatapointResult { Datapoints = subresult.ToList(), CreatedOn = DateTime.UtcNow.Ticks, Metadata = stats.Serialize() };
}
}
//figure out the metadata
//var finalMetadata = " [@" + AzureInterface.Instance.CurrentInstanceId + "] Calculated Datapoints:\r\n" + string.Join("\r\n", results.Values.Select(x => x.Metadata));
stats.NodeId = HostEnvironment.Instance.CurrentInstanceId;
stats.Notes = "Computed Datapoints";
//Trace.WriteLine("total mentions processed: " + mentionCount);
//var datapoints = results.Values.SelectMany(x => x.Datapoints);
if (datapoints == null)
{
return new BermudaResult()
{
Metadata = new BermudaNodeStatistic {
Notes = "No Results"
}
}
}
;
//foreach (var p in datapoints) if (p.IsCount) p.Value = p.Count;
var mergeFunc = GetMergeFunc(merge, mapreduce, itemType, resultType);
if (mergeFunc != null)
{
var mergeFuncInvoke = mergeFunc.GetType().GetMethod("Invoke");
datapoints = mergeFuncInvoke.Invoke(mergeFunc, new object[] { datapoints });
}
stats.LinqExecutionTime = sw.Elapsed;
var arraylol = ToArrayCollection(datapoints, resultType);
if (json == null && datapoints != null)
{
json = JsonConvert.SerializeObject(arraylol);
}
//var json = JsonConvert.SerializeObject(datapoints);
var originalData = makeCursor ? arraylol : null;
var result = CachedData[queryHash] = new BermudaResult {
DataType = LinqRuntimeTypeBuilder.GetTypeKey(resultType), OriginalData = originalData, Data = json, CreatedOn = DateTime.Now.Ticks, Metadata = stats
};
sw.Stop();
return(result);
}
}
}