public EntityPrimaryKeyMap()
: base()
{
object entityconfig = new TC();
object config = _configurationPropertyInfo.GetValue(entityconfig);
var tableName = (string)_tableNamePropertyInfo.GetValue(config);
string schema = (string)_schemaNamePropertyInfo.GetValue(config);
this.ToTable(tableName, schema);
IEnumerable <PropertyInfo> keys = (IEnumerable <PropertyInfo>)_keyPropertiesPropertyInfo.GetValue(config);
Type dynamicType = LinqRuntimeTypeBuilder.GetDynamicType(keys);
var paramEx = Expression.Parameter(typeof(T), "m");
ConstructorInfo ci = dynamicType.GetConstructor(keys.Select(k => k.PropertyType).ToArray());
NewExpression nexp = Expression.New(ci, keys.Select(k => Expression.Property(paramEx, k)), dynamicType.GetFields());
Expression selector = Expression.Lambda(nexp, paramEx);
_hasKeyMethodInfo.MakeGenericMethod(dynamicType).Invoke(this, new object[] { selector });
foreach (var otherprop in typeof(T).GetProperties().Where(p => !keys.Any(k => k.Name == p.Name)))
{
Expression ignoreSelector = Expression.Lambda(Expression.Property(paramEx, otherprop), paramEx);
_ignoreMethodInfo.MakeGenericMethod(otherprop.PropertyType).Invoke(this, new object[] { ignoreSelector });
}
}
//public static IQueryable SelectDynamic(this IQueryable source, string select)
//{
// JavaScriptSerializer js = new JavaScriptSerializer();
// var d = (Dictionary<string,string>) js.Deserialize(select, typeof(Dictionary<string, string>));
// return SelectDynamic(source, (Dictionary<string, string>)d);
//}
public static IQueryable SelectDynamic(this IQueryable source, Dictionary <string, string> fieldNames)
{
Dictionary <string, PropertyInfo> sourceProperties = fieldNames.ToDictionary(name => name.Key, name => source.ElementType.GetDynamicProperty(string.IsNullOrWhiteSpace(name.Value) ? name.Key : name.Value));
ParameterExpression sourceItem = Expression.Parameter(source.ElementType, "t");
LinqFields fields = new LinqFields();
foreach (var item in fieldNames.OrderBy(x => x.Key))
{
LinqField f = new LinqField {
};
f.Name = item.Key;
f.PropertyPath = string.IsNullOrWhiteSpace(item.Value) ? item.Key : item.Value;
f.Property = source.ElementType.GetDynamicProperty(f.PropertyPath);
f.Type = f.Property.PropertyType;
f.Expression = LinqRuntimeTypeBuilder.NestedProperty(sourceItem, source.ElementType, f.PropertyPath);
fields.Add(f);
}
Type dynamicType = LinqRuntimeTypeBuilder.GetDynamicType(fields);
Expression selector = Expression.Lambda(Expression.MemberInit(Expression.New(dynamicType),
fields.Select(x => Expression.Bind(dynamicType.GetField("_" + x.Name), x.Expression)).ToArray()), sourceItem);
return(source.Provider.CreateQuery(Expression.Call(typeof(Queryable), "Select", new Type[] { source.ElementType, dynamicType },
Expression.Constant(source), selector)));
}
public void Main()
{
// This expression creates a new TestMemberInitClass object
// and assigns 10 to its sample property.
var sourceProperties = new Dictionary <string, Type>()
{
{ "PTestEntity", typeof(TestInput) }
};
Type dynamicType = LinqRuntimeTypeBuilder.GetDynamicType(sourceProperties);
// var pExpresion = Expression.Parameter(typeof(TestInput), "i");
var paramerExpression = sourceProperties.Select(t => Expression.Parameter(t.Value, t.Key));
var memberInitExpression = paramerExpression.Select(t => Expression.Bind(dynamicType.GetMember(t.Name).FirstOrDefault(), t));
Expression testExpr = Expression.MemberInit(
Expression.New(dynamicType),
memberInitExpression
);
// The following statement first creates an expression tree,
// then compiles it, and then runs it.
var testLambda = Expression.Lambda <Func <TestInput, dynamic> >(testExpr, paramerExpression);
var test = testLambda.Compile()(new TestInput()
{
sample = 12
});
var a = test.sample.sample;
Console.WriteLine(test.sample);
}
private static void AddFieldWithIdArgumentIfExists <TContextType>(MappedSchemaProvider <TContextType> schema, Type contextType, Field fieldProp)
{
if (!fieldProp.Resolve.Type.IsEnumerableOrArray())
{
return;
}
var schemaType = schema.Type(fieldProp.ReturnTypeClrSingle);
// Find the first field named "id" or "<fieldProp.Name>Id" to turn into a field with arguments
var idFieldDef = schemaType.GetFields().FirstOrDefault(f => f.Name.ToLower() == "id" || f.Name.ToLower() == $"{fieldProp.Name.ToLower()}id");
if (idFieldDef == null)
{
return;
}
// Save a little bit of typing and clean things up.
var idFieldName = idFieldDef.Name;
// We need to build an anonymous type with id = RequiredField<idFieldDef.Resolve.Type>()
// Resulting lambda is (a, p) => a.Where(b => b.Id == p.Id).First()
// This allows us to "insert" .Select() (and .Include()) before the .First()
var requiredFieldType = typeof(RequiredField <>).MakeGenericType(idFieldDef.Resolve.Type);
var fieldNameAndType = new Dictionary <string, Type> {
{ idFieldName, requiredFieldType }
};
var argTypes = LinqRuntimeTypeBuilder.GetDynamicType(fieldNameAndType);
var argTypesValue = argTypes.GetTypeInfo().GetConstructors()[0].Invoke(new Type[0]);
var argTypeParam = Expression.Parameter(argTypes);
Type arrayContextType = schema.Type(fieldProp.ReturnTypeClrSingle).ContextType;
var arrayContextParam = Expression.Parameter(arrayContextType);
var ctxId = Expression.PropertyOrField(arrayContextParam, $"{char.ToUpper(idFieldName[0])}{idFieldName.Substring(1)}");
Expression argId = Expression.PropertyOrField(argTypeParam, idFieldName);
argId = Expression.Property(argId, "Value"); // call RequiredField<>.Value to get the real type without a convert
var idBody = Expression.MakeBinary(ExpressionType.Equal, ctxId, argId);
var idLambda = Expression.Lambda(idBody, new[] { arrayContextParam });
Expression body = ExpressionUtil.MakeExpressionCall(new[] { typeof(Queryable), typeof(Enumerable) }, "Where", new Type[] { arrayContextType }, fieldProp.Resolve, idLambda);
body = ExpressionUtil.MakeExpressionCall(new[] { typeof(Queryable), typeof(Enumerable) }, "FirstOrDefault", new Type[] { arrayContextType }, body);
var contextParam = Expression.Parameter(contextType);
var lambdaParams = new[] { contextParam, argTypeParam };
body = new ParameterReplacer().ReplaceByType(body, contextType, contextParam);
var selectionExpression = Expression.Lambda(body, lambdaParams);
var name = fieldProp.Name.Singularize();
if (name == null)
{
// If we can't singularize it just use the name plus something as GraphQL doesn't support field overloads
name = $"{fieldProp.Name}";
}
var field = new Field(name, selectionExpression, $"Return a {fieldProp.ReturnTypeClrSingle} by its Id", fieldProp.ReturnTypeClrSingle, argTypesValue);
schema.AddField(field);
}
public static IQueryable SelectDynamic(this IQueryable source, params string[] fieldNames)
{
Dictionary <string, PropertyInfo> sourceProperties = fieldNames.ToDictionary(name => name, name => source.ElementType.GetProperty(name));
Type dynamicType = LinqRuntimeTypeBuilder.GetDynamicType(sourceProperties.Values);
ParameterExpression sourceItem = Expression.Parameter(source.ElementType, "t");
IEnumerable <MemberBinding> bindings = dynamicType.GetFields().Select(p => Expression.Bind(p, Expression.Property(sourceItem, sourceProperties[p.Name]))).OfType <MemberBinding>();
Expression selector = Expression.Lambda(Expression.MemberInit(Expression.New(dynamicType.GetConstructor(Type.EmptyTypes)), bindings), sourceItem);
return(source.Provider.CreateQuery(Expression.Call(typeof(Queryable), "Select", new Type[] { source.ElementType, dynamicType }, Expression.Constant(source), selector)));
}
public static Expression CreateNewExpression(Expression currentContext, IEnumerable <DataApiNode> fieldExpressions, ISchemaProvider schemaProvider, out Type dynamicType)
{
var fieldExpressionsByName = fieldExpressions.ToDictionary(f => f.Name, f => f.Expression);
dynamicType = LinqRuntimeTypeBuilder.GetDynamicType(fieldExpressions.ToDictionary(f => f.Name, f => f.Expression.Type));
var bindings = dynamicType.GetFields().Select(p => Expression.Bind(p, fieldExpressionsByName[p.Name])).OfType <MemberBinding>();
var newExp = Expression.New(dynamicType.GetConstructor(Type.EmptyTypes));
var mi = Expression.MemberInit(newExp, bindings);
return(mi);
}
private static void AddFieldWithIdArgumentIfExists <TContextType>(SchemaProvider <TContextType> schema, Type contextType, Field fieldProp)
{
if (!fieldProp.Resolve.Type.IsEnumerableOrArray())
{
return;
}
var schemaType = fieldProp.ReturnType.SchemaType;
var idFieldDef = schemaType.GetFields().FirstOrDefault(f => f.Name == "id");
if (idFieldDef == null)
{
return;
}
// We need to build an anonymous type with id = RequiredField<idFieldDef.Resolve.Type>()
// Resulting lambda is (a, p) => a.Where(b => b.Id == p.Id).First()
// This allows us to "insert" .Select() (and .Include()) before the .First()
var requiredFieldType = typeof(RequiredField <>).MakeGenericType(idFieldDef.Resolve.Type);
var fieldNameAndType = new Dictionary <string, Type> {
{ "id", requiredFieldType }
};
var argTypes = LinqRuntimeTypeBuilder.GetDynamicType(fieldNameAndType);
var argTypesValue = argTypes.GetTypeInfo().GetConstructors()[0].Invoke(new Type[0]);
var argTypeParam = Expression.Parameter(argTypes, $"args_{argTypes.Name}");
Type arrayContextType = schemaType.ContextType;
var arrayContextParam = Expression.Parameter(arrayContextType, $"arrcxt_{arrayContextType.Name}");
var ctxId = Expression.PropertyOrField(arrayContextParam, "Id");
Expression argId = Expression.PropertyOrField(argTypeParam, "id");
argId = Expression.Property(argId, "Value"); // call RequiredField<>.Value to get the real type without a convert
var idBody = Expression.MakeBinary(ExpressionType.Equal, ctxId, argId);
var idLambda = Expression.Lambda(idBody, new[] { arrayContextParam });
Expression body = ExpressionUtil.MakeCallOnQueryable("Where", new Type[] { arrayContextType }, fieldProp.Resolve, idLambda);
body = ExpressionUtil.MakeCallOnQueryable("FirstOrDefault", new Type[] { arrayContextType }, body);
var contextParam = Expression.Parameter(contextType, $"cxt_{contextType.Name}");
var lambdaParams = new[] { contextParam, argTypeParam };
body = new ParameterReplacer().ReplaceByType(body, contextType, contextParam);
var selectionExpression = Expression.Lambda(body, lambdaParams);
var name = fieldProp.Name.Singularize();
if (name == null)
{
// If we can't singularize it just use the name plus something as GraphQL doesn't support field overloads
name = $"{fieldProp.Name}ById";
}
var field = new Field(schema, name, selectionExpression, $"Return a {fieldProp.ReturnType.SchemaType.Name} by its Id", argTypesValue, new GqlTypeInfo(fieldProp.ReturnType.SchemaTypeGetter, selectionExpression.Body.Type), fieldProp.AuthorizeClaims);
schema.AddField(field);
}
private static void AddFieldWithIdArgumentIfExists <TContextType>(MappedSchemaProvider <TContextType> schema, Type contextType, Field fieldProp)
{
if (!fieldProp.Resolve.Type.IsEnumerable())
{
return;
}
var schemaType = schema.Type(fieldProp.ReturnSchemaType);
var idFieldDef = schemaType.GetFields().FirstOrDefault(f => f.Name == "Id");
if (idFieldDef == null)
{
return;
}
// We need to build an anonymous type with id = RequiredField<idFieldDef.Resolve.Type>()
// Resulting lambda is (a, p) => a.Where(b => b.Id == p.Id).First()
// This allows us to "insert" .Select() (and .Include()) before the .First()
var requiredFieldType = typeof(RequiredField <>).MakeGenericType(idFieldDef.Resolve.Type);
var fieldNameAndType = new Dictionary <string, Type> {
{ "id", requiredFieldType }
};
var argTypes = LinqRuntimeTypeBuilder.GetDynamicType(fieldNameAndType);
var argTypesValue = argTypes.GetTypeInfo().GetConstructors()[0].Invoke(new Type[0]);
var argTypeParam = Expression.Parameter(argTypes);
Type arrayContextType = schema.Type(fieldProp.ReturnSchemaType).ContextType;
var arrayContextParam = Expression.Parameter(arrayContextType);
var ctxId = Expression.PropertyOrField(arrayContextParam, "Id");
Expression argId = Expression.PropertyOrField(argTypeParam, "id");
argId = Expression.Property(argId, "Value"); // call RequiredField<>.Value to get the real type without a convert
var idBody = Expression.MakeBinary(ExpressionType.Equal, ctxId, argId);
var idLambda = Expression.Lambda(idBody, new[] { arrayContextParam });
Expression body = ExpressionUtil.MakeExpressionCall(new[] { typeof(Queryable), typeof(Enumerable) }, "Where", new Type[] { arrayContextType }, fieldProp.Resolve, idLambda);
body = ExpressionUtil.MakeExpressionCall(new[] { typeof(Queryable), typeof(Enumerable) }, "FirstOrDefault", new Type[] { arrayContextType }, body);
var contextParam = Expression.Parameter(contextType);
var lambdaParams = new[] { contextParam, argTypeParam };
body = new ParameterReplacer().ReplaceByType(body, contextType, contextParam);
var selectionExpression = Expression.Lambda(body, lambdaParams);
var field = new Field(fieldProp.Name.Singularize(), selectionExpression, $"Return {fieldProp.ReturnSchemaType} by Id", fieldProp.ReturnSchemaType, argTypesValue);
schema.AddField(field);
}
/// <summary>
/// Select properties according to property names from source.
/// </summary>
/// <remarks>
/// This method is copied from StackOverflow answer (http://stackoverflow.com/a/723018/1380428).
/// Alterations and additions made by Adil Mammadov.
/// </remarks>
/// <exception cref="ArgumentNullException">
/// When source or propertyNames is null.
/// </exception>
/// <exception cref="ArgumentOutOfRangeException">
/// When there is no item in propertyNames.
/// </exception>
/// <param name="source">Source to select from.</param>
/// <param name="propertyNames">Name of properties to get data according to.</param>
/// <returns>The result as IQueryable.</returns>
public static IQueryable <object> SelectDynamic(
this IQueryable source,
IEnumerable <string> propertyNames)
{
if (source == null)
{
throw new ArgumentNullException(nameof(source));
}
if (propertyNames == null)
{
throw new ArgumentNullException(nameof(propertyNames));
}
if (propertyNames.Count() == 0)
{
throw new ArgumentOutOfRangeException("propertyNames", "There is no propertyName in propertyNames");
}
Dictionary <string, PropertyInfo> sourceProperties = propertyNames
.ToDictionary(name => name, name => source.ElementType.GetProperty(name));
// Create anonymous type at runtime
Type dynamicType = LinqRuntimeTypeBuilder.GetDynamicType(sourceProperties.Values);
ParameterExpression sourceItem = Expression.Parameter(source.ElementType, "m");
IEnumerable <MemberBinding> bindings = dynamicType.GetProperties()
.Select(p => Expression.Bind(p, Expression.Property(sourceItem, sourceProperties[p.Name])))
.OfType <MemberBinding>();
Expression selector = Expression.Lambda(Expression.MemberInit(
Expression.New(dynamicType.GetConstructor(Type.EmptyTypes)), bindings), sourceItem);
IQueryable queryResult = source.Provider.CreateQuery(
Expression.Call(typeof(Queryable),
"Select",
new Type[] { source.ElementType, dynamicType },
source.Expression,
selector));
// Cast to IQueryable<object> to be able to use Linq
return((IQueryable <object>)queryResult);
}
public static IQueryable <dynamic> Join <TOuter, TInner, TKey>(this IQueryable <TOuter> outer,
IEnumerable <TInner> inner, Expression <Func <TOuter, TKey> > outerKeySelector,
Expression <Func <TInner, TKey> > innerKeySelector)
{
var sourceProperties = new Dictionary <string, Type>()
{
// { "AbpUsers2",typeof(AbpUsers) }, { "AbpUsers4",typeof(AbpUsers) },
};
Type dynamicType = LinqRuntimeTypeBuilder.GetDynamicType(sourceProperties);
NewExpression selector = Expression.New(dynamicType.GetConstructor(Type.EmptyTypes));
var lambdaLInq = Expression.Lambda <Func <TOuter, TInner, dynamic> >(selector);
return(outer.Provider.CreateQuery <dynamic>(Expression.Call(null,
GetMethodInfo1 <IQueryable <TOuter>, IEnumerable <TInner>, Expression <Func <TOuter, TKey> >, Expression <Func <TInner, TKey> >, Expression <Func <TOuter, TInner, dynamic> >, IQueryable <dynamic> >(
Queryable.Join, outer, inner, outerKeySelector, innerKeySelector),
outer.Expression, GetSourceExpression(inner), (Expression)Expression.Quote(outerKeySelector),
(Expression)Expression.Quote(innerKeySelector), (Expression)Expression.Quote(selector))));
}
public static Expression <Func <TSource, TResult> > SelectBuild <TSource, TResult>(IEnumerable <string> properties)
{
Type sourceType = typeof(TSource);
Dictionary <string, PropertyInfo> sourceProperties = sourceType.GetProperties(BindingFlags.Public | BindingFlags.Instance)
.Where(prop =>
{
if (!prop.CanRead)
{
return(false);
}
return(properties.Contains(prop.Name, new MemberNameEqualityComparer()));
}).ToDictionary(p => p.Name, p => p);
Type dynamicType = LinqRuntimeTypeBuilder.GetDynamicType(sourceProperties.Values);
ParameterExpression sourceItem = Expression.Parameter(sourceType, "t");
IEnumerable <MemberBinding> bindings = dynamicType.GetRuntimeProperties().Select(p => Expression.Bind(p, Expression.Property(sourceItem, sourceProperties[p.Name]))).OfType <MemberBinding>();
return(Expression.Lambda <Func <TSource, TResult> >(Expression.MemberInit(
Expression.New(dynamicType.GetConstructor(Type.EmptyTypes)), bindings), sourceItem));
// Expression<Func<TSource, TResult>> selector = null;
// var tSourceType = typeof(TSource);
// var left=Expression.Parameter(tSourceType,"x");
// System.Linq.Expressions.NewExpression newExpression = System.Linq.Expressions.Expression.New(tSourceType);
// var memberInfos = tSourceType.GetMembers(BindingFlags.Public | BindingFlags.Instance);
// var memberInfoFilters = memberInfos.Where(x => properties.Contains(x.Name, new MemberNameEqualityComparer()));
// var MemberBindings = memberInfoFilters.Select(x =>
// System.Linq.Expressions.Expression.Bind(
// x,
// Expression.Property(left,x.Name)));
// System.Linq.Expressions.MemberInitExpression memberInitExpression =
// System.Linq.Expressions.Expression.MemberInit(newExpression,MemberBindings);
// selector= Expression.Lambda<Func<TSource, TResult>>(memberInitExpression,new ParameterExpression[]{left});
// return selector;
}
/// <summary>
/// execute the proper fetch
/// </summary>
/// <param name="page"></param>
/// <returns></returns>
public bool ExecuteFetch(long page)
{
bool ret = false;
try
{
//open the client connection
using (var client = ExternalServiceClient.GetClient(m_Properties.Server))
{
//temp parameters until we get SQL TO BQL
//var mapreduce = "SELECT Count() GROUP BY Date INTERVAL Day";
var mapreduce = Sql;
//var mapreduce = "Get Mentions";
var minDate = DateTime.UtcNow.AddDays(-100);
var maxDate = DateTime.UtcNow;
string query = "";
//string command = "-nocount";
string command = "";
//create the paging string
//??how do I specify the page??
string paging = "";// string.Format("__paging__ORDERED BY Date LIMIT {0}", m_Properties.RowsToFetch);
//init the data
m_Data.Clear();
//make the query
//var cursor = client.GetCursor(m_Properties.Catalog, "__ql__" + query, "__ql__" + mapreduce, "__default__", null, command);
var datapoints = client.GetData(m_Properties.Catalog, "__ql__" + query, "__ql__" + mapreduce, "__default__", null, command);
//var datapoints = client.GetData(m_Properties.Catalog, "__ql__" + query, null, null, paging, minDate, maxDate, command);
//this is the call to pass the sql through to bermuda
//var datapoints = client.GetData(Sql);
//check results
if (datapoints != null && !string.IsNullOrWhiteSpace(datapoints.Data) && !string.IsNullOrWhiteSpace(datapoints.DataType))
{
//build the returned data
object obj = LinqRuntimeTypeBuilder.DeserializeJson(datapoints.Data, datapoints.DataType, true);
Type type = obj.GetType();
Array array = obj as Array;
//check for results returned
if (array.Length > 0)
{
//parse the results
foreach (var item in array)
{
//check if we need to init the columns
if (m_Columns.Count == 0)
{
//init the columns
InitializeColumns(item);
}
//add this row
AddRow(item);
}
}
//set the actual row size
//RowCount = TotalItemsToFetch;
}
}
}
catch (Exception ex)
{
System.Diagnostics.Trace.WriteLine(ex.ToString());
}
return(ret);
}
/// <summary>
/// Gets the queryable.
/// </summary>
/// <param name="entityType">Type of the entity.</param>
/// <param name="entityFields">The entity fields.</param>
/// <param name="attributes">The attributes.</param>
/// <param name="selectComponents">The select components.</param>
/// <param name="sortProperty">The sort property.</param>
/// <param name="dataViewFilterOverrides">The data view filter overrides.</param>
/// <param name="databaseTimeoutSeconds">The database timeout seconds.</param>
/// <param name="isCommunication">if set to <c>true</c> [is communication].</param>
/// <param name="errorMessages">The error messages.</param>
/// <param name="reportDbContext">The report database context.</param>
/// <returns></returns>
/// <exception cref="Exception"></exception>
public IQueryable GetQueryable(Type entityType, Dictionary <int, EntityField> entityFields, Dictionary <int, AttributeCache> attributes, Dictionary <int, ReportField> selectComponents, Rock.Web.UI.Controls.SortProperty sortProperty, DataViewFilterOverrides dataViewFilterOverrides, int?databaseTimeoutSeconds, bool isCommunication, out List <string> errorMessages, out System.Data.Entity.DbContext reportDbContext)
{
errorMessages = new List <string>();
reportDbContext = null;
if (entityType != null)
{
reportDbContext = Reflection.GetDbContextForEntityType(entityType);
IService serviceInstance = Reflection.GetServiceForEntityType(entityType, reportDbContext);
if (databaseTimeoutSeconds.HasValue)
{
reportDbContext.Database.CommandTimeout = databaseTimeoutSeconds.Value;
}
if (serviceInstance != null)
{
ParameterExpression paramExpression = serviceInstance.ParameterExpression;
MemberExpression idExpression = Expression.Property(paramExpression, "Id");
// Get AttributeValue queryable and parameter
var attributeValues = reportDbContext.Set <AttributeValue>();
ParameterExpression attributeValueParameter = Expression.Parameter(typeof(AttributeValue), "v");
// Create the dynamic type
var dynamicFields = new Dictionary <string, Type>();
dynamicFields.Add("Id", typeof(int));
foreach (var f in entityFields)
{
dynamicFields.Add(string.Format("Entity_{0}_{1}", f.Value.Name, f.Key), f.Value.PropertyType);
}
foreach (var a in attributes)
{
dynamicFields.Add(string.Format("Attribute_{0}_{1}", a.Value.Id, a.Key), a.Value.FieldType.Field.AttributeValueFieldType);
}
foreach (var reportField in selectComponents)
{
DataSelectComponent selectComponent = DataSelectContainer.GetComponent(reportField.Value.DataSelectComponentEntityType.Name);
if (selectComponent != null)
{
dynamicFields.Add(string.Format("Data_{0}_{1}", selectComponent.ColumnPropertyName, reportField.Key), selectComponent.ColumnFieldType);
var customSortProperties = selectComponent.SortProperties(reportField.Value.Selection);
if (customSortProperties != null)
{
foreach (var customSortProperty in customSortProperties.Split(','))
{
if (!string.IsNullOrWhiteSpace(customSortProperty))
{
var customSortPropertyType = entityType.GetPropertyType(customSortProperty);
dynamicFields.Add(string.Format("Sort_{0}_{1}", customSortProperty, reportField.Key), customSortPropertyType ?? typeof(string));
}
}
}
if (isCommunication && selectComponent is IRecipientDataSelect)
{
dynamicFields.Add($"Recipient_{selectComponent.ColumnPropertyName}_{reportField.Key}", ((IRecipientDataSelect)selectComponent).RecipientColumnFieldType);
}
}
}
if (dynamicFields.Count == 0)
{
errorMessages.Add("At least one field must be defined");
return(null);
}
Type dynamicType = LinqRuntimeTypeBuilder.GetDynamicType(dynamicFields);
ConstructorInfo methodFromHandle = dynamicType.GetConstructor(Type.EmptyTypes);
// Bind the dynamic fields to their expressions
var bindings = new List <MemberAssignment>();
bindings.Add(Expression.Bind(dynamicType.GetField("id"), idExpression));
foreach (var f in entityFields)
{
bindings.Add(Expression.Bind(dynamicType.GetField(string.Format("entity_{0}_{1}", f.Value.Name, f.Key)), Expression.Property(paramExpression, f.Value.Name)));
}
foreach (var a in attributes)
{
bindings.Add(Expression.Bind(dynamicType.GetField(string.Format("attribute_{0}_{1}", a.Value.Id, a.Key)), GetAttributeValueExpression(attributeValues, attributeValueParameter, idExpression, a.Value.Id)));
}
foreach (var reportField in selectComponents)
{
DataSelectComponent selectComponent = DataSelectContainer.GetComponent(reportField.Value.DataSelectComponentEntityType.Name);
if (selectComponent != null)
{
try
{
var componentExpression = selectComponent.GetExpression(reportDbContext, idExpression, reportField.Value.Selection ?? string.Empty);
if (componentExpression == null)
{
componentExpression = Expression.Constant(null, typeof(string));
}
bindings.Add(Expression.Bind(dynamicType.GetField(string.Format("data_{0}_{1}", selectComponent.ColumnPropertyName, reportField.Key)), componentExpression));
if (isCommunication && selectComponent is IRecipientDataSelect)
{
var recipientPersonIdExpression = ((IRecipientDataSelect)selectComponent).GetRecipientPersonIdExpression(reportDbContext, idExpression, reportField.Value.Selection ?? string.Empty);
if (recipientPersonIdExpression != null)
{
bindings.Add(Expression.Bind(dynamicType.GetField(string.Format("recipient_{0}_{1}", selectComponent.ColumnPropertyName, reportField.Key)), recipientPersonIdExpression));
}
}
var customSortProperties = selectComponent.SortProperties(reportField.Value.Selection);
if (!string.IsNullOrEmpty(customSortProperties))
{
foreach (var customSortProperty in customSortProperties.Split(','))
{
var customSortPropertyParts = customSortProperty.Split('.');
MemberInfo memberInfo = dynamicType.GetField(string.Format("sort_{0}_{1}", customSortProperty, reportField.Key));
Expression memberExpression = null;
foreach (var customSortPropertyPart in customSortPropertyParts)
{
memberExpression = Expression.Property(memberExpression ?? paramExpression, customSortPropertyPart);
}
bindings.Add(Expression.Bind(memberInfo, memberExpression));
}
}
}
catch (Exception ex)
{
throw new Exception(string.Format("Exception in {0}", selectComponent), ex);
}
}
}
ConstructorInfo constructorInfo = dynamicType.GetConstructor(Type.EmptyTypes);
NewExpression newExpression = Expression.New(constructorInfo);
MemberInitExpression memberInitExpression = Expression.MemberInit(newExpression, bindings);
LambdaExpression selector = Expression.Lambda(memberInitExpression, paramExpression);
// NOTE: having a NULL Dataview is OK.
Expression whereExpression = null;
if (this.DataView != null)
{
whereExpression = this.DataView.GetExpression(serviceInstance, paramExpression, dataViewFilterOverrides, out errorMessages);
}
MethodInfo getMethod = serviceInstance.GetType().GetMethod("Get", new Type[] { typeof(ParameterExpression), typeof(Expression), typeof(Rock.Web.UI.Controls.SortProperty), typeof(int?) });
if (getMethod != null)
{
var getResult = getMethod.Invoke(serviceInstance, new object[] { paramExpression, whereExpression, null, null });
var qry = getResult as IQueryable <IEntity>;
var qryExpression = qry.Expression;
// apply the OrderBy clauses to the Expression from whatever columns are specified in sortProperty.Property
string orderByMethod = "OrderBy";
if (sortProperty == null)
{
// if no sorting was specified, sort by Id
sortProperty = new Web.UI.Controls.SortProperty {
Direction = SortDirection.Ascending, Property = "Id"
};
}
/*
* NOTE: The sort property sorting rules can be a little confusing. Here is how it works:
* - SortProperty.Direction of Ascending means sort exactly as what the Columns specification says
* - SortProperty.Direction of Descending means sort the _opposite_ of what the Columns specification says
* Examples:
* 1) SortProperty.Property "LastName desc, FirstName, BirthDate desc" and SortProperty.Direction = Ascending
* OrderBy should be: "order by LastName desc, FirstName, BirthDate desc"
* 2) SortProperty.Property "LastName desc, FirstName, BirthDate desc" and SortProperty.Direction = Descending
* OrderBy should be: "order by LastName, FirstName desc, BirthDate"
*/
foreach (var column in sortProperty.Property.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
{
string propertyName;
var direction = sortProperty.Direction;
if (column.EndsWith(" desc", StringComparison.OrdinalIgnoreCase))
{
propertyName = column.Left(column.Length - 5);
// if the column ends with " desc", toggle the direction if sortProperty is Descending
direction = sortProperty.Direction == SortDirection.Ascending ? SortDirection.Descending : SortDirection.Ascending;
}
else
{
propertyName = column;
}
string methodName = direction == SortDirection.Descending ? orderByMethod + "Descending" : orderByMethod;
// Call OrderBy on whatever the Expression is for that Column
var sortMember = bindings.FirstOrDefault(a => a.Member.Name.Equals(propertyName, StringComparison.OrdinalIgnoreCase));
LambdaExpression sortSelector = Expression.Lambda(sortMember.Expression, paramExpression);
qryExpression = Expression.Call(typeof(Queryable), methodName, new Type[] { qry.ElementType, sortSelector.ReturnType }, qryExpression, sortSelector);
orderByMethod = "ThenBy";
}
var selectExpression = Expression.Call(typeof(Queryable), "Select", new Type[] { qry.ElementType, dynamicType }, qryExpression, selector);
var query = qry.Provider.CreateQuery(selectExpression).AsNoTracking();
// cast to a dynamic so that we can do a Queryable.Take (the compiler figures out the T in IQueryable at runtime)
dynamic dquery = query;
if (FetchTop.HasValue)
{
dquery = Queryable.Take(dquery, FetchTop.Value);
}
return(dquery as IQueryable);
}
}
}
return(null);
}
static void Main(string[] args)
{
AssemblyBuilder _assemblyBuilder = AppDomain.CurrentDomain.DefineDynamicAssembly(new AssemblyName("MyBuilder"), AssemblyBuilderAccess.Run);
ModuleBuilder _moduleBuilder = _assemblyBuilder.DefineDynamicModule("MyModule");
var typeBuilder = _moduleBuilder.DefineType("lulz43", TypeAttributes.Class | TypeAttributes.Public);
var fieldBuilder = typeBuilder.DefineField("zomglist", typeof(List <long>), FieldAttributes.Public);
var cctor = typeBuilder.DefineConstructor(MethodAttributes.Public, CallingConventions.Standard | CallingConventions.HasThis, new Type[0]);
var ilGenerator = cctor.GetILGenerator();
MakeCtor(new FieldBuilder[] { fieldBuilder }, ilGenerator);
var type32 = typeBuilder.CreateType();
//var lolzfda = Activator.CreateInstance(type32);
var param = Expression.Parameter(typeof(Mention), "x");
var tags = Expression.MakeMemberAccess(param, typeof(Mention).GetField("Tags"));
var ds = Expression.MakeMemberAccess(param, typeof(Mention).GetField("Datasources"));
var a = new List <int> {
1, 2
};
var b = new List <int> {
3, 4
};
var c = new List <int> {
5, 6
};
var joinedlistsexpressions = UnionEnumerableExpressions(new Expression[] { tags, ds, tags, ds });
var joinedlists = ToArrayCollection(UnionEnumerables(new object[] { a, b, c }), typeof(int));
//var lol = Convert.ChangeType("5", typeof(string));
var listlol = DataHelper.GetMentions(10000);
//var sqlexpr = new EvoSQLExpression("SELECT Type, Color, Count() FROM Cats GROUP BY Color, Fur");
//Console.WriteLine(sqlexpr.Tree.ToString());
var tzs = TimeZoneInfo.GetSystemTimeZones();
var query22 = "SELECT Type FROM (SELECT Type GROUP BY Type)";
var query23 = "SELECT Sentiment FROM (SELECT Sentiment, Id FROM Mentions)";
var query24 = @"
SELECT SUM({fn CONVERT(1, SQL_BIGINT)}) AS ""sum_Number_of_Records_qk""
,
{
fn TIMESTAMPADD
(
SQL_TSI_MONTH,
CAST
(
{fn TRUNCATE((3 * (CAST({fn TRUNCATE({fn QUARTER(""Mentions"".""Date"")},0)} AS INTEGER) - 1)),0)} AS INTEGER),
{fn TIMESTAMPADD(SQL_TSI_DAY,CAST({fn TRUNCATE(-({fn DAYOFYEAR(""Mentions"".""Date"")} - 1),0)} AS INTEGER), CAST(""Mentions"".""Date"" AS DATE))}
)
} AS ""tqr_Date_qk""
FROM ""Bermuda"".""Mentions"" ""Mentions""
GROUP BY ""tqr_Date_qk""";
var query24mini = @" SELECT
SUM({fn CONVERT(1, SQL_BIGINT)}) AS ""sum_Number_of_Records_qk"",
{ fn TIMESTAMPADD( SQL_TSI_MONTH, CAST({fn TRUNCATE(3,3)} as sql_bigint), Date ) } AS ""tqr_Date_qk""
FROM ""Bermuda"".""Mentions"" ""Mentions""
GROUP BY ""tqr_Date_qk""";
var reduceexpr1 = EvoQLBuilder.GetReduceExpression("SELECT Average(NulNum), * GROUP BY IsDisabled, Interval(OccurredOn, \"Year\", 0)", typeof(MentionTest2), true);
var reduceexpr11 = EvoQLBuilder.GetReduceExpression("SELECT Count() WHERE IsDisabled NulNum>5 Tags:5", typeof(MentionTest2), true);
var sellol52 = EvoQLBuilder.GetReduceExpression("SELECT Count() as Value, * GROUP BY TOP 5 Tags as Id VIA Count(), Interval(Date,Day,-5) as Id2", typeof(Mention), true);
//var whereexpr2 = EvoQLBuilder.GetWhereExpression("GET Activity WHERE Whateverz:50 DONCAER:OK jkdsadasjd DATE>\"2012/06/04 16:19:58\"", typeof(Mention));
var sellol3 = EvoQLBuilder.GetReduceExpression(query24, typeof(Mention), true);
var sellol4 = EvoQLBuilder.GetMergeExpression(query24, typeof(Mention), null);
//var sellol5 = EvoQLBuilder.GetReduceExpression("SELECT SUM({fn CONVERT(CONVERT(Sentiment, SQL_BIGINT), SQL_BIGINT)}) \"sum_Number_of_Records_qk\" FROM Nulz", typeof(Mention), true);
var sellol7 = EvoQLBuilder.GetReduceExpression("SELECT -Sentiment+Influence as Ten", typeof(Mention), true);
var sellol6 = EvoQLBuilder.GetReduceExpression("SELECT Sentiment+Influence as Ten, -Interval(Date, Day,-5) as DLOL , -Sentiment as neg, cat cat, Sentiment as S FROM Lulzcol Lulzforsur", typeof(Mention), true);
var sellol = EvoQLBuilder.GetReduceExpression("SELECT \"Mentions\".\"Name\", Sentiment, {fn DatePart(Date,Hour,-5)} as Hour FROM Whatever as Lol", typeof(Mention), true);
var kdsadsa = EvoQLBuilder.GetCollections(null, "SELECT Sentiment FROM Mentions metn", null, null);
//var whereexpr = EvoQLBuilder.GetWhereExpression("Themes:lol NOT:(lol or hi or bye) (Tag:((@52 OR @75) AND NOT:(@53)) ) Tags:5 DATE:-13.551235..-12.551235 (Sentiment:(-100..100 OR -100..100) ) ", typeof(Mention));
var mergeexpr = EvoQLBuilder.GetMergeExpression("SELECT Average(Sentiment) as C GROUP BY Interval(Date, Day, -5) as IntervalTheKilla FROM sigh", typeof(Mention), null);
var zomg = "SELECT 5+5 as lol FROM Mentions";
var red = EvoQLBuilder.GetReduceExpression(zomg, typeof(Mention), true);
var red2 = EvoQLBuilder.GetMergeExpression(zomg, typeof(Mention), null);
var cols = EvoQLBuilder.GetCollections("GET Mention DATE:\"2012/04/09 04:00:00\"..\"2012/04/10 04:00:00\" ( ( (Tag:@2 )))", "SELECT Sum(Followers) AS Count, * FROM Catz", null, null);
//var reduceexpr = EvoQLBuilder.GetReduceExpression("SELECT Count(*) as Value, Average(Sentiment) as Retard GROUP BY TOP 5 Type VIA Count(), OccurredOn as X INTERVAL Day", typeof(MentionTest));
//var reducestring = "SELECT Count() AS Value GROUP BY TOP 10 Tags VIA Count() AS Id, Datepart(OccurredOn, QuarterHour, -5) AS Id2";
var reducestring = "SELECT Sentiment FROM (SELECT Sentiment, Id FROM Mentions)";
var reduceexpr = EvoQLBuilder.GetReduceExpression(reducestring, typeof(MentionTest));
//var mergeexpr = EvoQLBuilder.GetMergeExpression("SELECT Average(Sentiment), Count() as Lulz GROUP BY Tags, OccurredOn INTERVAL Day WHERE Name:no", typeof(Mention), null);
var pagingexpr = EvoQLBuilder.GetPagingExpression("ORDERED BY Date LIMIT 5,25", typeof(Mention));
//var reduceexpr = EvoQLBuilder.GetReduceExpression("CHART Count BY TOP 11 Tags VIA Sentiment OVER Date INTERVAL Day WHERE cats TAG:@1");
//var reduceexpr = ReduceExpressionGeneration.MakeExpression
//(
// null,
// "Count",
// new ReduceDimension[]
// {
// new ReduceDimension{ GroupBy = "Type" },
// new ReduceDimension{ GroupBy = "OccurredOn", Interval = IntervalTypes.Day }
// }
//);
var mytype = LinqRuntimeTypeBuilder.GetDynamicType(new Dictionary <string, Type> {
{ "Count", typeof(long) }, { "Value", typeof(double) }, { "Zomg", typeof(string) }
});
var ctor = Activator.CreateInstance(mytype, false);
var type = LinqRuntimeTypeBuilder.GetTypeKey(listlol.GetType().GetGenericArguments().FirstOrDefault());
var json = LinqRuntimeTypeBuilder.SerializeObject(listlol.Take(5).ToList());
var res = LinqRuntimeTypeBuilder.DeserializeJson(json, type, true);
var type2 = LinqRuntimeTypeBuilder.GetTypeKey(ReduceExpressionGeneration.GetTypeOfEnumerable(res.GetType()));
var type_consistent = type == type2;
//var listlol = DataHelper.GetPremadeMentions();
var dsadsadas = listlol.GroupBy(x => x).Select(g => g.FirstOrDefault());
//Console.WriteLine(reduceexpr);
Console.WriteLine(mergeexpr);
//int serverCount = 20;
//int bucketCount = 131;
//ServerId == BucketId % ServerCount
//var parititions = Enumerable.Range(0, bucketCount).Split(serverCount);
//foreach (var s in parititions){ foreach (var b in s) Console.Write(b + "\t"); Console.WriteLine();}
//Console.WriteLine();
//parititions = Enumerable.Range(0, bucketCount).Split(serverCount + 3);
//foreach (var s in parititions) { foreach (var b in s) Console.Write(b + "\t"); Console.WriteLine(); }
//########################################################################
//## TEST REDUCE EXPRESSION
//########################################################################
//var poitsad = listlol.AsParallel().GroupBy(m => m.Type).Select(g => new EnumMetadata<IGrouping<string, MentionTest>>() { Enum = g }).Select(gmd => new { Type = gmd.Enum.Key, Type_Hash = (long)(gmd.Enum.Key.GetHashCode()), _Count = gmd.Enum.LongCount() }).ToArray();
var compileMethod = reduceexpr.GetType().GetMethods().FirstOrDefault(x => x.Name == "Compile" && x.GetParameters().Length == 0);
var mapreduceFunc = compileMethod.Invoke(reduceexpr, new object[0]);
var mapReduceFuncInvoke = mapreduceFunc.GetType().GetMethod("Invoke");
var pointenum = mapReduceFuncInvoke.Invoke(mapreduceFunc, new object[] { listlol });
var genericToArrayInfos = typeof(Enumerable).GetMethods().Where(x => x.Name == "ToArray" && x.IsGenericMethod && x.GetParameters().Length == 1);
var genericToArrayInfo = genericToArrayInfos.FirstOrDefault();
var enumType = pointenum.GetType();
var pointType = enumType.GetGenericArguments().Last();
var toArrayInfo = genericToArrayInfo.MakeGenericMethod(pointType);
Stopwatch sw2 = new Stopwatch();
sw2.Start();
var data = toArrayInfo.Invoke(null, new object[] { pointenum });
sw2.Stop();
var jsonz = new JavaScriptSerializer().Serialize(pointenum);
Console.WriteLine(sw2.Elapsed);
//var filter = EvoQLBuilder.GetFilterLambda("(b and a) or 23");
//var filtered = listlol.Where(filter.Compile());
//var result = filtered.ToArray();
Console.ReadKey();
//########################################################################
//## BENCHMARK
//########################################################################
#region playing with expressions
//var expr = EvoQLBuilder.GetReduceExpression("CHART Count BY Type OVER Date WHERE cat");
/*
* var validTags = new long[] { 1, 2 };
*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> reduce = collection =>
* from mention in collection
* from tag in mention.Tags
* from datasource in mention.Datasources
* group mention by new MentionGroup { Id = tag, Id2 = datasource } into g
* select new Datapoint { Id = g.Key.Id, Id2 = g.Key.Id2, Timestamp = g.Key.Timestamp };
*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Tag>, IEnumerable<Datapoint>>> reduce2 = (mentions, tags) =>
* from mention in mentions
* join tag in tags on mention.Id equals tag.MentionId
* group mention by new MentionGroup { Timestamp = mention.OccurredOnTicks, Id = tag.TagId } into g
* select new Datapoint { Id = g.Key.Id, Timestamp = g.Key.Timestamp };
*
* var dayTicks = TimeSpan.FromDays(1).Ticks;
*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> reduce34 = mentions =>
* from mention in mentions
* group mention by mention.OccurredOnTicks - mention.OccurredOnTicks % dayTicks into g
* select new Datapoint { Timestamp = g.Key };
*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> reduce33 = mentions =>
* from mention in mentions
* group mention by mention.OccurredOn.Date.Ticks into g
* select new Datapoint { Timestamp = g.Key };
*
*
*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> reduce3 = mentions =>
* from mention in mentions
* from tag in mention.Tags
* where validTags.Contains(tag)
* group mention by new MentionGroup { Timestamp = mention.OccurredOnTicks, Id = tag } into g
* select new Datapoint { Id = g.Key.Id, Timestamp = g.Key.Timestamp };
*
*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> mapreduce3 = collection =>
* collection
* .SelectMany(m => m.Tags, (m, t) => new MentionMetadata2 { Mention = m, Id = t })
* .SelectMany(md => md.Mention.Datasources, (md, ds) => new MentionMetadata2 { Child = md, Id = ds })
* .GroupBy(md => new MentionGroup { Timestamp = md.Child.Mention.OccurredOnTicks, Id = md.Child.Id, Id2 = md.Id })
* .Select(x => new Datapoint { Timestamp = x.Key.Timestamp, Id = x.Key.Id, Id2 = x.Key.Id2 });
*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> mapreduce4 = collection =>
* collection
* .SelectMany(m => m.Tags, (m, t) => new MentionMetadata2 { Mention = m, Id = t })
* .GroupBy(mt => mt.Id)
* .OrderByDescending(g => g.Count())
* .Take(5)
* .SelectMany(x => x)
* .GroupBy(md => new MentionGroup { Timestamp = md.Child.Mention.OccurredOnTicks, Id = md.Child.Id, Id2 = md.Id })
* .Select(x => new Datapoint { Timestamp = x.Key.Timestamp, Id = x.Key.Id, Id2 = x.Key.Id2 });
*
*
*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> mapreduce5 = collection =>
* collection
* .SelectMany(m => m.Tags, (m, t) => new MentionMetadata { Item = m, Id = t })
* .GroupBy(md => md.Id)
* .Select(g => new { Group = g, Value = g.Count() })
* .OrderByDescending(md => md.Value)
* .Take(5)
* .Select(x => new Datapoint());
*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> mapreduce9 = collection =>
* collection
* .AsParallel()
* .SelectMany(m => m.Themes, (m, t) => new MentionMetadata { Item = m, Id = t })
*
* .GroupBy(md => md.Id, md => md.Item)
* .Select(g => new GroupMetadata { Group = g, Value = g.Count() })
* .OrderByDescending(g => g.Value)
* .Take(5)
*
* .SelectMany
* (
* gmd =>
*
* gmd.Group.GroupBy(md => md.OccurredOnTicks)
* .Select(g => new GroupMetadata { Group = g })
* .Select(gmd2 => new Datapoint { Id = gmd.Group.Key, Id2 = gmd2.Group.Key, CountValue = gmd.Value })
* );
*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> mapreduce6 = collection =>
*
* collection
* .AsParallel()
* .SelectMany(m => m.Themes, (m, t) => new MentionMetadata { Item = m, Id = t })
* .GroupBy(md => md.Id, md => md.Item)
* .Select(g => new GroupMetadata { Group = g, Value = g.Count() })
* .OrderByDescending(g => g.Value)
* .Take(5)
*
* .SelectMany
* (
* gmd =>
*
* gmd.Group
* .SelectMany(m => m.Tags, (m, t) => new MentionMetadata { Item = m, Id = t })
* .GroupBy(md => md.Id, md => md.Item)
* .Select(g => new GroupMetadata { Group = g, Value = gmd.Value })
* .OrderByDescending(g => g.Value)
* .Take(2)
*
* .Select(gmd2 => new Datapoint { Id = gmd.Group.Key, Id2 = gmd2.Group.Key, CountValue = gmd2.Value })
*
*
* );
*/
//ExpressionSerializer serializer = new ExpressionSerializer();
//var xml = serializer.Serialize(mapreduce6);
//Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> dsajkdsa = collection => collection.SelectMany(m => m.Ngrams, (m, t) => new MentionMetadata() { Mention = m, Id = t }).GroupBy(md => new MentionGroup() { Timestamp = md.Mention.OccurredOnTicks, Id2 = md.Id }).Select(x => new Datapoint() { Timestamp = x.Key.Timestamp, Id2 = x.Key.Id2 });
/*
* Expression<Func<IEnumerable<Mention>, IEnumerable<Datapoint>>> mapreduce11 = collection =>
* collection
* .SelectMany(m => m.Themes, (m, t) => new MentionMetadata { Item = m, Id = t })
* .GroupBy(md => new MentionGroup { Timestamp = md.Item.OccurredOnTicks, Id2 = md.Id })
* .Select(x => new Datapoint { Timestamp = x.Key.Timestamp, Id2 = x.Key.Id2 });*/
#endregion
var exprs = new Expression <Func <IEnumerable <Mention>, IEnumerable <Datapoint> > >[] { };
var sw = new Stopwatch();
foreach (var expr in exprs)
{
Console.WriteLine(expr);
var func = expr.Compile();
sw.Restart();
for (int i = 0; i < 100; i++)
{
//var dsadqw = func(listlol).ToArray();
}
sw.Stop();
Console.WriteLine(sw.Elapsed);
}
Console.ReadKey();
/*
* Console.WriteLine("WAITING");
*
*
* //http://localhost:13866/Connection.External/ExternalService.svc
*
* var client = GetServiceClient();
*
*
* Random rng = new Random();
*
* while (true)
* {
* char randomChar = (char)(rng.Next(0, 10) + '0');
*
* //ParameterExpression param = Expression.Parameter(typeof(ThriftMention));
*
* object[] parameters = new object[] { randomChar };
*
* Expression<Func<ThriftMention, object[], bool>> query = (x, p) => x.Description.Contains((char)p[0]);
*
* Expression<Func<IEnumerable<ThriftMention>, IEnumerable<ThriftDatapoint>>> mapreduce = x => x.SelectMany(y => y.Tags).GroupBy(y => y).Select(y => new ThriftDatapoint { Count = y.Count(), Value = y.Count(), EntityId = y.Key });
*
* Expression<Func<IEnumerable<ThriftDatapoint>, double>> merge = x => x.Sum(y => y.Value);
*
* var domain = "evoapp";
* var minDate = DateTime.Today.AddDays(-600);
* var maxDate = DateTime.Today.AddDays(1);
*
* Stopwatch sw = new Stopwatch();
* sw.Start();
* var datapoints = client.GetDatapointList(domain, query, mapreduce, merge, minDate, maxDate, parameters);
* sw.Stop();
*
* Console.WriteLine("QUERY: " + randomChar);
* Console.WriteLine(sw.Elapsed.ToString());
*
* Console.WriteLine(string.Join(" ", datapoints.Select(x => "tag" + x.EntityId + ": " + x.Value + " (" + x.Count + ")")));
* Console.WriteLine(string.Empty);
*
* Thread.Sleep(1000);
* }
*/
}
/// <summary>
/// Gets the data source.
/// </summary>
/// <param name="context">The context.</param>
/// <param name="entityType">Type of the entity.</param>
/// <param name="entityFields">The entity fields.</param>
/// <param name="attributes">The attributes.</param>
/// <param name="selectComponents">The select components.</param>
/// <param name="sortProperty">The sort property.</param>
/// <param name="errorMessages">The error messages.</param>
/// <returns></returns>
public List <object> GetDataSource(RockContext context, Type entityType, Dictionary <int, EntityField> entityFields, Dictionary <int, AttributeCache> attributes, Dictionary <int, ReportField> selectComponents, Rock.Web.UI.Controls.SortProperty sortProperty, out List <string> errorMessages)
{
errorMessages = new List <string>();
if (entityType != null)
{
Type[] modelType = { entityType };
Type genericServiceType = typeof(Rock.Data.Service <>);
Type modelServiceType = genericServiceType.MakeGenericType(modelType);
object serviceInstance = Activator.CreateInstance(modelServiceType, new object[] { context });
if (serviceInstance != null)
{
ParameterExpression paramExpression = serviceInstance.GetType().GetProperty("ParameterExpression").GetValue(serviceInstance) as ParameterExpression;
MemberExpression idExpression = Expression.Property(paramExpression, "Id");
// Get AttributeValue queryable and parameter
var attributeValues = context.Set <AttributeValue>();
ParameterExpression attributeValueParameter = Expression.Parameter(typeof(AttributeValue), "v");
// Create the dynamic type
var dynamicFields = new Dictionary <string, Type>();
foreach (var f in entityFields)
{
dynamicFields.Add(string.Format("Entity_{0}_{1}", f.Value.Name, f.Key), f.Value.PropertyType);
}
foreach (var a in attributes)
{
dynamicFields.Add(string.Format("Attribute_{0}_{1}", a.Value.Id, a.Key), typeof(string));
}
foreach (var reportField in selectComponents)
{
DataSelectComponent selectComponent = DataSelectContainer.GetComponent(reportField.Value.DataSelectComponentEntityType.Name);
if (selectComponent != null)
{
dynamicFields.Add(string.Format("Data_{0}_{1}", selectComponent.ColumnPropertyName, reportField.Key), selectComponent.ColumnFieldType);
}
}
if (dynamicFields.Count == 0)
{
errorMessages.Add("At least one field must be defined");
return(null);
}
Type dynamicType = LinqRuntimeTypeBuilder.GetDynamicType(dynamicFields);
ConstructorInfo methodFromHandle = dynamicType.GetConstructor(Type.EmptyTypes);
// Bind the dynamic fields to their expressions
var bindings = new List <MemberBinding>();
foreach (var f in entityFields)
{
bindings.Add(Expression.Bind(dynamicType.GetField(string.Format("entity_{0}_{1}", f.Value.Name, f.Key)), Expression.Property(paramExpression, f.Value.Name)));
}
foreach (var a in attributes)
{
bindings.Add(Expression.Bind(dynamicType.GetField(string.Format("attribute_{0}_{1}", a.Value.Id, a.Key)), GetAttributeValueExpression(attributeValues, attributeValueParameter, idExpression, a.Value.Id)));
}
foreach (var reportField in selectComponents)
{
DataSelectComponent selectComponent = DataSelectContainer.GetComponent(reportField.Value.DataSelectComponentEntityType.Name);
if (selectComponent != null)
{
bindings.Add(Expression.Bind(dynamicType.GetField(string.Format("data_{0}_{1}", selectComponent.ColumnPropertyName, reportField.Key)), selectComponent.GetExpression(context, idExpression, reportField.Value.Selection)));
}
}
ConstructorInfo constructorInfo = dynamicType.GetConstructor(Type.EmptyTypes);
NewExpression newExpression = Expression.New(constructorInfo);
MemberInitExpression memberInitExpression = Expression.MemberInit(newExpression, bindings);
Expression selector = Expression.Lambda(memberInitExpression, paramExpression);
Expression whereExpression = this.DataView.GetExpression(serviceInstance, paramExpression, out errorMessages);
MethodInfo getMethod = serviceInstance.GetType().GetMethod("Get", new Type[] { typeof(ParameterExpression), typeof(Expression), typeof(Rock.Web.UI.Controls.SortProperty) });
if (getMethod != null)
{
var getResult = getMethod.Invoke(serviceInstance, new object[] { paramExpression, whereExpression, sortProperty });
var qry = getResult as IQueryable <IEntity>;
var selectExpression = Expression.Call(typeof(Queryable), "Select", new Type[] { qry.ElementType, dynamicType }, Expression.Constant(qry), selector);
var query = qry.Provider.CreateQuery(selectExpression).AsNoTracking();
// enumerate thru the query results and put into a list
var reportResult = new List <object>();
var enumerator = query.GetEnumerator();
while (enumerator.MoveNext())
{
reportResult.Add(enumerator.Current);
}
return(reportResult);
}
}
}
return(null);
}
public BermudaResult GetData(string domain, string query, string mapreduce, string merge, string paging, int remdepth, string command, string cursor, string paging2)
{
var args = ParseCommand(command);
bool noCache = args.Contains("-nocache");
bool makeCursor = cursor == MakeCursorToken;
bool useCursor = !makeCursor && !string.IsNullOrWhiteSpace(cursor);
DateTime minDate = DateTime.MinValue;
DateTime maxDate = DateTime.MaxValue;
if (remdepth > 0)
{
//map
var queryHash = cursor ?? GetQueryHash(domain, query, mapreduce, merge, paging, null);
//reduce
BermudaResult cachedDatapoints;
if (!noCache && CachedData.TryGetValue(queryHash, out cachedDatapoints) && (DateTime.Now.Ticks - cachedDatapoints.CreatedOn) < CacheLifetime)
{
#if DEBUG
if (CacheTraceMessageLevel < 3)
{
Trace.WriteLine("returned CACHED BLOBS DATAPOINTS results FOR ENTIRE BLOB SET [REMDEPTH:" + remdepth + "]");
}
#endif
if (useCursor)
{
var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(cachedDatapoints.DataType);
return(GetCursorData(paging2, cachedDatapoints, dataType));
}
else
{
return(new BermudaResult {
DataType = cachedDatapoints.DataType, Data = cachedDatapoints.Data, Metadata = new BermudaNodeStatistic {
Notes = "Cache_Hit_1"
}, CacheKey = cachedDatapoints.CacheKey
});
}
}
else
{
if (useCursor)
{
throw new Exception("Cursor " + cursor + " not found");
}
//var assignments = PartitionBlobs(domain, blobInterfaces, minDate, maxDate, false, true);
var reducers = HostEnvironment.Instance.GetAvailablePeerConnections();
if (!reducers.Any())
{
throw new Exception("Specified dataset not loaded: " + domain);
}
ConcurrentDictionary <PeerInfo, BermudaResult> results = new ConcurrentDictionary <PeerInfo, BermudaResult>();
Stopwatch sw = new Stopwatch();
sw.Start();
List <Task> tasks = new List <Task>();
foreach (var reducer in reducers)
{
Task t = new Task((peerObj) =>
{
var peerInfo = peerObj as PeerInfo;
var initiated = DateTime.Now;
var subqueryHash = GetQueryHash(domain, query, mapreduce, merge, paging, peerInfo.ToString());
Stopwatch sw3 = new Stopwatch();
sw3.Start();
//see if the cache contains a matching result and return it if it's not outdated
BermudaResult cachedDatapoints2;
if (!noCache && CachedData.TryGetValue(subqueryHash, out cachedDatapoints2) && (DateTime.Now.Ticks - cachedDatapoints2.CreatedOn) < CacheLifetime)
{
if (CacheTraceMessageLevel < 2)
{
Trace.WriteLine("returned CACHED BLOB DATAPOINT results FOR BLOB SUBSET [REMDEPTH:" + remdepth + "]");
}
BermudaResult res = null;
if (useCursor)
{
var dataType2 = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(cachedDatapoints2.DataType);
res = GetCursorData(paging2, cachedDatapoints2, dataType2);
}
else
{
res = new BermudaResult {
DataType = cachedDatapoints2.DataType, Data = cachedDatapoints2.Data, Metadata = new BermudaNodeStatistic {
Notes = "Cache_Hit_2"
}
};
}
results[peerInfo] = res;
}
else
{
try
{
Stopwatch sw2 = new Stopwatch();
sw2.Start();
BermudaResult subresult = null;
if (peerInfo.Equals(Endpoint))
{
subresult = GetData(domain, query, mapreduce, merge, paging, remdepth - 1, command, cursor, paging2);
}
else
{
using (var client = HostEnvironment.GetServiceClient(peerInfo))
{
subresult = client.GetData(domain, query, mapreduce, merge, paging, remdepth - 1, command, cursor, paging2);
}
//subresult = GetDataFromPeer(domain, query, mapreduce, merge, minDate, maxDate, remdepth - 1, command, assignment.PeerEndpoint.Endpoint);
}
sw2.Stop();
subresult.CreatedOn = DateTime.Now.Ticks;
subresult.Metadata.Initiated = initiated;
subresult.Metadata.Completed = DateTime.Now;
subresult.Metadata.OperationTime = sw2.Elapsed;
results[peerInfo] = CachedData[subqueryHash] = subresult;
}
catch (Exception ex)
{
results[peerInfo] = new BermudaResult {
Error = "[Failed Node] " + ex
};
}
}
}, reducer, TaskCreationOptions.LongRunning);
tasks.Add(t);
t.Start();
}
Task.WaitAll(tasks.ToArray());
sw.Stop();
#if DEBUG
Trace.WriteLine("Join Time:" + sw.Elapsed);
#endif
if (results.Any(x => x.Value.Error != null))
{
throw new BermudaException("Some nodes failed:\r\n" + string.Join("\r\n", results.Select(x => x.Value.Error)));
}
if (results.All(x => x.Value.Data == null))
{
return new BermudaResult {
Metadata = new BermudaNodeStatistic {
Notes = "No Data"
}
}
}
;
//if all results are not the same time throw an error
if (results.GroupBy(x => x.Value.DataType).Count() > 1)
{
throw new BermudaException("Subresults must all return the same type");
}
var dataTypeDescriptor = results.Select(x => x.Value.DataType).FirstOrDefault(x => x != null);
if (dataTypeDescriptor == null)
{
return new BermudaResult {
Error = "Could not determine the merge type, none of the nodes provided type info"
}
}
;
//use the passed combine espression to make multiple datapoint sets into one
var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(dataTypeDescriptor);
//allItems = results.Values.SelectMany(x => x.DataObject)
var totalJson = "[" + string.Join(",", results.Values.Where(x => !string.IsNullOrWhiteSpace(x.Data)).Select(x => x.Data.Trim('[', ']')).Where(x => !string.IsNullOrWhiteSpace(x))) + "]";
var allItems = LinqRuntimeTypeBuilder.DeserializeJson(totalJson, dataTypeDescriptor, true);
//var aaa = new JavaScriptSerializer().Deserialize<Datapoint[]>(totalJson);
//var ggc = aaa.GroupBy(x => new { x.Id, x.Id2 }).Count();
//InvokeSelectManyViaReflectionTheKilla(results.Values.Select(x => x.DataObject), dataType);
var mergeFunc = GetMergeFunc(merge, mapreduce, dataType, dataType);
if (mergeFunc != null)
{
//var dataType = "kdsajkdsa";
var mergeInvokeMethod = mergeFunc.GetType().GetMethod("Invoke");
allItems = mergeInvokeMethod.Invoke(mergeFunc, new object[] { allItems }); // MergeDatapoints(results.Values.Where(x => x.Data != null).SelectMany(x => x.Data), mergeFunc);
}
var pagingFunc = GetPagingFunc(paging, dataType);
if (pagingFunc != null)
{
var pagingInvokeMethod = pagingFunc.GetType().GetMethod("Invoke");
allItems = pagingInvokeMethod.Invoke(pagingFunc, new object[] { allItems });
}
//figure out the metadata
var finalMetadata = new BermudaNodeStatistic {
Notes = "Merged Datapoints in " + sw.Elapsed, NodeId = HostEnvironment.Instance.CurrentInstanceId, ChildNodes = results.Values.Select(x => x.Metadata).ToArray()
};
var arraylol = ToArrayCollection(allItems, dataType);
var json = JsonConvert.SerializeObject(arraylol);
//var json = JsonConvert.SerializeObject(allItems);
var originalData = makeCursor ? arraylol : null;
var finalResult = new BermudaResult {
DataType = dataTypeDescriptor, OriginalData = originalData, Data = json, CreatedOn = DateTime.Now.Ticks, Metadata = finalMetadata, CacheKey = queryHash
};
CachedData[queryHash] = finalResult;
return(finalResult);
}
}
else
{
ConcurrentDictionary <string, BermudaResult> results = new ConcurrentDictionary <string, BermudaResult>();
BermudaNodeStatistic stats = new BermudaNodeStatistic();
var bucketInterfaces = HostEnvironment.Instance.GetBucketInterfacesForDomain(domain);
if (!bucketInterfaces.Any())
{
throw new BermudaException("Data not loaded for: " + domain);
}
if (bucketInterfaces.Count() > 1)
{
throw new BermudaException("Multiple buckets not supported by BermudaMapReduce");
}
var queryHash = GetQueryHash(domain, query, mapreduce, merge, paging, Endpoint.ToString());
BermudaResult cachedDatapoints;
if (!noCache && CachedData.TryGetValue(queryHash, out cachedDatapoints) && (DateTime.Now.Ticks - cachedDatapoints.CreatedOn) < CacheLifetime)
{
if (CacheTraceMessageLevel < 2)
{
Trace.WriteLine("returned CACHED BLOB SET DATAPOINT results [REMDEPTH:" + remdepth + "]");
}
if (useCursor)
{
var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(cachedDatapoints.DataType);
return(GetCursorData(paging2, cachedDatapoints, dataType));
}
else
{
return(new BermudaResult {
DataType = cachedDatapoints.DataType, Data = cachedDatapoints.Data, Metadata = new BermudaNodeStatistic {
Notes = "Cache_Hit_3"
}, CacheKey = queryHash
});
}
}
else
{
//Chad: short circuiting to test WCF response time in Azure
//return new DatapointResult() { Datapoints = new List<Datapoint>(), CreatedOn = DateTime.Now.Ticks, Metadata = new BermudaNodeStatistic() };
//IEnumerable<Datapoint> datapoints = null;
object datapoints = null;
Stopwatch sw = new Stopwatch();
sw.Start();
Type itemType = null;
Type resultType = null;
string json = null;
foreach (var bucketInterface in bucketInterfaces)
{
var bucketKey = GetQueryHash(domain, query, mapreduce, merge, paging, Endpoint.ToString());
//see if the cache contains a matching result and return it if it's not outdated
BermudaResult cachedDatapoints2;
if (!noCache && CachedData.TryGetValue(bucketKey, out cachedDatapoints2) && (DateTime.Now.Ticks - cachedDatapoints2.CreatedOn) < CacheLifetime)
{
if (CacheTraceMessageLevel < 1)
{
Trace.WriteLine("returned CACHED BLOB DATAPOINT results [REMDEPTH:" + remdepth + "]");
}
if (useCursor)
{
if (cachedDatapoints2.OriginalData == null)
{
throw new Exception("Cursor " + cursor + " contains null data");
}
var dataType = LinqRuntimeTypeBuilder.GetTypeFromTypeKey(cachedDatapoints2.DataType);
results[bucketInterface.Name] = GetCursorData(paging2, cachedDatapoints2, dataType);
}
else
{
results[bucketInterface.Name] = new BermudaResult {
DataType = cachedDatapoints2.DataType, Data = cachedDatapoints2.Data, Metadata = new BermudaNodeStatistic {
Notes = "Cache_Hit_4"
}
};
json = cachedDatapoints2.Data;
}
}
else
{
//get mentions
var collections = GetCollections(query, mapreduce);
if (collections.Count() > 1)
{
throw new BermudaException("More than one collection specified: " + string.Join(",", collections));
}
var table = collections.FirstOrDefault();
var tableName = table == null ? null : table.Source;
var raw = bucketInterface.GetData(tableName);
//var rawType = raw.GetType();
//itemType = ReduceExpressionGeneration.GetTypeOfEnumerable(rawType);
itemType = bucketInterface.GetDataType(tableName);
var mapreduceFunc = GetMapReduceFunc(mapreduce, itemType, out resultType);
var queryFunc = GetFilterFunc(query, itemType);
var pagingFunc = GetPagingFunc(paging, resultType);
var minDateTicks = minDate.Ticks;
var maxDateTicks = maxDate.Ticks;
object subresult = raw;
//queryFunc == null ?
// raw.AsParallel() :
//minDate == DateTime.MinValue && maxDate == DateTime.MaxValue ?
// raw.AsParallel().Where(x => queryFunc) :
// raw.AsParallel().Where(x => x.OccurredOnTicks >= minDateTicks && x.OccurredOnTicks <= maxDateTicks && queryFunc(x, parameters));
if (json == null)
{
if (queryFunc != null)
{
var queryFuncInvoke = queryFunc.GetType().GetMethod("Invoke");
subresult = queryFuncInvoke.Invoke(queryFunc, new object[] { subresult });
}
//reduce them using the passed expression
if (mapreduceFunc != null)
{
var mapReduceFuncInvoke = mapreduceFunc.GetType().GetMethod("Invoke");
subresult = mapReduceFuncInvoke.Invoke(mapreduceFunc, new object[] { subresult });
}
if (pagingFunc != null)
{
var pagingInvokeMethod = pagingFunc.GetType().GetMethod("Invoke");
subresult = pagingInvokeMethod.Invoke(pagingFunc, new object[] { subresult });
}
datapoints = subresult;
}
//format a metada string
if (!args.Contains("-nocount"))
{
stats.TotalItems = bucketInterface.GetCount(tableName);
//stats.FilteredItems = filtered.Count();
//stats.ReducedItems = subresult.Count();
}
//cache the result
//results[blobInterface.Name] = new DatapointResult { Datapoints = subresult, CreatedOn = DateTime.UtcNow.Ticks, Metadata = stats.Serialize() };
//CachedDatapoints[blobKey] = new DatapointResult { Datapoints = subresult.ToList(), CreatedOn = DateTime.UtcNow.Ticks, Metadata = stats.Serialize() };
}
}
//figure out the metadata
//var finalMetadata = " [@" + AzureInterface.Instance.CurrentInstanceId + "] Calculated Datapoints:\r\n" + string.Join("\r\n", results.Values.Select(x => x.Metadata));
stats.NodeId = HostEnvironment.Instance.CurrentInstanceId;
stats.Notes = "Computed Datapoints";
//Trace.WriteLine("total mentions processed: " + mentionCount);
//var datapoints = results.Values.SelectMany(x => x.Datapoints);
if (datapoints == null)
{
return new BermudaResult()
{
Metadata = new BermudaNodeStatistic {
Notes = "No Results"
}
}
}
;
//foreach (var p in datapoints) if (p.IsCount) p.Value = p.Count;
var mergeFunc = GetMergeFunc(merge, mapreduce, itemType, resultType);
if (mergeFunc != null)
{
var mergeFuncInvoke = mergeFunc.GetType().GetMethod("Invoke");
datapoints = mergeFuncInvoke.Invoke(mergeFunc, new object[] { datapoints });
}
stats.LinqExecutionTime = sw.Elapsed;
var arraylol = ToArrayCollection(datapoints, resultType);
if (json == null && datapoints != null)
{
json = JsonConvert.SerializeObject(arraylol);
}
//var json = JsonConvert.SerializeObject(datapoints);
var originalData = makeCursor ? arraylol : null;
var result = CachedData[queryHash] = new BermudaResult {
DataType = LinqRuntimeTypeBuilder.GetTypeKey(resultType), OriginalData = originalData, Data = json, CreatedOn = DateTime.Now.Ticks, Metadata = stats
};
sw.Stop();
return(result);
}
}
}
