public DateTimeOffset GetDateTimeOffset(int ordinal)
{
LastOrdinal = ordinal;
LastMethodName = nameof(GetDateTimeOffset);
switch (typeCodes[ordinal])
{
case tcDateTimeOffset:
if (isPostgres)
{
throw new InvalidOperationException("DateTimeOffset not supported in Postgres");
}
return(((SqlDataReader)reader).GetDateTimeOffset(ordinal));
default:
return(ReflectionTools.ChangeType <DateTimeOffset>(reader.GetValue(ordinal)));
}
}
public static Dictionary <QueryToken, object?> ParseMainKeys(this PredictorPredictContext pctx, Dictionary <string, object?> mainKeys)
{
Dictionary <QueryToken, object?> filters = new Dictionary <QueryToken, object?>();
var options = SignumServer.JsonSerializerOptions;
var qd = QueryLogic.Queries.QueryDescription(pctx.Predictor.MainQuery.Query.ToQueryName());
foreach (var kvp in mainKeys)
{
var qt = QueryUtils.Parse(kvp.Key, qd, SubTokensOptions.CanElement | SubTokensOptions.CanAggregate);
var obj = kvp.Value is JsonElement jt?jt.ToObject(qt.Type, options) : ReflectionTools.ChangeType(kvp.Value, qt.Type);
filters.Add(qt, obj);
}
return(filters);
}
private static Result <object?> TryParseInternal(string?stringValue, Type type)
{
FilterType filterType = QueryUtils.GetFilterType(type);
List <IFilterValueConverter>?converters = SpecificConverters.TryGetC(filterType);
if (converters != null)
{
foreach (var fvc in converters)
{
var res = fvc.TryParseValue(stringValue, type);
if (res != null)
{
if (res is Result <object?> .Success s)
{
try
{
var v = ReflectionTools.ChangeType(s.Value, type);
return(new Result <object?> .Success(v));
}
catch (Exception e)
{
return(new Result <object?> .Error(e.Message));
}
}
else
{
return(res);
}
}
}
}
if (ReflectionTools.TryParse(stringValue, type, CultureInfo.InvariantCulture, out var result))
{
return(new Result <object?> .Success(result));
}
else
{
return(new Result <object?> .Error("Invalid format"));
}
}
public TimeSpan GetTimeSpan(int ordinal)
{
LastOrdinal = ordinal;
LastMethodName = nameof(GetTimeSpan);
switch (typeCodes[ordinal])
{
case tcTimeSpan:
if (isPostgres)
{
return(((NpgsqlDataReader)reader).GetTimeSpan(ordinal));
}
else
{
return(((SqlDataReader)reader).GetTimeSpan(ordinal));
}
default:
return(ReflectionTools.ChangeType <TimeSpan>(reader.GetValue(ordinal)));
}
}
public DateTime GetDateTime(int ordinal)
{
LastOrdinal = ordinal;
DateTime dt;
switch (typeCodes[ordinal])
{
case TypeCode.DateTime:
dt = reader.GetDateTime(ordinal);
break;
default:
dt = ReflectionTools.ChangeType <DateTime>(reader.GetValue(ordinal));
break;
}
if (Schema.Current.TimeZoneMode == TimeZoneMode.Utc)
{
return(new DateTime(dt.Ticks, DateTimeKind.Utc));
}
return(dt);
}
public Date GetDate(int ordinal)
{
LastOrdinal = ordinal;
LastMethodName = nameof(GetDate);
Date dt;
switch (typeCodes[ordinal])
{
case TypeCode.DateTime:
dt = new Date(reader.GetDateTime(ordinal));
break;
case FieldReader.tcNpgsqlDate:
dt = new Date((DateTime)((NpgsqlDataReader)reader).GetDate(ordinal));
break;
default:
dt = new Date(ReflectionTools.ChangeType <DateTime>(reader.GetValue(ordinal)));
break;
}
return(dt);
}
public static object Eval(Expression expression)
{
switch (expression.NodeType)
{
case ExpressionType.Constant:
return(((ConstantExpression)expression).Value);
case ExpressionType.MemberAccess:
{
var me = (MemberExpression)expression;
if (me.Expression == null)
{
return(GetStaticGetter(me.Member)());
}
else
{
return(GetInstanceGetter(me.Member)(Eval(me.Expression)));
}
}
case ExpressionType.Convert:
{
var conv = (UnaryExpression)expression;
var operand = Eval(conv.Operand);
if (conv.Method != null)
{
return(GetExtensionMethodCaller(conv.Method)(operand));
}
if (operand is IConvertible)
{
return(ReflectionTools.ChangeType(operand, conv.Type));
}
return(operand);
}
case ExpressionType.Call:
{
var call = (MethodCallExpression)expression;
if (call.Method.IsStatic)
{
if (call.Arguments.Count == 0)
{
return(GetStaticMethodCaller(call.Method)());
}
if (call.Arguments.Count == 1)
{
return(GetExtensionMethodCaller(call.Method)(Eval(call.Arguments[0])));
}
}
else
{
if (call.Arguments.Count == 0)
{
return(GetInstanceMethodCaller(call.Method)(Eval(call.Object)));
}
}
break;
}
case ExpressionType.Equal:
{
var be = (BinaryExpression)expression;
return(object.Equals(Eval(be.Left), Eval(be.Right)));
}
case ExpressionType.NotEqual:
{
var be = (BinaryExpression)expression;
return(!object.Equals(Eval(be.Left), Eval(be.Right)));
}
}
Delegate fn;
using (HeavyProfiler.LogNoStackTrace("Comp"))
{
fn = Expression.Lambda(expression).Compile();
}
try
{
return(fn.DynamicInvoke(null));
}
catch (TargetInvocationException ex)
{
ex.InnerException.PreserveStackTrace();
throw ex.InnerException;
}
}
public UniqueKeyException(Exception inner) : base(null, inner)
{
foreach (var rx in regexes)
{
Match m = rx.Match(inner.Message);
if (m.Success)
{
TableName = m.Groups["table"].Value;
IndexName = m.Groups["index"].Value;
Values = m.Groups["value"].Value;
Table = cachedTables.GetOrAdd(TableName, tn => Schema.Current.Tables.Values.FirstOrDefault(t => t.Name.ToString() == tn));
if (Table != null)
{
var tuple = cachedLookups.GetOrAdd((Table, IndexName), tup =>
{
var index = tup.table.GeneratAllIndexes().OfType <UniqueTableIndex>().FirstOrDefault(ix => ix.IndexName == tup.indexName);
if (index == null)
{
return(null);
}
var properties = (from f in tup.table.Fields.Values
let cols = f.Field.Columns()
where cols.Any() && cols.All(c => index.Columns.Contains(c))
select Reflector.TryFindPropertyInfo(f.FieldInfo)).NotNull().ToList();
if (properties.IsEmpty())
{
return(null);
}
return(index, properties);
});
if (tuple != null)
{
Index = tuple.Value.index;
Properties = tuple.Value.properties;
try
{
var values = Values.Split(", ");
if (values.Length == Index.Columns.Length)
{
var colValues = Index.Columns.Zip(values).ToDictionary(a => a.First, a => a.Second == "<NULL>" ? null : a.Second.Trim().Trim('\''));
HumanValues = Properties.Select(p =>
{
var f = Table.GetField(p);
if (f is FieldValue fv)
{
return(colValues.GetOrThrow(fv));
}
if (f is FieldEnum fe)
{
return(colValues.GetOrThrow(fe)?.Let(a => ReflectionTools.ChangeType(a, fe.Type)));
}
if (f is FieldReference fr)
{
colValues.GetOrThrow(fr)?.Let(a => Database.RetrieveLite(fr.Type, PrimaryKey.Parse(a, fr.Type)));
}
if (f is FieldImplementedBy ib)
{
var imp = ib.ImplementationColumns.SingleOrDefault(ic => colValues.TryGetCN(ic.Value) != null);
if (imp.Key == null)
{
return(null);
}
return(Database.RetrieveLite(imp.Key, PrimaryKey.Parse(colValues.GetOrThrow(imp.Value) !, imp.Key)));
}
if (f is FieldImplementedByAll iba)
{
var typeId = colValues.GetOrThrow(iba.ColumnType);
if (typeId == null)
{
return(null);
}
var type = TypeLogic.IdToType.GetOrThrow(PrimaryKey.Parse(typeId, typeof(TypeEntity)));
return(Database.RetrieveLite(type, PrimaryKey.Parse(colValues.GetOrThrow(iba.Column) !, type)));
}
throw new UnexpectedValueException(f);
}).ToArray();
}
}
catch
{
//
}
}
}
}
}
}
static MList <T> ReadJsonInternal <T>(JsonReader reader, IMListPrivate <T> existingValue, JsonSerializer serializer)
{
var errors = new List <string>();
serializer.Error += delegate(object?sender, ErrorEventArgs args)
{
// only log an error once
if (args.CurrentObject == args.ErrorContext.OriginalObject)
{
errors.Add(args.ErrorContext.Error.Message);
}
};
var dic = existingValue == null ? new Dictionary <PrimaryKey, MList <T> .RowIdElement>() :
existingValue.InnerList.Where(a => a.RowId.HasValue).ToDictionary(a => a.RowId !.Value, a => a);
var newList = new List <MList <T> .RowIdElement>();
var pr = JsonSerializerExtensions.CurrentPropertyRoute !;
var elementPr = pr.Add("Item");
var rowIdType = GetRowIdTypeFromAttribute(pr);
reader.Assert(JsonToken.StartArray);
reader.Read();
using (JsonSerializerExtensions.SetCurrentPropertyRoute(elementPr))
{
while (reader.TokenType == JsonToken.StartObject)
{
reader.Read();
reader.Assert(JsonToken.PropertyName);
if (((string)reader.Value !) != "rowId")
{
throw new JsonSerializationException($"member 'rowId' expected in {reader.Path}");
}
reader.Read();
var rowIdValue = reader.Value;
reader.Read();
reader.Assert(JsonToken.PropertyName);
if (((string)reader.Value !) != "element")
{
throw new JsonSerializationException($"member 'element' expected in {reader.Path}");
}
reader.Read();
if (rowIdValue != null && !rowIdValue.Equals(GraphExplorer.DummyRowId.Object))
{
var rowId = new PrimaryKey((IComparable)ReflectionTools.ChangeType(rowIdValue, rowIdType) !);
var oldValue = dic.TryGetS(rowId);
if (oldValue == null)
{
T newValue = (T)serializer.DeserializeValue(reader, typeof(T), null) !;
newList.Add(new MList <T> .RowIdElement(newValue, rowId, null));
}
else
{
T newValue = (T)serializer.DeserializeValue(reader, typeof(T), oldValue.Value.Element) !;
if (oldValue.Value.Element !.Equals(newValue))
{
newList.Add(new MList <T> .RowIdElement(newValue, rowId, oldValue.Value.OldIndex));
}
else
{
newList.Add(new MList <T> .RowIdElement(newValue));
}
}
}
public override MList <T>?Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options, MList <T>?existingValue)
{
var existingMList = (IMListPrivate <T>?)existingValue;
var dic = existingMList == null ? new Dictionary <PrimaryKey, MList <T> .RowIdElement>() :
existingMList.InnerList.Where(a => a.RowId.HasValue).ToDictionary(a => a.RowId !.Value, a => a);
var newList = new List <MList <T> .RowIdElement>();
var tup = EntityJsonContext.CurrentPropertyRouteAndEntity !.Value;
var elementPr = tup.pr.Add("Item");
var rowIdType = GetRowIdTypeFromAttribute(tup.pr);
reader.Assert(JsonTokenType.StartArray);
reader.Read();
while (reader.TokenType == JsonTokenType.StartObject)
{
reader.Read();
reader.Assert(JsonTokenType.PropertyName);
if (reader.GetString() != "rowId")
{
throw new JsonException($"member 'rowId' expected in {reader.CurrentState}");
}
reader.Read();
var rowIdValue = reader.GetLiteralValue();
reader.Read();
reader.Assert(JsonTokenType.PropertyName);
if (reader.GetString() != "element")
{
throw new JsonException($"member 'element' expected in {reader.CurrentState}");
}
reader.Read();
using (EntityJsonConverterFactory.SetPath($"[{newList.Count}].element"))
{
if (rowIdValue != null && !rowIdValue.Equals(GraphExplorer.DummyRowId.Object))
{
var rowId = new PrimaryKey((IComparable)ReflectionTools.ChangeType(rowIdValue, rowIdType) !);
var oldValue = dic.TryGetS(rowId);
using (EntityJsonContext.SetCurrentPropertyRouteAndEntity((elementPr, tup.mod, rowId)))
{
if (oldValue == null)
{
T newValue = (T)converter.Read(ref reader, typeof(T), options) !;
newList.Add(new MList <T> .RowIdElement(newValue, rowId, null));
}
else
{
T newValue = converter is JsonConverterWithExisting <T> jcwe ?
(T)jcwe.Read(ref reader, typeof(T), options, oldValue.Value.Element !) ! :
(T)converter.Read(ref reader, typeof(T), options) !;
if (oldValue.Value.Element !.Equals(newValue))
{
newList.Add(new MList <T> .RowIdElement(newValue, rowId, oldValue.Value.OldIndex));
}
else
{
newList.Add(new MList <T> .RowIdElement(newValue));
}
}
}
}
public object?DecodeValue(PredictorColumnBase column, List <PredictorCodification> codifications, float[] outputValues, PredictionOptions options)
{
var c = codifications.SingleEx();
return(ReflectionTools.ChangeType(outputValues[c.Index], column.Token.Type));
}
public override object?DecodeSingleValue(float value, PredictorCodification c)
{
var newValue = (float)Math.Exp((double)value);
return(ReflectionTools.ChangeType(newValue, c.Column.Token.Type));
}
public override object?DecodeSingleValue(float value, PredictorCodification c)
{
var newValue = c.Min !.Value + ((c.Max !.Value - c.Min.Value) * value);
return(ReflectionTools.ChangeType(newValue, c.Column.Token.Type));
}
public override object?DecodeSingleValue(float value, PredictorCodification c)
{
var newValue = c.Average !+(c.StdDev !*value);
return(ReflectionTools.ChangeType(newValue, c.Column.Token.Type));
}
static MList <T> ReadJsonInternal <T>(JsonReader reader, IMListPrivate <T> existingValue, JsonSerializer serializer)
{
var dic = existingValue == null ? new Dictionary <PrimaryKey, MList <T> .RowIdElement>() :
existingValue.InnerList.Where(a => a.RowId.HasValue).ToDictionary(a => a.RowId.Value, a => a);
var newList = new List <MList <T> .RowIdElement>();
var pr = JsonSerializerExtensions.CurrentPropertyRoute;
var elementPr = pr.Add("Item");
var rowIdType = GetRowIdTypeFromAttribute(pr);
reader.Assert(JsonToken.StartArray);
reader.Read();
using (JsonSerializerExtensions.SetCurrentPropertyRoute(elementPr))
{
while (reader.TokenType == JsonToken.StartObject)
{
reader.Read();
reader.Assert(JsonToken.PropertyName);
if (((string)reader.Value) != "rowId")
{
throw new JsonSerializationException($"member 'rowId' expected in {reader.Path}");
}
reader.Read();
var rowIdValue = reader.Value;
reader.Read();
reader.Assert(JsonToken.PropertyName);
if (((string)reader.Value) != "element")
{
throw new JsonSerializationException($"member 'element' expected in {reader.Path}");
}
reader.Read();
if (rowIdValue != null && !rowIdValue.Equals(GraphExplorer.DummyRowId.Object))
{
var rowId = new PrimaryKey((IComparable)ReflectionTools.ChangeType(rowIdValue, rowIdType));
var oldValue = dic.TryGetS(rowId);
if (oldValue == null)
{
T newValue = (T)serializer.DeserializeValue(reader, typeof(T), null);
newList.Add(new MList <T> .RowIdElement(newValue, rowId, null));
}
else
{
T newValue = (T)serializer.DeserializeValue(reader, typeof(T), oldValue.Value.Element);
if (oldValue.Value.Element.Equals(newValue))
{
newList.Add(new MList <T> .RowIdElement(newValue, rowId, oldValue.Value.OldIndex));
}
else
{
newList.Add(new MList <T> .RowIdElement(newValue));
}
}
}
else
{
var newValue = (T)serializer.DeserializeValue(reader, typeof(T), null);
newList.Add(new MList <T> .RowIdElement(newValue));
}
reader.Read();
reader.Assert(JsonToken.EndObject);
reader.Read();
}
}
reader.Assert(JsonToken.EndArray);
if (existingValue == null) //Strange case...
{
if (newList.IsEmpty())
{
return(null);
}
else
{
existingValue = new MList <T>();
}
}
bool orderMatters = GetPreserveOrderFromAttribute(pr);
if (!existingValue.IsEqualTo(newList, orderMatters))
{
EntityJsonConverter.AssertCanWrite(pr);
existingValue.AssignMList(newList);
}
return((MList <T>)existingValue);
}
public Filter(QueryToken token, FilterOperation operation, object value)
{
this.token = token;
this.operation = operation;
this.value = ReflectionTools.ChangeType(value, operation == FilterOperation.IsIn ? typeof(IEnumerable <>).MakeGenericType(Token.Type.Nullify()) : Token.Type);
}
public FilterCondition(QueryToken token, FilterOperation operation, object value)
{
this.Token = token;
this.Operation = operation;
this.Value = ReflectionTools.ChangeType(value, operation.IsList() ? typeof(IEnumerable <>).MakeGenericType(Token.Type.Nullify()) : Token.Type);
}
private static ConstantExpression Constant(decimal multiplier, Type type)
{
return(Expression.Constant(ReflectionTools.ChangeType(multiplier, type), type));
}
public void SavePredictions(PredictorTrainingContext ctx)
{
using (HeavyProfiler.Log("SavePredictions"))
{
var p = ctx.Predictor.ToLite();
var outputColumn = AssertOnlyOutput(ctx.Predictor);
var isCategorical = outputColumn.Encoding.Is(DefaultColumnEncodings.OneHot);
var keys = !ctx.Predictor.MainQuery.GroupResults ? null : ctx.Predictor.MainQuery.Columns.Where(c => !(c.Token.Token is AggregateToken)).ToList();
var key0 = keys?.ElementAtOrDefault(0);
var key1 = keys?.ElementAtOrDefault(1);
var key2 = keys?.ElementAtOrDefault(2);
using (HeavyProfiler.Log("Delete Old Predictions"))
{
ctx.ReportProgress($"Deleting old {typeof(PredictSimpleResultEntity).NicePluralName()}");
{
var query = Database.Query <PredictSimpleResultEntity>().Where(a => a.Predictor.Is(p));
int chunkSize = 5000;
var totalCount = query.Count();
var deleted = 0;
while (totalCount - deleted > 0)
{
int num = query.OrderBy(a => a.Id).Take(chunkSize).UnsafeDelete();
deleted += num;
ctx.ReportProgress($"Deleting old {typeof(PredictSimpleResultEntity).NicePluralName()}", deleted / (decimal)totalCount);
}
}
}
using (HeavyProfiler.Log("SavePredictions"))
{
ctx.ReportProgress($"Creating {typeof(PredictSimpleResultEntity).NicePluralName()}");
{
var dictionary = ctx.ToPredictDictionaries();
var toInsert = new List <PredictSimpleResultEntity>();
var pc = PredictorPredictLogic.CreatePredictContext(ctx.Predictor);
int grIndex = 0;
foreach (var gr in dictionary.Chunk(PredictionBatchSize))
{
using (HeavyProfiler.LogNoStackTrace("Group"))
{
ctx.ReportProgress($"Creating {typeof(PredictSimpleResultEntity).NicePluralName()}", (grIndex++ *PredictionBatchSize) / (decimal)dictionary.Count);
var inputs = gr.Select(a => a.Value).ToList();
var outputs = pc.Algorithm.PredictMultiple(pc, inputs);
using (HeavyProfiler.LogNoStackTrace("Create SimpleResults"))
{
for (int i = 0; i < inputs.Count; i++)
{
PredictDictionary input = inputs[i];
PredictDictionary output = outputs[i];
object?inValue = input.MainQueryValues.GetOrThrow(outputColumn);
object?outValue = output.MainQueryValues.GetOrThrow(outputColumn);
toInsert.Add(new PredictSimpleResultEntity
{
Predictor = p,
Target = ctx.Predictor.MainQuery.GroupResults ? null : input.Entity,
Type = ctx.Validation.Contains(gr[i].Key) ? PredictionSet.Validation : PredictionSet.Training,
Key0 = key0 == null ? null : input.MainQueryValues.GetOrThrow(key0)?.ToString(),
Key1 = key1 == null ? null : input.MainQueryValues.GetOrThrow(key1)?.ToString(),
Key2 = key2 == null ? null : input.MainQueryValues.GetOrThrow(key2)?.ToString(),
OriginalValue = isCategorical ? null : ReflectionTools.ChangeType <double?>(inValue),
OriginalCategory = isCategorical ? inValue?.ToString() : null,
PredictedValue = isCategorical ? null : ReflectionTools.ChangeType <double?>(outValue),
PredictedCategory = isCategorical ? outValue?.ToString() : null,
});
}
}
}
}
ctx.Predictor.RegressionTraining = isCategorical ? null : GetRegressionStats(toInsert.Where(a => a.Type == PredictionSet.Training).ToList());
ctx.Predictor.RegressionValidation = isCategorical ? null : GetRegressionStats(toInsert.Where(a => a.Type == PredictionSet.Validation).ToList());
ctx.Predictor.ClassificationTraining = !isCategorical ? null : GetClassificationStats(toInsert.Where(a => a.Type == PredictionSet.Training).ToList());
ctx.Predictor.ClassificationValidation = !isCategorical ? null : GetClassificationStats(toInsert.Where(a => a.Type == PredictionSet.Validation).ToList());
using (OperationLogic.AllowSave <PredictorEntity>())
ctx.Predictor.Save();
if (SaveAllResults)
{
var groups = toInsert.Chunk(PredictionBatchSize).ToList();
foreach (var iter in groups.Iterate())
{
ctx.ReportProgress($"Inserting {typeof(PredictSimpleResultEntity).NicePluralName()}", iter.Position / (decimal)groups.Count);
iter.Value.BulkInsert();
}
}
}
}
}
}
