void Complete(Table table)
{
using (HeavyProfiler.LogNoStackTrace("Complete", () => table.Type.Name))
using (var tr = HeavyProfiler.LogNoStackTrace("GetPrimaryKeyAttribute", () => table.Type.Name))
{
Type type = table.Type;
table.IdentityBehaviour = GetPrimaryKeyAttribute(type).IdentityBehaviour;
tr.Switch("GenerateTableName");
table.Name = GenerateTableName(type, Settings.TypeAttribute <TableNameAttribute>(type));
tr.Switch("GenerateCleanTypeName");
table.CleanTypeName = GenerateCleanTypeName(type);
tr.Switch("GenerateFields");
table.Fields = GenerateFields(PropertyRoute.Root(type), table, NameSequence.Void, forceNull: false, inMList: false);
tr.Switch("GenerateMixins");
table.Mixins = GenerateMixins(PropertyRoute.Root(type), table, NameSequence.Void);
tr.Switch("GenerateTemporal");
table.SystemVersioned = ToSystemVersionedInfo(Settings.TypeAttribute <SystemVersionedAttribute>(type), table.Name);
tr.Switch("GenerateColumns");
table.GenerateColumns();
}
}
protected internal override int ExecuteNonQuery(SqlPreCommandSimple preCommand)
{
using (SqlCeConnection con = EnsureConnection())
using (SqlCeCommand cmd = NewCommand(preCommand, con))
using (HeavyProfiler.Log("SQL", () => cmd.CommandText))
{
try
{
int result = cmd.ExecuteNonQuery();
return(result);
}
catch (SqlCeException ex)
{
var nex = HandleException(ex);
if (nex == ex)
{
throw;
}
throw nex;
}
}
}
public static List <Lite <T> > RetrieveAllLite <T>()
where T : Entity
{
try
{
using (HeavyProfiler.Log("DBRetrieve", () => "All Lite<{0}>".FormatWith(typeof(T).TypeName())))
{
var cc = GetCacheController <T>();
if (cc != null && GetFilterQuery <T>() == null)
{
return(cc.GetAllIds().Select(id => (Lite <T>) new LiteImp <T>(id, cc.GetToString(id))).ToList());
}
return(Database.Query <T>().Select(e => e.ToLite()).ToList());
}
}
catch (Exception e)
{
e.Data["type"] = typeof(T).TypeName();
throw;
}
}
private PredictDictionary GetPredictionDictionary(float[] outputValues, PredictorPredictContext ctx, PredictionOptions options)
{
using (HeavyProfiler.LogNoStackTrace("GetPredictionDictionary"))
{
return(new PredictDictionary(ctx.Predictor, options, null)
{
MainQueryValues = ctx.MainOutputCodifications.SelectDictionary(col => col,
(col, list) => Encodings.GetOrThrow(col.Encoding).DecodeValue(list.First().Column, list, outputValues, options)),
SubQueries = ctx.Predictor.SubQueries.ToDictionary(sq => sq, sq => new PredictSubQueryDictionary(sq)
{
SubQueryGroups = ctx.SubQueryOutputCodifications.TryGetC(sq)?.Groups.ToDictionary(
kvp => kvp.Key,
kvp => kvp.Value
.Where(a => a.Key.Usage == PredictorSubQueryColumnUsage.Output)
.ToDictionary(a => a.Key, a => Encodings.GetOrThrow(a.Key.Encoding).DecodeValue(a.Value.FirstEx().Column, a.Value, outputValues, options)),
ObjectArrayComparer.Instance
) ?? new Dictionary <object?[], Dictionary <PredictorSubQueryColumnEmbedded, object?> >(ObjectArrayComparer.Instance),
})
});
}
}
public async Task FillAsync(Dictionary <LookupToken, IEnumerable> lookups, IRetriever retriever, CancellationToken token)
{
Dictionary <K, MList <V> >?requests = (Dictionary <K, MList <V> >?)lookups.TryGetC(Token);
if (requests == null)
{
return;
}
using (HeavyProfiler.Log("SQL", () => Command.sp_executesql()))
using (var reader = await Executor.UnsafeExecuteDataReaderAsync(Command, token: token))
{
ProjectionRowEnumerator <KeyValuePair <K, MList <V> .RowIdElement> > enumerator = new ProjectionRowEnumerator <KeyValuePair <K, MList <V> .RowIdElement> >(reader.Reader, ProjectorExpression, lookups, retriever, token);
IEnumerable <KeyValuePair <K, MList <V> .RowIdElement> > enumerabe = new ProjectionRowEnumerable <KeyValuePair <K, MList <V> .RowIdElement> >(enumerator);
try
{
var lookUp = enumerabe.ToLookup(a => a.Key, a => a.Value);
foreach (var kvp in requests)
{
var results = lookUp[kvp.Key];
((IMListPrivate <V>)kvp.Value).AssignMList(results.ToList());
((IMListPrivate <V>)kvp.Value).InnerListModified(results.Select(a => a.Element).ToList(), null);
retriever.ModifiablePostRetrieving(kvp.Value);
}
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
FieldReaderException fieldEx = enumerator.Reader.CreateFieldReaderException(ex);
fieldEx.Command = Command;
fieldEx.Row = enumerator.Row;
fieldEx.Projector = ProjectorExpression;
throw fieldEx;
}
}
}
public async Task <HttpResponseMessage> ExecuteAuthorizationFilterAsync(HttpActionContext actionContext, CancellationToken cancellationToken, Func <Task <HttpResponseMessage> > continuation)
{
string action = ProfilerActionSplitterAttribute.GetActionDescription(actionContext);
try
{
using (TimeTracker.Start(action))
{
using (HeavyProfiler.Log("Web.API " + actionContext.Request.Method, () => actionContext.Request.RequestUri.ToString()))
{
//if (ProfilerLogic.SessionTimeout != null)
//{
// IDisposable sessionTimeout = Connector.CommandTimeoutScope(ProfilerLogic.SessionTimeout.Value);
// if (sessionTimeout != null)
// actionContext.Request.RegisterForDispose(sessionTimeout);
//}
actionContext.Request.Properties[SavedRequestKey] = await actionContext.Request.Content.ReadAsStringAsync();
using (Authenticate(actionContext))
{
using (GetCurrentCultures?.Invoke(actionContext))
{
if (actionContext.Response != null)
{
return(actionContext.Response);
}
return(await continuation());
}
}
}
}
}
finally
{
Statics.CleanThreadContextAndAssert();
}
}
private BlobContainerClient CalculateSuffixWithRenames(IFilePath fp)
{
using (HeavyProfiler.LogNoStackTrace("CalculateSuffixWithRenames"))
{
string suffix = CalculateSuffix(fp);
if (!suffix.HasText())
{
throw new InvalidOperationException("Suffix not set");
}
fp.SetPrefixPair(GetPrefixPair(fp));
var client = GetClient(fp);
if (CreateBlobContainerIfNotExists)
{
using (HeavyProfiler.LogNoStackTrace("AzureBlobStorage CreateIfNotExists"))
{
client.CreateIfNotExists();
CreateBlobContainerIfNotExists = false;
}
}
int i = 2;
fp.Suffix = suffix.Replace("\\", "/");
if (RenameAlgorithm != null)
{
while (HeavyProfiler.LogNoStackTrace("ExistBlob").Using(_ => client.ExistsBlob(fp.Suffix)))
{
fp.Suffix = RenameAlgorithm(suffix, i).Replace("\\", "/");
i++;
}
}
return(client);
}
}
public static List <T> RetrieveAll <T>()
where T : Entity
{
try
{
using (HeavyProfiler.Log("DBRetrieve", () => "All {0}".FormatWith(typeof(T).TypeName())))
{
var cc = GetCacheController <T>();
if (cc != null)
{
var filter = GetFilterQuery <T>();
if (filter == null || filter.InMemoryFunction != null)
{
List <T> result;
using (new EntityCache())
using (var r = EntityCache.NewRetriever())
{
result = cc.GetAllIds().Select(id => r.Request <T>(id)).ToList();
}
if (filter != null)
{
result = result.Where(filter.InMemoryFunction).ToList();
}
return(result);
}
}
return(Database.Query <T>().ToList());
}
}
catch (Exception e)
{
e.Data["type"] = typeof(T).TypeName();
throw;
}
}
public void Fill(Dictionary <LookupToken, IEnumerable> lookups, IRetriever retriever)
{
Dictionary <K, MList <V> > requests = (Dictionary <K, MList <V> >)lookups.TryGetC(Token);
if (requests == null)
{
return;
}
using (HeavyProfiler.Log("SQL", () => Command.Sql))
using (DbDataReader reader = Executor.UnsafeExecuteDataReader(Command))
{
ProjectionRowEnumerator <KeyValuePair <K, MList <V> .RowIdElement> > enumerator = new ProjectionRowEnumerator <KeyValuePair <K, MList <V> .RowIdElement> >(reader, ProjectorExpression, lookups, retriever);
IEnumerable <KeyValuePair <K, MList <V> .RowIdElement> > enumerabe = new ProjectionRowEnumerable <KeyValuePair <K, MList <V> .RowIdElement> >(enumerator);
try
{
var lookUp = enumerabe.ToLookup(a => a.Key, a => a.Value);
foreach (var kvp in requests)
{
var results = lookUp[kvp.Key];
((IMListPrivate <V>)kvp.Value).InnerList.AddRange(results);
((IMListPrivate <V>)kvp.Value).InnerListModified(results.Select(a => a.Element).ToList(), null);
retriever.ModifiablePostRetrieving(kvp.Value);
}
}
catch (Exception ex)
{
FieldReaderException fieldEx = enumerator.Reader.CreateFieldReaderException(ex);
fieldEx.Command = Command;
fieldEx.Row = enumerator.Row;
fieldEx.Projector = ProjectorExpression;
throw fieldEx;
}
}
}
public static string GetToStr <T>(PrimaryKey id)
where T : Entity
{
try
{
using (HeavyProfiler.Log("DBRetrieve", () => "GetToStr<{0}>".FormatWith(typeof(T).TypeName())))
{
var cc = GetCacheController <T>();
if (cc != null && GetFilterQuery <T>() == null)
{
return(cc.GetToString(id));
}
return(Database.Query <T>().Where(a => a.Id == id).Select(a => a.ToString()).FirstEx());
}
}
catch (Exception e)
{
e.Data["type"] = typeof(T).TypeName();
e.Data["id"] = id;
throw;
}
}
protected internal override int ExecuteNonQuery(SqlPreCommandSimple preCommand, CommandType commandType)
{
using (SqlConnection? con = EnsureConnection())
using (SqlCommand cmd = NewCommand(preCommand, con, commandType))
using (HeavyProfiler.Log("SQL", () => preCommand.sp_executesql()))
{
try
{
int result = cmd.ExecuteNonQuery();
return(result);
}
catch (Exception ex)
{
var nex = HandleException(ex, preCommand);
if (nex == ex)
{
throw;
}
throw nex;
}
}
}
public List <StackTraceTS>?StackTrace(string fullIndex)
{
ProfilerPermission.ViewHeavyProfiler.AssertAuthorized();
var e = HeavyProfiler.Find(fullIndex);
if (e == null)
{
return(null);
}
if (e.ExternalStackTrace != null)
{
return((from est in e.ExternalStackTrace
select new StackTraceTS
{
Method = est.MethodName,
Color = est.Namespace == null ? null : ColorExtensions.ToHtmlColor(est.Namespace.Split('.').Take(2).ToString(".").GetHashCode()),
Type = est.Type,
Namespace = est.Namespace !,
FileName = est.FileName,
LineNumber = est.LineNumber ?? 0
}).ToList());
protected internal override void BulkCopy(DataTable dt, ObjectName destinationTable, SqlBulkCopyOptions options, int?timeout)
{
using (SqlConnection con = EnsureConnection())
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(
options.HasFlag(SqlBulkCopyOptions.UseInternalTransaction) ? con : (SqlConnection)Transaction.CurrentConnection,
options,
options.HasFlag(SqlBulkCopyOptions.UseInternalTransaction) ? null : (SqlTransaction)Transaction.CurrentTransaccion))
using (HeavyProfiler.Log("SQL", () => destinationTable.ToString() + " Rows:" + dt.Rows.Count))
{
if (timeout.HasValue)
{
bulkCopy.BulkCopyTimeout = timeout.Value;
}
foreach (DataColumn c in dt.Columns)
{
bulkCopy.ColumnMappings.Add(new SqlBulkCopyColumnMapping(c.ColumnName, c.ColumnName));
}
bulkCopy.DestinationTableName = destinationTable.ToString();
bulkCopy.WriteToServer(dt);
}
}
protected internal override void BulkCopy(DataTable dt, ObjectName destinationTable, SqlBulkCopyOptions options, int?timeout)
{
EnsureConnectionRetry(con =>
{
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(
options.HasFlag(SqlBulkCopyOptions.UseInternalTransaction) ? con : (SqlConnection)Transaction.CurrentConnection !,
options,
options.HasFlag(SqlBulkCopyOptions.UseInternalTransaction) ? null : (SqlTransaction)Transaction.CurrentTransaccion !))
using (HeavyProfiler.Log("SQL", () => destinationTable.ToString() + " Rows:" + dt.Rows.Count))
{
bulkCopy.BulkCopyTimeout = timeout ?? Connector.ScopeTimeout ?? this.CommandTimeout ?? bulkCopy.BulkCopyTimeout;
foreach (var c in dt.Columns.Cast <DataColumn>())
{
bulkCopy.ColumnMappings.Add(new SqlBulkCopyColumnMapping(c.ColumnName, c.ColumnName));
}
bulkCopy.DestinationTableName = destinationTable.ToString();
bulkCopy.WriteToServer(dt);
return(0);
}
});
}
public static PropertyInfo TryFindPropertyInfo(FieldInfo fi)
{
using (HeavyProfiler.LogNoStackTrace("TryFindPropertyInfo", () => fi.Name))
{
const BindingFlags flags = BindingFlags.IgnoreCase | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic;
string propertyName = null;
if (fi.Name.StartsWith("<"))
{
CheckSignumProcessed(fi);
propertyName = fi.Name.After('<').Before('>');
}
else
{
propertyName = fi.Name.FirstUpper();
}
var result = fi.DeclaringType.GetProperty(propertyName, flags, null, null, new Type[0], null);
if (result != null)
{
return(result);
}
foreach (Type i in fi.DeclaringType.GetInterfaces())
{
result = fi.DeclaringType.GetProperty(i.FullName + "." + propertyName, flags);
if (result != null)
{
return(result);
}
}
return(null);
}
}
static string SerializeToken(AuthToken token)
{
using (HeavyProfiler.LogNoStackTrace("SerializeToken"))
{
var array = new MemoryStream().Using(ms =>
{
using (DeflateStream ds = new DeflateStream(ms, CompressionMode.Compress))
{
using (Utf8JsonWriter writer = new Utf8JsonWriter(ds))
{
JsonSerializer.Serialize(writer, token, EntityJsonContext.FullJsonSerializerOptions);
}
}
return(ms.ToArray());
});
//var str = Encoding.UTF8.GetString(array);
array = Encrypt(array);
return(Convert.ToBase64String(array));
}
}
public static int Execute(this IUpdateable update, string message = null)
{
if (message != null)
{
return(SafeConsole.WaitRows(message == "auto" ? UnsafeMessage(update) : message,
() => update.Execute(message: null)));
}
using (HeavyProfiler.Log("DBUnsafeUpdate", () => update.EntityType.TypeName()))
{
if (update == null)
{
throw new ArgumentNullException("update");
}
using (Transaction tr = new Transaction())
{
Schema.Current.OnPreUnsafeUpdate(update);
int rows = DbQueryProvider.Single.Update(update, sql => (int)sql.ExecuteScalar());
return(tr.Commit(rows));
}
}
}
public ExtensionInfo Register <E, S>(Expression <Func <E, S> > lambdaToMethodOrProperty, Func <string>?niceName = null)
{
using (HeavyProfiler.LogNoStackTrace("RegisterExpression"))
{
if (lambdaToMethodOrProperty.Body.NodeType == ExpressionType.Call)
{
var mi = ReflectionTools.GetMethodInfo(lambdaToMethodOrProperty);
AssertExtensionMethod(mi);
return(Register <E, S>(lambdaToMethodOrProperty, niceName ?? (() => mi.Name.NiceName()), mi.Name));
}
else if (lambdaToMethodOrProperty.Body.NodeType == ExpressionType.MemberAccess)
{
var pi = ReflectionTools.GetPropertyInfo(lambdaToMethodOrProperty);
return(Register <E, S>(lambdaToMethodOrProperty, niceName ?? (() => pi.NiceName()), pi.Name));
}
else
{
throw new InvalidOperationException("argument 'lambdaToMethodOrProperty' should be a simple lambda calling a method or property: {0}".FormatWith(lambdaToMethodOrProperty.ToString()));
}
}
}
protected internal override DataSet ExecuteDataSet(SqlPreCommandSimple preCommand)
{
using (SqlCeConnection con = EnsureConnection())
using (SqlCeCommand cmd = NewCommand(preCommand, con))
using (HeavyProfiler.Log("SQL", () => cmd.CommandText))
{
try
{
SqlCeDataAdapter da = new SqlCeDataAdapter(cmd);
DataSet result = new DataSet();
da.Fill(result);
return(result);
}
catch (SqlCeException ex)
{
var nex = HandleException(ex);
if (nex == ex)
{
throw;
}
throw nex;
}
}
}
public static void StartRunningProcesses()
{
if (running)
{
throw new InvalidOperationException("ProcessLogic is running");
}
using (ExecutionContext.SuppressFlow())
Task.Factory.StartNew(() =>
{
var database = Schema.Current.Table(typeof(ProcessEntity)).Name.Schema?.Database;
SystemEventLogLogic.Log("Start ProcessRunner");
ExceptionEntity exception = null;
using (AuthLogic.Disable())
{
try
{
running = true;
(from p in Database.Query <ProcessEntity>()
where p.IsMine() && (p.State == ProcessState.Executing || p.State == ProcessState.Suspending || p.State == ProcessState.Suspended) ||
p.IsShared() && p.State == ProcessState.Suspended
select p).SetAsQueued();
CancelNewProcesses = new CancellationTokenSource();
autoResetEvent.Set();
timerNextExecution = new Timer(ob => WakeUp("TimerNextExecution", null), // main timer
null,
Timeout.Infinite,
Timeout.Infinite);
if (!CacheLogic.WithSqlDependency)
{
timerPeriodic = new Timer(ob => WakeUp("TimerPeriodic", null), null, PoolingPeriodMilliseconds, PoolingPeriodMilliseconds);
}
while (autoResetEvent.WaitOne())
{
if (CancelNewProcesses.IsCancellationRequested)
{
return;
}
using (HeavyProfiler.Log("PWL", () => "Process Runner"))
{
(from p in Database.Query <ProcessEntity>()
where p.State == ProcessState.Planned && p.PlannedDate <= TimeZoneManager.Now
select p).SetAsQueued();
var list = Database.Query <ProcessEntity>()
.Where(p => p.IsMine() || p.IsShared())
.Where(p => p.State == ProcessState.Planned)
.Select(p => p.PlannedDate)
.ToListWakeup("Planned dependency");
SetNextPannedExecution(list.Min());
lock (executing)
{
int remaining = MaxDegreeOfParallelism - executing.Count;
if (remaining > 0)
{
retry:
var queued = Database.Query <ProcessEntity>()
.Where(p => p.State == ProcessState.Queued)
.Where(p => p.IsMine() || p.IsShared())
.Select(a => new { Process = a.ToLite(), a.QueuedDate, a.MachineName })
.ToListWakeup("Planned dependency");
var afordable = queued
.OrderByDescending(p => p.MachineName == Environment.MachineName)
.OrderBy(a => a.QueuedDate)
.Take(remaining).ToList();
var taken = afordable.Where(p => p.MachineName == ProcessEntity.None).Select(a => a.Process).ToList();
if (taken.Any())
{
using (Transaction tr = Transaction.ForceNew())
{
Database.Query <ProcessEntity>()
.Where(p => taken.Contains(p.ToLite()) && p.MachineName == ProcessEntity.None)
.UnsafeUpdate()
.Set(p => p.MachineName, p => Environment.MachineName)
.Set(p => p.ApplicationName, p => Schema.Current.ApplicationName)
.Execute();
tr.Commit();
}
goto retry;
}
foreach (var pair in afordable)
{
ProcessEntity pro = pair.Process.Retrieve();
IProcessAlgorithm algorithm = ProcessLogic.GetProcessAlgorithm(pro.Algorithm);
ExecutingProcess executingProcess = new ExecutingProcess(algorithm, pro);
executing.Add(pro.ToLite(), executingProcess);
executingProcess.TakeForThisMachine();
using (ExecutionContext.SuppressFlow())
Task.Run(() =>
{
try
{
executingProcess.Execute();
}
catch (Exception ex)
{
try
{
ex.LogException(edn =>
{
edn.ControllerName = "ProcessWorker";
edn.ActionName = executingProcess.CurrentProcess.ToLite().Key();
});
}
catch { }
}
finally
{
lock (executing)
{
executing.Remove(pro.ToLite());
WakeUp("Process ended", null);
}
}
});
}
var suspending = Database.Query <ProcessEntity>()
.Where(p => p.State == ProcessState.Suspending)
.Where(p => p.IsMine())
.Select(a => a.ToLite())
.ToListWakeup("Suspending dependency");
foreach (var s in suspending)
{
ExecutingProcess execProc = executing.GetOrThrow(s);
if (execProc.CurrentProcess.State != ProcessState.Finished)
{
execProc.CurrentProcess = s.Retrieve();
execProc.CancelationSource.Cancel();
}
}
}
}
}
}
}
catch (ThreadAbortException)
{
//Ignore
}
catch (Exception e)
{
try
{
exception = e.LogException(edn =>
{
edn.ControllerName = "ProcessWorker";
edn.ActionName = "MainLoop";
});
}
catch { }
}
finally
{
lock (executing)
executing.Clear();
SystemEventLogLogic.Log("Stop ProcessRunner", exception);
running = false;
}
}
}, TaskCreationOptions.LongRunning);
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
public void Configure(IApplicationBuilder app, IWebHostEnvironment env, IHostApplicationLifetime lifetime)
{
app.UseDeveloperExceptionPage();
app.UseStaticFiles();
//HeavyProfiler.Enabled = true;
using (HeavyProfiler.Log("Startup"))
using (var log = HeavyProfiler.Log("Initial"))
{
DynamicCode.CodeGenDirectory = env.ContentRootPath + "/CodeGen";
Starter.Start(Configuration.GetConnectionString("ConnectionString"), Configuration.GetValue <bool>("IsPostgres"), detectSqlVersion: false);
Statics.SessionFactory = new ScopeSessionFactory(new VoidSessionFactory());
log.Switch("WebStart");
WebStart(app, env, lifetime);
log.Switch("UseEndpoints");
//Enable middleware to serve generated Swagger as a JSON endpoint.
app.UseSwagger();
app.UseSwaggerUI(c =>
{
c.SwaggerEndpoint("../swagger/v1/swagger.json", "Southwind API");
});//Swagger Configure
app.UseWhen(req => req.Request.Path.StartsWithSegments("/api/reflection/types"), builder =>
{
builder.UseResponseCompression();
});
app.UseRouting();
app.UseEndpoints(routes =>
{
routes.MapControllers();
routes.MapControllerRoute(
name: "spa-fallback",
pattern: "{*url}",
constraints: new { url = new NoAPIContraint() },
defaults: new { controller = "Home", action = "Index" });
});
}
SignumInitializeFilterAttribute.InitializeDatabase = () =>
{
using (HeavyProfiler.Log("Startup"))
using (var log = HeavyProfiler.Log("Initial"))
{
log.Switch("Initialize");
using (AuthLogic.Disable())
Schema.Current.Initialize();
if (Configuration.GetValue <bool>("StartBackgroundProcesses"))
{
log.Switch("StartRunningProcesses");
ProcessRunnerLogic.StartRunningProcesses(5 * 1000);
log.Switch("StartScheduledTasks");
SchedulerLogic.StartScheduledTasks();
log.Switch("StartRunningEmailSenderAsync");
AsyncEmailSenderLogic.StartRunningEmailSenderAsync(5 * 1000);
}
SystemEventServer.LogStartStop(app, lifetime);
}
};
}
public void ExecuteDataReaderDependency(SqlPreCommandSimple preCommand, OnChangeEventHandler change, Action reconect, Action <FieldReader> forEach, CommandType commandType)
{
bool reconected = false;
retry:
try
{
using (SqlConnection con = EnsureConnection())
using (SqlCommand cmd = NewCommand(preCommand, con, commandType))
using (HeavyProfiler.Log("SQL-Dependency"))
using (HeavyProfiler.Log("SQL", () => preCommand.PlainSql()))
{
try
{
if (change != null)
{
SqlDependency dep = new SqlDependency(cmd);
dep.OnChange += change;
}
using (SqlDataReader reader = cmd.ExecuteReader())
{
FieldReader fr = new FieldReader(reader);
int row = -1;
try
{
while (reader.Read())
{
row++;
forEach(fr);
}
}
catch (Exception ex)
{
FieldReaderException fieldEx = fr.CreateFieldReaderException(ex);
fieldEx.Command = preCommand;
fieldEx.Row = row;
throw fieldEx;
}
}
}
catch (Exception ex)
{
var nex = HandleException(ex, preCommand);
if (nex == ex)
{
throw;
}
throw nex;
}
}
}
catch (InvalidOperationException ioe)
{
if (ioe.Message.Contains("SqlDependency.Start()") && !reconected)
{
reconect();
reconected = true;
goto retry;
}
throw;
}
}
public static int BulkInsertMListTable <E, V>(
this IEnumerable <MListElement <E, V> > mlistElements,
Expression <Func <E, MList <V> > > mListProperty,
SqlBulkCopyOptions copyOptions = SqlBulkCopyOptions.Default,
int?timeout = null,
bool?updateParentTicks = null, /*Needed for concurrency and Temporal tables*/
string?message = null)
where E : Entity
{
using (HeavyProfiler.Log(nameof(BulkInsertMListTable), () => $"{mListProperty} ({typeof(MListElement<E, V>).TypeName()})"))
{
if (message != null)
{
return(SafeConsole.WaitRows(message == "auto" ? $"BulkInsering MList<{ typeof(V).TypeName()}> in { typeof(E).TypeName()}" : message,
() => BulkInsertMListTable(mlistElements, mListProperty, copyOptions, timeout, updateParentTicks, message: null)));
}
if (copyOptions.HasFlag(SqlBulkCopyOptions.UseInternalTransaction))
{
throw new InvalidOperationException("BulkInsertDisableIdentity not compatible with UseInternalTransaction");
}
var mlistTable = ((FieldMList)Schema.Current.Field(mListProperty)).TableMList;
if (updateParentTicks == null)
{
updateParentTicks = mlistTable.PrimaryKey.Type != typeof(Guid) && mlistTable.BackReference.ReferenceTable.Ticks != null;
}
var maxRowId = updateParentTicks.Value ? Database.MListQuery(mListProperty).Max(a => (PrimaryKey?)a.RowId) : null;
DataTable dt = new DataTable();
var columns = mlistTable.Columns.Values.Where(c => !(c is SystemVersionedInfo.SqlServerPeriodColumn) && !c.IdentityBehaviour).ToList();
foreach (var c in columns)
{
dt.Columns.Add(new DataColumn(c.Name, ConvertType(c.Type)));
}
var list = mlistElements.ToList();
foreach (var e in list)
{
dt.Rows.Add(mlistTable.BulkInsertDataRow(e.Parent, e.Element, e.Order));
}
using (Transaction tr = new Transaction())
{
Schema.Current.OnPreBulkInsert(typeof(E), inMListTable: true);
Executor.BulkCopy(dt, columns, mlistTable.Name, copyOptions, timeout);
var result = list.Count;
if (updateParentTicks.Value)
{
Database.MListQuery(mListProperty)
.Where(a => maxRowId == null || a.RowId > maxRowId)
.Select(a => a.Parent)
.UnsafeUpdate()
.Set(e => e.Ticks, a => TimeZoneManager.Now.Ticks)
.Execute();
}
return(tr.Commit(result));
}
}
}
public static int BulkInsertTable <T>(IEnumerable <T> entities,
SqlBulkCopyOptions copyOptions = SqlBulkCopyOptions.Default,
bool preSaving = true,
bool validateFirst = true,
bool disableIdentity = false,
int?timeout = null,
string?message = null)
where T : Entity
{
using (HeavyProfiler.Log(nameof(BulkInsertTable), () => typeof(T).TypeName()))
{
if (message != null)
{
return(SafeConsole.WaitRows(message == "auto" ? $"BulkInsering {entities.Count()} {typeof(T).TypeName()}" : message,
() => BulkInsertTable(entities, copyOptions, preSaving, validateFirst, disableIdentity, timeout, message: null)));
}
if (disableIdentity)
{
copyOptions |= SqlBulkCopyOptions.KeepIdentity;
}
if (copyOptions.HasFlag(SqlBulkCopyOptions.UseInternalTransaction))
{
throw new InvalidOperationException("BulkInsertDisableIdentity not compatible with UseInternalTransaction");
}
var list = entities.ToList();
if (preSaving)
{
Saver.PreSaving(() => GraphExplorer.FromRoots(list));
}
if (validateFirst)
{
Validate <T>(list);
}
var t = Schema.Current.Table <T>();
bool disableIdentityBehaviour = copyOptions.HasFlag(SqlBulkCopyOptions.KeepIdentity);
DataTable dt = new DataTable();
var columns = t.Columns.Values.Where(c => !(c is SystemVersionedInfo.SqlServerPeriodColumn) && (disableIdentityBehaviour || !c.IdentityBehaviour)).ToList();
foreach (var c in columns)
{
dt.Columns.Add(new DataColumn(c.Name, ConvertType(c.Type)));
}
using (disableIdentityBehaviour ? Administrator.DisableIdentity(t, behaviourOnly: true) : null)
{
foreach (var e in list)
{
if (!e.IsNew)
{
throw new InvalidOperationException("Entites should be new");
}
t.SetToStrField(e);
dt.Rows.Add(t.BulkInsertDataRow(e));
}
}
using (Transaction tr = new Transaction())
{
Schema.Current.OnPreBulkInsert(typeof(T), inMListTable: false);
Executor.BulkCopy(dt, columns, t.Name, copyOptions, timeout);
foreach (var item in list)
{
item.SetNotModified();
}
return(tr.Commit(list.Count));
}
}
}
public static CompilationResult Compile(IEnumerable <MetadataReference> references, string code)
{
return(resultCache.GetOrAdd(code, (Func <string, CompilationResult>)(_ =>
{
using (HeavyProfiler.Log("COMPILE", () => code))
{
try
{
var tree = SyntaxFactory.ParseSyntaxTree(code, options: new CSharpParseOptions(LanguageVersion.CSharp8));
var compilation = CSharpCompilation.Create($"{Guid.NewGuid()}.dll")
.WithOptions(new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary, nullableContextOptions: NullableContextOptions.Enable))
.AddReferences(references)
.AddSyntaxTrees(tree);
using (MemoryStream ms = new MemoryStream())
{
var emit = compilation.Emit(ms);
if (!emit.Success)
{
var lines = code.Lines();
var errors = emit.Diagnostics.Where(a => a.DefaultSeverity == DiagnosticSeverity.Error);
return new CompilationResult
{
CompilationErrors = errors.Count() + " Errors:\r\n" + errors.ToString(e =>
{
var line = e.Location.GetLineSpan().StartLinePosition.Line;
return "Line {0}: {1}".FormatWith(line, e.GetMessage() + "\r\n" + lines[line]);
}, "\r\n\r\n")
};
}
if (DynamicCode.GetCustomErrors != null)
{
var allCustomErrors = DynamicCode.GetCustomErrors.GetInvocationListTyped()
.SelectMany((Func <string, List <CustomCompilerError> > a) => a(code) ?? Enumerable.Empty <CustomCompilerError>()).ToList();
if (allCustomErrors.Any())
{
var lines = code.Split('\n');
return new CompilationResult
{
CompilationErrors = allCustomErrors.Count() + " Errors:\r\n" + allCustomErrors.ToString(e => {
return "Line {0}: {1}".FormatWith(e.Line, e.ErrorText) + "\r\n" + lines[e.Line - 1];
}, "\r\n\r\n")
};
}
}
ms.Seek(0, SeekOrigin.Begin);
var assembly = AssemblyLoadContext.Default.LoadFromStream(ms);
Type type = assembly.GetTypes().Where(a => typeof(T).IsAssignableFrom(a)).SingleEx();
T algorithm = (T)assembly.CreateInstance(type.FullName !) !;
return new CompilationResult {
Algorithm = algorithm
};
}
}
catch (Exception e)
{
return new CompilationResult {
CompilationErrors = e.Message
};
}
}
})));
}
internal virtual Dictionary <Guid, IntegrityCheck> EntityIntegrityCheckBase()
{
using (HeavyProfiler.LogNoStackTrace("EntityIntegrityCheckBase", () => GetType().Name))
return(GraphExplorer.EntityIntegrityCheck(GraphExplorer.FromRootEntity(this)));
}
public static void Save(Entity[] entities)
{
if (entities == null || entities.Any(e => e == null))
{
throw new ArgumentNullException("entity");
}
using (var log = HeavyProfiler.LogNoStackTrace("PreSaving"))
{
Schema schema = Schema.Current;
DirectedGraph <Modifiable> modifiables = PreSaving(() => GraphExplorer.FromRoots(entities));
HashSet <Entity> wasNew = modifiables.OfType <Entity>().Where(a => a.IsNew).ToHashSet(ReferenceEqualityComparer <Entity> .Default);
HashSet <Entity> wasSelfModified = modifiables.OfType <Entity>().Where(a => a.Modified == ModifiedState.SelfModified).ToHashSet(ReferenceEqualityComparer <Entity> .Default);
log.Switch("Integrity");
var error = GraphExplorer.FullIntegrityCheck(modifiables);
if (error != null)
{
#if DEBUG
throw new IntegrityCheckException(error.WithEntities(modifiables));
#else
throw new IntegrityCheckException(error);
#endif
}
log.Switch("Graph");
GraphExplorer.PropagateModifications(modifiables.Inverse());
//colapsa modifiables (collections and embeddeds) keeping indentifiables only
DirectedGraph <Entity> identifiables = GraphExplorer.ColapseIdentifiables(modifiables);
foreach (var node in identifiables)
{
schema.OnSaving(node);
}
//Remove all the edges that doesn't mean a dependency
identifiables.RemoveEdges(identifiables.Edges.Where(e => !e.To.IsNew).ToList());
//Remove all the nodes that are not modified
List <Entity> notModified = identifiables.Where(node => !node.IsGraphModified).ToList();
notModified.ForEach(node => identifiables.RemoveFullNode(node, None));
log.Switch("SaveGroups");
SaveGraph(schema, identifiables);
foreach (var node in identifiables)
{
schema.OnSaved(node, new SavedEventArgs
{
IsRoot = entities.Contains(node),
WasNew = wasNew.Contains(node),
WasSelfModified = wasSelfModified.Contains(node),
});
}
EntityCache.Add(identifiables);
EntityCache.Add(notModified);
GraphExplorer.CleanModifications(modifiables);
}
}
public override void WriteJson(JsonWriter writer, object?value, JsonSerializer serializer)
{
using (HeavyProfiler.LogNoStackTrace("WriteJson", () => value !.GetType().Name))
{
PropertyRoute pr = GetCurrentPropertyRoute(value !);
ModifiableEntity mod = (ModifiableEntity)value !;
writer.WriteStartObject();
if (mod is Entity entity)
{
writer.WritePropertyName("Type");
writer.WriteValue(TypeLogic.TryGetCleanName(mod.GetType()));
writer.WritePropertyName("id");
writer.WriteValue(entity.IdOrNull == null ? null : entity.Id.Object);
if (entity.IsNew)
{
writer.WritePropertyName("isNew");
writer.WriteValue(true);
}
if (Schema.Current.Table(entity.GetType()).Ticks != null)
{
writer.WritePropertyName("ticks");
writer.WriteValue(entity.Ticks.ToString());
}
}
else
{
writer.WritePropertyName("Type");
writer.WriteValue(mod.GetType().Name);
}
if (!(mod is MixinEntity))
{
writer.WritePropertyName("toStr");
writer.WriteValue(mod.ToString());
}
writer.WritePropertyName("modified");
writer.WriteValue(mod.Modified == ModifiedState.Modified || mod.Modified == ModifiedState.SelfModified);
foreach (var kvp in PropertyConverter.GetPropertyConverters(value !.GetType()))
{
WriteJsonProperty(writer, serializer, mod, kvp.Key, kvp.Value, pr);
}
if (mod.Mixins.Any())
{
writer.WritePropertyName("mixins");
writer.WriteStartObject();
foreach (var m in mod.Mixins)
{
var prm = pr.Add(m.GetType());
using (JsonSerializerExtensions.SetCurrentPropertyRoute(prm))
{
writer.WritePropertyName(m.GetType().Name);
serializer.Serialize(writer, m);
}
}
writer.WriteEndObject();
}
writer.WriteEndObject();
}
}
public static void RetrieveData(PredictorTrainingContext ctx)
{
using (HeavyProfiler.Log("RetrieveData"))
{
ctx.ReportProgress($"Executing MainQuery for {ctx.Predictor}");
QueryRequest mainQueryRequest = GetMainQueryRequest(ctx.Predictor.MainQuery);
ResultTable mainResult = QueryLogic.Queries.ExecuteQuery(mainQueryRequest);
ctx.MainQuery = new MainQuery
{
QueryRequest = mainQueryRequest,
ResultTable = mainResult,
};
if (!mainQueryRequest.GroupResults)
{
ctx.MainQuery.GetParentKey = (ResultRow row) => new object[] { row.Entity };
}
else
{
var rcs = mainResult.Columns.Where(a => !(a.Column.Token is AggregateToken)).ToArray();
ctx.MainQuery.GetParentKey = (ResultRow row) => row.GetValues(rcs);
}
var algorithm = PredictorLogic.Algorithms.GetOrThrow(ctx.Predictor.Algorithm);
ctx.SubQueries = new Dictionary <PredictorSubQueryEntity, SubQuery>();
foreach (var sqe in ctx.Predictor.SubQueries)
{
ctx.ReportProgress($"Executing SubQuery {sqe}");
QueryRequest queryGroupRequest = ToMultiColumnQuery(ctx.Predictor.MainQuery, sqe);
ResultTable groupResult = QueryLogic.Queries.ExecuteQuery(queryGroupRequest);
var pairs = groupResult.Columns.Zip(sqe.Columns, (rc, sqc) => (rc, sqc)).ToList();
var parentKeys = pairs.Extract(a => a.sqc.Usage == PredictorSubQueryColumnUsage.ParentKey).Select(a => a.rc).ToArray();
var splitKeys = pairs.Extract(a => a.sqc.Usage == PredictorSubQueryColumnUsage.SplitBy).Select(a => a.rc).ToArray();
var values = pairs.Select(a => a.rc).ToArray();
var groupedValues = groupResult.Rows.AgGroupToDictionary(
row => row.GetValues(parentKeys),
gr => gr.ToDictionaryEx(
row => row.GetValues(splitKeys),
row => row.GetValues(values),
ObjectArrayComparer.Instance));
ctx.SubQueries.Add(sqe, new SubQuery
{
SubQueryEntity = sqe,
QueryGroupRequest = queryGroupRequest,
ResultTable = groupResult,
GroupedValues = groupedValues,
SplitBy = splitKeys,
ValueColumns = values,
ColumnIndexToValueIndex = values.Select((r, i) => KeyValuePair.Create(r.Index, i)).ToDictionary()
});
}
ctx.ReportProgress($"Creating Columns");
var codifications = new List <PredictorCodification>();
using (HeavyProfiler.Log("MainQuery"))
{
for (int i = 0; i < mainResult.Columns.Length; i++)
{
var col = ctx.Predictor.MainQuery.Columns[i];
using (HeavyProfiler.Log("Columns", () => col.Token.Token.ToString()))
{
var mainCol = new PredictorColumnMain(col, i);
var mainCodifications = algorithm.GenerateCodifications(col.Encoding, mainResult.Columns[i], mainCol);
codifications.AddRange(mainCodifications);
}
}
}
foreach (var sq in ctx.SubQueries.Values)
{
using (HeavyProfiler.Log("SubQuery", () => sq.ToString() !))
{
var distinctKeys = sq.GroupedValues.SelectMany(a => a.Value.Keys).Distinct(ObjectArrayComparer.Instance).ToList();
distinctKeys.Sort(ObjectArrayComparer.Instance);
foreach (var ks in distinctKeys)
{
using (HeavyProfiler.Log("Keys", () => ks.ToString(k => k?.ToString(), ", ")))
{
foreach (var vc in sq.ValueColumns)
{
var col = sq.SubQueryEntity.Columns[vc.Index];
using (HeavyProfiler.Log("Columns", () => col.Token.Token.ToString()))
{
var subCol = new PredictorColumnSubQuery(col, vc.Index, sq.SubQueryEntity, ks);
var subQueryCodifications = algorithm.GenerateCodifications(col.Encoding, vc, subCol);
codifications.AddRange(subQueryCodifications);
}
}
}
}
}
}
ctx.SetCodifications(codifications.ToArray());
}
}
public override IDisposable GetResource(ResourceExecutingContext context)
{
return(HeavyProfiler.Log("Web.API " + context.HttpContext.Request.Method, () => context.HttpContext.Request.GetDisplayUrl()));
}
