public void Execute(ExecutingProcess ep)
{
var conf = (AutoconfigureNeuralNetworkEntity)ep.Data !;
var initial = conf.InitialPredictor.RetrieveAndForget();
Random r = conf.Seed == null ?
new Random():
new Random(conf.Seed.Value);
var mutationProbability = conf.InitialMutationProbability;
List <PredictorEntity> population = 0.To(conf.Population).Select(p => initial.ConstructFrom(PredictorOperation.Clone)).ToList();
population.ForEach(p => Mutate(p, conf, mutationProbability, r));
Dictionary <PredictorEntity, double> evaluatedPopulation = EvaluatePopulation(ep, conf, population, 0);
for (int gen = 1; gen < conf.Generations + 1; gen++)
{
population = CrossOverPopulation(evaluatedPopulation, initial, conf, r);
population.ForEach(p => Mutate(p, conf, mutationProbability, r));
evaluatedPopulation = EvaluatePopulation(ep, conf, population, gen);
}
}
public void Execute(ExecutingProcess executingProcess)
{
SMSSendPackageEntity package = (SMSSendPackageEntity)executingProcess.Data !;
executingProcess.ForEachLine(package.SMSMessages().Where(s => s.State == SMSMessageState.Created),
sms => sms.Execute(SMSMessageOperation.Send));
}
public void Execute(ExecutingProcess executingProcess)
{
SMSSendPackageEntity package = (SMSSendPackageEntity)executingProcess.Data;
executingProcess.ForEachLine(package.SMSMessages().Where(s => s.State == SMSMessageState.Created),
sms => sms.Execute(SMSMessageOperation.Send));
}
public void Execute(ExecutingProcess executingProcess)
{
NewsletterEntity newsletter = (NewsletterEntity)executingProcess.Data !;
var queryName = QueryLogic.ToQueryName(newsletter.Query !.Key);
QueryDescription qd = QueryLogic.Queries.QueryDescription(queryName);
List <QueryToken> list = new List <QueryToken>();
using (ExecutionMode.Global())
{
list.Add(QueryUtils.Parse("Entity", qd, 0));
list.Add(QueryUtils.Parse(".".Combine("Entity", "NewsletterDeliveries", "Element"), qd, SubTokensOptions.CanElement));
list.Add(QueryUtils.Parse(".".Combine("Entity", "EmailOwnerData"), qd, 0));
TextTemplateParser.Parse(newsletter.Subject, qd, null).FillQueryTokens(list);
TextTemplateParser.Parse(newsletter.Text, qd, null).FillQueryTokens(list);
list = list.Distinct().ToList();
}
var columns = list.Select(qt => new Column(qt, null)).ToList();
//var columns = new List<QueryToken>();
//columns.Add(QueryUtils.Parse("Entity.NewsletterDeliveries.Element", qd, canAggregate: false));
//columns.Add(QueryUtils.Parse("Entity.Email", qd, canAggregate: false));
//columns.AddRange(NewsletterLogic.GetTokens(queryName, newsletter.Subject));
//columns.AddRange(NewsletterLogic.GetTokens(queryName, newsletter.Text));
columns = columns.Distinct().ToList();
var resultTable = QueryLogic.Queries.ExecuteQuery(new QueryRequest
{
QueryName = queryName,
Filters = new List <Filter>
{
new FilterCondition(QueryUtils.Parse("Entity.NewsletterDeliveries.Element.Newsletter", qd, SubTokensOptions.CanElement), FilterOperation.EqualTo, newsletter.ToLite()),
new FilterCondition(QueryUtils.Parse("Entity.NewsletterDeliveries.Element.Sent", qd, SubTokensOptions.CanElement), FilterOperation.EqualTo, false),
},
Orders = new List <Order>(),
Columns = columns,
Pagination = new Pagination.All(),
});
var dicTokenColumn = resultTable.Columns.ToDictionary(rc => rc.Column.Token);
var entityColumn = resultTable.Columns.SingleEx(c => c.Column.Token.FullKey() == "Entity");
var deliveryColumn = resultTable.Columns.SingleEx(c => c.Column.Token.FullKey() == "Entity.NewsletterDeliveries.Element");
var emailOwnerColumn = resultTable.Columns.SingleEx(c => c.Column.Token.FullKey() == "Entity.EmailOwnerData");
var lines = resultTable.Rows.GroupBy(r => (Lite <Entity>)r[entityColumn] !).Select(g => new SendLine
{
NewsletterDelivery = (Lite <NewsletterDeliveryEntity>)g.DistinctSingle(deliveryColumn) !,
Email = (EmailOwnerData)g.DistinctSingle(emailOwnerColumn) !,
Rows = g,
}).ToList();
public void Execute(ExecutingProcess executingProcess)
{
SMSUpdatePackageEntity package = (SMSUpdatePackageEntity)executingProcess.Data !;
executingProcess.ForEachLine(package.SMSMessages().Where(sms => sms.State == SMSMessageState.Sent && !sms.UpdatePackageProcessed), sms =>
{
sms.Execute(SMSMessageOperation.UpdateStatus);
});
}
public void Execute(ExecutingProcess executingProcess)
{
SMSUpdatePackageEntity package = (SMSUpdatePackageEntity)executingProcess.Data;
executingProcess.ForEachLine(package.SMSMessages().Where(sms => sms.State == SMSMessageState.Sent && !sms.UpdatePackageProcessed), sms =>
{
sms.Execute(SMSMessageOperation.UpdateStatus);
});
}
public void Execute(ExecutingProcess executingProcess)
{
PrintPackageEntity package = (PrintPackageEntity)executingProcess.Data !;
executingProcess.ForEachLine(package.Lines().Where(a => a.State != PrintLineState.Printed), line =>
{
PrintLineGraph.Print(line);
});
}
public virtual void Execute(ExecutingProcess executingProcess)
{
PackageEntity package = (PackageEntity)executingProcess.Data !;
var args = package.GetOperationArgs();
executingProcess.ForEachLine(package.Lines().Where(a => a.FinishTime == null), line =>
{
((T)(object)line.Target).Execute(Symbol, args);
line.FinishTime = Clock.Now;
line.Save();
});
}
public void Execute(ExecutingProcess executingProcess)
{
EmailPackageEntity package = (EmailPackageEntity)executingProcess.Data !;
List <Lite <EmailMessageEntity> > emails = package.RemainingMessages()
.OrderBy(e => e.CreationDate)
.Select(e => e.ToLite())
.ToList();
int counter = 0;
using (AuthLogic.Disable())
{
foreach (var group in emails.GroupsOf(EmailLogic.Configuration.ChunkSizeSendingEmails))
{
var retrieved = group.RetrieveFromListOfLite();
foreach (var m in retrieved)
{
executingProcess.CancellationToken.ThrowIfCancellationRequested();
counter++;
try
{
using (Transaction tr = Transaction.ForceNew())
{
EmailLogic.SenderManager.Send(m);
tr.Commit();
}
executingProcess.ProgressChanged(counter, emails.Count);
}
catch
{
try
{
if (m.SendRetries < EmailLogic.Configuration.MaxEmailSendRetries)
{
using (Transaction tr = Transaction.ForceNew())
{
var nm = m.ToLite().RetrieveAndRemember();
nm.SendRetries += 1;
nm.State = EmailMessageState.ReadyToSend;
nm.Save();
tr.Commit();
}
}
}
catch { }
}
}
}
}
}
public Dictionary <PredictorEntity, double> EvaluatePopulation(ExecutingProcess ep, AutoconfigureNeuralNetworkEntity conf, List <PredictorEntity> population, int gen)
{
var total = conf.Population * (conf.Generations + 1);
var evaluatedPopulation = new Dictionary <PredictorEntity, double>();
for (int i = 0; i < population.Count; i++)
{
ep.CancellationToken.ThrowIfCancellationRequested();
var current = gen * population.Count + i;
ep.ProgressChanged(current / (decimal)total);
var p = population[i];
double lastValidation = Evaluate(ep, p, onProgress: val => ep.ProgressChanged((current + val ?? 0) / (decimal)total));
evaluatedPopulation.Add(p, lastValidation);
}
return(evaluatedPopulation);
}
public virtual void Execute(ExecutingProcess executingProcess)
{
PackageEntity package = (PackageEntity)executingProcess.Data !;
var args = package.OperationArgs;
var template = args.GetArg <Lite <EmailTemplateEntity> >();
executingProcess.ForEachLine(package.Lines().Where(a => a.FinishTime == null), line =>
{
var emails = template.CreateEmailMessage(line.Target).ToList();
foreach (var email in emails)
{
email.SendMailAsync();
}
line.Result = emails.Only()?.ToLite();
line.FinishTime = TimeZoneManager.Now;
line.Save();
});
}
public void Execute(ExecutingProcess executingProcess)
{
PackageOperationEntity package = (PackageOperationEntity)executingProcess.Data !;
OperationSymbol operationSymbol = package.Operation;
var args = package.GetOperationArgs();
executingProcess.ForEachLine(package.Lines().Where(a => a.FinishTime == null), line =>
{
OperationType operationType = OperationLogic.OperationType(line.Target.GetType(), operationSymbol);
switch (operationType)
{
case OperationType.Execute:
OperationLogic.ServiceExecute(line.Target, operationSymbol, args);
break;
case OperationType.Delete:
OperationLogic.ServiceDelete(line.Target, operationSymbol, args);
break;
case OperationType.ConstructorFrom:
{
var result = OperationLogic.ServiceConstructFrom(line.Target, operationSymbol, args);
line.Result = result.ToLite();
}
break;
default:
throw new InvalidOperationException("Unexpected operation type {0}".FormatWith(operationType));
}
line.FinishTime = Clock.Now;
line.Save();
});
}
public virtual void Execute(ExecutingProcess executingProcess)
{
PackageEntity package = (PackageEntity)executingProcess.Data !;
var args = package.GetOperationArgs();
executingProcess.ForEachLine(package.Lines().Where(a => a.FinishTime == null), line =>
{
var result = ((F)(object)line.Target).ConstructFrom(Symbol, args);
if (result != null)
{
if (result.IsNew)
{
using (OperationLogic.AllowSave <T>())
result.Save();
}
line.Result = ((Entity)(IEntity)result).ToLite();
}
line.FinishTime = Clock.Now;
line.Save();
});
}
public virtual void Execute(ExecutingProcess executingProcess)
{
PackageEntity package = (PackageEntity)executingProcess.Data;
var args = package.OperationArgs;
var template = args.GetArg<Lite<EmailTemplateEntity>>();
executingProcess.ForEachLine(package.Lines().Where(a => a.FinishTime == null), line =>
{
var emails = template.CreateEmailMessage(line.Target).ToList();
foreach (var email in emails)
email.SendMailAsync();
line.Result = emails.Only()?.ToLite();
line.FinishTime = TimeZoneManager.Now;
line.Save();
});
}
public void Execute(ExecutingProcess executingProcess)
{
EmailPackageEntity package = (EmailPackageEntity)executingProcess.Data;
List<Lite<EmailMessageEntity>> emails = package.RemainingMessages()
.OrderBy(e => e.CreationDate)
.Select(e => e.ToLite())
.ToList();
int counter = 0;
using (AuthLogic.Disable())
{
foreach (var group in emails.GroupsOf(EmailLogic.Configuration.ChunkSizeSendingEmails))
{
var retrieved = group.RetrieveFromListOfLite();
foreach (var m in retrieved)
{
executingProcess.CancellationToken.ThrowIfCancellationRequested();
counter++;
try
{
using (Transaction tr = Transaction.ForceNew())
{
EmailLogic.SenderManager.Send(m);
tr.Commit();
}
executingProcess.ProgressChanged(counter, emails.Count);
}
catch
{
try
{
if (m.SendRetries < EmailLogic.Configuration.MaxEmailSendRetries)
{
using (Transaction tr = Transaction.ForceNew())
{
var nm = m.ToLite().Retrieve();
nm.SendRetries += 1;
nm.State = EmailMessageState.ReadyToSend;
nm.Save();
tr.Commit();
}
}
}
catch { }
}
}
}
}
}
private static double Evaluate(ExecutingProcess ep, PredictorEntity p, Action <decimal?> onProgress)
{
PredictorLogic.TrainSync(p, onReportProgres: (str, val) => onProgress(val));
return(p.ResultValidation !.Loss.Value);
}
public static void StartRunningProcesses()
{
if (running)
{
throw new InvalidOperationException("ProcessLogic is running");
}
using (ExecutionContext.SuppressFlow())
Task.Factory.StartNew(() =>
{
var database = Schema.Current.Table(typeof(ProcessEntity)).Name.Schema?.Database;
SystemEventLogLogic.Log("Start ProcessRunner");
ExceptionEntity?exception = null;
using (AuthLogic.Disable())
{
try
{
running = true;
(from p in Database.Query <ProcessEntity>()
where p.IsMine() && (p.State == ProcessState.Executing || p.State == ProcessState.Suspending || p.State == ProcessState.Suspended) ||
p.IsShared() && p.State == ProcessState.Suspended
select p).SetAsQueued();
CancelNewProcesses = new CancellationTokenSource();
autoResetEvent.Set();
timerNextExecution = new Timer(ob => WakeUp("TimerNextExecution", null), // main timer
null,
Timeout.Infinite,
Timeout.Infinite);
if (!CacheLogic.WithSqlDependency)
{
timerPeriodic = new Timer(ob => WakeUp("TimerPeriodic", null), null, PoolingPeriodMilliseconds, PoolingPeriodMilliseconds);
}
while (autoResetEvent.WaitOne())
{
if (CancelNewProcesses.IsCancellationRequested)
{
return;
}
using (HeavyProfiler.Log("PWL", () => "Process Runner"))
{
(from p in Database.Query <ProcessEntity>()
where p.State == ProcessState.Planned && p.PlannedDate <= Clock.Now
select p).SetAsQueued();
var list = Database.Query <ProcessEntity>()
.Where(p => p.IsMine() || p.IsShared())
.Where(p => p.State == ProcessState.Planned)
.Select(p => p.PlannedDate)
.ToListWakeup("Planned dependency");
SetNextPannedExecution(list.Min());
lock (executing)
{
int remaining = MaxDegreeOfParallelism - executing.Count;
if (remaining > 0)
{
retry:
var queued = Database.Query <ProcessEntity>()
.Where(p => p.State == ProcessState.Queued)
.Where(p => p.IsMine() || p.IsShared())
.Select(a => new { Process = a.ToLite(), a.QueuedDate, a.MachineName })
.ToListWakeup("Planned dependency");
var afordable = queued
.OrderByDescending(p => p.MachineName == Environment.MachineName)
.OrderBy(a => a.QueuedDate)
.Take(remaining).ToList();
var taken = afordable.Where(p => p.MachineName == ProcessEntity.None).Select(a => a.Process).ToList();
if (taken.Any())
{
using (var tr = Transaction.ForceNew())
{
Database.Query <ProcessEntity>()
.Where(p => taken.Contains(p.ToLite()) && p.MachineName == ProcessEntity.None)
.UnsafeUpdate()
.Set(p => p.MachineName, p => Environment.MachineName)
.Set(p => p.ApplicationName, p => Schema.Current.ApplicationName)
.Execute();
tr.Commit();
}
goto retry;
}
foreach (var pair in afordable)
{
ProcessEntity pro = pair.Process !.RetrieveAndRemember();
IProcessAlgorithm algorithm = ProcessLogic.GetProcessAlgorithm(pro.Algorithm);
ExecutingProcess executingProcess = new ExecutingProcess(algorithm, pro);
executing.Add(pro.ToLite(), executingProcess);
executingProcess.TakeForThisMachine();
using (ExecutionContext.SuppressFlow())
Task.Run(() =>
{
try
{
executingProcess.Execute();
}
catch (Exception ex)
{
try
{
ex.LogException(edn =>
{
edn.ControllerName = "ProcessWorker";
edn.ActionName = executingProcess.CurrentProcess.ToLite().Key();
});
}
catch { }
}
finally
{
lock (executing)
{
executing.Remove(pro.ToLite());
WakeUp("Process ended", null);
}
}
});
}
var suspending = Database.Query <ProcessEntity>()
.Where(p => p.State == ProcessState.Suspending)
.Where(p => p.IsMine())
.Select(a => a.ToLite())
.ToListWakeup("Suspending dependency");
foreach (var s in suspending)
{
ExecutingProcess execProc = executing.GetOrThrow(s);
if (execProc.CurrentProcess.State != ProcessState.Finished)
{
execProc.CurrentProcess = s.RetrieveAndRemember();
execProc.CancelationSource.Cancel();
}
}
}
}
}
}
}
catch (ThreadAbortException)
{
//Ignore
}
catch (Exception e)
{
try
{
exception = e.LogException(edn =>
{
edn.ControllerName = "ProcessWorker";
edn.ActionName = "MainLoop";
});
}
catch { }
}
finally
{
lock (executing)
executing.Clear();
SystemEventLogLogic.Log("Stop ProcessRunner", exception);
running = false;
}
}
}, TaskCreationOptions.LongRunning);
}
public override void Execute(ExecutingProcess executingProcess)
{
base.Execute(executingProcess); // Override if necessary
}
public void Execute(ExecutingProcess executingProcess)
{
NewsletterEntity newsletter = (NewsletterEntity)executingProcess.Data;
var queryName = QueryLogic.ToQueryName(newsletter.Query.Key);
QueryDescription qd = DynamicQueryManager.Current.QueryDescription(queryName);
List <QueryToken> list = new List <QueryToken>();
using (ExecutionMode.Global())
{
list.Add(QueryUtils.Parse("Entity", qd, 0));
list.Add(QueryUtils.Parse(".".Combine("Entity", "NewsletterDeliveries", "Element"), qd, SubTokensOptions.CanElement));
list.Add(QueryUtils.Parse(".".Combine("Entity", "EmailOwnerData"), qd, 0));
EmailTemplateParser.Parse(newsletter.Subject, qd, null).FillQueryTokens(list);
EmailTemplateParser.Parse(newsletter.Text, qd, null).FillQueryTokens(list);
list = list.Distinct().ToList();
}
var columns = list.Select(qt => new Column(qt, null)).ToList();
//var columns = new List<QueryToken>();
//columns.Add(QueryUtils.Parse("Entity.NewsletterDeliveries.Element", qd, canAggregate: false));
//columns.Add(QueryUtils.Parse("Entity.Email", qd, canAggregate: false));
//columns.AddRange(NewsletterLogic.GetTokens(queryName, newsletter.Subject));
//columns.AddRange(NewsletterLogic.GetTokens(queryName, newsletter.Text));
columns = columns.Distinct().ToList();
var resultTable = DynamicQueryManager.Current.ExecuteQuery(new QueryRequest
{
QueryName = queryName,
Filters = new List <Filter>
{
new Filter(QueryUtils.Parse("Entity.NewsletterDeliveries.Element.Newsletter", qd, SubTokensOptions.CanElement), FilterOperation.EqualTo, newsletter.ToLite()),
new Filter(QueryUtils.Parse("Entity.NewsletterDeliveries.Element.Sent", qd, SubTokensOptions.CanElement), FilterOperation.EqualTo, false),
},
Orders = new List <Order>(),
Columns = columns,
Pagination = new Pagination.All(),
});
var dicTokenColumn = resultTable.Columns.ToDictionary(rc => rc.Column.Token);
var entityColumn = resultTable.Columns.SingleEx(c => c.Column.Token.FullKey() == "Entity");
var deliveryColumn = resultTable.Columns.SingleEx(c => c.Column.Token.FullKey() == "Entity.NewsletterDeliveries.Element");
var emailOwnerColumn = resultTable.Columns.SingleEx(c => c.Column.Token.FullKey() == "Entity.EmailOwnerData");
var lines = resultTable.Rows.GroupBy(r => (Lite <Entity>)r[entityColumn]).Select(g => new SendLine
{
NewsletterDelivery = (Lite <NewsletterDeliveryEntity>)g.DistinctSingle(deliveryColumn),
Email = (EmailOwnerData)g.DistinctSingle(emailOwnerColumn),
Rows = g,
}).ToList();
if (newsletter.SubjectParsedNode == null)
{
newsletter.SubjectParsedNode = EmailTemplateParser.Parse(newsletter.Subject, qd, null);
}
if (newsletter.TextParsedNode == null)
{
newsletter.TextParsedNode = EmailTemplateParser.Parse(newsletter.Text, qd, null);
}
var conf = EmailLogic.Configuration;
int processed = 0;
foreach (var group in lines.GroupsOf(20))
{
processed += group.Count;
executingProcess.CancellationToken.ThrowIfCancellationRequested();
if (conf.SendEmails)
{
Parallel.ForEach(group, s =>
{
try
{
var smtpConfig = NewsletterLogic.GetStmpConfiguration(newsletter);
var client = smtpConfig.GenerateSmtpClient();
var message = new MailMessage();
if (newsletter.From.HasText())
{
message.From = new MailAddress(newsletter.From, newsletter.DisplayFrom);
}
else
{
message.From = smtpConfig.DefaultFrom.ToMailAddress();
}
message.To.Add(conf.OverrideEmailAddress.DefaultText(s.Email.Email));
message.Subject = ((EmailTemplateParser.BlockNode)newsletter.SubjectParsedNode).Print(
new EmailTemplateParameters(null, null, dicTokenColumn, s.Rows)
{
IsHtml = false,
});
message.Body = ((EmailTemplateParser.BlockNode)newsletter.TextParsedNode).Print(
new EmailTemplateParameters(null, null, dicTokenColumn, s.Rows)
{
IsHtml = true,
});
message.IsBodyHtml = true;
client.Send(message);
}
catch (Exception ex)
{
s.Exception = ex;
}
});
}
var failed = group.Extract(sl => sl.Exception != null);
foreach (var f in failed)
{
new ProcessExceptionLineEntity
{
Exception = f.Exception.LogException().ToLite(),
Line = f.NewsletterDelivery,
Process = executingProcess.CurrentProcess.ToLite(),
}.Save();
}
var sent = group.Select(sl => sl.NewsletterDelivery).ToList();
if (sent.Any())
{
Database.Query <NewsletterDeliveryEntity>()
.Where(nd => sent.Contains(nd.ToLite()))
.UnsafeUpdate()
.Set(nd => nd.Sent, nd => true)
.Set(nd => nd.SendDate, nd => TimeZoneManager.Now.TrimToSeconds())
.Execute();
}
executingProcess.ProgressChanged(processed, lines.Count);
}
newsletter.State = NewsletterState.Sent;
using (OperationLogic.AllowSave <NewsletterEntity>())
newsletter.Save();
}
