private TodoFrame toFrame(TodoQueue queue, BSONDocument doc)
{
try
{
var result = new TodoFrame();
result.ID = RowConverter.GDID_BSONtoCLR(doc[Query._ID] as BSONBinaryElement);
result.Type = Guid.Parse(((BSONStringElement)doc[FLD_TODO_TYPE]).Value);
result.CreateTimestampUTC = ((BSONDateTimeElement)doc[FLD_TODO_CREATETIMESTAMP]).Value;
result.ShardingKey = elmStr(doc[FLD_TODO_SHARDINGKEY]);
result.ParallelKey = elmStr(doc[FLD_TODO_PARALLELKEY]);
result.Priority = ((BSONInt32Element)doc[FLD_TODO_PRIORITY]).Value;
result.StartDate = ((BSONDateTimeElement)doc[FLD_TODO_STARTDATE]).Value;
result.CorrelationKey = elmStr(doc[FLD_TODO_CORRELATIONKEY]);
result.State = ((BSONInt32Element)doc[FLD_TODO_STATE]).Value;
result.Tries = ((BSONInt32Element)doc[FLD_TODO_TRIES]).Value;
result.Serializer = ((BSONInt32Element)doc[FLD_TODO_SERIALIZER]).Value;
result.Content = elmBin(doc[FLD_TODO_CONTENT]);
return(result);
}
catch (Exception error)
{
throw new MongoWorkersException(StringConsts.TODO_QUEUE_BSON_READ_ERROR.Args(queue, error.ToMessageWithType()), error);
}
}
public override TodoFrame FetchLatestCorrelated(TodoQueue queue, string correlationKey, DateTime utcStartingFrom)
{
if (DisposeStarted || queue == null || correlationKey == null)
{
return(new TodoFrame());
}
var collection = m_Database[queue.Name];
var query = new Query(
@"{
'$query': { cky: '$$cky', sd: {'$gt': '$$utcFrom'} },
'$orderby': { sd: -1}
}",
true, new TemplateArg(new BSONStringElement("cky", correlationKey)),
new TemplateArg(new BSONDateTimeElement("utcFrom", utcStartingFrom)));
var doc = collection.FindOne(query);
if (doc == null)
{
return(new TodoFrame());
}
var frame = toFrame(queue, doc);
return(frame);
}
private void put(TodoQueue queue, TodoFrame todo, object transaction = null, bool leak = false)
{
try
{
m_QueueStore.Put(queue, todo, transaction);
if (InstrumentationEnabled)
{
m_stat_PutTodoCount.IncrementLong(ALL);
m_stat_PutTodoCount.IncrementLong(queue.Name);
}
}
catch (Exception e)
{
var from = "put('{0}')".Args(queue.Name);
Log(MessageType.Critical, from, "{0} Leaked: {1}".Args(todo, e.ToMessageWithType()), e);
if (InstrumentationEnabled)
{
m_stat_QueueOperationErrorCount.IncrementLong(ALL);
m_stat_QueueOperationErrorCount.IncrementLong(from);
}
if (leak)
{
throw;
}
}
}
private bool checkDups(TodoQueue queue, TodoFrame todo)
{
var mode = queue.DuplicationHandling;
if (mode == TodoQueue.DuplicationHandlingMode.NotDetected)
{
return(false); //never a duplicate
}
var result = checkDupsInMemory(todo);
if (result || mode == TodoQueue.DuplicationHandlingMode.HostFastDetection)
{
return(result);
}
result = checkDupsInQueueStore(queue, todo);
if (result || mode == TodoQueue.DuplicationHandlingMode.HostAccurateDetection)
{
return(result);
}
result = checkDupsInHostset(queue, todo);
return(result);
}
public override void Complete(TodoQueue queue, TodoFrame todo, Exception error = null, object transaction = null)
{
if (DisposeStarted || queue == null || !todo.Assigned)
{
return;
}
//this provider ignores exceptions passed in error var
m_Database[queue.Name].DeleteOne(Query.ID_EQ_GDID(todo.ID));
}
public override void Put(TodoQueue queue, TodoFrame todo, object transaction)
{
if (DisposeStarted || queue == null || !todo.Assigned)
{
return;
}
var tName = queue.Name;
var doc = toBSON(queue, todo);
m_Database[tName].Insert(doc);
}
public override void Update(TodoQueue queue, Todo todo, bool sysOnly, object transaction)
{
if (DisposeStarted || queue == null || todo == null)
{
return;
}
var frame = new TodoFrame(todo, sysOnly ? (int?)null : TodoFrame.SERIALIZER_BSON);
var doc = toBSONUpdate(queue, frame, sysOnly);
m_Database[queue.Name].Update(new UpdateEntry(Query.ID_EQ_GDID(todo.SysID), doc, false, false));
}
public override IEnumerable <TodoFrame> Fetch(TodoQueue queue, DateTime utcNow)
{
if (DisposeStarted || queue == null)
{
return(Enumerable.Empty <TodoFrame>());
}
var collection = m_Database[queue.Name];
//todo Napisat query - kak sdelat server-side SORT? mopjet sdelat ID ne GDID a vremya?
//{ $query: {}, $orderby: { dt: 1 } }
// akto sdelaet v mongo index? mojet ne xranit null v __sd a xranit MIN date vmesto null?
//Mojet zdes nado sdelat Db.RunCommand i peredat sort tuda? Nujen index!
//https://docs.mongodb.com/manual/reference/command/find/#dbcmd.find
//kak eto vliyaet na mongo sort size buffer (32 mb? tam ili chtoto typa togo)?
var query = new Query(
@"{
'$query': { sd: {'$lt': '$$now'} },
'$orderby': { sd: 1}
}",
true, new TemplateArg(new BSONDateTimeElement("now", utcNow)));
var total = queue.BatchSize;
var result = new List <TodoFrame>();
var fby = FetchBy;
if (fby > total)
{
fby = total;
}
using (var cursor = collection.Find(query, fetchBy: fby))
foreach (var doc in cursor)
{
var todo = toFrame(queue, doc);
result.Add(todo);
var left = total - result.Count;
if (left == 0)
{
break;
}
if (cursor.FetchBy > left)
{
cursor.FetchBy = left;
}
}
return(result);
}
private void processOneQueueBatch(TodoQueue queue, IEnumerable <TodoFrame> batch, DateTime utcNow)
{
if (queue.Mode == TodoQueue.ExecuteMode.Sequential)
{
batch.OrderBy(t => t.StartDate).ForEach(todo => executeOne(queue, todo, utcNow));
}
else if (queue.Mode == TodoQueue.ExecuteMode.Parallel)
{
Parallel.ForEach(batch.OrderBy(t => t.StartDate), todo => executeOne(queue, todo, utcNow));
}
else//ParallelByKey
{
var tasks = new List <Task>();
var parallelTodos = batch.Where(t => t.ParallelKey == null).ToArray();
if (parallelTodos.Length > 0)
{
tasks.Add(Task.Factory.StartNew(ts => Parallel.ForEach(((IEnumerable <TodoFrame>)ts).OrderBy(t => t.StartDate), todo => executeOne(queue, todo, utcNow)), parallelTodos));
}
List <TodoFrame> todos = null;
string parallelKey = null;
foreach (var todo in batch.Where(t => t.ParallelKey != null).OrderBy(t => t.ParallelKey))
{
if (parallelKey != todo.ParallelKey)
{
if (todos != null)
{
tasks.Add(Task.Factory.StartNew(ts => ((IEnumerable <TodoFrame>)ts).OrderBy(t => t.StartDate).ForEach(t => executeOne(queue, t, utcNow)), todos));
}
todos = new List <TodoFrame>();
parallelKey = todo.ParallelKey;
}
todos.Add(todo);
}
if (todos != null)
{
tasks.Add(Task.Factory.StartNew(ts => ((IEnumerable <TodoFrame>)ts).OrderBy(t => t.StartDate).ForEach(t => executeOne(queue, t, utcNow)), todos));
}
Task.WaitAll(tasks.ToArray());
}
}
private void processOneQueueCore(TodoQueue queue, DateTime utcNow)
{
var fetched = m_QueueStore.Fetch(queue, utcNow);
try
{
var batch = fetched.Where(todo =>
{
if (todo.CorrelationKey == null)
{
return(true);
}
return(tryLockCorrelatedProcessing(todo.CorrelationKey, todo.StartDate));
}).ToList();//must materialize not to double lock
if (InstrumentationEnabled)
{
var fetchCount = fetched.Count();
m_stat_FetchedTodoCount.AddLong(ALL, fetchCount);
m_stat_FetchedTodoCount.AddLong(queue.Name, fetchCount);
}
processOneQueueBatch(queue, batch, utcNow);
if (InstrumentationEnabled)
{
m_stat_ProcessedTodoCount.AddLong(ALL, batch.Count);
m_stat_ProcessedTodoCount.AddLong(queue.Name, batch.Count);
}
}
finally
{
fetched.ForEach(todo =>
{
if (todo.CorrelationKey == null)
{
return;
}
releaseCorrelatedProcessing(todo.CorrelationKey);
});
}
}
private void complete(TodoQueue queue, TodoFrame todo, Exception error, object transaction = null)
{
try
{
m_QueueStore.Complete(queue, todo, error, transaction);
if (error != null)
{
Log(MessageType.Error, "complete('{0}')".Args(queue.Name), "Completed with error: " + error.ToMessageWithType(), error);
}
if (InstrumentationEnabled)
{
m_stat_CompletedTodoCount.IncrementLong(ALL);
m_stat_CompletedTodoCount.IncrementLong(queue.Name);
if (error != null)
{
m_stat_CompletedErrorTodoCount.IncrementLong(ALL);
m_stat_CompletedErrorTodoCount.IncrementLong(queue.Name);
}
else
{
m_stat_CompletedOkTodoCount.IncrementLong(ALL);
m_stat_CompletedOkTodoCount.IncrementLong(queue.Name);
}
}
}
catch (Exception e)
{
var from = "complete('{0}')".Args(queue.Name);
Log(MessageType.Critical, from, "{0} Leaked: {1}".Args(todo, e.ToMessageWithType()), e);
if (InstrumentationEnabled)
{
m_stat_QueueOperationErrorCount.IncrementLong(ALL);
m_stat_QueueOperationErrorCount.IncrementLong(from);
}
}
}
private BSONDocument toBSON(TodoQueue queue, TodoFrame todo)
{
var result = new BSONDocument();
var t = todo.GetType();
result.Set(RowConverter.GDID_CLRtoBSON(Query._ID, todo.ID));
result.Set(new BSONStringElement(FLD_TODO_TYPE, todo.Type.ToString()));
result.Set(new BSONDateTimeElement(FLD_TODO_CREATETIMESTAMP, todo.CreateTimestampUTC));
result.Set(elmStr(FLD_TODO_SHARDINGKEY, todo.ShardingKey));
result.Set(elmStr(FLD_TODO_PARALLELKEY, todo.ParallelKey));
result.Set(new BSONInt32Element(FLD_TODO_PRIORITY, todo.Priority));
result.Set(new BSONDateTimeElement(FLD_TODO_STARTDATE, todo.StartDate));
result.Set(elmStr(FLD_TODO_CORRELATIONKEY, todo.CorrelationKey));
result.Set(new BSONInt32Element(FLD_TODO_STATE, todo.State));
result.Set(new BSONInt32Element(FLD_TODO_TRIES, todo.Tries));
result.Set(new BSONInt32Element(FLD_TODO_SERIALIZER, todo.Serializer));
result.Set(elmBin(FLD_TODO_CONTENT, todo.Content));
return(result);
}
private BSONDocument toBSONUpdate(TodoQueue queue, TodoFrame todo, bool sysOnly)
{
var setDoc = new BSONDocument();
setDoc.Set(elmStr(FLD_TODO_SHARDINGKEY, todo.ShardingKey));
setDoc.Set(elmStr(FLD_TODO_PARALLELKEY, todo.ParallelKey));
setDoc.Set(new BSONInt32Element(FLD_TODO_PRIORITY, todo.Priority));
setDoc.Set(new BSONDateTimeElement(FLD_TODO_STARTDATE, todo.StartDate));
setDoc.Set(elmStr(FLD_TODO_CORRELATIONKEY, todo.CorrelationKey));
setDoc.Set(new BSONInt32Element(FLD_TODO_STATE, todo.State));
setDoc.Set(new BSONInt32Element(FLD_TODO_TRIES, todo.Tries));
if (!sysOnly)
{
setDoc.Set(new BSONInt32Element(FLD_TODO_SERIALIZER, todo.Serializer));
setDoc.Set(elmBin(FLD_TODO_CONTENT, todo.Content));
}
var result = new BSONDocument();
result.Set(new BSONDocumentElement("$set", setDoc));
return(result);
}
private void enqueue(TodoQueue queue, TodoFrame[] todos)
{
var tx = m_QueueStore.BeginTransaction(queue);
try
{
enqueueCore(queue, todos, tx);
m_QueueStore.CommitTransaction(queue, tx);
}
catch (Exception error)
{
m_QueueStore.RollbackTransaction(queue, tx);
var from = "enqueue('{0}')".Args(queue.Name);
Log(MessageType.CatastrophicError, from, error.ToMessageWithType(), error);
if (InstrumentationEnabled)
{
m_stat_QueueOperationErrorCount.IncrementLong(ALL);
m_stat_QueueOperationErrorCount.IncrementLong(from);
}
throw new WorkersException(StringConsts.TODO_ENQUEUE_TX_BODY_ERROR.Args(queue.Name, error.ToMessageWithType()), error);
}
}
private void executeOne(TodoQueue queue, TodoFrame todoFrame, DateTime utcNow)//must not leak
{
try
{
if (!Running)
{
return;
}
Todo todo;
try
{
todo = todoFrame.Materialize(AgniSystem.ProcessManager.TodoTypeResolver);
}
catch (Exception me)
{
var from = "executeOne('{0}').Materialize".Args(queue.Name);
Log(MessageType.Critical, from, "Frame materialization: " + me.ToMessageWithType(), me);
if (InstrumentationEnabled)
{
m_stat_QueueOperationErrorCount.IncrementLong(ALL);
m_stat_QueueOperationErrorCount.IncrementLong(from);
}
throw;
}
var wasState = todo.SysState;
while (todo.SysState != Todo.ExecuteState.Complete)
{
var nextState = todo.Execute(this, utcNow);
if (nextState == Todo.ExecuteState.ReexecuteUpdatedAfterError)
{
todo.SysTries++;
var ms = todo.RetryAfterErrorInMs(utcNow);
if (ms > 0)
{
todo.SysStartDate = todo.SysStartDate.AddMilliseconds(ms);
}
update(queue, todo, false);
return;
}
if (nextState == Todo.ExecuteState.ReexecuteAfterError)
{
todo.SysTries++;
var ms = todo.RetryAfterErrorInMs(utcNow);
if (ms > 0)
{
todo.SysStartDate = todo.SysStartDate.AddMilliseconds(ms);
}
if (ms >= 0 || todo.SysState != wasState)
{
update(queue, todo, true);
}
return;
}
if (nextState == Todo.ExecuteState.ReexecuteUpdated)
{
update(queue, todo, false);
return;
}
if (nextState == Todo.ExecuteState.ReexecuteSysUpdated)
{
update(queue, todo, true);
return;
}
if (nextState == Todo.ExecuteState.Reexecute)
{
if (todo.SysState != wasState)
{
update(queue, todo, true);
}
return;
}
todo.SysTries = 0;
todo.SysState = nextState;
}
complete(queue, todoFrame, null);
}
catch (Exception error)
{
complete(queue, todoFrame, error);
}
}
public abstract void CommitTransaction(TodoQueue queue, object transaction);
public override int GetInboundCapacity(TodoQueue queue)
{
return(TodoQueueService.FULL_BATCH_SIZE);
}
public override void RollbackTransaction(TodoQueue queue, object transaction)
{
}
public override void CommitTransaction(TodoQueue queue, object transaction)
{
}
public override object BeginTransaction(TodoQueue queue)
{
return(null);
}
private bool checkDupsInHostset(TodoQueue queue, TodoFrame todo)
{
return(false); // TODO implement
}
private void enqueueCore(TodoQueue queue, TodoFrame[] todos, object tx)
{
foreach (var todo in todos)
{
if (!todo.Assigned)
{
continue;
}
//20171114 If the todo was already submitted, then do nothing
var isdup = checkDups(queue, todo);
if (isdup)
{
if (InstrumentationEnabled)
{
var from = "enqueue-dup('{0}')".Args(queue.Name);
m_stat_TodoDuplicationCount.IncrementLong(ALL);
m_stat_TodoDuplicationCount.IncrementLong(from);
}
continue;
}
if (todo.CorrelationKey == null)//regular todos just add
{
put(queue, todo, tx, true);
continue;
}
//Correlated -------------------------------------------------------------
var utcNow = App.TimeSource.UTCNow;//warning locking depends on this accurate date
var utcCorrelateSD = lockCorrelatedEnqueue(todo.CorrelationKey, utcNow);
try
{
var existing = m_QueueStore.FetchLatestCorrelated(queue, todo.CorrelationKey, utcCorrelateSD);
if (!existing.Assigned) //no existing with same correlation, just add
{
put(queue, todo, tx, true);
continue;
}
var todoExisting = existing.Materialize(AgniSystem.ProcessManager.TodoTypeResolver) as CorrelatedTodo;
var todoAnother = todo.Materialize(AgniSystem.ProcessManager.TodoTypeResolver) as CorrelatedTodo;
if (todoExisting == null || todoAnother == null)//safeguard
{
put(queue, todo, tx, true);
continue;
}
CorrelatedTodo.MergeResult mergeResult;
try
{
mergeResult = todoExisting.Merge(this, utcNow, todoAnother);
}
catch (Exception error)
{
throw new WorkersException(StringConsts.TODO_CORRELATED_MERGE_ERROR.Args(todoExisting, todoAnother, error.ToMessageWithType()), error);
}
if (InstrumentationEnabled)
{
m_stat_MergedTodoCount.IncrementLong(ALL);
m_stat_MergedTodoCount.IncrementLong(queue.Name);
m_stat_MergedTodoCount.IncrementLong(queue.Name + ":" + mergeResult);
}
if (mergeResult == CorrelatedTodo.MergeResult.Merged)
{
update(queue, todoExisting, sysOnly: false, leak: true, transaction: tx);
continue;
}
else if (mergeResult == CorrelatedTodo.MergeResult.IgnoreAnother)
{
continue;//do nothing, just drop another as-if never existed
}
//Else, merge == CorrelatedTodo.MergeResult.None
put(queue, todo, tx, true);
}
finally
{
releaseCorrelatedEnqueue(todo.CorrelationKey);
}
}//foreach
}
public abstract void Put(TodoQueue queue, TodoFrame todo, object transaction);
public abstract TodoFrame FetchLatestCorrelated(TodoQueue queue, string correlationKey, DateTime utcStartingFrom);
public abstract void Update(TodoQueue queue, Todo todo, bool sysOnly, object transaction);
public abstract void Complete(TodoQueue queue, TodoFrame todo, Exception error = null, object transaction = null);
public abstract object BeginTransaction(TodoQueue queue);
public abstract void RollbackTransaction(TodoQueue queue, object transaction);
public abstract IEnumerable <TodoFrame> Fetch(TodoQueue queue, DateTime utcNow);
public abstract int GetInboundCapacity(TodoQueue queue);