public virtual void StartProcessing(CancellationToken cancellationToken, Action <object, AppContext> execute)
{
if (_settings.IsDisabled)
{
return;
}
RegisterClassMaps();
if (_settings.ResetCollection)
{
DropCollection();
}
CreateCollection();
//Create query based on latest document id
BsonValue lastId = BsonMinKey.Value;
var collection = _database.GetCollection <QueueJob>(_settings.CollectionName);
var query = collection
.FindAs <QueueJob>(Query.GT("_id", lastId))
.SetFlags(QueryFlags.AwaitData | QueryFlags.TailableCursor);
var cursor = new MongoCursorEnumerator <QueueJob>(query);
while (true)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
if (!cursor.MoveNext())
{
continue;
}
var queueJob = cursor.Current;
if (queueJob.IsComplete || !string.IsNullOrWhiteSpace(queueJob.Error))
{
continue;
}
try
{
execute(queueJob.Command, queueJob.Context);
queueJob.IsComplete = true;
collection.Save(queueJob);
}
catch (ThreadAbortException)
{
//ignore
}
catch (Exception ex)
{
queueJob.SetError(ex.Message);
collection.Save(queueJob);
}
}
}
public T Receive()
{
// for reading, we give the impression to the client that we provide a single message at a time
// which means we maintain a cursor and enumerator in the background and hide it from the caller
if (_enumerator == null)
{
_enumerator = InitializeCursor();
}
// there is no end when you need to sit and wait for messages to arrive
while (true)
{
try
{
// do we have a message waiting?
// this may block on the server for a few seconds but will return as soon as something is available
if (_enumerator.MoveNext())
{
// yes - record the current position and return it to the client
_startedReading = true;
_lastId = _enumerator.Current.Id;
_position.Update(_positionQuery, Update.Set("last", _lastId), UpdateFlags.Upsert, SafeMode.False);
var message = _enumerator.Current.Message;
var delay = DateTime.UtcNow - _enumerator.Current.Enqueued;
Log.Debug(m => m("Received {0} after {1}", _queueName, delay));
return(message);
}
if (!_startedReading)
{
// for an empty collection, we'll need to re-query to be notified of new records
Log.Debug("Cursor Empty");
Thread.Sleep(100);
_enumerator.Dispose();
_enumerator = InitializeCursor();
}
else
{
// if the cursor is dead then we need to re-query, otherwise we just go back to iterating over it
if (_enumerator.IsDead)
{
Log.Debug("Cursor Dead");
_enumerator.Dispose();
_enumerator = InitializeCursor();
}
}
}
catch (Exception ex)
{
// cursor died or was killed
_enumerator.Dispose();
_enumerator = InitializeCursor();
}
}
}
public bool CursorMoveNext()
{
MongoCollection <CustomMongoEntity> collection = new MongoCollection <CustomMongoEntity>(_db, _collectionName, new MongoCollectionSettings());
collection.Insert(new CustomMongoEntity {
Id = new ObjectId(), Name = "Genghis Khan"
});
var cursor = collection.Find(Query <CustomMongoEntity> .EQ(e => e.Name, "Genghis Khan"));
var enumerator = new MongoCursorEnumerator <CustomMongoEntity>(cursor);
return(enumerator.MoveNext());
}
/// <summary>
/// Receives an event (or action) from queue. Does not change the message state. Used for PubSub (1 to Many - all gets the same messages) use cases
/// </summary>
/// <returns></returns>
private T ReceiveEvent()
{
if (_enumerator == null)
{
_enumerator = InitializeCursor();
}
// running untill we have something to return
while (true)
{
var t1 = DateTime.UtcNow;
try
{
// do we have a message waiting?
// this may block on the server for a few seconds but will return as soon as something is available
if (_enumerator.MoveNext())
{
// yes - record the current position and return it to the client
Interlocked.Increment(ref _totalReceived);
_lastId = _enumerator.Current.Id;
var current = _enumerator.Current;
var message = current.Message;
var delay = DateTime.UtcNow - _enumerator.Current.Enqueued;
Log.Debug(m => m("Received {0} after {1}", _queueName, delay));
return(message);
}
// if the cursor is dead then we need to re-query, otherwise we just go back to iterating over it
if (_enumerator.IsDead)
{
Log.Debug("Cursor Dead");
_enumerator.Dispose();
_enumerator = InitializeCursor();
}
// If we are here it means that the server returned without items, usually it sits in a 2 seconds block on the server, but if initial request is empty then return immedietly.
//This is a safety mechanism to protect the mongo server from our DOS...(due to this cursor behavior, and from unknown bugs / 'behaviors')
var iterationTime = DateTime.UtcNow - t1;
if (iterationTime.TotalMilliseconds < SHORT_SLEEP_INTERVAL) // minimal polling interval : TODO - move to Config
{
Thread.Sleep(SHORT_SLEEP_INTERVAL);
}
}
catch (Exception ex)
{
Log.Warn("[ReceiveEvents inside while true] Got exception. Will reinitialize and continue. Ex={0}", ex);
// cursor died or was killed
if (_enumerator != null)
{
_enumerator.Dispose();
}
_enumerator = InitializeCursor();
//This is a safety mechanism to protect the mongo server from our DOS...
var iterationTime = DateTime.UtcNow - t1;
if (iterationTime.TotalMilliseconds < SHORT_SLEEP_INTERVAL) // minimal polling interval : TODO - move to Config
{
Thread.Sleep(SHORT_SLEEP_INTERVAL);
}
}
}
}
public void Start()
{
try
{
Sitecore.Diagnostics.Log.Info("MongoOplogCacheClearer: Starting", this);
var client = new MongoClient(_connectionString);
var server = client.GetServer();
var mappingDatabase = server.GetDatabase(_mongoDatabase);
var mappingCollection = mappingDatabase.GetCollection(_mappingCollection);
//run a query against the oplog with a tailable cursor, which will block until new records are created
var mongoDatabase = server.GetDatabase(LOCAL);
var opLog = mongoDatabase.GetCollection(OPLOG);
var queryCollection = string.Format("{0}.{1}", _mongoDatabase, _mongoCollection);
var queryDoc = new QueryDocument("ns", queryCollection);
var query = opLog.Find(queryDoc)
.SetFlags(QueryFlags.AwaitData | QueryFlags.NoCursorTimeout | QueryFlags.TailableCursor);
var cursor = new MongoCursorEnumerator <BsonDocument>(query);
while (true)
{
if (cursor.MoveNext())
{
var document = cursor.Current;
if (document["op"].AsString == "d")
{
//get the deleted document
document = document["o"].AsBsonDocument;
//TODO: on delete, delete from the mapping collection as well?
}
else if (document["op"].AsString == "u")
{
//get the updated document
document = document["o2"].AsBsonDocument;
}
else
{
continue;
}
var objectId = document["_id"].AsObjectId;
//look in our mapping table for the associated sitecore item id
var mapping = mappingCollection.FindOne(new QueryDocument("mongoId", objectId.ToString()));
if (mapping == null)
{
continue;
}
var id = Sitecore.Data.ShortID.Parse(mapping["sitecoreId"].AsString).ToID();
//clear caches for all configured databases, simulate item save
foreach (var database in _databases)
{
var item = database.GetItem(id);
if (item == null)
{
continue;
}
Sitecore.Diagnostics.Log.Info(
string.Format("MongoOplogCacheClearer: Product {0}://{1} updated", database.Name, id), this);
database.Caches.ItemCache.RemoveItem(id);
database.Caches.DataCache.RemoveItemInformation(id);
database.Engines.DataEngine.RaiseSavedItem(item, true);
var args = new ItemSavedEventArgs(item);
Sitecore.Events.Event.RaiseItemSaved(this, args);
//TODO: Something to cause reindex in web, new indexing strategy?
//TODO: Clear HTML cache?
}
}
else if (cursor.IsDead)
{
//TODO: restart mechanism?
Sitecore.Diagnostics.Log.Info("MongoOplogCacheClearer: Dead", this);
break;
}
}
}
catch (Exception e)
{
Sitecore.Diagnostics.Log.Error("Exception starting MongoDb oplog monitoring", e, this);
}
}
public void Start()
{
try
{
Sitecore.Diagnostics.Log.Info("MongoOplogCacheClearer: Starting", this);
var client = new MongoClient(_connectionString);
var server = client.GetServer();
var mappingDatabase = server.GetDatabase(_mongoDatabase);
var mappingCollection = mappingDatabase.GetCollection(_mappingCollection);
//run a query against the oplog with a tailable cursor, which will block until new records are created
var mongoDatabase = server.GetDatabase(LOCAL);
var opLog = mongoDatabase.GetCollection(OPLOG);
var queryCollection = string.Format("{0}.{1}", _mongoDatabase, _mongoCollection);
var queryDoc = new QueryDocument("ns", queryCollection);
var query = opLog.Find(queryDoc)
.SetFlags(QueryFlags.AwaitData | QueryFlags.NoCursorTimeout | QueryFlags.TailableCursor);
var cursor = new MongoCursorEnumerator<BsonDocument>(query);
while (true)
{
if (cursor.MoveNext())
{
var document = cursor.Current;
if (document["op"].AsString == "d")
{
//get the deleted document
document = document["o"].AsBsonDocument;
//TODO: on delete, delete from the mapping collection as well?
}
else if (document["op"].AsString == "u")
{
//get the updated document
document = document["o2"].AsBsonDocument;
}
else
{
continue;
}
var objectId = document["_id"].AsObjectId;
//look in our mapping table for the associated sitecore item id
var mapping = mappingCollection.FindOne(new QueryDocument("mongoId", objectId.ToString()));
if (mapping == null)
{
continue;
}
var id = Sitecore.Data.ShortID.Parse(mapping["sitecoreId"].AsString).ToID();
//clear caches for all configured databases, simulate item save
foreach (var database in _databases)
{
var item = database.GetItem(id);
if (item == null)
{
continue;
}
Sitecore.Diagnostics.Log.Info(
string.Format("MongoOplogCacheClearer: Product {0}://{1} updated", database.Name, id), this);
database.Caches.ItemCache.RemoveItem(id);
database.Caches.DataCache.RemoveItemInformation(id);
database.Engines.DataEngine.RaiseSavedItem(item, true);
var args = new ItemSavedEventArgs(item);
Sitecore.Events.Event.RaiseItemSaved(this, args);
//TODO: Something to cause reindex in web, new indexing strategy?
//TODO: Clear HTML cache?
}
}
else if (cursor.IsDead)
{
//TODO: restart mechanism?
Sitecore.Diagnostics.Log.Info("MongoOplogCacheClearer: Dead", this);
break;
}
}
}
catch (Exception e)
{
Sitecore.Diagnostics.Log.Error("Exception starting MongoDb oplog monitoring", e, this);
}
}
