示例#1
0
        public virtual void StartProcessing(CancellationToken cancellationToken, Action <object, AppContext> execute)
        {
            if (_settings.IsDisabled)
            {
                return;
            }

            RegisterClassMaps();
            if (_settings.ResetCollection)
            {
                DropCollection();
            }
            CreateCollection();

            //Create query based on latest document id
            BsonValue lastId     = BsonMinKey.Value;
            var       collection = _database.GetCollection <QueueJob>(_settings.CollectionName);
            var       query      = collection
                                   .FindAs <QueueJob>(Query.GT("_id", lastId))
                                   .SetFlags(QueryFlags.AwaitData | QueryFlags.TailableCursor);

            var cursor = new MongoCursorEnumerator <QueueJob>(query);

            while (true)
            {
                if (cancellationToken.IsCancellationRequested)
                {
                    break;
                }

                if (!cursor.MoveNext())
                {
                    continue;
                }

                var queueJob = cursor.Current;
                if (queueJob.IsComplete || !string.IsNullOrWhiteSpace(queueJob.Error))
                {
                    continue;
                }

                try
                {
                    execute(queueJob.Command, queueJob.Context);
                    queueJob.IsComplete = true;
                    collection.Save(queueJob);
                }
                catch (ThreadAbortException)
                {
                    //ignore
                }
                catch (Exception ex)
                {
                    queueJob.SetError(ex.Message);
                    collection.Save(queueJob);
                }
            }
        }
示例#2
0
        public T Receive()
        {
            // for reading, we give the impression to the client that we provide a single message at a time
            // which means we maintain a cursor and enumerator in the background and hide it from the caller

            if (_enumerator == null)
            {
                _enumerator = InitializeCursor();
            }

            // there is no end when you need to sit and wait for messages to arrive
            while (true)
            {
                try
                {
                    // do we have a message waiting?
                    // this may block on the server for a few seconds but will return as soon as something is available
                    if (_enumerator.MoveNext())
                    {
                        // yes - record the current position and return it to the client
                        _startedReading = true;
                        _lastId         = _enumerator.Current.Id;
                        _position.Update(_positionQuery, Update.Set("last", _lastId), UpdateFlags.Upsert, SafeMode.False);

                        var message = _enumerator.Current.Message;
                        var delay   = DateTime.UtcNow - _enumerator.Current.Enqueued;
                        Log.Debug(m => m("Received {0} after {1}", _queueName, delay));
                        return(message);
                    }

                    if (!_startedReading)
                    {
                        // for an empty collection, we'll need to re-query to be notified of new records
                        Log.Debug("Cursor Empty");
                        Thread.Sleep(100);
                        _enumerator.Dispose();
                        _enumerator = InitializeCursor();
                    }
                    else
                    {
                        // if the cursor is dead then we need to re-query, otherwise we just go back to iterating over it
                        if (_enumerator.IsDead)
                        {
                            Log.Debug("Cursor Dead");
                            _enumerator.Dispose();
                            _enumerator = InitializeCursor();
                        }
                    }
                }
                catch (Exception ex)
                {
                    // cursor died or was killed
                    _enumerator.Dispose();
                    _enumerator = InitializeCursor();
                }
            }
        }
        public bool CursorMoveNext()
        {
            MongoCollection <CustomMongoEntity> collection = new MongoCollection <CustomMongoEntity>(_db, _collectionName, new MongoCollectionSettings());

            collection.Insert(new CustomMongoEntity {
                Id = new ObjectId(), Name = "Genghis Khan"
            });
            var cursor     = collection.Find(Query <CustomMongoEntity> .EQ(e => e.Name, "Genghis Khan"));
            var enumerator = new MongoCursorEnumerator <CustomMongoEntity>(cursor);

            return(enumerator.MoveNext());
        }
示例#4
0
        /// <summary>
        /// Receives an event (or action) from queue. Does not change the message state. Used for PubSub (1 to Many - all gets the same messages) use cases
        /// </summary>
        /// <returns></returns>
        private T ReceiveEvent()
        {
            if (_enumerator == null)
            {
                _enumerator = InitializeCursor();
            }

            // running untill we have something to return
            while (true)
            {
                var t1 = DateTime.UtcNow;
                try
                {
                    // do we have a message waiting?
                    // this may block on the server for a few seconds but will return as soon as something is available

                    if (_enumerator.MoveNext())
                    {
                        // yes - record the current position and return it to the client
                        Interlocked.Increment(ref _totalReceived);
                        _lastId = _enumerator.Current.Id;

                        var current = _enumerator.Current;
                        var message = current.Message;
                        var delay   = DateTime.UtcNow - _enumerator.Current.Enqueued;
                        Log.Debug(m => m("Received {0} after {1}", _queueName, delay));
                        return(message);
                    }

                    // if the cursor is dead then we need to re-query, otherwise we just go back to iterating over it
                    if (_enumerator.IsDead)
                    {
                        Log.Debug("Cursor Dead");
                        _enumerator.Dispose();
                        _enumerator = InitializeCursor();
                    }

                    // If we are here it means that the server returned without items, usually it sits in a 2 seconds block on the server, but if initial request is empty then return immedietly.
                    //This is a safety mechanism to protect the mongo server from our DOS...(due to this cursor behavior, and from unknown bugs / 'behaviors')
                    var iterationTime = DateTime.UtcNow - t1;
                    if (iterationTime.TotalMilliseconds < SHORT_SLEEP_INTERVAL) // minimal polling interval : TODO - move to Config
                    {
                        Thread.Sleep(SHORT_SLEEP_INTERVAL);
                    }
                }
                catch (Exception ex)
                {
                    Log.Warn("[ReceiveEvents inside while true] Got exception. Will reinitialize and continue. Ex={0}", ex);
                    // cursor died or was killed
                    if (_enumerator != null)
                    {
                        _enumerator.Dispose();
                    }
                    _enumerator = InitializeCursor();

                    //This is a safety mechanism to protect the mongo server from our DOS...
                    var iterationTime = DateTime.UtcNow - t1;
                    if (iterationTime.TotalMilliseconds < SHORT_SLEEP_INTERVAL) // minimal polling interval : TODO - move to Config
                    {
                        Thread.Sleep(SHORT_SLEEP_INTERVAL);
                    }
                }
            }
        }
示例#5
0
        public void Start()
        {
            try
            {
                Sitecore.Diagnostics.Log.Info("MongoOplogCacheClearer: Starting", this);
                var client            = new MongoClient(_connectionString);
                var server            = client.GetServer();
                var mappingDatabase   = server.GetDatabase(_mongoDatabase);
                var mappingCollection = mappingDatabase.GetCollection(_mappingCollection);

                //run a query against the oplog with a tailable cursor, which will block until new records are created
                var mongoDatabase   = server.GetDatabase(LOCAL);
                var opLog           = mongoDatabase.GetCollection(OPLOG);
                var queryCollection = string.Format("{0}.{1}", _mongoDatabase, _mongoCollection);
                var queryDoc        = new QueryDocument("ns", queryCollection);
                var query           = opLog.Find(queryDoc)
                                      .SetFlags(QueryFlags.AwaitData | QueryFlags.NoCursorTimeout | QueryFlags.TailableCursor);
                var cursor = new MongoCursorEnumerator <BsonDocument>(query);
                while (true)
                {
                    if (cursor.MoveNext())
                    {
                        var document = cursor.Current;
                        if (document["op"].AsString == "d")
                        {
                            //get the deleted document
                            document = document["o"].AsBsonDocument;

                            //TODO: on delete, delete from the mapping collection as well?
                        }
                        else if (document["op"].AsString == "u")
                        {
                            //get the updated document
                            document = document["o2"].AsBsonDocument;
                        }
                        else
                        {
                            continue;
                        }
                        var objectId = document["_id"].AsObjectId;

                        //look in our mapping table for the associated sitecore item id
                        var mapping = mappingCollection.FindOne(new QueryDocument("mongoId", objectId.ToString()));
                        if (mapping == null)
                        {
                            continue;
                        }
                        var id = Sitecore.Data.ShortID.Parse(mapping["sitecoreId"].AsString).ToID();

                        //clear caches for all configured databases, simulate item save
                        foreach (var database in _databases)
                        {
                            var item = database.GetItem(id);
                            if (item == null)
                            {
                                continue;
                            }
                            Sitecore.Diagnostics.Log.Info(
                                string.Format("MongoOplogCacheClearer: Product {0}://{1} updated", database.Name, id), this);
                            database.Caches.ItemCache.RemoveItem(id);
                            database.Caches.DataCache.RemoveItemInformation(id);
                            database.Engines.DataEngine.RaiseSavedItem(item, true);
                            var args = new ItemSavedEventArgs(item);
                            Sitecore.Events.Event.RaiseItemSaved(this, args);
                            //TODO: Something to cause reindex in web, new indexing strategy?
                            //TODO: Clear HTML cache?
                        }
                    }
                    else if (cursor.IsDead)
                    {
                        //TODO: restart mechanism?
                        Sitecore.Diagnostics.Log.Info("MongoOplogCacheClearer: Dead", this);
                        break;
                    }
                }
            }
            catch (Exception e)
            {
                Sitecore.Diagnostics.Log.Error("Exception starting MongoDb oplog monitoring", e, this);
            }
        }
        public void Start()
        {
            try
            {
                Sitecore.Diagnostics.Log.Info("MongoOplogCacheClearer: Starting", this);
                var client = new MongoClient(_connectionString);
                var server = client.GetServer();
                var mappingDatabase = server.GetDatabase(_mongoDatabase);
                var mappingCollection = mappingDatabase.GetCollection(_mappingCollection);

                //run a query against the oplog with a tailable cursor, which will block until new records are created
                var mongoDatabase = server.GetDatabase(LOCAL);
                var opLog = mongoDatabase.GetCollection(OPLOG);
                var queryCollection = string.Format("{0}.{1}", _mongoDatabase, _mongoCollection);
                var queryDoc = new QueryDocument("ns", queryCollection);
                var query = opLog.Find(queryDoc)
                                .SetFlags(QueryFlags.AwaitData | QueryFlags.NoCursorTimeout | QueryFlags.TailableCursor);
                var cursor = new MongoCursorEnumerator<BsonDocument>(query);
                while (true)
                {
                    if (cursor.MoveNext())
                    {
                        var document = cursor.Current;
                        if (document["op"].AsString == "d")
                        {
                            //get the deleted document
                            document = document["o"].AsBsonDocument;

                            //TODO: on delete, delete from the mapping collection as well?
                        }
                        else if (document["op"].AsString == "u")
                        {
                            //get the updated document
                            document = document["o2"].AsBsonDocument;
                        }
                        else
                        {
                            continue;
                        }
                        var objectId = document["_id"].AsObjectId;

                        //look in our mapping table for the associated sitecore item id
                        var mapping = mappingCollection.FindOne(new QueryDocument("mongoId", objectId.ToString()));
                        if (mapping == null)
                        {
                            continue;
                        }
                        var id = Sitecore.Data.ShortID.Parse(mapping["sitecoreId"].AsString).ToID();

                        //clear caches for all configured databases, simulate item save
                        foreach (var database in _databases)
                        {
                            var item = database.GetItem(id);
                            if (item == null)
                            {
                                continue;
                            }
                            Sitecore.Diagnostics.Log.Info(
                                string.Format("MongoOplogCacheClearer: Product {0}://{1} updated", database.Name, id), this);
                            database.Caches.ItemCache.RemoveItem(id);
                            database.Caches.DataCache.RemoveItemInformation(id);
                            database.Engines.DataEngine.RaiseSavedItem(item, true);
                            var args = new ItemSavedEventArgs(item);
                            Sitecore.Events.Event.RaiseItemSaved(this, args);
                            //TODO: Something to cause reindex in web, new indexing strategy?
                            //TODO: Clear HTML cache?
                        }
                    }
                    else if (cursor.IsDead)
                    {
                        //TODO: restart mechanism?
                        Sitecore.Diagnostics.Log.Info("MongoOplogCacheClearer: Dead", this);
                        break;
                    }
                }
            }
            catch (Exception e)
            {
                Sitecore.Diagnostics.Log.Error("Exception starting MongoDb oplog monitoring", e, this);
            }
        }