示例#1
0
        public void Process()
        {
            if (_processingQueue)
            {
                return;
            }

            if (!_config.Enabled)
            {
                _log.Info(typeof(DefaultEventQueue), "Configuration is disabled. The queue will not be processed.");
                return;
            }

            _log.Trace(typeof(DefaultEventQueue), "Processing queue...");
            _processingQueue = true;

            try {
                _storage.CleanupQueueFiles(_config.GetQueueName(), _config.QueueMaxAge, _config.QueueMaxAttempts);
                _storage.ReleaseStaleLocks(_config.GetQueueName());

                DateTime maxCreatedDate = DateTime.Now;
                int      batchSize      = _config.SubmissionBatchSize;
                var      batch          = _storage.GetEventBatch(_config.GetQueueName(), _serializer, batchSize, maxCreatedDate);
                while (batch.Any())
                {
                    bool deleteBatch = true;

                    try {
                        var events   = batch.Select(b => b.Item2).ToList();
                        var response = _client.PostEvents(events, _config, _serializer);
                        if (response.Success)
                        {
                            _log.FormattedInfo(typeof(DefaultEventQueue), "Sent {0} events to \"{1}\".", batch.Count, _config.ServerUrl);
                        }
                        else if (response.ServiceUnavailable)
                        {
                            // You are currently over your rate limit or the servers are under stress.
                            _log.Error(typeof(DefaultEventQueue), "Server returned service unavailable.");
                            SuspendProcessing();
                            deleteBatch = false;
                        }
                        else if (response.PaymentRequired)
                        {
                            // If the organization over the rate limit then discard the event.
                            _log.Info(typeof(DefaultEventQueue), "Too many events have been submitted, please upgrade your plan.");
                            SuspendProcessing(discardFutureQueuedItems: true, clearQueue: true);
                        }
                        else if (response.UnableToAuthenticate)
                        {
                            // The api key was suspended or could not be authorized.
                            _log.Info(typeof(DefaultEventQueue), "Unable to authenticate, please check your configuration. The event will not be submitted.");
                            SuspendProcessing(TimeSpan.FromMinutes(15));
                        }
                        else if (response.NotFound || response.BadRequest)
                        {
                            // The service end point could not be found.
                            _log.FormattedError(typeof(DefaultEventQueue), "Error while trying to submit data: {0}", response.Message);
                            SuspendProcessing(TimeSpan.FromHours(4));
                        }
                        else if (response.RequestEntityTooLarge)
                        {
                            if (batchSize > 1)
                            {
                                _log.Error(typeof(DefaultEventQueue), "Event submission discarded for being too large. The event will be retried with a smaller batch size.");
                                batchSize   = Math.Max(1, (int)Math.Round(batchSize / 1.5d, 0));
                                deleteBatch = false;
                            }
                            else
                            {
                                _log.Error(typeof(DefaultEventQueue), "Event submission discarded for being too large. The event will not be submitted.");
                            }
                        }
                        else if (!response.Success)
                        {
                            _log.Error(typeof(DefaultEventQueue), String.Concat("An error occurred while submitting events: ", response.Message));
                            SuspendProcessing();
                            deleteBatch = false;
                        }

                        OnEventsPosted(new EventsPostedEventArgs {
                            Events = events, Response = response
                        });
                    } catch (AggregateException ex) {
                        _log.Error(typeof(DefaultEventQueue), ex, String.Concat("An error occurred while submitting events: ", ex.Flatten().Message));
                        SuspendProcessing();
                        deleteBatch = false;
                    } catch (Exception ex) {
                        _log.Error(typeof(DefaultEventQueue), ex, String.Concat("An error occurred while submitting events: ", ex.Message));
                        SuspendProcessing();
                        deleteBatch = false;
                    }

                    if (deleteBatch)
                    {
                        _storage.DeleteBatch(batch);
                    }
                    else
                    {
                        _storage.ReleaseBatch(batch);
                    }

                    if (!deleteBatch || IsQueueProcessingSuspended)
                    {
                        break;
                    }

                    batch = _storage.GetEventBatch(_config.GetQueueName(), _serializer, batchSize, maxCreatedDate);
                }
            } catch (Exception ex) {
                _log.Error(typeof(DefaultEventQueue), ex, String.Concat("An error occurred while processing the queue: ", ex.Message));
                SuspendProcessing();
            } finally {
                _processingQueue = false;
            }
        }
        public void CanManageQueue()
        {
            Reset();

            IObjectStorage storage   = GetStorage();
            const string   queueName = "test";

            IJsonSerializer serializer = new DefaultJsonSerializer();
            var             ev         = new Event {
                Type = Event.KnownTypes.Log, Message = "test"
            };

            storage.Enqueue(queueName, ev);
            storage.SaveObject("test.txt", "test");
            Assert.True(storage.GetObjectList().Any(f => f.Path.StartsWith(queueName + "\\q\\") && f.Path.EndsWith("0.json")));
            Assert.Equal(2, storage.GetObjectList().Count());

            Assert.True(storage.GetQueueFiles(queueName).All(f => f.Path.EndsWith("0.json")));
            Assert.Equal(1, storage.GetQueueFiles(queueName).Count());

            storage.DeleteObject("test.txt");
            Assert.Equal(1, storage.GetObjectList().Count());

            Assert.True(storage.LockFile(storage.GetObjectList().FirstOrDefault()));
            Assert.True(storage.GetQueueFiles(queueName).All(f => f.Path.EndsWith("0.json.x")));
            Assert.True(storage.ReleaseFile(storage.GetObjectList().FirstOrDefault()));

            var batch = storage.GetEventBatch(queueName, serializer);

            Assert.Equal(1, batch.Count);

            Assert.True(storage.GetObjectList().All(f => f.Path.StartsWith(queueName + "\\q\\") && f.Path.EndsWith("1.json.x")));
            Assert.Equal(1, storage.GetObjectList().Count());

            Assert.Equal(0, storage.GetQueueFiles(queueName).Count());
            Assert.Equal(0, storage.GetEventBatch(queueName, serializer).Count());

            Assert.False(storage.LockFile(storage.GetObjectList().FirstOrDefault()));

            storage.ReleaseBatch(batch);
            Assert.True(storage.GetObjectList().All(f => f.Path.StartsWith(queueName + "\\q\\") && f.Path.EndsWith("1.json")));
            Assert.Equal(1, storage.GetObjectList().Count());
            Assert.Equal(1, storage.GetQueueFiles(queueName).Count());

            var file = storage.GetObjectList().FirstOrDefault();

            storage.IncrementAttempts(file);
            Assert.True(storage.GetObjectList().All(f => f.Path.StartsWith(queueName + "\\q\\") && f.Path.EndsWith("2.json")));
            storage.IncrementAttempts(file);
            Assert.True(storage.GetObjectList().All(f => f.Path.StartsWith(queueName + "\\q\\") && f.Path.EndsWith("3.json")));

            Assert.True(storage.LockFile(file));
            Assert.NotNull(file);
            Assert.True(storage.GetObjectList().All(f => f.Path.StartsWith(queueName + "\\q\\") && f.Path.EndsWith("3.json.x")));
            Thread.Sleep(TimeSpan.FromMilliseconds(1));
            storage.ReleaseStaleLocks(queueName, TimeSpan.Zero);
            Assert.True(storage.GetObjectList().All(f => f.Path.StartsWith(queueName + "\\q\\") && f.Path.EndsWith("3.json")));

            batch = storage.GetEventBatch(queueName, serializer);
            Assert.Equal(1, batch.Count);
            Assert.True(storage.GetObjectList().All(f => f.Path.StartsWith(queueName + "\\q\\") && f.Path.EndsWith("4.json.x")));
            storage.DeleteBatch(batch);
            Assert.Equal(0, storage.GetQueueFiles(queueName).Count());

            ev = new Event {
                Type = Event.KnownTypes.Log, Message = "test"
            };
            storage.Enqueue(queueName, ev);
            file = storage.GetObjectList().FirstOrDefault();
            Assert.NotNull(file);
            Thread.Sleep(TimeSpan.FromMilliseconds(1));
            storage.CleanupQueueFiles(queueName, TimeSpan.Zero);
            Assert.Equal(0, storage.GetQueueFiles(queueName).Count());
        }