public void CanConcurrentlyManageFiles()
{
Reset();
IFileStorage storage = GetStorage();
IJsonSerializer serializer = new DefaultJsonSerializer();
const string queueName = "test";
Parallel.For(0, 25, i => {
var ev = new Event {
Type = Event.KnownTypes.Log,
Message = "test" + i
};
storage.Enqueue(queueName, ev, serializer);
});
Assert.Equal(25, storage.GetFileList().Count());
var working = new ConcurrentDictionary <string, object>();
Parallel.For(0, 50, i => {
var fileBatch = storage.GetEventBatch(queueName, serializer, 2);
foreach (var f in fileBatch)
{
if (working.ContainsKey(f.Item1.Path))
{
Debug.WriteLine(f.Item1.Path);
}
Assert.False(working.ContainsKey(f.Item1.Path));
working.TryAdd(f.Item1.Path, null);
}
if (RandomHelper.GetBool())
{
object o;
foreach (var f in fileBatch)
{
working.TryRemove(f.Item1.Path, out o);
}
storage.ReleaseBatch(fileBatch);
}
else
{
storage.DeleteBatch(fileBatch);
}
});
Assert.Equal(25, working.Count + storage.GetQueueFiles(queueName).Count);
}
public void Process()
{
if (_processingQueue)
{
return;
}
_log.Info(typeof(DefaultEventQueue), "Processing queue...");
if (!_config.Enabled)
{
_log.Info(typeof(DefaultEventQueue), "Configuration is disabled. The queue will not be processed.");
return;
}
_processingQueue = true;
try {
_storage.CleanupQueueFiles(_config.GetQueueName());
_storage.ReleaseStaleLocks(_config.GetQueueName());
var batch = _storage.GetEventBatch(_config.GetQueueName(), _serializer);
while (batch.Any())
{
bool deleteBatch = true;
try {
var response = _client.PostEvents(batch.Select(b => b.Item2), _config, _serializer);
if (response.ServiceUnavailable)
{
// You are currently over your rate limit or the servers are under stress.
_log.Error(typeof(DefaultEventQueue), "Server returned service unavailable.");
SuspendProcessing();
deleteBatch = false;
}
else if (response.PaymentRequired)
{
// If the organization over the rate limit then discard the event.
_log.Info(typeof(DefaultEventQueue), "Too many events have been submitted, please upgrade your plan.");
SuspendProcessing(discardFutureQueuedItems: true, clearQueue: true);
}
else if (response.UnableToAuthenticate)
{
// The api key was suspended or could not be authorized.
_log.Info(typeof(DefaultEventQueue), "Unable to authenticate, please check your configuration. The event will not be submitted.");
SuspendProcessing(TimeSpan.FromMinutes(15));
}
else if (response.NotFound || response.BadRequest)
{
// The service end point could not be found.
_log.Error(typeof(DefaultEventQueue), "Unable to reach the service end point, please check your configuration. The event will not be submitted.");
SuspendProcessing(TimeSpan.FromHours(4));
}
else if (!response.Success)
{
_log.Error(typeof(DefaultEventQueue), String.Format("An error occurred while submitting events: {0}", response.Message));
SuspendProcessing();
deleteBatch = false;
}
} catch (Exception ex) {
_log.Error(typeof(DefaultEventQueue), ex, String.Concat("An error occurred while submitting events: ", ex.Message));
SuspendProcessing();
deleteBatch = false;
}
if (deleteBatch)
{
_storage.DeleteBatch(batch);
}
else
{
_storage.ReleaseBatch(batch);
}
if (!deleteBatch || IsQueueProcessingSuspended)
{
break;
}
batch = _storage.GetEventBatch(_config.GetQueueName(), _serializer);
}
} catch (Exception ex) {
_log.Error(typeof(DefaultEventQueue), ex, String.Concat("An error occurred while processing the queue: ", ex.Message));
SuspendProcessing();
} finally {
_processingQueue = false;
}
}
public void CanManageQueue()
{
Reset();
IFileStorage storage = GetStorage();
const string queueName = "test";
IJsonSerializer serializer = new DefaultJsonSerializer();
var ev = new Event {
Type = Event.KnownTypes.Log, Message = "test"
};
storage.Enqueue(queueName, ev, serializer);
storage.SaveFile("test.txt", "test");
Assert.True(storage.GetFileList().Any(f => f.Path.StartsWith(queueName + "\\q\\") && f.Path.EndsWith("0.json")));
Assert.Equal(2, storage.GetFileList().Count());
Assert.True(storage.GetQueueFiles(queueName).All(f => f.Path.EndsWith("0.json")));
Assert.Equal(1, storage.GetQueueFiles(queueName).Count());
storage.DeleteFile("test.txt");
Assert.Equal(1, storage.GetFileList().Count());
Assert.True(storage.LockFile(storage.GetFileList().FirstOrDefault()));
Assert.True(storage.GetQueueFiles(queueName).All(f => f.Path.EndsWith("0.json.x")));
Assert.True(storage.ReleaseFile(storage.GetFileList().FirstOrDefault()));
var batch = storage.GetEventBatch(queueName, serializer);
Assert.Equal(1, batch.Count);
Assert.True(storage.GetFileList().All(f => f.Path.StartsWith(queueName + "\\q\\") && f.Path.EndsWith("1.json.x")));
Assert.Equal(1, storage.GetFileList().Count());
Assert.Equal(0, storage.GetQueueFiles(queueName).Count());
Assert.Equal(0, storage.GetEventBatch(queueName, serializer).Count());
Assert.False(storage.LockFile(storage.GetFileList().FirstOrDefault()));
storage.ReleaseBatch(batch);
Assert.True(storage.GetFileList().All(f => f.Path.StartsWith(queueName + "\\q\\") && f.Path.EndsWith("1.json")));
Assert.Equal(1, storage.GetFileList().Count());
Assert.Equal(1, storage.GetQueueFiles(queueName).Count());
var file = storage.GetFileList().FirstOrDefault();
storage.IncrementAttempts(file);
Assert.True(storage.GetFileList().All(f => f.Path.StartsWith(queueName + "\\q\\") && f.Path.EndsWith("2.json")));
storage.IncrementAttempts(file);
Assert.True(storage.GetFileList().All(f => f.Path.StartsWith(queueName + "\\q\\") && f.Path.EndsWith("3.json")));
Assert.True(storage.LockFile(file));
Assert.NotNull(file);
Assert.True(storage.GetFileList().All(f => f.Path.StartsWith(queueName + "\\q\\") && f.Path.EndsWith("3.json.x")));
Thread.Sleep(TimeSpan.FromMilliseconds(1));
storage.ReleaseStaleLocks(queueName, TimeSpan.Zero);
Assert.True(storage.GetFileList().All(f => f.Path.StartsWith(queueName + "\\q\\") && f.Path.EndsWith("3.json")));
batch = storage.GetEventBatch(queueName, serializer);
Assert.Equal(1, batch.Count);
Assert.True(storage.GetFileList().All(f => f.Path.StartsWith(queueName + "\\q\\") && f.Path.EndsWith("4.json.x")));
storage.DeleteBatch(batch);
Assert.Equal(0, storage.GetQueueFiles(queueName).Count());
ev = new Event {
Type = Event.KnownTypes.Log, Message = "test"
};
storage.Enqueue(queueName, ev, serializer);
file = storage.GetFileList().FirstOrDefault();
Assert.NotNull(file);
Thread.Sleep(TimeSpan.FromMilliseconds(1));
storage.CleanupQueueFiles(queueName, TimeSpan.Zero);
Assert.Equal(0, storage.GetQueueFiles(queueName).Count());
}
public void Process()
{
if (IsQueueProcessingSuspended)
{
return;
}
_log.Info(typeof(DefaultEventQueue), "Processing queue...");
if (!_config.Enabled)
{
_log.Info(typeof(DefaultEventQueue), "Configuration is disabled. The queue will not be processed.");
return;
}
if (_processingQueue)
{
_log.Info(typeof(DefaultEventQueue), "The queue is already being processed.");
return;
}
_processingQueue = true;
try {
var batch = _storage.GetEventBatch(_serializer);
if (!batch.Any())
{
_log.Info(typeof(DefaultEventQueue), "There are no events in the queue to process.");
return;
}
bool deleteBatch = true;
try {
var response = _client.Submit(batch.Select(b => b.Item2), _config, _serializer);
if (response.ServiceUnavailable)
{
// You are currently over your rate limit or the servers are under stress.
_log.Error(typeof(DefaultEventQueue), "Server returned service unavailable.");
SuspendProcessing();
deleteBatch = false;
}
else if (response.PaymentRequired)
{
// If the organization over the rate limit then discard the error.
_log.Info(typeof(DefaultEventQueue), "Too many errors have been submitted, please upgrade your plan.");
SuspendProcessing(discardFutureQueuedItems: true, clearQueue: true);
}
else if (response.UnableToAuthenticate)
{
// The api key was suspended or could not be authorized.
_log.Info(typeof(DefaultEventQueue), "Unable to authenticate, please check your configuration. The error will not be submitted.");
SuspendProcessing(TimeSpan.FromMinutes(15));
}
else if (response.NotFound)
{
// The service end point could not be found.
_log.Error(typeof(DefaultEventQueue), "Unable to reach the service end point, please check your configuration. The error will not be submitted.");
SuspendProcessing(TimeSpan.FromHours(4));
}
else if (!response.Success)
{
deleteBatch = false;
}
} catch (Exception ex) {
_log.Error(typeof(DefaultEventQueue), ex, String.Concat("An error occurred while submitting events: ", ex.Message));
deleteBatch = false;
}
if (deleteBatch)
{
_storage.DeleteBatch(batch);
}
else
{
_storage.ReleaseBatch(batch);
}
} catch (Exception ex) {
_log.Error(typeof(DefaultEventQueue), ex, String.Concat("An error occurred while processing the queue: ", ex.Message));
} finally {
_processingQueue = false;
}
//if (response.ShouldUpdateConfiguration(LocalConfiguration.CurrentConfigurationVersion))
// UpdateConfiguration(true);
//completed = new SendErrorCompletedEventArgs(id, exception, false, error);
//OnSendErrorCompleted(completed);
// TODO: Check to see if the configuration needs to be updated.
}
