public void Dispose()
{
if (_connection == null)
{
return;
}
Logger.Debug("Closing ZkClient...");
EventLock.Lock();
try
{
ShutdownTrigger = true;
_eventThread.Interrupt();
_eventThread.Join(2000);
_connection.Dispose();
_connection = null;
}
catch (ThreadInterruptedException e)
{
throw new ZkInterruptedException(e);
}
finally
{
EventLock.Unlock();
}
Logger.Debug("Closing ZkClient...done");
}
public void writeMessage(Message msg)
{
if (stopped)
{
throw InterruptedException();
}
EventLock lk = new EventLock(writeEvent);
LevinProtocol proto = new LevinProtocol(connection);
switch (msg.messageType)
{
case P2pContext.Message.NOTIFY:
proto.sendMessage(new uint(msg.type), msg.data, false);
break;
case P2pContext.Message.REQUEST:
proto.sendMessage(new uint(msg.type), msg.data, true);
break;
case P2pContext.Message.REPLY:
proto.sendReply(new uint(msg.type), msg.data, new uint(msg.returnCode));
break;
}
}
public bool WaitForKeeperState(KeeperState keeperState, TimeSpan timeout)
{
if (_zookeeperEventThread != null && Thread.CurrentThread == _zookeeperEventThread)
{
throw new Exception("Must not be done in the zookeeper event thread.");
}
Logger.Debug("Waiting for keeper state " + keeperState);
this.AcquireEventLock();
try
{
bool stillWaiting = true;
while (CurrentState != keeperState)
{
if (!stillWaiting)
{
return(false);
}
stillWaiting = EventLock.StateChangedCondition.Await(timeout);
}
Logger.Debug("State is " + CurrentState);
return(true);
}
catch (ThreadInterruptedException e)
{
throw new ZkInterruptedException(e);
}
finally
{
EventLock.Unlock();
}
}
public bool WaitUntilExists(String path, TimeSpan time)
{
DateTime timeout = DateTime.Now + time;
Logger.Debug("Waiting until znode '" + path + "' becomes available.");
if (Exists(path))
{
return(true);
}
AcquireEventLock();
try
{
while (!Exists(path, true))
{
var gotSignal = EventLock.ZNodeEventCondition.AwaitUntil(timeout);
if (!gotSignal)
{
return(false);
}
}
return(true);
}
catch (ThreadInterruptedException e)
{
throw new ZkInterruptedException("Thread interrupted", e);
}
finally
{
EventLock.Unlock();
}
}
/// <inheritdoc />
public async Task Release(EventLock eventLock)
{
using (var conn = this.connectionFactory.GetEddsPerformanceConnection())
{
await conn.ExecuteAsync(Resources.EventLock_Release, eventLock);
}
}
private void AcquireEventLock()
{
try
{
EventLock.LockInterruptibly();
}
catch (ThreadInterruptedException e)
{
throw new ZkInterruptedException(e);
}
}
public bool readCommand(LevinProtocol.Command cmd)
{
if (stopped)
{
throw InterruptedException();
}
EventLock lk = new EventLock(readEvent);
bool result = new LevinProtocol(connection).readCommand(cmd);
lastReadTime = Clock.now();
return(result);
}
public async Task HandleEvent_Success(bool eventResultSuccess)
{
var eventId = 1234;
var eventType = EventSourceType.ScoreCategoryScore;
var evnt = new Event {
Id = eventId, SourceType = eventType, Status = EventStatus.PendingHangfire
};
this.eventTaskFactory.Setup(f => f.GetEventTask(It.IsAny <IKernel>(), evnt.SourceType)).Returns(this.eventTask.Object);
this.eventRepository.Setup(m => m.ReadAsync(eventId)).ReturnsAsync(evnt);
var eventResult = new EventResult(evnt.SourceType)
{
Succeeded = eventResultSuccess
};
this.eventTask.Setup(t => t.ProcessEvent(It.IsAny <Event>())).ReturnsAsync(eventResult);
var kernelWrapper = new Mock <IKernelWrapper>();
var kernel = new Mock <IKernel>();
kernelWrapper.SetupGet(w => w.Kernel).Returns(kernel.Object);
this.eventChildKernelFactory.Setup(f => f.CreateChildKernel(evnt))
.Returns(kernelWrapper.Object);
var eventLock = new EventLock
{
Id = 1,
EventId = evnt.Id,
EventTypeId = evnt.SourceTypeId,
SourceId = evnt.SourceId
};
this.eventLockRepositoryMock.Setup(m => m.Claim(evnt, It.IsAny <int>())).ReturnsAsync(eventLock);
this.eventLockRepositoryMock.Setup(m => m.Release(eventLock)).Returns(Task.Delay(5));
this.eventRepository.Setup(
m => m.UpdateAsync(It.Is <Event>(e => e.Id == eventId && e.Status == EventStatus.InProgress))).Returns(Task.Delay(5));
// Act
await this.eventHandler.HandleEvent(eventId, eventType);
// Assert
this.eventTaskFactory.Verify(m => m.GetEventTask(It.IsAny <IKernel>(), evnt.SourceType), Times.Once);
this.eventRepository.Verify(m => m.ReadAsync(evnt.Id), Times.Once);
this.eventLockRepositoryMock.Verify(m => m.Claim(evnt, It.IsAny <int>()), Times.Once);
this.eventRepository.Verify(m => m.UpdateAsync(It.Is <Event>(e => e.Status == EventStatus.InProgress && e.Id == evnt.Id)), Times.Once);
this.eventTask.Verify(m => m.ProcessEvent(It.Is <Event>(e => e.Id == evnt.Id)), Times.Once);
this.eventTask.Verify(m => m.MarkEventResultAsync(eventResult, evnt), Times.Once);
this.eventLockRepositoryMock.Verify(m => m.Release(eventLock), Times.Once);
this.eventTask.Verify(m => m.CreateNextEvents(eventResult, It.Is <Event>(e => e.Id == evnt.Id)), Times.Once());
}
private void Reconnect()
{
EventLock.Lock();
try
{
_connection.Dispose();
_connection.Connect(this);
}
catch (ThreadInterruptedException e)
{
throw new ZkInterruptedException(e);
}
finally
{
EventLock.Unlock();
}
}
public async Task HandleEvent_ExceptionWithLock()
{
var eventId = 1234;
var eventType = EventSourceType.ScoreCategoryScore;
var evnt = new Event {
Id = eventId, SourceType = eventType, Status = EventStatus.PendingHangfire
};
this.eventTaskFactory.Setup(f => f.GetEventTask(It.IsAny <IKernel>(), evnt.SourceType)).Returns(this.eventTask.Object);
this.eventRepository.Setup(m => m.ReadAsync(eventId)).ReturnsAsync(evnt);
var kernelWrapper = new Mock <IKernelWrapper>();
var kernel = new Mock <IKernel>();
kernelWrapper.SetupGet(w => w.Kernel).Returns(kernel.Object);
this.eventChildKernelFactory.Setup(f => f.CreateChildKernel(evnt))
.Returns(kernelWrapper.Object);
var eventLock = new EventLock
{
Id = 1,
EventId = evnt.Id,
EventTypeId = evnt.SourceTypeId,
SourceId = evnt.SourceId
};
this.eventLockRepositoryMock.Setup(m => m.Claim(evnt, It.IsAny <int>())).ReturnsAsync(eventLock);
this.eventLockRepositoryMock.Setup(m => m.Release(eventLock)).Returns(Task.Delay(5));
this.eventRepository.Setup(m => m.UpdateAsync(It.IsAny <Event>())).Returns(Task.Delay(5));
this.eventTask.Setup(t => t.ProcessEvent(It.IsAny <Event>())).Throws <Exception>();
// Act
await this.eventHandler.HandleEvent(eventId, eventType);
// Assert
this.eventLockRepositoryMock.Verify(m => m.Claim(evnt, It.IsAny <int>()), Times.Once);
this.eventTaskFactory.Verify(m => m.GetEventTask(It.IsAny <IKernel>(), evnt.SourceType), Times.Once);
this.eventRepository.Verify(m => m.ReadAsync(evnt.Id), Times.Exactly(2)); // once for initial read and again to mark for error
this.eventRepository.Verify(m => m.UpdateAsync(It.Is <Event>(e => e.Status == EventStatus.Error))); // for when it errors
this.eventTask.Verify(m => m.ProcessEvent(It.Is <Event>(e => e.Id == evnt.Id)), Times.Once);
this.eventLockRepositoryMock.Verify(m => m.Release(eventLock), Times.Once);
}
public long GetCreationTime(String path)
{
try
{
EventLock.LockInterruptibly();
return(_connection.GetCreateTime(path));
}
catch (KeeperException e)
{
throw ZkException.Create(e);
}
catch (ThreadInterruptedException e)
{
throw new ZkInterruptedException(e);
}
finally
{
EventLock.Unlock();
}
}
public void Connect(long maxMsToWaitUntilConnected, IWatcher watcher)
{
bool started = false;
try
{
EventLock.LockInterruptibly();
ShutdownTrigger = false;
_eventThread = new ZkEventThread(_connection.Servers);
_eventThread.Start();
_connection.Connect(watcher);
Logger.Debug("Awaiting connection to Zookeeper server");
if (!WaitUntilConnected(TimeSpan.FromMilliseconds(maxMsToWaitUntilConnected)))
{
throw new ZkTimeoutException("Unable to connect to zookeeper server within timeout: " + maxMsToWaitUntilConnected);
}
started = true;
}
catch (ThreadInterruptedException)
{
ZooKeeper.States state = _connection.ZookeeperState;
throw new Exception("Not connected with zookeeper server yet. Current state is " + state);
}
finally
{
EventLock.Unlock();
// we should close the zookeeper instance, otherwise it would keep
// on trying to connect
if (!started)
{
this.Dispose();
}
}
}
public async Task HandleEvent(long eventId, EventSourceType eventType)
{
EventLock eventLock = null;
try
{
// Grab the event
var evnt = await this.eventRepository.ReadAsync(eventId);
// if the event is not pending hangire then we wont process it. This can happen because of event migration or orphaned events.
if (evnt.Status != EventStatus.PendingHangfire)
{
this.logger.LogError($"Cannot process event {eventId} with status: {evnt.Status}");
// Mark the event errored (unless it's already errored, cancelled, or expired)
if (evnt.Status != EventStatus.Cancelled &&
evnt.Status != EventStatus.Error &&
evnt.Status != EventStatus.Expired)
{
await this.MarkEventErrored(eventId);
}
return;
}
// Add the event to the kernel context
using (var eventKernel = this.eventChildKernelFactory.CreateChildKernel(evnt))
{
// Try to put a lock on it
var currentWorker = await this.eventWorkerService.GetCurrentWorker();
ThrowOn.IsNull(currentWorker, "Event worker");
eventLock = await this.eventLockRepository.Claim(evnt, currentWorker.Id);
if (eventLock != null)
{
// If successful...
// Update to In Progress (from Hangfire Pending) and save
evnt.Status = EventStatus.InProgress;
await this.eventRepository.UpdateAsync(evnt);
// Grab the event task
var eventTask = this.eventTaskFactory.GetEventTask(eventKernel.Kernel, eventType);
// Process the event
var eventResult = await eventTask.ProcessEvent(evnt);
// Release the lock
await this.eventLockRepository.Release(eventLock);
await eventTask.MarkEventResultAsync(eventResult, evnt);
// Create the next events to call after this
await eventTask.CreateNextEvents(eventResult, evnt);
}
else
{
// If not successful, mark event as a duplicate and not execute
evnt.Status = EventStatus.Duplicate;
await this.eventRepository.UpdateAsync(evnt);
}
}
}
catch (Exception ex)
{
this.logger.LogError($"Failed to run handleEvent: {eventId}", ex);
// Attempt to mark the event errored
await this.MarkEventErrored(eventId);
// If we have a lock that we made, but failed after event execution
if (eventLock != null && eventLock.EventId == eventId)
{
// Attempt to release the lock
await this.eventLockRepository.Release(eventLock);
}
}
}
