public async Task TryLockAsync_WithContention_NoRetry_DoesNotPollForLease()
{
CancellationToken cancellationToken = new CancellationToken();
_mockStorageBlob.Setup(p => p.UploadTextAsync(string.Empty, null, It.Is <AccessCondition>(q => q.IfNoneMatchETag == "*"), null, null, cancellationToken)).Returns(Task.FromResult(true));
int count = 0;
_mockStorageBlob.Setup(p => p.AcquireLeaseAsync(_singletonConfig.LockPeriod, null, cancellationToken))
.Returns(() =>
{
count++;
return(Task.FromResult <string>(null));
});
SingletonAttribute attribute = new SingletonAttribute();
SingletonManager.SingletonLockHandle lockHandle = (SingletonManager.SingletonLockHandle)
await _singletonManager.TryLockAsync(TestLockId, TestInstanceId, attribute, cancellationToken, retry : false);
Assert.Null(lockHandle);
Assert.Equal(1, count);
_mockStorageBlob.VerifyAll();
}
public async Task TryLockAsync_CreatesBlob_WhenItDoesNotExist()
{
CancellationToken cancellationToken = new CancellationToken();
RequestResult storageResult = new RequestResult
{
HttpStatusCode = 404
};
StorageException storageException = new StorageException(storageResult, null, null);
int count = 0;
_mockStorageBlob.Setup(p => p.AcquireLeaseAsync(_singletonConfig.LockPeriod, null, cancellationToken)).Returns(() =>
{
if (count++ == 0)
{
throw storageException;
}
return(Task.FromResult(TestLeaseId));
});
_mockStorageBlob.Setup(p => p.UploadTextAsync(string.Empty, null, null, null, null, cancellationToken)).Returns(Task.FromResult(true));
_mockStorageBlob.SetupGet(p => p.Metadata).Returns(_mockBlobMetadata);
_mockStorageBlob.Setup(p => p.SetMetadataAsync(It.Is <AccessCondition>(q => q.LeaseId == TestLeaseId), null, null, cancellationToken)).Returns(Task.FromResult(true));
_mockStorageBlob.Setup(p => p.ReleaseLeaseAsync(It.Is <AccessCondition>(q => q.LeaseId == TestLeaseId), null, null, cancellationToken)).Returns(Task.FromResult(true));
SingletonAttribute attribute = new SingletonAttribute();
SingletonManager.SingletonLockHandle lockHandle = (SingletonManager.SingletonLockHandle) await _singletonManager.TryLockAsync(TestLockId, TestInstanceId, attribute, cancellationToken);
Assert.Same(_mockStorageBlob.Object, lockHandle.Blob);
Assert.Equal(TestLeaseId, lockHandle.LeaseId);
Assert.Equal(1, _mockStorageBlob.Object.Metadata.Keys.Count);
Assert.Equal(_mockStorageBlob.Object.Metadata[SingletonManager.FunctionInstanceMetadataKey], TestInstanceId);
}
public async Task TryLockAsync_WithContention_PollsForLease()
{
CancellationToken cancellationToken = new CancellationToken();
_mockStorageBlob.SetupGet(p => p.Metadata).Returns(_mockBlobMetadata);
_mockStorageBlob.Setup(p => p.UploadTextAsync(string.Empty, null, It.Is <AccessCondition>(q => q.IfNoneMatchETag == "*"), null, null, cancellationToken)).Returns(Task.FromResult(true));
_mockStorageBlob.Setup(p => p.SetMetadataAsync(It.Is <AccessCondition>(q => q.LeaseId == TestLeaseId), null, null, cancellationToken)).Returns(Task.FromResult(true));
int numRetries = 3;
int count = 0;
_mockStorageBlob.Setup(p => p.AcquireLeaseAsync(_singletonConfig.LockPeriod, null, cancellationToken))
.Returns(() =>
{
count++;
return(Task.FromResult <string>(count > numRetries ? TestLeaseId : null));
});
SingletonAttribute attribute = new SingletonAttribute();
SingletonManager.SingletonLockHandle lockHandle = (SingletonManager.SingletonLockHandle) await _singletonManager.TryLockAsync(TestLockId, TestInstanceId, attribute, cancellationToken);
Assert.Equal(TestLeaseId, lockHandle.LeaseId);
Assert.Equal(numRetries, count - 1);
Assert.NotNull(lockHandle.LeaseRenewalTimer);
_mockStorageBlob.VerifyAll();
}
public async Task ReleaseAsync_InvokesSingletonManager_WithExpectedValues()
{
CancellationToken cancellationToken = new CancellationToken();
SingletonAttribute attribute = new SingletonAttribute();
SingletonManager.SingletonLockHandle handle = new SingletonManager.SingletonLockHandle();
Mock <SingletonManager> mockSingletonManager = new Mock <SingletonManager>(MockBehavior.Strict);
mockSingletonManager.Setup(p => p.LockAsync(TestLockId, TestInstanceId, attribute, cancellationToken)).ReturnsAsync(handle);
mockSingletonManager.Setup(p => p.ReleaseLockAsync(handle, cancellationToken)).Returns(Task.FromResult(true));
SingletonLock singletonLock = new SingletonLock(TestLockId, TestInstanceId, attribute, mockSingletonManager.Object);
Assert.False(singletonLock.IsHeld);
await singletonLock.AcquireAsync(cancellationToken);
Assert.True(singletonLock.IsHeld);
await singletonLock.ReleaseAsync(cancellationToken);
Assert.False(singletonLock.IsHeld);
Assert.NotNull(singletonLock.AcquireStartTime);
Assert.NotNull(singletonLock.AcquireEndTime);
Assert.NotNull(singletonLock.ReleaseTime);
}
public async Task TryLockAsync_CreatesBlobLease_WithAutoRenewal()
{
CancellationToken cancellationToken = new CancellationToken();
_mockStorageBlob.SetupGet(p => p.Metadata).Returns(_mockBlobMetadata);
_mockStorageBlob.Setup(p => p.AcquireLeaseAsync(_singletonConfig.LockPeriod, null, cancellationToken)).ReturnsAsync(TestLeaseId);
_mockStorageBlob.Setup(p => p.SetMetadataAsync(It.Is <AccessCondition>(q => q.LeaseId == TestLeaseId), null, null, cancellationToken)).Returns(Task.FromResult(true));
_mockStorageBlob.Setup(p => p.ReleaseLeaseAsync(It.Is <AccessCondition>(q => q.LeaseId == TestLeaseId), null, null, cancellationToken)).Returns(Task.FromResult(true));
int renewCount = 0;
_mockStorageBlob.Setup(p => p.RenewLeaseAsync(It.Is <AccessCondition>(q => q.LeaseId == TestLeaseId), null, null, It.IsAny <CancellationToken>()))
.Callback <AccessCondition, BlobRequestOptions, OperationContext, CancellationToken>(
(mockAccessCondition, mockOptions, mockContext, mockCancellationToken) =>
{
renewCount++;
}).Returns(Task.FromResult(true));
SingletonAttribute attribute = new SingletonAttribute();
SingletonManager.SingletonLockHandle lockHandle = (SingletonManager.SingletonLockHandle) await _singletonManager.TryLockAsync(TestLockId, TestInstanceId, attribute, cancellationToken);
Assert.Same(_mockStorageBlob.Object, lockHandle.Blob);
Assert.Equal(TestLeaseId, lockHandle.LeaseId);
Assert.Equal(1, _mockStorageBlob.Object.Metadata.Keys.Count);
Assert.Equal(_mockStorageBlob.Object.Metadata[SingletonManager.FunctionInstanceMetadataKey], TestInstanceId);
// wait for enough time that we expect some lease renewals to occur
int duration = 2000;
int expectedRenewalCount = (int)(duration / (_singletonConfig.LockPeriod.TotalMilliseconds / 2)) - 1;
await Task.Delay(duration);
Assert.Equal(expectedRenewalCount, renewCount);
// now release the lock and verify no more renewals
await _singletonManager.ReleaseLockAsync(lockHandle, cancellationToken);
// verify the traces
Assert.Equal(1, _trace.Traces.Count(p => p.ToString().Contains("Verbose Singleton lock acquired (testid)")));
Assert.Equal(1, _trace.Traces.Count(p => p.ToString().Contains("Verbose Singleton lock released (testid)")));
// verify the logger
TestLogger logger = _loggerProvider.CreatedLoggers.Single() as TestLogger;
Assert.Equal(LogCategories.Singleton, logger.Category);
Assert.Equal(2, logger.LogMessages.Count);
Assert.NotNull(logger.LogMessages.Single(m => m.Level == Extensions.Logging.LogLevel.Debug && m.FormattedMessage == "Singleton lock acquired (testid)"));
Assert.NotNull(logger.LogMessages.Single(m => m.Level == Extensions.Logging.LogLevel.Debug && m.FormattedMessage == "Singleton lock released (testid)"));
renewCount = 0;
await Task.Delay(1000);
Assert.Equal(0, renewCount);
_mockStorageBlob.VerifyAll();
}
public async Task StartAsync_StartsListener_WhenLockAcquired()
{
CancellationToken cancellationToken = new CancellationToken();
SingletonManager.SingletonLockHandle lockHandle = new SingletonManager.SingletonLockHandle();
_mockSingletonManager.Setup(p => p.TryLockAsync(_lockId, null, _attribute, cancellationToken)).ReturnsAsync(lockHandle);
_mockInnerListener.Setup(p => p.StartAsync(cancellationToken)).Returns(Task.FromResult(true));
await _listener.StartAsync(cancellationToken);
_mockSingletonManager.VerifyAll();
_mockInnerListener.VerifyAll();
}
public async Task StartAsync_StartsListener_WhenLockAcquired()
{
CancellationToken cancellationToken = new CancellationToken();
SingletonManager.SingletonLockHandle lockHandle = new SingletonManager.SingletonLockHandle();
_mockSingletonManager.Setup(p => p.TryLockAsync(_lockId, null, _attribute, cancellationToken)).ReturnsAsync(lockHandle);
_mockInnerListener.Setup(p => p.StartAsync(cancellationToken)).Returns(Task.FromResult(true));
await _listener.StartAsync(cancellationToken);
_mockSingletonManager.VerifyAll();
_mockInnerListener.VerifyAll();
}
public async Task TryLockAsync_CreatesBlobLease_WithAutoRenewal()
{
CancellationToken cancellationToken = new CancellationToken();
_mockStorageBlob.SetupGet(p => p.Metadata).Returns(_mockBlobMetadata);
_mockStorageBlob.Setup(p => p.UploadTextAsync(string.Empty, null, It.Is <AccessCondition>(q => q.IfNoneMatchETag == "*"), null, null, cancellationToken)).Returns(Task.FromResult(true));
_mockStorageBlob.Setup(p => p.AcquireLeaseAsync(_singletonConfig.LockPeriod, null, cancellationToken)).ReturnsAsync(TestLeaseId);
_mockStorageBlob.Setup(p => p.SetMetadataAsync(It.Is <AccessCondition>(q => q.LeaseId == TestLeaseId), null, null, cancellationToken)).Returns(Task.FromResult(true));
_mockStorageBlob.Setup(p => p.ReleaseLeaseAsync(It.Is <AccessCondition>(q => q.LeaseId == TestLeaseId), null, null, cancellationToken)).Returns(Task.FromResult(true));
int renewCount = 0;
_mockStorageBlob.Setup(p => p.RenewLeaseAsync(It.Is <AccessCondition>(q => q.LeaseId == TestLeaseId), null, null, It.IsAny <CancellationToken>()))
.Callback <AccessCondition, BlobRequestOptions, OperationContext, CancellationToken>(
(mockAccessCondition, mockOptions, mockContext, mockCancellationToken) =>
{
renewCount++;
}).Returns(Task.FromResult(true));
SingletonAttribute attribute = new SingletonAttribute();
SingletonManager.SingletonLockHandle lockHandle = (SingletonManager.SingletonLockHandle) await _singletonManager.TryLockAsync(TestLockId, TestInstanceId, attribute, cancellationToken);
Assert.Same(_mockStorageBlob.Object, lockHandle.Blob);
Assert.Equal(TestLeaseId, lockHandle.LeaseId);
Assert.Equal(1, _mockStorageBlob.Object.Metadata.Keys.Count);
Assert.Equal(_mockStorageBlob.Object.Metadata[SingletonManager.FunctionInstanceMetadataKey], TestInstanceId);
// wait for enough time that we expect some lease renewals to occur
int duration = 2000;
int expectedRenewalCount = (int)(duration / (_singletonConfig.LockPeriod.TotalMilliseconds / 2)) - 1;
await Task.Delay(duration);
Assert.Equal(expectedRenewalCount, renewCount);
// now release the lock and verify no more renewals
await _singletonManager.ReleaseLockAsync(lockHandle, cancellationToken);
// verify the traces
Assert.Equal(1, _trace.Traces.Count(p => p.Contains("Verbose WebJobs.Execution Waiting for Singleton lock (testid)")));
Assert.Equal(1, _trace.Traces.Count(p => p.Contains("Verbose WebJobs.Execution Singleton lock acquired (testid)")));
Assert.Equal(renewCount, _trace.Traces.Count(p => p.Contains("Renewing Singleton lock (testid)")));
Assert.Equal(1, _trace.Traces.Count(p => p.Contains("Verbose WebJobs.Execution Singleton lock released (testid)")));
renewCount = 0;
await Task.Delay(1000);
Assert.Equal(0, renewCount);
_mockStorageBlob.VerifyAll();
}
public async Task TryAcquireLock_WhenLockAcquired_StopsLockTimerAndStartsListener()
{
_listener.LockTimer = new System.Timers.Timer
{
Interval = 30 * 1000
};
_listener.LockTimer.Start();
SingletonManager.SingletonLockHandle lockHandle = new SingletonManager.SingletonLockHandle();
_mockSingletonManager.Setup(p => p.TryLockAsync(_lockId, null, _attribute, CancellationToken.None, false))
.ReturnsAsync(lockHandle);
_mockInnerListener.Setup(p => p.StartAsync(CancellationToken.None)).Returns(Task.FromResult(true));
await _listener.TryAcquireLock();
Assert.Null(_listener.LockTimer);
}
public async Task AquireAsync_InvokesSingletonManager_WithExpectedValues()
{
CancellationToken cancellationToken = new CancellationToken();
SingletonAttribute attribute = new SingletonAttribute();
SingletonManager.SingletonLockHandle handle = new SingletonManager.SingletonLockHandle();
Mock<SingletonManager> mockSingletonManager = new Mock<SingletonManager>(MockBehavior.Strict);
mockSingletonManager.Setup(p => p.LockAsync(TestLockId, TestInstanceId, attribute, cancellationToken)).ReturnsAsync(handle);
SingletonLock singletonLock = new SingletonLock(TestLockId, TestInstanceId, attribute, mockSingletonManager.Object);
Assert.False(singletonLock.IsHeld);
await singletonLock.AcquireAsync(cancellationToken);
Assert.NotNull(singletonLock.AcquireStartTime);
Assert.NotNull(singletonLock.AcquireEndTime);
Assert.True(singletonLock.IsHeld);
}
public async Task ReleaseLockAsync_StopsRenewalTimerAndReleasesLease()
{
CancellationToken cancellationToken = new CancellationToken();
Mock <ITaskSeriesTimer> mockRenewalTimer = new Mock <ITaskSeriesTimer>(MockBehavior.Strict);
mockRenewalTimer.Setup(p => p.StopAsync(cancellationToken)).Returns(Task.FromResult(true));
_mockStorageBlob.Setup(p => p.ReleaseLeaseAsync(It.Is <AccessCondition>(q => q.LeaseId == TestLeaseId), null, null, cancellationToken)).Returns(Task.FromResult(true));
SingletonManager.SingletonLockHandle handle = new SingletonManager.SingletonLockHandle
{
Blob = _mockStorageBlob.Object,
LeaseId = TestLeaseId,
LeaseRenewalTimer = mockRenewalTimer.Object
};
await _singletonManager.ReleaseLockAsync(handle, cancellationToken);
mockRenewalTimer.VerifyAll();
}
public async Task StartAsync_DefaultsAcquisitionTimeout()
{
int expectedTimeoutSeconds = (int)_config.LockPeriod.TotalSeconds;
CancellationToken cancellationToken = new CancellationToken();
SingletonManager.SingletonLockHandle lockHandle = new SingletonManager.SingletonLockHandle();
_mockSingletonManager.Setup(p => p.TryLockAsync(_lockId, null, _attribute, cancellationToken))
.Callback<string, string, SingletonAttribute, CancellationToken>(
(mockLockId, mockInstanceId, mockAttribute, mockCancellationToken) =>
{
Assert.Equal(expectedTimeoutSeconds, mockAttribute.LockAcquisitionTimeout);
})
.ReturnsAsync(lockHandle);
_mockInnerListener.Setup(p => p.StartAsync(cancellationToken)).Returns(Task.FromResult(true));
Assert.Null(_attribute.LockAcquisitionTimeout);
await _listener.StartAsync(cancellationToken);
Assert.Equal(expectedTimeoutSeconds, _attribute.LockAcquisitionTimeout);
_mockSingletonManager.VerifyAll();
_mockInnerListener.VerifyAll();
}
public async Task StartAsync_DefaultsAcquisitionTimeout()
{
int expectedTimeoutSeconds = (int)_config.LockPeriod.TotalSeconds;
CancellationToken cancellationToken = new CancellationToken();
SingletonManager.SingletonLockHandle lockHandle = new SingletonManager.SingletonLockHandle();
_mockSingletonManager.Setup(p => p.TryLockAsync(_lockId, null, _attribute, cancellationToken))
.Callback <string, string, SingletonAttribute, CancellationToken>(
(mockLockId, mockInstanceId, mockAttribute, mockCancellationToken) =>
{
Assert.Equal(expectedTimeoutSeconds, mockAttribute.LockAcquisitionTimeout);
})
.ReturnsAsync(lockHandle);
_mockInnerListener.Setup(p => p.StartAsync(cancellationToken)).Returns(Task.FromResult(true));
Assert.Null(_attribute.LockAcquisitionTimeout);
await _listener.StartAsync(cancellationToken);
Assert.Equal(expectedTimeoutSeconds, _attribute.LockAcquisitionTimeout);
_mockSingletonManager.VerifyAll();
_mockInnerListener.VerifyAll();
}
public async Task TryAcquireLock_WhenLockAcquired_StopsLockTimerAndStartsListener()
{
_listener.LockTimer = new System.Timers.Timer
{
Interval = 30 * 1000
};
_listener.LockTimer.Start();
SingletonManager.SingletonLockHandle lockHandle = new SingletonManager.SingletonLockHandle();
_mockSingletonManager.Setup(p => p.TryLockAsync(_lockId, null, _attribute, CancellationToken.None, false))
.ReturnsAsync(lockHandle);
_mockInnerListener.Setup(p => p.StartAsync(CancellationToken.None)).Returns(Task.FromResult(true));
await _listener.TryAcquireLock();
Assert.Null(_listener.LockTimer);
}
public async Task ReleaseLockAsync_StopsRenewalTimerAndReleasesLease()
{
CancellationToken cancellationToken = new CancellationToken();
Mock<ITaskSeriesTimer> mockRenewalTimer = new Mock<ITaskSeriesTimer>(MockBehavior.Strict);
mockRenewalTimer.Setup(p => p.StopAsync(cancellationToken)).Returns(Task.FromResult(true));
_mockStorageBlob.Setup(p => p.ReleaseLeaseAsync(It.Is<AccessCondition>(q => q.LeaseId == TestLeaseId), null, null, cancellationToken)).Returns(Task.FromResult(true));
SingletonManager.SingletonLockHandle handle = new SingletonManager.SingletonLockHandle
{
Blob = _mockStorageBlob.Object,
LeaseId = TestLeaseId,
LeaseRenewalTimer = mockRenewalTimer.Object
};
await _singletonManager.ReleaseLockAsync(handle, cancellationToken);
mockRenewalTimer.VerifyAll();
}
