public async Task HasLeaseChanged_WhenLeaseIsAcquired() { var host = CreateHost(); string connectionString = host.GetStorageConnectionString(); string hostId = host.GetHostId(); ICloudBlob blob = await GetLockBlobAsync(connectionString, hostId); // Acquire a lease on the host lock blob string leaseId = await blob.AcquireLeaseAsync(TimeSpan.FromMinutes(1)); using (host) { await host.StartAsync(); var primaryState = host.Services.GetService <IPrimaryHostStateProvider>(); // The test owns the lease, so the host doesn't have it. Assert.False(primaryState.IsPrimary); // Now release it, and we should reclaim it. await blob.ReleaseLeaseAsync(new AccessCondition { LeaseId = leaseId }); await TestHelpers.Await(() => primaryState.IsPrimary, userMessageCallback : () => $"{nameof(IPrimaryHostStateProvider.IsPrimary)} was not correctly set to 'true' when lease was acquired."); await host.StopAsync(); } await ClearLeaseBlob(connectionString, hostId); }
public async Task HasLeaseChanged_WhenLeaseIsAcquiredAndStateChanges_IsFired() { string hostId = Guid.NewGuid().ToString(); string instanceId = Guid.NewGuid().ToString(); var resetEvent = new ManualResetEventSlim(); string connectionString = AmbientConnectionStringProvider.Instance.GetConnectionString(ConnectionStringNames.Storage); ICloudBlob blob = await GetLockBlobAsync(connectionString, hostId); // Acquire a lease on the host lock blob string leaseId = await blob.AcquireLeaseAsync(TimeSpan.FromSeconds(15)); PrimaryHostCoordinator manager = null; try { manager = PrimaryHostCoordinator.Create(CreateLockManager(), TimeSpan.FromSeconds(15), hostId, instanceId, _loggerFactory); manager.HasLeaseChanged += (s, a) => resetEvent.Set(); } finally { await blob.ReleaseLeaseAsync(new AccessCondition { LeaseId = leaseId }); } resetEvent.Wait(TimeSpan.FromSeconds(15)); bool hasLease = manager.HasLease; manager.Dispose(); Assert.True(resetEvent.IsSet); Assert.True(hasLease, $"{nameof(PrimaryHostCoordinator.HasLease)} was not correctly set to 'true' when lease was acquired."); await ClearLeaseBlob(hostId); }
public async Task Dispose_ReleasesBlobLease() { string hostId = Guid.NewGuid().ToString(); string instanceId = Guid.NewGuid().ToString(); string connectionString = AmbientConnectionStringProvider.Instance.GetConnectionString(ConnectionStringNames.Storage); var traceWriter = new TestTraceWriter(System.Diagnostics.TraceLevel.Verbose); using (var manager = await BlobLeaseManager.CreateAsync(connectionString, TimeSpan.FromSeconds(15), hostId, instanceId, traceWriter)) { await TestHelpers.Await(() => manager.HasLease); } ICloudBlob blob = await GetLockBlobAsync(connectionString, hostId); string leaseId = null; try { // Acquire a lease on the host lock blob leaseId = await blob.AcquireLeaseAsync(TimeSpan.FromSeconds(15)); await blob.ReleaseLeaseAsync(new AccessCondition { LeaseId = leaseId }); } catch (StorageException exc) when(exc.RequestInformation.HttpStatusCode == 409) { } Assert.False(string.IsNullOrEmpty(leaseId), "Failed to acquire a blob lease. The lease was not properly released."); await ClearLeaseBlob(hostId); }
private async Task AcquireOrRenewLeaseAsync() { try { DateTime requestStart = DateTime.UtcNow; if (HasLease) { await _lockBlob.RenewLeaseAsync(new AccessCondition { LeaseId = LeaseId }); _lastRenewal = DateTime.UtcNow; _lastRenewalLatency = _lastRenewal - requestStart; } else { LeaseId = await _lockBlob.AcquireLeaseAsync(_leaseTimeout, _instanceId); _lastRenewal = DateTime.UtcNow; _lastRenewalLatency = _lastRenewal - requestStart; _traceWriter.Info($"Host lock lease acquired by instance ID '{_instanceId}'."); // We've successfully acquired the lease, change the timer to use our renewal interval SetTimerInterval(_renewalInterval); } } catch (StorageException exc) { if (exc.RequestInformation.HttpStatusCode == 409) { // If we did not have the lease already, a 409 indicates that another host had it. This is // normal and does not warrant any logging. if (HasLease) { // The lease was 'stolen'. Log details for debugging. string lastRenewalFormatted = _lastRenewal.ToString("yyyy-MM-ddTHH:mm:ss.FFFZ", CultureInfo.InvariantCulture); int millisecondsSinceLastSuccess = (int)(DateTime.UtcNow - _lastRenewal).TotalMilliseconds; int lastRenewalMilliseconds = (int)_lastRenewalLatency.TotalMilliseconds; ProcessLeaseError($"Another host has acquired the lease. The last successful renewal completed at {lastRenewalFormatted} ({millisecondsSinceLastSuccess} milliseconds ago) with a duration of {lastRenewalMilliseconds} milliseconds."); } } else if (exc.RequestInformation.HttpStatusCode >= 500) { ProcessLeaseError($"Server error {exc.RequestInformation.HttpStatusMessage}."); } else if (exc.RequestInformation.HttpStatusCode == 404) { // The blob or container do not exist, reset the lease information ResetLease(); // Create the blob and retry _lockBlob = await GetLockBlobAsync(_lockBlob.ServiceClient, GetBlobName(_hostId)); await AcquireOrRenewLeaseAsync(); } else { throw; } } }
public static async Task <string> TryAcquireLeaseAsync(this ICloudBlob blob, TimeSpan leaseTime, CancellationToken cancellationToken) { string leaseId; try { var sourceBlobExists = await blob.ExistsAsync(cancellationToken); if (!sourceBlobExists) { return(null); } leaseId = await blob.AcquireLeaseAsync(leaseTime, null, cancellationToken); } catch (StorageException storageException) { // check if this is a 409 Conflict with a StatusDescription stating that "There is already a lease present." // or 404 NotFound (might have been removed by another other instance of this job) var webException = storageException.InnerException as WebException; var httpWebResponse = webException?.Response as HttpWebResponse; if (httpWebResponse != null) { if ((httpWebResponse.StatusCode == HttpStatusCode.Conflict && httpWebResponse.StatusDescription == "There is already a lease present.") || httpWebResponse.StatusCode == HttpStatusCode.NotFound) { return(null); } } throw; } return(leaseId); }
private async Task AcquireOrRenewLeaseAsync() { try { if (HasLease) { await _lockBlob.RenewLeaseAsync(new AccessCondition { LeaseId = LeaseId }); _traceWriter.Verbose("Host lock lease renewed."); } else { LeaseId = await _lockBlob.AcquireLeaseAsync(_leaseTimeout, _instanceId); _traceWriter.Info($"Host lock lease acquired by instance ID '{_instanceId}'."); // We've successfully acquired the lease, change the timer to use our renewal interval SetTimerInterval(_renewalInterval); } } catch (StorageException exc) { if (exc.RequestInformation.HttpStatusCode == 409) { ProcessLeaseError("Another host has an active lease."); } else if (exc.RequestInformation.HttpStatusCode >= 500) { ProcessLeaseError($"Server error {exc.RequestInformation.HttpStatusMessage}."); } else if (exc.RequestInformation.HttpStatusCode == 404) { // The blob or container do not exist, reset the lease information ResetLease(); // Create the blob and retry _lockBlob = await GetLockBlobAsync(_lockBlob.ServiceClient, GetBlobName(_hostId)); await AcquireOrRenewLeaseAsync(); } else { throw; } } }
public async Task HasLeaseChanged_WhenLeaseIsLostAndStateChanges_IsFired() { string hostId = Guid.NewGuid().ToString(); string instanceId = Guid.NewGuid().ToString(); string connectionString = AmbientConnectionStringProvider.Instance.GetConnectionString(ConnectionStringNames.Storage); ICloudBlob blob = await GetLockBlobAsync(connectionString, hostId); var traceWriter = new TestTraceWriter(TraceLevel.Verbose); var resetEvent = new ManualResetEventSlim(); PrimaryHostCoordinator manager = null; string tempLeaseId = null; var lockManager = CreateLockManager(); var renewalInterval = TimeSpan.FromSeconds(3); using (manager = PrimaryHostCoordinator.Create(lockManager, TimeSpan.FromSeconds(15), hostId, instanceId, traceWriter, null, renewalInterval)) { try { await TestHelpers.Await(() => manager.HasLease); manager.HasLeaseChanged += (s, a) => resetEvent.Set(); // Release the manager's lease and acquire one with a different id await lockManager.ReleaseLockAsync(manager.LockHandle, CancellationToken.None); tempLeaseId = await blob.AcquireLeaseAsync(TimeSpan.FromSeconds(30), Guid.NewGuid().ToString()); } finally { if (tempLeaseId != null) { await blob.ReleaseLeaseAsync(new AccessCondition { LeaseId = tempLeaseId }); } } resetEvent.Wait(TimeSpan.FromSeconds(15)); } Assert.True(resetEvent.IsSet); Assert.False(manager.HasLease, $"{nameof(PrimaryHostCoordinator.HasLease)} was not correctly set to 'false' when lease lost."); await ClearLeaseBlob(hostId); }
private async Task <string> TryAcquireLeaseAsync(ICloudBlob blob) { string leaseId; var blobUriString = blob.Uri.ToString(); try { var sourceBlobExists = await blob.ExistsAsync(); if (!sourceBlobExists) { return(null); } _logger.LogDebug("Beginning to acquire lease for blob {BlobUri}.", blobUriString); leaseId = await blob.AcquireLeaseAsync(_defaultLeaseTime); _logger.LogInformation("Finishing to acquire lease for blob {BlobUri}.", blobUriString); } catch (StorageException storageException) { // check if this is a 409 Conflict with a StatusDescription stating that "There is already a lease present." // or 404 NotFound (might have been removed by another other instance of this job) var webException = storageException.InnerException as WebException; if (webException != null) { var httpWebResponse = webException.Response as HttpWebResponse; if (httpWebResponse != null) { if ((httpWebResponse.StatusCode == HttpStatusCode.Conflict && httpWebResponse.StatusDescription == "There is already a lease present.") || httpWebResponse.StatusCode == HttpStatusCode.NotFound) { _logger.LogDebug("Failed to acquire lease for blob {BlobUri}.", blobUriString); // no need to report these in Application Insights return(null); } } } _logger.LogError(LogEvents.FailedBlobLease, storageException, "Failed to acquire lease for blob {BlobUri}.", blobUriString); throw; } return(leaseId); }
public async Task HasLeaseChanged_WhenLeaseIsLost() { var host = CreateHost(); string connectionString = host.GetStorageConnectionString(); string hostId = host.GetHostId(); using (host) { ICloudBlob blob = await GetLockBlobAsync(connectionString, hostId); var primaryState = host.Services.GetService <IPrimaryHostStateProvider>(); var manager = host.Services.GetServices <IHostedService>().OfType <PrimaryHostCoordinator>().Single(); var lockManager = host.Services.GetService <IDistributedLockManager>(); string tempLeaseId = null; await host.StartAsync(); try { await TestHelpers.Await(() => primaryState.IsPrimary); // Release the manager's lease and acquire one with a different id await lockManager.ReleaseLockAsync(manager.LockHandle, CancellationToken.None); tempLeaseId = await blob.AcquireLeaseAsync(TimeSpan.FromSeconds(30), Guid.NewGuid().ToString()); await TestHelpers.Await(() => !primaryState.IsPrimary, userMessageCallback : () => $"{nameof(IPrimaryHostStateProvider.IsPrimary)} was not correctly set to 'false' when lease lost."); } finally { if (tempLeaseId != null) { await blob.ReleaseLeaseAsync(new AccessCondition { LeaseId = tempLeaseId }); } } await host.StopAsync(); } await ClearLeaseBlob(connectionString, hostId); }
public async Task <string> TryAcquireLease() { try { var leaseTime = TimeSpan.FromSeconds(_renewIntervalInSeconds); _leaseId = await _blob.AcquireLeaseAsync(leaseTime, null); KeepAliveRxTimer = Observable.Interval(TimeSpan.FromSeconds(_renewIntervalInSeconds - 9.0)).Subscribe(async l => await RenewLease(_blob, _leaseId)); } catch (StorageException ex) { _leaseId = null; Trace.TraceInformation(string.Format("Failed to acquire the lease on blob {0}", _blob), ex); } if (_leaseId == null) { await DisableTimer(); } return(_leaseId); }
private async Task<string> TryAcquireLeaseAsync(ICloudBlob blob) { string leaseId; var blobUriString = blob.Uri.ToString(); try { var sourceBlobExists = await blob.ExistsAsync(); if (!sourceBlobExists) { return null; } _jobEventSource.BeginningAcquireLease(blobUriString); leaseId = await blob.AcquireLeaseAsync(_defaultLeaseTime, null); _jobEventSource.FinishedAcquireLease(blobUriString); } catch (StorageException storageException) { // check if this is a 409 Conflict with a StatusDescription stating that "There is already a lease present." // or 404 NotFound (might have been removed by another other instance of this job) var webException = storageException.InnerException as WebException; if (webException != null) { var httpWebResponse = webException.Response as HttpWebResponse; if (httpWebResponse != null) { if ((httpWebResponse.StatusCode == HttpStatusCode.Conflict && httpWebResponse.StatusDescription == "There is already a lease present.") || httpWebResponse.StatusCode == HttpStatusCode.NotFound) { _jobEventSource.FailedAcquireLease(blobUriString); // no need to report these in Application Insights return null; } } } _jobEventSource.FailedAcquireLease(blobUriString); ApplicationInsights.TrackException(storageException); throw; } return leaseId; }
public async Task Dispose_ReleasesBlobLease() { var host = CreateHost(); string connectionString = host.GetStorageConnectionString(); string hostId = host.GetHostId(); var primaryHostCoordinator = host.Services.GetServices <IHostedService>().OfType <PrimaryHostCoordinator>().Single(); using (host) { await host.StartAsync(); var primaryState = host.Services.GetService <IPrimaryHostStateProvider>(); await TestHelpers.Await(() => primaryState.IsPrimary); await host.StopAsync(); } // Container disposal is a fire-and-forget so this service disposal could be delayed. This will force it. primaryHostCoordinator.Dispose(); ICloudBlob blob = await GetLockBlobAsync(connectionString, hostId); string leaseId = null; try { // Acquire a lease on the host lock blob leaseId = await blob.AcquireLeaseAsync(TimeSpan.FromSeconds(15)); await blob.ReleaseLeaseAsync(new AccessCondition { LeaseId = leaseId }); } catch (StorageException exc) when(exc.RequestInformation.HttpStatusCode == 409) { } Assert.False(string.IsNullOrEmpty(leaseId), "Failed to acquire a blob lease. The lease was not properly released."); await ClearLeaseBlob(connectionString, hostId); }
public async Task HasLeaseChanged_WhenLeaseIsAcquiredAndStateChanges_IsFired() { string hostId = Guid.NewGuid().ToString(); string instanceId = Guid.NewGuid().ToString(); var traceWriter = new TestTraceWriter(TraceLevel.Verbose); var resetEvent = new ManualResetEventSlim(); string connectionString = AmbientConnectionStringProvider.Instance.GetConnectionString(ConnectionStringNames.Storage); ICloudBlob blob = await GetLockBlobAsync(connectionString, hostId); // Acquire a lease on the host lock blob string leaseId = await blob.AcquireLeaseAsync(TimeSpan.FromSeconds(15)); BlobLeaseManager manager = null; try { manager = await BlobLeaseManager.CreateAsync(connectionString, TimeSpan.FromSeconds(15), hostId, instanceId, traceWriter); manager.HasLeaseChanged += (s, a) => resetEvent.Set(); } finally { await blob.ReleaseLeaseAsync(new AccessCondition { LeaseId = leaseId }); } resetEvent.Wait(TimeSpan.FromSeconds(15)); manager.Dispose(); Assert.True(resetEvent.IsSet); Assert.True(manager.HasLease, $"{nameof(BlobLeaseManager.HasLease)} was not correctly set to 'true' when lease was acquired."); Assert.Equal(instanceId, manager.LeaseId); await ClearLeaseBlob(hostId); }
public static async Task <IActionResult> RunAsync( [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequest request, [Blob("locks/" + nameof(GenerateMissingTwitterCards), Connection = "AzureWebJobsStorage")] ICloudBlob lockBlob, ILogger log, ExecutionContext context) { // Lock present? string lockIdentifier = null; if (!await lockBlob.ExistsAsync()) { await lockBlob.UploadFromByteArrayAsync(new byte[1], 0, 1); } try { lockIdentifier = await lockBlob.AcquireLeaseAsync(TimeSpan.FromMinutes(1)); } catch (StorageException ex) when(ex.RequestInformation?.HttpStatusCode == (int)HttpStatusCode.Conflict || ex.RequestInformation?.HttpStatusCode == (int)HttpStatusCode.PreconditionFailed) { return(new OkObjectResult("Could not acquire lock, skipping execution.")); } // Read configuration var configuration = new ConfigurationBuilder() .SetBasePath(context.FunctionAppDirectory) .AddJsonFile("local.settings.json", optional: true, reloadOnChange: true) .AddEnvironmentVariables() .Build(); // Defaults var cardStyle = CardStyle.DarkWithBackgroundImage; var cardWidth = 876; var cardHeight = 438; var textPadding = 25; var shadowOffset = 10; var titleSize = 42; var authorSize = 28; var titleLocation = cardStyle == CardStyle.Light ? new PointF(textPadding, cardHeight / 3) : new PointF(textPadding, cardHeight / 3.6f); var authorLocation = cardStyle == CardStyle.Light ? new PointF(textPadding, cardHeight / 3 + authorSize) : new PointF(textPadding, cardHeight / 4 + authorSize * 2); var font = Environment.OSVersion.Platform == PlatformID.Unix ? SystemFonts.Find("DejaVu Sans") : SystemFonts.Find("Segoe UI"); var yamlDeserializer = new DeserializerBuilder() .WithNamingConvention(CamelCaseNamingConvention.Instance) .IgnoreUnmatchedProperties() .Build(); // Create the Octokit client var github = new GitHubClient(new ProductHeaderValue("PostCommentToPullRequest"), new Octokit.Internal.InMemoryCredentialStore(new Credentials(configuration["GitHubToken"]))); // Get all contents from our repo var repoOwnerName = configuration["PullRequestRepository"].Split('/'); var repo = await github.Repository.Get(repoOwnerName[0], repoOwnerName[1]); var defaultBranch = await github.Repository.Branch.Get(repo.Id, repo.DefaultBranch); var repoContentsPosts = await github.Repository.Content.GetAllContents(repoOwnerName[0], repoOwnerName[1], "_posts"); var repoContentsCards = await github.Repository.Content.GetAllContents(repoOwnerName[0], repoOwnerName[1], "images/cards"); Reference newBranch = null; var itemsCreated = 0; foreach (var repoPost in repoContentsPosts) { // Is there a card? if (repoContentsCards.Any(it => it.Name == repoPost.Name + ".png")) { continue; } // If not, generate one! var postData = await github.Repository.Content.GetRawContent(repoOwnerName[0], repoOwnerName[1], repoPost.Path); if (postData == null) { continue; } var frontMatterYaml = Encoding.UTF8.GetString(postData); var frontMatterYamlTemp = frontMatterYaml.Split("---"); if (frontMatterYamlTemp.Length >= 2) { // Deserialize front matter frontMatterYaml = frontMatterYamlTemp[1]; var frontMatter = yamlDeserializer .Deserialize <FrontMatter>(frontMatterYaml); // Cleanup front matter frontMatter.Title = frontMatter.Title.Replace("&", "&"); // Generate card image using var cardImage = new Image <Rgba32>(cardWidth, cardHeight); if (cardStyle == CardStyle.Light) { // Shadow and box DrawRectangle(cardImage, shadowOffset, shadowOffset, cardWidth - shadowOffset, cardHeight - shadowOffset, Color.Gray); DrawRectangle(cardImage, 0, 0, cardWidth - shadowOffset, cardHeight - shadowOffset, Color.White); // Title DrawText(cardImage, titleLocation.X, titleLocation.Y, cardWidth - textPadding - textPadding - textPadding - shadowOffset, Color.Black, font.CreateFont(titleSize, FontStyle.Bold), frontMatter.Title); // Author & date DrawText(cardImage, authorLocation.X, authorLocation.Y, cardWidth - textPadding - textPadding - textPadding - shadowOffset, Color.DarkGray, font.CreateFont(authorSize), (frontMatter.Author ?? "") + (frontMatter.Date?.ToString(" | MMMM dd, yyyy", CultureInfo.InvariantCulture) ?? "")); } else if (cardStyle == CardStyle.DarkWithBackgroundImage) { // Draw background image using var backgroundImage = Image.Load(System.IO.Path.Combine(context.FunctionAppDirectory, "Images", "TwitterCardBackground.png")); DrawImage(cardImage, 0, 0, cardWidth, cardHeight, backgroundImage); // Title DrawText(cardImage, titleLocation.X, titleLocation.Y, cardWidth - textPadding - textPadding - textPadding - textPadding, Color.White, font.CreateFont(titleSize, FontStyle.Bold), frontMatter.Title); // Author & date DrawText(cardImage, authorLocation.X, authorLocation.Y, cardWidth - textPadding - textPadding - textPadding - textPadding, Color.White, font.CreateFont(authorSize, FontStyle.Italic), (frontMatter.Author ?? "") + (frontMatter.Date?.ToString(" | MMMM dd, yyyy", CultureInfo.InvariantCulture) ?? "")); } // Render card image await using var memoryStream = new MemoryStream(); cardImage.Save(memoryStream, PngFormat.Instance); memoryStream.Position = 0; // Create a pull request for it if (newBranch == null) { newBranch = await github.Git.Reference.Create(repo.Id, new NewReference($"refs/heads/twitter-cards-" + Guid.NewGuid(), defaultBranch.Commit.Sha)); } var latestCommit = await github.Git.Commit.Get(repo.Id, newBranch.Object.Sha); var file = new NewBlob { Encoding = EncodingType.Base64, Content = Convert.ToBase64String(memoryStream.ToArray()) }; var blob = await github.Git.Blob.Create(repo.Id, file); var nt = new NewTree { BaseTree = latestCommit.Sha }; nt.Tree.Add(new NewTreeItem { Path = $"images/cards/{repoPost.Name}.png", Mode = "100644", Type = TreeType.Blob, Sha = blob.Sha }); var newTree = await github.Git.Tree.Create(repo.Id, nt); var newCommit = new NewCommit($"[Automatic] Add a Twitter card for: {frontMatter.Title}", newTree.Sha, latestCommit.Sha); var commit = await github.Git.Commit.Create(repo.Id, newCommit); newBranch = await github.Git.Reference.Update(repo.Id, newBranch.Ref, new ReferenceUpdate(commit.Sha)); log.LogInformation($"Generated Twitter card for: {frontMatter.Title}"); // Renew lease try { await lockBlob.RenewLeaseAsync(new AccessCondition { LeaseId = lockIdentifier }); } catch (StorageException ex) when(ex.RequestInformation?.HttpStatusCode == (int)HttpStatusCode.Conflict || ex.RequestInformation?.HttpStatusCode == (int)HttpStatusCode.PreconditionFailed) { break; } // Stop after X items if (++itemsCreated >= 25) { break; } } } // Create PR if (newBranch != null) { await github.Repository.PullRequest.Create(repo.Id, new NewPullRequest($"[Automatic] Add missing Twitter cards", newBranch.Ref, defaultBranch.Name) { Body = $"Add Twitter cards for various posts" }); } // Release lock await lockBlob.ReleaseLeaseAsync(new AccessCondition { LeaseId = lockIdentifier }); return(new OkObjectResult("Done.")); }
/// <summary> /// Puts the lease on the given blob in a leased state. /// </summary> /// <param name="blob">The blob with the lease.</param> /// <param name="leaseTime">The amount of time on the new lease.</param> /// <returns>The lease ID of the current lease.</returns> internal static string SetLeasedStateTask(ICloudBlob blob, TimeSpan? leaseTime) { string leaseId = Guid.NewGuid().ToString(); SetAvailableStateTask(blob); return blob.AcquireLeaseAsync(leaseTime, leaseId).Result; }
/// <summary> /// Puts the lease on the given blob in a leased state. /// </summary> /// <param name="blob">The blob with the lease.</param> /// <param name="leaseTime">The amount of time on the new lease.</param> /// <returns>The lease ID of the current lease.</returns> internal static async Task<string> SetLeasedStateAsync(ICloudBlob blob, TimeSpan? leaseTime) { string leaseId = Guid.NewGuid().ToString(); await SetAvailableStateAsync(blob); return await blob.AcquireLeaseAsync(leaseTime, leaseId); }
/// <summary>Ensures that 1 machine will execute an operation periodically.</summary> /// <param name="startTime">The base time; all machines should use the same exact base time.</param> /// <param name="period">Indicates how frequently you want the operation performed.</param> /// <param name="timeBetweenLeaseRetries">Indicates how frequently a machine that is not elected to perform the work should retry the work in case the elected machine crashes while performing the work.</param> /// <param name="blob">The blob that all the machines should be using to acquire a lease.</param> /// <param name="cancellationToken">Indicates when the operation should no longer be performed in the future.</param> /// <param name="winnerWorker">A method that is called periodically by whatever machine wins the election.</param> /// <returns>A Task which you can use to catch an exception or known then this method has been canceled.</returns> public static async Task RunAsync(DateTimeOffset startTime, TimeSpan period, TimeSpan timeBetweenLeaseRetries, ICloudBlob blob, CancellationToken cancellationToken, Func <Task> winnerWorker) { var bro = new BlobRequestOptions { RetryPolicy = new ExponentialRetry() }; const Int32 leaseDurationSeconds = 60; // 15-60 seconds DateTimeOffset nextElectionTime = NextElectionTime(startTime, period); while (true) { TimeSpan timeToWait = nextElectionTime - DateTimeOffset.UtcNow; timeToWait = (timeToWait < TimeSpan.Zero) ? TimeSpan.Zero : timeToWait; await Task.Delay(timeToWait, cancellationToken).ConfigureAwait(false); // Wait until time to check ElectionError electionError = ElectionError.Unknown; try { // Try to acquire lease & check metadata to see if this period has been processed String leaseId = await blob.AcquireLeaseAsync(TimeSpan.FromSeconds(leaseDurationSeconds), null, AccessCondition.GenerateIfNotModifiedSinceCondition(nextElectionTime), bro, null, CancellationToken.None).ConfigureAwait(false); try { // Got lease: do elected work (periodically renew lease) Task winnerWork = Task.Run(winnerWorker); while (true) { // Verify if winnerWork throws, we exit this loop & release the lease to try again Task wakeUp = await Task.WhenAny(Task.Delay(TimeSpan.FromSeconds(leaseDurationSeconds - 15)), winnerWork).ConfigureAwait(false); if (wakeUp == winnerWork) // Winner work is done { if (winnerWork.IsFaulted) { throw winnerWork.Exception; } else { break; } } await blob.RenewLeaseAsync(AccessCondition.GenerateLeaseCondition(leaseId)).ConfigureAwait(false); } // After work done, write to blob to indicate elected winner sucessfully did work blob.UploadFromByteArray(new Byte[1], 0, 1, AccessCondition.GenerateLeaseCondition(leaseId), bro, null); nextElectionTime = NextElectionTime(nextElectionTime + period, period); } finally { blob.ReleaseLease(AccessCondition.GenerateLeaseCondition(leaseId)); } } catch (StorageException ex) { if (ex.Matches(HttpStatusCode.Conflict, BlobErrorCodeStrings.LeaseAlreadyPresent)) { electionError = ElectionError.LeaseAlreadyPresent; } else if (ex.Matches(HttpStatusCode.PreconditionFailed)) { electionError = ElectionError.ElectionOver; } else { throw; } } switch (electionError) { case ElectionError.ElectionOver: // If access condition failed, the election is over, wait until next election time nextElectionTime = NextElectionTime(nextElectionTime + period, period); break; case ElectionError.LeaseAlreadyPresent: // if failed to get lease, wait a bit and retry again await Task.Delay(timeBetweenLeaseRetries).ConfigureAwait(false); break; } } // We never get here }
private async Task<Tuple<IDisposable, string>> LockInternal(ICloudBlob blob) { string leaseId; try { leaseId = await blob.AcquireLeaseAsync(TimeSpan.FromMinutes(1), null).ConfigureAwait(false); LeoTrace.WriteLine("Leased Blob: " + blob.Name); } catch (StorageException e) { // If we have a conflict this blob is already locked... if (e.RequestInformation.HttpStatusCode == 409) { return null; } if (e.RequestInformation.HttpStatusCode == 404) { leaseId = null; } else { throw e.Wrap(blob.Name); } } // May not have had a blob pushed... if (leaseId == null) { try { using (var stream = new MemoryStream(new byte[1])) { try { await blob.UploadFromStreamAsync(stream).ConfigureAwait(false); } catch (StorageException) { } // Just eat storage exceptions at this point... something was created obviously } leaseId = await blob.AcquireLeaseAsync(TimeSpan.FromMinutes(1), null).ConfigureAwait(false); LeoTrace.WriteLine("Created new blob and lease (2 calls): " + blob.Name); } catch (StorageException e) { // If we have a conflict this blob is already locked... if (e.RequestInformation.HttpStatusCode == 409) { return null; } if (e.RequestInformation.HttpStatusCode == 404) { leaseId = null; } else { throw e.Wrap(blob.Name); } } } var condition = AccessCondition.GenerateLeaseCondition(leaseId); // Every 30 secs keep the lock renewed var keepAlive = AsyncEnumerableEx.CreateTimer(TimeSpan.FromSeconds(30)) .Select(t => { LeoTrace.WriteLine("Renewed Lease: " + blob.Name); return blob.RenewLeaseAsync(condition); }) .Unwrap() .TakeUntilDisposed(null, t => { try { blob.ReleaseLease(condition); } catch (Exception e) { LeoTrace.WriteLine("Release failed: " + e.Message); } }); return Tuple.Create((IDisposable)keepAlive, leaseId); }
private async Task <Tuple <IDisposable, string> > LockInternal(ICloudBlob blob) { string leaseId; try { leaseId = await blob.AcquireLeaseAsync(TimeSpan.FromMinutes(1), null).ConfigureAwait(false); LeoTrace.WriteLine("Leased Blob: " + blob.Name); } catch (StorageException e) { // If we have a conflict this blob is already locked... if (e.RequestInformation.HttpStatusCode == 409) { return(null); } if (e.RequestInformation.HttpStatusCode == 404) { leaseId = null; } else { throw e.Wrap(blob.Name); } } // May not have had a blob pushed... if (leaseId == null) { try { using (var stream = new MemoryStream(new byte[1])) { try { await blob.UploadFromStreamAsync(stream).ConfigureAwait(false); } catch (StorageException) { } // Just eat storage exceptions at this point... something was created obviously } leaseId = await blob.AcquireLeaseAsync(TimeSpan.FromMinutes(1), null).ConfigureAwait(false); LeoTrace.WriteLine("Created new blob and lease (2 calls): " + blob.Name); } catch (StorageException e) { // If we have a conflict this blob is already locked... if (e.RequestInformation.HttpStatusCode == 409) { return(null); } if (e.RequestInformation.HttpStatusCode == 404) { return(null); } else { throw e.Wrap(blob.Name); } } } var condition = AccessCondition.GenerateLeaseCondition(leaseId); // Every 30 secs keep the lock renewed var keepAlive = AsyncEnumerableEx.CreateTimer(TimeSpan.FromSeconds(30)) .Select(t => { LeoTrace.WriteLine("Renewed Lease: " + blob.Name); return(blob.RenewLeaseAsync(condition)); }) .Unwrap() .TakeUntilDisposed(null, t => { try { // We need to do this to make sure after the dispose the lease is gone blob.ReleaseLeaseAsync(condition).GetAwaiter().GetResult(); } catch (Exception e) { LeoTrace.WriteLine("Release failed: " + e.Message); } }); return(Tuple.Create((IDisposable)keepAlive, leaseId)); }