internal AzureBlobLease(AzureBlobLease source)
: base(source)
{
this.Offset = source.Offset;
this.SequenceNumber = source.SequenceNumber;
this.Blob = source.Blob;
}
async Task <bool> ReleaseLeaseCoreAsync(AzureBlobLease lease)
{
ProcessorEventSource.Log.AzureStorageManagerInfo(this.host.Id, lease.PartitionId, "Releasing lease");
CloudBlockBlob leaseBlob = lease.Blob;
bool retval = true;
string partitionId = lease.PartitionId;
try
{
string leaseId = lease.Token;
AzureBlobLease releasedCopy = new AzureBlobLease(lease)
{
Token = string.Empty,
Owner = string.Empty
};
await leaseBlob.UploadTextAsync(JsonConvert.SerializeObject(releasedCopy), null, AccessCondition.GenerateLeaseCondition(leaseId), null, null).ConfigureAwait(false);
await leaseBlob.ReleaseLeaseAsync(AccessCondition.GenerateLeaseCondition(leaseId)).ConfigureAwait(false);
}
catch (StorageException se)
{
if (WasLeaseLost(partitionId, se))
{
retval = false;
}
else
{
throw;
}
}
return(retval);
}
internal AzureBlobLease(AzureBlobLease source, CloudBlockBlob blob) : base(source)
{
this.Offset = source.Offset;
this.SequenceNumber = source.SequenceNumber;
this.Blob = blob;
this.isOwned = blob.Properties.LeaseState == LeaseState.Leased;
}
public async Task UpdateCheckpointAsync(Lease lease, Checkpoint checkpoint)
{
AzureBlobLease newLease = new AzureBlobLease((AzureBlobLease)lease);
newLease.Offset = checkpoint.Offset;
newLease.SequenceNumber = checkpoint.SequenceNumber;
await this.UpdateLeaseAsync(newLease).ConfigureAwait(false);
}
public async Task <Checkpoint> CreateCheckpointIfNotExistsAsync(string partitionId)
{
// Normally the lease will already be created, checkpoint store is initialized after lease store.
AzureBlobLease lease = (AzureBlobLease)(await CreateLeaseIfNotExistsAsync(partitionId));
Checkpoint checkpoint = new Checkpoint(partitionId, lease.Offset, lease.SequenceNumber);
return(checkpoint);
}
async Task <bool> AcquireLeaseCoreAsync(AzureBlobLease lease)
{
CloudBlockBlob leaseBlob = lease.Blob;
bool retval = true;
string newLeaseId = Guid.NewGuid().ToString();
string partitionId = lease.PartitionId;
try
{
string newToken;
await leaseBlob.FetchAttributesAsync().ConfigureAwait(false);
if (leaseBlob.Properties.LeaseState == LeaseState.Leased)
{
if (string.IsNullOrEmpty(lease.Token))
{
// We reach here in a race condition: when this instance of EventProcessorHost scanned the
// lease blobs, this partition was unowned (token is empty) but between then and now, another
// instance of EPH has established a lease (getLeaseState() is LEASED). We normally enforce
// that we only steal the lease if it is still owned by the instance which owned it when we
// scanned, but we can't do that when we don't know who owns it. The safest thing to do is just
// fail the acquisition. If that means that one EPH instance gets more partitions than it should,
// rebalancing will take care of that quickly enough.
return(false);
}
ProcessorEventSource.Log.AzureStorageManagerInfo(this.host.HostName, lease.PartitionId, "Need to ChangeLease");
newToken = await leaseBlob.ChangeLeaseAsync(newLeaseId, AccessCondition.GenerateLeaseCondition(lease.Token)).ConfigureAwait(false);
}
else
{
ProcessorEventSource.Log.AzureStorageManagerInfo(this.host.HostName, lease.PartitionId, "Need to AcquireLease");
try
{
newToken = await leaseBlob.AcquireLeaseAsync(leaseDuration, newLeaseId).ConfigureAwait(false);
}
catch (StorageException se)
when(se.RequestInformation != null &&
se.RequestInformation.ErrorCode.Equals(BlobErrorCodeStrings.LeaseAlreadyPresent, StringComparison.OrdinalIgnoreCase))
{
// Either some other host grabbed the lease or checkpoint call renewed it.
return(false);
}
}
lease.Token = newToken;
lease.Owner = this.host.HostName;
lease.IncrementEpoch(); // Increment epoch each time lease is acquired or stolen by a new host
await leaseBlob.UploadTextAsync(JsonConvert.SerializeObject(lease), null, AccessCondition.GenerateLeaseCondition(lease.Token), null, null).ConfigureAwait(false);
}
catch (StorageException se)
{
throw HandleStorageException(partitionId, se);
}
return(retval);
}
public async Task UpdateCheckpointAsync(Checkpoint checkpoint)
{
// Need to fetch the most current lease data so that we can update it correctly.
AzureBlobLease lease = (AzureBlobLease) await GetLeaseAsync(checkpoint.PartitionId).ConfigureAwait(false);
lease.Offset = checkpoint.Offset;
lease.SequenceNumber = checkpoint.SequenceNumber;
await UpdateLeaseAsync(lease).ConfigureAwait(false);
}
async Task <AzureBlobLease> DownloadLeaseAsync(string partitionId, CloudBlockBlob blob) // throws StorageException, IOException
{
string jsonLease = await blob.DownloadTextAsync();
ProcessorEventSource.Log.AzureStorageManagerInfo(this.host.Id, partitionId, "Raw JSON downloaded: " + jsonLease);
AzureBlobLease rehydrated = (AzureBlobLease)JsonConvert.DeserializeObject(jsonLease, typeof(AzureBlobLease));
AzureBlobLease blobLease = new AzureBlobLease(rehydrated, blob);
return(blobLease);
}
async Task <Lease> DownloadLeaseAsync(string partitionId, CloudBlockBlob blob) // throws StorageException, IOException
{
string jsonLease = await blob.DownloadTextAsync(null, null, this.defaultRequestOptions, this.operationContext).ConfigureAwait(false);
ProcessorEventSource.Log.AzureStorageManagerInfo(this.host.HostName, partitionId, "Raw JSON downloaded: " + jsonLease);
AzureBlobLease rehydrated = (AzureBlobLease)JsonConvert.DeserializeObject(jsonLease, typeof(AzureBlobLease));
AzureBlobLease blobLease = new AzureBlobLease(rehydrated, blob);
return(blobLease);
}
public async Task <Lease> GetLeaseAsync(string partitionId) // throws URISyntaxException, IOException, StorageException
{
AzureBlobLease retval = null;
CloudBlockBlob leaseBlob = GetBlockBlobReference(partitionId);
if (await leaseBlob.ExistsAsync().ConfigureAwait(false))
{
retval = await DownloadLeaseAsync(partitionId, leaseBlob).ConfigureAwait(false);
}
return(retval);
}
public async Task <Lease> GetLeaseAsync(string partitionId) // throws URISyntaxException, IOException, StorageException
{
AzureBlobLease retval = null;
CloudBlockBlob leaseBlob = this.consumerGroupDirectory.GetBlockBlobReference(partitionId);
if (await leaseBlob.ExistsAsync())
{
retval = await DownloadLeaseAsync(partitionId, leaseBlob);
}
return(retval);
}
public async Task <Lease> CreateLeaseIfNotExistsAsync(string partitionId) // throws URISyntaxException, IOException, StorageException
{
AzureBlobLease returnLease;
try
{
CloudBlockBlob leaseBlob = GetBlockBlobReference(partitionId);
returnLease = new AzureBlobLease(partitionId, leaseBlob);
string jsonLease = JsonConvert.SerializeObject(returnLease);
ProcessorEventSource.Log.AzureStorageManagerInfo(
this.host.HostName,
partitionId,
"CreateLeaseIfNotExist - leaseContainerName: " + this.leaseContainerName +
" consumerGroupName: " + this.host.ConsumerGroupName + " storageBlobPrefix: " + this.storageBlobPrefix);
// Don't provide default request options for upload call.
// This request will respect client's default options.
await leaseBlob.UploadTextAsync(
jsonLease,
null,
AccessCondition.GenerateIfNoneMatchCondition("*"),
null,
this.operationContext).ConfigureAwait(false);
}
catch (StorageException se)
{
if (se.RequestInformation.ErrorCode == BlobErrorCodeStrings.BlobAlreadyExists ||
se.RequestInformation.ErrorCode == BlobErrorCodeStrings.LeaseIdMissing) // occurs when somebody else already has leased the blob
{
// The blob already exists.
ProcessorEventSource.Log.AzureStorageManagerInfo(this.host.HostName, partitionId, "Lease already exists");
returnLease = (AzureBlobLease) await GetLeaseAsync(partitionId).ConfigureAwait(false);
}
else
{
ProcessorEventSource.Log.AzureStorageManagerError(
this.host.HostName,
partitionId,
"CreateLeaseIfNotExist StorageException - leaseContainerName: " + this.leaseContainerName +
" consumerGroupName: " + this.host.ConsumerGroupName + " storageBlobPrefix: " + this.storageBlobPrefix,
se.ToString());
throw;
}
}
return(returnLease);
}
public async Task <Checkpoint> GetCheckpointAsync(string partitionId)
{
AzureBlobLease lease = (AzureBlobLease)(await GetLeaseAsync(partitionId));
Checkpoint checkpoint = null;
if (lease != null && lease.Offset != null)
{
checkpoint = new Checkpoint(partitionId)
{
Offset = lease.Offset,
SequenceNumber = lease.SequenceNumber
};
}
return(checkpoint);
}
async Task <bool> RenewLeaseCoreAsync(AzureBlobLease lease)
{
CloudBlockBlob leaseBlob = lease.Blob;
string partitionId = lease.PartitionId;
try
{
await leaseBlob.RenewLeaseAsync(AccessCondition.GenerateLeaseCondition(lease.Token), this.renewRequestOptions, null).ConfigureAwait(false);
}
catch (StorageException se)
{
throw HandleStorageException(partitionId, se);
}
return(true);
}
public async Task <Checkpoint> GetCheckpointAsync(string partitionId)
{
AzureBlobLease lease = (AzureBlobLease) await GetLeaseAsync(partitionId).ConfigureAwait(false);
Checkpoint checkpoint = null;
if (lease != null && !string.IsNullOrEmpty(lease.Offset))
{
checkpoint = new Checkpoint(partitionId)
{
Offset = lease.Offset,
SequenceNumber = lease.SequenceNumber
};
}
return(checkpoint);
}
async Task <bool> UpdateLeaseCoreAsync(AzureBlobLease lease)
{
if (lease == null)
{
return(false);
}
string partitionId = lease.PartitionId;
ProcessorEventSource.Log.AzureStorageManagerInfo(this.host.Id, partitionId, "Updating lease");
string token = lease.Token;
if (string.IsNullOrEmpty(token))
{
return(false);
}
// First, renew the lease to make sure the update will go through.
if (!await this.RenewLeaseAsync(lease).ConfigureAwait(false))
{
return(false);
}
CloudBlockBlob leaseBlob = lease.Blob;
try
{
string jsonToUpload = JsonConvert.SerializeObject(lease);
ProcessorEventSource.Log.AzureStorageManagerInfo(this.host.Id, lease.PartitionId, $"Raw JSON uploading: {jsonToUpload}");
await leaseBlob.UploadTextAsync(jsonToUpload, null, AccessCondition.GenerateLeaseCondition(token), null, null).ConfigureAwait(false);
}
catch (StorageException se)
{
if (WasLeaseLost(partitionId, se))
{
throw new LeaseLostException(lease, se);
}
throw;
}
return(true);
}
public async Task <Lease> CreateLeaseIfNotExistsAsync(string partitionId) // throws URISyntaxException, IOException, StorageException
{
AzureBlobLease returnLease;
try
{
CloudBlockBlob leaseBlob = this.consumerGroupDirectory.GetBlockBlobReference(partitionId);
returnLease = new AzureBlobLease(partitionId, leaseBlob);
string jsonLease = JsonConvert.SerializeObject(returnLease);
ProcessorEventSource.Log.AzureStorageManagerInfo(
this.host.Id,
partitionId,
"CreateLeaseIfNotExist - leaseContainerName: " + this.leaseContainerName + " consumerGroupName: " + this.host.ConsumerGroupName);
await leaseBlob.UploadTextAsync(jsonLease, null, AccessCondition.GenerateIfNoneMatchCondition("*"), null, null);
}
catch (StorageException se)
{
StorageExtendedErrorInformation extendedErrorInfo = se.RequestInformation.ExtendedErrorInformation;
if (extendedErrorInfo != null &&
(extendedErrorInfo.ErrorCode == BlobErrorCodeStrings.BlobAlreadyExists ||
extendedErrorInfo.ErrorCode == BlobErrorCodeStrings.LeaseIdMissing)) // occurs when somebody else already has leased the blob
{
// The blob already exists.
ProcessorEventSource.Log.AzureStorageManagerInfo(this.host.Id, partitionId, "Lease already exists");
returnLease = (AzureBlobLease)(await GetLeaseAsync(partitionId));
}
else
{
Console.WriteLine("errorCode " + extendedErrorInfo.ErrorCode);
Console.WriteLine("errorString " + extendedErrorInfo.ErrorMessage);
ProcessorEventSource.Log.AzureStorageManagerError(
this.host.Id,
partitionId,
"CreateLeaseIfNotExist StorageException - leaseContainerName: " + this.leaseContainerName + " consumerGroupName: " + this.host.ConsumerGroupName,
se.ToString());
throw;
}
}
return(returnLease);
}
async Task <bool> AcquireLeaseCoreAsync(AzureBlobLease lease)
{
CloudBlockBlob leaseBlob = lease.Blob;
bool retval = true;
string newLeaseId = Guid.NewGuid().ToString();
string partitionId = lease.PartitionId;
try
{
string newToken;
await leaseBlob.FetchAttributesAsync().ConfigureAwait(false);
if (leaseBlob.Properties.LeaseState == LeaseState.Leased)
{
ProcessorEventSource.Log.AzureStorageManagerInfo(this.host.Id, lease.PartitionId, "Need to ChangeLease");
newToken = await leaseBlob.ChangeLeaseAsync(newLeaseId, AccessCondition.GenerateLeaseCondition(lease.Token)).ConfigureAwait(false);
}
else
{
ProcessorEventSource.Log.AzureStorageManagerInfo(this.host.Id, lease.PartitionId, "Need to AcquireLease");
newToken = await leaseBlob.AcquireLeaseAsync(leaseDuration, newLeaseId).ConfigureAwait(false);
}
lease.Token = newToken;
lease.Owner = this.host.HostName;
lease.IncrementEpoch(); // Increment epoch each time lease is acquired or stolen by a new host
await leaseBlob.UploadTextAsync(JsonConvert.SerializeObject(lease), null, AccessCondition.GenerateLeaseCondition(lease.Token), null, null).ConfigureAwait(false);
}
catch (StorageException se)
{
if (WasLeaseLost(partitionId, se))
{
retval = false;
}
else
{
throw;
}
}
return(retval);
}
async Task <bool> UpdateLeaseCoreAsync(AzureBlobLease lease)
{
if (lease == null)
{
return(false);
}
string partitionId = lease.PartitionId;
ProcessorEventSource.Log.AzureStorageManagerInfo(this.host.HostName, partitionId, "Updating lease");
string token = lease.Token;
if (string.IsNullOrEmpty(token))
{
return(false);
}
CloudBlockBlob leaseBlob = lease.Blob;
try
{
string jsonToUpload = JsonConvert.SerializeObject(lease);
ProcessorEventSource.Log.AzureStorageManagerInfo(this.host.HostName, lease.PartitionId, $"Raw JSON uploading: {jsonToUpload}");
// This is on the code path of checkpoint call thus don't provide default request options for upload call.
// This request will respect client's default options.
await leaseBlob.UploadTextAsync(
jsonToUpload,
null,
AccessCondition.GenerateLeaseCondition(token),
null,
this.operationContext).ConfigureAwait(false);
}
catch (StorageException se)
{
throw HandleStorageException(partitionId, se);
}
return(true);
}
async Task <bool> ReleaseLeaseCoreAsync(AzureBlobLease lease)
{
ProcessorEventSource.Log.AzureStorageManagerInfo(this.host.HostName, lease.PartitionId, "Releasing lease");
CloudBlockBlob leaseBlob = lease.Blob;
string partitionId = lease.PartitionId;
try
{
string leaseId = lease.Token;
AzureBlobLease releasedCopy = new AzureBlobLease(lease)
{
Token = string.Empty,
Owner = string.Empty
};
// Remove owner in the metadata.
leaseBlob.Metadata.Remove(MetaDataOwnerName);
await leaseBlob.SetMetadataAsync(
AccessCondition.GenerateLeaseCondition(leaseId),
this.defaultRequestOptions,
this.operationContext).ConfigureAwait(false);
await leaseBlob.UploadTextAsync(
JsonConvert.SerializeObject(releasedCopy),
null,
AccessCondition.GenerateLeaseCondition(leaseId),
this.defaultRequestOptions,
this.operationContext).ConfigureAwait(false);
await leaseBlob.ReleaseLeaseAsync(AccessCondition.GenerateLeaseCondition(leaseId), this.defaultRequestOptions, this.operationContext).ConfigureAwait(false);
}
catch (StorageException se)
{
throw HandleStorageException(partitionId, se);
}
return(true);
}
async Task <bool> UpdateLeaseCoreAsync(AzureBlobLease lease)
{
if (lease == null)
{
return(false);
}
string partitionId = lease.PartitionId;
ProcessorEventSource.Log.AzureStorageManagerInfo(this.host.HostName, partitionId, "Updating lease");
string token = lease.Token;
if (string.IsNullOrEmpty(token))
{
return(false);
}
CloudBlockBlob leaseBlob = lease.Blob;
try
{
string jsonToUpload = JsonConvert.SerializeObject(lease);
ProcessorEventSource.Log.AzureStorageManagerInfo(this.host.HostName, lease.PartitionId, $"Raw JSON uploading: {jsonToUpload}");
await leaseBlob.UploadTextAsync(
jsonToUpload,
null,
AccessCondition.GenerateLeaseCondition(token),
this.defaultRequestOptions,
this.operationContext).ConfigureAwait(false);
}
catch (StorageException se)
{
throw HandleStorageException(partitionId, se);
}
return(true);
}
async Task <bool> RenewLeaseCoreAsync(AzureBlobLease lease)
{
CloudBlockBlob leaseBlob = lease.Blob;
bool retval = true;
string partitionId = lease.PartitionId;
try
{
await leaseBlob.RenewLeaseAsync(AccessCondition.GenerateLeaseCondition(lease.Token), this.renewRequestOptions, null).ConfigureAwait(false);
}
catch (StorageException se)
{
if (WasLeaseLost(partitionId, se))
{
retval = false;
}
else
{
throw;
}
}
return(retval);
}
async Task <bool> AcquireLeaseCoreAsync(AzureBlobLease lease)
{
CloudBlockBlob leaseBlob = lease.Blob;
bool retval = true;
string newLeaseId = Guid.NewGuid().ToString();
string partitionId = lease.PartitionId;
try
{
bool renewLease = false;
string newToken;
await leaseBlob.FetchAttributesAsync(null, this.defaultRequestOptions, this.operationContext).ConfigureAwait(false);
if (leaseBlob.Properties.LeaseState == LeaseState.Leased)
{
if (string.IsNullOrEmpty(lease.Token))
{
// We reach here in a race condition: when this instance of EventProcessorHost scanned the
// lease blobs, this partition was unowned (token is empty) but between then and now, another
// instance of EPH has established a lease (getLeaseState() is LEASED). We normally enforce
// that we only steal the lease if it is still owned by the instance which owned it when we
// scanned, but we can't do that when we don't know who owns it. The safest thing to do is just
// fail the acquisition. If that means that one EPH instance gets more partitions than it should,
// rebalancing will take care of that quickly enough.
return(false);
}
ProcessorEventSource.Log.AzureStorageManagerInfo(this.host.HostName, lease.PartitionId, "Need to ChangeLease");
renewLease = true;
newToken = await leaseBlob.ChangeLeaseAsync(
newLeaseId,
AccessCondition.GenerateLeaseCondition(lease.Token),
this.defaultRequestOptions,
this.operationContext).ConfigureAwait(false);
}
else
{
ProcessorEventSource.Log.AzureStorageManagerInfo(this.host.HostName, lease.PartitionId, "Need to AcquireLease");
newToken = await leaseBlob.AcquireLeaseAsync(
leaseDuration,
newLeaseId,
null,
this.defaultRequestOptions,
this.operationContext).ConfigureAwait(false);
}
lease.Token = newToken;
lease.Owner = this.host.HostName;
lease.IncrementEpoch(); // Increment epoch each time lease is acquired or stolen by a new host
// Renew lease here if needed?
// ChangeLease doesn't renew so we should avoid lease expiring before next renew interval.
if (renewLease)
{
await this.RenewLeaseCoreAsync(lease).ConfigureAwait(false);
}
// Update owner in the metadata first since clients get ownership information by looking at metadata.
lease.Blob.Metadata[MetaDataOwnerName] = lease.Owner;
await lease.Blob.SetMetadataAsync(
AccessCondition.GenerateLeaseCondition(lease.Token),
this.defaultRequestOptions,
this.operationContext).ConfigureAwait(false);
// Then update deserialized lease content.
await leaseBlob.UploadTextAsync(
JsonConvert.SerializeObject(lease),
null,
AccessCondition.GenerateLeaseCondition(lease.Token),
this.defaultRequestOptions,
this.operationContext).ConfigureAwait(false);
}
catch (StorageException se)
{
throw HandleStorageException(partitionId, se);
}
return(retval);
}
internal AzureBlobLease(AzureBlobLease source, CloudBlockBlob blob) : base(source)
{
this.Offset = source.Offset;
this.SequenceNumber = source.SequenceNumber;
this.Blob = blob;
}