private Metadata GetActualMetadata(BlobItem item)
{
// Pass all custom metadata through the converter
var convertedMeta = item.Metadata.ToDictionary(k => k.Key, k => AzureStoreMetadataEncoder.DecodeMetadata(k.Value));
var metadata = new Metadata(convertedMeta);
metadata.StoredLastModified = item.Properties.LastModified.Value.UtcDateTime;
if (!metadata.LastModified.HasValue)
{
metadata.LastModified = metadata.StoredLastModified;
}
metadata.StoredContentLength = item.Properties.ContentLength;
metadata.StoredContentType = item.Properties.ContentType;
metadata.ETag = item.Properties.ETag.ToString();
// Remove the snapshot key at this point if we have it
if (metadata.ContainsKey(InternalSnapshotKey))
{
metadata.Remove(InternalSnapshotKey);
}
// Remove the store key as well...
if (metadata.ContainsKey(StoreVersionKey))
{
metadata.Remove(StoreVersionKey);
}
if (_enableSnapshots && !string.IsNullOrEmpty(item.Snapshot))
{
metadata.Snapshot = item.Snapshot;
}
return(metadata);
}
public async Task SoftDelete(StoreLocation location, UpdateAuditInfo audit)
{
// In Azure we cannot delete the blob as this will loose the snapshots
// Instead we will just add some metadata
var blob = GetBlockBlob(location);
var props = await GetBlobProperties(blob);
var meta = await GetActualMetadata(blob, props, null);
// If already deleted don't worry about it!
if (meta == null || meta.ContainsKey(_deletedKey))
{
return;
}
var fullInfo = TransformAuditInformation(meta, audit);
meta.Audit = fullInfo;
props.Metadata[MetadataConstants.AuditMetadataKey] = AzureStoreMetadataEncoder.EncodeMetadata(meta[MetadataConstants.AuditMetadataKey]);
props.Metadata[_deletedKey] = DateTime.UtcNow.Ticks.ToString();
await blob.SetMetadataAsync(props.Metadata);
LeoTrace.WriteLine("Soft deleted (2 calls): " + blob.Name);
}
public async Task <Metadata> SaveMetadata(StoreLocation location, Metadata metadata)
{
var blob = GetBlockBlob(location);
var props = await GetBlobProperties(blob);
if (props == null)
{
return(null);
}
foreach (var m in metadata)
{
// Do not override audit information!
if (m.Key != MetadataConstants.AuditMetadataKey)
{
props.Metadata[m.Key] = AzureStoreMetadataEncoder.EncodeMetadata(m.Value);
}
}
await blob.SetMetadataAsync(props.Metadata);
return(await GetActualMetadata(blob, props, null));
}
private async ValueTask <Metadata> GetActualMetadata(BlockBlobClient blob, BlobProperties props, string snapshot)
{
if (props == null)
{
return(null);
}
// Pass all custom metadata through the converter
var convertedMeta = props.Metadata.ToDictionary(k => k.Key, k => AzureStoreMetadataEncoder.DecodeMetadata(k.Value));
var metadata = new Metadata(convertedMeta);
metadata.StoredLastModified = props.LastModified.UtcDateTime;
if (!metadata.LastModified.HasValue)
{
metadata.LastModified = metadata.StoredLastModified;
}
metadata.StoredContentLength = props.ContentLength;
metadata.StoredContentType = props.ContentType;
metadata.ETag = props.ETag.ToString();
// Remove the snapshot key at this point if we have it
if (metadata.ContainsKey(InternalSnapshotKey))
{
metadata.Remove(InternalSnapshotKey);
}
// Remove the store key as well...
if (metadata.ContainsKey(StoreVersionKey))
{
metadata.Remove(StoreVersionKey);
}
if (_enableSnapshots)
{
if (!string.IsNullOrEmpty(snapshot))
{
metadata.Snapshot = snapshot;
}
else if (props.Metadata.ContainsKey(InternalSnapshotKey))
{
// Try and use our save snapshot instead of making more calls...
var date = props.Metadata[InternalSnapshotKey];
if (long.TryParse(date, out var ticks))
{
// This is in old format, convert back to snapshot format date
date = new DateTime(ticks, DateTimeKind.Utc).ToString("o", CultureInfo.InvariantCulture);
}
metadata.Snapshot = date;
}
else
{
// Try and find last snapshot
// Unfortunately we need to make a request since this information isn't on the actual blob that we are working with...
var container = _blobStorage.GetBlobContainerClient(blob.BlobContainerName);
var snapBlob = await ListBlobs(container, blob.Name, BlobTraits.None, BlobStates.Snapshots)
.Where(b => !string.IsNullOrEmpty(b.Snapshot) && b.Name == blob.Name)
.AggregateAsync((BlobItem)null, (a, b) => a == null || b == null ? (a ?? b) : (ParseSnapshotDate(b.Snapshot) > ParseSnapshotDate(b.Snapshot) ? a : b));
metadata.Snapshot = snapBlob?.Snapshot;
}
}
return(metadata);
}
private async Task <OptimisticStoreWriteResult> SaveDataInternal(StoreLocation location, Metadata metadata, UpdateAuditInfo audit, Func <IWriteAsyncStream, Task <long?> > savingFunc, CancellationToken token, bool isOptimistic)
{
var blob = GetBlockBlob(location);
// We always want to save the new audit information when saving!
var props = await GetBlobProperties(blob);
var currentMetadata = await GetActualMetadata(blob, props, null);
var auditInfo = TransformAuditInformation(currentMetadata, audit);
metadata = metadata ?? new Metadata();
metadata.Audit = auditInfo;
var result = new OptimisticStoreWriteResult()
{
Result = true
};
try
{
// If the ETag value is empty then the store value must not exist yet...
var condition = isOptimistic ?
(string.IsNullOrEmpty(metadata.ETag) ? new BlobRequestConditions {
IfNoneMatch = ETag.All
} : new BlobRequestConditions {
IfMatch = new ETag(metadata.ETag)
})
: null;
// Copy the metadata across
var newMeta = new Dictionary <string, string>();
foreach (var m in metadata)
{
newMeta[m.Key] = AzureStoreMetadataEncoder.EncodeMetadata(m.Value);
}
// Always store the version - We use this to do more efficient things on read
newMeta[StoreVersionKey] = StoreVersionValue;
long?length;
using (var stream = new AzureWriteBlockBlobStream(blob, condition, newMeta))
{
length = await savingFunc(stream);
await stream.Complete(token);
}
if (length.HasValue && (metadata == null || !metadata.ContentLength.HasValue))
{
newMeta[MetadataConstants.ContentLengthMetadataKey] = length.Value.ToString(CultureInfo.InvariantCulture);
// Save the length straight away before the snapshot...
await blob.SetMetadataAsync(newMeta, cancellationToken : token);
}
// Create a snapshot straight away on azure
// Note: this shouldnt matter for cost as any blocks that are the same do not cost extra
if (_enableSnapshots)
{
var snapshotBlob = await blob.CreateSnapshotAsync(cancellationToken : token);
// Save the snapshot back to original blob...
newMeta[InternalSnapshotKey] = snapshotBlob.Value.Snapshot;
await blob.SetMetadataAsync(newMeta, cancellationToken : token);
LeoTrace.WriteLine("Created Snapshot: " + blob.Name);
}
var newProps = await blob.GetPropertiesAsync();
result.Metadata = await GetActualMetadata(blob, newProps, null);
}
catch (RequestFailedException e)
{
if (isOptimistic)
{
// First condition occurrs when the eTags do not match
// Second condition when we specified no eTag (ie must be new blob)
if (e.Status == (int)HttpStatusCode.PreconditionFailed ||
(e.Status == (int)HttpStatusCode.Conflict && e.ErrorCode == BlobErrorCode.BlobAlreadyExists))
{
result.Result = false;
}
else
{
// Might have been a different error?
throw;
}
}
else
{
if (e.Status == (int)HttpStatusCode.Conflict || e.ErrorCode == BlobErrorCode.LeaseIdMissing)
{
throw new LockException("The underlying storage is currently locked for save");
}
// Might have been a different error?
throw;
}
}
return(result);
}