private Metadata GetActualMetadata(BlobItem item) { // Pass all custom metadata through the converter var convertedMeta = item.Metadata.ToDictionary(k => k.Key, k => AzureStoreMetadataEncoder.DecodeMetadata(k.Value)); var metadata = new Metadata(convertedMeta); metadata.StoredLastModified = item.Properties.LastModified.Value.UtcDateTime; if (!metadata.LastModified.HasValue) { metadata.LastModified = metadata.StoredLastModified; } metadata.StoredContentLength = item.Properties.ContentLength; metadata.StoredContentType = item.Properties.ContentType; metadata.ETag = item.Properties.ETag.ToString(); // Remove the snapshot key at this point if we have it if (metadata.ContainsKey(InternalSnapshotKey)) { metadata.Remove(InternalSnapshotKey); } // Remove the store key as well... if (metadata.ContainsKey(StoreVersionKey)) { metadata.Remove(StoreVersionKey); } if (_enableSnapshots && !string.IsNullOrEmpty(item.Snapshot)) { metadata.Snapshot = item.Snapshot; } return(metadata); }
public async Task SoftDelete(StoreLocation location, UpdateAuditInfo audit) { // In Azure we cannot delete the blob as this will loose the snapshots // Instead we will just add some metadata var blob = GetBlockBlob(location); var props = await GetBlobProperties(blob); var meta = await GetActualMetadata(blob, props, null); // If already deleted don't worry about it! if (meta == null || meta.ContainsKey(_deletedKey)) { return; } var fullInfo = TransformAuditInformation(meta, audit); meta.Audit = fullInfo; props.Metadata[MetadataConstants.AuditMetadataKey] = AzureStoreMetadataEncoder.EncodeMetadata(meta[MetadataConstants.AuditMetadataKey]); props.Metadata[_deletedKey] = DateTime.UtcNow.Ticks.ToString(); await blob.SetMetadataAsync(props.Metadata); LeoTrace.WriteLine("Soft deleted (2 calls): " + blob.Name); }
public async Task <Metadata> SaveMetadata(StoreLocation location, Metadata metadata) { var blob = GetBlockBlob(location); var props = await GetBlobProperties(blob); if (props == null) { return(null); } foreach (var m in metadata) { // Do not override audit information! if (m.Key != MetadataConstants.AuditMetadataKey) { props.Metadata[m.Key] = AzureStoreMetadataEncoder.EncodeMetadata(m.Value); } } await blob.SetMetadataAsync(props.Metadata); return(await GetActualMetadata(blob, props, null)); }
private async ValueTask <Metadata> GetActualMetadata(BlockBlobClient blob, BlobProperties props, string snapshot) { if (props == null) { return(null); } // Pass all custom metadata through the converter var convertedMeta = props.Metadata.ToDictionary(k => k.Key, k => AzureStoreMetadataEncoder.DecodeMetadata(k.Value)); var metadata = new Metadata(convertedMeta); metadata.StoredLastModified = props.LastModified.UtcDateTime; if (!metadata.LastModified.HasValue) { metadata.LastModified = metadata.StoredLastModified; } metadata.StoredContentLength = props.ContentLength; metadata.StoredContentType = props.ContentType; metadata.ETag = props.ETag.ToString(); // Remove the snapshot key at this point if we have it if (metadata.ContainsKey(InternalSnapshotKey)) { metadata.Remove(InternalSnapshotKey); } // Remove the store key as well... if (metadata.ContainsKey(StoreVersionKey)) { metadata.Remove(StoreVersionKey); } if (_enableSnapshots) { if (!string.IsNullOrEmpty(snapshot)) { metadata.Snapshot = snapshot; } else if (props.Metadata.ContainsKey(InternalSnapshotKey)) { // Try and use our save snapshot instead of making more calls... var date = props.Metadata[InternalSnapshotKey]; if (long.TryParse(date, out var ticks)) { // This is in old format, convert back to snapshot format date date = new DateTime(ticks, DateTimeKind.Utc).ToString("o", CultureInfo.InvariantCulture); } metadata.Snapshot = date; } else { // Try and find last snapshot // Unfortunately we need to make a request since this information isn't on the actual blob that we are working with... var container = _blobStorage.GetBlobContainerClient(blob.BlobContainerName); var snapBlob = await ListBlobs(container, blob.Name, BlobTraits.None, BlobStates.Snapshots) .Where(b => !string.IsNullOrEmpty(b.Snapshot) && b.Name == blob.Name) .AggregateAsync((BlobItem)null, (a, b) => a == null || b == null ? (a ?? b) : (ParseSnapshotDate(b.Snapshot) > ParseSnapshotDate(b.Snapshot) ? a : b)); metadata.Snapshot = snapBlob?.Snapshot; } } return(metadata); }
private async Task <OptimisticStoreWriteResult> SaveDataInternal(StoreLocation location, Metadata metadata, UpdateAuditInfo audit, Func <IWriteAsyncStream, Task <long?> > savingFunc, CancellationToken token, bool isOptimistic) { var blob = GetBlockBlob(location); // We always want to save the new audit information when saving! var props = await GetBlobProperties(blob); var currentMetadata = await GetActualMetadata(blob, props, null); var auditInfo = TransformAuditInformation(currentMetadata, audit); metadata = metadata ?? new Metadata(); metadata.Audit = auditInfo; var result = new OptimisticStoreWriteResult() { Result = true }; try { // If the ETag value is empty then the store value must not exist yet... var condition = isOptimistic ? (string.IsNullOrEmpty(metadata.ETag) ? new BlobRequestConditions { IfNoneMatch = ETag.All } : new BlobRequestConditions { IfMatch = new ETag(metadata.ETag) }) : null; // Copy the metadata across var newMeta = new Dictionary <string, string>(); foreach (var m in metadata) { newMeta[m.Key] = AzureStoreMetadataEncoder.EncodeMetadata(m.Value); } // Always store the version - We use this to do more efficient things on read newMeta[StoreVersionKey] = StoreVersionValue; long?length; using (var stream = new AzureWriteBlockBlobStream(blob, condition, newMeta)) { length = await savingFunc(stream); await stream.Complete(token); } if (length.HasValue && (metadata == null || !metadata.ContentLength.HasValue)) { newMeta[MetadataConstants.ContentLengthMetadataKey] = length.Value.ToString(CultureInfo.InvariantCulture); // Save the length straight away before the snapshot... await blob.SetMetadataAsync(newMeta, cancellationToken : token); } // Create a snapshot straight away on azure // Note: this shouldnt matter for cost as any blocks that are the same do not cost extra if (_enableSnapshots) { var snapshotBlob = await blob.CreateSnapshotAsync(cancellationToken : token); // Save the snapshot back to original blob... newMeta[InternalSnapshotKey] = snapshotBlob.Value.Snapshot; await blob.SetMetadataAsync(newMeta, cancellationToken : token); LeoTrace.WriteLine("Created Snapshot: " + blob.Name); } var newProps = await blob.GetPropertiesAsync(); result.Metadata = await GetActualMetadata(blob, newProps, null); } catch (RequestFailedException e) { if (isOptimistic) { // First condition occurrs when the eTags do not match // Second condition when we specified no eTag (ie must be new blob) if (e.Status == (int)HttpStatusCode.PreconditionFailed || (e.Status == (int)HttpStatusCode.Conflict && e.ErrorCode == BlobErrorCode.BlobAlreadyExists)) { result.Result = false; } else { // Might have been a different error? throw; } } else { if (e.Status == (int)HttpStatusCode.Conflict || e.ErrorCode == BlobErrorCode.LeaseIdMissing) { throw new LockException("The underlying storage is currently locked for save"); } // Might have been a different error? throw; } } return(result); }