public async Task<Metadata> SaveData(StoreLocation location, Metadata metadata, UpdateAuditInfo audit, Func<IWriteAsyncStream, Task<long?>> savingFunc, CancellationToken token)
{
var current = await GetMetadata(location).ConfigureAwait(false);
var info = current == null ? new AuditInfo() : current.Audit;
info.UpdatedBy = audit == null ? "0" : audit.UpdatedBy;
info.UpdatedByName = audit == null ? string.Empty : audit.UpdatedByName;
info.UpdatedOn = DateTime.UtcNow;
info.CreatedBy = info.CreatedBy ?? info.UpdatedBy;
info.CreatedByName = info.CreatedByName ?? info.UpdatedByName;
info.CreatedOn = info.CreatedOn ?? info.UpdatedOn;
metadata = metadata ?? new Metadata();
metadata.Audit = info;
var key = GetObjectKey(location);
long? length = null;
using(var stream = new AmazonMultiUploadStream(_client, _bucket, key, metadata))
{
length = await savingFunc(stream).ConfigureAwait(false);
await stream.Complete(token).ConfigureAwait(false);
metadata.Snapshot = stream.VersionId;
}
if (length.HasValue && (metadata == null || !metadata.ContentLength.HasValue))
{
metadata[MetadataConstants.ContentLengthMetadataKey] = length.Value.ToString(CultureInfo.InvariantCulture);
// Save the length straight away before the snapshot...
metadata = await SaveMetadata(location, metadata).ConfigureAwait(false);
}
return metadata;
}
public async Task SoftDelete(StoreLocation location, UpdateAuditInfo audit)
{
// In Azure we cannot delete the blob as this will loose the snapshots
// Instead we will just add some metadata
var blob = GetBlockBlob(location);
var props = await GetBlobProperties(blob);
var meta = await GetActualMetadata(blob, props, null);
// If already deleted don't worry about it!
if (meta == null || meta.ContainsKey(_deletedKey))
{
return;
}
var fullInfo = TransformAuditInformation(meta, audit);
meta.Audit = fullInfo;
props.Metadata[MetadataConstants.AuditMetadataKey] = AzureStoreMetadataEncoder.EncodeMetadata(meta[MetadataConstants.AuditMetadataKey]);
props.Metadata[_deletedKey] = DateTime.UtcNow.Ticks.ToString();
await blob.SetMetadataAsync(props.Metadata);
LeoTrace.WriteLine("Soft deleted (2 calls): " + blob.Name);
}
public Task SoftDelete(StoreLocation location, UpdateAuditInfo audit)
{
// If we support snapshots then we can just delete the record in amazon...
var key = GetObjectKey(location);
var request = new DeleteObjectRequest
{
BucketName = _bucket,
Key = key
};
return(_client.DeleteObjectAsync(request));
}
private AuditInfo TransformAuditInformation(Metadata current, UpdateAuditInfo newAudit)
{
var info = current == null ? new AuditInfo() : current.Audit;
info.UpdatedBy = newAudit == null ? "0" : newAudit.UpdatedBy;
info.UpdatedByName = newAudit == null ? string.Empty : newAudit.UpdatedByName;
info.UpdatedOn = DateTime.UtcNow;
info.CreatedBy = info.CreatedBy ?? info.UpdatedBy;
info.CreatedByName = info.CreatedByName ?? info.UpdatedByName;
info.CreatedOn = info.CreatedOn ?? info.UpdatedOn;
return(info);
}
public async Task Delete(StoreLocation location, UpdateAuditInfo audit, SecureStoreOptions options = SecureStoreOptions.All)
{
var metadata = await _store.GetMetadata(location);
if (metadata == null)
{
return;
}
if (options.HasFlag(SecureStoreOptions.KeepDeletes))
{
await _store.SoftDelete(location, audit);
}
else
{
await _store.PermanentDelete(location);
}
// The rest of the tasks are done asyncly
var tasks = new List <Task>();
if (options.HasFlag(SecureStoreOptions.Backup))
{
if (_backupQueue == null)
{
throw new ArgumentException("Backup option should not be used if no backup queue has been defined", "options");
}
tasks.Add(_backupQueue.SendMessage(GetMessageDetails(location, metadata)));
}
if (options.HasFlag(SecureStoreOptions.Index))
{
if (_indexQueue == null)
{
throw new ArgumentException("Index option should not be used if no index queue has been defined", "options");
}
tasks.Add(_indexQueue.SendMessage(GetMessageDetails(location, metadata)));
}
if (tasks.Count > 0)
{
await Task.WhenAll(tasks);
}
}
public async Task SoftDelete(StoreLocation location, UpdateAuditInfo audit)
{
// In Azure we cannot delete the blob as this will loose the snapshots
// Instead we will just add some metadata
var blob = GetBlockBlob(location);
var metadata = await GetBlobMetadata(blob).ConfigureAwait(false);
// If already deleted don't worry about it!
if (metadata == null || metadata.ContainsKey(_deletedKey))
{
return;
}
var fullInfo = TransformAuditInformation(metadata, audit);
metadata.Audit = fullInfo;
blob.Metadata[MetadataConstants.AuditMetadataKey] = metadata[MetadataConstants.AuditMetadataKey];
blob.Metadata[_deletedKey] = DateTime.UtcNow.Ticks.ToString();
await blob.ExecuteWrap(b => b.SetMetadataAsync()).ConfigureAwait(false);
LeoTrace.WriteLine("Soft deleted (2 calls): " + blob.Name);
}
public async Task<Metadata> SaveData(StoreLocation location, Metadata metadata, UpdateAuditInfo audit, Func<IWriteAsyncStream, Task<long?>> savingFunc, CancellationToken token)
{
var result = await SaveDataInternal(location, metadata, audit, savingFunc, token, false).ConfigureAwait(false);
return result.Metadata;
}
private AuditInfo TransformAuditInformation(Metadata current, UpdateAuditInfo newAudit)
{
var info = current == null ? new AuditInfo() : current.Audit;
info.UpdatedBy = newAudit == null ? "0" : newAudit.UpdatedBy;
info.UpdatedByName = newAudit == null ? string.Empty : newAudit.UpdatedByName;
info.UpdatedOn = DateTime.UtcNow;
info.CreatedBy = info.CreatedBy ?? info.UpdatedBy;
info.CreatedByName = info.CreatedByName ?? info.UpdatedByName;
info.CreatedOn = info.CreatedOn ?? info.UpdatedOn;
return info;
}
public async Task SoftDelete(StoreLocation location, UpdateAuditInfo audit)
{
// In Azure we cannot delete the blob as this will loose the snapshots
// Instead we will just add some metadata
var blob = GetBlockBlob(location);
var metadata = await GetBlobMetadata(blob).ConfigureAwait(false);
// If already deleted don't worry about it!
if(metadata == null || metadata.ContainsKey(_deletedKey)) { return; }
var fullInfo = TransformAuditInformation(metadata, audit);
metadata.Audit = fullInfo;
blob.Metadata[MetadataConstants.AuditMetadataKey] = metadata[MetadataConstants.AuditMetadataKey];
blob.Metadata[_deletedKey] = DateTime.UtcNow.Ticks.ToString();
await blob.ExecuteWrap(b => b.SetMetadataAsync()).ConfigureAwait(false);
LeoTrace.WriteLine("Soft deleted (2 calls): " + blob.Name);
}
public async Task <Metadata> SaveData(StoreLocation location, Metadata metadata, UpdateAuditInfo audit, Func <IWriteAsyncStream, Task <long?> > savingFunc, CancellationToken token)
{
var current = await GetMetadata(location).ConfigureAwait(false);
var info = current == null ? new AuditInfo() : current.Audit;
info.UpdatedBy = audit == null ? "0" : audit.UpdatedBy;
info.UpdatedByName = audit == null ? string.Empty : audit.UpdatedByName;
info.UpdatedOn = DateTime.UtcNow;
info.CreatedBy = info.CreatedBy ?? info.UpdatedBy;
info.CreatedByName = info.CreatedByName ?? info.UpdatedByName;
info.CreatedOn = info.CreatedOn ?? info.UpdatedOn;
metadata = metadata ?? new Metadata();
metadata.Audit = info;
var key = GetObjectKey(location);
long?length = null;
using (var stream = new AmazonMultiUploadStream(_client, _bucket, key, metadata))
{
length = await savingFunc(stream).ConfigureAwait(false);
await stream.Complete(token).ConfigureAwait(false);
metadata.Snapshot = stream.VersionId;
}
if (length.HasValue && (metadata == null || !metadata.ContentLength.HasValue))
{
metadata[MetadataConstants.ContentLengthMetadataKey] = length.Value.ToString(CultureInfo.InvariantCulture);
// Save the length straight away before the snapshot...
metadata = await SaveMetadata(location, metadata).ConfigureAwait(false);
}
return(metadata);
}
public Task SoftDelete(StoreLocation location, UpdateAuditInfo audit)
{
// If we support snapshots then we can just delete the record in amazon...
var key = GetObjectKey(location);
var request = new DeleteObjectRequest
{
BucketName = _bucket,
Key = key
};
return _client.DeleteObjectAsync(request);
}
public Task <OptimisticStoreWriteResult> TryOptimisticWrite(StoreLocation location, Metadata metadata, UpdateAuditInfo audit, Func <IWriteAsyncStream, Task <long?> > savingFunc, CancellationToken token)
{
return(SaveDataInternal(location, metadata, audit, savingFunc, token, true));
}
private async Task <OptimisticStoreWriteResult> SaveDataInternal(StoreLocation location, Metadata metadata, UpdateAuditInfo audit, Func <IWriteAsyncStream, Task <long?> > savingFunc, CancellationToken token, bool isOptimistic)
{
var blob = GetBlockBlob(location);
// We always want to save the new audit information when saving!
var currentMetadata = await GetBlobMetadata(blob).ConfigureAwait(false);
var auditInfo = TransformAuditInformation(currentMetadata, audit);
metadata = metadata ?? new Metadata();
metadata.Audit = auditInfo;
var result = new OptimisticStoreWriteResult()
{
Result = true
};
try
{
// If the ETag value is empty then the store value must not exist yet...
var condition = isOptimistic ?
(string.IsNullOrEmpty(metadata.ETag) ? AccessCondition.GenerateIfNoneMatchCondition("*") : AccessCondition.GenerateIfMatchCondition(metadata.ETag))
: null;
// Copy the metadata across
blob.Metadata.Clear();
foreach (var m in metadata)
{
blob.Metadata[m.Key] = m.Value;
}
// Always store the version - We use this to do more efficient things on read
blob.Metadata[StoreVersionKey] = StoreVersionValue;
long?length;
using (var stream = new AzureWriteBlockBlobStream(blob, condition))
{
length = await savingFunc(stream).ConfigureAwait(false);
await stream.Complete(token).ConfigureAwait(false);
}
if (length.HasValue && (metadata == null || !metadata.ContentLength.HasValue))
{
blob.Metadata[MetadataConstants.ContentLengthMetadataKey] = length.Value.ToString(CultureInfo.InvariantCulture);
// Save the length straight away before the snapshot...
await blob.SetMetadataAsync(null, null, null, token).ConfigureAwait(false);
}
// Create a snapshot straight away on azure
// Note: this shouldnt matter for cost as any blocks that are the same do not cost extra
if (_enableSnapshots)
{
var snapshotBlob = await blob.CreateSnapshotAsync(blob.Metadata, null, null, null, token).ConfigureAwait(false);
var snapshot = snapshotBlob.SnapshotTime.Value.UtcTicks.ToString(CultureInfo.InvariantCulture);
// Save the snapshot back to original blob...
blob.Metadata[InternalSnapshotKey] = snapshot;
await blob.SetMetadataAsync(null, null, null, token).ConfigureAwait(false);
LeoTrace.WriteLine("Created Snapshot: " + blob.Name);
}
result.Metadata = await GetActualMetadata(blob).ConfigureAwait(false);
}
catch (StorageException exc)
{
if (isOptimistic)
{
// First condition occurrs when the eTags do not match
// Second condition when we specified no eTag (ie must be new blob)
if (exc.RequestInformation.HttpStatusCode == (int)HttpStatusCode.PreconditionFailed ||
(exc.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Conflict && exc.RequestInformation.ExtendedErrorInformation.ErrorCode == "BlobAlreadyExists"))
{
result.Result = false;
}
else
{
// Might have been a different error?
throw exc.Wrap(blob.Name);
}
}
else
{
if (exc.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Conflict ||
exc.RequestInformation.ExtendedErrorInformation.ErrorCode == "LeaseIdMissing")
{
throw new LockException("The underlying storage is currently locked for save");
}
// Might have been a different error?
throw exc.Wrap(blob.Name);
}
}
return(result);
}
private async Task <OptimisticStoreWriteResult> SaveDataInternal(StoreLocation location, Metadata metadata, UpdateAuditInfo audit, Func <IWriteAsyncStream, Task <long?> > savingFunc, CancellationToken token, bool isOptimistic)
{
var blob = GetBlockBlob(location);
// We always want to save the new audit information when saving!
var props = await GetBlobProperties(blob);
var currentMetadata = await GetActualMetadata(blob, props, null);
var auditInfo = TransformAuditInformation(currentMetadata, audit);
metadata = metadata ?? new Metadata();
metadata.Audit = auditInfo;
var result = new OptimisticStoreWriteResult()
{
Result = true
};
try
{
// If the ETag value is empty then the store value must not exist yet...
var condition = isOptimistic ?
(string.IsNullOrEmpty(metadata.ETag) ? new BlobRequestConditions {
IfNoneMatch = ETag.All
} : new BlobRequestConditions {
IfMatch = new ETag(metadata.ETag)
})
: null;
// Copy the metadata across
var newMeta = new Dictionary <string, string>();
foreach (var m in metadata)
{
newMeta[m.Key] = AzureStoreMetadataEncoder.EncodeMetadata(m.Value);
}
// Always store the version - We use this to do more efficient things on read
newMeta[StoreVersionKey] = StoreVersionValue;
long?length;
using (var stream = new AzureWriteBlockBlobStream(blob, condition, newMeta))
{
length = await savingFunc(stream);
await stream.Complete(token);
}
if (length.HasValue && (metadata == null || !metadata.ContentLength.HasValue))
{
newMeta[MetadataConstants.ContentLengthMetadataKey] = length.Value.ToString(CultureInfo.InvariantCulture);
// Save the length straight away before the snapshot...
await blob.SetMetadataAsync(newMeta, cancellationToken : token);
}
// Create a snapshot straight away on azure
// Note: this shouldnt matter for cost as any blocks that are the same do not cost extra
if (_enableSnapshots)
{
var snapshotBlob = await blob.CreateSnapshotAsync(cancellationToken : token);
// Save the snapshot back to original blob...
newMeta[InternalSnapshotKey] = snapshotBlob.Value.Snapshot;
await blob.SetMetadataAsync(newMeta, cancellationToken : token);
LeoTrace.WriteLine("Created Snapshot: " + blob.Name);
}
var newProps = await blob.GetPropertiesAsync();
result.Metadata = await GetActualMetadata(blob, newProps, null);
}
catch (RequestFailedException e)
{
if (isOptimistic)
{
// First condition occurrs when the eTags do not match
// Second condition when we specified no eTag (ie must be new blob)
if (e.Status == (int)HttpStatusCode.PreconditionFailed ||
(e.Status == (int)HttpStatusCode.Conflict && e.ErrorCode == BlobErrorCode.BlobAlreadyExists))
{
result.Result = false;
}
else
{
// Might have been a different error?
throw;
}
}
else
{
if (e.Status == (int)HttpStatusCode.Conflict || e.ErrorCode == BlobErrorCode.LeaseIdMissing)
{
throw new LockException("The underlying storage is currently locked for save");
}
// Might have been a different error?
throw;
}
}
return(result);
}
private async Task<OptimisticStoreWriteResult> SaveDataInternal(StoreLocation location, Metadata metadata, UpdateAuditInfo audit, Func<IWriteAsyncStream, Task<long?>> savingFunc, CancellationToken token, bool isOptimistic)
{
var blob = GetBlockBlob(location);
// We always want to save the new audit information when saving!
var currentMetadata = await GetBlobMetadata(blob).ConfigureAwait(false);
var auditInfo = TransformAuditInformation(currentMetadata, audit);
metadata = metadata ?? new Metadata();
metadata.Audit = auditInfo;
var result = new OptimisticStoreWriteResult() { Result = true };
try
{
// If the ETag value is empty then the store value must not exist yet...
var condition = isOptimistic ?
(string.IsNullOrEmpty(metadata.ETag) ? AccessCondition.GenerateIfNoneMatchCondition("*") : AccessCondition.GenerateIfMatchCondition(metadata.ETag))
: null;
// Copy the metadata across
blob.Metadata.Clear();
foreach (var m in metadata)
{
blob.Metadata[m.Key] = m.Value;
}
// Always store the version - We use this to do more efficient things on read
blob.Metadata[StoreVersionKey] = StoreVersionValue;
long? length;
using (var stream = new AzureWriteBlockBlobStream(blob, condition))
{
length = await savingFunc(stream).ConfigureAwait(false);
await stream.Complete(token).ConfigureAwait(false);
}
if (length.HasValue && (metadata == null || !metadata.ContentLength.HasValue))
{
blob.Metadata[MetadataConstants.ContentLengthMetadataKey] = length.Value.ToString(CultureInfo.InvariantCulture);
// Save the length straight away before the snapshot...
await blob.SetMetadataAsync(token).ConfigureAwait(false);
}
// Create a snapshot straight away on azure
// Note: this shouldnt matter for cost as any blocks that are the same do not cost extra
if (_enableSnapshots)
{
var snapshotBlob = await blob.CreateSnapshotAsync(token).ConfigureAwait(false);
var snapshot = snapshotBlob.SnapshotTime.Value.UtcTicks.ToString(CultureInfo.InvariantCulture);
// Save the snapshot back to original blob...
blob.Metadata[InternalSnapshotKey] = snapshot;
await blob.SetMetadataAsync(token).ConfigureAwait(false);
LeoTrace.WriteLine("Created Snapshot: " + blob.Name);
}
result.Metadata = await GetActualMetadata(blob).ConfigureAwait(false);
}
catch (StorageException exc)
{
if (isOptimistic)
{
// First condition occurrs when the eTags do not match
// Second condition when we specified no eTag (ie must be new blob)
if (exc.RequestInformation.HttpStatusCode == (int)HttpStatusCode.PreconditionFailed
|| (exc.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Conflict && exc.RequestInformation.ExtendedErrorInformation.ErrorCode == "BlobAlreadyExists"))
{
result.Result = false;
}
else
{
// Might have been a different error?
throw exc.Wrap(blob.Name);
}
}
else
{
if (exc.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Conflict
|| exc.RequestInformation.ExtendedErrorInformation.ErrorCode == "LeaseIdMissing")
{
throw new LockException("The underlying storage is currently locked for save");
}
// Might have been a different error?
throw exc.Wrap(blob.Name);
}
}
return result;
}
public async Task <Metadata> SaveData(StoreLocation location, Metadata mdata, UpdateAuditInfo audit, Func <IWriteAsyncStream, Task> savingFunc, CancellationToken token, IEncryptor encryptor = null, SecureStoreOptions options = SecureStoreOptions.All)
{
LeoTrace.WriteLine("Saving: " + location.Container + ", " + location.BasePath + ", " + (location.Id.HasValue ? location.Id.Value.ToString() : "null"));
var metadata = new Metadata(mdata);
/****************************************************
* SETUP METADATA
* ***************************************************/
if (encryptor != null)
{
metadata[MetadataConstants.EncryptionMetadataKey] = encryptor.Algorithm;
}
else
{
metadata.Remove(MetadataConstants.EncryptionMetadataKey);
}
if (options.HasFlag(SecureStoreOptions.Compress))
{
if (_compressor == null)
{
throw new ArgumentException("Compression option should not be used if no compressor has been implemented", "options");
}
metadata[MetadataConstants.CompressionMetadataKey] = _compressor.Algorithm;
}
else
{
metadata.Remove(MetadataConstants.CompressionMetadataKey);
}
/****************************************************
* PREPARE THE SAVE STREAM
* ***************************************************/
var m = await _store.SaveData(location, metadata, audit, async (stream) =>
{
LengthCounterStream counter = null;
stream = stream.AddTransformer(s =>
{
// Encrypt just before writing to the stream (if we need)
if (encryptor != null)
{
s = encryptor.Encrypt(s, false);
}
// Compression comes right before encryption
if (options.HasFlag(SecureStoreOptions.Compress))
{
s = _compressor.CompressWriteStream(s);
}
// Always place the length counter stream
counter = new LengthCounterStream(s);
return(counter);
});
await savingFunc(stream);
await stream.Complete(token);
return(counter.Length);
}, token);
/****************************************************
* POST SAVE TASKS (BACKUP, INDEX)
* ***************************************************/
// The rest of the tasks are done asyncly
var tasks = new List <Task>();
if (options.HasFlag(SecureStoreOptions.Backup))
{
if (_backupQueue == null)
{
throw new ArgumentException("Backup option should not be used if no backup queue has been defined", "options");
}
tasks.Add(_backupQueue.SendMessage(GetMessageDetails(location, metadata)));
}
if (options.HasFlag(SecureStoreOptions.Index))
{
tasks.Add(ForceIndex(location, mdata));
}
if (tasks.Count > 0)
{
await Task.WhenAll(tasks);
}
return(m);
}
public async Task <Metadata> SaveObject <T>(StoreLocation location, ObjectWithMetadata <T> obj, UpdateAuditInfo audit, IEncryptor encryptor = null, SecureStoreOptions options = SecureStoreOptions.All)
where T : ObjectWithAuditInfo
{
// Serialise to json as more cross platform
obj.Data.HideAuditInfo = true;
var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(obj.Data));
obj.Data.HideAuditInfo = false;
obj.Metadata[MetadataConstants.TypeMetadataKey] = typeof(T).FullName;
var ct = CancellationToken.None;
var metadata = await SaveData(location, obj.Metadata, audit, (s) => s.WriteAsync(data, 0, data.Length, ct), ct, encryptor, options);
obj.Data.Audit = metadata.Audit;
return(metadata);
}
public Task<OptimisticStoreWriteResult> TryOptimisticWrite(StoreLocation location, Metadata metadata, UpdateAuditInfo audit, Func<IWriteAsyncStream, Task<long?>> savingFunc, CancellationToken token)
{
return SaveDataInternal(location, metadata, audit, savingFunc, token, true);
}
public async Task <Metadata> SaveData(StoreLocation location, Metadata metadata, UpdateAuditInfo audit, Func <IWriteAsyncStream, Task <long?> > savingFunc, CancellationToken token)
{
var result = await SaveDataInternal(location, metadata, audit, savingFunc, token, false).ConfigureAwait(false);
return(result.Metadata);
}
