public virtual void Init() { _store = new AzureStore(CloudStorageAccount.DevelopmentStorageAccount.CreateCloudBlobClient(), true); _blob = AzureTestsHelper.GetBlockBlob("kalix-leo-tests", "AzureStoreTests.testdata", true); _location = new StoreLocation("kalix-leo-tests", "AzureStoreTests.testdata"); }
private async Task RunOnce(StoreLocation location, Func<Task> action, TimeSpan pollingFrequency) { var blob = GetBlockBlob(location); // blob.Exists has the side effect of calling blob.FetchAttributes, which populates the metadata collection while (!(await blob.ExecuteWrap(b => b.ExistsAsync()).ConfigureAwait(false)) || !blob.Metadata.ContainsKey("progress") || blob.Metadata["progress"] != "done") { var lease = await LockInternal(blob).ConfigureAwait(false); if (lease != null) { using (var arl = lease.Item1) { // Once we have the lock make sure we are not done! if (!blob.Metadata.ContainsKey("progress") || blob.Metadata["progress"] != "done") { await action().ConfigureAwait(false); blob.Metadata["progress"] = "done"; await blob.ExecuteWrap(b => b.SetMetadataAsync(AccessCondition.GenerateLeaseCondition(lease.Item2), null, null)).ConfigureAwait(false); } } } else { await Task.Delay(pollingFrequency).ConfigureAwait(false); } } }
public virtual void Init() { _bucket = "kalixtest"; _client = AmazonTestsHelper.SetupBlob(_bucket, "kalix-leo-tests\\AmazonStoreTests.testdata"); _location = new StoreLocation("kalix-leo-tests", "AmazonStoreTests.testdata"); _store = new AmazonStore(_client, _bucket); }
public static async Task<IEncryptor> CreateEncryptor(IOptimisticStore store, StoreLocation keyLocation, RSAServiceProvider rsaCert) { bool isFound; byte[] blob; do { var data = await store.LoadData(keyLocation).ConfigureAwait(false); if (data == null) { // Have to create a new key blob = AESBlob.CreateBlob(DefaultKeySize, rsaCert); var ct = CancellationToken.None; // We use an optimistic write so that it will only create the file IF THE FILE DOES NOT EXIST // This will catch rare cases where two server calls may try to create two keys var result = await store.TryOptimisticWrite(keyLocation, null, null, async (s) => { await s.WriteAsync(blob, 0, blob.Length, ct).ConfigureAwait(false); return blob.Length; }, ct).ConfigureAwait(false); isFound = result.Result; } else { blob = await data.Stream.ReadBytes().ConfigureAwait(false); isFound = true; } } while (!isFound); var encryptor = AESBlob.CreateEncryptor(blob, rsaCert); return new CertProtectedEncryptor(keyLocation.Container, encryptor); }
public async Task<Metadata> SaveData(StoreLocation location, Metadata metadata, UpdateAuditInfo audit, Func<IWriteAsyncStream, Task<long?>> savingFunc, CancellationToken token) { var current = await GetMetadata(location).ConfigureAwait(false); var info = current == null ? new AuditInfo() : current.Audit; info.UpdatedBy = audit == null ? "0" : audit.UpdatedBy; info.UpdatedByName = audit == null ? string.Empty : audit.UpdatedByName; info.UpdatedOn = DateTime.UtcNow; info.CreatedBy = info.CreatedBy ?? info.UpdatedBy; info.CreatedByName = info.CreatedByName ?? info.UpdatedByName; info.CreatedOn = info.CreatedOn ?? info.UpdatedOn; metadata = metadata ?? new Metadata(); metadata.Audit = info; var key = GetObjectKey(location); long? length = null; using(var stream = new AmazonMultiUploadStream(_client, _bucket, key, metadata)) { length = await savingFunc(stream).ConfigureAwait(false); await stream.Complete(token).ConfigureAwait(false); metadata.Snapshot = stream.VersionId; } if (length.HasValue && (metadata == null || !metadata.ContentLength.HasValue)) { metadata[MetadataConstants.ContentLengthMetadataKey] = length.Value.ToString(CultureInfo.InvariantCulture); // Save the length straight away before the snapshot... metadata = await SaveMetadata(location, metadata).ConfigureAwait(false); } return metadata; }
protected OptimisticStoreWriteResult TryOptimisticWrite(StoreLocation location, Metadata m, byte[] data) { var ct = CancellationToken.None; return _store.TryOptimisticWrite(location, m, null, async (s) => { await s.WriteAsync(data, 0, data.Length, ct).ConfigureAwait(false); return data.Length; }, ct).Result; }
protected void WriteData(StoreLocation location, Metadata m, byte[] data) { var ct = CancellationToken.None; _store.SaveData(location, m, null, async (s) => { await s.WriteAsync(data, 0, data.Length, ct).ConfigureAwait(false); return data.Length; }, ct).Wait(); }
private IDictionary<string, string> GetMetadata(StoreLocation location) { var resp = _client.GetObjectMetadata(new GetObjectMetadataRequest { BucketName = _bucket, Key = Path.Combine(location.Container, location.BasePath), }); return resp.Metadata.Keys.ToDictionary(s => s.Replace("x-amz-meta-", string.Empty), s => resp.Metadata[s]); }
public async Task<Metadata> SaveMetadata(StoreLocation location, Metadata metadata) { // Copy so that we are not modifying original! metadata = new Metadata(metadata); // Do not change the audit information! var current = await GetMetadata(location).ConfigureAwait(false); metadata.Audit = current.Audit; var key = GetObjectKey(location); var request = new CopyObjectRequest { SourceBucket = _bucket, SourceKey = key, DestinationBucket = _bucket, DestinationKey = key, MetadataDirective = S3MetadataDirective.REPLACE }; foreach(var m in metadata) { request.Metadata.Add("x-amz-meta-" + m.Key, m.Value); } // Copy the object (only way to update metadata) string versionToRemove; try { var copyResponse = await _client.CopyObjectAsync(request).ConfigureAwait(false); versionToRemove = copyResponse.SourceVersionId; } catch (AmazonS3Exception e) { if (e.StatusCode == HttpStatusCode.NotFound) { return null; } throw; } // We will be grabbing the metadata var metadataTask = GetMetadata(location); var tasks = new List<Task> { metadataTask }; // Remove the double up so that we don't get heaps of extra snapshots... if (versionToRemove != null) { tasks.Add(_client.DeleteObjectAsync(_bucket, key, versionToRemove)); } await Task.WhenAll(tasks).ConfigureAwait(false); return metadataTask.Result; }
public UniqueIdGenerator( IOptimisticStore store, StoreLocation location, int rangeSize = 10, int maxRetries = 25) { _taskLock = new object(); _rangeSize = rangeSize; _maxRetries = maxRetries; _store = store; _location = location; }
public SecureStoreIndexInput(SecureStoreDirectory directory, Directory cache, ISecureStore store, IEncryptor encryptor, StoreLocation location, string cachePath) { _directory = directory; _cache = cache; _name = cachePath; _fileMutex = BlobMutexManager.GrabMutex(_name); _fileMutex.WaitOne(); try { InitialiseFile(store, encryptor, location).WaitAndWrap(); } finally { _fileMutex.ReleaseMutex(); } }
public async Task<Metadata> SaveMetadata(StoreLocation location, Metadata metadata) { var blob = GetBlockBlob(location); if(!await blob.ExecuteWrap(b => b.FetchAttributesAsync(), true).ConfigureAwait(false)) { return null; } foreach (var m in metadata) { // Do not override audit information! if (m.Key != MetadataConstants.AuditMetadataKey) { blob.Metadata[m.Key] = m.Value; } } await blob.ExecuteWrap(b => b.SetMetadataAsync()).ConfigureAwait(false); return await GetActualMetadata(blob).ConfigureAwait(false); }
public Task RunOnce(StoreLocation location, Func<Task> action) { return RunOnce(location, action, TimeSpan.FromSeconds(5)); }
public IAsyncEnumerable <bool> RunEvery(StoreLocation location, TimeSpan interval, Action <Exception> unhandledExceptions = null) { return(_store.RunEvery(location, interval, unhandledExceptions)); }
public Task<OptimisticStoreWriteResult> TryOptimisticWrite(StoreLocation location, Metadata metadata, UpdateAuditInfo audit, Func<IWriteAsyncStream, Task<long?>> savingFunc, CancellationToken token) { return SaveDataInternal(location, metadata, audit, savingFunc, token, true); }
public async Task<IDisposable> Lock(StoreLocation location) { var blob = GetBlockBlob(location); var l = await LockInternal(blob).ConfigureAwait(false); return l == null ? null : l.Item1; }
public IUniqueIdGenerator GetIdGenerator(StoreLocation location) { return(new UniqueIdGenerator(_store, location)); }
private CloudBlockBlob GetBlockBlob(StoreLocation location, string snapshot = null) { DateTime? snapshotDate = null; if (snapshot != null) { snapshotDate = new DateTime(long.Parse(snapshot, CultureInfo.InvariantCulture)); } var container = _blobStorage.GetContainerReference(SafeContainerName(location.Container)); var offset = snapshotDate.HasValue ? new DateTimeOffset(snapshotDate.Value, new TimeSpan(0)) : (DateTimeOffset?)null; CloudBlockBlob blob; if (location.Id.HasValue) { if (!string.IsNullOrEmpty(location.BasePath)) { var dir = container.GetDirectoryReference(SafePath.MakeSafeFilePath(location.BasePath)); blob = dir.GetBlockBlobReference(location.Id.ToString() + IdExtension, offset); } else { blob = container.GetBlockBlobReference(location.Id.ToString() + IdExtension, offset); } } else { blob = container.GetBlockBlobReference(SafePath.MakeSafeFilePath(location.BasePath), offset); } return blob; }
public void SubItemBlobSnapshotsAreNotIncluded() { var data = AmazonTestsHelper.RandomData(1); WriteData(_location, null, data); AmazonTestsHelper.SetupBlob(_bucket, "kalix-leo-tests\\AzureStoreTests.testdata\\subitem.data"); var location2 = new StoreLocation("kalix-leo-tests", "AzureStoreTests.testdata\\subitem.data"); WriteData(location2, null, data); var snapshots = _store.FindSnapshots(_location).ToList().Result; Assert.AreEqual(1, snapshots.Count()); }
public async Task<Metadata> GetMetadata(StoreLocation location, string snapshot = null) { var blob = GetBlockBlob(location, snapshot); var metadata = await GetBlobMetadata(blob).ConfigureAwait(false); return metadata == null || metadata.ContainsKey(_deletedKey) ? null : metadata; }
public SecureStoreLock(ISecureStore store, StoreLocation location) { _store = store; _location = location; }
public void SubItemBlobSnapshotsAreNotIncluded() { var data = AzureTestsHelper.RandomData(1); WriteData(_location, null, data); var blob2 = AzureTestsHelper.GetBlockBlob("kalix-leo-tests", "AzureStoreTests.testdata/subitem.data", true); var location2 = new StoreLocation("kalix-leo-tests", "AzureStoreTests.testdata/subitem.data"); WriteData(location2, null, data); var snapshots = _store.FindSnapshots(_location).ToEnumerable(); Assert.AreEqual(1, snapshots.Count()); }
public IAsyncEnumerable<LocationWithMetadata> FindFiles(string container, string prefix = null) { var c = _blobStorage.GetContainerReference(SafeContainerName(container)); return ListBlobs(c, prefix, BlobListingDetails.Metadata) .Where(b => !b.Metadata.ContainsKey(_deletedKey)) // Do not include blobs which are soft deleted .Select(async b => { long? id = null; string path = b.Name; if (path.EndsWith(IdExtension)) { long tempId; if (long.TryParse(Path.GetFileNameWithoutExtension(path), out tempId)) { id = tempId; path = Path.GetDirectoryName(path); } } var loc = new StoreLocation(container, path, id); return new LocationWithMetadata(loc, await GetActualMetadata(b).ConfigureAwait(false)); }) .Unwrap(); }
public IAsyncEnumerable<Snapshot> FindSnapshots(StoreLocation location) { if (!_enableSnapshots) { return AsyncEnumerable.Empty<Snapshot>(); } var blob = GetBlockBlob(location); return ListBlobs(blob.Container, blob.Name, BlobListingDetails.Snapshots | BlobListingDetails.Metadata) .Where(b => b.IsSnapshot && b.Uri == blob.Uri && b.SnapshotTime.HasValue) .Select(async b => new Snapshot { Id = b.SnapshotTime.Value.UtcTicks.ToString(CultureInfo.InvariantCulture), Metadata = await GetActualMetadata(b).ConfigureAwait(false) }) .Unwrap(); }
public async Task<DataWithMetadata> LoadData(StoreLocation location, string snapshot = null) { var blob = GetBlockBlob(location, snapshot); // Get metadata first - this record might be deleted so quicker to test this than to download whole record... var metadata = await GetBlobMetadata(blob).ConfigureAwait(false); // If deleted then return null... if (metadata == null || metadata.ContainsKey(_deletedKey)) { return null; } // Older versions need to check the block list before loading... var needsToUseBlockList = !blob.Metadata.ContainsKey(StoreVersionKey); return new DataWithMetadata(new AzureReadBlockBlobStream(blob, needsToUseBlockList), metadata); }
public void Init() { _loc = new StoreLocation(); _store = Substitute.For<IOptimisticStore>(); }
public Task <Metadata> GetMetadata(StoreLocation location, string snapshot = null) { return(_store.GetMetadata(location, snapshot)); }
public Task RunOnce(StoreLocation location, Func <Task> action) { return(_store.RunOnce(location, action)); }
public IAsyncEnumerable<bool> RunEvery(StoreLocation location, TimeSpan interval, Action<Exception> unhandledExceptions = null) { return AsyncEnumerableEx.Create<bool>(async (y) => { var blob = GetBlockBlob(location); var minimum = TimeSpan.FromSeconds(5); // so we're not polling the leased blob too fast while (!y.CancellationToken.IsCancellationRequested) { var timeLeft = TimeSpan.FromSeconds(0); // Don't allow you to throw to get out of the loop... try { var lastPerformed = DateTimeOffset.MinValue; var lease = await LockInternal(blob).ConfigureAwait(false); if (lease != null) { using (var arl = lease.Item1) { await blob.ExecuteWrap(b => b.FetchAttributesAsync(y.CancellationToken)).ConfigureAwait(false); if (blob.Metadata.ContainsKey("lastPerformed")) { DateTimeOffset.TryParseExact(blob.Metadata["lastPerformed"], "R", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, out lastPerformed); } if (DateTimeOffset.UtcNow >= lastPerformed + interval) { await y.YieldReturn(true).ConfigureAwait(false); lastPerformed = DateTimeOffset.UtcNow; blob.Metadata["lastPerformed"] = lastPerformed.ToString("R", CultureInfo.InvariantCulture); await blob.ExecuteWrap(b => b.SetMetadataAsync(AccessCondition.GenerateLeaseCondition(lease.Item2), null, null, y.CancellationToken)).ConfigureAwait(false); } } } timeLeft = (lastPerformed + interval) - DateTimeOffset.UtcNow; } catch (TaskCanceledException) { throw; } catch (Exception e) { unhandledExceptions?.Invoke(e); LeoTrace.WriteLine("Error on lock loop: " + e.Message); } // Do this outside the exception to prevent it going out of control await Task.Delay(timeLeft > minimum ? timeLeft : minimum, y.CancellationToken).ConfigureAwait(false); } }); }
public Task <IAsyncDisposable> Lock(StoreLocation location) { return(_store.Lock(location)); }
public async Task<Metadata> SaveData(StoreLocation location, Metadata metadata, UpdateAuditInfo audit, Func<IWriteAsyncStream, Task<long?>> savingFunc, CancellationToken token) { var result = await SaveDataInternal(location, metadata, audit, savingFunc, token, false).ConfigureAwait(false); return result.Metadata; }
public async Task <Metadata> SaveData(StoreLocation location, Metadata mdata, UpdateAuditInfo audit, Func <IWriteAsyncStream, Task> savingFunc, CancellationToken token, IEncryptor encryptor = null, SecureStoreOptions options = SecureStoreOptions.All) { LeoTrace.WriteLine("Saving: " + location.Container + ", " + location.BasePath + ", " + (location.Id.HasValue ? location.Id.Value.ToString() : "null")); var metadata = new Metadata(mdata); /**************************************************** * SETUP METADATA * ***************************************************/ if (encryptor != null) { metadata[MetadataConstants.EncryptionMetadataKey] = encryptor.Algorithm; } else { metadata.Remove(MetadataConstants.EncryptionMetadataKey); } if (options.HasFlag(SecureStoreOptions.Compress)) { if (_compressor == null) { throw new ArgumentException("Compression option should not be used if no compressor has been implemented", "options"); } metadata[MetadataConstants.CompressionMetadataKey] = _compressor.Algorithm; } else { metadata.Remove(MetadataConstants.CompressionMetadataKey); } /**************************************************** * PREPARE THE SAVE STREAM * ***************************************************/ var m = await _store.SaveData(location, metadata, audit, async (stream) => { LengthCounterStream counter = null; stream = stream.AddTransformer(s => { // Encrypt just before writing to the stream (if we need) if (encryptor != null) { s = encryptor.Encrypt(s, false); } // Compression comes right before encryption if (options.HasFlag(SecureStoreOptions.Compress)) { s = _compressor.CompressWriteStream(s); } // Always place the length counter stream counter = new LengthCounterStream(s); return(counter); }); await savingFunc(stream); await stream.Complete(token); return(counter.Length); }, token); /**************************************************** * POST SAVE TASKS (BACKUP, INDEX) * ***************************************************/ // The rest of the tasks are done asyncly var tasks = new List <Task>(); if (options.HasFlag(SecureStoreOptions.Backup)) { if (_backupQueue == null) { throw new ArgumentException("Backup option should not be used if no backup queue has been defined", "options"); } tasks.Add(_backupQueue.SendMessage(GetMessageDetails(location, metadata))); } if (options.HasFlag(SecureStoreOptions.Index)) { tasks.Add(ForceIndex(location, mdata)); } if (tasks.Count > 0) { await Task.WhenAll(tasks); } return(m); }
private async Task<OptimisticStoreWriteResult> SaveDataInternal(StoreLocation location, Metadata metadata, UpdateAuditInfo audit, Func<IWriteAsyncStream, Task<long?>> savingFunc, CancellationToken token, bool isOptimistic) { var blob = GetBlockBlob(location); // We always want to save the new audit information when saving! var currentMetadata = await GetBlobMetadata(blob).ConfigureAwait(false); var auditInfo = TransformAuditInformation(currentMetadata, audit); metadata = metadata ?? new Metadata(); metadata.Audit = auditInfo; var result = new OptimisticStoreWriteResult() { Result = true }; try { // If the ETag value is empty then the store value must not exist yet... var condition = isOptimistic ? (string.IsNullOrEmpty(metadata.ETag) ? AccessCondition.GenerateIfNoneMatchCondition("*") : AccessCondition.GenerateIfMatchCondition(metadata.ETag)) : null; // Copy the metadata across blob.Metadata.Clear(); foreach (var m in metadata) { blob.Metadata[m.Key] = m.Value; } // Always store the version - We use this to do more efficient things on read blob.Metadata[StoreVersionKey] = StoreVersionValue; long? length; using (var stream = new AzureWriteBlockBlobStream(blob, condition)) { length = await savingFunc(stream).ConfigureAwait(false); await stream.Complete(token).ConfigureAwait(false); } if (length.HasValue && (metadata == null || !metadata.ContentLength.HasValue)) { blob.Metadata[MetadataConstants.ContentLengthMetadataKey] = length.Value.ToString(CultureInfo.InvariantCulture); // Save the length straight away before the snapshot... await blob.SetMetadataAsync(token).ConfigureAwait(false); } // Create a snapshot straight away on azure // Note: this shouldnt matter for cost as any blocks that are the same do not cost extra if (_enableSnapshots) { var snapshotBlob = await blob.CreateSnapshotAsync(token).ConfigureAwait(false); var snapshot = snapshotBlob.SnapshotTime.Value.UtcTicks.ToString(CultureInfo.InvariantCulture); // Save the snapshot back to original blob... blob.Metadata[InternalSnapshotKey] = snapshot; await blob.SetMetadataAsync(token).ConfigureAwait(false); LeoTrace.WriteLine("Created Snapshot: " + blob.Name); } result.Metadata = await GetActualMetadata(blob).ConfigureAwait(false); } catch (StorageException exc) { if (isOptimistic) { // First condition occurrs when the eTags do not match // Second condition when we specified no eTag (ie must be new blob) if (exc.RequestInformation.HttpStatusCode == (int)HttpStatusCode.PreconditionFailed || (exc.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Conflict && exc.RequestInformation.ExtendedErrorInformation.ErrorCode == "BlobAlreadyExists")) { result.Result = false; } else { // Might have been a different error? throw exc.Wrap(blob.Name); } } else { if (exc.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Conflict || exc.RequestInformation.ExtendedErrorInformation.ErrorCode == "LeaseIdMissing") { throw new LockException("The underlying storage is currently locked for save"); } // Might have been a different error? throw exc.Wrap(blob.Name); } } return result; }
public async Task SoftDelete(StoreLocation location, UpdateAuditInfo audit) { // In Azure we cannot delete the blob as this will loose the snapshots // Instead we will just add some metadata var blob = GetBlockBlob(location); var metadata = await GetBlobMetadata(blob).ConfigureAwait(false); // If already deleted don't worry about it! if(metadata == null || metadata.ContainsKey(_deletedKey)) { return; } var fullInfo = TransformAuditInformation(metadata, audit); metadata.Audit = fullInfo; blob.Metadata[MetadataConstants.AuditMetadataKey] = metadata[MetadataConstants.AuditMetadataKey]; blob.Metadata[_deletedKey] = DateTime.UtcNow.Ticks.ToString(); await blob.ExecuteWrap(b => b.SetMetadataAsync()).ConfigureAwait(false); LeoTrace.WriteLine("Soft deleted (2 calls): " + blob.Name); }
public IAsyncEnumerable <Snapshot> FindSnapshots(StoreLocation location) { return(_store.FindSnapshots(location)); }
private async Task InitialiseFile(ISecureStore store, IEncryptor encryptor, StoreLocation location) { bool fFileNeeded = false; if (!_cache.FileExists(_name)) { fFileNeeded = true; } else { long cachedLength = _cache.FileLength(_name); var metadata = await store.GetMetadata(location).ConfigureAwait(false); if (metadata == null) { throw new System.IO.FileNotFoundException(_name); } var blobLength = metadata.ContentLength ?? 0; var blobLastModifiedUTC = metadata.LastModified ?? DateTime.UtcNow; if (cachedLength != blobLength) { fFileNeeded = true; } else { // there seems to be an error of 1 tick which happens every once in a while // for now we will say that if they are within 1 tick of each other and same length var elapsed = _cache.FileModified(_name); // normalize RAMDirectory and FSDirectory times if (elapsed > ticks1970) { elapsed -= ticks1970; } var cachedLastModifiedUTC = new DateTime(elapsed, DateTimeKind.Local).ToUniversalTime(); if (cachedLastModifiedUTC != blobLastModifiedUTC) { var timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC); if (timeSpan.TotalSeconds > 1) { fFileNeeded = true; } } } } // if the file does not exist // or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage) if (fFileNeeded) { using (StreamOutput fileStream = _directory.CreateCachedOutputAsStream(_name)) { var data = await store.LoadData(location, null, encryptor).ConfigureAwait(false); if (data == null) { throw new System.IO.FileNotFoundException(_name); } await data.Stream.CopyToStream(fileStream, CancellationToken.None).ConfigureAwait(false); } // and open it as an input _indexInput = _cache.OpenInput(_name); } else { // open the file in read only mode _indexInput = _cache.OpenInput(_name); } }
public Task PermanentDelete(StoreLocation location) { var blob = GetBlockBlob(location); LeoTrace.WriteLine("Deleted blob: " + blob.Name); return blob.ExecuteWrap(b => b.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, null, null, null)); }