public async Task <int> IncrementDeletedInstanceRetryAsync(VersionedInstanceIdentifier versionedInstanceIdentifier, DateTimeOffset cleanupAfter, CancellationToken cancellationToken = default)
{
await _sqlServerIndexSchema.EnsureInitialized();
using (SqlConnectionWrapper sqlConnectionWrapper = _sqlConnectionFactoryWrapper.ObtainSqlConnectionWrapper(true))
using (SqlCommandWrapper sqlCommandWrapper = sqlConnectionWrapper.CreateSqlCommand())
{
VLatest.IncrementDeletedInstanceRetry.PopulateCommand(
sqlCommandWrapper,
versionedInstanceIdentifier.StudyInstanceUid,
versionedInstanceIdentifier.SeriesInstanceUid,
versionedInstanceIdentifier.SopInstanceUid,
versionedInstanceIdentifier.Version,
cleanupAfter);
try
{
return((int)(await sqlCommandWrapper.ExecuteScalarAsync(cancellationToken)));
}
catch (SqlException ex)
{
throw new DataStoreException(ex);
}
}
}
/// <inheritdoc />
public async Task <DicomDataset> GetInstanceMetadataAsync(VersionedInstanceIdentifier versionedInstanceIdentifier, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(versionedInstanceIdentifier, nameof(versionedInstanceIdentifier));
BlockBlobClient cloudBlockBlob = GetInstanceBlockBlob(versionedInstanceIdentifier);
DicomDataset dicomDataset = null;
await ExecuteAsync(async() =>
{
await using (Stream stream = _recyclableMemoryStreamManager.GetStream(GetInstanceMetadataStreamTagName))
{
await cloudBlockBlob.DownloadToAsync(stream, cancellationToken);
stream.Seek(0, SeekOrigin.Begin);
using (var streamReader = new StreamReader(stream, MetadataEncoding))
using (var jsonTextReader = new JsonTextReader(streamReader))
{
dicomDataset = _jsonSerializer.Deserialize <DicomDataset>(jsonTextReader);
}
}
});
return(dicomDataset);
}
private async Task <Uri> AddFileAsync(VersionedInstanceIdentifier versionedInstanceIdentifier, byte[] bytes, string tag, CancellationToken cancellationToken = default)
{
await using (var stream = _recyclableMemoryStreamManager.GetStream(tag, bytes, 0, bytes.Length))
{
return(await _blobDataStore.StoreFileAsync(versionedInstanceIdentifier, stream, cancellationToken));
}
}
/// <inheritdoc />
public async Task <DicomDataset> GetInstanceMetadataAsync(VersionedInstanceIdentifier versionedInstanceIdentifier, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(versionedInstanceIdentifier, nameof(versionedInstanceIdentifier));
string instanceIdentifierInString = versionedInstanceIdentifier.ToString();
LogGetInstanceMetadataDelegate(_logger, instanceIdentifierInString, null);
try
{
DicomDataset dicomDataset = await _metadataStore.GetInstanceMetadataAsync(versionedInstanceIdentifier, cancellationToken);
LogOperationSucceededDelegate(_logger, null);
return(dicomDataset);
}
catch (ItemNotFoundException ex)
{
LogMetadataDoesNotExistDelegate(_logger, instanceIdentifierInString, ex);
throw;
}
catch (Exception ex)
{
LogOperationFailedDelegate(_logger, ex);
throw;
}
}
public async Task GivenAValidFileStream_WhenStored_ThenItCanBeRetrievedAndDeleted()
{
VersionedInstanceIdentifier instanceIdentifier = GenerateIdentifier();
var fileData = new byte[] { 4, 7, 2 };
// Store the file.
Uri fileLocation = await AddFileAsync(instanceIdentifier, fileData, $"{nameof(GivenAValidFileStream_WhenStored_ThenItCanBeRetrievedAndDeleted)}.fileData");
Assert.NotNull(fileLocation);
// Should be able to retrieve.
await using (Stream resultStream = await _blobDataStore.GetFileAsync(instanceIdentifier))
{
Assert.Equal(
fileData,
await ConvertStreamToByteArrayAsync(resultStream));
}
// Should be able to delete.
await _blobDataStore.DeleteFileIfExistsAsync(instanceIdentifier);
// The file should no longer exists.
await Assert.ThrowsAsync <ItemNotFoundException>(() => _blobDataStore.GetFileAsync(instanceIdentifier));
}
/// <inheritdoc />
public async Task <Uri> StoreFileAsync(
VersionedInstanceIdentifier versionedInstanceIdentifier,
Stream stream,
CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(versionedInstanceIdentifier, nameof(versionedInstanceIdentifier));
EnsureArg.IsNotNull(stream, nameof(stream));
BlockBlobClient blob = GetInstanceBlockBlob(versionedInstanceIdentifier);
stream.Seek(0, SeekOrigin.Begin);
var blobUploadOptions = new BlobUploadOptions {
TransferOptions = _options.Upload
};
try
{
await blob.UploadAsync(
stream,
blobUploadOptions,
cancellationToken);
return(blob.Uri);
}
catch (Exception ex)
{
throw new DataStoreException(ex);
}
}
public async Task <bool> ReindexInstanceAsync(
IReadOnlyCollection <ExtendedQueryTagStoreEntry> entries,
VersionedInstanceIdentifier versionedInstanceId,
CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(entries, nameof(entries));
EnsureArg.IsNotNull(versionedInstanceId, nameof(versionedInstanceId));
DicomDataset dataset;
try
{
dataset = await _metadataStore.GetInstanceMetadataAsync(versionedInstanceId, cancellationToken);
}
catch (ItemNotFoundException)
{
_logger.LogWarning("Could not find metadata for instance with {Identifier}", versionedInstanceId);
return(false);
}
// Only reindex on valid query tags
IReadOnlyCollection <QueryTag> validQueryTags = await _dicomDatasetReindexValidator.ValidateAsync(
dataset,
versionedInstanceId.Version,
entries.Select(x => new QueryTag(x)).ToList(),
cancellationToken);
await _indexDataStore.ReindexInstanceAsync(dataset, versionedInstanceId.Version, validQueryTags, cancellationToken);
return(true);
}
/// <inheritdoc />
public async Task <Uri> StoreFileAsync(
VersionedInstanceIdentifier versionedInstanceIdentifier,
Stream stream,
CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(versionedInstanceIdentifier, nameof(versionedInstanceIdentifier));
EnsureArg.IsNotNull(stream, nameof(stream));
BlockBlobClient blob = GetInstanceBlockBlob(versionedInstanceIdentifier);
stream.Seek(0, SeekOrigin.Begin);
try
{
await blob.UploadAsync(
stream,
new BlobHttpHeaders()
{
ContentType = KnownContentTypes.ApplicationDicom,
},
metadata : null,
conditions : null,
accessTier : null,
progressHandler : null,
cancellationToken);
return(blob.Uri);
}
catch (Exception ex)
{
throw new DataStoreException(ex);
}
}
/// <inheritdoc />
public async Task DeleteInstanceMetadataIfExistsAsync(VersionedInstanceIdentifier versionedInstanceIdentifier, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(versionedInstanceIdentifier, nameof(versionedInstanceIdentifier));
BlockBlobClient blob = GetInstanceBlockBlob(versionedInstanceIdentifier);
await ExecuteAsync(() => blob.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, conditions: null, cancellationToken));
}
/// <inheritdoc />
public async Task <Stream> GetFileAsync(VersionedInstanceIdentifier versionedInstanceIdentifier, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(versionedInstanceIdentifier, nameof(versionedInstanceIdentifier));
string instanceIdentifierInString = versionedInstanceIdentifier.ToString();
LogGetFileDelegate(_logger, instanceIdentifierInString, null);
try
{
Stream stream = await _fileStore.GetFileAsync(versionedInstanceIdentifier, cancellationToken);
LogOperationSucceededDelegate(_logger, null);
return(stream);
}
catch (ItemNotFoundException ex)
{
LogFileDoesNotExistDelegate(_logger, instanceIdentifierInString, ex);
throw;
}
catch (Exception ex)
{
LogOperationFailedDelegate(_logger, ex);
throw;
}
}
/// <inheritdoc />
public async Task StoreDicomInstanceEntryAsync(
IDicomInstanceEntry dicomInstanceEntry,
CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(dicomInstanceEntry, nameof(dicomInstanceEntry));
DicomDataset dicomDataset = await dicomInstanceEntry.GetDicomDatasetAsync(cancellationToken);
long version = await _indexDataStore.CreateInstanceIndexAsync(dicomDataset, cancellationToken);
VersionedInstanceIdentifier versionedInstanceIdentifier = dicomDataset.ToVersionedInstanceIdentifier(version);
try
{
// We have successfully created the index, store the files.
Task[] tasks = new[]
{
StoreFileAsync(versionedInstanceIdentifier, dicomInstanceEntry, cancellationToken),
StoreInstanceMetadataAsync(dicomDataset, version, cancellationToken),
};
await Task.WhenAll(tasks);
// Successfully uploaded the files. Update the status to be available.
await _indexDataStore.UpdateInstanceIndexStatusAsync(versionedInstanceIdentifier, IndexStatus.Created, cancellationToken);
}
catch (Exception)
{
// Exception occurred while storing the file. Try delete the index.
await TryCleanupInstanceIndexAsync(versionedInstanceIdentifier);
throw;
}
}
private async Task ValidateNoChangeFeed(VersionedInstanceIdentifier dicomInstanceIdentifier)
{
IReadOnlyList <ChangeFeedRow> result = await _fixture.DicomIndexDataStoreTestHelper.GetChangeFeedRowsAsync(
dicomInstanceIdentifier.StudyInstanceUid,
dicomInstanceIdentifier.SeriesInstanceUid,
dicomInstanceIdentifier.SopInstanceUid);
Assert.NotNull(result);
Assert.Equal(0, result.Count);
}
public async Task GivenANonExistingMetadata_WhenRetrievingInstanceMetadata_ThenItemNotFoundExceptionShouldBeThrown()
{
var instanceIdentifier = new VersionedInstanceIdentifier(
studyInstanceUid: TestUidGenerator.Generate(),
seriesInstanceUid: TestUidGenerator.Generate(),
sopInstanceUid: TestUidGenerator.Generate(),
version: 0);
await Assert.ThrowsAsync <ItemNotFoundException>(
() => _metadataStore.GetInstanceMetadataAsync(instanceIdentifier));
}
public async Task GivenRetrieveInstanceMetadataRequestForInstance_WhenFailsToRetrieve_ThenDicomInstanceNotFoundExceptionIsThrownAsync()
{
VersionedInstanceIdentifier sopInstanceIdentifier = SetupInstanceIdentifiersList(ResourceType.Instance).First();
_metadataStore.GetInstanceMetadataAsync(sopInstanceIdentifier, DefaultCancellationToken).Throws(new InstanceNotFoundException());
string ifNoneMatch = null;
var exception = await Assert.ThrowsAsync <InstanceNotFoundException>(() => _retrieveMetadataService.RetrieveSopInstanceMetadataAsync(_studyInstanceUid, _seriesInstanceUid, _sopInstanceUid, ifNoneMatch, DefaultCancellationToken));
Assert.Equal("The specified instance cannot be found.", exception.Message);
}
private async Task ValidateRemoval(bool success, int retrievedInstanceCount, VersionedInstanceIdentifier versionedInstanceIdentifier)
{
Assert.True(success);
Assert.Equal(1, retrievedInstanceCount);
await Assert.ThrowsAsync <ItemNotFoundException>(async() => await _fixture.MetadataStore.GetInstanceMetadataAsync(versionedInstanceIdentifier));
await Assert.ThrowsAsync <ItemNotFoundException>(async() => await _fixture.FileStore.GetFileAsync(versionedInstanceIdentifier));
Assert.Empty(await _fixture.IndexDataStoreTestHelper.GetDeletedInstanceEntriesAsync(versionedInstanceIdentifier.StudyInstanceUid, versionedInstanceIdentifier.SeriesInstanceUid, versionedInstanceIdentifier.SopInstanceUid));
}
public async Task GivenRetrieveInstanceMetadataRequestForInstance_WhenIsSuccessful_ThenSuccessStatusCodeIsReturnedAsync()
{
VersionedInstanceIdentifier sopInstanceIdentifier = SetupInstanceIdentifiersList(ResourceType.Instance).First();
_metadataStore.GetInstanceMetadataAsync(sopInstanceIdentifier, DefaultCancellationToken).Returns(new DicomDataset());
string ifNoneMatch = null;
RetrieveMetadataResponse response = await _retrieveMetadataService.RetrieveSopInstanceMetadataAsync(_studyInstanceUid, _seriesInstanceUid, _sopInstanceUid, ifNoneMatch, DefaultCancellationToken);
Assert.Single(response.ResponseMetadata);
}
private async Task StoreFileAsync(
VersionedInstanceIdentifier versionedInstanceIdentifier,
IDicomInstanceEntry dicomInstanceEntry,
CancellationToken cancellationToken)
{
Stream stream = await dicomInstanceEntry.GetStreamAsync(cancellationToken);
await _fileStore.StoreFileAsync(
versionedInstanceIdentifier,
stream,
cancellationToken);
}
private async Task PopulateMetadata(ChangeFeedEntry entry, CancellationToken cancellationToken)
{
if (entry.State == ChangeFeedState.Deleted || entry.CurrentVersion == null)
{
return;
}
var identifier = new VersionedInstanceIdentifier(entry.StudyInstanceUid, entry.SeriesInstanceUid, entry.SopInstanceUid, entry.CurrentVersion.Value);
entry.Metadata = await _metadataStore.GetInstanceMetadataAsync(identifier, cancellationToken);
entry.IncludeMetadata = true;
}
public async Task GivenADeletedDicomInstance_WhenIncrementingRetryCount_NewRetryCountShouldBeReturned()
{
string studyInstanceUid = TestUidGenerator.Generate();
string seriesInstanceUid = TestUidGenerator.Generate();
string sopInstanceUid = TestUidGenerator.Generate();
Instance instance1 = await CreateIndexAndVerifyInstance(studyInstanceUid, seriesInstanceUid, sopInstanceUid);
await _indexDataStore.DeleteInstanceIndexAsync(studyInstanceUid, seriesInstanceUid, sopInstanceUid, Clock.UtcNow);
DeletedInstance deletedEntry = (await _testHelper.GetDeletedInstanceEntriesAsync(studyInstanceUid, seriesInstanceUid, sopInstanceUid)).First();
var versionedDicomInstanceIdentifier = new VersionedInstanceIdentifier(studyInstanceUid, seriesInstanceUid, sopInstanceUid, deletedEntry.Watermark);
var retryCount = await _indexDataStore.IncrementDeletedInstanceRetryAsync(versionedDicomInstanceIdentifier, Clock.UtcNow);
Assert.Equal(1, retryCount);
}
private async Task ValidateDeleteFeed(VersionedInstanceIdentifier dicomInstanceIdentifier, int expectedCount)
{
IReadOnlyList <ChangeFeedRow> result = await _fixture.DicomIndexDataStoreTestHelper.GetChangeFeedRowsAsync(
dicomInstanceIdentifier.StudyInstanceUid,
dicomInstanceIdentifier.SeriesInstanceUid,
dicomInstanceIdentifier.SopInstanceUid);
Assert.NotNull(result);
Assert.Equal(expectedCount, result.Count);
Assert.Equal((int)ChangeFeedAction.Delete, result.Last().Action);
foreach (ChangeFeedRow row in result)
{
Assert.Null(row.CurrentWatermark);
}
}
/// <inheritdoc />
public Task <DicomDataset> GetInstanceMetadataAsync(VersionedInstanceIdentifier versionedInstanceIdentifier, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(versionedInstanceIdentifier, nameof(versionedInstanceIdentifier));
BlockBlobClient cloudBlockBlob = GetInstanceBlockBlob(versionedInstanceIdentifier);
return(ExecuteAsync(async t =>
{
await using (Stream stream = _recyclableMemoryStreamManager.GetStream(GetInstanceMetadataStreamTagName))
{
await cloudBlockBlob.DownloadToAsync(stream, cancellationToken);
stream.Seek(0, SeekOrigin.Begin);
return await JsonSerializer.DeserializeAsync <DicomDataset>(stream, _jsonSerializerOptions, t);
}
}, cancellationToken));
}
/// <inheritdoc />
public async Task DeleteInstanceMetadataIfExistsAsync(VersionedInstanceIdentifier versionedInstanceIdentifier, CancellationToken cancellationToken)
{
LogDeleteInstanceMetadataDelegate(_logger, versionedInstanceIdentifier.ToString(), null);
try
{
await _metadataStore.DeleteInstanceMetadataIfExistsAsync(versionedInstanceIdentifier, cancellationToken);
LogOperationSucceededDelegate(_logger, null);
}
catch (Exception ex)
{
LogOperationFailedDelegate(_logger, ex);
throw;
}
}
/// <inheritdoc />
public async Task UpdateInstanceIndexStatusAsync(VersionedInstanceIdentifier versionedInstanceIdentifier, IndexStatus status, CancellationToken cancellationToken)
{
LogUpdateInstanceIndexStatusDelegate(_logger, versionedInstanceIdentifier, status, null);
try
{
await IndexDataStore.UpdateInstanceIndexStatusAsync(versionedInstanceIdentifier, status, cancellationToken);
LogOperationSucceededDelegate(_logger, null);
}
catch (Exception ex)
{
LogOperationFailedDelegate(_logger, ex);
throw;
}
}
/// <inheritdoc />
public async Task DeleteDeletedInstanceAsync(VersionedInstanceIdentifier versionedInstanceIdentifier, CancellationToken cancellationToken)
{
LogDeleteDeletedInstanceAsyncDelegate(_logger, versionedInstanceIdentifier, null);
try
{
await IndexDataStore.DeleteDeletedInstanceAsync(versionedInstanceIdentifier, cancellationToken);
LogOperationSucceededDelegate(_logger, null);
}
catch (Exception ex)
{
LogOperationFailedDelegate(_logger, ex);
throw;
}
}
private async Task TryCleanupInstanceIndexAsync(VersionedInstanceIdentifier versionedInstanceIdentifier)
{
try
{
// In case the request is canceled and one of the operation failed, we still want to cleanup.
// Therefore, we will not be using the same cancellation token as the request itself.
await _deleteService.DeleteInstanceNowAsync(
versionedInstanceIdentifier.StudyInstanceUid,
versionedInstanceIdentifier.SeriesInstanceUid,
versionedInstanceIdentifier.SopInstanceUid,
CancellationToken.None);
}
catch (Exception)
{
// Fire and forget.
}
}
/// <inheritdoc />
public async Task <Stream> GetFileAsync(
VersionedInstanceIdentifier versionedInstanceIdentifier,
CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(versionedInstanceIdentifier, nameof(versionedInstanceIdentifier));
BlockBlobClient blob = GetInstanceBlockBlob(versionedInstanceIdentifier);
Stream stream = null;
var blobOpenReadOptions = new BlobOpenReadOptions(allowModifications: false);
await ExecuteAsync(async() =>
{
stream = await blob.OpenReadAsync(blobOpenReadOptions, cancellationToken);
});
return(stream);
}
/// <inheritdoc />
public async Task <int> IncrementDeletedInstanceRetryAsync(VersionedInstanceIdentifier versionedInstanceIdentifier, DateTimeOffset cleanupAfter, CancellationToken cancellationToken)
{
LogIncrementDeletedInstanceRetryAsyncDelegate(_logger, versionedInstanceIdentifier, cleanupAfter, null);
try
{
int returnValue = await IndexDataStore.IncrementDeletedInstanceRetryAsync(versionedInstanceIdentifier, cleanupAfter, cancellationToken);
LogOperationSucceededDelegate(_logger, null);
return(returnValue);
}
catch (Exception ex)
{
LogOperationFailedDelegate(_logger, ex);
throw;
}
}
/// <inheritdoc />
public async Task DeleteFileIfExistsAsync(VersionedInstanceIdentifier versionedInstanceIdentifier, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(versionedInstanceIdentifier, nameof(versionedInstanceIdentifier));
LogDeleteFileDelegate(_logger, versionedInstanceIdentifier.ToString(), null);
try
{
await _fileStore.DeleteFileIfExistsAsync(versionedInstanceIdentifier, cancellationToken);
LogOperationSucceededDelegate(_logger, null);
}
catch (Exception ex)
{
LogOperationFailedDelegate(_logger, ex);
throw;
}
}
/// <inheritdoc />
public async Task <Uri> StoreFileAsync(VersionedInstanceIdentifier versionedInstanceIdentifier, Stream stream, CancellationToken cancellationToken)
{
EnsureArg.IsNotNull(versionedInstanceIdentifier, nameof(versionedInstanceIdentifier));
LogStoreFileDelegate(_logger, versionedInstanceIdentifier.ToString(), null);
try
{
Uri uri = await _fileStore.StoreFileAsync(versionedInstanceIdentifier, stream, cancellationToken);
LogOperationSucceededDelegate(_logger, null);
return(uri);
}
catch (Exception ex)
{
LogOperationFailedDelegate(_logger, ex);
throw;
}
}
private async Task ValidateInsertFeed(VersionedInstanceIdentifier dicomInstanceIdentifier, int expectedCount)
{
IReadOnlyList <ChangeFeedRow> result = await _fixture.DicomIndexDataStoreTestHelper.GetChangeFeedRowsAsync(
dicomInstanceIdentifier.StudyInstanceUid,
dicomInstanceIdentifier.SeriesInstanceUid,
dicomInstanceIdentifier.SopInstanceUid);
Assert.NotNull(result);
Assert.Equal(expectedCount, result.Count);
Assert.Equal((int)ChangeFeedAction.Create, result.Last().Action);
Assert.Equal(result.Last().OriginalWatermark, result.Last().CurrentWatermark);
int i = 0;
while (i < expectedCount - 1)
{
ChangeFeedRow r = result[i];
Assert.NotEqual(r.OriginalWatermark, r.CurrentWatermark);
i++;
}
}