private async Task ProcessSearchResultsAsync(IEnumerable <SearchResultEntry> searchResults, string partId, IAnonymizer anonymizer, CancellationToken cancellationToken)
{
foreach (SearchResultEntry result in searchResults)
{
ResourceWrapper resourceWrapper = result.Resource;
string resourceType = resourceWrapper.ResourceTypeName;
// Check whether we already have an existing file for the current resource type.
if (!_resourceTypeToFileInfoMapping.TryGetValue(resourceType, out ExportFileInfo exportFileInfo))
{
// Check whether we have seen this file previously (in situations where we are resuming an export)
if (_exportJobRecord.Output.TryGetValue(resourceType, out exportFileInfo))
{
// A file already exists for this resource type. Let us open the file on the client.
await _exportDestinationClient.OpenFileAsync(exportFileInfo.FileUri, cancellationToken);
}
else
{
// File does not exist. Create it.
string fileName;
if (_exportJobRecord.StorageAccountContainerName.Equals(_exportJobRecord.Id, StringComparison.OrdinalIgnoreCase))
{
fileName = $"{resourceType}.ndjson";
}
else
{
string dateTime = _exportJobRecord.QueuedTime.UtcDateTime.ToString("s")
.Replace("-", string.Empty, StringComparison.OrdinalIgnoreCase)
.Replace(":", string.Empty, StringComparison.OrdinalIgnoreCase);
fileName = $"{dateTime}-{_exportJobRecord.Id}/{resourceType}.ndjson";
}
Uri fileUri = await _exportDestinationClient.CreateFileAsync(fileName, cancellationToken);
exportFileInfo = new ExportFileInfo(resourceType, fileUri, sequence: 0);
// Since we created a new file the JobRecord Output also needs to know about it.
_exportJobRecord.Output.TryAdd(resourceType, exportFileInfo);
}
_resourceTypeToFileInfoMapping.Add(resourceType, exportFileInfo);
}
ResourceElement element = _resourceDeserializer.Deserialize(resourceWrapper);
if (anonymizer != null)
{
element = anonymizer.Anonymize(element);
}
// Serialize into NDJson and write to the file.
byte[] bytesToWrite = _resourceToByteArraySerializer.Serialize(element);
await _exportDestinationClient.WriteFilePartAsync(exportFileInfo.FileUri, partId, bytesToWrite, cancellationToken);
// Increment the file information.
exportFileInfo.IncrementCount(bytesToWrite.Length);
}
}
private async Task <ExportFileInfo> CreateNewFileAndUpdateMappings(string resourceType, int fileSequence, CancellationToken cancellationToken)
{
string fileName;
if (_exportJobRecord.SchemaVersion == 1)
{
fileName = $"{_exportJobRecord.ExportFormat}.ndjson";
}
else
{
fileName = $"{_exportJobRecord.ExportFormat}-{fileSequence}.ndjson";
}
string dateTime = _exportJobRecord.QueuedTime.UtcDateTime.ToString("s")
.Replace("-", string.Empty, StringComparison.OrdinalIgnoreCase)
.Replace(":", string.Empty, StringComparison.OrdinalIgnoreCase);
fileName = fileName.Replace(ExportFormatTags.Timestamp, dateTime, StringComparison.OrdinalIgnoreCase);
fileName = fileName.Replace(ExportFormatTags.Id, _exportJobRecord.Id, StringComparison.OrdinalIgnoreCase);
fileName = fileName.Replace(ExportFormatTags.ResourceName, resourceType, StringComparison.OrdinalIgnoreCase);
Uri fileUri = await _exportDestinationClient.CreateFileAsync(fileName, cancellationToken);
var newFile = new ExportFileInfo(resourceType, fileUri, sequence: fileSequence);
// Since we created a new file the ExportJobRecord Output also needs to be updated.
if (_exportJobRecord.Output.TryGetValue(resourceType, out List <ExportFileInfo> fileList))
{
fileList.Add(newFile);
}
else
{
_exportJobRecord.Output.Add(resourceType, new List <ExportFileInfo>()
{
newFile
});
}
// Update internal mapping with new file for the resource type.
if (_resourceTypeToFileInfoMapping.ContainsKey(resourceType))
{
_resourceTypeToFileInfoMapping[resourceType] = newFile;
}
else
{
_resourceTypeToFileInfoMapping.Add(resourceType, newFile);
}
return(newFile);
}
public ExportFileManagerTests()
{
_exportDestinationClient = Substitute.For <IExportDestinationClient>();
_exportDestinationClient
.CreateFileAsync(Arg.Any <string>(), Arg.Any <CancellationToken>())
.Returns <Uri>(callInfo => new Uri("https://localhost/" + callInfo.ArgAt <string>(0)));
_exportJobConfigurationFormat = $"{ExportFormatTags.ResourceName}";
_exportJobRecord = new ExportJobRecord(
new Uri("https://localhost/$export"),
ExportJobType.All,
_exportJobConfigurationFormat,
resourceType: null,
filters: null,
"hash",
rollingFileSizeInMB: 1);
_exportFileManager = new ExportFileManager(_exportJobRecord, _exportDestinationClient);
}
private async Task ProcessSearchResultsAsync(IEnumerable <SearchResultEntry> searchResults, string partId, CancellationToken cancellationToken)
{
foreach (SearchResultEntry result in searchResults)
{
ResourceWrapper resourceWrapper = result.Resource;
string resourceType = resourceWrapper.ResourceTypeName;
// Check whether we already have an existing file for the current resource type.
if (!_resourceTypeToFileInfoMapping.TryGetValue(resourceType, out ExportFileInfo exportFileInfo))
{
// Check whether we have seen this file previously (in situations where we are resuming an export)
if (_exportJobRecord.Output.TryGetValue(resourceType, out exportFileInfo))
{
// A file already exists for this resource type. Let us open the file on the client.
await _exportDestinationClient.OpenFileAsync(exportFileInfo.FileUri, cancellationToken);
}
else
{
// File does not exist. Create it.
string fileName = resourceType + ".ndjson";
Uri fileUri = await _exportDestinationClient.CreateFileAsync(fileName, cancellationToken);
exportFileInfo = new ExportFileInfo(resourceType, fileUri, sequence: 0);
// Since we created a new file the JobRecord Output also needs to know about it.
_exportJobRecord.Output.TryAdd(resourceType, exportFileInfo);
}
_resourceTypeToFileInfoMapping.Add(resourceType, exportFileInfo);
}
// Serialize into NDJson and write to the file.
byte[] bytesToWrite = _resourceToByteArraySerializer.Serialize(resourceWrapper);
await _exportDestinationClient.WriteFilePartAsync(exportFileInfo.FileUri, partId, bytesToWrite, cancellationToken);
// Increment the file information.
exportFileInfo.IncrementCount(bytesToWrite.Length);
}
}
private async Task ProcessResourceWrapperAsync(ResourceWrapper resourceWrapper, uint partId, CancellationToken cancellationToken)
{
string resourceType = resourceWrapper.ResourceTypeName;
// Check whether we already have an existing file for the current resource type.
if (!_resourceTypeToFileInfoMapping.TryGetValue(resourceType, out ExportFileInfo exportFileInfo))
{
string fileName = resourceType + ".ndjson";
Uri fileUri = await _exportDestinationClient.CreateFileAsync(fileName, cancellationToken);
exportFileInfo = new ExportFileInfo(resourceType, fileUri, sequence: 0);
_resourceTypeToFileInfoMapping.Add(resourceType, exportFileInfo);
}
// Serialize into NDJson and write to the file.
byte[] bytesToWrite = _resourceToByteArraySerializer.Serialize(resourceWrapper);
await _exportDestinationClient.WriteFilePartAsync(exportFileInfo.FileUri, partId, bytesToWrite, cancellationToken);
// Increment the file information.
exportFileInfo.IncrementCount(bytesToWrite.Length);
}