public async Task <IEnumerable <RootIndexRow> > UpdateIndexFileWithValues(IList <string> values,
string indexFilename, string originFilename)
{
//todo: test after refactor
//todo: test update integrity
var downloadedFilename =
CreateNewColumnIndexFileName();
if (indexFilename != CommonKeys.END_OF_INDEX_FLAG)
{
downloadedFilename = await _amazonAdapter.DownloadObjectAsync(_bucketName, indexFilename);
}
else
{
indexFilename = downloadedFilename;
}
var rootIndexRows = await _indexFileAccess.CreateUpdatedIndexFileWithValues(downloadedFilename, values, originFilename);
for (var i = 0; i < rootIndexRows.Count; i++)
{
var rootIndexRow = rootIndexRows[i];
var localFileName = rootIndexRow.FileName;
rootIndexRow.FileName = i == 0 ? indexFilename : CreateNewColumnIndexFileName();
await _amazonAdapter.UploadObjectAsync(_bucketName, rootIndexRow.FileName, localFileName);
}
return(rootIndexRows);
}
protected override async Task HandleMessage(string filename)
{
var fileType = filename.GetExtensionFromPath();
var tableWrapperFactory = _tableWrapperFactories.AsParallel().First(extractor => extractor.IsFileTypeSupported(fileType));
var downloadedFileName = await _amazonAdapter.DownloadObjectAsync(_bucketName, filename);
//todo: add handling of no available
var tableWrapper = tableWrapperFactory.CreateTableWrapperForFile(downloadedFileName);
await _contentIndexer.IndexTableAsync(tableWrapper, filename);
}
public async Task UpdateColumnRanges(SortedSet <RootIndexColumnUpdate> columnMappings)
{
await _lockMechanism.TakeLockAsync(CommonKeys.ROOT_INDEX_FILE_LOCK_KEY, _lockTimeSpan);
var downloadedFileName = await _amazonAdapter.DownloadObjectAsync(_bucketName, _rootIndexName);
using var stream = OptionalFileStream.CreateOptionalFileReadStream(downloadedFileName);
var indexRows = GetIndexRowsFromFile(stream);
var outputRows = MergeIndexWithUpdate(indexRows, columnMappings);
var outputFileName = downloadedFileName + "_new";
await WriteIndexRowsToFile(outputFileName, outputRows);
await _amazonAdapter.UploadObjectAsync(_bucketName, _rootIndexName, outputFileName);
await _lockMechanism.ReleaseLockAsync(CommonKeys.ROOT_INDEX_FILE_LOCK_KEY);
await _cache.UpdateColumnRanges(columnMappings);
}
protected async Task <string> DownloadIndexFile(string filename)
{
try
{
return(await _amazonAdapter.DownloadObjectAsync(_bucketName, filename));
}
catch (ResourceNotFoundException)
{
return(null);
}
}
protected override async Task HandleMessage(string filename)
{
var downloadedFileName = await _amazonAdapter.DownloadObjectAsync(_bucketName, filename);
var allMetadata = _extractors.AsParallel().SelectMany(extractor => extractor.ExtractMetadata(downloadedFileName))
.ToDictionary(kvp => kvp.Key, kvp => kvp.Value);
var tasks = new List <Task>();
foreach (var metadata in allMetadata)
{
tasks.AddRange(_indexers.Select(indexer => IndexMetadataAsTask(indexer, metadata)));
}
await Task.WhenAll(tasks);
}
public async Task IndexMetadata(KeyValuePair <string, string> data)
{
var indexFileName = GetIndexFilePath(data.Key);
await _fileLock.TakeLockAsync(indexFileName, _lockTimeSpan);
string downloadedIndexFile;
try
{
downloadedIndexFile = await _amazonAdapter.DownloadObjectAsync(_bucketName, indexFileName);
}
catch (Exception)
{
downloadedIndexFile = null;
}
var indexObject = GetIndexObjectFromFile(downloadedIndexFile);
UpdateIndexObjectWithMetadata(indexObject, data.Value);
var outputFileName = WriteIndexObjectToFile(indexObject);
await _amazonAdapter.UploadObjectAsync(_bucketName, indexFileName, outputFileName);
await _fileLock.ReleaseLockAsync(indexFileName);
}