public async Task <IList <RootIndexRow> > CreateUpdatedIndexFileWithValues(string sourceIndexFileName, IList <string> values,
string valuesFileName
)
{
var originalIndexValues = AsyncEnumerable.Empty <IndexValueModel>();
using var fileStream = OptionalFileStream.CreateOptionalFileReadStream(sourceIndexFileName);
if (fileStream.BaseStream != null)
{
originalIndexValues = GetIndexValuesFromFile(fileStream.BaseStream);
}
var indexValues = MergeIndexWithUpdatedValues(originalIndexValues, values, valuesFileName);
var rootIndexRows = new List <RootIndexRow>();
var indexValueBatches = indexValues.Batch(_maxRowsPerFile);
await foreach (var batch in indexValueBatches)
{
var outputFilename = Guid.NewGuid().ToString();
var rootIndexRow = await MergeIndexValuesToFile(batch, outputFilename);
rootIndexRows.Add(rootIndexRow);
}
return(rootIndexRows);
}
private static IAsyncEnumerable <RootIndexRow> GetIndexRowsFromFile(OptionalFileStream stream)
{
var indexFile = stream.BaseStream;
var rows = AsyncEnumerable.Empty <RootIndexRow>();
if (indexFile != null)
{
rows = indexFile.GetDeserializedRowsFromFileAsync <RootIndexRow>(indexFile.Length);
}
return(rows);
}
public async Task <string> GetFileNameForColumnAndValue(string column, string val)
{
var cached = await _cache.GetFileNameForColumnAndValue(column, val);
if (cached != null)
{
return(cached);
}
await _lockMechanism.TakeLockAsync(CommonKeys.ROOT_INDEX_FILE_LOCK_KEY, _lockTimeSpan);
var downloadedFileName = "__NOT_EXISTING__";
try
{
downloadedFileName = await _amazonAdapter.DownloadObjectAsync(_bucketName, _rootIndexName);
}
catch (ResourceNotFoundException)
{
await CreateRootIndexFile();
}
await _lockMechanism.ReleaseLockAsync(CommonKeys.ROOT_INDEX_FILE_LOCK_KEY);
using var stream = OptionalFileStream.CreateOptionalFileReadStream(downloadedFileName);
var indexRows = GetIndexRowsFromFile(stream);
var relevantIndexRow =
await indexRows.Where(row => ValidateRowWithRequest(column, val, row)).FirstOrDefaultAsync();
if (relevantIndexRow == default(RootIndexRow))
{
return(CommonKeys.END_OF_INDEX_FLAG);
}
var update = new RootIndexColumnUpdate
{
ColumnName = column,
Rows = new SortedSet <RootIndexRow> {
relevantIndexRow
}
};
await _cache.UpdateColumnRanges(new SortedSet <RootIndexColumnUpdate> {
update
});
return(relevantIndexRow.FileName);
}
public async Task UpdateColumnRanges(SortedSet <RootIndexColumnUpdate> columnMappings)
{
await _lockMechanism.TakeLockAsync(CommonKeys.ROOT_INDEX_FILE_LOCK_KEY, _lockTimeSpan);
var downloadedFileName = await _amazonAdapter.DownloadObjectAsync(_bucketName, _rootIndexName);
using var stream = OptionalFileStream.CreateOptionalFileReadStream(downloadedFileName);
var indexRows = GetIndexRowsFromFile(stream);
var outputRows = MergeIndexWithUpdate(indexRows, columnMappings);
var outputFileName = downloadedFileName + "_new";
await WriteIndexRowsToFile(outputFileName, outputRows);
await _amazonAdapter.UploadObjectAsync(_bucketName, _rootIndexName, outputFileName);
await _lockMechanism.ReleaseLockAsync(CommonKeys.ROOT_INDEX_FILE_LOCK_KEY);
await _cache.UpdateColumnRanges(columnMappings);
}
