public async Task <IList <RootIndexRow> > CreateUpdatedIndexFileWithValues(string sourceIndexFileName, IList <string> values, string valuesFileName ) { var originalIndexValues = AsyncEnumerable.Empty <IndexValueModel>(); using var fileStream = OptionalFileStream.CreateOptionalFileReadStream(sourceIndexFileName); if (fileStream.BaseStream != null) { originalIndexValues = GetIndexValuesFromFile(fileStream.BaseStream); } var indexValues = MergeIndexWithUpdatedValues(originalIndexValues, values, valuesFileName); var rootIndexRows = new List <RootIndexRow>(); var indexValueBatches = indexValues.Batch(_maxRowsPerFile); await foreach (var batch in indexValueBatches) { var outputFilename = Guid.NewGuid().ToString(); var rootIndexRow = await MergeIndexValuesToFile(batch, outputFilename); rootIndexRows.Add(rootIndexRow); } return(rootIndexRows); }
public async Task <string> GetFileNameForColumnAndValue(string column, string val) { var cached = await _cache.GetFileNameForColumnAndValue(column, val); if (cached != null) { return(cached); } await _lockMechanism.TakeLockAsync(CommonKeys.ROOT_INDEX_FILE_LOCK_KEY, _lockTimeSpan); var downloadedFileName = "__NOT_EXISTING__"; try { downloadedFileName = await _amazonAdapter.DownloadObjectAsync(_bucketName, _rootIndexName); } catch (ResourceNotFoundException) { await CreateRootIndexFile(); } await _lockMechanism.ReleaseLockAsync(CommonKeys.ROOT_INDEX_FILE_LOCK_KEY); using var stream = OptionalFileStream.CreateOptionalFileReadStream(downloadedFileName); var indexRows = GetIndexRowsFromFile(stream); var relevantIndexRow = await indexRows.Where(row => ValidateRowWithRequest(column, val, row)).FirstOrDefaultAsync(); if (relevantIndexRow == default(RootIndexRow)) { return(CommonKeys.END_OF_INDEX_FLAG); } var update = new RootIndexColumnUpdate { ColumnName = column, Rows = new SortedSet <RootIndexRow> { relevantIndexRow } }; await _cache.UpdateColumnRanges(new SortedSet <RootIndexColumnUpdate> { update }); return(relevantIndexRow.FileName); }
public async Task UpdateColumnRanges(SortedSet <RootIndexColumnUpdate> columnMappings) { await _lockMechanism.TakeLockAsync(CommonKeys.ROOT_INDEX_FILE_LOCK_KEY, _lockTimeSpan); var downloadedFileName = await _amazonAdapter.DownloadObjectAsync(_bucketName, _rootIndexName); using var stream = OptionalFileStream.CreateOptionalFileReadStream(downloadedFileName); var indexRows = GetIndexRowsFromFile(stream); var outputRows = MergeIndexWithUpdate(indexRows, columnMappings); var outputFileName = downloadedFileName + "_new"; await WriteIndexRowsToFile(outputFileName, outputRows); await _amazonAdapter.UploadObjectAsync(_bucketName, _rootIndexName, outputFileName); await _lockMechanism.ReleaseLockAsync(CommonKeys.ROOT_INDEX_FILE_LOCK_KEY); await _cache.UpdateColumnRanges(columnMappings); }