private bool TryFetchEntities(string table, TableConfig config, StagingMetadata metadata)
{
int chunkSize = config.ChunkSize ?? CommonConfig.DefaultChunkSize;
// TODO: Currently writing in GetEntitiesByTimestamp, need to rethink status writes
//string lastUpdateText = metadata.LastTimestamp != null ?
// metadata.LastTimestamp.Value.ToString("yyyy-MM-dd HH:mm:ss") :
// "forever";
//Console.WriteLine($"Retrieving {table} updated since {lastUpdateText}");
var entities = _context.RemoteStore.GetEntitiesByTimestamp(
table,
config,
metadata.LastTimestamp,
metadata.LastIds);
if (_context.Config.FetchLimit != null)
{
entities = entities.Take(_context.Config.FetchLimit.Value);
}
using var enumerator = new EntityEnumerator(entities);
bool first = true;
while (!enumerator.Finished)
{
var chunk = _context.LocalStore.WriteStagedEntities(table, enumerator.Take(chunkSize));
if (chunk != null)
{
metadata.LastTimestamp = enumerator.LastTimestamp;
metadata.LastIds.ReplaceWith(enumerator.LastIds);
if (config.DetailFields.Count > 0 && config.Detail != DetailBehavior.Skip)
{
metadata.FetchDetail.Remove(chunk); // in case we have same chunk as previously downloaded
metadata.FetchDetail.Add(chunk);
}
else
{
metadata.Merge.Remove(chunk); // in case we have same chunk as previously downloaded
metadata.Merge.Add(chunk);
}
_context.LocalStore.SaveStagingMetadata(table, metadata);
Console.WriteLine($"Wrote {chunk} to staging");
}
else if (first)
{
Console.WriteLine($"{table} already up to date.");
}
first = false;
}
return(true);
}
internal EntityEnumerator(
FlowContext context,
string table,
Metadata tableMetadata,
StagingMetadata stagingMetadata)
{
_context = context;
_table = table;
_tableMetadata = tableMetadata;
_stagingMetadata = stagingMetadata;
_chunksToRemove = new List <string>();
_entities = ReadEntitiesToMerge().GetEnumerator();
}
public bool TryExecute()
{
IEnumerable <string> tables;
if (_context.Tables.Count > 0)
{
tables = _context.Tables;
}
else
{
tables = _context.LocalStore.GetTables();
}
foreach (string table in tables)
{
if (!_context.LocalStore.TryLoadMetadata(table, out var tableMetadata))
{
return(false);
}
// NOTE: overrides are applied transiently (not persisted to store)
tableMetadata.Config.OverrideWith(_context.Config);
// TODO: inherit fields/detailFields from base config if not specified in metadata config
if (!_context.LocalStore.TryLoadStagingMetadata(table, out var stagingMetadata))
{
stagingMetadata = new StagingMetadata
{
LastTimestamp = tableMetadata.LastTimestamp,
LastIds = { tableMetadata.LastIds },
};
}
if (_context.Config.FetchIds == null && !TryFetchEntities(table, tableMetadata.Config, stagingMetadata))
{
return(false);
}
if (!TryFetchDetail(table, tableMetadata, stagingMetadata))
{
return(false);
}
}
return(true);
}
private bool TryMergeEntities(string table, Metadata tableMetadata, StagingMetadata stagingMetadata)
{
// TODO: make this configurable, consider making it memory-based rather than count-based (after switching to System.Text.Json)
const int batchSize = 10000;
using var enumerator = new EntityEnumerator(_context, table, tableMetadata, stagingMetadata);
while (!enumerator.Finished)
{
if (!_context.LocalStore.TryUpsertEntities(table, tableMetadata, enumerator.Take(batchSize)))
{
return(false);
}
enumerator.Flush();
}
return(true);
}
private bool TryFetchDetail(string table, Metadata tableMetadata, StagingMetadata metadata)
{
var config = tableMetadata.Config;
int chunkSize = config.ChunkSize ?? CommonConfig.DefaultDetailChunkSize;
while (metadata.FetchDetail.Count > 0)
{
var chunk = metadata.FetchDetail[0];
var entities = GetDetailForEntities(
table,
config,
_context.LocalStore.ReadStagedEntities(chunk));
// TODO: chunk this into smaller files or create smaller fetchDetail chunks in the first place (change the chunkSize in TryFetchEntities)
var detailChunk = _context.LocalStore.WriteStagedEntities(table, entities, detailForChunk: chunk);
if (detailChunk != null)
{
metadata.Merge.Remove(detailChunk);
metadata.Merge.Add(detailChunk);
Console.WriteLine($"Wrote {detailChunk} to staging");
}
metadata.FetchDetail.Remove(chunk);
_context.LocalStore.SaveStagingMetadata(table, metadata);
_context.LocalStore.RemoveStagedEntities(chunk);
}
if (_context.Config.FetchIds != null || config.Detail == DetailBehavior.Backfill)
{
var entities = _context.Config.FetchIds != null?
_context.Config.FetchIds.Select(id => _context.RemoteStore.GetEntityDetail(table, config, id)) :
GetDetailForEntities(
table,
config,
GetEntitiesToBackfillDetail(table, tableMetadata, metadata.DetailLastId));
using var enumerator = new EntityEnumerator(entities);
while (!enumerator.Finished)
{
var chunk = _context.LocalStore.WriteStagedEntities(
table,
enumerator.Take(chunkSize),
detailChunkById: true);
if (chunk == null)
{
continue;
}
if (config.Detail == DetailBehavior.Backfill)
{
metadata.DetailLastId = enumerator.LastId;
}
metadata.Merge.Remove(chunk);
metadata.Merge.Add(chunk);
_context.LocalStore.SaveStagingMetadata(table, metadata);
Console.WriteLine($"Wrote {chunk} to staging");
}
}
return(true);
}