/// <summary>
/// Get all parts containing this table
/// Could be multiple parts, since the table may be spread across multiples files
/// </summary>
public IEnumerable <SyncTable> GetTable(string tableName, string schemaName)
{
var tableInfo = new BatchPartTableInfo(tableName, schemaName);
if (InMemory)
{
if (this.InMemoryData.HasTables)
{
yield return(this.InMemoryData.Tables[tableName, schemaName]);
}
}
else
{
foreach (var batchPartinInfo in this.BatchPartsInfo.OrderBy(bpi => bpi.Index))
{
if (batchPartinInfo.Tables != null && batchPartinInfo.Tables.Any(t => t == tableInfo))
{
batchPartinInfo.LoadBatchAsync(schema, GetDirectoryFullPath()).ConfigureAwait(false).GetAwaiter().GetResult();
// Get the table from the batchPartInfo
// generate a tmp SyncTable for
var batchTable = batchPartinInfo.Data.Tables.FirstOrDefault(bt => bt == new SyncTable(tableName, schemaName));
if (batchTable != null)
{
yield return(batchTable);
// Once read, clear it
batchPartinInfo.Data.Clear();
batchPartinInfo.Data = null;
}
}
}
}
}
/// <summary>
/// Check if this batchinfo has some data (in memory or not)
/// </summary>
public bool HasData(string tableName, string schemaName)
{
if (this.SanitizedSchema == null)
{
throw new NullReferenceException("Batch info schema should not be null");
}
if (InMemory && InMemoryData != null && InMemoryData.HasTables)
{
var table = InMemoryData.Tables[tableName, schemaName];
if (table == null)
{
return(false);
}
return(table.HasRows);
}
if (!InMemory && BatchPartsInfo != null && BatchPartsInfo.Count > 0)
{
var tableInfo = new BatchPartTableInfo(tableName, schemaName);
var bptis = BatchPartsInfo.SelectMany(bpi => bpi.Tables.Where(t => t.EqualsByName(tableInfo)));
if (bptis == null)
{
return(false);
}
return(bptis.Sum(bpti => bpti.RowsCount) > 0);
}
return(false);
}
public async IAsyncEnumerable <SyncTable> GetTableAsync(string tableName, string schemaName, ISerializerFactory serializerFactory = default, BaseOrchestrator orchestrator = null)
{
if (this.SanitizedSchema == null)
{
throw new NullReferenceException("Batch info schema should not be null");
}
var tableInfo = new BatchPartTableInfo(tableName, schemaName);
if (InMemory)
{
this.SerializerFactoryKey = null;
if (this.InMemoryData != null && this.InMemoryData.HasTables)
{
yield return(this.InMemoryData.Tables[tableName, schemaName]);
}
}
else
{
this.SerializerFactoryKey = serializerFactory.Key;
var bpiTables = BatchPartsInfo.Where(bpi => bpi.RowsCount > 0 && bpi.Tables.Any(t => t.EqualsByName(tableInfo))).OrderBy(t => t.Index);
if (bpiTables != null)
{
foreach (var batchPartinInfo in bpiTables)
{
// load only if not already loaded in memory
if (batchPartinInfo.Data == null)
{
await batchPartinInfo.LoadBatchAsync(this.SanitizedSchema, GetDirectoryFullPath(), serializerFactory, orchestrator).ConfigureAwait(false);
}
// Get the table from the batchPartInfo
// generate a tmp SyncTable for
var batchTable = batchPartinInfo.Data.Tables.FirstOrDefault(bt => bt.EqualsByName(new SyncTable(tableName, schemaName)));
if (batchTable != null)
{
yield return(batchTable);
// We may need this same BatchPartInfo for another table,
// but we dispose it anyway, because memory can be quickly a bottleneck
// if batchpartinfos are resident in memory
batchPartinInfo.Data.Dispose();
batchPartinInfo.Data = null;
}
}
}
}
}
/// <summary>
/// Get all parts containing this table
/// Could be multiple parts, since the table may be spread across multiples files
/// </summary>
public IEnumerable <SyncTable> GetTable(string tableName, string schemaName, BaseOrchestrator orchestrator = null)
{
if (this.SanitizedSchema == null)
{
throw new NullReferenceException("Batch info schema should not be null");
}
var tableInfo = new BatchPartTableInfo(tableName, schemaName);
if (InMemory)
{
if (this.InMemoryData != null && this.InMemoryData.HasTables)
{
yield return(this.InMemoryData.Tables[tableName, schemaName]);
}
}
else
{
foreach (var batchPartinInfo in this.BatchPartsInfo.OrderBy(bpi => bpi.Index))
{
if (batchPartinInfo.Tables != null && batchPartinInfo.Tables.Any(t => t.EqualsByName(tableInfo)))
{
// TODO : Need to implement IAsyncEnumerable
// Need to use GetAwaiter().GetResult() until we have await in foreach yield
// IAsyncEnumerable is part of .Net Standard 2.1
// But .Net Standard 2.1 is not compatible with .Net Framework 4.8
// So far, we can't use it, until we decide to abandon .Net FX 4.8
// For sure, it will be replaced when .Net 5 will appears ...
batchPartinInfo.LoadBatchAsync(this.SanitizedSchema, GetDirectoryFullPath(), orchestrator).ConfigureAwait(false)
.GetAwaiter().GetResult();
// Get the table from the batchPartInfo
// generate a tmp SyncTable for
var batchTable = batchPartinInfo.Data.Tables.FirstOrDefault(bt => bt.EqualsByName(new SyncTable(tableName, schemaName)));
if (batchTable != null)
{
yield return(batchTable);
//// Once read, clear it
//batchPartinInfo.Data.Clear();
//batchPartinInfo.Data = null;
}
}
}
}
}
public async IAsyncEnumerable <SyncTable> GetTableAsync(string tableName, string schemaName, BaseOrchestrator orchestrator = null)
{
if (this.SanitizedSchema == null)
{
throw new NullReferenceException("Batch info schema should not be null");
}
var tableInfo = new BatchPartTableInfo(tableName, schemaName);
if (InMemory)
{
if (this.InMemoryData != null && this.InMemoryData.HasTables)
{
yield return(this.InMemoryData.Tables[tableName, schemaName]);
}
}
else
{
var bpiTables = BatchPartsInfo.Where(bpi => bpi.RowsCount > 0 && bpi.Tables.Any(t => t.EqualsByName(tableInfo))).OrderBy(t => t.Index);
if (bpiTables != null)
{
foreach (var batchPartinInfo in bpiTables)
{
// load only if not already loaded in memory
if (batchPartinInfo.Data == null)
{
await batchPartinInfo.LoadBatchAsync(this.SanitizedSchema, GetDirectoryFullPath(), orchestrator).ConfigureAwait(false);
}
// Get the table from the batchPartInfo
// generate a tmp SyncTable for
var batchTable = batchPartinInfo.Data.Tables.FirstOrDefault(bt => bt.EqualsByName(new SyncTable(tableName, schemaName)));
if (batchTable != null)
{
yield return(batchTable);
// once loaded and yield, can dispose
batchPartinInfo.Data.Dispose();
batchPartinInfo.Data = null;
}
}
}
}
}
public static BatchPartInfo NewBatchPartInfo(string batchPartFileName, string tableName, string schemaName, int rowsCount, int batchIndex)
{
var bpi = new BatchPartInfo {
FileName = batchPartFileName
};
// Create the info on the batch part
BatchPartTableInfo tableInfo = new BatchPartTableInfo
{
TableName = tableName,
SchemaName = schemaName,
RowsCount = rowsCount
};
bpi.Tables = new BatchPartTableInfo[] { tableInfo };
bpi.RowsCount = rowsCount;
bpi.IsLastBatch = false;
bpi.Index = batchIndex;
return(bpi);
}