/// <summary>Copies the given files into staging tables MOTE: filesToCopy is relative to the store path. The resulting
/// LoadHistoryRow.File paths are converted to be equivalent</summary>
public static async Task <LoadHistoryRow[]> CopyInto(this StageTableCfg t, ISimpleFileStore store, string sfStage, ILoggedConnection <IDbConnection> db,
[CanBeNull] SPath[] filesToCopy, ILogger log)
{
if (filesToCopy?.Length > 1000)
{
throw new("copying 1k+ files not implemented");
}
var stagePath = new string[] { sfStage, store.BasePathSansContainer().Dot(c => c.IsEmpty ? null : c) }.Concat(t.Dir.Tokens).NotNull().Join("/");
var cols = await t.TableCols(db); // support subsets of columns (e.g. no loaded or updated columns
var selectCols = cols.Join(",", c => c.column_name.ToLowerInvariant() switch {
"v" => "$1 v",
"loaded" => "sysdate() loaded",
"updated" => "v:Updated::timestamp_ntz updated",
_ => throw new($"stage column {c.column_name} not supported")
});
public static async Task <SfCol[]> TableCols(this StageTableCfg t, ILoggedConnection <IDbConnection> db)
{
var cols = await db.QueryAsync <SfCol>("show columns", $"show columns in table {t.Table}").ToArrayAsync();
return(cols);
}
public static async Task <DateTime?> LatestTimestamp(this StageTableCfg t, ILoggedConnection <IDbConnection> db) =>
await db.ExecuteScalar <DateTime?>("latest timestamp", $"select max(v:{t.TsCol ?? "Updated"}::timestamp_ntz) from {t.Table}");
