async Task <ByteSize> LoadBLobData(SyncTableCfg tableCfg, ILogger log, string loadId, string sourceSql, object maxTs, TableId loadTable)
{
var path = StringPath.Relative("sync", tableCfg.Name, loadId);
var copyTask = Source.CopyTo(path, sourceSql, tableCfg, maxTs);
var loadedFiles = new KeyedCollection <StringPath, FileListItem>(f => f.Path);
while (true) // load as the files are created
{
if (copyTask.IsFaulted)
{
break;
}
var toLoad = (await Store.List(path).SelectManyList())
.Where(f => !loadedFiles.ContainsKey(f.Path)).ToArray();
if (toLoad.None())
{
if (copyTask.IsCompleted)
{
break;
}
await 5.Seconds().Delay();
continue;
}
log.Debug("Sync {Table} - loading: {Files}", tableCfg.Name, toLoad.Join("|", l => l.Path.ToString()));
await Dest.LoadFrom(toLoad.Select(f => f.Path), loadTable);
loadedFiles.AddRange(toLoad);
await toLoad.BlockAction(f => Store.Delete(f.Path, log), parallel : 8);
}
log.Information("Sync {Table} - copied {Files} files ({Size})", tableCfg.Name, loadedFiles.Count, loadedFiles.Sum(f => f.Bytes).Bytes().Humanize("#,#"));
return(loadedFiles.Sum(f => f.Bytes).Bytes());
}
public void AddRange(string[] keys, WizardPanel[] items)
{
m_Panels.AddRange(keys, items);
if (AddPanelRange != null)
{
AddPanelRange(items);
}
}
public static KeyedCollection <T> ToKeyedCollection <T>(this IEnumerable <T> list) where T : IKeyed
{
KeyedCollection <T> items = new KeyedCollection <T>();
items.AddRange(list);
return(items);
}