public static async Task Execute(string targetConnectionString, string sourceConnectionString)
{
var now = DateTime.Now;
using var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
await using var snowflakeConnection = await CreateSnowflakeConnection(targetConnectionString);
using var databaseConnection = OracleDatabaseSource.Create(sourceConnectionString);
using var tablesQueue = new PipelineQueue <DataSourceSlice>("tables", token);
using var processQueue = new PipelineQueue <DataSourceFile>("process", token);
using var uploadQueue = new PipelineQueue <UploadItem>("upload", token);
using var resolveQueue = new PipelineQueue <DataSource>("resolve", token);
var sources = Primer.GetSources().ToDictionary(k => k.LifetimeKey);
await DoIt(
new StorageManager(snowflakeConnection),
databaseConnection,
sources,
tablesQueue,
processQueue,
uploadQueue,
resolveQueue,
new LifetimeReference <string>(
// ReSharper disable once AccessToDisposedClosure
v => resolveQueue.Enqueue(sources[v])
),
token
);
Console.WriteLine($@"start:{now} end:{DateTime.Now} diff:{DateTime.Now - now}");
}
protected override void Awake()
{
base.Awake();
Ins = this;
_LogQueue = new PipelineQueue <string>(MaxBufferSize);
var l =
(from Transform rectTran
in RectTrans
select rectTran.GetComponent <TextMeshProUGUI>()).ToList();
l.Reverse();
_TextQueue = l.ToArray();
RePaint(0);
}
public void Test1()
{
string inQ = "in", outQ = "out";
var steps = new List <PipelineStep>()
{
new PipelineAction(item => { item.Content = (int)item.Content + 5; item.Status = ItemStatus.OnGoing; return(item); }, inQ, "has5", 2),
new PipelineRouter(item => ((int)item.Content % 2 == 0?"pair":"impair"), "has5", new[] { "pair", "impair" }, 2),
new PipelineAction(item => { Assert.AreEqual(0, (int)item.Content % 2); item.Status = ItemStatus.Complete; return(item); }, "pair", outQ, 2),
new PipelineAction(item => { Assert.AreEqual(1, (int)item.Content % 2); item.Status = ItemStatus.Complete; return(item); }, "impair", outQ, 2)
};
var outP = new PipelineQueue(outQ);
var queues = new List <PipelineQueue>()
{
new PipelineQueue(inQ),
outP,
new PipelineQueue("has5"),
new PipelineQueue("pair"),
new PipelineQueue("impair")
};
Task.Run(() => {
foreach (var o in outP.GetConsumingEnumerable())
{
Console.WriteLine(string.Concat(o.Treatment.Message));
}
});
var p = PipelineFactory.CreatePipeline(
inQ,
outQ,
steps,
queues,
item => {
Console.WriteLine(string.Concat(item.Treatment.Message));
return(true);
}
);
p.Execute(new item()
{
Content = 5
});
}
public static Task[] CreatePump <TIn, TOut>(
this PipelineQueue <TIn> queue, int threadCount,
Func <IPipelineTask <TIn, TOut> > pipeFactory,
Action <TOut> action)
{
var tasks = new Task[threadCount];
for (var i = 0; i < threadCount; i++)
{
tasks[i] = Task.Run(async() =>
{
var p = pipeFactory();
await p.Start();
await foreach (var item in queue.GetConsumingEnumerable())
{
var result = await p.Execute(item);
action?.Invoke(result);
}
await p.End();
});
Console.WriteLine($"Completed Pump {queue}::{i}");
}
return(tasks);
}
private void Awake()
{
Instance = this;
_BulletHoles = new PipelineQueue <BulletHole>(HolesLimit);
_ClearCounter = new AutoCounter(this, ClearTime).OnComplete(Clear);
}
private static async Task DoIt(
StorageManager credentialManager,
IDatabaseSource databaseConnection,
IDictionary <string, DataSource> sources,
PipelineQueue <DataSourceSlice> tablesQueue,
PipelineQueue <DataSourceFile> processQueue,
PipelineQueue <UploadItem> uploadQueue,
PipelineQueue <DataSource> resolveQueue,
LifetimeReference <string> lifetimeController,
CancellationToken token
)
{
var tableTasks = tablesQueue.CreatePump(
pipeFactory: () => new ExportPipelineTask(
connection: databaseConnection,
emitFile: state =>
{
lifetimeController.AddRef(state.DataSourceSlice.LifetimeKey);
processQueue.Enqueue(
new DataSourceFile(
state.Filename,
state.DataSourceSlice.LifetimeKey,
state.RowCount)
);
},
feedback: state =>
{
if (state.State != ExportPipelineTask.ExportFeedbackState.Complete)
{
return;
}
Console.Out.WriteLine(
$@"{state.Filename:-40}:State:{state.State}:Rows:{state.RowCount:D16}:TotalRows:{state.TotalRowCount:D16}"
);
}
),
threadCount: 4,
action: slice => lifetimeController.Release(slice.LifetimeKey)
);
var processTasks = processQueue.CreatePump(
pipeFactory: () => new ProcessPipelineTask(credentialManager),
threadCount: 1,
action: batch => uploadQueue.Enqueue(batch)
);
var uploadTasks = uploadQueue.CreatePump(
pipeFactory: () => new UploadPipelineTask(),
threadCount: 4,
action: batch => lifetimeController.Release(batch.LifetimeKey)
);
var resolveTasks = resolveQueue.CreatePump(
pipeFactory: () => new ResolvePipelineTask(),
threadCount: 1,
action: batch => Console.WriteLine($@"**************** Done File {batch}")
);
await Task.Run(
() =>
{
foreach (var item in Primer.Prime(sources.Values))
{
lifetimeController.AddRef(item.LifetimeKey);
tablesQueue.Enqueue(item);
}
tablesQueue.Complete();
},
token
);
await Task.WhenAll(tableTasks);
processQueue.Complete();
await Task.WhenAll(processTasks);
uploadQueue.Complete();
await Task.WhenAll(uploadTasks);
resolveQueue.Complete();
await Task.WhenAll(resolveTasks);
}
internal void AddQueue(PipelineQueue queue)
{
PipelineQueues.Add(queue.Name, queue);
}
