public async Task SaveIntegrationTaskAsync(IntegrationTask integrationTask,
CancellationToken cancellationToken)
{
var integrationSagaEntity = _mapper.Map <IntegrationTaskEntity>(integrationTask);
await TasksCollection.InsertOneAsync(integrationSagaEntity, null, cancellationToken);
}
public void ExecuteInThread_ShouldReturn_OK()
{
var task = new IntegrationTask();
var r = Player.ServiceThreads.ExecuteInThread(task);
this.AreEqual(ActionResultType.OK, r.ResultType, fieldName: "ExecuteInThread.ResultType");
}
public Instance[] AnalyseInstance(JArray objs, JObject inherited, string path)
{
List <IntegrationInstance> instances = new List <IntegrationInstance>();
foreach (JObject obj in objs)
{
IntegrationTable[][] tables = AnalyseTable($"{path}\\{GetJValue(obj, inherited, "tables")}",
out string parms);
IntegrationTask[][] tasks = new IntegrationTask[tables.Length][];
List <IntegrationTask> taskList = new List <IntegrationTask>();
for (int i = 0; i < tasks.Length; i++)
{
if (tables[i].Length > 0)
{
tasks[i] = new IntegrationTask[tables[i].Length];
for (int j = 0; j < tasks[i].Length; j++)
{
IntegrationTask task = new IntegrationTask()
{
Name = tables[i][j].DestName,
Progress = 0,
Total = 0,
Status = DataStates.Idle,
StartTick = 0,
Sources = AnalyseDatabases(obj["sources"] as JArray, inherited),
Dest = AnalyseDatabase(obj["dest"] as JObject, inherited),
Params = parms,
Table = tables[i][j],
ReadPages = uint.Parse(GetJValue(obj, inherited, "readPages"))
};
taskList.Add(task);
tasks[i][j] = task;
}
}
}
if (taskList.Count > 0)
{
IntegrationInstance ins = new IntegrationInstance()
{
Name = $"{taskList[0].Dest.Server}:{taskList[0].Dest.Port}/{taskList[0].Dest.DB}",
Tasks = taskList.ToArray(),
ActualTasks = tasks,
Threads = uint.Parse(GetJValue(obj, inherited, "threads"))
};
instances.Add(ins);
}
}
return(instances.ToArray());
}
public void WaitFor_ShouldReturn_OK()
{
var task = new IntegrationTask(() =>
{
Thread.Sleep(TimeSpan.FromSeconds(5));
});
var r = Player.ServiceThreads.ExecuteInThread(task);
this.AreEqual(ActionResultType.OK, r.ResultType, fieldName: "ExecuteInThread.ResultType");
var waitResult = Player.ServiceThreads.WaitFor(r.Result);
this.AreEqual(ActionResultType.OK, waitResult.ResultType, fieldName: "WaitFor.Cancel");
}
public void Cancel_ShouldReturn_OK()
{
var task = new IntegrationTask(() =>
{
Thread.Sleep(TimeSpan.FromMinutes(2));
});
var r = Player.ServiceThreads.ExecuteInThread(task);
this.AreEqual(ActionResultType.OK, r.ResultType, fieldName: "ExecuteInThread.ResultType");
var cancelResult = Player.ServiceThreads.Cancel(r.Result, AimpServiceThreadPoolType.None);
this.AreEqual(ActionResultType.OK, cancelResult.ResultType, fieldName: "ExecuteInThread.Cancel");
}
private void IntegrateTask(IntegrationTask task, out string reason)
{
reason = "取消操作";
if (status.Stopped)
{
return;
}
if (Connect(task, null, out _, task.Dest, out IDBMSWriter dest))
{
Dictionary <string, object> parms = new Dictionary <string, object>();
dest.QueryParam(task.Params, parms);
dest.BeginTransaction();
try
{
// 汇集数据
IntegrateTaskWithScript(task, parms, dest, out reason);
if (task.Status == DataStates.Done)
{
dest.CommitTransaction();
}
else
{
dest.RollbackTransaction();
}
}
catch (Exception ex)
{
dest.RollbackTransaction();
task.Status = DataStates.Error;
reason = ex.Message;
}
dest.Close();
}
else
{
task.Status = DataStates.Error;
reason = task.ErrorMsg;
}
}
public override string ToString()
{
return
($"ArchiveID: {Id}\r\nOptions to download archive:\r\n{string.Join(Environment.NewLine, new[] {"From the web-based interface (Portal)", $"Run the following command: {IntegrationTask.NameOf<DumpArchiveTask>()} {Id}"}.Concat(_additionalDownloadOptions).Select(x => $" - { x } "))}");
}
private void IntegrateTaskWithScript(IntegrationTask task, Dictionary <string, object> parms, IDBMSWriter dest,
out string failReason)
{
ConcurrentQueue <object> scripts = new ConcurrentQueue <object>();
bool read = false;
string reason = "取消操作";
Parallel.ForEach(CreateThreadAction(), act =>
{
try
{
// “读”线程：个别数据源读取失败，容错继续执行
if ("read".Equals(act))
{
uint bufSize = task.ReadPages * 3;
IDataFilter filter = DataFilterFactory.GetFilterByName(task.Table.Filter);
IDataWrapper data = null;
foreach (Database db in task.Sources)
{
if (status.Stopped || task.Status == DataStates.Error)
{
break;
}
uint fromRow = 1, toRow = task.ReadPages * task.Table.PageSize;
Dictionary <string, object> tmpParams = new Dictionary <string, object>(parms);
// 连接数据源
if (!Connect(task, db, out IDBMSReader source, null, out _))
{
task.Status = DataStates.RunningError;
reason = task.ErrorMsg;
break;
}
while (true)
{
// 等待缓冲区可用
while (scripts.Count > bufSize && !status.Stopped && task.Status != DataStates.Error)
{
Thread.Sleep(50);
}
if (status.Stopped || task.Status == DataStates.Error)
{
break;
}
// 取数
if (source.QueryPage(task.Table, fromRow, toRow, WithEnums.NoLock, tmpParams, out data))
{
try
{
object script = null;
data.MapFields(task.Table.DestFields);
while (dest.BuildScript(task.Table, data, filter, out script) &&
!status.Stopped && task.Status != DataStates.Error)
{
scripts.Enqueue(script);
}
// 获取不到预期的记录数，作最后一页处理
if (data.ReadCount != task.ReadPages * task.Table.PageSize ||
status.Stopped)
{
break;
}
}
finally
{
data.Close();
}
}
else
{
task.Status = DataStates.RunningError;
reason = source.LastError;
break;
}
fromRow = toRow + 1;
toRow += task.ReadPages * task.Table.PageSize;
}
source.Close();
}
private IntegrationTask GetTask(List <List <IntegrationTask> > lst, List <IntegrationTask> runList)
{
while (true)
{
lock (runList)
{
int dependCount = 0;
// 先从依赖树取
for (int i = 1; i < lst.Count; i++)
{
dependCount += lst[i].Count;
for (int j = 0; j < lst[i].Count; j++)
{
// 无外键依赖
if (lst[i][j].Table.References.Length == 0)
{
IntegrationTask rst = lst[i][j];
lst[i].RemoveAt(j);
runList.Add(rst);
return(rst);
}
else
{
bool inTree = false;
foreach (string s in lst[i][j].Table.References)
{
for (int k = 1; k < i; k++)
{
for (int l = 0; l < lst[k].Count; l++)
{
if (s.Equals(lst[k][l].Table.DestName))
{
inTree = true;
break;
}
}
if (inTree)
{
break;
}
}
if (inTree)
{
break;
}
for (int k = 0; k < runList.Count; k++)
{
if (s.Equals(runList[k].Table.DestName))
{
inTree = true;
break;
}
}
}
if (!inTree)
{
IntegrationTask rst = lst[i][j];
lst[i].RemoveAt(j);
runList.Add(rst);
return(rst);
}
}
}
}
// 再从独立表取
if (lst[0].Count > 0)
{
IntegrationTask rst = lst[0][0];
lst[0].RemoveAt(0);
runList.Add(rst);
return(rst);
}
else if (dependCount == 0)
{
return(null);
}
}
Thread.Sleep(50);
}
}
public void Execute(Instance ins, IStopStatus status)
{
this.status = status;
if (ins is IntegrationInstance instance && instance.ActualTasks.Length > 0)
{
// 构建待汇集任务清单：lst[0] = 独立任务，lst[1+] = 依赖树
List <List <IntegrationTask> > lst = new List <List <IntegrationTask> >
{
new List <IntegrationTask>(instance.ActualTasks[0]),
new List <IntegrationTask>()
};
for (int i = 1; i < instance.ActualTasks.Length; i++)
{
for (int j = 0; j < instance.ActualTasks[i].Length; j++)
{
IntegrationTask ta = instance.ActualTasks[i][j];
for (int k = 0; k < ta.Table.References.Length; k++)
{
for (int l = 0; l < lst[0].Count; l++)
{
if (ta.Table.References[k].Equals(lst[0][l].Table.DestName))
{
lst[1].Add(lst[0][l]);
lst[0].RemoveAt(l);
break;
}
}
}
}
lst.Add(new List <IntegrationTask>(instance.ActualTasks[i]));
}
TaskComparer comparer = new TaskComparer();
foreach (List <IntegrationTask> tasks in lst)
{
tasks.Sort(comparer);
}
List <IntegrationTask> runList = new List <IntegrationTask>();
// 开始汇集
Parallel.ForEach(CreateThreadAction((int)instance.Threads), i =>
{
IntegrationTask task = GetTask(lst, runList);
while (task != null)
{
try
{
Logger.WriteLog($"{task.Dest.Server}/{task.Dest.DB}.{task.Table.DestName}", "汇集开始...");
task.StartTick = WinAPI.GetTickCount();
task.Status = DataStates.Running;
IntegrateTask(task, out string reason);
if (task.Status == DataStates.Done)
{
Logger.WriteLog($"{task.Dest.Server}/{task.Dest.DB}.{task.Table.DestName}", "汇集成功。");
Logger.WriteRpt(task.Dest.Server, task.Dest.DB, task.Table.DestName, "成功",
task.Table.Progress.ToString("#,##0"));
}
else
{
task.Progress -= task.Table.Progress;
task.ErrorMsg = reason;
Logger.WriteLog($"{task.Dest.Server}/{task.Dest.DB}.{task.Table.DestName}",
$"汇集失败！{reason}");
Logger.WriteRpt(task.Dest.Server, task.Dest.DB, task.Table.DestName, "失败", reason);
}
task.StartTick = WinAPI.GetTickCount() - task.StartTick;
lock (runList) { runList.Remove(task); }
task = GetTask(lst, runList);
}
catch (Exception ex)
{
task.Status = DataStates.Error;
task.ErrorMsg = ex.Message;
Logger.WriteLog($"{task.Dest.Server}/{task.Dest.DB}", $"汇集失败！{ex.Message}");
}
}
});
foreach (Common.Task task in instance.Tasks)
{
if (task.Status != DataStates.Done)
{
Logger.WriteLog(instance.Name, "汇集失败！");
return;
}
}
Logger.WriteLog(instance.Name, "汇集成功。");
}
}
public ITask Get <TTask>() where TTask : class, ITask
{
return(Get(IntegrationTask.NameOf <TTask>()));
}
