public string ImportFinanceialReport(Guid dataModelingId, Tenant tenant, CrawlerAction action, string guidSerial = "")
{
if (dataModelingId == null || dataModelingId == Guid.Empty)
{
throw new ArgumentException("dataModelingId");
}
if (tenant == null)
{
throw new ArgumentNullException("tenant");
}
var taskId = !string.IsNullOrEmpty(guidSerial) ? new Guid(guidSerial) : Guid.NewGuid();
using DataContext context = new DataContext(tenant.ConnectionStrings.Master);
var dataModeling = context.DataModeling.Find(dataModelingId);
var dataSource = context.DataSource.Find(dataModeling.DsId);
string message;
if (dataSource != null)
{
RepairSchema(taskId, dataSource, tenant);
ClearIndustryData(taskId, dataSource, tenant);
message = ImportFinanceialReportToDataTable(taskId, context, dataModeling, dataSource, tenant);
if (action == CrawlerAction.UpdateForTenant)
{
dataSource.Hashcode = HashUtil.CreateHashcode();
dataSource.UpdateDate = DateTime.Now;
dataSource.EndDate = dataSource.UpdateDate.ToString("yyyy-MM-dd HH:mm:ss");
}
context.SaveChanges();
}
else
{
message = $"DataModeling.DsId ({dataModeling.DsId}) 未找到工作表記錄";
_logger.LogWarning($"taskId={taskId}, {message}");
}
return(message);
}
public void UpdateTask(Tenant tenant, Guid dataSourceId, UpdateMode updateMode)
{
var taskId = Guid.NewGuid();
_logger.LogInformation($"taskId={taskId}，开始执行调度任务");
try
{
DateTime beginDateTime = DateTime.Now;//记录更新开始时间
using var ctx = new DataContext(tenant.ConnectionStrings.Master);
var dataSource = ctx.DataSource.Find(dataSourceId);
if (dataSource == null)
{
throw new KeyNotFoundException(dataSourceId.ToString());
}
Tuple <bool, bool> result = _dispatchService.Update(taskId, tenant, ctx, dataSource, HashUtil.CreateHashcode(dataSourceId, ctx), updateMode, beginDateTime);
ctx.SaveChanges();
_logger.LogInformation($"taskId={taskId}，调度任务已完成");
if (result.Item1)//更新成功，触发上游表更新
{
var relations = ctx.TableRelation.Where(_ => _.Id == dataSource.DataSourceId).ToArray();
if (relations.Length > 0)
{
foreach (var item in relations)
{
var relationds = ctx.DataSource.Find(item.ParentId);
if (relationds != null)
{
_logger.LogInformation($"taskId={taskId}，更新下游表，tablerelation.DataSource(ParentId).Name={relationds.Name}");
BackgroundJob.Enqueue <IEngineService>(services => services.UpdateTask(tenant, relationds.DataSourceId, updateMode));
}
}
}
try
{
//触发转置数据源更新
string url = $"{tenant.ApplicationUrl}/api/data_source/transpose/{dataSourceId}/refresh";
using HttpClient httpClient = new HttpClient
{
BaseAddress = new Uri(url)
};
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, string.Empty);
HttpResponseMessage response = httpClient.SendAsync(request).Result;
_logger.LogInformation($"触发转置数据源更新, url={url}, status={response.StatusCode}, result={response.Content.ReadAsStringAsync().Result}");
}
catch (Exception ex)
{
_logger.LogError(ex, $"触发转置数据源更新发生错误：{ex.Message}");
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, $"taskId={taskId}，调度任务执行失败: params tenant={JsonConvert.SerializeObject(tenant)}, params dataSourceId={dataSourceId}");
//throw ex;//自动更新整个过程都是DB级别的操作，一旦出现错误都是致命性的，重试已经没有意义了
}
}
public void AutoUpdate()
{
lock (_updateObject)
{
if (_autoUpdateIsBusy)
{
return;
}
_autoUpdateIsBusy = true;
}
string updateCode = $"{Guid.NewGuid():N}";
try
{
_logger.LogInformation($"updateCode={updateCode}，自动更新开始");
foreach (var group in _tenants.GroupBy(_ => _.ConnectionStrings.Master))
{
Tenant tenant = group.First();
_logger.LogInformation($"updateCode={updateCode}，Tenant.Name={tenant.Name}");
try
{
using DataContext context = new DataContext(group.Key);
//1、找到关系网最下层节点（基表，Sync、Excel）
var basedslist = context.DataSource.Where(_ =>
_.ProjectId != null &&//排除文件夹
(_.Reference == "Sync" || _.Reference == "Excel" || _.Reference == "TRANSPOSE") //底层表的类型限定为：同步客户端上传、Excel导入、二维转一维
//&& _.TableName != "indicatorwarehouse"//排除指标数据源
&& !string.IsNullOrEmpty(_.Hashcode) //Hashcode是新增字段，增加此条件是为了兼容旧数据，如果工作表更新过，Hashcode会重现生成
).ToArray();
IList <Guid> composedsidlist = new List <Guid>();
foreach (var baseds in basedslist)
{
foreach (var downstreamrelation in context.TableRelation.Where(_ => _.Id == baseds.DataSourceId).ToArray())
{
//2、找到合表第一层关系节点
if (!composedsidlist.Contains(downstreamrelation.ParentId))
{
composedsidlist.Add(downstreamrelation.ParentId);
}
}
}
foreach (var id in composedsidlist)
{
var dataSource = context.DataSource.Find(id);
if (dataSource != null)
{
if (dataSource.Reference == "jointable" || dataSource.Reference == "uniontable" || dataSource.Reference == "grouptable")
{
if (string.IsNullOrWhiteSpace(dataSource.Hashcode) || HashUtil.CreateHashcode(id, context) != dataSource.Hashcode)
{
BackgroundJob.Enqueue <IEngineService>(services => services.UpdateTask(tenant, id, UpdateMode.AutoUpdate));
}
}
}
else
{
_logger.LogError($"updateCode={updateCode}, Tenent.Name={tenant.Name}, 工作表（DataSourceId={id}）未找到");
}
}
}
catch (Exception ex)
{
//单个租户发生异常，不影响其他租户自动更新
_logger.LogError(ex, $"updateCode={updateCode}，自动更新发生错误，租户信息：{JsonConvert.SerializeObject(tenant)}");
}
}
_autoUpdateIsBusy = false;
}
catch (Exception ex)
{
_autoUpdateIsBusy = false;
_logger.LogError(ex, $"updateCode={updateCode}，检索租户配置信息发生错误");
throw ex;
}
_logger.LogInformation($"updateCode={updateCode}，自动更新结束");
}
/// <summary>
/// 更新实体表
/// </summary>
/// <param name="taskId"></param>
/// <param name="tenant"></param>
/// <param name="dataContext"></param>
/// <param name="dataSource"></param>
/// <param name="hashcode"></param>
/// <param name="updateMode"></param>
/// <param name="beginDateTime"></param>
/// <returns></returns>
public Tuple <bool, bool> Update(Guid taskId, Tenant tenant, DataContext dataContext, DataSource dataSource, string hashcode, UpdateMode updateMode, DateTime beginDateTime)
{
_logger.LogInformation($"taskId={taskId}, update DataSource, DataSourceId={dataSource.DataSourceId}, Name={dataSource.Name}, UpdateStatue={dataSource.UpdateStatus}，params beginDateTime={beginDateTime: yyyy-MM-dd HH:mm:ss.fffffff}");
if (string.IsNullOrWhiteSpace(dataSource.UpdateSql))
{
dataSource.UpdateStatus = UpdateStatusType.Normal;
_logger.LogInformation($"taskId={taskId}, datasource.UpdateSql is null, skip the update procressing, DataSourceId={dataSource.DataSourceId}, Name={dataSource.Name}");
return(new Tuple <bool, bool>(false, false));
}
////获取上游表
//Func<IEnumerable<DataSource>> GetUpstreamDs = () => {
// IEnumerable<Guid> upstreamDids = dataContext.TableRelation.Where(_ => _.ParentId == dataSource.DataSourceId).Select(_ => _.Id);
// return dataContext.DataSource.Where(_ => upstreamDids.Contains(_.DataSourceId)).ToArray();
//};
UtilsTools.Timewatch.Restart();
DateTime startDate = DateTime.Now;
Tuple <int, int, bool> updateContext = null;
try
{
string dataconnectionstring = dataSource.Connection;
#if DEBUG
dataconnectionstring = tenant.ConnectionStrings.Data;
#endif
//更新物理表数据（全量）
using MySqlConnection connection = new MySqlConnection(dataconnectionstring);
try
{
connection.Open();
updateContext = CreatePhysicalTable(taskId, tenant, dataSource, connection);
}
catch (Exception)
{
throw;
}
finally
{
if (connection != null && connection.State == ConnectionState.Open)
{
connection.Close();
}
}
//在合表更新逻辑开始之前，记录那个时间点的hash1值，在更新完毕后回写（
//更新逻辑结束后从数据库再取hash2值，判断hash1是否等于hash2，
// 如果一致，就回写hash值；
// 如果不一致，说明上游表在更新这段时间内又发生了变更，这个时候直接退出，不修改当前hash值，下一轮重新生成合表；）
if (hashcode != HashUtil.CreateHashcode(dataSource.DataSourceId, dataContext))
{
return(new Tuple <bool, bool>(false, updateContext.Item3));
}
var dateTime = DateTime.Now;
dataSource.UpdateDate = dateTime;
dataSource.EndDate = dateTime.ToString("yyyy-MM-dd HH:mm:ss");
dataSource.UpdateStatus = UpdateStatusType.Finish;
dataSource.Hashcode = hashcode;
dataContext.Entry(dataSource).State = EntityState.Modified;
}
catch (Exception ex)
{
_logger.LogError(ex, $"taskId={taskId}，更新实体表发生错误，DataSourceId={dataSource.DataSourceId}，Name={dataSource.Name}");
dataSource.UpdateStatus = UpdateStatusType.Fail;
SaveLog(dataContext, dataSource, $"调度任务执行失败：{ex.Message}", UpdateLogCategory.Exception);
throw ex;
}
_logger.LogInformation($"taskId={taskId}, UpdateStatus={dataSource.UpdateStatus}");
int timewatchUsed = UtilsTools.TimewatchUsed;
_logger.LogInformation($"taskId={taskId}, TimewatchUsed={timewatchUsed}millisecond({Math.Round((double)timewatchUsed / 60000, 2)}min)");
UtilsTools.Timewatch.Stop();
//清理数据源缓存
try
{
RemoveTbbCache(taskId, tenant, dataSource);
}
catch (Exception ex)
{
_logger.LogError(ex, $"taskId={taskId}，清理数据源缓存发生错误");
}
//保存更新日志
try
{
//AddDataSourceUpdateLog(taskId, tenant, dataSource, new Tuple<int, int, DateTime>(updateContext.Item1, updateContext.Item2, startDate));
SaveLog(dataContext, dataSource, updateContext.Item3 ? "新建成功" : "更新成功", UpdateLogCategory.Information);
}
catch (Exception ex)
{
_logger.LogError(ex, $"taskId={taskId}，保存更新日志发生错误");
}
return(new Tuple <bool, bool>(true, updateContext.Item3));
}