/// <summary> /// 操作 /// </summary> /// <param name="item"></param> /// <returns></returns> public IActionResult OnPostBusinessDetailsForm(Data_Business_Details item) { var info = new WriteReturn(); using (var db = new DataContext(AppEtl.Db)) { var table = IFast.Query <Data_Business>(a => a.Id == item.Id).ToItem <Data_Business>(db); var source = IFast.Query <Data_Source>(a => a.Id == item.DataSourceId).ToItem <Data_Source>(db); var key = string.Format(AppEtl.CacheKey.Column, source.Host, item.TableName); var colunm = RedisInfo.Get <List <CacheColumn> >(key, AppEtl.CacheDb).Find(a => a.Name == item.ColumnName); db.BeginTrans(); if (IFast.Query <Data_Business_Details>(a => a.FieldId == item.FieldId).ToCount(db) == 0) { item.FieldId = Guid.NewGuid().ToStr(); info = IFast.Add(item); if (info.IsSuccess) { info = DataSchema.AddColumn(db, table, item, colunm, source); if (info.IsSuccess) { DataSchema.UpdateColumnComment(db, table, item, colunm, source); } } } else { info = IFast.Update <Data_Business_Details>(item, a => a.FieldId == item.FieldId); if (info.IsSuccess) { info.IsSuccess = DataSchema.UpdateColumn(db, table, item, colunm, source); if (info.IsSuccess) { DataSchema.UpdateColumnComment(db, table, item, colunm, source); } } } if (info.IsSuccess) { db.SubmitTrans(); return(new JsonResult(new { success = true, msg = "操作成功" })); } else { db.RollbackTrans(); return(new JsonResult(new { success = false, msg = "操作失败" })); } } }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) { try { while (!stoppingToken.IsCancellationRequested) { lock (thisLock) { using (var db = new DataContext(AppEtl.Db)) { var tableLink = DataSchema.TableLink(db); BaseLog.SaveLog("开始抽取", "FastEtlService"); var list = IFast.Query <Data_Business>(a => a.Id != null).ToList <Data_Business>(db); foreach (var item in list) { var isExec = item.UpdateTime == DateTime.Now.Hour; if (item.UpdateTime == 99) { isExec = true; } if (DataSchema.IsExistsTable(db, item.TableName) && isExec) { Parallel.Invoke(() => { var leaf = IFast.Query <Data_Business_Details>(a => a.Id == item.Id).ToList <Data_Business_Details>(db) ?? new List <Data_Business_Details>(); if (leaf.Count > 0 && leaf.Exists(a => a.Key.ToStr() != "")) { var isAdd = true; var dt = DataSchema.GetTable(tableLink, item.TableName); var columnName = dt.Columns[2].ColumnName.ToLower(); if (leaf.Exists(a => a.FieldName.ToLower() == columnName)) { DataSchema.ExpireData(db, item); //第一列 var link = DataSchema.InitColLink(leaf, db, IFast); var tempLeaf = leaf.Find(a => a.FieldName.ToLower() == columnName); var pageInfo = DataSchema.GetTableCount(tempLeaf, item); for (var i = 1; i <= pageInfo.pageCount; i++) { var log = new Data_Log(); log.Id = Guid.NewGuid().ToStr(); log.TableName = string.Format("{0}_page_{1}", item.TableName, i); log.BeginDateTime = DateTime.Now; pageInfo.pageId = i; var tempLink = link.Find(a => a.GetValue("id").ToStr() == tempLeaf.DataSourceId); var pageData = DataSchema.GetFirstColumnData(tempLink, tempLeaf, item, pageInfo); //遍历填充table for (var row = 0; row < pageData.list.Count; row++) { var dtRow = dt.NewRow(); dtRow["EtlAddTime"] = DateTime.Now; dtRow["EtlKey"] = pageData.list[row].GetValue("key"); dtRow[columnName] = pageData.list[row].GetValue("data"); //字典对照 if (!string.IsNullOrEmpty(tempLeaf.Dic)) { dtRow[columnName] = IFast.Query <Data_Dic_Details>(a => a.Value.ToLower() == dtRow[columnName].ToStr().ToLower() && a.DicId == tempLeaf.Dic, a => new { a.ContrastValue }).ToDic(db).GetValue("ContrastValue"); } //数据策略 isAdd = DataSchema.DataPolicy(db, item, dtRow["EtlKey"], columnName, dtRow[columnName]); for (var col = 2; col < dt.Columns.Count; col++) { columnName = dt.Columns[col].ColumnName.ToLower(); if (leaf.Exists(a => a.FieldName.ToLower() == columnName)) { tempLeaf = leaf.Find(a => a.FieldName.ToLower() == columnName); tempLink = link.Find(a => a.GetValue("id").ToStr() == tempLeaf.DataSourceId); dtRow[columnName] = DataSchema.GetColumnData(tempLink, tempLeaf, dtRow["EtlKey"]); //字典对照 if (!string.IsNullOrEmpty(tempLeaf.Dic)) { dtRow[columnName] = IFast.Query <Data_Dic_Details>(a => a.Value.ToLower() == dtRow[columnName].ToStr().ToLower() && a.DicId == tempLeaf.Dic, a => new { a.ContrastValue }).ToDic(db).GetValue("ContrastValue"); } //数据策略 if (item.Policy == "2") { isAdd = DataSchema.DataPolicy(db, item, dtRow["EtlKey"], columnName, dtRow[columnName]); } } } if (isAdd) { dt.Rows.Add(dtRow); } } if (dt.Rows.Count > 0) { DataSchema.AddList(db, dt, ref log); } db.Add(log); dt.Clear(); } DataSchema.CloseLink(link); item.LastUpdateTime = DateTime.Now; IFast.Update <Data_Business>(item, a => a.Id == item.Id, a => new { a.LastUpdateTime }, db); } } }); } } BaseLog.SaveLog("结束抽取", "FastEtlService"); DataSchema.CloseTableLink(tableLink); } } await Task.Delay(1000 * 60 * 30, stoppingToken); } } catch (Exception ex) { BaseLog.SaveLog(ex.ToString(), "FastEtlServiceError"); } }