/// <summary> /// Áбí /// </summary> /// <returns></returns> public IActionResult OnPostDicList(int PageSize, int PageId, string DicId) { using (var db = new DataContext(AppEtl.Db)) { var page = new FastUntility.Core.Page.PageModel(); page.PageId = PageId == 0 ? 1 : PageId; page.PageSize = PageSize == 0 ? 10 : PageSize; var list = new FastUntility.Core.Page.PageResult(); if (string.IsNullOrEmpty(DicId)) { list = IFast.Query <Data_Dic_Details>(a => a.DicId != null).OrderBy <Data_Dic_Details>(a => new { a.DicId }).ToPage(page, db); } else { list = IFast.Query <Data_Dic_Details>(a => a.DicId == DicId).OrderBy <Data_Dic_Details>(a => new { a.DicId }).ToPage(page, db); } return(new PartialViewResult { ViewName = "Partial/DicList", ViewData = new ViewDataDictionary <FastUntility.Core.Page.PageResult>(ViewData, list) }); } }
public void OnPost(string id) { using (var db = new DataContext(AppEtl.Db)) { if (!string.IsNullOrEmpty(id)) { info = IFast.Query <Data_Dic>(a => a.Id == id).ToItem <Data_Dic>(db); } } }
/// <summary> /// Áбí /// </summary> /// <returns></returns> public IActionResult OnPostDataList(int PageSize, int PageId) { using (var db = new DataContext(AppEtl.Db)) { var page = new FastUntility.Core.Page.PageModel(); page.PageId = PageId == 0 ? 1 : PageId; page.PageSize = PageSize == 0 ? 10 : PageSize; var list = IFast.Query <Data_Source>(a => a.Id != "").ToPage(page, db); return(new PartialViewResult { ViewName = "Partial/DataList", ViewData = new ViewDataDictionary <FastUntility.Core.Page.PageResult>(ViewData, list) }); } }
/// <summary> /// ɾ³ý /// </summary> /// <returns></returns> public IActionResult OnPostDel(string id) { if (string.IsNullOrEmpty(id)) { return(new JsonResult(new { success = false, msg = "ɾ³ýʧ°Ü" })); } using (var db = new DataContext(AppEtl.Db)) { if (IFast.Query <Data_Business_Details>(a => a.Id == id).ToCount(db) == 0) { if (IFast.Delete <Data_Business>(a => a.Id == id, db).IsSuccess) { return(new JsonResult(new { success = true, msg = "ɾ³ý³É¹¦" })); } else { return(new JsonResult(new { success = false, msg = "ɾ³ýʧ°Ü" })); } } else { return(new JsonResult(new { success = false, msg = "ÒµÎñ¼¯´æÔÚʹÓÃÖÐ" })); } } }
/// <summary> /// 初始化取数据长连接 /// </summary> /// <returns></returns> public static List <Dictionary <string, object> > InitColLink(List <Data_Business_Details> list, DataContext db, IFastRepository IFast) { var result = new List <Dictionary <string, object> >(); foreach (var item in list) { if (result.Exists(a => a.GetValue("id").ToStr() == item.DataSourceId)) { continue; } var link = IFast.Query <Data_Source>(a => a.Id == item.DataSourceId).ToItem <Data_Source>(db); if (link.Type.ToLower() == AppEtl.DataDbType.SqlServer.ToLower()) { var dic = new Dictionary <string, object>(); var conn = DbProviderFactories.GetFactory(AppEtl.DataDbType.SqlServer).CreateConnection(); conn.ConnectionString = GetConnStr(link); conn.Open(); dic.Add("id", item.DataSourceId); dic.Add("conn", conn); dic.Add("type", AppEtl.DataDbType.SqlServer.ToLower()); result.Add(dic); } if (link.Type.ToLower() == AppEtl.DataDbType.MySql.ToLower()) { var dic = new Dictionary <string, object>(); var conn = DbProviderFactories.GetFactory(AppEtl.DataDbType.MySql).CreateConnection(); conn.ConnectionString = GetConnStr(link); conn.Open(); dic.Add("id", item.DataSourceId); dic.Add("conn", conn); dic.Add("type", AppEtl.DataDbType.MySql.ToLower()); result.Add(dic); } if (link.Type.ToLower() == AppEtl.DataDbType.Oracle.ToLower()) { var dic = new Dictionary <string, object>(); var conn = DbProviderFactories.GetFactory(AppEtl.DataDbType.Oracle).CreateConnection(); conn.ConnectionString = GetConnStr(link); conn.Open(); dic.Add("id", item.DataSourceId); dic.Add("conn", conn); dic.Add("type", AppEtl.DataDbType.Oracle.ToLower()); result.Add(dic); } } return(result); }
public async Task ContentAsync(HttpContext context, IFastRepository IFast) { var urlParam = HttpUtility.UrlDecode(GetUrlParam(context)); var isSuccess = true; var dic = new Dictionary <string, object>(); var stopwatch = new Stopwatch(); stopwatch.Start(); context.Response.ContentType = "application/Json"; var name = context.Request.Path.Value.ToStr().Substring(1, context.Request.Path.Value.ToStr().Length - 1).ToLower(); if (IFast.IsExists(name)) { var data = new List <Dictionary <string, object> >(); var param = new List <DbParameter>(); foreach (var item in IFast.MapParam(name)) { var checkKey = IFast.MapCheckMap(name, item); var existsKey = IFast.MapExistsMap(name, item); var tempParam = DbProviderFactories.GetFactory(IFast.MapDb(name)).CreateParameter(); tempParam.ParameterName = item; tempParam.Value = GetUrlParam(urlParam, item); param.Add(tempParam); if (!string.IsNullOrEmpty(IFast.MapRequired(name, item))) { if (!(IFast.MapRequired(name, item).ToLower() == "true" && !string.IsNullOrEmpty(tempParam.Value.ToStr()))) { dic.Add("error", string.Format("{0}不能为空", item)); param.Remove(tempParam); break; } } if (IFast.MapMaxlength(name, item).ToInt(0) != 0) { if (!(IFast.MapMaxlength(name, item).ToInt(0) >= tempParam.Value.ToStr().Length)) { dic.Add("error", string.Format("{0}:{1},最大长度{2}", item, tempParam.Value.ToStr(), IFast.MapMaxlength(name, item))); param.Remove(tempParam); break; } } if (!string.IsNullOrEmpty(existsKey)) { var existsListParam = new List <DbParameter>(); var existsParam = DbProviderFactories.GetFactory(IFast.MapDb(existsKey)).CreateParameter(); existsParam.ParameterName = item; existsParam.Value = tempParam.Value; existsListParam.Add(existsParam); var checkData = IFast.Query(existsKey, existsListParam.ToArray())?.First() ?? new Dictionary <string, object>(); if (checkData.GetValue("count").ToStr().ToInt(0) >= 1) { dic.Add("error", string.Format("{0}:{1}已存在", item, tempParam.Value)); param.Remove(tempParam); break; } } if (!string.IsNullOrEmpty(checkKey)) { var checkListParam = new List <DbParameter>(); var checkParam = DbProviderFactories.GetFactory(IFast.MapDb(checkKey)).CreateParameter(); checkParam.ParameterName = item; checkParam.Value = GetUrlParam(urlParam, item); checkListParam.Add(checkParam); var checkData = IFast.Query(existsKey, checkListParam.ToArray())?.First() ?? new Dictionary <string, object>(); if (checkData.GetValue("count").ToStr().ToInt(0) < 1) { dic.Add("error", string.Format("{0}:{1}不存在", item, tempParam.Value)); param.Remove(tempParam); break; } } if (IFast.MapDate(name, item).ToLower() == "true") { if (!BaseRegular.IsDate(tempParam.Value.ToStr())) { dic.Add("error", string.Format("{0}:{1},不是日期类型", item, tempParam.Value)); param.Remove(tempParam); break; } tempParam.Value = tempParam.Value.ToDate(); tempParam.DbType = System.Data.DbType.DateTime; } if (tempParam.Value.ToStr() == "") { param.Remove(tempParam); } } if (IFast.MapType(name).ToLower() == AppConfig.PageAll && dic.Count == 0) { var pageSize = GetUrlParam(urlParam, "PageSize"); var pageId = GetUrlParam(urlParam, "PageId"); isSuccess = true; var page = new PageModel(); page.PageSize = pageSize.ToInt(0) == 0 ? 10 : pageSize.ToInt(0); page.PageId = pageId.ToInt(0) == 0 ? 1 : pageId.ToInt(0); var info = IFast.QueryPage(page, name, param.ToArray()); dic.Add("data", info.list); dic.Add("page", info.pModel); } else if (IFast.MapType(name).ToLower() == AppConfig.Page && param.Count > 0) { var pageSize = GetUrlParam(urlParam, "PageSize"); var pageId = GetUrlParam(urlParam, "PageId"); isSuccess = true; var page = new PageModel(); page.PageSize = pageSize.ToInt(0) == 0 ? 10 : pageSize.ToInt(0); page.PageId = pageId.ToInt(0) == 0 ? 1 : pageId.ToInt(0); var info = IFast.QueryPage(page, name, param.ToArray()); dic.Add("data", info.list); dic.Add("page", info.pModel); } else if (IFast.MapType(name).ToLower() == AppConfig.All && dic.Count == 0) { isSuccess = true; data = IFast.Query(name, param.ToArray()); dic.Add("data", data); } else if (IFast.MapType(name).ToLower() == AppConfig.Write && param.Count > 0) { var result = IFast.Write(name, param.ToArray()); if (result.IsSuccess) { isSuccess = true; } else { isSuccess = false; dic.Add("error", result.Message); } } else { if (param.Count > 0) { isSuccess = true; data = IFast.Query(name, param.ToArray()); dic.Add("data", data); } else { isSuccess = false; } } } else { isSuccess = false; dic.Add("error", "接口不存在"); } dic.Add("isSuccess", isSuccess); context.Response.StatusCode = 200; await context.Response.WriteAsync(BaseJson.ModelToJson(dic), Encoding.UTF8).ConfigureAwait(false); }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) { try { while (!stoppingToken.IsCancellationRequested) { lock (thisLock) { using (var db = new DataContext(AppEtl.Db)) { var tableLink = DataSchema.TableLink(db); BaseLog.SaveLog("开始抽取", "FastEtlService"); var list = IFast.Query <Data_Business>(a => a.Id != null).ToList <Data_Business>(db); foreach (var item in list) { var isExec = item.UpdateTime == DateTime.Now.Hour; if (item.UpdateTime == 99) { isExec = true; } if (DataSchema.IsExistsTable(db, item.TableName) && isExec) { Parallel.Invoke(() => { var leaf = IFast.Query <Data_Business_Details>(a => a.Id == item.Id).ToList <Data_Business_Details>(db) ?? new List <Data_Business_Details>(); if (leaf.Count > 0 && leaf.Exists(a => a.Key.ToStr() != "")) { var isAdd = true; var dt = DataSchema.GetTable(tableLink, item.TableName); var columnName = dt.Columns[2].ColumnName.ToLower(); if (leaf.Exists(a => a.FieldName.ToLower() == columnName)) { DataSchema.ExpireData(db, item); //第一列 var link = DataSchema.InitColLink(leaf, db, IFast); var tempLeaf = leaf.Find(a => a.FieldName.ToLower() == columnName); var pageInfo = DataSchema.GetTableCount(tempLeaf, item); for (var i = 1; i <= pageInfo.pageCount; i++) { var log = new Data_Log(); log.Id = Guid.NewGuid().ToStr(); log.TableName = string.Format("{0}_page_{1}", item.TableName, i); log.BeginDateTime = DateTime.Now; pageInfo.pageId = i; var tempLink = link.Find(a => a.GetValue("id").ToStr() == tempLeaf.DataSourceId); var pageData = DataSchema.GetFirstColumnData(tempLink, tempLeaf, item, pageInfo); //遍历填充table for (var row = 0; row < pageData.list.Count; row++) { var dtRow = dt.NewRow(); dtRow["EtlAddTime"] = DateTime.Now; dtRow["EtlKey"] = pageData.list[row].GetValue("key"); dtRow[columnName] = pageData.list[row].GetValue("data"); //字典对照 if (!string.IsNullOrEmpty(tempLeaf.Dic)) { dtRow[columnName] = IFast.Query <Data_Dic_Details>(a => a.Value.ToLower() == dtRow[columnName].ToStr().ToLower() && a.DicId == tempLeaf.Dic, a => new { a.ContrastValue }).ToDic(db).GetValue("ContrastValue"); } //数据策略 isAdd = DataSchema.DataPolicy(db, item, dtRow["EtlKey"], columnName, dtRow[columnName]); for (var col = 2; col < dt.Columns.Count; col++) { columnName = dt.Columns[col].ColumnName.ToLower(); if (leaf.Exists(a => a.FieldName.ToLower() == columnName)) { tempLeaf = leaf.Find(a => a.FieldName.ToLower() == columnName); tempLink = link.Find(a => a.GetValue("id").ToStr() == tempLeaf.DataSourceId); dtRow[columnName] = DataSchema.GetColumnData(tempLink, tempLeaf, dtRow["EtlKey"]); //字典对照 if (!string.IsNullOrEmpty(tempLeaf.Dic)) { dtRow[columnName] = IFast.Query <Data_Dic_Details>(a => a.Value.ToLower() == dtRow[columnName].ToStr().ToLower() && a.DicId == tempLeaf.Dic, a => new { a.ContrastValue }).ToDic(db).GetValue("ContrastValue"); } //数据策略 if (item.Policy == "2") { isAdd = DataSchema.DataPolicy(db, item, dtRow["EtlKey"], columnName, dtRow[columnName]); } } } if (isAdd) { dt.Rows.Add(dtRow); } } if (dt.Rows.Count > 0) { DataSchema.AddList(db, dt, ref log); } db.Add(log); dt.Clear(); } DataSchema.CloseLink(link); item.LastUpdateTime = DateTime.Now; IFast.Update <Data_Business>(item, a => a.Id == item.Id, a => new { a.LastUpdateTime }, db); } } }); } } BaseLog.SaveLog("结束抽取", "FastEtlService"); DataSchema.CloseTableLink(tableLink); } } await Task.Delay(1000 * 60 * 30, stoppingToken); } } catch (Exception ex) { BaseLog.SaveLog(ex.ToString(), "FastEtlServiceError"); } }
public async Task ContentAsync(HttpContext context, IFastRepository IFast, bool IsResource, string dbFile = "db.json") { var name = context.Request.Path.Value.ToStr().Substring(1, context.Request.Path.Value.ToStr().Length - 1).ToLower(); var urlParam = HttpUtility.UrlDecode(GetUrlParam(context)); var success = true; var dic = new Dictionary <string, object>(); var data = new List <Dictionary <string, object> >(); var dbKey = IFast.MapDb(name).ToStr(); var pageInfo = new PageResult(); context.Response.StatusCode = 200; context.Response.ContentType = "application/Json"; if (!IFast.IsExists(name)) { dic.Add("success", false); dic.Add("error", "接口不存在"); await context.Response.WriteAsync(BaseJson.ModelToJson(dic), Encoding.UTF8).ConfigureAwait(false); } else if (dbKey == "") { dic.Add("success", false); dic.Add("error", string.Format("map id {0}的db没有配置", name)); await context.Response.WriteAsync(BaseJson.ModelToJson(dic), Encoding.UTF8).ConfigureAwait(false); } else if (IFast.IsExists(name)) { var param = new List <DbParameter>(); foreach (var item in IFast.MapParam(name)) { var checkKey = IFast.MapCheck(name, item); var existsKey = IFast.MapExists(name, item); var tempParam = DbProviderFactories.GetFactory(IFast.MapDb(name), IsResource, dbFile).CreateParameter(); tempParam.ParameterName = item; tempParam.Value = GetUrlParam(urlParam, item); param.Add(tempParam); if (!string.IsNullOrEmpty(IFast.MapRequired(name, item))) { if (!(string.Compare(IFast.MapRequired(name, item), "true", true) == 0 && !string.IsNullOrEmpty(tempParam.Value.ToStr()))) { dic.Add("success", false); dic.Add("error", string.Format("{0}不能为空", item)); param.Remove(tempParam); break; } } if (IFast.MapMaxlength(name, item).ToInt(0) != 0) { if (!(IFast.MapMaxlength(name, item).ToInt(0) >= tempParam.Value.ToStr().Length)) { dic.Add("success", false); dic.Add("error", string.Format("{0}:{1},最大长度{2}", item, tempParam.Value.ToStr(), IFast.MapMaxlength(name, item))); param.Remove(tempParam); break; } } if (!string.IsNullOrEmpty(existsKey)) { var existsListParam = new List <DbParameter>(); var existsParam = DbProviderFactories.GetFactory(IFast.MapDb(existsKey), IsResource, dbFile).CreateParameter(); existsParam.ParameterName = item; existsParam.Value = tempParam.Value; existsListParam.Add(existsParam); var checkData = IFast.Query(existsKey, existsListParam.ToArray())?.First() ?? new Dictionary <string, object>(); if (checkData.GetValue("count").ToStr().ToInt(0) >= 1) { dic.Add("success", false); dic.Add("error", string.Format("{0}:{1}已存在", item, tempParam.Value)); param.Remove(tempParam); break; } } if (!string.IsNullOrEmpty(checkKey)) { var checkListParam = new List <DbParameter>(); var checkParam = DbProviderFactories.GetFactory(IFast.MapDb(checkKey), IsResource, dbFile).CreateParameter(); checkParam.ParameterName = item; checkParam.Value = GetUrlParam(urlParam, item); checkListParam.Add(checkParam); var checkData = IFast.Query(existsKey, checkListParam.ToArray())?.First() ?? new Dictionary <string, object>(); if (checkData.GetValue("count").ToStr().ToInt(0) < 1) { dic.Add("success", false); dic.Add("error", string.Format("{0}:{1}不存在", item, tempParam.Value)); param.Remove(tempParam); break; } } if (string.Compare(IFast.MapDate(name, item).ToStr(), "true", true) == 0) { if (!BaseRegular.IsDate(tempParam.Value.ToStr())) { dic.Add("success", false); dic.Add("error", string.Format("{0}:{1},不是日期类型", item, tempParam.Value)); param.Remove(tempParam); break; } tempParam.Value = tempParam.Value.ToDate(); tempParam.DbType = System.Data.DbType.DateTime; } if (tempParam.Value.ToStr() == "") { param.Remove(tempParam); } } using (var db = new DataContext(dbKey)) { var tempParam = param.ToArray(); var sql = MapXml.GetMapSql(name, ref tempParam, db, dbKey); if (dic.Count > 0) { await context.Response.WriteAsync(BaseJson.ModelToJson(dic), Encoding.UTF8).ConfigureAwait(false); } else if (string.Compare(IFast.MapType(name).ToStr(), AppConfig.PageAll, true) == 0 || string.Compare(IFast.MapType(name).ToStr(), AppConfig.Page, true) == 0) { success = true; var pageSize = GetUrlParam(urlParam, "PageSize"); var pageId = GetUrlParam(urlParam, "PageId"); var page = new PageModel(); page.PageSize = pageSize.ToInt(0) == 0 ? 10 : pageSize.ToInt(0); page.PageId = pageId.ToInt(0) == 0 ? 1 : pageId.ToInt(0); pageInfo = db.GetPageSql(page, sql, tempParam).PageResult; if (IFast.MapView(name).ToStr() == "") { dic.Add("data", pageInfo.list); dic.Add("page", pageInfo.pModel); } } else if (string.Compare(IFast.MapType(name).ToStr(), AppConfig.All, true) == 0) { success = true; data = db.ExecuteSqlList(sql, tempParam, false).DicList; dic.Add("data", data); } else if (string.Compare(IFast.MapType(name).ToStr(), AppConfig.Write, true) == 0 && param.Count > 0) { var result = db.ExecuteSqlList(sql, tempParam, false).writeReturn; if (result.IsSuccess) { success = true; } else { success = false; dic.Add("error", result.Message); } } else { if (param.Count > 0) { success = true; data = db.ExecuteSqlList(sql, tempParam, false).DicList; dic.Add("data", data); } else { success = false; } } } dic.Add("success", success); await context.Response.WriteAsync(BaseJson.ModelToJson(dic), Encoding.UTF8).ConfigureAwait(false); } }
public void OnGet(string id) { using (var db = new DataContext(AppEtl.Db)) { var param = new List <DbParameter>(); var item = DbProviderFactories.GetFactory(db.config.DbType).CreateParameter(); item.ParameterName = "id"; item.Value = id; param.Add(item); list = IFast.Query("Business.Details", param.ToArray(), db); BusinessId = id; } }
/// <summary> /// 操作 /// </summary> /// <param name="item"></param> /// <returns></returns> public IActionResult OnPostBusinessFormList(Data_Business_List item) { var result = new WriteReturn(); result.IsSuccess = true; using (var db = new DataContext(AppEtl.Db)) { if (IFast.Query <Data_Source>(a => a.Id == item.DataId).ToCount(db) == 0) { return(new JsonResult(new { success = false, msg = "数据源不存在" })); } var data = IFast.Query <Data_Source>(a => a.Id == item.DataId).ToItem <Data_Source>(db); var tableKey = string.Format(AppEtl.CacheKey.Table, data.Host); if (!RedisInfo.Exists(tableKey, AppEtl.CacheDb)) { DataSchema.InitTable(data, false); } var tableList = RedisInfo.Get <List <CacheTable> >(tableKey, AppEtl.CacheDb); foreach (var table in tableList) { var columnKey = string.Format(AppEtl.CacheKey.Column, data.Host, table.Name); if (!RedisInfo.Exists(columnKey, AppEtl.CacheDb)) { DataSchema.InitColumn(data, false, table.Name); } var tableModel = BaseMap.CopyModel <Data_Business, Data_Business_List>(item); tableModel.Id = Guid.NewGuid().ToStr(); tableModel.Name = string.IsNullOrEmpty(table.Comments) ? table.Name : table.Comments; tableModel.TableName = table.Name; if (result.IsSuccess) { result = db.Add(tableModel).writeReturn; } if (result.IsSuccess) { result = DataSchema.CreateTable(db, tableModel); } var columnList = RedisInfo.Get <List <CacheColumn> >(columnKey, AppEtl.CacheDb); var keyName = columnList.Find(a => a.IsKey == true)?.Name; var keyList = columnList.FindAll(a => a.IsKey == true); columnList.ForEach(column => { var columnModel = new Data_Business_Details(); columnModel.FieldId = Guid.NewGuid().ToStr(); columnModel.Id = tableModel.Id; columnModel.DataSourceId = data.Id; columnModel.TableName = table.Name; columnModel.ColumnName = column.Name; columnModel.FieldName = column.Name; columnModel.Key = keyName; if (result.IsSuccess) { result = db.Add(columnModel).writeReturn; } else { BaseLog.SaveLog(string.Format("tableName:{0},error:", table.Name, result.Message), "Error_CreateTable"); } if (result.IsSuccess) { if ((keyList.Count > 1 && keyList.Exists(a => a.Name == columnModel.FieldName))) { result = DataSchema.AddColumn(db, tableModel, columnModel, column, data, false); } else { result = DataSchema.AddColumn(db, tableModel, columnModel, column, data); } if (result.IsSuccess) { DataSchema.UpdateColumnComment(db, tableModel, columnModel, column, data); } } result.IsSuccess = true; }); if (keyList.Count > 1) { DataSchema.AddColumnMoreKey(db, tableModel, keyList); } } if (result.IsSuccess) { return(new JsonResult(new { success = true, msg = "操作成功" })); } else { return(new JsonResult(new { success = false, msg = result.Message })); } } }