Example #1
0
        /// <summary>
        /// 定时任务同步产品数据
        /// </summary>
        /// <param name="source"></param>
        /// <param name="e"></param>
        public void SynData(object source, ElapsedEventArgs e)
        {
            try
            {
                //防止重复执行
                lock (thisLock)
                {
                    using (var db = new DataContext())
                    {
                        BaseLog.SaveLog("开始抽取", "FastEtlService");

                        //不允许停止服务
                        this.CanStop = false;

                        var list = FastRead.Query <Data_Business>(a => a.Id != null).ToList <Data_Business>(db);

                        foreach (var item in list)
                        {
                            if (DataSchema.IsExistsTable(db, item.TableName) && item.UpdateTime == DateTime.Now.Hour && item.LastUpdateTime.Day + item.UpdateDay >= DateTime.Now.Day)
                            {
                                Parallel.Invoke(() =>
                                {
                                    var leaf = FastRead.Query <Data_Business_Details>(a => a.Id == item.Id).ToList <Data_Business_Details>(db);

                                    if (leaf.Count > 0)
                                    {
                                        var isAdd      = true;
                                        var dt         = DataSchema.GetTable(db, item.TableName);
                                        var columnName = dt.Columns[3].ColumnName.ToLower();

                                        if (leaf.Exists(a => a.FieldName.ToLower() == columnName))
                                        {
                                            DataSchema.ExpireData(db, item);

                                            //第一列
                                            var link     = DataSchema.InitColLink(leaf, db);
                                            var tempLeaf = leaf.Find(a => a.FieldName.ToLower() == columnName);
                                            var pageInfo = DataSchema.GetTableCount(link[0], tempLeaf, item);

                                            for (var i = 1; i <= pageInfo.pageCount; i++)
                                            {
                                                var log           = new Data_Log();
                                                log.Id            = Guid.NewGuid().ToStr();
                                                log.TableName     = string.Format("{0}_page_{1}", item.TableName, i);
                                                log.BeginDateTime = DateTime.Now;

                                                pageInfo.pageId = i;
                                                var pageData    = DataSchema.GetFirstColumnData(link[0], tempLeaf, item, pageInfo);

                                                //遍历填充table
                                                for (var row = 0; row < pageData.list.Count; row++)
                                                {
                                                    var dtRow         = dt.NewRow();
                                                    dtRow["Id"]       = Guid.NewGuid().ToString();
                                                    dtRow["AddTime"]  = DateTime.Now;
                                                    dtRow["Key"]      = pageData.list[row].GetValue("key");
                                                    dtRow[columnName] = pageData.list[row].GetValue("data");

                                                    //字典对照
                                                    if (!string.IsNullOrEmpty(tempLeaf.Dic))
                                                    {
                                                        dtRow[columnName] = FastRead.Query <Data_Dic_Details>(a => a.Value.ToLower() == dtRow[columnName].ToStr().ToLower() && a.DicId == tempLeaf.Dic, a => new { a.ContrastValue }).ToDic(db).GetValue("ContrastValue");
                                                    }

                                                    //数据策略
                                                    isAdd = DataSchema.DataPolicy(db, item, dtRow["Key"], columnName, dtRow[columnName]);

                                                    for (var col = 3; col < dt.Columns.Count; col++)
                                                    {
                                                        columnName = dt.Columns[col].ColumnName.ToLower();
                                                        if (leaf.Exists(a => a.FieldName.ToLower() == columnName))
                                                        {
                                                            tempLeaf          = leaf.Find(a => a.FieldName.ToLower() == columnName);
                                                            dtRow[columnName] = DataSchema.GetColumnData(link[col - 3], tempLeaf, dtRow["Key"]);

                                                            //字典对照
                                                            if (!string.IsNullOrEmpty(tempLeaf.Dic))
                                                            {
                                                                dtRow[columnName] = FastRead.Query <Data_Dic_Details>(a => a.Value.ToLower() == dtRow[columnName].ToStr().ToLower() && a.DicId == tempLeaf.Dic, a => new { a.ContrastValue }).ToDic(db).GetValue("ContrastValue");
                                                            }

                                                            //数据策略
                                                            if (item.Policy == "2")
                                                            {
                                                                isAdd = DataSchema.DataPolicy(db, item, dtRow["Key"], columnName, dtRow[columnName]);
                                                            }
                                                        }
                                                    }

                                                    if (isAdd)
                                                    {
                                                        dt.Rows.Add(dtRow);
                                                    }
                                                }

                                                if (dt.Rows.Count > 0)
                                                {
                                                    DataSchema.AddList(db, dt, ref log);
                                                }
                                                db.Add(log);
                                                dt.Clear();
                                            }

                                            DataSchema.CloseLink(link);
                                            item.LastUpdateTime = DateTime.Now;
                                            FastWrite.Update <Data_Business>(item, a => a.Id == item.Id, a => new { a.LastUpdateTime }, db);
                                        }
                                    }
                                });
                            }
                        }

                        //允许停止服务
                        this.CanStop = true;

                        BaseLog.SaveLog("结束抽取", "FastEtlService");
                    }
                }
            }
            catch (Exception ex)
            {
                BaseLog.SaveLog(ex.ToString(), "FastEtlServiceError");
            }
        }
Example #2
0
        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            try
            {
                while (!stoppingToken.IsCancellationRequested)
                {
                    lock (thisLock)
                    {
                        using (var db = new DataContext(AppEtl.Db))
                        {
                            var tableLink = DataSchema.TableLink(db);
                            BaseLog.SaveLog("开始抽取", "FastEtlService");

                            var list = IFast.Query <Data_Business>(a => a.Id != null).ToList <Data_Business>(db);

                            foreach (var item in list)
                            {
                                var isExec = item.UpdateTime == DateTime.Now.Hour;
                                if (item.UpdateTime == 99)
                                {
                                    isExec = true;
                                }

                                if (DataSchema.IsExistsTable(db, item.TableName) && isExec)
                                {
                                    Parallel.Invoke(() =>
                                    {
                                        var leaf = IFast.Query <Data_Business_Details>(a => a.Id == item.Id).ToList <Data_Business_Details>(db) ?? new List <Data_Business_Details>();

                                        if (leaf.Count > 0 && leaf.Exists(a => a.Key.ToStr() != ""))
                                        {
                                            var isAdd = true;
                                            var dt    = DataSchema.GetTable(tableLink, item.TableName);

                                            var columnName = dt.Columns[2].ColumnName.ToLower();

                                            if (leaf.Exists(a => a.FieldName.ToLower() == columnName))
                                            {
                                                DataSchema.ExpireData(db, item);

                                                //第一列
                                                var link     = DataSchema.InitColLink(leaf, db, IFast);
                                                var tempLeaf = leaf.Find(a => a.FieldName.ToLower() == columnName);
                                                var pageInfo = DataSchema.GetTableCount(tempLeaf, item);

                                                for (var i = 1; i <= pageInfo.pageCount; i++)
                                                {
                                                    var log           = new Data_Log();
                                                    log.Id            = Guid.NewGuid().ToStr();
                                                    log.TableName     = string.Format("{0}_page_{1}", item.TableName, i);
                                                    log.BeginDateTime = DateTime.Now;

                                                    pageInfo.pageId = i;
                                                    var tempLink    = link.Find(a => a.GetValue("id").ToStr() == tempLeaf.DataSourceId);
                                                    var pageData    = DataSchema.GetFirstColumnData(tempLink, tempLeaf, item, pageInfo);

                                                    //遍历填充table
                                                    for (var row = 0; row < pageData.list.Count; row++)
                                                    {
                                                        var dtRow           = dt.NewRow();
                                                        dtRow["EtlAddTime"] = DateTime.Now;
                                                        dtRow["EtlKey"]     = pageData.list[row].GetValue("key");
                                                        dtRow[columnName]   = pageData.list[row].GetValue("data");

                                                        //字典对照
                                                        if (!string.IsNullOrEmpty(tempLeaf.Dic))
                                                        {
                                                            dtRow[columnName] = IFast.Query <Data_Dic_Details>(a => a.Value.ToLower() == dtRow[columnName].ToStr().ToLower() && a.DicId == tempLeaf.Dic, a => new { a.ContrastValue }).ToDic(db).GetValue("ContrastValue");
                                                        }

                                                        //数据策略
                                                        isAdd = DataSchema.DataPolicy(db, item, dtRow["EtlKey"], columnName, dtRow[columnName]);

                                                        for (var col = 2; col < dt.Columns.Count; col++)
                                                        {
                                                            columnName = dt.Columns[col].ColumnName.ToLower();
                                                            if (leaf.Exists(a => a.FieldName.ToLower() == columnName))
                                                            {
                                                                tempLeaf          = leaf.Find(a => a.FieldName.ToLower() == columnName);
                                                                tempLink          = link.Find(a => a.GetValue("id").ToStr() == tempLeaf.DataSourceId);
                                                                dtRow[columnName] = DataSchema.GetColumnData(tempLink, tempLeaf, dtRow["EtlKey"]);

                                                                //字典对照
                                                                if (!string.IsNullOrEmpty(tempLeaf.Dic))
                                                                {
                                                                    dtRow[columnName] = IFast.Query <Data_Dic_Details>(a => a.Value.ToLower() == dtRow[columnName].ToStr().ToLower() && a.DicId == tempLeaf.Dic, a => new { a.ContrastValue }).ToDic(db).GetValue("ContrastValue");
                                                                }

                                                                //数据策略
                                                                if (item.Policy == "2")
                                                                {
                                                                    isAdd = DataSchema.DataPolicy(db, item, dtRow["EtlKey"], columnName, dtRow[columnName]);
                                                                }
                                                            }
                                                        }

                                                        if (isAdd)
                                                        {
                                                            dt.Rows.Add(dtRow);
                                                        }
                                                    }

                                                    if (dt.Rows.Count > 0)
                                                    {
                                                        DataSchema.AddList(db, dt, ref log);
                                                    }
                                                    db.Add(log);
                                                    dt.Clear();
                                                }

                                                DataSchema.CloseLink(link);
                                                item.LastUpdateTime = DateTime.Now;
                                                IFast.Update <Data_Business>(item, a => a.Id == item.Id, a => new { a.LastUpdateTime }, db);
                                            }
                                        }
                                    });
                                }
                            }

                            BaseLog.SaveLog("结束抽取", "FastEtlService");
                            DataSchema.CloseTableLink(tableLink);
                        }
                    }
                    await Task.Delay(1000 * 60 * 30, stoppingToken);
                }
            }
            catch (Exception ex)
            {
                BaseLog.SaveLog(ex.ToString(), "FastEtlServiceError");
            }
        }