private static void BatchCaseData(int batchCommit, DataConv.Consumer.ProviderServiceRef.ProviderServiceClient psc, List <DataConv.Consumer.ProviderServiceRef.DataCase> lst_sc, string tableName) { if (lst_sc.Count > 0) { RemoveDuplicate(lst_sc);//去除重复数据 if (lst_sc.Count > 0) { PriceFilter(lst_sc);//价格过滤 } } int length = lst_sc.Count; //把List转成数组,WCF必须传数组过去才行 DataConv.Consumer.ProviderServiceRef.DataCase[] dc_tmp = new DataConv.Consumer.ProviderServiceRef.DataCase[length]; for (int n = 0; n < length; n++) { dc_tmp[n] = lst_sc[n]; } //批量入库,返回成功插入数据库的记录数 int insertCount = psc.BatchInsertDataCase(dc_tmp, tableName); logger.Info("当前批量插入[" + insertCount + "]行数据..."); }
public static void RunMain() { String[] cityId = ConfigurationManager.AppSettings["CityList"].Split(','); int length = cityId.Length; int[] arrays = new int[length]; for (int i = 0; i < length; i++) { arrays[i] = Convert.ToInt32(cityId[i]); } string startTime = Convert.ToString(ConfigurationManager.AppSettings["StartTime"]); string endTime = Convert.ToString(ConfigurationManager.AppSettings["EndTime"]); int batchCommit = Convert.ToInt32(ConfigurationManager.AppSettings["BatchCommit"]); bool isRemoveData = Convert.ToBoolean(ConfigurationManager.AppSettings["RemoveData"]); logger.Info("配置参数加载完成,获取远程数据信息"); //获取所有城市数据 DataConv.Consumer.ProviderServiceRef.ProviderServiceClient psc = new DataConv.Consumer.ProviderServiceRef.ProviderServiceClient(); int page = 1; List <DataConv.Consumer.ProviderServiceRef.City> list_city = new List <DataConv.Consumer.ProviderServiceRef.City>(); while (true) { List <DataConv.Consumer.ProviderServiceRef.City> list = ServiceRef.QueryCityInfoList(psc, page); if (list.Count > 0) { foreach (DataConv.Consumer.ProviderServiceRef.City item in list) { logger.DebugFormat("City name={0}, ID={1}", item.CityName, item.CityID); list_city.Add(item); } } else { break; } page++; } Dictionary <string, int> PurposeMap = ServiceRef.QueryPurposeInfoMap(psc); Dictionary <string, int> FrontMap = ServiceRef.QueryFrontInfoMap(psc); Dictionary <string, int> BuildingTypeMap = ServiceRef.QueryBuildingTypeInfoMap(psc); Dictionary <string, int> HouseTypeMap = ServiceRef.QueryHouseTypeInfoMap(psc); Dictionary <string, int> StructureMap = ServiceRef.QueryStructureInfoMap(psc); Dictionary <string, int> MoneyUnitInfoMap = ServiceRef.QueryMoneyUnitInfoMap(psc); if (PurposeMap.Keys.Count == 0 || FrontMap.Keys.Count == 0 || BuildingTypeMap.Keys.Count == 0 || HouseTypeMap.Keys.Count == 0 || StructureMap.Keys.Count == 0) { logger.Info("基础数据获取不全,无法完成后续操作!"); Console.ReadLine(); } else { CaseDao caseDao = new CaseDao(); Dictionary <string, string> paramMap = new Dictionary <string, string>(); paramMap.Add("[#StartTime#]", startTime); paramMap.Add("[#EndTime#]", endTime); foreach (DataConv.Consumer.ProviderServiceRef.City item in list_city) { Console.WriteLine(item.CityName + " 转换数据开始时间= " + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); logger.InfoFormat("{0} 转换数据开始时间= {1}", item.CityName, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); if (paramMap.ContainsKey("[#CityID#]")) { paramMap.Remove("[#CityID#]"); } paramMap.Add("[#CityID#]", Convert.ToString(item.CityID)); //此处可以根据item,取出City的各个值 logger.Info("CityName = " + item.CityName + "\t" + item.ProjectTable + "\t" + item.CaseTable); //该集合中的对象到15个时,执行一次插入,并清空 List <DataConv.Consumer.ProviderServiceRef.DataCase> lst_sc = new List <DataConv.Consumer.ProviderServiceRef.DataCase>(); //循环获取房源信息【住宅案例_出售】 int pageIndex = 1; ProviderServiceRef.Area[] a = psc.QueryAreaInfoMap(item.CityID); //获取城市对应的行政区 Dictionary <string, int> dict = new Dictionary <string, int>(); foreach (ProviderServiceRef.Area aitem in a) { //logger.Debug(aitem.AreaName + "\t" + aitem.AreaId); dict.Add(aitem.AreaName, aitem.AreaId); } item.AreaMap = dict; //获取城市对应的楼盘信息表(标准数据,对照源) //List<ProviderServiceRef.DataProject> lst_project = ServiceRef.QueryDataProjectList(psc, item.CityID, 0, item.ProjectTable); List <ProviderServiceRef.DataProject> lst_project = new List <ProviderServiceRef.DataProject>(); while (true) { List <ProviderServiceRef.DataProject> list = ServiceRef.PagingQueryDataProjectList(psc, item.CityID, 0, item.ProjectTable, pageIndex); if (list.Count == 0) { break; } foreach (ProviderServiceRef.DataProject item1 in list) { lst_project.Add(item1); } pageIndex++; } pageIndex = 1; while (true) { List <DataCase> list = caseDao.PagingQueryDataCase(pageIndex, 10, paramMap); if (list.Count == 0) { //取不到数据,就退出循环 break; } foreach (DataCase data in list) { int projectId = ValidatorUtils.IsValidProject(lst_project, data); bool isValid = ValidatorUtils.IsValidData(item, data, arrays, PurposeMap, FrontMap, BuildingTypeMap, HouseTypeMap, StructureMap, MoneyUnitInfoMap); if (isValid && projectId != -1) { #region 符合条件的数据 lst_sc.Add(new DataConv.Consumer.ProviderServiceRef.DataCase() { ProjectId = projectId, CaseDate = data.CaseDate, AreaId = data.AreaId == null ? -1 : data.AreaId, AreaName = data.AreaName == null ? null : data.AreaName, BuildingArea = data.BuildingArea = data.BuildingArea, PurposeCode = data.PurposeCode == null ? -1 : data.PurposeCode, BuildingDate = data.BuildingDate == null ? null : data.BuildingDate, BuildingTypeCode = data.BuildingTypeCode == null ? -1 : data.BuildingTypeCode, CaseTypeCode = data.CaseTypeCode == null ? -1 : data.CaseTypeCode, CityID = data.CityID == null ? -1 : data.CityID, CaseTypeName = data.CaseTypeName == null ? null : data.CaseTypeName, BuildingTypeName = data.BuildingTypeName == null ? null : data.BuildingTypeName, FloorNumber = data.FloorNumber, FrontCode = data.FrontCode == null ? -1 : data.FrontCode, FrontName = data.FrontName == null ? null : data.FrontName, HouseTypeCode = data.HouseTypeCode == null ? -1 : data.HouseTypeCode, HouseTypeName = data.HouseTypeName == null ? null : data.HouseTypeName, MoneyUnitCode = data.MoneyUnitCode == null ? -1 : data.MoneyUnitCode, MoneyUnitName = data.MoneyUnitName == null ? null : data.MoneyUnitName, PeiTao = data.PeiTao == null ? null : data.PeiTao, PurposeName = data.PurposeName == null ? null : data.PurposeName, ProjectName = data.ProjectName == null ? null : data.ProjectName, RecordWeek = data.RecordWeek, RemainYear = data.RemainYear, Remark = data.Remark == null ? null : data.Remark, SourceLink = data.SourceLink == null ? null : data.SourceLink, SourceName = data.SourceName == null ? null : data.SourceName, SightCode = data.SightCode == null ? -1 : data.SightCode, SourcePhone = data.SourcePhone == null ? null : data.SourcePhone, StructureCode = data.StructureCode == null ? -1 : data.StructureCode, StructureName = data.StructureName == null ? null : data.StructureName, CreateDate = DateTime.Now, Creator = data.Creator == null ? null : data.Creator, SurveyId = data.SurveyId == null ? 0 : data.SurveyId, TotalFloor = data.TotalFloor, TotalPrice = data.TotalPrice = data.TotalPrice, UnitPrice = data.UnitPrice = data.UnitPrice, Valid = data.Valid == null ? 1 : data.Valid, ZhuangXiu = data.ZhuangXiu == null ? null : data.ZhuangXiu, Depreciation = data.Depreciation }); #endregion } else { #region 失败信息,需要入库 logger.InfoFormat("<<< INVALID DATA CityName={0}, ProjectName={1}, ProjectId={2}, isValid={3}", data.CityID, data.ProjectName, projectId, isValid); caseDao.AddExceptionData(data); #endregion } //if (lst_sc.Count == batchCommit) //{ // #region List中的对象数到15条,就执行一次入库 // BatchCaseData(batchCommit, psc, lst_sc, item.CaseTable); // lst_sc = new List<ProviderServiceRef.DataCase>(); // #endregion //} } //if (lst_sc.Count > 0) //{ // #region List中的对象数到15条,就执行一次入库 // BatchCaseData(batchCommit, psc, lst_sc, item.CaseTable); // lst_sc = new List<ProviderServiceRef.DataCase>(); // #endregion //} pageIndex++; } if (lst_sc.Count > 0) { #region List中的对象数到15条,就执行一次入库 BatchCaseData(batchCommit, psc, lst_sc, item.CaseTable); lst_sc = new List <ProviderServiceRef.DataCase>(); #endregion } if (isRemoveData) { //从源库删除这些已经处理转换过的案例 int cnt = caseDao.RemoveDataCase(paramMap); logger.Info("当前批量删除已经处理转换过的案例DataCase[" + cnt + "]行数据..."); } Console.WriteLine(item.CityName + " 转换数据结束时间= " + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); logger.InfoFormat("{0} 转换数据结束时间= {1}", item.CityName, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); } Console.WriteLine("================>> 数据转换完成,时间 = " + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); logger.InfoFormat("================================================>> 数据转换完成,时间 = {0}", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); } }