static void Main(string[] args) { try { Console.WriteLine("Initialization...... "); //Aggregate_Save obj = new Aggregate_Save(); //obj.Update_Hour_Aggregation(3628); // Interpolation.Get_Previous_and_Next_Good_Value(DateTime.Parse("2015-09-01 00:30:00"), 3575); //data_staus is used for the interpoliation of data string[] data_status = ConfigurationManagerHelper.AppSettings["bad_data_status"].ToString().Split(','); InnonAnalyticsEngineEntities _dbcontext = new InnonAnalyticsEngineEntities(); IElementRepository _ElementRepository = new ElementRepository(_dbcontext); //Get All Elements from the tblElement where Elemet Type is point (point has data) IList <ElementDTO> list_element_dto = _ElementRepository.Get_Element_By_Element_Tag_Type(Common.Enums.TagType.point, false); //list_element_dto = list_element_dto.Where(e => e.ID == 2366).ToList(); //GenerateMissingData(list_element_dto[0]); ITagRepository _TagRepository = new TagRepository(_dbcontext); //Get All non deleted tag type mapping IList <Tag_Type_Mapping_DTO> list_tag_type_dto = _TagRepository.Get_Tag_Type_Mapping(false); Console.WriteLine("Updating point dim "); IPoint_Dim_Service _Point_dim_service = new Point_Dim_Service(); _Point_dim_service.Update_Point_Dim(list_element_dto, list_tag_type_dto); Console.WriteLine("Start Data Migration "); IDatawareHouseMigrationLogRepository _DatawareHouseMigrationLogRepository = new DatawareHouseMigrationLogRepository(_dbcontext); //Get all element data migration history from [tblDatawareHouseMigrationLog] IList <DatawareHouseMigrationLogDTO> list_datawaredousemigrationLog_dto = _DatawareHouseMigrationLogRepository.get_Data_Migration_Logs(); //Creat an arry of tasks, Number of Point element found in tblElement table Task[] taskArray = new Task[list_element_dto.Count]; DateTime starttime = DateTime.Now; Console.WriteLine("Start " + starttime); //Initialize Connector Repository for getting the data connection info of each table IConnectorRepository _connector_rep = _connector_rep = new ConnectorRepository(_dbcontext); ConnectorDTO _dto_connector = new ConnectorDTO(); //running loop on each task in the task array () for (int i = 0; i < taskArray.Length; i++) { DateTime default_from_dt, default_to_dt = System.DateTime.Now;; taskArray[i] = Task.Factory.StartNew(() => { try { //First Step get database or data connector information of each points //The connection info is same as the previous we will not get the connector info again if (_dto_connector.ConnectionID != list_element_dto[i].ConnectionInfo.ConnectionID) { _dto_connector = _connector_rep.GetExistingConnectionById(list_element_dto[i].ConnectionInfo.ConnectionID); } //Console.WriteLine("No." + i + " " + _dto_connector.ConnectionID + " -- " + _dto_connector.ConnectionName); //Sec Step set from_date and to_date //If the element is not runing first time than change the from datetime (get last run datetime), All log save in the tbldatawaredousemigrationLog if (list_datawaredousemigrationLog_dto.Any(element => element.Element_ID == list_element_dto[i].ID)) { DateTime?_timestamp_to = list_datawaredousemigrationLog_dto.Where(element => element.Element_ID == list_element_dto[i].ID).OrderByDescending(t => t.Last_Run_Start).First().Timestamp_To; if (_timestamp_to.HasValue) { default_from_dt = Convert.ToDateTime(_timestamp_to.Value.AddMilliseconds(2).ToString("yyyy-MM-dd,HH:mm:ss.fff")); // GetRowsForEachTable(list_element_dto[j], _dto_connector, j); } else { default_from_dt = CheckLastRecordInsterted(list_element_dto[i].ID); // GetRowsForEachTable(list_element_dto[j], _dto_connector, j); } } else { default_from_dt = CheckLastRecordInsterted(list_element_dto[i].ID); //GetRowsForEachTable(list_element_dto[j], _dto_connector, j); } Console.WriteLine("No." + i + " " + list_element_dto[i].ID + " " + default_from_dt + " -- " + default_to_dt); //End sec //Third Get Raw Data; IList <RawDataDTO> list_RawDataDTO = new List <RawDataDTO>(); list_RawDataDTO = GetRawData(list_element_dto[i], default_from_dt, default_to_dt, _dto_connector, _dbcontext); //End Get Raw Data Console.WriteLine("Total Records " + list_RawDataDTO.Count()); //Save DataMirgrationLog tblDatawareHouseMigrationLog tbldatamigrationlog = SaveDataMigrationLog(list_element_dto[i], default_to_dt); //End if (list_RawDataDTO.Count > 0) { //Get Convert Raw data in to Point_Mesaure_Fact_DTO IList <Point_Measure_Fact> point_measure_fact_list_dto = Convert_Raw_Data_To_Point_Measure_Fact_DTO(list_element_dto[i], list_RawDataDTO); //End //Saving in to the Datawarehouse table Point_Measure_Fact Final_Insert_In_To_DatawareHouse_Table(point_measure_fact_list_dto); //End //Update DataMirgrationLog if (point_measure_fact_list_dto.Count > 0) { UpdateDataMigrationLog(tbldatamigrationlog, point_measure_fact_list_dto.Count(), point_measure_fact_list_dto.LastOrDefault().Timestamp_From); } //End } //Aggregation //Task aggregationtask = GetAggregation(list_element_dto[i]); //EndAggregation Console.WriteLine("Missing Data " + list_element_dto[i].Source_Element_Name_History); try { //Get Raw missing data if any thing accedently during the process than we need to get the missing data IList <RawDataDTO> list_Mising_RawDataDTO = GetRawMissingData(list_element_dto[i], _dto_connector); Console.WriteLine("Get Missing Data " + list_Mising_RawDataDTO.Count); if (list_Mising_RawDataDTO.Count > 0) { //Get Convert Raw data in to Point_Mesaure_Fact_DTO IList <Point_Measure_Fact> missing_point_measure_fact_list_dto = Convert_Raw_Data_To_Point_Measure_Fact_DTO(list_element_dto[i], list_Mising_RawDataDTO); //End //Saving in to the Datawarehouse table Point_Measure_Fact Final_Insert_In_To_DatawareHouse_Table(missing_point_measure_fact_list_dto); //End } //End End of Raw missing data saving } catch (Exception ex) { Console.WriteLine("Error in Missing Data " + ex.Message); } Console.WriteLine("Generate Missing Data "); try { IList <RawDataDTO> list_Genrated_Mising_RawDataDTO = GenerateMissingData(list_element_dto[i]); //Get Convert Raw data in to Point_Mesaure_Fact_DTO IList <Point_Measure_Fact> genrated_missing_point_measure_fact_list_dto = Convert_Raw_Data_To_Point_Measure_Fact_DTO(list_element_dto[i], list_Genrated_Mising_RawDataDTO); //End //Saving in to the Datawarehouse table Point_Measure_Fact Final_Insert_In_To_DatawareHouse_Table(genrated_missing_point_measure_fact_list_dto); //End } catch (Exception ex) { Helper.WriteToFile(ex.Message); } try { //Console.WriteLine("Deleting Data " + list_element_dto[i].Source_Element_Name_History); //Deleting Any Duplication Delete_Duplicate(list_element_dto[i]); //End } catch (Exception ex) { Console.WriteLine("Error in Delete duplicate Data " + ex.Message); } try { //Check interpolation Console.WriteLine("Interpolation"); bool Is_Read_For_Aggregation = Interpolation.InterPolateData(default_from_dt, default_to_dt, list_element_dto[i]); Console.WriteLine("End Interpolation"); //End Interpolation if (Is_Read_For_Aggregation) { Console.WriteLine("Aggregate Data Migration "); IAggregation_Service agr_service = new Aggregation_Service(); agr_service.Calculate_and_Save_Aggregation_WareHouse_By_ElementID(list_element_dto[i].ID); Console.WriteLine("End Aggregate Data Migration "); } } catch (Exception ex) { Console.WriteLine("Error in Aggregation Data " + ex.Message); } // aggregationtask.Wait(); } catch (Exception ex) { Console.WriteLine("Exception " + ex.Message); } }); taskArray[i].Wait(); } Console.WriteLine("End Min " + DateTime.Now.Subtract(starttime).TotalMinutes); Console.WriteLine("End Sec " + DateTime.Now.Subtract(starttime).TotalSeconds); Helper.WriteToFile("Finished Start Time " + starttime + ", End Time " + System.DateTime.Now + " -- " + " Total Min " + DateTime.Now.Subtract(starttime).TotalMinutes); } catch (Exception ex) { Console.WriteLine("Exception in mian " + ex.Message); } //Console.ReadLine(); }
public async static void GetData() { IElementRepository _ElementRepository; IDatawareHouseMigrationLogRepository _DatawareHouseMigrationLogRepository; _dbcontext = new InnonAnalyticsEngineEntities(); _ElementRepository = new ElementRepository(_dbcontext); _DatawareHouseMigrationLogRepository = new DatawareHouseMigrationLogRepository(_dbcontext); IList <ElementDTO> list_element_dto = _ElementRepository.Get_Element_By_Element_Tag_Type(Common.Enums.TagType.point, false); //Get all element data migration history from [tblDatawareHouseMigrationLog] IList <DatawareHouseMigrationLogDTO> list_datawaredousemigrationLog_dto = _DatawareHouseMigrationLogRepository.get_Data_Migration_Logs(); Console.WriteLine("Total Table " + list_element_dto.Count(), EventLogEntryType.Information); try { _dbcontext = new InnonAnalyticsEngineEntities(); IConnectorRepository _connector_rep; ConnectorDTO _dto_connector; for (int j = 0; j < list_element_dto.Count(); j++) { default_to_dt = System.DateTime.Now; //One year back datetime //default_from_dt = Convert.ToDateTime(default_to_dt.AddYears(-1).ToShortDateString() + " 00:00:00"); Console.WriteLine(j + "Initialize " + list_element_dto[j].Element_Name); _connector_rep = new ConnectorRepository(_dbcontext); _dto_connector = _connector_rep.GetExistingConnectionById(list_element_dto[j].ConnectionInfo.ConnectionID); //If the element previously run than change the from datetime if (list_datawaredousemigrationLog_dto.Any(element => element.Element_ID == list_element_dto[j].ID)) { DateTime?_timestamp_to = list_datawaredousemigrationLog_dto.Where(element => element.Element_ID == list_element_dto[j].ID).OrderByDescending(t => t.Last_Run_Start).First().Timestamp_To; if (_timestamp_to.HasValue) { default_from_dt = Convert.ToDateTime(_timestamp_to.Value.AddMilliseconds(2).ToString("yyyy-MM-dd,HH:mm:ss.fff")); await Task.Factory.StartNew(() => { GetRowsForEachTable(list_element_dto[j], _dto_connector, j); }); } else { await Task.Run(() => { CheckLastRecordInsterted(list_element_dto[j].ID); }) .ContinueWith((z) => { GetRowsForEachTable(list_element_dto[j], _dto_connector, j); }); } //default_from_dt = Convert.ToDateTime(list_datawaredousemigrationLog_dto.Where(element => element.Element_ID == list_element_dto[j].ID).OrderByDescending(t => t.Last_Run_Start).First().Timestamp_To.AddMilliseconds(2).ToString("yyyy-MM-dd,HH:mm:ss.fff")); } else { // await Task.Factory.StartNew(() => { GetRowsForEachTable(list_element_dto[j]); }); await Task.Run(() => { CheckLastRecordInsterted(list_element_dto[j].ID); }) .ContinueWith((z) => { GetRowsForEachTable(list_element_dto[j], _dto_connector, j); }); } // Task.Run(() => { Delete_Duplicate(list_element_dto[j]); }); //Task[] taskArray = { Task.Factory.StartNew(() => { Delete_Duplicate(list_element_dto[j]); }) }; //Task.WaitAll(taskArray); } } catch (Exception ex) { Helper.WriteToFile("Error GetData " + "- Error" + ex.Message + " - Inner exception " + ex.InnerException); } }