public static IEnumerable <T> ReadSqlData <T>(OleDbDataReader reader, Func <IDataRecord, T> selector) { while (reader.Read()) { yield return(selector(reader)); } reader.CloseDataReader(); }
public void ProcessPhoneCalls() { var phoneCallsFunc = new PhoneCallsImpl(); DataTable toBeInsertedDataTable; OleDbDataReader dataReader = null; var exceptions = new ConcurrentQueue <Exception>(); var column = string.Empty; var lastImportedPhoneCallDate = DateTime.MinValue; //OPEN CONNECTIONS _sourceDbConnector.Open(); _destinationDbConnector.Open(); dataReader = DbRoutines.Executereader(SqLs.GetLastImportedPhonecallDate(_phoneCallsTableName, false), _destinationDbConnector); if (dataReader.Read() && !dataReader.IsDBNull(0)) { lastImportedPhoneCallDate = dataReader.GetDateTime(dataReader.GetOrdinal("SessionIdTime")); lastImportedPhoneCallDate = lastImportedPhoneCallDate.AddDays(+1); dataReader.CloseDataReader(); } else { //Table is empty in this case we need to read from the source that we will import the data from dataReader = DbRoutines.Executereader(SqLs.GetLastImportedPhonecallDate("DialogsView", true), _sourceDbConnector); if (dataReader.Read() && !dataReader.IsDBNull(0)) { lastImportedPhoneCallDate = dataReader.GetDateTime(dataReader.GetOrdinal("SessionIdTime")); } dataReader.CloseDataReader(); } while (lastImportedPhoneCallDate <= DateTime.Now) { //Construct CREATE_IMPORT_PHONE_CALLS_QUERY var sql = SqLs.CreateImportCallsQueryLync2013(lastImportedPhoneCallDate); if (lastImportedPhoneCallDate > DateTime.MinValue) { Console.WriteLine("Importing PhoneCalls from " + _phoneCallsTableName + " since " + lastImportedPhoneCallDate); } else { Console.WriteLine("Importing PhoneCalls from " + _phoneCallsTableName + " since the begining"); } //Read DB and map it to List of PhoneCalls var phoneCalls = Db.ReadSqlData(DbRoutines.Executereader(sql, _sourceDbConnector), Db.PhoneCallsSelector).ToList(); if (phoneCalls.Count() > 0) { var status = new object(); var partitionsize = Partitioner.Create(0, phoneCalls.Count()); Parallel.ForEach(partitionsize, (range, loopStet) => { for (var i = range.Item1; i < range.Item2; i++) { phoneCallsFunc.ProcessPhoneCall(phoneCalls[i]); } }); // Bulk insert toBeInsertedDataTable = phoneCalls.ConvertToDataTable(); toBeInsertedDataTable.BulkInsert(_phoneCallsTableName, _destinationDbConnector.ConnectionString); toBeInsertedDataTable.Dispose(); Console.WriteLine(" [+] Imported: " + phoneCalls.Count + " phone calls."); } // Increment the datetime object by 1 day. lastImportedPhoneCallDate = lastImportedPhoneCallDate.AddDays(+1); GC.Collect(); } //Close All Connection and DataReaders _sourceDbConnector.Close(); _destinationDbConnector.Close(); if (dataReader.IsClosed == false) { dataReader.Close(); } Console.WriteLine("Finish importing Calls from " + _phoneCallsTableName); }