protected override bool LoadBatch(IDataReader reader, BatchParams batchParams, LoadCounters rowCounters) { bool read = true; //batch size = 0 works as a single batch since --batchParams.Size != 0 will be never reached while (--batchParams.Size != 0 && (read = reader.Read())) { try { if (RowExists(reader)) { if (_updateCmd != null) { DoUpdate(reader); rowCounters.Updated++; } } else { DoInsert(reader); rowCounters.Added++; } } catch (Exception ex) { OnLoadError(ex); } } return read; }
public void ClearData(object key, LoadCounters counters) { using (var connection = new SqlConnection(_connString)) { connection.Open(); try { //Проверка в списке собственных складов var cnt = LockerManager.CheckOwnStore((int)key, connection); //Если поставщик находится в собственных складах наличия то пытаемся получить лок на базу данных, если не получаем то вываливаемся с исключением if (cnt > 0 && LockerManager.AquireCheckLock(connection) < 0) { //Если здесь вываливаемся, то попадаем counters.Aborted = -1; throw new Exception(); } BatchUtil.ExecBatched( () => ClearBatch(connection, _batchParams, _targetTable, key, counters), _batchParams, counters); } catch { //Если удалить не удалось, то пытаемся сделать это еще раз (флаг означает, что ) counters.Aborted = -1; } finally { //if (counters.Aborted == 0) // LockerManager._releaseAppLock(connection); } } }
public static LoadCounters ClearData(string format, object key) { if (string.IsNullOrEmpty(format)) throw new ArgumentException("csv data format doesn't specified", "format"); if (key == null) throw new ArgumentNullException("key"); if (format != _prices && format != _priceFactors) { throw new InvalidOperationException(format + " not supported"); } LoadCounters counters = new LoadCounters(); (new PricesCleaner(GetSettings(format))).ClearData(key, counters); return counters; }
protected override bool ClearBatch(SqlConnection connection, BatchParams batchParams, string targetTable, object key, LoadCounters counters) { if (!(key is int)) throw new ArgumentException("SupplierID key must be integer", "key"); using (SqlCommand cmd = new SqlCommand(string.Format( "DELETE " + (batchParams.Size == 0 ? "" : "TOP({0}) ") + "FROM {1} WHERE SupplierID = @SupplierID", batchParams.Size, targetTable), connection)) { cmd.CommandTimeout = batchParams.Timeout; cmd.Parameters.AddWithValue("@SupplierID", key); int rowsAffected = cmd.ExecuteNonQuery(); counters.Deleted += rowsAffected; return rowsAffected >= batchParams.Size && batchParams.Size > 0; //batchParams.Size = 0 works as a single batch } }
public static void ExecBatched(Func<bool> processBatch, BatchParams batchParams, LoadCounters counters) { Stopwatch sw = Stopwatch.StartNew(); while (processBatch()) { sw.Stop(); int delay = batchParams.Delay; if (batchParams.DelayMode == DelayMode.Percentage) { delay = (int)(sw.ElapsedMilliseconds * delay / 100m); } counters.BatchCount++; counters.BatchTime += sw.Elapsed; counters.DelayTime += new TimeSpan(0, 0, 0, 0, delay); System.Threading.Thread.Sleep(delay); sw.Reset(); sw.Start(); } }
protected override bool LoadBatch(IDataReader reader, BatchParams batchParams, LoadCounters rowCounters) { LoadCounters impCounters = new LoadCounters(); bool aborted; int deleted; CreateTempTable(_TargetTable, _dataTable); try { //вставка ключей удаляемых записей во временную таблицу aborted = base.LoadBatch(reader, batchParams, impCounters); //удаление данных на основе содержимого временной таблицы deleted = this.DeleteData(batchParams, _TargetTable, _dataTable); //подсчет строк rowCounters.Deleted += deleted; rowCounters.Skipped += (impCounters.Added - deleted); } finally { DropTempTable(_TargetTable); } return aborted; }
protected override void DoWorkInternal() { AlarmFile alarmFile; while ((alarmFile = DequeueAlarmFile()) != null) { Encoding encoding = Encoding.GetEncoding(Configuration.LogEncoding); using (var log = new StreamWriter( Path.ChangeExtension(alarmFile.FileInfo.FullName, _LogExt), false, encoding)) { //log.WriteLine( alarmFile.Extension.Substring( 1 ) ); switch (alarmFile.FileType) { case AlarmFile.AlarmFileType.Import: log.WriteLine("imp"); break; case AlarmFile.AlarmFileType.Delete: log.WriteLine("del"); break; default: throw new Exception("Unknown alarm file type: " + alarmFile.FileType.ToString()); } LoadCounters counters = new LoadCounters(); int rowcount = 0; int skiprowcount = 0; int errorcode = 0; FileInfo dataFile = null; if (_traceMode) //TODO: Необходимо записывать тип события в лог LogMessage("Start processing {0} file \"{1}\".", this.ServiceName, alarmFile.ImportEntity, alarmFile.FileInfo.Name); try { switch (alarmFile.ImportEntity) { //Тот же самый case ImportEntity.PriceFactors. case ImportEntity.Prices: case ImportEntity.PriceFactors: if (alarmFile.FileType == AlarmFile.AlarmFileType.Import) { dataFile = PickUpDataFile(alarmFile.FileInfo); //Вернуть флаг, что отвалились counters = ProcessImportTask( alarmFile.ImportEntity, ImportMode.BulkInsert, dataFile, Path.ChangeExtension(alarmFile.FileInfo.FullName, _ErrExt), ref rowcount, ref skiprowcount, ref errorcode); } else { counters = ProcessDeleteTask(alarmFile.ImportEntity, alarmFile.FileInfo, ref rowcount); } break; case ImportEntity.Crosses: case ImportEntity.CrossesBrands: case ImportEntity.CrossesGroups: case ImportEntity.CrossesLinks: dataFile = PickUpDataFile(alarmFile.FileInfo); counters = ProcessImportTask( alarmFile.ImportEntity, alarmFile.FileType == AlarmFile.AlarmFileType.Import ? ImportMode.BulkInsertWithSkipped : ImportMode.BulkDelete, dataFile, Path.ChangeExtension(alarmFile.FileInfo.FullName, _ErrExt), ref rowcount, ref skiprowcount, ref errorcode); break; } } catch (Exception ex) { LogException(ex, log); errorcode = 2900; } finally { //проверить counters.Aborted и не удалять если отвалились if (counters.Aborted == 0) { log.WriteLine("rowcount={0}", rowcount); if (alarmFile.ImportEntity == ImportEntity.Crosses) log.WriteLine("skiprowcount={0}", skiprowcount); log.WriteLine("errorcode={0}", errorcode); alarmFile.Delete(); if (dataFile != null) { DeleteDataFile(dataFile); } } if (_traceMode) { LogMessage("Finished processing {0} file \"{1}\" (ErrorCode = {2}, " + "RowCount = {3}, BatchCount = {4}, BatchTime = {5}, DelayTime = {6}).", this.ServiceName, alarmFile.ImportEntity, alarmFile.FileInfo.Name, errorcode, rowcount + skiprowcount, counters.BatchCount, counters.BatchTime, counters.DelayTime); } } } } }
protected abstract bool LoadBatch(IDataReader reader, BatchParams batchParams, LoadCounters rowCounters);
public void LoadData(IDataReader reader, out LoadCounters rowCounters) { if (_disposed) throw new ObjectDisposedException("DataLoaderBase"); if (reader == null) throw new ArgumentNullException("reader"); var counters = new LoadCounters(); BatchUtil.ExecBatched( () => LoadBatch(reader, _batchParams, counters), _batchParams, counters); rowCounters = counters; }
protected abstract bool ClearBatch(SqlConnection connection, BatchParams batchParams, string targetTable, object key, LoadCounters counters);