Example #1
0
        protected override bool LoadBatch(IDataReader reader, BatchParams batchParams, LoadCounters rowCounters)
        {
            bool read = true;
            //batch size = 0 works as a single batch since --batchParams.Size != 0 will be never reached
            while (--batchParams.Size != 0 && (read = reader.Read()))
            {
                try
                {
                    if (RowExists(reader))
                    {
                        if (_updateCmd != null)
                        {
                            DoUpdate(reader);
                            rowCounters.Updated++;
                        }
                    }
                    else
                    {
                        DoInsert(reader);
                        rowCounters.Added++;
                    }
                }
                catch (Exception ex)
                {
                    OnLoadError(ex);
                }
            }

            return read;
        }
Example #2
0
        public void ClearData(object key, LoadCounters counters)
        {
            using (var connection = new SqlConnection(_connString))
            {
                connection.Open();

                try
                {
                    //Проверка в списке собственных складов
                    var cnt = LockerManager.CheckOwnStore((int)key, connection);

                    //Если поставщик находится в собственных складах наличия то пытаемся получить лок на базу данных, если не получаем то вываливаемся с исключением
                    if (cnt > 0 && LockerManager.AquireCheckLock(connection) < 0)
                    {
                        //Если здесь вываливаемся, то попадаем
                        counters.Aborted = -1;
                        throw new Exception();
                    }

                    BatchUtil.ExecBatched(
                        () => ClearBatch(connection, _batchParams, _targetTable, key, counters),
                        _batchParams, counters);
                }
                catch
                {
                    //Если удалить не удалось, то пытаемся сделать это еще раз (флаг означает, что )
                    counters.Aborted = -1;
                }
                finally
                {
                    //if (counters.Aborted == 0)
                       // LockerManager._releaseAppLock(connection);
                }
            }
        }
Example #3
0
        public static LoadCounters ClearData(string format, object key)
        {
            if (string.IsNullOrEmpty(format))
                throw new ArgumentException("csv data format doesn't specified", "format");
            if (key == null)
                throw new ArgumentNullException("key");

            if (format != _prices && format != _priceFactors)
            {
                throw new InvalidOperationException(format + " not supported");
            }

            LoadCounters counters = new LoadCounters();
            (new PricesCleaner(GetSettings(format))).ClearData(key, counters);
            return counters;
        }
Example #4
0
        protected override bool ClearBatch(SqlConnection connection,
            BatchParams batchParams, string targetTable, object key, LoadCounters counters)
        {
            if (!(key is int)) throw new
                ArgumentException("SupplierID key must be integer", "key");

            using (SqlCommand cmd = new SqlCommand(string.Format(
                "DELETE " + (batchParams.Size == 0 ? "" : "TOP({0}) ") +
                "FROM {1} WHERE SupplierID = @SupplierID",
                batchParams.Size, targetTable), connection))
            {
                cmd.CommandTimeout = batchParams.Timeout;
                cmd.Parameters.AddWithValue("@SupplierID", key);
                int rowsAffected = cmd.ExecuteNonQuery();

                counters.Deleted += rowsAffected;
                return rowsAffected >= batchParams.Size && batchParams.Size > 0;
                //batchParams.Size = 0 works as a single batch
            }
        }
Example #5
0
        public static void ExecBatched(Func<bool> processBatch,
            BatchParams batchParams, LoadCounters counters)
        {
            Stopwatch sw = Stopwatch.StartNew();
            while (processBatch())
            {
                sw.Stop();
                int delay = batchParams.Delay;
                if (batchParams.DelayMode == DelayMode.Percentage)
                {
                    delay = (int)(sw.ElapsedMilliseconds * delay / 100m);
                }
                counters.BatchCount++;
                counters.BatchTime += sw.Elapsed;
                counters.DelayTime += new TimeSpan(0, 0, 0, 0, delay);

                System.Threading.Thread.Sleep(delay);

                sw.Reset();
                sw.Start();
            }
        }
Example #6
0
        protected override bool LoadBatch(IDataReader reader, BatchParams batchParams, LoadCounters rowCounters)
        {
            LoadCounters impCounters = new LoadCounters();
            bool aborted;
            int deleted;

            CreateTempTable(_TargetTable, _dataTable);
            try
            {
                //вставка ключей удаляемых записей во временную таблицу
                aborted = base.LoadBatch(reader, batchParams, impCounters);
                //удаление данных на основе содержимого временной таблицы
                deleted = this.DeleteData(batchParams, _TargetTable, _dataTable);
                //подсчет строк
                rowCounters.Deleted += deleted;
                rowCounters.Skipped += (impCounters.Added - deleted);
            }
            finally
            {
                DropTempTable(_TargetTable);
            }

            return aborted;
        }
Example #7
0
        protected override void DoWorkInternal()
        {
            AlarmFile alarmFile;

                while ((alarmFile = DequeueAlarmFile()) != null)
                {
                    Encoding encoding = Encoding.GetEncoding(Configuration.LogEncoding);
                    using (var log = new StreamWriter(
                        Path.ChangeExtension(alarmFile.FileInfo.FullName, _LogExt), false, encoding))
                    {
                        //log.WriteLine( alarmFile.Extension.Substring( 1 ) );
                        switch (alarmFile.FileType)
                        {
                            case AlarmFile.AlarmFileType.Import: log.WriteLine("imp"); break;
                            case AlarmFile.AlarmFileType.Delete: log.WriteLine("del"); break;
                            default: throw new Exception("Unknown alarm file type: " + alarmFile.FileType.ToString());
                        }

                        LoadCounters counters = new LoadCounters();
                        int rowcount = 0;
                        int skiprowcount = 0;
                        int errorcode = 0;
                        FileInfo dataFile = null;

                        if (_traceMode)
                            //TODO: Необходимо записывать тип события в лог
                            LogMessage("Start processing {0} file \"{1}\".", this.ServiceName, alarmFile.ImportEntity, alarmFile.FileInfo.Name);
                        try
                        {
                            switch (alarmFile.ImportEntity)
                            {
                                //Тот же самый case ImportEntity.PriceFactors.
                                case ImportEntity.Prices:
                                case ImportEntity.PriceFactors:
                                    if (alarmFile.FileType == AlarmFile.AlarmFileType.Import)
                                    {
                                        dataFile = PickUpDataFile(alarmFile.FileInfo);
                                        //Вернуть флаг, что отвалились
                                        counters = ProcessImportTask(
                                            alarmFile.ImportEntity,
                                            ImportMode.BulkInsert,
                                            dataFile,
                                            Path.ChangeExtension(alarmFile.FileInfo.FullName, _ErrExt),
                                            ref rowcount, ref skiprowcount, ref errorcode);
                                    }
                                    else
                                    {
                                        counters = ProcessDeleteTask(alarmFile.ImportEntity, alarmFile.FileInfo, ref rowcount);
                                    }
                                    break;

                                case ImportEntity.Crosses:
                                case ImportEntity.CrossesBrands:
                                case ImportEntity.CrossesGroups:
                                case ImportEntity.CrossesLinks:

                                    dataFile = PickUpDataFile(alarmFile.FileInfo);
                                    counters = ProcessImportTask(
                                        alarmFile.ImportEntity,
                                        alarmFile.FileType == AlarmFile.AlarmFileType.Import ? ImportMode.BulkInsertWithSkipped : ImportMode.BulkDelete,
                                        dataFile,
                                        Path.ChangeExtension(alarmFile.FileInfo.FullName, _ErrExt),
                                        ref rowcount, ref skiprowcount, ref errorcode);

                                    break;
                            }

                        }
                        catch (Exception ex)
                        {
                            LogException(ex, log);
                            errorcode = 2900;
                        }
                        finally
                        {

                            //проверить counters.Aborted и не удалять если отвалились
                            if (counters.Aborted == 0)
                            {
                                log.WriteLine("rowcount={0}", rowcount);
                                if (alarmFile.ImportEntity == ImportEntity.Crosses)
                                    log.WriteLine("skiprowcount={0}", skiprowcount);
                                log.WriteLine("errorcode={0}", errorcode);

                                alarmFile.Delete();

                                if (dataFile != null)
                                {
                                    DeleteDataFile(dataFile);
                                }
                            }

                            if (_traceMode)
                            {
                                LogMessage("Finished processing {0} file \"{1}\" (ErrorCode = {2}, " +
                                "RowCount = {3}, BatchCount = {4}, BatchTime = {5}, DelayTime = {6}).", this.ServiceName,
                                alarmFile.ImportEntity, alarmFile.FileInfo.Name, errorcode, rowcount + skiprowcount,
                                counters.BatchCount, counters.BatchTime, counters.DelayTime);
                            }
                        }
                    }
                }
        }
Example #8
0
 protected abstract bool LoadBatch(IDataReader reader,
     BatchParams batchParams, LoadCounters rowCounters);
Example #9
0
        public void LoadData(IDataReader reader, out LoadCounters rowCounters)
        {
            if (_disposed)
                throw new ObjectDisposedException("DataLoaderBase");
            if (reader == null)
                throw new ArgumentNullException("reader");

            var counters = new LoadCounters();

            BatchUtil.ExecBatched(
                () => LoadBatch(reader, _batchParams, counters),
                _batchParams, counters);

            rowCounters = counters;
        }
Example #10
0
 protected abstract bool ClearBatch(SqlConnection connection,
     BatchParams batchParams, string targetTable, object key, LoadCounters counters);