internal DefaultKclProcess(IRecordProcessor processor, IoHandler iohandler)
 {
     _processor = processor;
     _iohandler = iohandler;
     _checkpointer = new InternalCheckpointer(this);
     ConfigureStateMachine();
 }
 internal DefaultKclProcess(IRecordProcessor processor, IoHandler iohandler)
 {
     _processor    = processor;
     _iohandler    = iohandler;
     _checkpointer = new InternalCheckpointer(this);
     ConfigureStateMachine();
 }
        public async Task <Result> Start(IRecordProcessor recordProcessor, string streamName = null)
        {
            if (_isRunning)
            {
                return(Result.Create(false, "KinesisNet has already started. Please stop it first."));
            }

            if (!string.IsNullOrEmpty(streamName))
            {
                _utilities.SetStreamName(streamName);
            }

            if (string.IsNullOrEmpty(_utilities.StreamName))
            {
                return(Result.Create(false, "Please set a stream name."));
            }

            await _dynamoDb.Init();

            _isRunning = true;
            _currentRecordsProcessing = 0;
            _cancellationTokenSource  = new CancellationTokenSource();


            Log.Debug("Beginning to process shards");
            await Task.Run(() => ProcessShardsAsync(recordProcessor));

            return(Result.Create(true, "Processing System"));
        }
Beispiel #4
0
        private async Task ProcessShardsAsync(IRecordProcessor processor)
        {
            try
            {
                var processShardsTask = new ConcurrentDictionary <Task <RecordResponse>, KShard>();

                await WatchForShardUpdates(processShardsTask);

                while (!_cancellationTokenSource.IsCancellationRequested)
                {
                    if (processShardsTask.Count > 0)
                    {
                        var task = await Task.WhenAny(processShardsTask.Select(m => m.Key)).WithCancellation(_cancellationTokenSource.Token);

                        KShard shard;
                        if (task != null && task.IsCompleted && !task.IsCanceled && !task.IsFaulted)
                        {
                            Interlocked.Add(ref _currentRecordsProcessing, 1);

                            if (processShardsTask.TryRemove(task, out shard))
                            {
                                var record = await task;

                                if (record != null && record.GetRecordsResponse.Records.Any())
                                {
                                    processor.Process(shard.ShardId, record.GetRecordsResponse.Records.LastOrDefault().SequenceNumber, shard.LastUpdateUtc, record.GetRecordsResponse.Records, SaveCheckpoint);

                                    if (record.GetRecordsResponse.NextShardIterator != null)
                                    {
                                        shard.UpdateShardInformation(record.GetRecordsResponse);

                                        var getRecordsTask = GetRecordResponse(shard, record.CancellationToken);

                                        processShardsTask.TryAdd(getRecordsTask, shard);
                                    }
                                }
                            }

                            Interlocked.Decrement(ref _currentRecordsProcessing);
                        }
                        else //the task was cancelled or faulted or there is some error
                        {
                            if (task != null)
                            {
                                processShardsTask.TryRemove(task, out shard);
                            }
                        }
                    }
                    else
                    {
                        await Task.Delay(2000); //sleep and hope that we get some cool shards to work with
                    }
                }
            }
            catch (OperationCanceledException e)
            {
                Log.Debug(e, "ProcessShardsAsync was cancelled");
            }
        }
 public void SetRecordProcessor(string filename)
 {
     if (rp != null && rp.GetType().Name == Path.GetFileNameWithoutExtension(filename))
     {
         return;                  //already loaded
     }
     rp = TableProcessor.LoadRecordProcessor(filename);
 }
Beispiel #6
0
 public DynamoStreamReader(
     IAmazonDynamoDB dynamoDb,
     IAmazonDynamoDBStreams dynamoDbStreams,
     IRecordProcessor <StreamRecord> recordProcessor,
     ILogger <Pump> logger = null)
 {
     _dynamoDb        = dynamoDb;
     _dynamoDbStreams = dynamoDbStreams;
     _recordProcessor = recordProcessor;
     _logger          = logger;
 }
 public void Process(IRecordProcessor rp)
 {
     //Try to copy original to output database
     if (ProcessMode == ProcessMode.pmEdit)
     {
         ProcessEditMode(rp);
     }
     else
     {
         ProcessETLMode(rp);
     }
 }
Beispiel #8
0
 public RawMeasurementsKinesisShardReader(IAmazonKinesis client,
                                          ICheckpointRepository checkpointRepository,
                                          IHeartbeatService heartbeatService,
                                          IProcessedMeasurementStreamWriter processedMeasurementStreamWriter,
                                          IRecordProcessor recordProcessor,
                                          ILogger logger)
     : base(client, checkpointRepository, heartbeatService)
 {
     _processedMeasurementStreamWriter = processedMeasurementStreamWriter;
     _recordProcessor = recordProcessor;
     _logger          = logger;
 }
        public void ProcessETLMode(IRecordProcessor rp)
        {
            inputDatabase.Connect();
            MapFields(rp);

            rp.setDatasetName(inputDatabase.NormTableName(tableName));


            string[] sqls = rp.GetSelectStatements();

            if (Progress != null)
            {
                Progress.ReportProgress(0);
            }
            foreach (string sql in sqls)
            {
                try
                {
                    DbDataReader reader = inputDatabase.Execute(sql);
                    int          count  = 1;
                    rp.Reset();
                    //Extract
                    while (reader.Read())
                    {
                        //Transform
                        rp.ProcessRecord(reader);
                        if (Progress != null)
                        {
                            Progress.ReportProgress(0, "Read records: " + count.ToString());
                        }
                        count++;
                    }
                    reader.Close();
                    reader = null;

                    //Load
                    if (outputDatabase.FileName != null)
                    {
                        //if output is file - Write one
                        WriteData();
                    }
                }
                catch (Exception ex)
                {
                    throw ex;
                }
                // Call Close when done reading.
            }
            inputDatabase.Close();
        }
        void MapFields(IRecordProcessor rp)
        {
            Dictionary <string, int> inputFields = inputDatabase.GetFields(tableName);
            Dictionary <string, int> fieldsMap   = new Dictionary <string, int>();

            foreach (KeyValuePair <string, string> fieldpair in inputFieldNamesMap)
            {
                string transferColName = fieldpair.Value;
                if (fieldpair.Key.Length > 0)
                {
                    int fieldIndex = inputFields[fieldpair.Key];
                    fieldsMap[transferColName] = fieldIndex;
                }
            }
            rp.setFieldsMap(fieldsMap);
        }
        public static IRecordProcessor LoadRecordProcessor(Assembly Dynamic)
        {
            Type TypeofIRecordProcessor = typeof(IRecordProcessor);

            foreach (Type Exported in Dynamic.GetExportedTypes())
            {
                if (TypeofIRecordProcessor.IsAssignableFrom(Exported))
                {
                    //return (IRecordProcessor)Activator.CreateInstance(InputElementClass);
                    ConstructorInfo Constructor = Exported.GetConstructor(System.Type.EmptyTypes);
                    if (Constructor == null)
                    {
                        throw new Exception("Failed to bind " + Dynamic.FullName + ". Classes has no proper Constructor");
                    }

                    // Type supports IMatch and has an no-parameter constructor
                    IRecordProcessor Instance = (IRecordProcessor)Constructor.Invoke(System.Type.EmptyTypes);
                    return(Instance);
                }
            }
            throw new Exception("Failed to bind " + Dynamic.FullName + ". No acceptable Classes found");
        }
 /// <summary>
 /// Create an instance of KclProcess that uses the given IRecordProcessor to process records.
 /// </summary>
 /// <param name="recordProcessor">IRecordProcessor used to process records.</param>
 public static KclProcess Create(IRecordProcessor recordProcessor)
 {
     return Create(recordProcessor, new IoHandler());
 }
 internal static KclProcess Create(IRecordProcessor recordProcessor, IoHandler ioHandler)
 {
     return new DefaultKclProcess(recordProcessor, ioHandler);
 }
 internal static KclProcess Create(IRecordProcessor recordProcessor, IoHandler ioHandler)
 {
     return(new DefaultKclProcess(recordProcessor, ioHandler));
 }
        public void ProcessEditMode(IRecordProcessor rp)
        {
            inputDatabase.Connect();
            outputDatabase.Connect();
            MapFields(rp);
            rp.setDatasetName(inputDatabase.NormTableName(tableName));

            if (Progress != null)
            {
                Progress.ReportProgress(0);
            }
            try
            {
                int count          = ProcessOffset;
                int processedCount = 0;
                int totalCount;

                rp.Reset();

                //Extract
                inputDatabase.SetTable(tableName);
                outputDatabase.SetTable(tableName);

                totalCount = inputDatabase.GetRowsCount();

                if (ProcessOffset > 0)
                {
                    inputDatabase.SeekRecord(ProcessOffset);
                }
                else
                {
                    inputDatabase.FirstRecord();
                }

                //!! Skip first line as it Header
                while (inputDatabase.NextRecord())
                {
                    DataRow row = inputDatabase.CurrentRow();

                    //Transform
                    if (rp.ProcessRow(row))
                    {
                        //Load if row changed
                        outputDatabase.UpdateRow(count + 1, row);

                        if (Progress != null)
                        {
                            Progress.ReportProgress((processedCount * 100) / (totalCount - ProcessOffset), string.Format("Records: Read {0} (Processed {1}) of {2}", count, processedCount, totalCount));
                        }

                        processedCount++;
                    }
                    count++;
                }

                //Load
                if (outputDatabase.FileName != null)
                {
                    outputDatabase.Write(outputDatabase.FileName);
                }
                //Set complete
                Progress.ReportProgress(100, string.Format("Read records: {0}, Processed {1}", count, processedCount));
            }
            catch (Exception ex)
            {
                throw ex;
            }
            // Call Close when done reading.

            inputDatabase.Close();
            outputDatabase.Close();
        }
 /// <summary>
 /// Create an instance of KclProcess that uses the given IRecordProcessor to process records.
 /// </summary>
 /// <param name="recordProcessor">IRecordProcessor used to process records.</param>
 public static KclProcess Create(IRecordProcessor recordProcessor)
 {
     return(Create(recordProcessor, new IoHandler()));
 }
Beispiel #17
0
 /// <summary>
 /// Конструктор парсера записей FTP-сервера.
 /// </summary>
 /// <param name="processor">Обработчик строки</param>
 public DirectoryListParser(IRecordProcessor processor)
 {
     _recordProcessor = processor;
 }
 /// <summary>
 /// Create an instance of KclProcess that uses the given IRecordProcessor to process records.
 /// </summary>
 /// <param name="recordProcessor">IRecordProcessor used to process records.</param>
 public static KclProcess Create(IRecordProcessor recordProcessor)
 {
     #pragma warning disable CC0022 // Should dispose object
     return Create(recordProcessor, new IoHandler());
     #pragma warning restore CC0022 // Should dispose object
 }
 private void SetRecordProcessor(Assembly irp)
 {
     rp = TableProcessor.LoadRecordProcessor(irp);
 }
Beispiel #20
0
 internal ShardRecordProcessorToRecordProcessor(IRecordProcessor recordProcessor)
 {
     RecordProcessor = recordProcessor;
 }