internal DefaultKclProcess(IRecordProcessor processor, IoHandler iohandler) { _processor = processor; _iohandler = iohandler; _checkpointer = new InternalCheckpointer(this); ConfigureStateMachine(); }
public async Task <Result> Start(IRecordProcessor recordProcessor, string streamName = null) { if (_isRunning) { return(Result.Create(false, "KinesisNet has already started. Please stop it first.")); } if (!string.IsNullOrEmpty(streamName)) { _utilities.SetStreamName(streamName); } if (string.IsNullOrEmpty(_utilities.StreamName)) { return(Result.Create(false, "Please set a stream name.")); } await _dynamoDb.Init(); _isRunning = true; _currentRecordsProcessing = 0; _cancellationTokenSource = new CancellationTokenSource(); Log.Debug("Beginning to process shards"); await Task.Run(() => ProcessShardsAsync(recordProcessor)); return(Result.Create(true, "Processing System")); }
private async Task ProcessShardsAsync(IRecordProcessor processor) { try { var processShardsTask = new ConcurrentDictionary <Task <RecordResponse>, KShard>(); await WatchForShardUpdates(processShardsTask); while (!_cancellationTokenSource.IsCancellationRequested) { if (processShardsTask.Count > 0) { var task = await Task.WhenAny(processShardsTask.Select(m => m.Key)).WithCancellation(_cancellationTokenSource.Token); KShard shard; if (task != null && task.IsCompleted && !task.IsCanceled && !task.IsFaulted) { Interlocked.Add(ref _currentRecordsProcessing, 1); if (processShardsTask.TryRemove(task, out shard)) { var record = await task; if (record != null && record.GetRecordsResponse.Records.Any()) { processor.Process(shard.ShardId, record.GetRecordsResponse.Records.LastOrDefault().SequenceNumber, shard.LastUpdateUtc, record.GetRecordsResponse.Records, SaveCheckpoint); if (record.GetRecordsResponse.NextShardIterator != null) { shard.UpdateShardInformation(record.GetRecordsResponse); var getRecordsTask = GetRecordResponse(shard, record.CancellationToken); processShardsTask.TryAdd(getRecordsTask, shard); } } } Interlocked.Decrement(ref _currentRecordsProcessing); } else //the task was cancelled or faulted or there is some error { if (task != null) { processShardsTask.TryRemove(task, out shard); } } } else { await Task.Delay(2000); //sleep and hope that we get some cool shards to work with } } } catch (OperationCanceledException e) { Log.Debug(e, "ProcessShardsAsync was cancelled"); } }
public void SetRecordProcessor(string filename) { if (rp != null && rp.GetType().Name == Path.GetFileNameWithoutExtension(filename)) { return; //already loaded } rp = TableProcessor.LoadRecordProcessor(filename); }
public DynamoStreamReader( IAmazonDynamoDB dynamoDb, IAmazonDynamoDBStreams dynamoDbStreams, IRecordProcessor <StreamRecord> recordProcessor, ILogger <Pump> logger = null) { _dynamoDb = dynamoDb; _dynamoDbStreams = dynamoDbStreams; _recordProcessor = recordProcessor; _logger = logger; }
public void Process(IRecordProcessor rp) { //Try to copy original to output database if (ProcessMode == ProcessMode.pmEdit) { ProcessEditMode(rp); } else { ProcessETLMode(rp); } }
public RawMeasurementsKinesisShardReader(IAmazonKinesis client, ICheckpointRepository checkpointRepository, IHeartbeatService heartbeatService, IProcessedMeasurementStreamWriter processedMeasurementStreamWriter, IRecordProcessor recordProcessor, ILogger logger) : base(client, checkpointRepository, heartbeatService) { _processedMeasurementStreamWriter = processedMeasurementStreamWriter; _recordProcessor = recordProcessor; _logger = logger; }
public void ProcessETLMode(IRecordProcessor rp) { inputDatabase.Connect(); MapFields(rp); rp.setDatasetName(inputDatabase.NormTableName(tableName)); string[] sqls = rp.GetSelectStatements(); if (Progress != null) { Progress.ReportProgress(0); } foreach (string sql in sqls) { try { DbDataReader reader = inputDatabase.Execute(sql); int count = 1; rp.Reset(); //Extract while (reader.Read()) { //Transform rp.ProcessRecord(reader); if (Progress != null) { Progress.ReportProgress(0, "Read records: " + count.ToString()); } count++; } reader.Close(); reader = null; //Load if (outputDatabase.FileName != null) { //if output is file - Write one WriteData(); } } catch (Exception ex) { throw ex; } // Call Close when done reading. } inputDatabase.Close(); }
void MapFields(IRecordProcessor rp) { Dictionary <string, int> inputFields = inputDatabase.GetFields(tableName); Dictionary <string, int> fieldsMap = new Dictionary <string, int>(); foreach (KeyValuePair <string, string> fieldpair in inputFieldNamesMap) { string transferColName = fieldpair.Value; if (fieldpair.Key.Length > 0) { int fieldIndex = inputFields[fieldpair.Key]; fieldsMap[transferColName] = fieldIndex; } } rp.setFieldsMap(fieldsMap); }
public static IRecordProcessor LoadRecordProcessor(Assembly Dynamic) { Type TypeofIRecordProcessor = typeof(IRecordProcessor); foreach (Type Exported in Dynamic.GetExportedTypes()) { if (TypeofIRecordProcessor.IsAssignableFrom(Exported)) { //return (IRecordProcessor)Activator.CreateInstance(InputElementClass); ConstructorInfo Constructor = Exported.GetConstructor(System.Type.EmptyTypes); if (Constructor == null) { throw new Exception("Failed to bind " + Dynamic.FullName + ". Classes has no proper Constructor"); } // Type supports IMatch and has an no-parameter constructor IRecordProcessor Instance = (IRecordProcessor)Constructor.Invoke(System.Type.EmptyTypes); return(Instance); } } throw new Exception("Failed to bind " + Dynamic.FullName + ". No acceptable Classes found"); }
/// <summary> /// Create an instance of KclProcess that uses the given IRecordProcessor to process records. /// </summary> /// <param name="recordProcessor">IRecordProcessor used to process records.</param> public static KclProcess Create(IRecordProcessor recordProcessor) { return Create(recordProcessor, new IoHandler()); }
internal static KclProcess Create(IRecordProcessor recordProcessor, IoHandler ioHandler) { return new DefaultKclProcess(recordProcessor, ioHandler); }
internal static KclProcess Create(IRecordProcessor recordProcessor, IoHandler ioHandler) { return(new DefaultKclProcess(recordProcessor, ioHandler)); }
public void ProcessEditMode(IRecordProcessor rp) { inputDatabase.Connect(); outputDatabase.Connect(); MapFields(rp); rp.setDatasetName(inputDatabase.NormTableName(tableName)); if (Progress != null) { Progress.ReportProgress(0); } try { int count = ProcessOffset; int processedCount = 0; int totalCount; rp.Reset(); //Extract inputDatabase.SetTable(tableName); outputDatabase.SetTable(tableName); totalCount = inputDatabase.GetRowsCount(); if (ProcessOffset > 0) { inputDatabase.SeekRecord(ProcessOffset); } else { inputDatabase.FirstRecord(); } //!! Skip first line as it Header while (inputDatabase.NextRecord()) { DataRow row = inputDatabase.CurrentRow(); //Transform if (rp.ProcessRow(row)) { //Load if row changed outputDatabase.UpdateRow(count + 1, row); if (Progress != null) { Progress.ReportProgress((processedCount * 100) / (totalCount - ProcessOffset), string.Format("Records: Read {0} (Processed {1}) of {2}", count, processedCount, totalCount)); } processedCount++; } count++; } //Load if (outputDatabase.FileName != null) { outputDatabase.Write(outputDatabase.FileName); } //Set complete Progress.ReportProgress(100, string.Format("Read records: {0}, Processed {1}", count, processedCount)); } catch (Exception ex) { throw ex; } // Call Close when done reading. inputDatabase.Close(); outputDatabase.Close(); }
/// <summary> /// Create an instance of KclProcess that uses the given IRecordProcessor to process records. /// </summary> /// <param name="recordProcessor">IRecordProcessor used to process records.</param> public static KclProcess Create(IRecordProcessor recordProcessor) { return(Create(recordProcessor, new IoHandler())); }
/// <summary> /// Конструктор парсера записей FTP-сервера. /// </summary> /// <param name="processor">Обработчик строки</param> public DirectoryListParser(IRecordProcessor processor) { _recordProcessor = processor; }
/// <summary> /// Create an instance of KclProcess that uses the given IRecordProcessor to process records. /// </summary> /// <param name="recordProcessor">IRecordProcessor used to process records.</param> public static KclProcess Create(IRecordProcessor recordProcessor) { #pragma warning disable CC0022 // Should dispose object return Create(recordProcessor, new IoHandler()); #pragma warning restore CC0022 // Should dispose object }
private void SetRecordProcessor(Assembly irp) { rp = TableProcessor.LoadRecordProcessor(irp); }
internal ShardRecordProcessorToRecordProcessor(IRecordProcessor recordProcessor) { RecordProcessor = recordProcessor; }