internal DefaultKclProcess(IRecordProcessor processor, IoHandler iohandler)
{
_processor = processor;
_iohandler = iohandler;
_checkpointer = new InternalCheckpointer(this);
ConfigureStateMachine();
}
internal DefaultKclProcess(IRecordProcessor processor, IoHandler iohandler)
{
_processor = processor;
_iohandler = iohandler;
_checkpointer = new InternalCheckpointer(this);
ConfigureStateMachine();
}
public async Task <Result> Start(IRecordProcessor recordProcessor, string streamName = null)
{
if (_isRunning)
{
return(Result.Create(false, "KinesisNet has already started. Please stop it first."));
}
if (!string.IsNullOrEmpty(streamName))
{
_utilities.SetStreamName(streamName);
}
if (string.IsNullOrEmpty(_utilities.StreamName))
{
return(Result.Create(false, "Please set a stream name."));
}
await _dynamoDb.Init();
_isRunning = true;
_currentRecordsProcessing = 0;
_cancellationTokenSource = new CancellationTokenSource();
Log.Debug("Beginning to process shards");
await Task.Run(() => ProcessShardsAsync(recordProcessor));
return(Result.Create(true, "Processing System"));
}
private async Task ProcessShardsAsync(IRecordProcessor processor)
{
try
{
var processShardsTask = new ConcurrentDictionary <Task <RecordResponse>, KShard>();
await WatchForShardUpdates(processShardsTask);
while (!_cancellationTokenSource.IsCancellationRequested)
{
if (processShardsTask.Count > 0)
{
var task = await Task.WhenAny(processShardsTask.Select(m => m.Key)).WithCancellation(_cancellationTokenSource.Token);
KShard shard;
if (task != null && task.IsCompleted && !task.IsCanceled && !task.IsFaulted)
{
Interlocked.Add(ref _currentRecordsProcessing, 1);
if (processShardsTask.TryRemove(task, out shard))
{
var record = await task;
if (record != null && record.GetRecordsResponse.Records.Any())
{
processor.Process(shard.ShardId, record.GetRecordsResponse.Records.LastOrDefault().SequenceNumber, shard.LastUpdateUtc, record.GetRecordsResponse.Records, SaveCheckpoint);
if (record.GetRecordsResponse.NextShardIterator != null)
{
shard.UpdateShardInformation(record.GetRecordsResponse);
var getRecordsTask = GetRecordResponse(shard, record.CancellationToken);
processShardsTask.TryAdd(getRecordsTask, shard);
}
}
}
Interlocked.Decrement(ref _currentRecordsProcessing);
}
else //the task was cancelled or faulted or there is some error
{
if (task != null)
{
processShardsTask.TryRemove(task, out shard);
}
}
}
else
{
await Task.Delay(2000); //sleep and hope that we get some cool shards to work with
}
}
}
catch (OperationCanceledException e)
{
Log.Debug(e, "ProcessShardsAsync was cancelled");
}
}
public void SetRecordProcessor(string filename)
{
if (rp != null && rp.GetType().Name == Path.GetFileNameWithoutExtension(filename))
{
return; //already loaded
}
rp = TableProcessor.LoadRecordProcessor(filename);
}
public DynamoStreamReader(
IAmazonDynamoDB dynamoDb,
IAmazonDynamoDBStreams dynamoDbStreams,
IRecordProcessor <StreamRecord> recordProcessor,
ILogger <Pump> logger = null)
{
_dynamoDb = dynamoDb;
_dynamoDbStreams = dynamoDbStreams;
_recordProcessor = recordProcessor;
_logger = logger;
}
public void Process(IRecordProcessor rp)
{
//Try to copy original to output database
if (ProcessMode == ProcessMode.pmEdit)
{
ProcessEditMode(rp);
}
else
{
ProcessETLMode(rp);
}
}
public RawMeasurementsKinesisShardReader(IAmazonKinesis client,
ICheckpointRepository checkpointRepository,
IHeartbeatService heartbeatService,
IProcessedMeasurementStreamWriter processedMeasurementStreamWriter,
IRecordProcessor recordProcessor,
ILogger logger)
: base(client, checkpointRepository, heartbeatService)
{
_processedMeasurementStreamWriter = processedMeasurementStreamWriter;
_recordProcessor = recordProcessor;
_logger = logger;
}
public void ProcessETLMode(IRecordProcessor rp)
{
inputDatabase.Connect();
MapFields(rp);
rp.setDatasetName(inputDatabase.NormTableName(tableName));
string[] sqls = rp.GetSelectStatements();
if (Progress != null)
{
Progress.ReportProgress(0);
}
foreach (string sql in sqls)
{
try
{
DbDataReader reader = inputDatabase.Execute(sql);
int count = 1;
rp.Reset();
//Extract
while (reader.Read())
{
//Transform
rp.ProcessRecord(reader);
if (Progress != null)
{
Progress.ReportProgress(0, "Read records: " + count.ToString());
}
count++;
}
reader.Close();
reader = null;
//Load
if (outputDatabase.FileName != null)
{
//if output is file - Write one
WriteData();
}
}
catch (Exception ex)
{
throw ex;
}
// Call Close when done reading.
}
inputDatabase.Close();
}
void MapFields(IRecordProcessor rp)
{
Dictionary <string, int> inputFields = inputDatabase.GetFields(tableName);
Dictionary <string, int> fieldsMap = new Dictionary <string, int>();
foreach (KeyValuePair <string, string> fieldpair in inputFieldNamesMap)
{
string transferColName = fieldpair.Value;
if (fieldpair.Key.Length > 0)
{
int fieldIndex = inputFields[fieldpair.Key];
fieldsMap[transferColName] = fieldIndex;
}
}
rp.setFieldsMap(fieldsMap);
}
public static IRecordProcessor LoadRecordProcessor(Assembly Dynamic)
{
Type TypeofIRecordProcessor = typeof(IRecordProcessor);
foreach (Type Exported in Dynamic.GetExportedTypes())
{
if (TypeofIRecordProcessor.IsAssignableFrom(Exported))
{
//return (IRecordProcessor)Activator.CreateInstance(InputElementClass);
ConstructorInfo Constructor = Exported.GetConstructor(System.Type.EmptyTypes);
if (Constructor == null)
{
throw new Exception("Failed to bind " + Dynamic.FullName + ". Classes has no proper Constructor");
}
// Type supports IMatch and has an no-parameter constructor
IRecordProcessor Instance = (IRecordProcessor)Constructor.Invoke(System.Type.EmptyTypes);
return(Instance);
}
}
throw new Exception("Failed to bind " + Dynamic.FullName + ". No acceptable Classes found");
}
/// <summary>
/// Create an instance of KclProcess that uses the given IRecordProcessor to process records.
/// </summary>
/// <param name="recordProcessor">IRecordProcessor used to process records.</param>
public static KclProcess Create(IRecordProcessor recordProcessor)
{
return Create(recordProcessor, new IoHandler());
}
internal static KclProcess Create(IRecordProcessor recordProcessor, IoHandler ioHandler)
{
return new DefaultKclProcess(recordProcessor, ioHandler);
}
internal static KclProcess Create(IRecordProcessor recordProcessor, IoHandler ioHandler)
{
return(new DefaultKclProcess(recordProcessor, ioHandler));
}
public void ProcessEditMode(IRecordProcessor rp)
{
inputDatabase.Connect();
outputDatabase.Connect();
MapFields(rp);
rp.setDatasetName(inputDatabase.NormTableName(tableName));
if (Progress != null)
{
Progress.ReportProgress(0);
}
try
{
int count = ProcessOffset;
int processedCount = 0;
int totalCount;
rp.Reset();
//Extract
inputDatabase.SetTable(tableName);
outputDatabase.SetTable(tableName);
totalCount = inputDatabase.GetRowsCount();
if (ProcessOffset > 0)
{
inputDatabase.SeekRecord(ProcessOffset);
}
else
{
inputDatabase.FirstRecord();
}
//!! Skip first line as it Header
while (inputDatabase.NextRecord())
{
DataRow row = inputDatabase.CurrentRow();
//Transform
if (rp.ProcessRow(row))
{
//Load if row changed
outputDatabase.UpdateRow(count + 1, row);
if (Progress != null)
{
Progress.ReportProgress((processedCount * 100) / (totalCount - ProcessOffset), string.Format("Records: Read {0} (Processed {1}) of {2}", count, processedCount, totalCount));
}
processedCount++;
}
count++;
}
//Load
if (outputDatabase.FileName != null)
{
outputDatabase.Write(outputDatabase.FileName);
}
//Set complete
Progress.ReportProgress(100, string.Format("Read records: {0}, Processed {1}", count, processedCount));
}
catch (Exception ex)
{
throw ex;
}
// Call Close when done reading.
inputDatabase.Close();
outputDatabase.Close();
}
/// <summary>
/// Create an instance of KclProcess that uses the given IRecordProcessor to process records.
/// </summary>
/// <param name="recordProcessor">IRecordProcessor used to process records.</param>
public static KclProcess Create(IRecordProcessor recordProcessor)
{
return(Create(recordProcessor, new IoHandler()));
}
/// <summary>
/// Конструктор парсера записей FTP-сервера.
/// </summary>
/// <param name="processor">Обработчик строки</param>
public DirectoryListParser(IRecordProcessor processor)
{
_recordProcessor = processor;
}
/// <summary>
/// Create an instance of KclProcess that uses the given IRecordProcessor to process records.
/// </summary>
/// <param name="recordProcessor">IRecordProcessor used to process records.</param>
public static KclProcess Create(IRecordProcessor recordProcessor)
{
#pragma warning disable CC0022 // Should dispose object
return Create(recordProcessor, new IoHandler());
#pragma warning restore CC0022 // Should dispose object
}
private void SetRecordProcessor(Assembly irp)
{
rp = TableProcessor.LoadRecordProcessor(irp);
}
internal ShardRecordProcessorToRecordProcessor(IRecordProcessor recordProcessor)
{
RecordProcessor = recordProcessor;
}