public KafkaProducer(Dictionary <string, object> config, string topicName, int partition, EventHandler <IError> dlgtOnError)
{
this.config = config;
this.topicName = topicName;
this.partition = partition;
this.dlgtOnError = dlgtOnError;
var keySerializer = Factory.CreateSerializer <string>();
var valueSerializer = Factory.CreateSerializer <T>();
AssignSerializersIfDefault(ref keySerializer, ref valueSerializer);
producer = new Producer <string, T>(config, keySerializer, valueSerializer);
threadProcessing = new ThreadProcessing(() =>
{
if (ToBeContinued)
{
KeyValuePair <string, T> pair;
while (GetNextDataPair(out pair))
{
producer.ProduceAsync(topicName, pair.Key, pair.Value, partition, blockIfQueueFull: true);
}
}
// Tasks are not waited on synchronously (ContinueWith is not synchronous),
// so it's possible they may still in progress here.
//producer.Flush(TimeSpan.FromSeconds(10));
},
e =>
{
dlgtOnError(null, new Error((int)ErrorCode.Unknown, e.Message));
LogToScreen.Exception(e);
});
}
public StorageWriter(int receiverId, string host, int port, int firstDbNum, int periodInMs, int trimLength,
EventHandler <IError> dlgtOnError, string keySuffix,
Func <string, T, KeyContext, bool> dlgtFilter = null)
{
this.keySuffix = keySuffix;
// Init DataStore
dataStore = DataStore.Create <T>(host, database: firstDbNum, isDurationMeasured: true, port: port);
periodicProcessing = new PeriodicProcessing(
// PeriodicProc
() =>
{
KeyValuePair <string, T> pair;
while (cqueData.TryDequeue(out pair))
{
try
{
if (dlgtFilter?.Invoke(pair.Key, pair.Value, GetKeyContext(pair.Key)) != false)
{
dataStore.Add(pair.Key, pair.Value, trimLength);
// Publish DataStore "INSERTED" event
dataStore.PublishAsync("ReceiverChannel", $"INSERTED|DATA_RECEIVER|{receiverId}");
}
}
catch (Exception e)
{
LogToScreen.Exception(e);
}
}
},
e => dlgtOnError(this, new Error(e)),
periodInMs);
}
public KafkaConsumer(Dictionary <string, object> config, string topicName, int partition, int offset,
MessageProcessingDelegate <T> dlgtMessageProcessing, int consumerTimeoutMs,
EventHandler <IError> dlgtOnError)
{
if (dlgtMessageProcessing == null)
{
return;
}
var keyDeserializer = Factory.CreateDeserializer <string>();
var valueDeserializer = Factory.CreateDeserializer <T>();
AssignDeserializersIfDefault(ref keyDeserializer, ref valueDeserializer);
consumer = GetConsumer(config, topicName, partition, offset, (sender, e) => dlgtOnError(sender, (Error)e),
keyDeserializer, valueDeserializer);
consumer.OnMessage += (sender, msg) =>
{
Interlocked.Exchange(ref lastOffset, msg.Offset.Value);
Task.Run(() =>
{
try
{
dlgtMessageProcessing(new Message <T>(msg));
}
catch (Exception e)
{
dlgtOnError(null, new Error((int)ErrorCode.Unknown, e.Message));
LogToScreen.Exception(e);
}
});
};
threadProcessing = new ThreadProcessing(() =>
{
while (ToBeContinued)
{
consumer.Poll(consumerTimeoutMs);
}
},
e =>
{
dlgtOnError(null, new Error((int)ErrorCode.Unknown, e.Message));
LogToScreen.Exception(e);
});
}
public StorageReader(Action <int, IList <DSData> > dlgtAction, int processorId, string host, int port,
int firstDbNum, int periodInMs, int[] dsRange, int maxLagInMs,
EventHandler <IError> dlgtOnError,
string keySuffix)
{
this.keySuffix = keySuffix;
// Init DataStore
dataStore = DataStore.Create <T>(host, database: firstDbNum, isDurationMeasured: true, port: port);
if (periodInMs == Timeout.Infinite)
{
// Receive DataStore "INSERTED" event
dataStore.Subscribe("ReceiverChannel",
s =>
{
if (string.IsNullOrEmpty(s))
{
return;
}
var ss = s.Split('|');
int receiverId;
if (ss[0] == "INSERTED" && ss[1] == "DATA_RECEIVER" && int.TryParse(ss[2], out receiverId) &&
processorId == receiverId &&
periodicProcessing != null)
{
periodicProcessing.Execute();
}
});
}
periodicProcessing = new PeriodicProcessing(
// PeriodicProc
() =>
{
for (int i = dsRange[0]; i <= dsRange[1]; i++)
{
try
{
var list = dataStore.Get(GetKeyByDsNum(i));
if (list.Count > 0)
{
var lstData = new List <DSData>();
foreach (var d in list)
{
var item = Convert(d);
if (item != null)
{
lstData.Add(item);
}
}
if (lstData.Count > 0)
{
dlgtAction?.Invoke(i, lstData);
// Delay verification
if (/*lag*/ DateTime.UtcNow - lstData[0].Timestamp > TimeSpan.FromMilliseconds(maxLagInMs))
{
// Lag is too high
LogToScreen.Message("?");
}
}
// Verification of timestamps for LIFO in the list
for (int j = 0; j < list.Count - 1; j++)
{
var d0 = Convert(list[j]);
var d1 = Convert(list[j + 1]);
if (d0.Timestamp <= d1.Timestamp)
{
// Wrong chronologic order of data
LogToScreen.Message(" ?T ");
}
}
}
}
catch (Exception e)
{
LogToScreen.Exception(e);
}
}
},
e => dlgtOnError(this, new Error(e)),
periodInMs);
}
