public void ZKAwareProducerSends1Message()
{
int totalWaitTimeInMiliseconds = 0;
int waitSingle = 100;
var originalMessage = new Message(Encoding.UTF8.GetBytes("TestData"));
var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic);
multipleBrokersHelper.GetCurrentOffsets();
var producerConfig = new ProducerConfig(clientConfig);
var mockPartitioner = new MockAlwaysZeroPartitioner();
using (var producer = new Producer<string, Message>(producerConfig, mockPartitioner, new DefaultEncoder()))
{
var producerData = new ProducerData<string, Message>(
CurrentTestTopic, "somekey", new List<Message>() { originalMessage });
producer.Send(producerData);
while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged())
{
totalWaitTimeInMiliseconds += waitSingle;
Thread.Sleep(waitSingle);
if (totalWaitTimeInMiliseconds > MaxTestWaitTimeInMiliseconds)
{
Assert.Fail("None of the brokers changed their offset after sending a message");
}
}
totalWaitTimeInMiliseconds = 0;
var consumerConfig = new ConsumerConfig(clientConfig)
{
Host = multipleBrokersHelper.BrokerThatHasChanged.Address,
Port = multipleBrokersHelper.BrokerThatHasChanged.Port
};
IConsumer consumer = new Consumers.Consumer(consumerConfig);
var request = new FetchRequest(CurrentTestTopic, 0, multipleBrokersHelper.OffsetFromBeforeTheChange);
BufferedMessageSet response;
while (true)
{
Thread.Sleep(waitSingle);
response = consumer.Fetch(request);
if (response != null & response.Messages.Count() > 0)
{
break;
}
totalWaitTimeInMiliseconds += waitSingle;
if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds)
{
break;
}
}
Assert.NotNull(response);
Assert.AreEqual(1, response.Messages.Count());
Assert.AreEqual(originalMessage.ToString(), response.Messages.First().ToString());
}
}
public EditProducerViewModel(ProducerData producer)
{
NotificationId = producer.NotificationId;
ProducerId = producer.Id;
Address = producer.Address;
Contact = producer.Contact;
Business = new ProducerBusinessTypeViewModel(producer.Business);
}
public void ProducerSends1Message()
{
var prodConfig = this.ConfigBasedSyncProdConfig;
int totalWaitTimeInMiliseconds = 0;
int waitSingle = 100;
var originalMessage = new Message(Encoding.UTF8.GetBytes("TestData"));
var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic);
multipleBrokersHelper.GetCurrentOffsets(
new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 });
using (var producer = new Producer(prodConfig))
{
var producerData = new ProducerData<string, Message>(
CurrentTestTopic, new List<Message> { originalMessage });
producer.Send(producerData);
Thread.Sleep(waitSingle);
}
while (
!multipleBrokersHelper.CheckIfAnyBrokerHasChanged(
new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 }))
{
totalWaitTimeInMiliseconds += waitSingle;
Thread.Sleep(waitSingle);
if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds)
{
Assert.Fail("None of the brokers changed their offset after sending a message");
}
}
totalWaitTimeInMiliseconds = 0;
var consumerConfig = new ConsumerConfiguration(
multipleBrokersHelper.BrokerThatHasChanged.Host, multipleBrokersHelper.BrokerThatHasChanged.Port);
IConsumer consumer = new Consumer(consumerConfig);
var request1 = new FetchRequest(CurrentTestTopic, multipleBrokersHelper.PartitionThatHasChanged, multipleBrokersHelper.OffsetFromBeforeTheChange);
BufferedMessageSet response;
while (true)
{
Thread.Sleep(waitSingle);
response = consumer.Fetch(request1);
if (response != null && response.Messages.Count() > 0)
{
break;
}
totalWaitTimeInMiliseconds += waitSingle;
if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds)
{
break;
}
}
Assert.NotNull(response);
Assert.AreEqual(1, response.Messages.Count());
Assert.AreEqual(originalMessage.ToString(), response.Messages.First().ToString());
}
public void ZkAwareProducerSends3Messages()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
int totalWaitTimeInMiliseconds = 0;
int waitSingle = 100;
var originalMessage1 = new Message(Encoding.UTF8.GetBytes("TestData1"));
var originalMessage2 = new Message(Encoding.UTF8.GetBytes("TestData2"));
var originalMessage3 = new Message(Encoding.UTF8.GetBytes("TestData3"));
var originalMessageList = new List <Message> {
originalMessage1, originalMessage2, originalMessage3
};
var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic);
multipleBrokersHelper.GetCurrentOffsets(new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 });
var mockPartitioner = new MockAlwaysZeroPartitioner();
using (var producer = new Producer <string, Message>(prodConfig, mockPartitioner, new DefaultEncoder()))
{
var producerData = new ProducerData <string, Message>(CurrentTestTopic, "somekey", originalMessageList);
producer.Send(producerData);
while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged(new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 }))
{
totalWaitTimeInMiliseconds += waitSingle;
Thread.Sleep(waitSingle);
if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds)
{
Assert.Fail("None of the brokers changed their offset after sending a message");
}
}
totalWaitTimeInMiliseconds = 0;
var consumerConfig = new ConsumerConfiguration(
multipleBrokersHelper.BrokerThatHasChanged.Host,
multipleBrokersHelper.BrokerThatHasChanged.Port);
IConsumer consumer = new Consumer(consumerConfig);
var request = new FetchRequest(CurrentTestTopic, 0, multipleBrokersHelper.OffsetFromBeforeTheChange);
BufferedMessageSet response;
while (true)
{
Thread.Sleep(waitSingle);
response = consumer.Fetch(request);
if (response != null && response.Messages.Count() > 2)
{
break;
}
totalWaitTimeInMiliseconds += waitSingle;
if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds)
{
break;
}
}
Assert.NotNull(response);
Assert.AreEqual(3, response.Messages.Count());
Assert.AreEqual(originalMessage1.ToString(), response.Messages.First().ToString());
Assert.AreEqual(originalMessage2.ToString(), response.Messages.Skip(1).First().ToString());
Assert.AreEqual(originalMessage3.ToString(), response.Messages.Skip(2).First().ToString());
}
}
public MovieDetailsGetData GetMovieDetails(Guid movie_id)
{
SqlConnection sqlconnection = sqlsetup.SqlConnectionSetup();
MovieDetailsGetData moviedetails = new MovieDetailsGetData();
try
{
sqlconnection.Open();
List <Task> TableFetch = new List <Task>();
MovieData MovieDBList = new MovieData();
MovieProducerRelationData MovieProducerMapping = new MovieProducerRelationData();
List <ActorData> ActorDBList = new List <ActorData>();
List <ProducerData> ProducerList = new List <ProducerData>();
List <ActorMovieRelationData> MovieActorMapping = new List <ActorMovieRelationData>();
Dictionary <Guid, ActorData> dicActorData = new Dictionary <Guid, ActorData>();
Dictionary <Guid, ProducerData> dicProducer = new Dictionary <Guid, ProducerData>();
string ImageLocation = "/src/Images/";
TableFetch.Add(Task.Run(() => {
MovieDBList = sqlconnection.Query <MovieData>("SELECT * FROM IMDB.dbo.Movie WHERE movie_id = @movie_id;", new { movie_id }).FirstOrDefault();
}));
TableFetch.Add(Task.Run(() => {
ActorDBList = sqlconnection.Query <ActorData>("SELECT * FROM IMDB.dbo.Actors;").ToList();
if (ActorDBList?.Count > 0)
{
dicActorData = ActorDBList.ToDictionary(x => x.actor_id);
}
}));
TableFetch.Add(Task.Run(() => {
ProducerList = sqlconnection.Query <ProducerData>("SELECT * FROM IMDB.dbo.Producers;").ToList();
if (ProducerList?.Count > 0)
{
dicProducer = ProducerList.ToDictionary(x => x.producer_id);
}
}));
TableFetch.Add(Task.Run(() => {
MovieProducerMapping = sqlconnection.Query <MovieProducerRelationData>("SELECT * FROM IMDB.dbo.Mapping_Table_Producers_and_Movies WHERE movie_id = @movie_id;", new { movie_id }).FirstOrDefault();
}));
TableFetch.Add(Task.Run(() => {
MovieActorMapping = sqlconnection.Query <ActorMovieRelationData>("SELECT * FROM IMDB.dbo.Mapping_Table_Actors_and_movies WHERE movie_id = @movie_id;", new { movie_id }).ToList();
}));
Task.WaitAll(TableFetch.ToArray());
String Format = ".jpg";
moviedetails.MovieDetail = MovieDBList;
moviedetails.ImageFilePath = moviedetails.MovieDetail.Poster != null? ImageLocation + MovieDBList.Name + Format:null;
ProducerData producerBio = null;
dicProducer.TryGetValue(MovieProducerMapping.producer_id, out producerBio);
if (producerBio != null)
{
moviedetails.producerList = producerBio;
}
object obj = new object();
moviedetails.actorList = new List <ActorData>();
Parallel.ForEach(MovieActorMapping, x =>
{
ActorData actordata = null;
dicActorData.TryGetValue(x.actor_id, out actordata);
if (actordata != null)
{
lock (obj)
{
moviedetails.actorList.Add(actordata);
}
}
});
}
catch (Exception e)
{
}
finally
{
sqlconnection.Close();
}
return(moviedetails);
}
public void Send(ProducerData <string, Kafka.Client.Messages.Message> data)
{
_producer.Send(data);
}
private ProducerData <int, Message> BuildMessage(IMessage message)
{
var rv = new ProducerData <int, Message>(Topic, message.DeviceId, new Message(message.BinarySerialized()));
return(rv);
}
public void ProducerSends1Message()
{
var prodConfig = this.ConfigBasedSyncProdConfig;
int totalWaitTimeInMiliseconds = 0;
int waitSingle = 100;
var originalMessage = new Message(Encoding.UTF8.GetBytes("TestData"));
var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic);
multipleBrokersHelper.GetCurrentOffsets(
new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 });
using (var producer = new Producer(prodConfig))
{
var producerData = new ProducerData <string, Message>(
CurrentTestTopic, new List <Message> {
originalMessage
});
producer.Send(producerData);
Thread.Sleep(waitSingle);
}
while (
!multipleBrokersHelper.CheckIfAnyBrokerHasChanged(
new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 }))
{
totalWaitTimeInMiliseconds += waitSingle;
Thread.Sleep(waitSingle);
if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds)
{
Assert.Fail("None of the brokers changed their offset after sending a message");
}
}
totalWaitTimeInMiliseconds = 0;
var consumerConfig = new ConsumerConfiguration(
multipleBrokersHelper.BrokerThatHasChanged.Host, multipleBrokersHelper.BrokerThatHasChanged.Port);
IConsumer consumer = new Consumer(consumerConfig);
var request1 = new FetchRequest(CurrentTestTopic, multipleBrokersHelper.PartitionThatHasChanged, multipleBrokersHelper.OffsetFromBeforeTheChange);
BufferedMessageSet response;
while (true)
{
Thread.Sleep(waitSingle);
response = consumer.Fetch(request1);
if (response != null && response.Messages.Count() > 0)
{
break;
}
totalWaitTimeInMiliseconds += waitSingle;
if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds)
{
break;
}
}
Assert.NotNull(response);
Assert.AreEqual(1, response.Messages.Count());
Assert.AreEqual(originalMessage.ToString(), response.Messages.First().ToString());
}
void Start()
{
this.currentData = this.initialData;
this.lastEnergy = this.currentData.energy;
StartCoroutine(Live());
}
public void ZkAwareProducerSends3Messages()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
int totalWaitTimeInMiliseconds = 0;
int waitSingle = 100;
var originalMessage1 = new Message(Encoding.UTF8.GetBytes("TestData1"));
var originalMessage2 = new Message(Encoding.UTF8.GetBytes("TestData2"));
var originalMessage3 = new Message(Encoding.UTF8.GetBytes("TestData3"));
var originalMessageList = new List<Message> { originalMessage1, originalMessage2, originalMessage3 };
var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic);
multipleBrokersHelper.GetCurrentOffsets(new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 });
var mockPartitioner = new MockAlwaysZeroPartitioner();
using (var producer = new Producer<string, Message>(prodConfig, mockPartitioner, new DefaultEncoder()))
{
var producerData = new ProducerData<string, Message>(CurrentTestTopic, "somekey", originalMessageList);
producer.Send(producerData);
while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged(new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 }))
{
totalWaitTimeInMiliseconds += waitSingle;
Thread.Sleep(waitSingle);
if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds)
{
Assert.Fail("None of the brokers changed their offset after sending a message");
}
}
totalWaitTimeInMiliseconds = 0;
var consumerConfig = new ConsumerConfiguration(
multipleBrokersHelper.BrokerThatHasChanged.Host,
multipleBrokersHelper.BrokerThatHasChanged.Port);
IConsumer consumer = new Consumer(consumerConfig);
var request = new FetchRequest(CurrentTestTopic, 0, multipleBrokersHelper.OffsetFromBeforeTheChange);
BufferedMessageSet response;
while (true)
{
Thread.Sleep(waitSingle);
response = consumer.Fetch(request);
if (response != null && response.Messages.Count() > 2)
{
break;
}
totalWaitTimeInMiliseconds += waitSingle;
if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds)
{
break;
}
}
Assert.NotNull(response);
Assert.AreEqual(3, response.Messages.Count());
Assert.AreEqual(originalMessage1.ToString(), response.Messages.First().ToString());
Assert.AreEqual(originalMessage2.ToString(), response.Messages.Skip(1).First().ToString());
Assert.AreEqual(originalMessage3.ToString(), response.Messages.Skip(2).First().ToString());
}
}
public void PushMessage(IEnumerable<Message> msgs)
{
if (msgs == null || msgs.Count() == 0)
{
throw new ArgumentNullException("msg", "PushMessage received a null msg");
}
string s = string.Empty;
// var messages = new List<Message>(1) { msg };
int maxRetry = this.brokerConfList.Count;
int retryCount = 0;
bool success = false;
while (!success)
{
// find the leader first for every re-try
if (retryCount > 0)
{
this.ClearLeader();
}
if (this.FindLeader() == 0)
{
s = string.Format("PushMessege not found leader for topic {0}", this.topic);
//Logger.Error(s);
throw new ArgumentException(s);
}
//Logger.Debug(string.Format("KafkaClient.PushMessage,leader.First={0},retryCount={1},maxRetry={2}", leaders[0].ToString(), retryCount, maxRetry));
try
{
ProducerData<int, Message> data = null;
if (this.partitionIndex < 0)
{
data = new ProducerData<int, Message>(this.topic, msgs);
}
else
{
data = new ProducerData<int, Message>(this.topic, this.partitionIndex, msgs);
}
this.KafkaProducer.Send(data);
// simulate excepton in a debugger
//// bool t = false;
//// if (t)
//// {
//// throw new ArgumentException("simulate exception in PushMessge");
//// }
success = true;
}
catch (Exception e)
{
s = "PushMessage exception,leader={0},topic={1},retryCount={2},maxRetry={3},brokerConfigList.Count={4}";
Logger.Error(string.Format(s, leaders.Count == 0 ? "null" : leaders[0].ToString(), this.topic, retryCount, maxRetry, this.brokerConfList.Count), e);
if (retryCount >= maxRetry)
{
throw;
}
}
finally
{
retryCount++;
}
} // end of while
}
public ConsumerData Produce(ProducerData data)
{
ConsumerData path = new ConsumerData(data.FilePath);
return(path);
}
