public void DeleteProducer()
{
try
{
HttpClient client = new HttpClient();
client.BaseAddress = new Uri(baseUrl);
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", Token.access_token);
HttpResponseMessage response = client.PostAsJsonAsync(deleteAction, new IdModel(producerView.SelectedId)).Result;
response.EnsureSuccessStatusCode();
ProducerResponse producerResponse = response.Content.ReadAsAsync <ProducerResponse>().Result;
if (producerResponse.Success)
{
//Deleted!
LoadProducers();
PropertyCopier.CopyProperties(new Producer(), producerView.CurrentProducer);
producerView.CurrentProducer.RaiseUpdateProperties();
producerView.SelectedId = -1;
}
else
{
throw new ProducerException(producerResponse.ErrorCode, producerResponse.ErrorMessage);
}
}
catch (Exception ex)
{
StackTrace st = new StackTrace();
StackFrame sf = st.GetFrame(0);
MethodBase currentMethodName = sf.GetMethod();
Guid errorId = Guid.NewGuid();
//Log error here
producerView.HandleException(ex, currentMethodName.Name, errorId);
}
}
public ProducerResponse Send(ProducerRequest producerRequest)
{
var requestSize = producerRequest.SizeInBytes;
this.producerRequestStats.GetProducerRequestStats(this.BrokerInfo).RequestSizeHist.Update(requestSize);
this.producerRequestStats.GetProducerRequestAllBrokersStats().RequestSizeHist.Update(requestSize);
Receive response = null;
var specificTimer = this.producerRequestStats.GetProducerRequestStats(this.BrokerInfo).RequestTimer;
var aggregateTimer = this.producerRequestStats.GetProducerRequestAllBrokersStats().RequestTimer;
aggregateTimer.Time(() => specificTimer.Time(() =>
{
response = this.DoSend(producerRequest, producerRequest.RequiredAcks != 0);
}));
if (producerRequest.RequiredAcks != 0)
{
return(ProducerResponse.ReadFrom(response.Buffer));
}
else
{
return(null);
}
}
public void TestInsertUpdateErrors()
{
Token token = TokenHelper.GetToken(baseUrl, "Melvin3", "MelvinPass3");
JavaScriptSerializer javaScriptSerializer = new JavaScriptSerializer();
//Test insert
Producer producer = new Producer();
string jsonTicket = javaScriptSerializer.Serialize(producer);
//Post add ticket
HttpClient client = new HttpClient();
client.BaseAddress = new Uri(baseUrl);
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token.access_token);
//Test insert
HttpResponseMessage response = client.PostAsJsonAsync(insertAction, producer).Result;
Assert.IsTrue(response.IsSuccessStatusCode);
ProducerResponse producerResponse = response.Content.ReadAsAsync <ProducerResponse>().Result;
Assert.IsFalse(producerResponse.Success);
Assert.IsTrue(producerResponse.ErrorCode.HasFlag(ProducerError.InvalidName));
Assert.IsTrue(producerResponse.ErrorCode.HasFlag(ProducerError.InvalidCivilStatus));
Assert.IsTrue(producerResponse.ErrorCode.HasFlag(ProducerError.InvalidGender));
//Test update
response = client.PostAsJsonAsync(updateAction, producer).Result;
Assert.IsTrue(response.IsSuccessStatusCode);
producerResponse = response.Content.ReadAsAsync <ProducerResponse>().Result;
Assert.IsFalse(producerResponse.Success);
Assert.IsTrue(producerResponse.ErrorCode.HasFlag(ProducerError.InvalidName));
Assert.IsTrue(producerResponse.ErrorCode.HasFlag(ProducerError.InvalidCivilStatus));
Assert.IsTrue(producerResponse.ErrorCode.HasFlag(ProducerError.InvalidGender));
}
public void Start()
{
_connector.AddHost("tmq://localhost:48050");
_connector.Connected += Connected;
_connector.MessageReceived += MessageReceived;
_connector.Run();
_timer = new Timer(async o =>
{
if (_connector.IsConnected)
{
TmqClient client = _connector.GetClient();
TmqMessage message = new TmqMessage(MessageType.Client, "producer-id");
message.ContentType = ModelTypes.ConsumerRequest;
ConsumerRequest request = new ConsumerRequest();
request.Guid = Guid.NewGuid().ToString();
await message.SetJsonContent(request);
TmqMessage response = await client.Request(message);
ProducerResponse rmodel = await response.GetJsonContent <ProducerResponse>();
Console.WriteLine($"> response received for: {rmodel.RequestGuid}");
}
}, null, 6000, 6000);
}
public IHttpActionResult Update(Producer producer)
{
ProducerResponse response = new ProducerResponse();
try
{
Producer producerSaved = producerBL.UpdateProducer(producer);
response.Producer = producerSaved;
response.Success = true;
}
catch (ProducerException ex)
{
response.ErrorCode = ex.Error;
response.ErrorMessage = "Error. " + ex.Error.ToString();
response.Producer = null;
response.Success = false;
}
catch (Exception ex)
{
response.ErrorMessage = "Error. " + ex.Message;
response.Producer = null;
response.Success = false;
}
return(Ok(response));
}
public RequestResponseSerializationTest()
{
this.producerRequest = SerializationTestUtils.CreateTestProducerRequest();
this.producerResponse = SerializationTestUtils.CreateTestProducerResponse();
this.fetchRequest = SerializationTestUtils.CreateTestFetchRequest();
this.offsetRequest = SerializationTestUtils.CreateTestOffsetRequest();
this.offsetResponse = SerializationTestUtils.CreateTestOffsetResponse();
this.topicMetadataRequest = SerializationTestUtils.CreateTestTopicMetadataRequest();
this.topicMetadataResponse = SerializationTestUtils.CreateTestTopicMetadataResponse();
}
public void SaveProducer()
{
try
{
bool reLoadList = false;
HttpClient client = new HttpClient();
client.BaseAddress = new Uri(baseUrl);
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/bson"));
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", Token.access_token);
string action = string.Empty;
if (producerView.CurrentProducer.Id == 0)
{
//insert
action = insertAction;
reLoadList = true;
}
else
{
action = updateAction;
}
//update
MediaTypeFormatter bsonFormatter = new BsonMediaTypeFormatter();
HttpResponseMessage response = client.PostAsync(action, producerView.CurrentProducer, bsonFormatter).Result;
response.EnsureSuccessStatusCode();
MediaTypeFormatter[] formatters = new MediaTypeFormatter[] { bsonFormatter };
ProducerResponse producerResponse = response.Content.ReadAsAsync <ProducerResponse>(formatters).Result;
if (producerResponse.Success)
{
if (producerResponse.Producer != null)
{
if (reLoadList)
{
LoadProducers();
producerView.SelectedId = producerResponse.Producer.Id;
}
}
}
else
{
throw new ProducerException(producerResponse.ErrorCode, producerResponse.ErrorMessage);
}
}
catch (Exception ex)
{
StackTrace st = new StackTrace();
StackFrame sf = st.GetFrame(0);
MethodBase currentMethodName = sf.GetMethod();
Guid errorId = Guid.NewGuid();
//Log error here
producerView.HandleException(ex, currentMethodName.Name, errorId);
}
}
public void TestSerializationAndDeserialization()
{
var buffer = ByteBuffer.Allocate(this.producerRequest.SizeInBytes);
this.producerRequest.WriteTo(buffer);
buffer.Rewind();
var deserializedProducerRequest = ProducerRequest.ReadFrom(buffer);
Assert.Equal(this.producerRequest, deserializedProducerRequest);
buffer = ByteBuffer.Allocate(this.producerResponse.SizeInBytes);
this.producerResponse.WriteTo(buffer);
buffer.Rewind();
var deserializedProducerResponse = ProducerResponse.ReadFrom(buffer);
Assert.Equal(this.producerResponse, deserializedProducerResponse);
buffer = ByteBuffer.Allocate(this.fetchRequest.SizeInBytes);
this.fetchRequest.WriteTo(buffer);
buffer.Rewind();
var deserializedFetchRequest = FetchRequest.ReadFrom(buffer);
Assert.Equal(this.fetchRequest, deserializedFetchRequest);
buffer = ByteBuffer.Allocate(this.offsetRequest.SizeInBytes);
this.offsetRequest.WriteTo(buffer);
buffer.Rewind();
var deserializedOffsetRequest = OffsetRequest.ReadFrom(buffer);
Assert.Equal(this.offsetRequest, deserializedOffsetRequest);
buffer = ByteBuffer.Allocate(this.offsetResponse.SizeInBytes);
this.offsetResponse.WriteTo(buffer);
buffer.Rewind();
var deserializedOffsetResponse = OffsetResponse.ReadFrom(buffer);
Assert.Equal(this.offsetResponse, deserializedOffsetResponse);
buffer = ByteBuffer.Allocate(this.topicMetadataRequest.SizeInBytes);
this.topicMetadataRequest.WriteTo(buffer);
buffer.Rewind();
var deserializedTopicMetadataRequest = TopicMetadataRequest.ReadFrom(buffer);
Assert.Equal(this.topicMetadataRequest, deserializedTopicMetadataRequest);
buffer = ByteBuffer.Allocate(this.topicMetadataResponse.SizeInBytes);
this.topicMetadataResponse.WriteTo(buffer);
buffer.Rewind();
var deserializedTopicMetadataResponse = TopicMetadataResponse.ReadFrom(buffer);
Assert.Equal(this.topicMetadataResponse, deserializedTopicMetadataResponse);
}
public static ProducerResponse GetProducerResponse(byte[] buff, int len)
{
var resp = new ProducerResponse();
var stream = new MemoryStream(buff, 0, len);
stream.Position += 4; // skip message size
var count = BigEndianConverter.ReadInt32(stream);
resp.Topics = new ProducerResponse.TopicResponse[count];
for (int i = 0; i < count; i++)
{
resp.Topics[i] = DeserializeTopicResponse(stream);
}
return(resp);
}
public void TestDeleteErrors()
{
Token token = TokenHelper.GetToken(baseUrl, "Melvin3", "MelvinPass3");
JavaScriptSerializer javaScriptSerializer = new JavaScriptSerializer();
HttpClient client = new HttpClient();
client.BaseAddress = new Uri(baseUrl);
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token.access_token);
//Test delete
HttpResponseMessage response = client.PostAsJsonAsync(deleteAction, new IdModel(0)).Result;
Assert.IsTrue(response.IsSuccessStatusCode);
ProducerResponse producerResponse = response.Content.ReadAsAsync <ProducerResponse>().Result;
Assert.IsFalse(producerResponse.Success);
Assert.IsTrue(producerResponse.ErrorCode.HasFlag(ProducerError.InvalidId));
}
private void MessageReceived(ClientSocketBase <TmqMessage> client, TmqMessage message)
{
if (message.Type == MessageType.Client && message.ContentType == ModelTypes.ConsumerRequest)
{
ConsumerRequest request = message.GetJsonContent <ConsumerRequest>().Result;
Console.WriteLine($"Consumer request received: {request.Guid}");
ProducerResponse model = new ProducerResponse();
model.RequestGuid = request.Guid;
model.ResponseGuid = Guid.NewGuid().ToString();
TmqMessage response = message.CreateResponse();
response.ContentType = ModelTypes.ProducerResponse;
response.SetJsonContent(model);
TmqClient tmq = (TmqClient)client;
tmq.Send(response);
}
}
private void sendCommandAsync(Command cmd, Chan <ProducerResponse> doneChan, params object[] args)
{
ProducerResponse t = null;
try
{
if (_state != (int)State.Connected)
{
Connect();
}
// keep track of how many outstanding producers we're dealing with
// in order to later ensure that we clean them all up...
Interlocked.Increment(ref _concurrentProducers);
t = new ProducerResponse
{
_cmd = cmd,
_doneChan = doneChan,
Args = args
};
Select
.CaseSend(_transactionChan, t)
.CaseReceive(_exitChan, _throwErrStoppedAction)
.NoDefault();
Interlocked.Decrement(ref _concurrentProducers);
}
catch (Exception ex)
{
if (t != null)
{
Interlocked.Decrement(ref _concurrentProducers);
t.Error = ex;
GoFunc.Run(() => t.finish(), "Producer: t.finish()");
}
else
{
Thread.Sleep(1000); // slow down hammering Connect
throw;
}
}
}
public IHttpActionResult Delete(IdModel id)
{
ProducerResponse response = new ProducerResponse();
try
{
bool success = producerBL.DeleteProducer(id.Id);
response.Success = success;
}
catch (ProducerException ex)
{
response.ErrorCode = ex.Error;
response.ErrorMessage = "Error. " + ex.Error.ToString();
response.Producer = null;
response.Success = false;
}
catch (Exception ex)
{
response.ErrorMessage = "Error. " + ex.Message;
response.Producer = null;
response.Success = false;
}
return(Ok(response));
}
public void TestInsertUpdateAndGetProducer()
{
//Get token
Token token = TokenHelper.GetToken(baseUrl, "Melvin3", "MelvinPass3");
JavaScriptSerializer javaScriptSerializer = new JavaScriptSerializer();
//Test insert
Producer producer = ProducerHelper.CreateDummyProducer();
string jsonTicket = javaScriptSerializer.Serialize(producer);
//Post add ticket
HttpClient client = new HttpClient();
client.BaseAddress = new Uri(baseUrl);
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token.access_token);
HttpResponseMessage response = client.PostAsJsonAsync(insertAction, producer).Result;
Assert.IsTrue(response.IsSuccessStatusCode);
ProducerResponse producerResponse = response.Content.ReadAsAsync <ProducerResponse>().Result;
Assert.IsTrue(producerResponse.Success);
Assert.IsTrue(producerResponse != null);
Assert.IsTrue(producerResponse.Producer.Id > 0);
//get by id
string getByIdUrl = string.Format("{0}?id={1}", getByIdAction, producerResponse.Producer.Id.ToString());
response = client.GetAsync(getByIdUrl).Result;
Assert.IsTrue(response.IsSuccessStatusCode);
GetProducerResponse getProducerResponse = response.Content.ReadAsAsync <GetProducerResponse>().Result;
Assert.IsTrue(getProducerResponse.Success);
Assert.IsTrue(getProducerResponse.Producers.Count == 1);
Assert.IsTrue(getProducerResponse.Producers.ElementAt(0).Id == producerResponse.Producer.Id);
//get all
response = client.GetAsync(getAllAction).Result;
Assert.IsTrue(response.IsSuccessStatusCode);
getProducerResponse = response.Content.ReadAsAsync <GetProducerResponse>().Result;
Assert.IsTrue(getProducerResponse.Success);
Producer producerFound = (from produc in getProducerResponse.Producers where produc.Id == producerResponse.Producer.Id select produc).FirstOrDefault();
Assert.IsTrue(producerFound != null);
//test update
producer.Id = producerResponse.Producer.Id;
producer.RFC = "UpdatedProducerRFC";
producer.GenderId = 2;
response = client.PostAsJsonAsync(updateAction, producer).Result;
Assert.IsTrue(response.IsSuccessStatusCode);
producerResponse = response.Content.ReadAsAsync <ProducerResponse>().Result;
Assert.IsTrue(producerResponse.Success);
//Get ticket again and check it was updated
response = client.GetAsync(getByIdUrl).Result;
Assert.IsTrue(response.IsSuccessStatusCode);
getProducerResponse = response.Content.ReadAsAsync <GetProducerResponse>().Result;
Assert.IsTrue(getProducerResponse.Success);
Assert.IsTrue(getProducerResponse.Producers.Count == 1);
Assert.IsTrue(getProducerResponse.Producers.ElementAt(0).GenderId == 2);
Assert.IsTrue(getProducerResponse.Producers.ElementAt(0).RFC == "UpdatedProducerRFC");
//test delete
response = client.PostAsJsonAsync(deleteAction, new IdModel(producerResponse.Producer.Id)).Result;
Assert.IsTrue(response.IsSuccessStatusCode);
producerResponse = response.Content.ReadAsAsync <ProducerResponse>().Result;
Assert.IsTrue(producerResponse.Success);
//Get ticket again and check it is not found
response = client.GetAsync(getByIdUrl).Result;
Assert.IsTrue(response.IsSuccessStatusCode);
getProducerResponse = response.Content.ReadAsAsync <GetProducerResponse>().Result;
Assert.IsTrue(getProducerResponse.Success);
Assert.IsTrue(getProducerResponse.Producers.Count == 0);
}
/// <summary>
/// Send message of one broker.
/// </summary>
/// <param name="brokerId"></param>
/// <param name="messagesPerTopic"></param>
/// <returns></returns>
private ProducerSendResult <IEnumerable <Tuple <TopicAndPartition, ProducerResponseStatus> > > Send(int brokerId, IDictionary <TopicAndPartition, BufferedMessageSet> messagesPerTopic)
{
try
{
if (brokerId < 0)
{
throw new NoLeaderForPartitionException(
string.Format("No leader for some partition(s). And it try write to on invalid broker {0}. The assigned TopicAndPartition for the data is :{1} ", brokerId, messagesPerTopic.Any() ? messagesPerTopic.First().Key.ToString() : "(null)"));
}
if (messagesPerTopic.Any())
{
var producerRequest = new ProducerRequest(NextCorrelationId,
this.producerConfig.ClientId,
this.producerConfig.RequiredAcks,
this.producerConfig.AckTimeout,
messagesPerTopic);
ISyncProducer syncProducer = null;
try
{
syncProducer = this.syncProducerPool.GetProducer(brokerId);
}
catch (UnavailableProducerException e)
{
Logger.Error(e.Message);
// When initializing producer pool, some broker might be unavailable, and now it is healthy and is leader for some partitions.
// A new producer should be added to the pool, creating a TCP connection to the broker.
var broker =
this.brokerPartitionInfo.GetBrokerPartitionLeaders(messagesPerTopic.Keys.First().Topic)
.Values.FirstOrDefault(b => b.Id == brokerId);
if (broker != null)
{
this.syncProducerPool.AddProducer(broker);
syncProducer = this.syncProducerPool.GetProducer(brokerId);
}
}
if (producerConfig.Verbose)
{
Logger.DebugFormat("Kafka producer before sent messages for topics {0} to broker {1}", messagesPerTopic, brokerId);
}
ProducerResponse response = syncProducer.Send(producerRequest);
if (this.producerConfig.Verbose)
{
string msg = string.Format("Kafka producer sent messages for topics {0} to broker {1} on {2}:{3}",
messagesPerTopic, brokerId, syncProducer.Config.Host, syncProducer.Config.Port);
Logger.Debug(msg);
}
if (response != null)
{
int statusCount = response.Statuses.Count();
//In java version
//https://git-wip-us.apache.org/repos/asf?p=kafka.git;a=blob;f=core/src/main/scala/kafka/producer/async/DefaultEventHandler.scala;h=821901e4f434dfd9eec6eceabfc2e1e65507a57c;hb=HEAD#l260
//The producerRequest.data just the messagesPerTopic. So there compare the statusCount with producerRequest.data.size
//But in this C# version, the producerRequest.Data already grouped by topic. So here need compare with messagesPerTopic.Count()
int requestCount = messagesPerTopic.Count();
if (statusCount != requestCount)
{
StringBuilder sb = new StringBuilder();
sb.AppendFormat("Incomplete response count {0} for producer request count {1}. ", statusCount, requestCount);
sb.AppendFormat(" Broker {0} on {1}:{2}", brokerId, syncProducer.Config.Host, syncProducer.Config.Port);
sb.Append(" Message detail:");
sb.Append(string.Join(",", messagesPerTopic.Select(r => string.Format("{0},{1}", r.Key.Topic, r.Key.PartitionId))));
sb.Append(" Response status detail which has error:");
sb.Append(string.Join(",", response.Statuses.Where(r => r.Value.Error != (short)ErrorMapping.NoError).Select(r => r.ToString())));
throw new FailedToSendMessageException <TK>(sb.ToString());
}
return(new ProducerSendResult <IEnumerable <Tuple <TopicAndPartition, ProducerResponseStatus> > >(response.Statuses.Where(s => s.Value.Error != (short)ErrorMapping.NoError)
.Select(s => new Tuple <TopicAndPartition, ProducerResponseStatus>(s.Key, s.Value))));
}
}
}
catch (NoLeaderForPartitionException e)
{
Logger.Error(ExceptionUtil.GetExceptionDetailInfo(e));
return(new ProducerSendResult <IEnumerable <Tuple <TopicAndPartition, ProducerResponseStatus> > >(messagesPerTopic.Keys.Select(
s => new Tuple <TopicAndPartition, ProducerResponseStatus>(s, new ProducerResponseStatus {
Error = ErrorMapping.NotLeaderForPartitionCode
})), e));
}
catch (Exception e)
{
Logger.Error(ExceptionUtil.GetExceptionDetailInfo(e));
return(new ProducerSendResult <IEnumerable <Tuple <TopicAndPartition, ProducerResponseStatus> > >(messagesPerTopic.Keys.Select(
s => new Tuple <TopicAndPartition, ProducerResponseStatus>(s, new ProducerResponseStatus {
Error = ErrorMapping.UnknownCode
})), e));
}
return(new ProducerSendResult <IEnumerable <Tuple <TopicAndPartition, ProducerResponseStatus> > >(Enumerable.Empty <Tuple <TopicAndPartition, ProducerResponseStatus> >()));
}
