private bool ValidateCustomerAssetKafkaObject(bool isDissociate, KafkaMessage target, dynamic source) { var isValid = target.Key == source.CustomerUID.ToString() && CustomerTopics.Contains(target.Topic); var json = JObject.Parse(JsonConvert.SerializeObject(target.Message)); if (isDissociate) { var eventMsg = JsonConvert.DeserializeObject <DissociateCustomerAssetEvent>( json.SelectToken("DissociateCustomerAssetEvent").ToString()); return(isValid && eventMsg.CustomerUID == source.CustomerUID && eventMsg.AssetUID == source.AssetUID && eventMsg.ActionUTC == source.ActionUTC && eventMsg.ReceivedUTC == source.ReceivedUTC); } else { Enum.TryParse(source.RelationType, true, out TestDataEnums.RelationType relationType); var eventMsg = JsonConvert.DeserializeObject <AssociateCustomerAssetEvent>( json.SelectToken("AssociateCustomerAssetEvent").ToString()); return(isValid && eventMsg.CustomerUID == source.CustomerUID && eventMsg.AssetUID == source.AssetUID && eventMsg.ActionUTC == source.ActionUTC && eventMsg.RelationType == relationType.ToString() && eventMsg.ReceivedUTC == source.ReceivedUTC); } }
private bool ValidateCustomerKafkaObject(dynamic source, KafkaMessage target, string customerType, bool isIncludeID, DbCustomer sourceCustomerDetail) { Enum.TryParse(customerType, out CustomerType custType); var json = JObject.Parse(JsonConvert.SerializeObject(target.Message)); dynamic eventMsg; if (!isIncludeID) { eventMsg = JsonConvert.DeserializeObject <CreateCustomerEvent>( json.SelectToken($"CreateCustomerEvent").ToString()); } else { eventMsg = JsonConvert.DeserializeObject <UpdateCustomerEvent>( json.SelectToken($"UpdateCustomerEvent").ToString()); } return(CustomerTopics.Contains(target.Topic) && source.CustomerUID.ToString() == target.Key && eventMsg.CustomerName == source.CustomerName && eventMsg.CustomerUID == source.CustomerUID && eventMsg.BSSID == source.BSSID && eventMsg.DealerNetwork == source.DealerNetwork && eventMsg.NetworkDealerCode == source.NetworkDealerCode && eventMsg.DealerAccountCode == source.DealerAccountCode && eventMsg.NetworkCustomerCode == source.NetworkCustomerCode && eventMsg.FirstName == source.FirstName && eventMsg.LastName == source.LastName && eventMsg.PrimaryContactEmail == source.PrimaryContactEmail && !isIncludeID ? true : sourceCustomerDetail.fk_CustomerTypeID == (int)custType); }
public bool?DeleteUserPreference(DeleteUserPreferenceEvent userPreference) { var dbUserPreference = GetUserPreference(userPreference.UserUID, userPreference.PreferenceKeyUID, userPreference.PreferenceKeyName); if (dbUserPreference == null) { return(null); } var deleteQuery = string.Format( "Delete from md_preference_PreferenceUser where fk_UserUID = {0} and fk_PreferenceKeyID = {1}", userPreference.UserUID.ToStringAndWrapWithUnhex(), dbUserPreference.fk_PreferenceKeyID); var kafkaMessage = new KafkaMessage() { Key = userPreference.UserUID.ToString(), Message = userPreference }; var actions = new List <Action>(); actions.Add(() => transaction.Delete(deleteQuery)); actions.Add(() => topics.ToList().ForEach(topic => { kafkaMessage.Topic = topic; transaction.Publish(kafkaMessage); })); return(transaction.Execute(actions)); }
public bool?CreatePreferenceKey(CreatePreferenceKeyEvent preferenceKey) { if (GetPreferenceKey(preferenceKey.PreferenceKeyUID) != null || GetPreferenceKey(null, preferenceKey.PreferenceKeyName) != null) { return(null); } var currentUtc = DateTime.UtcNow; var createPreferenceKey = new DbPreferenceKey() { PreferenceKeyName = preferenceKey.PreferenceKeyName, PreferenceKeyUID = preferenceKey.PreferenceKeyUID.Value, InsertUTC = currentUtc, UpdateUTC = currentUtc }; var kafkaMessage = new KafkaMessage() { Key = preferenceKey.PreferenceKeyUID.ToString(), Message = new { CreateUserPreferenceKeyEvent = preferenceKey } }; var actions = new List <Action>(); actions.Add(() => transaction.Upsert <DbPreferenceKey>(createPreferenceKey)); actions.Add(() => topics.ToList().ForEach(topic => { kafkaMessage.Topic = topic; transaction.Publish(kafkaMessage); })); return(transaction.Execute(actions)); }
public async Task <InitiateKycResponseModel> InitiateKycAsync(InitiateKycRequestModel requestModel) { var kycEntity = await _kycRepository.AddAsync(new KycEntity(requestModel.UserId, KycStatus.Initiated)); var mrzSubmitRequest = _mapper.Map <MrzSubmitRequest>(requestModel); var mrzSubmitResponse = _mrzProcessor.ProcessMrzFile(mrzSubmitRequest); kycEntity.Status = KycStatus.Queued; await _kycRepository.UpdateAsync(kycEntity); var responseModel = _mapper.Map <InitiateKycResponseModel>(requestModel); responseModel.KycId = kycEntity.Id; responseModel.MrzTaskId = mrzSubmitResponse.TaskId; // Send Kafka Message var kafkaMessage = new KafkaMessage <string, InitiateKycResponseModel> { Key = Guid.NewGuid().ToString(), Value = responseModel, MessageType = nameof(InitiateKycResponseModel) }; await _initiateKycProducer.ProduceAsync(kafkaMessage); return(responseModel); }
public async Task <CheckMrzStatusResponseModel> CheckMrzTaskStatusAsync(CheckMrzStatusRequestModel requestModel) { var mrzStatusRequest = _mapper.Map <MrzStatusRequest>(requestModel); var mrzStatusResponse = _mrzProcessor.GetMrzTaskStatus(mrzStatusRequest); var entity = await _kycRepository.GetByKycIdAsync(requestModel.KycId); entity.Status = mrzStatusResponse.IsMrzCompleted ? KycStatus.MrzPassed : KycStatus.MrzFailed; await _kycRepository.UpdateAsync(entity); var responseModel = _mapper.Map <CheckMrzStatusResponseModel>(mrzStatusResponse); responseModel.UserId = requestModel.UserId; responseModel.KycId = requestModel.KycId; if (mrzStatusResponse.IsMrzCompleted) { // Send Kafka Message var kafkaMessage = new KafkaMessage <string, CheckMrzStatusResponseModel> { Key = Guid.NewGuid().ToString(), Value = responseModel, MessageType = nameof(CheckMrzStatusResponseModel) }; await _checkMrzStatusProducer.ProduceAsync(kafkaMessage); } return(responseModel); }
public async Task <VerificationResponseModel> VerifyIdentityAsync(VerificationRequestModel requestModel) { var identityVerificationRequest = _mapper.Map <IdentityVerificationRequest>(requestModel); var verified = await _verifier.VerifyAsync(identityVerificationRequest); var entity = await _kycRepository.GetByKycIdAsync(requestModel.KycId); entity.Status = verified ? KycStatus.VerificationPassed : KycStatus.VerificationFailed; await _kycRepository.UpdateAsync(entity); var responseModel = _mapper.Map <VerificationResponseModel>(identityVerificationRequest); responseModel.UserId = requestModel.UserId; responseModel.IsVerified = verified; // Send Kafka Message var kafkaMessage = new KafkaMessage <string, VerificationResponseModel> { Key = Guid.NewGuid().ToString(), Value = responseModel, MessageType = nameof(VerificationResponseModel) }; await _verificationProducer.ProduceAsync(kafkaMessage); return(responseModel); }
private static void GroupConsuemrTest() { var consumers = new List <KafkaConsumer <string, KafkaMessage> >(); for (var i = 0; i < 1; i++) { consumers.Add(CreateConsumer(commandQueue, i.ToString())); } var queueClient = new KafkaProducer <string, KafkaMessage>(commandQueue, brokerList, new StringSerializer(Encoding.UTF8), new KafkaMessageSerializer()); while (true) { var key = Console.ReadLine(); if (key.Equals("q")) { consumers.ForEach(consumer => consumer.Stop()); queueClient.Stop(); break; } var message = $"{key} @{DateTime.Now:yyyy-MM-dd HH:mm:ss.ffffff}"; var kafkaMessage = new KafkaMessage(message); var start = DateTime.Now; queueClient.SendAsync(key, kafkaMessage, CancellationToken.None) .ContinueWith(t => { var result = t.Result; Console.WriteLine($"send message: {message} partition:{result.Partition} offset:{result.Offset} cost: {(DateTime.Now - start).TotalMilliseconds}"); }); } }
Message <Null, string> GetMessage(IEvent sentEvent) { var attribute = sentEvent.GetType().GetCustomAttribute <EventAttribute> (); if (attribute == null) { throw new ArgumentException($"{nameof (EventAttribute)} missing on {nameof (sentEvent)}"); } if (string.IsNullOrEmpty(attribute.Name)) { throw new ArgumentNullException( $"{nameof (EventAttribute)}.Name missing on {nameof (sentEvent)}"); } var kMessage = new KafkaMessage { Name = attribute.Name, EventData = JsonConvert.SerializeObject(sentEvent, jsonSettings) }; var message = JsonConvert.SerializeObject(kMessage, jsonSettings); return(new Message <Null, string> { Value = message }); }
public async Task Run( [KafkaTrigger("KAFKA_BROKERS", "input-to-process-profile-page", ConsumerGroup = "input-processor-group")] KafkaMessage ev, FunctionContext context) { var logger = context.GetLogger("ProfilePageGeneratorFunc"); logger.LogInformation($"Start ProfilePageGeneratorFunc"); var msg = JsonSerializer.Deserialize <InputToProcessEvent>(ev.Value); if (msg is null) { logger.LogError($"value of event is empty | timestamp: {ev.Timestamp}"); return; } var storageClient = new MinioClient(MinioEndpoint, MinioAccessKey, MinioSecretKey); // 1, Download input file from S3 bucket (minio) var input = await DownloadInputData(logger, storageClient, msg.bucketName, msg.objectName); logger.LogInformation($"processing {input.template} input"); // 2. Download html template from s3 repo (should sync by other periodic function) var template = await GetTemplate(input.template); // 3. Combine these two sources into one html file var output = GenerateOutput(input.data, template); // 4. Put generated file to the S3 await UploadGeneratedOutput(storageClient, output, input.template); logger.LogInformation($"end ProfilePageGeneratorFunc"); }
private bool ValidateCustomerUserKafkaObject(bool isDissociate, KafkaMessage target, dynamic source) { var isValid = target.Key == source.CustomerUID.ToString() && CustomerTopics.Contains(target.Topic); var json = JObject.Parse(JsonConvert.SerializeObject(target.Message)); if (isDissociate) { var eventMsg = JsonConvert.DeserializeObject <DissociateCustomerUserEvent>( json.SelectToken("DissociateCustomerUserEvent").ToString()); return(isValid && eventMsg.CustomerUID == source.CustomerUID && eventMsg.UserUID == source.UserUID && eventMsg.ActionUTC == source.ActionUTC && eventMsg.ReceivedUTC == source.ReceivedUTC); } else { var eventMsg = JsonConvert.DeserializeObject <AssociateCustomerUserEvent>( json.SelectToken("AssociateCustomerUserEvent").ToString()); return(isValid && eventMsg.CustomerUID == source.CustomerUID && eventMsg.UserUID == source.UserUID && eventMsg.ActionUTC == source.ActionUTC && eventMsg.ReceivedUTC == source.ReceivedUTC); } }
public MessageContext(KafkaMessage kafkaMessage, int partition, long offset) { KafkaMessage = kafkaMessage; Offset = offset; Partition = partition; ToBeSentMessageContexts = new List <IMessageContext>(); }
public bool DissociateProjectSubscription(DissociateProjectSubscriptionEvent dissociateSubscription) { var toBeDissociated = GetExistingProjectSubscription(dissociateSubscription.SubscriptionUID); if (toBeDissociated == null) { throw new Exception("Invalid ProjectSubscriptionUID"); } toBeDissociated.fk_ProjectUID = null; toBeDissociated.UpdateUTC = DateTime.UtcNow; var kafkaMessage = new KafkaMessage() { Key = dissociateSubscription.SubscriptionUID.ToString(), Message = new { DissociateProjectSubscriptionEvent = dissociateSubscription } }; var actions = new List <Action>(); actions.Add(() => transaction.Upsert(toBeDissociated)); actions.Add(() => topics.ToList().ForEach(topic => { kafkaMessage.Topic = topic; transaction.Publish(kafkaMessage); })); return(transaction.Execute(actions)); }
private async Task <KafkaResult <TKey, TValue, TPass> > SendToProducer(KafkaMessage <TKey, TValue, TPass> msg) { using (var topic = producer.Topic(msg.Record.Topic)) { throw new NotImplementedException(); } }
/// <summary> /// 变更任务状态 /// </summary> /// <param name="id"></param> /// <param name="type"></param> /// <returns></returns> public static string ChangeTaskStatus(int id, int type, string desc, string accid) { string stat = RequirementManageBLL.ChangeTaskStatus(id, type, desc); if (stat != "0" && !string.IsNullOrEmpty(accid) && type == 3) { try { #region Kafka Message 反馈转为需求后 AfterImportReq iModel = new AfterImportReq(); iModel.EventId = 7; iModel.AccId = Convert.ToInt32(accid); iModel.RequirementDesc = desc; string specModel = CommonLib.Helper.JsonSerializeObject(iModel); KafkaMessage mSend = new KafkaMessage(); mSend.SendMsg(7, specModel); #endregion } catch (Exception ex) { Logger.Error("反馈转为需求后推送模板消息错误", ex); } } return(stat); }
public async Task Post([FromBody] KafkaMessage kafkaMessage) { try { int counterObject = 1; while (counterObject <= 5000) { Contact contact = new Contact { CustomeId = counterObject, Name = "vasu", Address = "test123", Email = counterObject + "*****@*****.**", }; string serializedContact = JsonConvert.SerializeObject(contact); kafkaMessage.Message = serializedContact; await _producer.ProduceAsync(kafkaMessage.Topic, kafkaMessage.Message); counterObject++; } } catch (Exception ex) { throw; } finally { } }
public async Task Run( [KafkaTrigger("KAFKA_BROKERS", "minio-events-inputs", ConsumerGroup = "input-processor-group")] KafkaMessage ev, FunctionContext executionContext) { var logger = executionContext.GetLogger("InputProcessorFunc"); logger.LogInformation("InputProcessorFunc start"); var minioEvent = JsonSerializer.Deserialize <S3ObjectEvent>(ev.Value); switch (minioEvent.EventName) { case "s3:ObjectRemoved:Delete": logger.LogInformation($"deleted {minioEvent.Key}"); break; case "s3:ObjectCreated:Put": { await InputObjectCreated(logger, minioEvent); break; } default: logger.LogInformation($"no executor for {minioEvent.EventName}"); break; } }
/// <summary> /// Task that write data in kafka every 1 second to emulate updates /// </summary> private static async Task FillKafkaWithDataAsync(string topicName, CancellationToken ct) { var config = new ProducerConfig { BootstrapServers = GetServers() }; using var p = new ProducerBuilder <Null, string>(config).Build(); while (!ct.IsCancellationRequested) { try { var newMessage = new KafkaMessage(_rnd.Next(1, 5), Guid.NewGuid()); var stringMessage = JsonConvert.SerializeObject(newMessage); var dr = await p.ProduceAsync(topicName, new Message <Null, string> { Value = stringMessage }, ct); Console.WriteLine($"Delivered '{dr.Value}' to '{dr.TopicPartitionOffset}'"); } catch (ProduceException <Null, string> e) { Console.WriteLine($"Delivery failed: {e.Error.Reason}"); } await Task.Delay(1000, ct); } }
public bool AssociateAssetDevice(AssociateDeviceAssetEvent associateDeviceAsset) { associateDeviceAsset.ReceivedUTC = DateTime.UtcNow; var assetDevice = new DbAssetDevice { fk_AssetUID = associateDeviceAsset.AssetUID, fk_DeviceUID = associateDeviceAsset.DeviceUID, ActionUTC = associateDeviceAsset.ActionUTC.Value, RowUpdatedUTC = associateDeviceAsset.ReceivedUTC }; var kafkaMessageList = new List <KafkaMessage>(); kafkaTopicNames.ForEach(topic => { var kafkaMessage = new KafkaMessage() { Key = associateDeviceAsset.DeviceUID.ToString(), Message = new { AssociateDeviceAssetEvent = associateDeviceAsset }, Topic = topic }; kafkaMessageList.Add(kafkaMessage); }); var actions = new List <Action>(); actions.Add(() => transactions.Upsert(assetDevice)); actions.Add(() => transactions.Publish(kafkaMessageList)); return(transactions.Execute(actions)); }
public static async Task ProduceMessage(MainConfig cfg, KafkaMessage kafkaMessage, ILogger logger) { var config = new ProducerConfig { BootstrapServers = cfg.Kafka.BootstrapServers, ClientId = cfg.Kafka.ClientId }; using (var producer = new ProducerBuilder <Null, string>(config).Build()) { var produceAsync = producer.ProduceAsync(cfg.Kafka.Topic, new Message <Null, string> { Value = JsonConvert.SerializeObject(kafkaMessage, Formatting.Indented) }); await produceAsync.ContinueWith(task => { if (task.IsFaulted) { logger.Error("Producing message faulted"); throw new Exception("FileGenerator was unable to send the message to Kafka"); } logger.Information($"Wrote to offset: {task.Result.Offset}"); }); } }
private List <KafkaMessage> GetGetAssetSourceSnapshotTopublish(dynamic assetDetails, DateTime receivedUTC, bool isAssetUpdate = false) { var kafkaMessageList = new List <KafkaMessage>(); if (isAssetUpdate) { assetTopicNames.ToList().ForEach(topic => { KafkaMessage kafkaMessage = new KafkaMessage() { Key = assetDetails.AssetUID.ToString(), Message = new { UpdateAssetEvent = new { AssetUID = assetDetails.AssetUID, LegacyAssetID = assetDetails.LegacyAssetID, Model = assetDetails.Model, AssetType = assetDetails.AssetTypeName, IconKey = assetDetails.IconKey, EquipmentVIN = assetDetails.EquipmentVIN, ModelYear = assetDetails.ModelYear, OwningCustomerUID = assetDetails.OwningCustomerUID, ObjectType = assetDetails.ObjectType, Category = assetDetails.Category, ProjectStatus = assetDetails.ProjectStatus, SortField = assetDetails.SortField, UserEnteredRuntimeHours = assetDetails.UserEnteredRuntimeHours, Classification = assetDetails.Classification, PlanningGroup = assetDetails.PlanningGroup, Source = assetDetails.Source, AssetName = assetDetails.AssetName, ActionUTC = assetDetails.UpdateUTC, ReceivedUTC = receivedUTC, } }, Topic = topic }; kafkaMessageList.Add(kafkaMessage); }); } else { var messagePayload = AssetSnapshotToPublish(assetDetails); messagePayload.ReceivedUTC = receivedUTC; messagePayload.ActionUTC = assetDetails.InsertUTC; assetTopicNames.ToList().ForEach(topic => { KafkaMessage kafkaMessage = new KafkaMessage() { Key = assetDetails.AssetUID.ToString(), Message = new { CreateAssetEvent = messagePayload }, Topic = topic }; kafkaMessageList.Add(kafkaMessage); }); } return(kafkaMessageList); }
protected override AggregatableMessage <ICommand> Process(KafkaMessage kafkaMessage) { return(new AggregatableMessage <ICommand> { TargetFlow = MessageFlow, Commands = _commandFactory.CreateCommands(kafkaMessage), }); }
public bool UpdateAccount(UpdateCustomerEvent updateAccount, DbAccount accountDetails) { try { var accountEvent = new AccountEvent { AccountName = updateAccount.CustomerName, AccountUID = updateAccount.CustomerUID, Action = Operation.Update.ToString(), BSSID = updateAccount.BSSID, DealerAccountCode = updateAccount.DealerAccountCode, NetworkCustomerCode = updateAccount.NetworkCustomerCode, fk_ParentCustomerUID = accountDetails.fk_ParentCustomerUID ?? null, fk_ChildCustomerUID = accountDetails.fk_ChildCustomerUID ?? null, ActionUTC = updateAccount.ActionUTC, ReceivedUTC = updateAccount.ReceivedUTC }; if (!FieldHelper.IsValidValuesFilled(accountEvent, accountDetails, logger)) { logger.LogError("DB Object expects typeOf IDbTable"); } FieldHelper.ReplaceEmptyFieldsByNull(accountEvent); var message = new KafkaMessage { Key = updateAccount.CustomerUID.ToString(), Topic = AccountTopic, Message = new { AccountEvent = accountEvent } }; var accountObj = new DbAccount { CustomerAccountID = accountDetails.CustomerAccountID, CustomerAccountUID = accountEvent.AccountUID, BSSID = accountEvent.BSSID, AccountName = accountEvent.AccountName, NetworkCustomerCode = accountEvent.NetworkCustomerCode, DealerAccountCode = accountEvent.DealerAccountCode, fk_ChildCustomerUID = accountDetails.fk_ChildCustomerUID ?? null, fk_ParentCustomerUID = accountDetails.fk_ParentCustomerUID ?? null, RowUpdatedUTC = DateTime.UtcNow }; var actions = new List <Action>() { () => transaction.Upsert(accountObj), () => transaction.Publish(message) }; return(transaction.Execute(actions)); } catch (Exception ex) { logger.LogError($"Error while updating account customer : {ex.Message}, {ex.StackTrace}"); throw; } }
Type GetHandlerType(KafkaMessage message) { if (message.Name == null) { throw new ArgumentNullException($"{nameof (Consumer)} exception: event Name is missing"); } return(options.Handlers.TryGetValue(message.Name, out var handlerType) ? handlerType : null); }
protected virtual async Task ProcessConsumeError(Exception ex, KafkaMessage kMessage) { if (kMessage != null) { ex.Data.Add("IntegrationMessage", kMessage); } logger.LogError(ex, "Error consuming message from the broker"); }
private byte[] Serialize(KafkaMessage km) { using (var ms = new MemoryStream()) { Serializer.Serialize(ms, km); return(ms.ToArray()); } }
public static string AddNewReqModel(int accId, int refId, int cateId, int reqType, string desc, int op, string originDesc, int device, int val, int diff) { Model.RequirementManage model = new Model.RequirementManage(); model.AccId = accId.ToString(); model.RefId = refId; model.CategoryId = cateId; model.RequirementType = reqType; model.Description = desc; model.Operator = op; model.Status = 0; model.OriginDesc = originDesc; model.Terminal = device; model.UserVal = val; model.Difficult = diff; int reqId = RequirementManageBLL.AddNewModel(model); if (reqId != 0) { if (Sys_TaskDailyBLL.UpdateReqId(reqId, refId) != 0) { try { #region Kafka Message 反馈转为需求后 AfterImportReq iModel = new AfterImportReq(); iModel.EventId = 6; iModel.AccId = accId; iModel.RequirementDesc = desc; string specModel = CommonLib.Helper.JsonSerializeObject(iModel); KafkaMessage mSend = new KafkaMessage(); mSend.SendMsg(6, specModel); #endregion } catch (Exception ex) { Logger.Error("反馈转为需求后推送模板消息错误", ex); } return("添加成功!"); } else { return("添加关联信息出错!请联系技术~"); } } else { return("添加新的需求记录出错!请联系技术~"); } }
/// <summary> /// Producer will deliver message to consumers in "random" order. /// Allows better performance by using all partitions. /// </summary> public async Task ProduceAsync(KafkaMessage message) { var result = await _producer.ProduceAsync( message.MetaData.MessageType.ToString(), new Message <string, string> { Key = null, Value = message.ToJson() }); Output(result); }
public async Task <IActionResult> PostItemAsync(string message) { var kafkaMessage = new KafkaMessage { Id = Guid.NewGuid(), MessageValue = message }; this.kafkaMessenger.MessageProducer(kafkaMessage); return(StatusCode(StatusCodes.Status201Created)); }
private KafkaMessage GenerateTestKM(string payload, DateTime messageGeneratedDateTime) { var ret = new KafkaMessage() { Id = (new Guid()).ToString(), DatetimeCreatedUtc = messageGeneratedDateTime.ToString("dd-MMM-yyyy HH:mm:ss"), Payload = payload, Source = "Test" }; return(ret); }