public async Task AddOrUpdateEvent(LogEvent logevents, InvitationLogEvent logevent = null, string tokenId = null) { try { if (string.IsNullOrWhiteSpace(logevents.Id)) { logevents.Id = ObjectId.GenerateNewId().ToString(); } logevents.TokenId = tokenId; var utcNow = DateTime.UtcNow; var filter = Builders <LogEvent> .Filter.Eq(x => x.Id, logevents.Id); var update = Builders <LogEvent> .Update; List <UpdateDefinition <LogEvent> > updates = new List <UpdateDefinition <LogEvent> >(); updates.Add(update.SetOnInsert(x => x.Created, utcNow)); updates.Add(update.SetOnInsert(x => x.Location, logevents.Location)); updates.Add(update.SetOnInsert(x => x.DeliveryWorkFlowId, logevents.DeliveryWorkFlowId)); updates.Add(update.SetOnInsert(x => x.Target, logevents.Target)); updates.Add(update.SetOnInsert(x => x.TargetHashed, logevents.TargetHashed)); updates.Add(update.SetOnInsert(x => x.LogMessage, logevents.LogMessage)); updates.Add(update.SetOnInsert(x => x.Events, logevents.Events)); if (logevent != null) { logevent.TimeStamp = utcNow; updates.Add(update.Push(x => x.Events, logevent)); } updates.Add(update.Set(x => x.Updated, utcNow)); var opt = new FindOneAndUpdateOptions <LogEvent> { IsUpsert = true, ReturnDocument = ReturnDocument.After }; var up = update.Combine(updates); await _EventLog.FindOneAndUpdateAsync(filter, up, opt); } catch (Exception ex) { await AddExceptionEvent(ex); } }
public async Task CheckDispatchData(List <DispatchRequest> batchRequests, string batchID, BatchResponse batchResponse) { try { HashAlgos hashAlgos = new HashAlgos(); CorrectDispatchData = new Dictionary <string, RequestPrefill>(); string wxmHashAlgo = string.Empty; BatchId = batchID; Settings settingsRes = null; settingsRes = JsonConvert.DeserializeObject <Settings>(DispatchSettings); //Fetching all Dispatches var allDispatches = JsonConvert.DeserializeObject <List <Dispatch> >(DispatchData); CheckDispatchID(ref batchRequests, ref batchResponse, allDispatches); //Check value in CorrectDispatchData if (CorrectDispatchData.Count == 0) { EventLogList.AddEventByLevel(2, SharedSettings.NovalidDispatchInTheBatch, BatchId); return; } var deliveryPlans = JsonConvert.DeserializeObject <List <DeliveryPlan> >(DeliveryPlanData); var activeQuestions = JsonConvert.DeserializeObject <List <Question> >(ActiveQuestions); //getting channels from delivery plan GetChannelFromDP(ref batchResponse, deliveryPlans); DateTime utcNow = DateTime.UtcNow; List <LogEvent> batchLogEvents = new List <LogEvent>(); foreach (var dispatch in CorrectDispatchData) { //If one dispatch fails the entire operation shouldn't fail. try { if (!islocationMigrated) { wxmHashAlgo = settingsRes?.locationList.Find(x => x.Name == dispatch.Value.QuestionnaireName)?.HashPIIBy; } else { if (!string.IsNullOrEmpty(SurverQuestionnaires)) { List <SurveyQuestionnaire> surveyQuestionnaire = JsonConvert.DeserializeObject <List <SurveyQuestionnaire> >(SurverQuestionnaires); wxmHashAlgo = surveyQuestionnaire?.Find(x => x.Name == dispatch.Value.QuestionnaireName)?.HashPIIBy; } } if (string.IsNullOrEmpty(wxmHashAlgo)) { EventLogList.AddEventByLevel(5, SharedSettings.NoHashAlgoConfigured, batchID, dispatch.Key); } else { EventLogList.AddEventByLevel(5, $"{SharedSettings.HashAlgoConfigured} {wxmHashAlgo}", batchID, dispatch.Key); } RequestBulkToken requestBulk = new RequestBulkToken() { DispatchId = dispatch.Key, PrefillReponse = new List <List <Response> >() }; int prefillFailCount = 0; List <List <Response> > prefillResponses = new List <List <Response> >(); Question batchprefill = new Question(); List <string> invalidQuestionIdOrPrefill = new List <string>(); foreach (var prefill in dispatch.Value.PreFill) { int recordChannel = 0; List <Response> responses = new List <Response>(); LogEvent logEvent = new LogEvent() { Id = ObjectId.GenerateNewId().ToString(), Created = utcNow, DispatchId = dispatch.Key, BatchId = batchID, DeliveryWorkFlowId = dispatch.Value.DeliveryPlanID, Location = dispatch.Value.QuestionnaireName, Prefills = new List <Prefill>(), Tags = new List <string> { "UserData" } }; bool failureFlag = false; bool uuidrecord = false; foreach (var record in prefill) { var question = activeQuestions.Find(x => x.Id == record.QuestionId && (x.StaffFill || x.ApiFill)); if (question == null) { invalidQuestionIdOrPrefill.Add(record.QuestionId); continue; } Response response = new Response { QuestionId = question.Id }; //Channel Check if (question.QuestionTags != null && question.QuestionTags.Contains("Email")) { // Email question if (dispatch.Value.Channels.Contains("Email")) { recordChannel++; bool emailStatus = Util.IsValidEmail(record.Input); if (!emailStatus) { failureFlag = true; } } } else if (question.QuestionTags != null && question.QuestionTags.Contains("Mobile")) { // SMS question if (dispatch.Value.Channels.Contains("SMS")) { recordChannel++; bool numberStatus = Util.IsValidMobile(record.Input); if (!numberStatus) { failureFlag = true; } } } //Common identifier check. if (question.Id == dispatch.Value.UniqueCustomerIDByPreFilledQuestionTag) { if (!string.IsNullOrWhiteSpace(record.Input)) { uuidrecord = true; } logEvent.Target = record.Input; if ((question.piiSettings != null) && (question.piiSettings.isPII && (question.piiSettings.piiType == "hash"))) { var hashedrecord = hashAlgos.GetHashedValue(record.Input, wxmHashAlgo); logEvent.TargetHashed = hashedrecord; } else { logEvent.TargetHashed = record.Input; } } //Normal prefill check if ((question.piiSettings != null) && (question.piiSettings.isPII && (question.piiSettings.piiType == "hash"))) { logEvent.Prefills.Add(new Prefill() { QuestionId = question.Id, Input = record.Input, Input_Hash = hashAlgos.GetHashedValue(record.Input, wxmHashAlgo) }); response.TextInput = hashAlgos.GetHashedValue(record.Input, wxmHashAlgo); } else { logEvent.Prefills.Add(new Prefill() { QuestionId = question.Id, Input = record.Input, Input_Hash = record.Input }); if (numberTypeRegEx.IsMatch(question.DisplayType)) { if (int.TryParse(record.Input, out int res)) { response.NumberInput = res; } } else { response.TextInput = record.Input; } } responses.Add(response); } var invitationLogEvent = new InvitationLogEvent() { Action = InvitationLogEvent.EventAction.Requested, Channel = InvitationLogEvent.EventChannel.DispatchAPI, TimeStamp = utcNow }; //Check for invalid mobile number or email if (failureFlag) { prefillFailCount++; invitationLogEvent.Action = InvitationLogEvent.EventAction.Rejected; invitationLogEvent.LogMessage = new LogMessage() { Message = SharedSettings.FailDueToEmailOrMobile }; logEvent.Events = new List <InvitationLogEvent>() { invitationLogEvent }; batchLogEvents.Add(logEvent); continue; } //Check for no channel question in record and Check for Common Identifier if (recordChannel == 0 || !uuidrecord) { prefillFailCount++; invitationLogEvent.Action = InvitationLogEvent.EventAction.Rejected; invitationLogEvent.LogMessage = new LogMessage() { Message = SharedSettings.FailDueToUUIDOrChannel }; logEvent.Events = new List <InvitationLogEvent>() { invitationLogEvent }; batchLogEvents.Add(logEvent); continue; } // Check for Duplication if (uuidrecord) { var dupRecord = batchLogEvents.Find(x => x.DispatchId == dispatch.Key && (x.Target?.ToLower() == logEvent.Target?.ToLower())); if (dupRecord != null) { prefillFailCount++; invitationLogEvent.Action = InvitationLogEvent.EventAction.Throttled; invitationLogEvent.LogMessage = new LogMessage() { Message = $"{ SharedSettings.DuplicateRecord} : {logEvent.Target}" }; logEvent.Events = new List <InvitationLogEvent>() { invitationLogEvent }; batchLogEvents.Add(logEvent); continue; } } // Unsubscribe data check bool unsubscribestatus = false; if (accountConfiguration.ExtendedProperties.TryGetValue("Unsubscriber", out string unsubcriberName)) { if (SharedSettings.AvailableUnsubscribeCheckers.TryGetValue(unsubcriberName, out IUnsubscribeChecker unsubscribeChecker)) { unsubscribestatus = await unsubscribeChecker.IsUnsubscribedAsync(invitationLogEvent.TargetId?.ToLower()); } } if (unsubscribestatus) { prefillFailCount++; invitationLogEvent.Action = InvitationLogEvent.EventAction.Supressed; logEvent.Events = new List <InvitationLogEvent>() { invitationLogEvent }; batchLogEvents.Add(logEvent); continue; } // Add DPID prefill for each record var dpPrefill = activeQuestions.Find(x => x.QuestionTags.Contains("DeliveryPlanId")); if (dpPrefill != null) { AddToResponsesAndEventLogs(ref responses, ref logEvent, dpPrefill.Id, dispatch.Value.DeliveryPlanID); } // Add batchID prefill for each record batchprefill = activeQuestions.Find(x => x.QuestionTags.Contains("BatchId")); if (batchprefill != null) { AddToResponsesAndEventLogs(ref responses, ref logEvent, batchprefill.Id, batchID); } // Add static prefills from AccountConfigurations set using SPA front-end var dispatchChannel = accountConfiguration.DispatchChannels?.Find(x => x.DispatchId == dispatch.Key); if (dispatchChannel != null) { foreach (var staticPrefill in dispatchChannel.StaticPrefills) { if (!string.IsNullOrWhiteSpace(staticPrefill.PrefillValue)) { var temp = responses.Find(x => x.QuestionId == staticPrefill.QuestionId); if (temp != null) { // Remove the old values and override with this one AddToResponsesAndEventLogs(ref responses, ref logEvent, staticPrefill.QuestionId, staticPrefill.PrefillValue, true); } else { // Add new value AddToResponsesAndEventLogs(ref responses, ref logEvent, staticPrefill.QuestionId, staticPrefill.PrefillValue); } } } } logEvent.Events = new List <InvitationLogEvent>() { invitationLogEvent }; batchLogEvents.Add(logEvent); //Add records to form Bulk token Request. prefillResponses.Add(responses); } // reject if all records are invalid if (prefillFailCount == dispatch.Value.PreFill.Count) { batchResponse.StatusByDispatch.Add(new StatusByDispatch() { DispatchId = dispatch.Key, DispatchStatus = "400", Message = SharedSettings.AllRecordsRejected }); EventLogList.AddEventByLevel(2, SharedSettings.AllRecordsRejected, BatchId, dispatch.Key); } else { if (invalidQuestionIdOrPrefill?.Count > 0) { //few records from the invites being removed with some question id EventLogList.AddEventByLevel(4, $"{SharedSettings.PrefillsMissing} {string.Join(',', invalidQuestionIdOrPrefill)}", BatchId, dispatch.Key); } requestBulk.PrefillReponse = prefillResponses; requestBulk.UUID = dispatch.Value.UniqueCustomerIDByPreFilledQuestionTag; requestBulk.Batchid = batchprefill?.Id; // Add records in batching queue if (accountConfiguration.ExtendedProperties.TryGetValue("BatchingQueue", out string queueName)) { if (SharedSettings.AvailableQueues.TryGetValue(queueName, out IBatchingQueue <RequestBulkToken> batchingQueue)) { batchingQueue.Insert(requestBulk); EventLogList.AddEventByLevel(5, $"{prefillResponses?.Count ?? 0} records added to queue {queueName} for bulk token creation", BatchId, dispatch.Key); } } else { EventLogList.AddEventByLevel(1, SharedSettings.BatchingQueueMissing, BatchId, dispatch.Key); return; } if (prefillFailCount == 0) { batchResponse.StatusByDispatch.Add(new StatusByDispatch() { DispatchId = dispatch.Key, DispatchStatus = "202", Message = SharedSettings.AcceptedForProcessing //When all records for DispatchID were successful. }); EventLogList.AddEventByLevel(5, $"{SharedSettings.AcceptedForProcessing}: {dispatch.Value.PreFill.Count} records", BatchId, dispatch.Key); } else { string message = SharedSettings.AcceptedForProcessing + " " + (dispatch.Value.PreFill.Count - prefillFailCount).ToString() + " " + "Rejected: " + prefillFailCount.ToString(); batchResponse.StatusByDispatch.Add(new StatusByDispatch() { DispatchId = dispatch.Key, DispatchStatus = "206", Message = message //Partial records were successfull. }); EventLogList.AddEventByLevel(5, message, BatchId, dispatch.Key); } } } catch (Exception ex0) { EventLogList.AddExceptionEvent(ex0, batchID, dispatch.Key, dispatch.Value?.DeliveryPlanID, dispatch.Value?.QuestionnaireName, SharedSettings.CheckDispatchDataEx1); return; } } //add for the entire batch await mongoDBConn.AddBulkEvents(batchLogEvents); } catch (Exception ex) { EventLogList.AddExceptionEvent(ex, batchID, null, null, null, SharedSettings.CheckDispatchDataEx2); return; } }
public async Task ReadQueue(CancellationToken cancellationToken) { EventLogList eventLog = new EventLogList(); try { DateTime stopTime = DateTime.Now.AddSeconds(30); Dictionary <string, RequestBulkToken> finalBulkStorage = new Dictionary <string, RequestBulkToken>(); while (DateTime.Now < stopTime) { RequestBulkToken requestBulkToken; if (!SingletonConcurrentQueue <RequestBulkToken> .Instance.TryPeek(out requestBulkToken)) { break; } else if (SingletonConcurrentQueue <RequestBulkToken> .Instance.TryDequeue(out requestBulkToken)) { if (finalBulkStorage.ContainsKey(requestBulkToken.DispatchId)) { finalBulkStorage[requestBulkToken.DispatchId].PrefillReponse.AddRange(requestBulkToken.PrefillReponse); } else { finalBulkStorage.Add(requestBulkToken.DispatchId, new RequestBulkToken() { DispatchId = requestBulkToken.DispatchId, UUID = requestBulkToken.UUID, Batchid = requestBulkToken.Batchid, PrefillReponse = requestBulkToken.PrefillReponse }); } } } if (finalBulkStorage.Count != 0) { HTTPWrapper hTTPWrapper = new HTTPWrapper(string.Empty, eventLog); string authToken = InvitationsMemoryCache.GetInstance().GetFromMemoryCache("AuthToken"); if (authToken == null) { AccountConfiguration accountConfiguration; string accountConfigurationCache = InvitationsMemoryCache.GetInstance().GetFromMemoryCache("accountconfig"); if (accountConfigurationCache == null) { accountConfiguration = await viaMongoDB.GetAccountConfiguration(); } else { accountConfiguration = Newtonsoft.Json.JsonConvert.DeserializeObject <AccountConfiguration>(accountConfigurationCache); } string username = accountConfiguration.WXMAdminUser; string apikey = accountConfiguration.WXMAPIKey; string responseBody = await hTTPWrapper.GetLoginToken(username, apikey); if (!string.IsNullOrEmpty(responseBody)) { BearerToken loginToken = Newtonsoft.Json.JsonConvert.DeserializeObject <BearerToken>(responseBody); authToken = "Bearer " + loginToken.AccessToken; var Expirationtime = loginToken.ExpiresIn - 300; // Expire 5 min before for uninterrupted token creation InvitationsMemoryCache.GetInstance().SetBulkTokenAuthToMemoryCache("AuthToken", authToken, Expirationtime); } else { //when login token api failed. eventLog.AddEventByLevel(1, SharedSettings.BearerTokenNotGenerated, null); await eventLog.AddEventLogs(viaMongoDB); } } // Calling bulk token api sequentially if (!string.IsNullOrWhiteSpace(authToken)) { List <(string, List <BulkTokenResult>)> status = new List <(string, List <BulkTokenResult>)>(); foreach (var request in finalBulkStorage) { var response = await hTTPWrapper.BulkTokenAPI(authToken, request.Value); status.Add(response); Thread.Sleep(1000); // Sleep for 1 second before making another call } /* * var bulkTokenAPITasks = finalBulkStorage.Values.ToList().Select(v => * { * return hTTPWrapper.BulkTokenAPI(authToken, v); * }); * * (string, List<BulkTokenResult>)[] status = await Task.WhenAll(bulkTokenAPITasks); * */ Dictionary <LogEvent, InvitationLogEvent> events = new Dictionary <LogEvent, InvitationLogEvent>(); DateTime utcNow = DateTime.UtcNow; //Update tokens in DB foreach (var item in status) { if (item.Item2 != null) { foreach (var perinvite in item.Item2) { var logEvent = new LogEvent() { DispatchId = item.Item1, BatchId = perinvite.Batchid, Updated = utcNow, TokenId = perinvite.Token, TargetHashed = perinvite.UUID }; var invitationEvent = new InvitationLogEvent() { Action = InvitationLogEvent.EventAction.TokenCreated, Channel = InvitationLogEvent.EventChannel.DispatchAPI, TimeStamp = utcNow, TargetId = perinvite.UUID }; events.Add(logEvent, invitationEvent); } } } if (events.Count() > 0) { await viaMongoDB.UpdateBulkEventLog(events); } eventLog.AddEventByLevel(5, $"{SharedSettings.DBUpdateCompleted} {events.Count()}", null); await eventLog.AddEventLogs(viaMongoDB); } } } catch (Exception ex) { eventLog.AddExceptionEvent(ex, null, null, null, null, SharedSettings.BulkTokenException); await eventLog.AddEventLogs(viaMongoDB); return; } }