/// <summary>
/// Insert 1 single FMGGroup + RemoteGroupMapping
/// </summary>
/// <param name="partyId"></param>
/// <param name="newTopic"></param>
/// <param name="logTrace"></param>
protected async Task InsertNewGroupAsync(int partyId, SalesforceTopicMapping newTopic, ILogTrace logTrace)
{
// insert FMG Group
var res = await AppStore.ContactsServiceClient.AddGroupAsync(
new AddGroupRequest { PartyID = partyId, Name = newTopic.SalesforceTopicName }
);
logTrace.Add(LogLevel.Info, "InsertNewGroupAsync() - Done"
, new { newTopic.SalesforceTopicName, res.Status });
if (res.Status != HttpStatusCode.Created)
{
logTrace?.Add(LogLevel.Error, "InsertNewGroupAsync() - Failed",
new { newTopic.SalesforceTopicName, res });
throw new Exception($"Failed to insert group {newTopic.SalesforceTopicName}");
}
// insert the RemoteGroupMapping
var groupId = res.Data.GroupID;
var upsertRemoteGroupMappingResponse = await AppStore.RemoteDataServiceClient
.UpsertRemoteGroupMappingAsync(
new UpsertRemoteGroupMappingRequestData
{
PartyId = partyId,
GroupId = groupId,
RemoteGroupId = newTopic.SalesforceTopicId,
RemoteGroupName = newTopic.SalesforceTopicName
}, logTrace);
logTrace.Add(LogLevel.Info, "UpsertRemoteGroupMapping() - Done"
, new { newTopic.SalesforceTopicName, upsertRemoteGroupMappingResponse.Status });
}
public override async Task <IList <TMessage> > ParseMessages(object reqData, ILogTrace logTrace)
{
var strMessage = JsonConvert.SerializeObject(reqData);
logTrace?.Add(LogLevel.Debug, "ParseMessages()", new { strMessage });
var sqsEvent = JsonConvert.DeserializeObject <SQSEvent>(strMessage);
logTrace?.Add(LogLevel.Info, "ParseMessages()", "Parsed SQSEvent");
return(ParseMessages(sqsEvent));
}
protected async Task InsertNewGroupsAsync(int partyId,
IList <SalesforceTopicMapping> salesforceTopicMappings,
IList <SalesforceTopicMapping> remoteTopicMappings,
IList <string> filteredRemoteGroupIds,
ILogTrace logTrace)
{
var startedAt = DateTime.UtcNow;
// logging
logTrace?.Add(LogLevel.Debug, "salesforceTopicTopicMappings", salesforceTopicMappings
.Select(topic => new { topic.SalesforceTopicId, topic.SalesforceTopicName })
.ToList());
// find the new topics
var newTopicMappings =
salesforceTopicMappings.Except(remoteTopicMappings, AppStore.SalesforceTopicMappingComparer).ToList();
logTrace?.Add(LogLevel.Debug, "newtopicMappings",
newTopicMappings.Select(topic => new { topic.SalesforceTopicId, topic.SalesforceTopicName }).ToList());
if (filteredRemoteGroupIds.Any())
{
var filteredTopicMappings = filteredRemoteGroupIds.Select(
remoteGroupId => new SalesforceTopicMapping {
SalesforceTopicId = remoteGroupId
});
newTopicMappings =
newTopicMappings.Intersect(filteredTopicMappings, AppStore.SalesforceTopicMappingComparer).ToList();
logTrace?.Add(LogLevel.Debug, "newtopicMappings after intersect",
newTopicMappings.Select(topic => new { topic.SalesforceTopicId, topic.SalesforceTopicName })
.ToList());
}
// logging
logTrace?.Add(
LogLevel.Info,
"New Groups",
newTopicMappings.Select(topic => topic.SalesforceTopicName).ToList()
);
// insert these as FMGGroups
var tasks = newTopicMappings.Select(newTopic => InsertNewGroupAsync(partyId, newTopic, logTrace)).ToList();
await Task.WhenAll(tasks);
// log
logTrace?.Add(
LogLevel.Info, "InsertNewGroupsAsync", $"{newTopicMappings.Count()} new groups inserted", startedAt
);
}
/// <summary>Gets all salesforce objects asynchronous.</summary>
/// <typeparam name="T"></typeparam>
/// <param name="partyId">The party identifier.</param>
/// <param name="getFirstPageFunc">The get first page function.</param>
/// <param name="svcSalesforce">The SVC salesforce.</param>
/// <param name="oauthTokenManager">The oauth token manager.</param>
/// <param name="logTrace">The log trace.</param>
/// <returns></returns>
public static async Task <IList <T> > GetAllSalesforceObjectsAsync <T>(int partyId,
Func <Task <ResponseBase <SalesforceQueryResponseData <T>, IList <SalesforceError> > > > getFirstPageFunc,
ISalesforceServiceClientV2 svcSalesforce, IOAuthTokenManager oauthTokenManager,
ILogTrace logTrace)
{
var startedAt = DateTime.UtcNow;
// send request to get the first batch from Salesforce
var firstTopicsRes = await GetFirstSalesforceObjectPageAsync(partyId,
getFirstPageFunc, svcSalesforce, oauthTokenManager, logTrace);
var sfObjects = firstTopicsRes.Data.Records;
// repeat until all the topics are retrieved, just in case, usually there won't be more than 1000 topics
var nextRecordUrl = firstTopicsRes.Data.NextRecordsUrl;
while (nextRecordUrl != null)
{
var nextTopicsRes = await svcSalesforce.GetNextPageAsync(firstTopicsRes.Data);
nextRecordUrl = nextTopicsRes.Data.NextRecordsUrl;
sfObjects = sfObjects.Concat(nextTopicsRes.Data.Records).ToList();
}
logTrace?.Add(LogLevel.Info, $"GetAllSalesforce{typeof(T).Name}Async() - Done", $"{typeof(T).Name}s #{sfObjects.Count}", startedAt);
return(sfObjects.ToList());
}
protected async Task DeleteGroupsAsync(
int partyId,
IList <SalesforceTopicMapping> salesforceTopicMappings,
IList <SalesforceTopicMapping> remoteTopicMappings,
IList <string> userSelectedGroupIds,
ILogTrace logTrace)
{
var startedAt = DateTime.UtcNow;
// deleted groups are the ones that currently is in the database but not in the remote list
var deletedGroups =
remoteTopicMappings.Except(salesforceTopicMappings, AppStore.SalesforceTopicMappingComparer);
// if user selected to sync only some groups, also delete the groups that is not in this list
if (userSelectedGroupIds.Any())
{
var userSelectedGroups = userSelectedGroupIds.Select(
groupId => new SalesforceTopicMapping {
SalesforceTopicId = groupId
}
).ToList();
var unselectedGroups =
remoteTopicMappings.Except(userSelectedGroups, AppStore.SalesforceTopicMappingComparer);
deletedGroups = deletedGroups.Concat(unselectedGroups).ToList();
}
// delete the FMG Groups first
var groupIds = deletedGroups.Select(group => group.FMGGroupId).Distinct().ToList();
var groupTasks = groupIds.Select(groupId => AppStore.ContactsServiceClient.DeleteGroupAsync(
new DeleteGroupRequest {
GroupID = groupId
}, logTrace
)).ToList();
await Task.WhenAll(groupTasks);
// delete the RemoteGroupMapping after deleting the FMG Groups because the GroupMapping objects hold the
// FMG Group ids
var remoteGroupIds = deletedGroups.Select(group => group.SalesforceTopicId).Distinct().ToList();
var groupMappingTasks = remoteGroupIds.Select(remoteGroupId =>
AppStore.RemoteDataServiceClient.DeleteRemoteGroupMappingAsync(
new DeleteRemoteGroupMappingRequestData
{
PartyId = partyId,
RemoteGroupId = remoteGroupId
}, logTrace
)).ToList();
await Task.WhenAll(groupMappingTasks);
// log
logTrace?.Add(LogLevel.Info, "DeleteGroupsAsync", $"{groupIds.Count()} deleted", startedAt);
}
/// <summary>
/// Processes the message and send next messages if any
/// </summary>
/// <returns>The message async.</returns>
/// <param name="message">Message.</param>
/// <param name="logTrace">Log trace.</param>
async Task ProcessMessageAsync(TMessage message, ILogTrace logTrace)
{
// for debugging if failed to process the message
logTrace?.Add(LogLevel.Debug, "ProcessMessageAsync()", new { message });
// process message based on business logic
var nextMessages = await Handler.ProcessAsync(message, logTrace);
// publish next messages to bus
// (delegate next logic into another async workers)
if (nextMessages != null && nextMessages.Any())
{
await SvcServiceBus.PublishMessages(nextMessages, logTrace);
}
}
/// <summary>
///
/// </summary>
/// <param name="partyId"></param>
/// <param name="logTrace"></param>
/// <returns></returns>
protected async Task <IList <SalesforceTopicMapping> > GetAllRemoteGroupMappingsAsync(
int partyId, ILogTrace logTrace)
{
// send request to get all the RemoteGroupMapping
var remoteGroupMappingRes = await AppStore.RemoteDataServiceClient.GetAllRemoteGroupMappingsAsync(
new GetAllRemoteGroupMappingsRequestData { PartyId = partyId }, logTrace
);
// map to SalesforceTopicMapping schema
var res = remoteGroupMappingRes.Data.RemoteGroupMappings.Select(groupMapping => new SalesforceTopicMapping
{
SalesforceTopicId = groupMapping.RemoteGroupId,
SalesforceTopicName = groupMapping.RemoteGroupName,
FMGGroupId = groupMapping.GroupId
}).ToList();
logTrace.Add(LogLevel.Info, "GetAllRemoteGroupMappingsAsync", $"# GroupMapping: {res.Count}");
return(res);
}
/// <summary>
/// Check the ApproximateReceiveCount with MaximumRetryCount
/// </summary>
/// <param name="message"></param>
/// <param name="reqData"></param>
/// <param name="logTrace"></param>
/// <returns></returns>
private bool IsAlertToSlack(TMessage message, object reqData, ILogTrace logTrace)
{
// alert to Slack if there is no config for SQS Maximum Retry.
if (SqsMaximumReceives == null)
{
return(true);
}
try
{
var sqsEvent = JsonConvert.DeserializeObject <SQSEvent>(JsonConvert.SerializeObject(reqData));
// Find the SQS message body have same message id that we generated for a message.
var sqsMessage = sqsEvent.Records.FirstOrDefault(x => JsonConvert.DeserializeObject <TMessage>(x.Body).MessageId.Equals(message.MessageId));
var isApproximateReceiveCount = sqsMessage.Attributes.TryGetValue("ApproximateReceiveCount", out string approximateReceiveCount);
logTrace.ExtendProps(new Dictionary <string, object> {
{ "approximateReceiveCount", approximateReceiveCount }
});
// Check the ApproximateReceiveCount of message.
if (sqsMessage != null && isApproximateReceiveCount &&
int.Parse(approximateReceiveCount) == SqsMaximumReceives)
{
return(true);
}
return(false);
}
catch (Exception ex)
{
logTrace.Add(LogLevel.Warning, "IsAlertToSlack()", new { ex.Message, ex.StackTrace });
// Alert to Slack if have any exception when check the ApproximateReceiveCount.
return(true);
}
}
/// <inheritdoc />
public override async Task <IEnumerable <IMessage> > ProcessAsync(SalesforceTopicsSyncerMessage message,
ILogTrace logTrace)
{
var partyId = message.PartyId;
// get the sync setting first
var syncSetting = await VerifySyncStatusAsync(partyId, logTrace);
var isSyncAll = syncSetting.IsSyncAll;
var isFullSync = syncSetting.IsFullSync;
logTrace?.Add(LogLevel.Info, "Sync Setting", new { isSyncAll, isFullSync });
// init the salesforce client
var svcSalesforce = InitSalesforceServiceClientAsync(syncSetting.OAuthToken, logTrace);
// get the list of SalesforceCampaigns and RemoteGroupMapping
// these 2 methods both return a list of SalesforceTopicMapping so we can compare those 2 and do some
// operations like Intersection, Difference,...
// NOTE: Initially, we support Salesforce Topics as Groups so these methods all return
// SalesforceTopicMapping. However, we changed to support Salesforce Campaigns as Groups later. We didn't
// want to change a lot of code here so we just changes the getSFGroupMappingFunc to get the Salesforce
// Campaigns instead of the Topics but it still returns the Topic schema.
var isSyncCampaign = true;
Func <Task <IList <SalesforceTopicMapping> > > getSFGroupMappingFunc =
() => GetAllSalesforceTopicsAsync(partyId, svcSalesforce, logTrace);
if (isSyncCampaign)
{
if (syncSetting.IntegrationType == CRMIntegrationTypes.Practifi)
{
getSFGroupMappingFunc = () => GetAllPractifiCampaignsAsync(partyId, svcSalesforce, logTrace);
}
else
{
getSFGroupMappingFunc = () => GetAllSalesforceCampaignsAsync(partyId, svcSalesforce, logTrace);
}
}
var groupMappingResults =
await Task.WhenAll(getSFGroupMappingFunc(), GetAllRemoteGroupMappingsAsync(partyId, logTrace));
// the TopicMappings from Salesforce, these objects don't have the FMGGroupId prop
var salesforceTopicMappings = groupMappingResults.ElementAt(0);
// the TopicMappings retrieved from RemoteData service, these objects have the linked FMGGroupId prop
var remoteTopicMappings = groupMappingResults.ElementAt(1);
// the list of GroupIds that user selected on UI
var userSelectedGroupIds = syncSetting.RemoteGroupIDs ?? new List <string>();
// add new groups
await InsertNewGroupsAsync(
partyId, salesforceTopicMappings, remoteTopicMappings, userSelectedGroupIds, logTrace
);
// delete non-existing groups
await DeleteGroupsAsync(partyId, salesforceTopicMappings, remoteTopicMappings, userSelectedGroupIds,
logTrace);
// update the names for all the groups that remain from the sync
await UpdateGroupNames(partyId, salesforceTopicMappings, remoteTopicMappings, userSelectedGroupIds,
logTrace);
return(new List <SalesforceTopicAssignmentsSyncerMessage>
{
new SalesforceTopicAssignmentsSyncerMessage {
PartyId = partyId
}
});
}
/// <summary>
/// Update group names for the groups that remain from last sync
/// </summary>
/// <param name="partyId"></param>
/// <param name="salesforceTopicMappings"></param>
/// <param name="remoteTopicMappings"></param>
/// <param name="logTrace"></param>
/// <returns></returns>
protected async Task UpdateGroupNames(int partyId,
IList <SalesforceTopicMapping> salesforceTopicMappings,
IList <SalesforceTopicMapping> remoteTopicMappings,
IList <string> userSelectedGroupIds,
ILogTrace logTrace)
{
var startedAt = DateTime.UtcNow;
// find the groups that remain from last sync
// Cannot use IList.Intersection because the salesforceTopicMapping items don't contain
// the FMGGroupId while the remoteTopicMappings items don't contain the new name from Salesforce
var remainGroups = new List <SalesforceTopicMapping>();
foreach (var remoteTopicMapping in remoteTopicMappings)
{
//
var salesforceTopicMapping = salesforceTopicMappings.FirstOrDefault(item =>
item.SalesforceTopicId == remoteTopicMapping.SalesforceTopicId);
if (salesforceTopicMapping == null)
{
continue;
}
salesforceTopicMapping.FMGGroupId = remoteTopicMapping.FMGGroupId;
remainGroups.Add(salesforceTopicMapping);
}
// is user selected to sync only some groups, filter to those groups only
if (userSelectedGroupIds.Any())
{
var userSelectedGroups = userSelectedGroupIds.Select(
groupId => new SalesforceTopicMapping {
SalesforceTopicId = groupId
}
).ToList();
remainGroups = remainGroups.Intersect(userSelectedGroups, AppStore.SalesforceTopicMappingComparer)
.ToList();
}
// update FMG Group
var groupTasks = remainGroups.Select(group => AppStore.ContactsServiceClient.UpdateGroupAsync(
new UpdateGroupRequest {
GroupID = group.FMGGroupId, Name = group.SalesforceTopicName
}
)).ToList();
foreach (var task in groupTasks)
{
await task;
}
logTrace.Add(LogLevel.Info, "Update FMG Groups", $"{groupTasks.Count()} updated", startedAt);
// update RemoteGroupMapping
startedAt = DateTime.UtcNow;
var remoteTasks = remainGroups.Select(group =>
AppStore.RemoteDataServiceClient.UpsertRemoteGroupMappingAsync(
new UpsertRemoteGroupMappingRequestData
{
GroupId = group.FMGGroupId,
PartyId = partyId,
RemoteGroupId = group.SalesforceTopicId,
RemoteGroupName = group.SalesforceTopicName
},
logTrace
)).ToList();
foreach (var task in remoteTasks)
{
await task;
}
logTrace.Add(LogLevel.Info, "Update RemoteGroupMapping", $"{remoteTasks.Count()} updated", startedAt);
}
GetFirstSalesforceObjectPageAsync <T>(int partyId,
Func <Task <ResponseBase <SalesforceQueryResponseData <T>, IList <SalesforceError> > > > getFirstPageFunc,
ISalesforceServiceClientV2 svcSalesforce, IOAuthTokenManager oauthTokenManager,
ILogTrace logTrace)
{
var domain = $"GetFirstSalesforce{typeof(T).Name}Page";
var startedAt = DateTime.UtcNow;
// send request to get the first batch from Salesforce
var res = await getFirstPageFunc();
logTrace?.Add(LogLevel.Info, domain, "First try", startedAt);
// success case
if (res.Status == HttpStatusCode.OK)
{
return(res);
}
// error case
var errorCode = res.Error?.FirstOrDefault()?.ErrorCode;
if (errorCode == null)
{
throw new Exception($"{domain} - Status {res.Status} - Message {res.ErrorString}");
}
if (errorCode != "INVALID_SESSION_ID")
{
throw new Exception(
$"{domain} - Status {res.Status} - Code {errorCode} - Message {res.ErrorString}"
);
}
logTrace?.Add(LogLevel.Info, domain, "AccessToken expired");
// try to refresh token
startedAt = DateTime.UtcNow;
var oauthToken = await oauthTokenManager.RefreshAccessTokenAsync <SalesforceOAuthError>(
new RefreshAccessTokenRequest
{
PartyId = partyId,
MaxRetries = 5,
RetryWaitSecondsIfFailed = 5
}, logTrace);
if (oauthToken == null)
{
logTrace?.Add(LogLevel.Warning, domain, "Refresh Token Failed.", startedAt);
throw new IgnoreProcessingMessageException("Refresh Token Failed.", $"Failed to RefreshAccessToken for partyId = {partyId}");
}
logTrace?.Add(LogLevel.Info, domain, "AccessToken refreshed", startedAt);
// update the access token in svcSalesforce
svcSalesforce.AccessToken = oauthToken.AccessToken;
// try again
startedAt = DateTime.UtcNow;
res = await getFirstPageFunc();
logTrace?.Add(LogLevel.Info, domain, "Task retried", startedAt);
// success case
if (res.Status == HttpStatusCode.OK)
{
return(res);
}
// error case
throw new Exception($"{domain} - Status {res.Status} - Code {errorCode} - Message {res.ErrorString}");
}