public async Task Handle(FetchDataForPlatformConnectionMessage message)
{
using var _ = _logger.BeginNamedScopeWithMessage(nameof(DataFetchCompleteHandler),
_messageContext.Message.GetMessageId(),
(LoggerPropertyNames.PlatformId, message.PlatformId),
(LoggerPropertyNames.UserId, message.UserId),
(LoggerPropertyNames.PlatformIntegrationType, message.PlatformIntegrationType));
_logger.LogInformation("Will start data fetching data for platform and user.");
var cancellationToken = _messageContext.GetCancellationToken();
using var session = _documentStore.OpenAsyncSession();
var user = await session.LoadAsync <User>(message.UserId, cancellationToken);
var syncLog = new DataSyncLog(user.Id, message.PlatformId);
using var __ = _logger.BeginPropertyScope((LoggerPropertyNames.DataSyncLogId, syncLog.ExternalId));
var platformConnection = user.PlatformConnections.SingleOrDefault(pc => pc.PlatformId == message.PlatformId);
if (platformConnection == null)
{
await LogErrorAndForwardMessageToErrorQueue("Platform connection for user for the given platform does not exist. Will forward message to error queue.");
return;
}
var connectionInfo = platformConnection.ConnectionInfo;
await session.StoreAsync(syncLog);
try
{
switch (message.PlatformIntegrationType)
{
case PlatformIntegrationType.AirbnbIntegration:
case PlatformIntegrationType.UpworkIntegration:
case PlatformIntegrationType.Manual:
var logMessage1 = "Data fetch for given platform integration type not implemented. Will forward message to error queue.";
await LogErrorAndForwardMessageToErrorQueue(logMessage1);
syncLog.Steps.Add(new DataSyncStep(DataSyncStepType.PlatformDataFetch, DataSyncStepState.Failed, logMessage: logMessage1));
await session.SaveChangesAsync(cancellationToken);
return;
case PlatformIntegrationType.GigDataPlatformIntegration:
var oauthOrEmailPlatformConnectionInfo = OAuthOrEmailPlatformConnectionInfo.FromIPlatformConnectionInfo(connectionInfo);
oauthOrEmailPlatformConnectionInfo = await _gigPlatformDataFetcher.StartDataFetch(message.UserId,
message.PlatformId, oauthOrEmailPlatformConnectionInfo,
platformConnection, syncLog, cancellationToken);
connectionInfo = OAuthOrEmailPlatformConnectionInfo.FromOAuthOrEmailPlatformConnectionInfo(oauthOrEmailPlatformConnectionInfo, connectionInfo);
break;
case PlatformIntegrationType.FreelancerIntegration:
connectionInfo = await _freelancerDataFetcher.StartDataFetch(message.UserId, message.PlatformId,
(OAuthPlatformConnectionInfo)connectionInfo, platformConnection, syncLog, cancellationToken);
syncLog.Steps.Add(new DataSyncStep(DataSyncStepType.PlatformDataFetch, DataSyncStepState.Started));
break;
default:
var logMessage2 = "Unrecognized platform integration type. Will forward message to error queue.";
await LogErrorAndForwardMessageToErrorQueue(logMessage2);
syncLog.Steps.Add(new DataSyncStep(DataSyncStepType.PlatformDataFetch, DataSyncStepState.Failed, logMessage: logMessage2));
await session.SaveChangesAsync(cancellationToken);
return;
}
}
catch (UnsupportedPlatformConnectionAuthenticationTypeException ex)
{
var logMessage = "Unsupported connection authentication type. Will forward message to error queue.";
await LogErrorAndForwardMessageToErrorQueue(
"Unsupported connection authentication type. Will forward message to error queue.", ex);
syncLog.Steps.Add(new DataSyncStep(DataSyncStepType.PlatformDataFetch, DataSyncStepState.Failed, logMessage: logMessage));
await session.SaveChangesAsync(cancellationToken);
return;
}
if (connectionInfo.IsDeleted)
{
//we no longer have a connection to the platform for the given user.
_logger.LogWarning("The connection is no longer valid and will be marked as deleted.");
}
platformConnection.ConnectionInfo = connectionInfo;
await session.SaveChangesAsync(cancellationToken);
_logger.LogInformation("Data fetch successfully initialized.");
}
public Task <TConnectionInfo> StartDataFetch(string userId, string platformId, TConnectionInfo connectionInfo,
PlatformConnection platformConnection, DataSyncLog syncLog, CancellationToken cancellationToken = default)
{
throw new NotImplementedException();
}
public new async Task <OAuthPlatformConnectionInfo> StartDataFetch(string userId, string platformId,
OAuthPlatformConnectionInfo connectionInfo, PlatformConnection platformConnection, DataSyncLog syncLog,
CancellationToken cancellationToken = default)
{
using var _ = Logger.BeginPropertyScope((LoggerPropertyNames.DataSyncLogId, syncLog.ExternalId));
Logger.LogTrace($"Will start data fetch from Freelancer for user with id {userId}");
if (connectionInfo.Token.HasExpired())
{
Logger.LogInformation($"Token has expired, will try to refresh it");
try
{
var newToken = await _freelancerAuthenticator.RefreshToken(new OAuthAccessToken
{
AccessToken = connectionInfo.Token.AccessToken,
RefreshToken = connectionInfo.Token.RefreshToken,
ExpiresIn = connectionInfo.Token.ExpiresInSeconds
});
connectionInfo.Token = new Token(newToken);
}
catch (UnauthorizedAccessException)
{
Logger.LogInformation(
"The consent to access Freelancer seems to have been revoked. Will act accordingly");
return(await HandleUnauthorized(userId, platformId, syncLog.Id));
}
}
_apiClient.SetAccessToken(connectionInfo.Token.AccessToken);
string rawReviews;
ReviewApiResult freelancerReviews;
string rawUserProfile;
UserInfoApiResult userProfile;
try
{
(userProfile, rawUserProfile) = await _apiClient.GetUserProfile();
(freelancerReviews, rawReviews) = await _apiClient.GetReviews(userProfile.Result.Id);
}
catch (UnauthorizedAccessException)
{
Logger.LogInformation(
"The consent to access Freelancer seems to have been revoked. Will act accordingly");
return(await HandleUnauthorized(userId, platformId, syncLog.Id));
}
var(ratings, reviews) = GetRatingsAndReviews(freelancerReviews);
var raw = $"{{data: [{rawUserProfile}, {rawReviews}]}}";
var numberOfCompletedJobs = userProfile.Result.Reputation.EntireHistory.Complete;
var averageRating = numberOfCompletedJobs == 0
? null
: new RatingDataFetchResult(userProfile.Result.Reputation.EntireHistory.Overall);
//achievements
var achievements = new List <AchievementFetchResult>();
var qualifications = userProfile.Result.Qualifications;
if (qualifications != null)
{
foreach (var qualification in qualifications)
{
var score = new AchievementScoreFetchResult(
qualification.ScorePercentage.ToString(CultureInfo.InvariantCulture), "percent");
var achievement = new AchievementFetchResult($"freelancer_qualification_{qualification.Id}",
qualification.Name,
"qualification", PlatformAchievementType.QualificationAssessment, qualification.Description,
$"https://www.freelancer.com{qualification.IconUrl}",
score);
achievements.Add(achievement);
}
}
var badges = userProfile.Result.Badges;
if (badges != null)
{
foreach (var badge in badges)
{
var achievement = new AchievementFetchResult($"freelancer_badge_{badge.Id}", badge.Name, "badge",
PlatformAchievementType.Badge, badge.Description, $"https://www.freelancer.com{badge.IconUrl}",
null);
achievements.Add(achievement);
}
}
var registrationDate = DateTimeOffset.FromUnixTimeSeconds(userProfile.Result.RegistrationDate);
var result = new PlatformDataFetchResult(numberOfCompletedJobs,
registrationDate, DateTimeOffset.UtcNow, averageRating, ratings, reviews, achievements, raw);
await CompleteDataFetch(userId, platformId, result, syncLog.Id);
Logger.LogTrace($"Freelancer data fetch completed for user with id {userId}");
return(connectionInfo);
}
public new async Task <OAuthOrEmailPlatformConnectionInfo> StartDataFetch(string userId, string platformId,
OAuthOrEmailPlatformConnectionInfo connectionInfo, PlatformConnection platformConnection, DataSyncLog syncLog,
CancellationToken cancellationToken = default)
{
using var _ = Logger.BeginPropertyScope((LoggerPropertyNames.UserId, userId),
(LoggerPropertyNames.PlatformId, platformId), (LoggerPropertyNames.PlatformName, platformConnection.PlatformName),
(LoggerPropertyNames.DataSyncLogId, syncLog.ExternalId));
Logger.LogInformation("Will start data fetch from a Gig platform integrated platform for user.");
if (connectionInfo.IsOAuthAuthentication)
{
Logger.LogError("Oauth connection not yet supported in Gig Platform API. Will throw.");
throw new UnsupportedPlatformConnectionAuthenticationTypeException("Oauth connection not yet supported in Gig Platform API");
}
try
{
var result = await _apiClient.RequestLatest(platformConnection.ExternalPlatformId, connectionInfo.Email, cancellationToken);
using var __ = Logger.BeginPropertyScope((LoggerPropertyNames.GigPlatformApiRequestId, result.RequestId));
await _intermittentDataManager.RegisterRequestData(result.RequestId, userId, platformId, syncLog.Id);
}
catch (ExternalServiceErrorException ex)
{
//Error that can occur here is that we get either a 500, which means there is something wrong with platform api server. Or we get a 404, which means that the
//platform we asked the api to fetch data from does not exist on the api side of things. Both these cases should result in a retry. So we throw here.
var logMessage = "Got error when regestering data fetch. Will throw.";
Logger.LogError(ex, logMessage);
syncLog.Steps.Add(new DataSyncStep(DataSyncStepType.PlatformDataFetch, DataSyncStepState.Failed, logMessage: logMessage));
throw new GigDataPlatformApiInitiateDataFetchException();
}
syncLog.Steps.Add(new DataSyncStep(DataSyncStepType.PlatformDataFetch, DataSyncStepState.Started));
Logger.LogInformation("Data fetch successfully started.");
return(connectionInfo);
}
public async Task Handle(PlatformConnectionUpdateNotificationMessage message)
{
using var _ = _logger.BeginNamedScopeWithMessage(nameof(DataFetchCompleteHandler),
_messageContext.Message.GetMessageId(),
(LoggerPropertyNames.ApplicationId, message.AppId),
(LoggerPropertyNames.UserId, message.UserId));
using var session = _documentStore.OpenAsyncSession();
DataSyncLog syncLog = null;
if (!string.IsNullOrEmpty(message.SyncLogId))
{
syncLog = await session.LoadAsync <DataSyncLog>(message.SyncLogId);
}
using var __ = _logger.BeginPropertyScope((LoggerPropertyNames.DataSyncLogId, syncLog?.ExternalId));
var cancellationToken = _messageContext.GetCancellationToken();
var user = await session.LoadAsync <User>(message.UserId, cancellationToken);
if (user == null)
{
var logMessage = "User with given id does not exist. Will move message to error queue.";
_logger.LogWarning(logMessage);
syncLog?.Steps.Add(new DataSyncStep(DataSyncStepType.AppNotification, DataSyncStepState.Failed, logMessage));
await _bus.Advanced.TransportMessage.Forward(_rebusConfiguration.ErrorQueueName);
await session.SaveChangesAsync();
return;
}
var app = await _appManager.GetAppFromId(message.AppId, session, cancellationToken);
if (app == null)
{
var logMessage = "App with given id does not exist. Will move message to error queue.";
_logger.LogWarning(logMessage);
syncLog?.Steps.Add(new DataSyncStep(DataSyncStepType.AppNotification, DataSyncStepState.Failed, logMessage));
await _bus.Advanced.TransportMessage.Forward(_rebusConfiguration.ErrorQueueName);
await session.SaveChangesAsync();
return;
}
var platform = await session.LoadAsync <Platform>(message.PlatformId, cancellationToken);
if (platform == null)
{
var logMessage = "Platform with given id does not exist. Will move message to error queue.";
_logger.LogWarning(logMessage);
syncLog?.Steps.Add(new DataSyncStep(DataSyncStepType.AppNotification, DataSyncStepState.Failed, logMessage, app.Id, app.DataUpdateCallbackUrl));
await _bus.Advanced.TransportMessage.Forward(_rebusConfiguration.ErrorQueueName);
await session.SaveChangesAsync();
return;
}
var notificationReason = message.Reason;
var platformConnectionState = message.PlatformConnectionState;
PlatformDataClaim?platformDataClaim = null;
if (notificationReason != NotificationReason.ConnectionDeleted)
{
if (platformConnectionState == PlatformConnectionState.Connected || platformConnectionState == PlatformConnectionState.Synced)
{
var activePlatformConnection = user.PlatformConnections.SingleOrDefault(pc =>
pc.PlatformId == message.PlatformId && !pc.ConnectionInfo.IsDeleted);
if (activePlatformConnection == null)
{
_logger.LogWarning(
$"No active connection exists for the given platform. Will notify app about removed connection.");
notificationReason = NotificationReason.ConnectionDeleted;
platformConnectionState = PlatformConnectionState.Removed;
}
else
{
var notificationInfoForApp =
activePlatformConnection.ConnectionInfo.NotificationInfos.SingleOrDefault(ni =>
ni.AppId == app.Id);
if (notificationInfoForApp == null)
{
_logger.LogWarning(
$"No active connection to the given app. Will notify app about removed connection.");
notificationReason = NotificationReason.ConnectionDeleted;
platformConnectionState = PlatformConnectionState.Removed;
}
else
{
platformDataClaim = notificationInfoForApp.PlatformDataClaim;
}
}
}
}
PlatformData platformData = null;
if (notificationReason == NotificationReason.DataUpdate)
{
platformData =
await _platformDataManager.GetPlatformData(user.Id, platform.Id, session, cancellationToken);
}
var updatePayload = CreatePayload(platform.ExternalId, user.ExternalId, app.SecretKey, platform.Name,
platformConnectionState, DateTimeOffset.UtcNow, notificationReason, platformData, platformDataClaim);
if (string.IsNullOrWhiteSpace(app.DataUpdateCallbackUrl))
{
var logMessage = "No notification endpoint was given. Will move message to error queue.";
_logger.LogWarning(logMessage);
syncLog?.Steps.Add(new DataSyncStep(DataSyncStepType.AppNotification, DataSyncStepState.Failed, logMessage, app.Id, app.DataUpdateCallbackUrl));
await _bus.Advanced.TransportMessage.Forward(_rebusConfiguration.ErrorQueueName);
await session.SaveChangesAsync();
return;
}
if (!Uri.TryCreate(app.DataUpdateCallbackUrl, UriKind.Absolute, out var uri))
{
_logger.LogError("Could not create uri from {DataCallbackUrl}. Will ignore message.", app.DataUpdateCallbackUrl);
syncLog?.Steps.Add(new DataSyncStep(DataSyncStepType.AppNotification, DataSyncStepState.Failed,
"Could not create uri from app data callback url. Will ignore message.", app.Id, app.DataUpdateCallbackUrl));
await session.SaveChangesAsync();
return;
}
if (IsLocalHost(uri))
{
_logger.LogWarning("Uri with url {DataCallbackUrl} is equal to localhost. Will ignore message.", app.DataUpdateCallbackUrl);
syncLog?.Steps.Add(new DataSyncStep(DataSyncStepType.AppNotification, DataSyncStepState.Failed,
"Uri is equal to localhost. Will ignore message.", app.Id, app.DataUpdateCallbackUrl));
await session.SaveChangesAsync();
return;
}
var serializerSettings = new JsonSerializerSettings
{
ContractResolver = new CamelCasePropertyNamesContractResolver()
};
var serializedPayload = JsonConvert.SerializeObject(updatePayload, serializerSettings);
_logger.LogTrace("Payload to be sent: {Payload}", serializedPayload);
var content = new StringContent(serializedPayload, Encoding.UTF8, "application/json");
content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
HttpResponseMessage response;
try
{
var httpClient = new HttpClient();
response = await httpClient.PostAsync(
uri,
content, cancellationToken);
}
catch (Exception e)
{
var logMessage = "Got error calling endpoint. Will schedule retry.";
_logger.LogError(e, logMessage);
syncLog?.Steps.Add(new DataSyncStep(DataSyncStepType.AppNotification, DataSyncStepState.Failed, logMessage, app.Id, app.DataUpdateCallbackUrl));
await _bus.DeferMessageLocalWithExponentialBackOff(message, _messageContext.Headers, MaxMessageRetries,
_rebusConfiguration.ErrorQueueName, logger : _logger);
await session.SaveChangesAsync();
return;
}
if (!response.IsSuccessStatusCode)
{
if ((int)response.StatusCode >= 400 && (int)response.StatusCode < 500)
{
//we got a 40x status code. Inspect further to determine action.
if (response.StatusCode == HttpStatusCode.Gone)
{
//If we get a 410 it means that the app is telling us that they don't want updates for the given platform for the user again.
//remove the app notification from the platformconnection.
_logger.LogInformation("App responded with {httpStatusCode}. Will remove notification info for app.", response.StatusCode);
var platformConnection = user.PlatformConnections.SingleOrDefault(pc => pc.PlatformId == message.PlatformId);
if (platformConnection != null)
{
int?notificationInfoIdx = null;
var counter = 0;
foreach (var notificationInfo in platformConnection.ConnectionInfo.NotificationInfos)
{
if (notificationInfo.AppId == message.AppId)
{
notificationInfoIdx = counter;
break;
}
counter++;
}
if (notificationInfoIdx.HasValue)
{
platformConnection.ConnectionInfo.NotificationInfos.RemoveAt(notificationInfoIdx.Value);
}
}
syncLog?.Steps.Add(new DataSyncStep(DataSyncStepType.AppNotification, DataSyncStepState.Succeeded,
logMessage: $"App responded with {response.StatusCode}, indicating that the notification should be removed. App will not be notified again.",
appId: app.Id,
appWebHookUrl: app.DataUpdateCallbackUrl));
}
else
{
//some other 400. We will not retry the request. But we will keep the connection info for the app and the app will be notified next update again.
_logger.LogInformation("App responded with {httpStatusCode}. Will not retry current request. App will be notified on next data update", response.StatusCode);
syncLog?.Steps.Add(new DataSyncStep(DataSyncStepType.AppNotification, DataSyncStepState.Failed,
logMessage: $"App responded with {response.StatusCode}. Current request will not be retried. App will be notified again on next update.",
appId: app.Id,
appWebHookUrl: app.DataUpdateCallbackUrl));
}
}
else
{
_logger.LogError("Got non success status code ({HttpStatusCode}) calling endpoint. Will schedule retry.", response.StatusCode);
syncLog?.Steps.Add(new DataSyncStep(DataSyncStepType.AppNotification, DataSyncStepState.Failed,
$"Got non success status code ({response.StatusCode}) calling endpoint. Will schedule retry.", app.Id, app.DataUpdateCallbackUrl));
await _bus.DeferMessageLocalWithExponentialBackOff(message, _messageContext.Headers, MaxMessageRetries,
_rebusConfiguration.ErrorQueueName, logger : _logger);
}
await session.SaveChangesAsync(cancellationToken);
return;
}
syncLog?.Steps.Add(new DataSyncStep(DataSyncStepType.AppNotification, DataSyncStepState.Succeeded, appId: app.Id, appWebHookUrl: app.DataUpdateCallbackUrl));
await session.SaveChangesAsync(cancellationToken);
_logger.LogInformation("App successfully notified about platform data update.");
}
public async Task Handle(PlatformConnectionRemovedMessage message)
{
using var _ = _logger.BeginPropertyScope((LoggerPropertyNames.PlatformId, message.PlatformId),
(LoggerPropertyNames.UserId, message.UserId),
("DeleteReason", message.DeleteReason));
_logger.LogInformation("Will remove platform connection");
using var session = _documentStore.OpenAsyncSession();
DataSyncLog syncLog = null;
if (!string.IsNullOrEmpty(message.SyncLogId))
{
syncLog = await session.LoadAsync <DataSyncLog>(message.SyncLogId);
}
using var __ = _logger.BeginPropertyScope((LoggerPropertyNames.DataSyncLogId, syncLog?.ExternalId));
//remove connection
var user = await session.LoadAsync <User>(message.UserId);
var index = 0;
var indexToRemove = -1;
PlatformConnection platformConnectionToRemove = null;
foreach (var userPlatformConnection in user.PlatformConnections)
{
if (userPlatformConnection.PlatformId == message.PlatformId)
{
indexToRemove = index;
platformConnectionToRemove = userPlatformConnection;
break;
}
index++;
}
if (indexToRemove == -1 || platformConnectionToRemove == null)
{
_logger.LogWarning("Platform connection with platform id {PlatformId} was not found for user with id {UserId}",
message.PlatformId, message.UserId);
syncLog?.Steps.Add(new DataSyncStep(DataSyncStepType.RemovePlatformConnection, DataSyncStepState.Failed,
$"Platform connection with platform id {message.PlatformId} was not found for user with id {message.UserId}"));
await session.SaveChangesAsync();
return;
}
if (message.DeleteReason != Common.PlatformConnectionDeleteReason.Undefined) //if we have a delete reason, do soft delete
{
_logger.LogInformation("Delete reason was {DeleteReason}. Will do soft delete", message.DeleteReason);
platformConnectionToRemove.ConnectionInfo.DeleteReason = message.DeleteReason;
platformConnectionToRemove.ConnectionInfo.IsDeleted = true;
}
else //no reason given, do hard delete
{
_logger.LogInformation("No delete reason was given. Will do hard delete");
user.PlatformConnections.RemoveAt(indexToRemove);
}
syncLog?.Steps.Add(new DataSyncStep(DataSyncStepType.RemovePlatformConnection, DataSyncStepState.Succeeded));
await session.SaveChangesAsync();
await _appNotificationManager.NotifyPlatformConnectionRemoved(message.UserId,
platformConnectionToRemove.ConnectionInfo.NotificationInfos.Select(ni => ni.AppId).ToList(),
platformConnectionToRemove.PlatformId, session);
}
