public async Task UpdateUserFromProfileAsync(string curUserId, string userProfileJson)
{
var userProfile = JObject.Parse(userProfileJson);
var identities = (JArray)userProfile["identities"];
JObject ptIdentity = identities.OfType <JObject>()
.FirstOrDefault(i => (string)i["connection"] == "paratext");
Regex emailRegex = new Regex(EMAIL_PATTERN);
using (IConnection conn = await _realtimeService.ConnectAsync(curUserId))
{
string name = (string)userProfile["name"];
IDocument <User> userDoc = await conn.FetchOrCreateAsync <User>(curUserId, () => new User
{
AuthId = (string)userProfile["user_id"],
DisplayName = string.IsNullOrWhiteSpace(name) || emailRegex.IsMatch(name) ?
(string)userProfile["nickname"] : name
});
await userDoc.SubmitJson0OpAsync(op =>
{
op.Set(u => u.Name, name);
op.Set(u => u.Email, (string)userProfile["email"]);
op.Set(u => u.AvatarUrl, (string)userProfile["picture"]);
op.Set(u => u.Role, (string)userProfile["app_metadata"]["xf_role"]);
if (ptIdentity != null)
{
var ptId = (string)ptIdentity["user_id"];
op.Set(u => u.ParatextId, GetIdpIdFromAuthId(ptId));
}
string language = userProfile["user_metadata"] == null ? null :
(string)userProfile["user_metadata"]["interface_language"];
string interfaceLanguage = string.IsNullOrWhiteSpace(language) ? "en" : language;
op.Set(u => u.InterfaceLanguage, interfaceLanguage);
string key = _siteOptions.Value.Id;
if (!userDoc.Data.Sites.ContainsKey(key))
{
op.Set(u => u.Sites[key], new Site());
}
});
}
if (ptIdentity != null)
{
var newPTTokens = new Tokens
{
AccessToken = (string)ptIdentity["access_token"],
RefreshToken = (string)ptIdentity["refresh_token"]
};
UserSecret userSecret = await _userSecrets.UpdateAsync(curUserId, update => update
.SetOnInsert(put => put.ParatextTokens, newPTTokens), true);
// Only update the PT tokens if they are newer
if (newPTTokens.IssuedAt > userSecret.ParatextTokens.IssuedAt)
{
await _userSecrets.UpdateAsync(curUserId,
update => update.Set(put => put.ParatextTokens, newPTTokens));
}
}
}
public async Task SyncAsync(string curUserId, string projectId, bool trainEngine)
{
string sourceProjectId = null;
using (IConnection conn = await _realtimeService.ConnectAsync(curUserId))
{
IDocument <SFProject> projectDoc = await conn.FetchAsync <SFProject>(projectId);
if (projectDoc.Data.SyncDisabled)
{
throw new ForbiddenException();
}
sourceProjectId = projectDoc.Data.TranslateConfig.Source?.ProjectRef;
await projectDoc.SubmitJson0OpAsync(op => op.Inc(pd => pd.Sync.QueuedCount));
// See if we can sync the source project
if (!string.IsNullOrWhiteSpace(sourceProjectId))
{
IDocument <SFProject> sourceProjectDoc = await conn.FetchAsync <SFProject>(sourceProjectId);
if (!sourceProjectDoc.IsLoaded || sourceProjectDoc.Data.SyncDisabled)
{
sourceProjectId = null;
}
else
{
await sourceProjectDoc.SubmitJson0OpAsync(op => op.Inc(pd => pd.Sync.QueuedCount));
}
}
}
if (!string.IsNullOrWhiteSpace(sourceProjectId))
{
// We need to sync the source first so that we can link the source texts and train the engine
string parentId = _backgroundJobClient.Enqueue <ParatextSyncRunner>(
r => r.RunAsync(sourceProjectId, curUserId, false));
_backgroundJobClient.ContinueJobWith <ParatextSyncRunner>(parentId,
r => r.RunAsync(projectId, curUserId, trainEngine));
}
else
{
_backgroundJobClient.Enqueue <ParatextSyncRunner>(r => r.RunAsync(projectId, curUserId, trainEngine));
}
}
public override async Task OnCompleted(BuildContext context)
{
using (IConnection conn = await _realtimeService.ConnectAsync())
{
IDocument <SFProject> project = await conn.FetchAsync <SFProject>(context.Engine.Projects.First());
if (!project.IsLoaded)
{
return;
}
var tasks = new List <Task>();
foreach (string userId in project.Data.UserRoles.Keys)
{
tasks.Add(ClearSelectedSegmentChecksum(conn, project.Id, userId));
}
await Task.WhenAll(tasks);
}
}
/// <summary>
/// First-stage migrator. Synchronize all SF projects to the Paratext Data Access server.
/// First query information that will show whether we should be able to sync all projects. In addition to
/// reporting information on projects and whether there is an admin that can sync the project, this method shows
/// that the admin can successfully perform queries to both the PT Registry and the PT Data Access web APIs, via
/// various ParatextService method calls.
/// If `doSynchronizations` is false, only do the above reporting. If true, also synchronize the SF DB with the
/// Paratext Data Access server.
/// </summary>
public async Task SynchronizeAllProjectsAsync(bool doSynchronizations,
ISet <string> sfProjectIdsToSynchronize = null, IDictionary <string, string> sfAdminsToUse = null)
{
List <SFProject> allSfProjects = _realtimeService.QuerySnapshots <SFProject>().ToList <SFProject>();
if (sfProjectIdsToSynchronize != null)
{
allSfProjects.RemoveAll((SFProject sfProject) => !sfProjectIdsToSynchronize.Contains(sfProject.Id));
string ids = string.Join(' ', allSfProjects.Select((SFProject sfProject) => sfProject.Id));
int count = allSfProjects.Count;
_logger.Log($"Only working on the subset of projects (count {count}) with these SF project ids: {ids}");
}
_realtimeServiceConnection = await _realtimeService.ConnectAsync();
List <Task> syncTasks = new List <Task>();
// Report on all SF projects.
foreach (SFProject sfProject in allSfProjects)
{
_logger.Log($"{Program.Bullet1} PT project {sfProject.ShortName}, "
+ $"PT project id {sfProject.ParatextId}, SF project id {sfProject.Id}.");
List <string> projectSfAdminUserIds = sfProject.UserRoles
.Where(ur => ur.Value == SFProjectRole.Administrator).Select(ur => ur.Key).ToList <string>();
if (projectSfAdminUserIds.Count() < 1)
{
List <string> projectSfUserIds = sfProject.UserRoles.Select(ur => ur.Key).ToList <string>();
string users = string.Join(", ", projectSfUserIds);
if (projectSfUserIds.Count() < 1)
{
users = "None";
}
_logger.Log($" {Program.Bullet2} Warning: no admin users. Non-admin users include: {users}");
}
// Report on all admins in a project
foreach (string sfUserId in projectSfAdminUserIds)
{
UserSecret userSecret = _userSecretRepo.Query()
.FirstOrDefault((UserSecret us) => us.Id == sfUserId);
string ptUsername = null;
string ptUserId = null;
try
{
ptUsername = _paratextService.GetParatextUsername(userSecret);
ptUserId = GetParatextUserId(userSecret);
}
catch (Exception e)
{
_logger.Log($" {Program.Bullet2} Failure getting SF user's PT username or PT user id. " +
$"Skipping. SF user id was {sfUserId}. If known, PT username was {ptUsername}. " +
$"Error with stack was {e}");
continue;
}
_logger.Log($" {Program.Bullet2} PT user '{ptUsername}', "
+ $"id {ptUserId}, using SF admin user id {sfUserId} on SF project.");
string rt = $"{userSecret.ParatextTokens.RefreshToken.Substring(0, 5)}..";
string at = $"{userSecret.ParatextTokens.AccessToken.Substring(0, 5)}..";
bool atv = userSecret.ParatextTokens.ValidateLifetime();
_logger.Log($" {Program.Bullet3} Paratext RefreshToken: {rt}, "
+ $"AccessToken: {at}, AccessToken initially valid: {atv}.");
// Demonstrate access to PT Registry, and report Registry's statement of role.
_logger.Log($" {Program.Bullet3} PT Registry report on role on PT project: ", false);
IReadOnlyDictionary <string, string> ptProjectRoles = null;
try
{
ptProjectRoles = await _paratextService.GetProjectRolesAsync(userSecret, sfProject.ParatextId);
}
catch (Exception e)
{
Console.WriteLine($" Failure fetching user's PT project roles. Skipping. " +
$"Error was {e.Message}");
continue;
}
if (ptProjectRoles.TryGetValue(ptUserId, out string ptRole))
{
Console.WriteLine($"{ptRole}");
}
else
{
Console.WriteLine($"Not found.");
}
// Demonstrate access to PT Data Access.
IReadOnlyList <ParatextProject> userPtProjects = null;
try
{
userPtProjects = await _paratextService.GetProjectsAsync(userSecret);
}
catch (Exception e)
{
_logger.Log($" {Program.Bullet3} Failure fetching user's PT projects. Skipping. "
+ $"Error was {e.Message}");
continue;
}
_logger.Log($" {Program.Bullet3} PT Data Access and PT Registry "
+ "based report on projects the user can access, narrowed to this project: ", false);
List <string> ptProjectNamesList = userPtProjects
.Where(ptProject => ptProject.ParatextId == sfProject.ParatextId)
.Select(ptProject => ptProject.ShortName).ToList();
string ptProjectNames = string.Join(',', ptProjectNamesList);
if (ptProjectNamesList.Count() < 1)
{
ptProjectNames = $"User is not on this project. " +
$"PT reports they are on this many PT projects: {userPtProjects.Count()}";
}
Console.WriteLine(ptProjectNames);
if (doSynchronizations)
{
if (sfAdminsToUse != null && sfAdminsToUse.ContainsKey(sfProject.Id))
{
sfAdminsToUse.TryGetValue(sfProject.Id, out string sfAdminIdToUse);
bool isUserAtHand = sfUserId == sfAdminIdToUse;
if (isUserAtHand)
{
_logger.Log($" {Program.Bullet2} For SF Project {sfProject.Id}, we were asked to use "
+ $"this SF user {sfUserId} to sync.");
}
else
{
_logger.Log($" {Program.Bullet2} For SF Project {sfProject.Id}, we were asked to use "
+ $"SF user {sfAdminIdToUse}, not {sfUserId}, to sync. So skipping this user.");
continue;
}
}
try
{
_logger.Log($" {Program.Bullet2} Starting an asynchronous synchronization for "
+ $"SF project {sfProject.Id} as SF user {sfUserId}.");
Task syncTask = SynchronizeProject(sfUserId, sfProject.Id);
var projectDoc = await _realtimeServiceConnection.FetchAsync <SFProject>(sfProject.Id);
// Increment the queued count (such as done in SyncService), since it gets decremented
// later by ParatextSyncRunner.
await projectDoc.SubmitJson0OpAsync(op => op.Inc(pd => pd.Sync.QueuedCount));
_logger.Log($" {Program.Bullet3} Synchronization task for SF project {sfProject.Id} as "
+ $"SF user {sfUserId} has Sync Task Id {syncTask.Id}.");
syncTasks.Add(syncTask);
break;
}
catch (Exception e)
{
// We probably won't get here. But just in case.
_logger.Log($" {Program.Bullet3} There was a problem with synchronizing. It might be "
+ $"tried next with another admin user. Exception is:{Environment.NewLine}{e}");
continue;
}
}
}
}
if (doSynchronizations)
{
_logger.Log("Waiting for synchronization tasks to finish (if any). "
+ $"There are this many tasks: {syncTasks.Count}");
try
{
Task allTasks = Task.WhenAll(syncTasks);
try
{
await allTasks.ConfigureAwait(false);
}
catch
{
if (allTasks.Exception == null)
{
throw;
}
ExceptionDispatchInfo.Capture(allTasks.Exception).Throw();
}
_logger.Log("Synchronization tasks are finished.");
}
catch (AggregateException e)
{
_logger.Log("There was a problem with one or more synchronization tasks. "
+ $"Exception is:{Environment.NewLine}{e}");
}
if (syncTasks.Any(task => !task.IsCompletedSuccessfully))
{
_logger.Log("One or more sync tasks did not complete successfully.");
}
else
{
_logger.Log("All sync tasks finished with a claimed Task status of Completed Successfully.");
}
_logger.Log($"{Program.Bullet1} Sync task completion results:");
foreach (Task task in syncTasks)
{
string exceptionInfo = $"with exception {task.Exception?.InnerException}.";
if (task.Exception == null)
{
exceptionInfo = "with no unhandled exception thrown.";
}
_logger.Log($" {Program.Bullet2} Sync task Id {task.Id} has status {task.Status} {exceptionInfo}");
if (task.Exception?.InnerExceptions?.Count > 1)
{
_logger.Log($" {Program.Bullet3} Sync task Id {task.Id} has more than one inner exception. "
+ "Sorry if this is redundant, but they are:");
foreach (var e in task.Exception.InnerExceptions)
{
_logger.Log($" {Program.Bullet3} Inner exception: {e}");
}
}
}
}
ReportLastSyncSuccesses(allSfProjects);
}
public async Task RunAsync(PerformContext context, IJobCancellationToken cancellationToken, string userId,
string jobId)
{
_job = await _jobs.UpdateAsync(j => j.Id == jobId, u => u
.Set(j => j.BackgroundJobId, context.BackgroundJob.Id)
.Set(j => j.State, SyncJobEntity.SyncingState));
if (_job == null)
{
return;
}
try
{
if ((await _users.TryGetAsync(userId)).TryResult(out UserEntity user) &&
(await _projects.TryGetAsync(_job.ProjectRef)).TryResult(out SFProjectEntity project))
{
if (!_fileSystemService.DirectoryExists(WorkingDir))
{
_fileSystemService.CreateDirectory(WorkingDir);
}
bool translateEnabled = project.TranslateConfig.Enabled;
using (IConnection conn = await _realtimeService.ConnectAsync())
{
string targetParatextId = project.ParatextId;
IReadOnlyList <string> targetBooks = await _paratextService.GetBooksAsync(user,
targetParatextId);
string sourceParatextId = project.TranslateConfig.SourceParatextId;
IReadOnlyList <string> sourceBooks = null;
if (translateEnabled)
{
sourceBooks = await _paratextService.GetBooksAsync(user, sourceParatextId);
}
var booksToSync = new HashSet <string>(targetBooks);
if (translateEnabled)
{
booksToSync.IntersectWith(sourceBooks);
}
var booksToDelete = new HashSet <string>(
GetBooksToDelete(project, targetParatextId, targetBooks));
if (translateEnabled)
{
booksToDelete.UnionWith(GetBooksToDelete(project, sourceParatextId, sourceBooks));
}
_step = 0;
_stepCount = booksToSync.Count * 3;
if (translateEnabled)
{
_stepCount *= 2;
}
_stepCount += booksToDelete.Count;
foreach (string bookId in booksToSync)
{
if (!BookNames.TryGetValue(bookId, out string name))
{
name = bookId;
}
TextEntity text = await _texts.UpdateAsync(
t => t.ProjectRef == project.Id && t.BookId == bookId,
u => u
.SetOnInsert(t => t.Name, name)
.SetOnInsert(t => t.BookId, bookId)
.SetOnInsert(t => t.ProjectRef, project.Id)
.SetOnInsert(t => t.OwnerRef, userId), upsert : true);
List <Chapter> newChapters = await SyncOrCloneBookUsxAsync(user, conn, project, text,
TextType.Target, targetParatextId, false);
if (translateEnabled)
{
var chaptersToInclude = new HashSet <int>(newChapters.Select(c => c.Number));
await SyncOrCloneBookUsxAsync(user, conn, project, text, TextType.Source,
sourceParatextId, true, chaptersToInclude);
}
await UpdateNotesData(conn, text, newChapters);
}
foreach (string bookId in booksToDelete)
{
TextEntity text = await _texts.DeleteAsync(
t => t.ProjectRef == project.Id && t.BookId == bookId);
await DeleteBookUsxAsync(conn, project, text, TextType.Target, targetParatextId);
if (translateEnabled)
{
await DeleteBookUsxAsync(conn, project, text, TextType.Source, sourceParatextId);
}
await DeleteNotesData(conn, text);
await UpdateProgress();
}
}
// TODO: Properly handle job cancellation
cancellationToken.ThrowIfCancellationRequested();
if (translateEnabled)
{
// start training Machine engine
await _engineService.StartBuildByProjectIdAsync(_job.ProjectRef);
}
await _projects.UpdateAsync(_job.ProjectRef, u => u
.Set(p => p.LastSyncedDate, DateTime.UtcNow)
.Unset(p => p.ActiveSyncJobRef));
}
else
{
await _projects.UpdateAsync(_job.ProjectRef, u => u.Unset(p => p.ActiveSyncJobRef));
}
_job = await _jobs.UpdateAsync(_job, u => u
.Set(j => j.State, SyncJobEntity.IdleState)
.Unset(j => j.BackgroundJobId));
}
/// <summary>
/// Iterates through SF projects on the server and identifies one administrator user on the project.
/// Using the administrator user's secrets, perform a send/receive with the Paratext server. Effectively,
/// this clones the project to the Scripture Forge server.
/// This will overwrite any un-synchronized data on SF.
/// </summary>
public async Task CloneSFProjects(string mode, IEnumerable <SFProject> projectsToClone)
{
string syncDir = Path.Combine(_siteOptions.Value.SiteDir, "sync");
bool doClone = mode == CLONE || mode == CLONE_AND_MOVE_OLD || mode == CLONE_SILENT;
string syncDirOld = Path.Combine(_siteOptions.Value.SiteDir, "sync_old");
if (mode == CLONE_AND_MOVE_OLD)
{
if (!_fileSystemService.DirectoryExists(syncDirOld))
{
_fileSystemService.CreateDirectory(syncDirOld);
}
}
IConnection connection = await _realtimeService.ConnectAsync();
// Get the paratext project ID and admin user for all SF Projects
foreach (SFProject proj in projectsToClone)
{
bool foundAdmin = false;
foreach (string userId in proj.UserRoles.Keys)
{
if (proj.UserRoles.TryGetValue(userId, out string role) && role == SFProjectRole.Administrator)
{
foundAdmin = true;
UserSecret userSecret = _userSecretRepo.Query().FirstOrDefault((UserSecret us) => us.Id == userId);
string ptUsername = _paratextService.GetParatextUsername(userSecret);
Log($"Project administrator identified on {proj.Name}: {ptUsername} ({userId})");
if (!doClone)
{
break;
}
try
{
var projectDoc = await connection.FetchAsync <SFProject>(proj.Id);
await projectDoc.SubmitJson0OpAsync(op =>
{
// Increment the queued count such as in SyncService
op.Inc(pd => pd.Sync.QueuedCount);
});
bool silent = mode == CLONE_SILENT;
// Clone the paratext project and update the SF database with the project data
await CloneAndSyncFromParatext(proj, userId, syncDir, silent);
if (mode == CLONE_AND_MOVE_OLD)
{
string projectDir = Path.Combine(syncDir, proj.Id);
string projectDirOld = Path.Combine(syncDirOld, proj.Id);
_fileSystemService.MoveDirectory(projectDir, projectDirOld);
}
break;
}
catch (Exception e)
{
Log($"Unable to clone {proj.Name} ({proj.Id}) as user: {userId}{Environment.NewLine}" +
$"Error was: {e}");
}
}
}
if (!foundAdmin)
{
Log($"ERROR: Unable to identify a project administrator on {proj.Name}");
}
}
}