public async Task UpdateUserFromProfileAsync(string curUserId, string userProfileJson) { var userProfile = JObject.Parse(userProfileJson); var identities = (JArray)userProfile["identities"]; JObject ptIdentity = identities.OfType <JObject>() .FirstOrDefault(i => (string)i["connection"] == "paratext"); Regex emailRegex = new Regex(EMAIL_PATTERN); using (IConnection conn = await _realtimeService.ConnectAsync(curUserId)) { string name = (string)userProfile["name"]; IDocument <User> userDoc = await conn.FetchOrCreateAsync <User>(curUserId, () => new User { AuthId = (string)userProfile["user_id"], DisplayName = string.IsNullOrWhiteSpace(name) || emailRegex.IsMatch(name) ? (string)userProfile["nickname"] : name }); await userDoc.SubmitJson0OpAsync(op => { op.Set(u => u.Name, name); op.Set(u => u.Email, (string)userProfile["email"]); op.Set(u => u.AvatarUrl, (string)userProfile["picture"]); op.Set(u => u.Role, (string)userProfile["app_metadata"]["xf_role"]); if (ptIdentity != null) { var ptId = (string)ptIdentity["user_id"]; op.Set(u => u.ParatextId, GetIdpIdFromAuthId(ptId)); } string language = userProfile["user_metadata"] == null ? null : (string)userProfile["user_metadata"]["interface_language"]; string interfaceLanguage = string.IsNullOrWhiteSpace(language) ? "en" : language; op.Set(u => u.InterfaceLanguage, interfaceLanguage); string key = _siteOptions.Value.Id; if (!userDoc.Data.Sites.ContainsKey(key)) { op.Set(u => u.Sites[key], new Site()); } }); } if (ptIdentity != null) { var newPTTokens = new Tokens { AccessToken = (string)ptIdentity["access_token"], RefreshToken = (string)ptIdentity["refresh_token"] }; UserSecret userSecret = await _userSecrets.UpdateAsync(curUserId, update => update .SetOnInsert(put => put.ParatextTokens, newPTTokens), true); // Only update the PT tokens if they are newer if (newPTTokens.IssuedAt > userSecret.ParatextTokens.IssuedAt) { await _userSecrets.UpdateAsync(curUserId, update => update.Set(put => put.ParatextTokens, newPTTokens)); } } }
public async Task SyncAsync(string curUserId, string projectId, bool trainEngine) { string sourceProjectId = null; using (IConnection conn = await _realtimeService.ConnectAsync(curUserId)) { IDocument <SFProject> projectDoc = await conn.FetchAsync <SFProject>(projectId); if (projectDoc.Data.SyncDisabled) { throw new ForbiddenException(); } sourceProjectId = projectDoc.Data.TranslateConfig.Source?.ProjectRef; await projectDoc.SubmitJson0OpAsync(op => op.Inc(pd => pd.Sync.QueuedCount)); // See if we can sync the source project if (!string.IsNullOrWhiteSpace(sourceProjectId)) { IDocument <SFProject> sourceProjectDoc = await conn.FetchAsync <SFProject>(sourceProjectId); if (!sourceProjectDoc.IsLoaded || sourceProjectDoc.Data.SyncDisabled) { sourceProjectId = null; } else { await sourceProjectDoc.SubmitJson0OpAsync(op => op.Inc(pd => pd.Sync.QueuedCount)); } } } if (!string.IsNullOrWhiteSpace(sourceProjectId)) { // We need to sync the source first so that we can link the source texts and train the engine string parentId = _backgroundJobClient.Enqueue <ParatextSyncRunner>( r => r.RunAsync(sourceProjectId, curUserId, false)); _backgroundJobClient.ContinueJobWith <ParatextSyncRunner>(parentId, r => r.RunAsync(projectId, curUserId, trainEngine)); } else { _backgroundJobClient.Enqueue <ParatextSyncRunner>(r => r.RunAsync(projectId, curUserId, trainEngine)); } }
public override async Task OnCompleted(BuildContext context) { using (IConnection conn = await _realtimeService.ConnectAsync()) { IDocument <SFProject> project = await conn.FetchAsync <SFProject>(context.Engine.Projects.First()); if (!project.IsLoaded) { return; } var tasks = new List <Task>(); foreach (string userId in project.Data.UserRoles.Keys) { tasks.Add(ClearSelectedSegmentChecksum(conn, project.Id, userId)); } await Task.WhenAll(tasks); } }
/// <summary> /// First-stage migrator. Synchronize all SF projects to the Paratext Data Access server. /// First query information that will show whether we should be able to sync all projects. In addition to /// reporting information on projects and whether there is an admin that can sync the project, this method shows /// that the admin can successfully perform queries to both the PT Registry and the PT Data Access web APIs, via /// various ParatextService method calls. /// If `doSynchronizations` is false, only do the above reporting. If true, also synchronize the SF DB with the /// Paratext Data Access server. /// </summary> public async Task SynchronizeAllProjectsAsync(bool doSynchronizations, ISet <string> sfProjectIdsToSynchronize = null, IDictionary <string, string> sfAdminsToUse = null) { List <SFProject> allSfProjects = _realtimeService.QuerySnapshots <SFProject>().ToList <SFProject>(); if (sfProjectIdsToSynchronize != null) { allSfProjects.RemoveAll((SFProject sfProject) => !sfProjectIdsToSynchronize.Contains(sfProject.Id)); string ids = string.Join(' ', allSfProjects.Select((SFProject sfProject) => sfProject.Id)); int count = allSfProjects.Count; _logger.Log($"Only working on the subset of projects (count {count}) with these SF project ids: {ids}"); } _realtimeServiceConnection = await _realtimeService.ConnectAsync(); List <Task> syncTasks = new List <Task>(); // Report on all SF projects. foreach (SFProject sfProject in allSfProjects) { _logger.Log($"{Program.Bullet1} PT project {sfProject.ShortName}, " + $"PT project id {sfProject.ParatextId}, SF project id {sfProject.Id}."); List <string> projectSfAdminUserIds = sfProject.UserRoles .Where(ur => ur.Value == SFProjectRole.Administrator).Select(ur => ur.Key).ToList <string>(); if (projectSfAdminUserIds.Count() < 1) { List <string> projectSfUserIds = sfProject.UserRoles.Select(ur => ur.Key).ToList <string>(); string users = string.Join(", ", projectSfUserIds); if (projectSfUserIds.Count() < 1) { users = "None"; } _logger.Log($" {Program.Bullet2} Warning: no admin users. Non-admin users include: {users}"); } // Report on all admins in a project foreach (string sfUserId in projectSfAdminUserIds) { UserSecret userSecret = _userSecretRepo.Query() .FirstOrDefault((UserSecret us) => us.Id == sfUserId); string ptUsername = null; string ptUserId = null; try { ptUsername = _paratextService.GetParatextUsername(userSecret); ptUserId = GetParatextUserId(userSecret); } catch (Exception e) { _logger.Log($" {Program.Bullet2} Failure getting SF user's PT username or PT user id. " + $"Skipping. SF user id was {sfUserId}. If known, PT username was {ptUsername}. " + $"Error with stack was {e}"); continue; } _logger.Log($" {Program.Bullet2} PT user '{ptUsername}', " + $"id {ptUserId}, using SF admin user id {sfUserId} on SF project."); string rt = $"{userSecret.ParatextTokens.RefreshToken.Substring(0, 5)}.."; string at = $"{userSecret.ParatextTokens.AccessToken.Substring(0, 5)}.."; bool atv = userSecret.ParatextTokens.ValidateLifetime(); _logger.Log($" {Program.Bullet3} Paratext RefreshToken: {rt}, " + $"AccessToken: {at}, AccessToken initially valid: {atv}."); // Demonstrate access to PT Registry, and report Registry's statement of role. _logger.Log($" {Program.Bullet3} PT Registry report on role on PT project: ", false); IReadOnlyDictionary <string, string> ptProjectRoles = null; try { ptProjectRoles = await _paratextService.GetProjectRolesAsync(userSecret, sfProject.ParatextId); } catch (Exception e) { Console.WriteLine($" Failure fetching user's PT project roles. Skipping. " + $"Error was {e.Message}"); continue; } if (ptProjectRoles.TryGetValue(ptUserId, out string ptRole)) { Console.WriteLine($"{ptRole}"); } else { Console.WriteLine($"Not found."); } // Demonstrate access to PT Data Access. IReadOnlyList <ParatextProject> userPtProjects = null; try { userPtProjects = await _paratextService.GetProjectsAsync(userSecret); } catch (Exception e) { _logger.Log($" {Program.Bullet3} Failure fetching user's PT projects. Skipping. " + $"Error was {e.Message}"); continue; } _logger.Log($" {Program.Bullet3} PT Data Access and PT Registry " + "based report on projects the user can access, narrowed to this project: ", false); List <string> ptProjectNamesList = userPtProjects .Where(ptProject => ptProject.ParatextId == sfProject.ParatextId) .Select(ptProject => ptProject.ShortName).ToList(); string ptProjectNames = string.Join(',', ptProjectNamesList); if (ptProjectNamesList.Count() < 1) { ptProjectNames = $"User is not on this project. " + $"PT reports they are on this many PT projects: {userPtProjects.Count()}"; } Console.WriteLine(ptProjectNames); if (doSynchronizations) { if (sfAdminsToUse != null && sfAdminsToUse.ContainsKey(sfProject.Id)) { sfAdminsToUse.TryGetValue(sfProject.Id, out string sfAdminIdToUse); bool isUserAtHand = sfUserId == sfAdminIdToUse; if (isUserAtHand) { _logger.Log($" {Program.Bullet2} For SF Project {sfProject.Id}, we were asked to use " + $"this SF user {sfUserId} to sync."); } else { _logger.Log($" {Program.Bullet2} For SF Project {sfProject.Id}, we were asked to use " + $"SF user {sfAdminIdToUse}, not {sfUserId}, to sync. So skipping this user."); continue; } } try { _logger.Log($" {Program.Bullet2} Starting an asynchronous synchronization for " + $"SF project {sfProject.Id} as SF user {sfUserId}."); Task syncTask = SynchronizeProject(sfUserId, sfProject.Id); var projectDoc = await _realtimeServiceConnection.FetchAsync <SFProject>(sfProject.Id); // Increment the queued count (such as done in SyncService), since it gets decremented // later by ParatextSyncRunner. await projectDoc.SubmitJson0OpAsync(op => op.Inc(pd => pd.Sync.QueuedCount)); _logger.Log($" {Program.Bullet3} Synchronization task for SF project {sfProject.Id} as " + $"SF user {sfUserId} has Sync Task Id {syncTask.Id}."); syncTasks.Add(syncTask); break; } catch (Exception e) { // We probably won't get here. But just in case. _logger.Log($" {Program.Bullet3} There was a problem with synchronizing. It might be " + $"tried next with another admin user. Exception is:{Environment.NewLine}{e}"); continue; } } } } if (doSynchronizations) { _logger.Log("Waiting for synchronization tasks to finish (if any). " + $"There are this many tasks: {syncTasks.Count}"); try { Task allTasks = Task.WhenAll(syncTasks); try { await allTasks.ConfigureAwait(false); } catch { if (allTasks.Exception == null) { throw; } ExceptionDispatchInfo.Capture(allTasks.Exception).Throw(); } _logger.Log("Synchronization tasks are finished."); } catch (AggregateException e) { _logger.Log("There was a problem with one or more synchronization tasks. " + $"Exception is:{Environment.NewLine}{e}"); } if (syncTasks.Any(task => !task.IsCompletedSuccessfully)) { _logger.Log("One or more sync tasks did not complete successfully."); } else { _logger.Log("All sync tasks finished with a claimed Task status of Completed Successfully."); } _logger.Log($"{Program.Bullet1} Sync task completion results:"); foreach (Task task in syncTasks) { string exceptionInfo = $"with exception {task.Exception?.InnerException}."; if (task.Exception == null) { exceptionInfo = "with no unhandled exception thrown."; } _logger.Log($" {Program.Bullet2} Sync task Id {task.Id} has status {task.Status} {exceptionInfo}"); if (task.Exception?.InnerExceptions?.Count > 1) { _logger.Log($" {Program.Bullet3} Sync task Id {task.Id} has more than one inner exception. " + "Sorry if this is redundant, but they are:"); foreach (var e in task.Exception.InnerExceptions) { _logger.Log($" {Program.Bullet3} Inner exception: {e}"); } } } } ReportLastSyncSuccesses(allSfProjects); }
public async Task RunAsync(PerformContext context, IJobCancellationToken cancellationToken, string userId, string jobId) { _job = await _jobs.UpdateAsync(j => j.Id == jobId, u => u .Set(j => j.BackgroundJobId, context.BackgroundJob.Id) .Set(j => j.State, SyncJobEntity.SyncingState)); if (_job == null) { return; } try { if ((await _users.TryGetAsync(userId)).TryResult(out UserEntity user) && (await _projects.TryGetAsync(_job.ProjectRef)).TryResult(out SFProjectEntity project)) { if (!_fileSystemService.DirectoryExists(WorkingDir)) { _fileSystemService.CreateDirectory(WorkingDir); } bool translateEnabled = project.TranslateConfig.Enabled; using (IConnection conn = await _realtimeService.ConnectAsync()) { string targetParatextId = project.ParatextId; IReadOnlyList <string> targetBooks = await _paratextService.GetBooksAsync(user, targetParatextId); string sourceParatextId = project.TranslateConfig.SourceParatextId; IReadOnlyList <string> sourceBooks = null; if (translateEnabled) { sourceBooks = await _paratextService.GetBooksAsync(user, sourceParatextId); } var booksToSync = new HashSet <string>(targetBooks); if (translateEnabled) { booksToSync.IntersectWith(sourceBooks); } var booksToDelete = new HashSet <string>( GetBooksToDelete(project, targetParatextId, targetBooks)); if (translateEnabled) { booksToDelete.UnionWith(GetBooksToDelete(project, sourceParatextId, sourceBooks)); } _step = 0; _stepCount = booksToSync.Count * 3; if (translateEnabled) { _stepCount *= 2; } _stepCount += booksToDelete.Count; foreach (string bookId in booksToSync) { if (!BookNames.TryGetValue(bookId, out string name)) { name = bookId; } TextEntity text = await _texts.UpdateAsync( t => t.ProjectRef == project.Id && t.BookId == bookId, u => u .SetOnInsert(t => t.Name, name) .SetOnInsert(t => t.BookId, bookId) .SetOnInsert(t => t.ProjectRef, project.Id) .SetOnInsert(t => t.OwnerRef, userId), upsert : true); List <Chapter> newChapters = await SyncOrCloneBookUsxAsync(user, conn, project, text, TextType.Target, targetParatextId, false); if (translateEnabled) { var chaptersToInclude = new HashSet <int>(newChapters.Select(c => c.Number)); await SyncOrCloneBookUsxAsync(user, conn, project, text, TextType.Source, sourceParatextId, true, chaptersToInclude); } await UpdateNotesData(conn, text, newChapters); } foreach (string bookId in booksToDelete) { TextEntity text = await _texts.DeleteAsync( t => t.ProjectRef == project.Id && t.BookId == bookId); await DeleteBookUsxAsync(conn, project, text, TextType.Target, targetParatextId); if (translateEnabled) { await DeleteBookUsxAsync(conn, project, text, TextType.Source, sourceParatextId); } await DeleteNotesData(conn, text); await UpdateProgress(); } } // TODO: Properly handle job cancellation cancellationToken.ThrowIfCancellationRequested(); if (translateEnabled) { // start training Machine engine await _engineService.StartBuildByProjectIdAsync(_job.ProjectRef); } await _projects.UpdateAsync(_job.ProjectRef, u => u .Set(p => p.LastSyncedDate, DateTime.UtcNow) .Unset(p => p.ActiveSyncJobRef)); } else { await _projects.UpdateAsync(_job.ProjectRef, u => u.Unset(p => p.ActiveSyncJobRef)); } _job = await _jobs.UpdateAsync(_job, u => u .Set(j => j.State, SyncJobEntity.IdleState) .Unset(j => j.BackgroundJobId)); }
/// <summary> /// Iterates through SF projects on the server and identifies one administrator user on the project. /// Using the administrator user's secrets, perform a send/receive with the Paratext server. Effectively, /// this clones the project to the Scripture Forge server. /// This will overwrite any un-synchronized data on SF. /// </summary> public async Task CloneSFProjects(string mode, IEnumerable <SFProject> projectsToClone) { string syncDir = Path.Combine(_siteOptions.Value.SiteDir, "sync"); bool doClone = mode == CLONE || mode == CLONE_AND_MOVE_OLD || mode == CLONE_SILENT; string syncDirOld = Path.Combine(_siteOptions.Value.SiteDir, "sync_old"); if (mode == CLONE_AND_MOVE_OLD) { if (!_fileSystemService.DirectoryExists(syncDirOld)) { _fileSystemService.CreateDirectory(syncDirOld); } } IConnection connection = await _realtimeService.ConnectAsync(); // Get the paratext project ID and admin user for all SF Projects foreach (SFProject proj in projectsToClone) { bool foundAdmin = false; foreach (string userId in proj.UserRoles.Keys) { if (proj.UserRoles.TryGetValue(userId, out string role) && role == SFProjectRole.Administrator) { foundAdmin = true; UserSecret userSecret = _userSecretRepo.Query().FirstOrDefault((UserSecret us) => us.Id == userId); string ptUsername = _paratextService.GetParatextUsername(userSecret); Log($"Project administrator identified on {proj.Name}: {ptUsername} ({userId})"); if (!doClone) { break; } try { var projectDoc = await connection.FetchAsync <SFProject>(proj.Id); await projectDoc.SubmitJson0OpAsync(op => { // Increment the queued count such as in SyncService op.Inc(pd => pd.Sync.QueuedCount); }); bool silent = mode == CLONE_SILENT; // Clone the paratext project and update the SF database with the project data await CloneAndSyncFromParatext(proj, userId, syncDir, silent); if (mode == CLONE_AND_MOVE_OLD) { string projectDir = Path.Combine(syncDir, proj.Id); string projectDirOld = Path.Combine(syncDirOld, proj.Id); _fileSystemService.MoveDirectory(projectDir, projectDirOld); } break; } catch (Exception e) { Log($"Unable to clone {proj.Name} ({proj.Id}) as user: {userId}{Environment.NewLine}" + $"Error was: {e}"); } } } if (!foundAdmin) { Log($"ERROR: Unable to identify a project administrator on {proj.Name}"); } } }