/// <summary>
/// Asynchronously creates a project for a Paratext resource.
/// </summary>
/// <param name="curUserId">The current user identifier.</param>
/// <param name="paratextId">The paratext resource identifier.</param>
/// <returns>SF project id of created project</returns>
/// <remarks>
/// This method will also work for a source project that has been deleted for some reason.
/// </remarks>
/// <exception cref="DataNotFoundException">
/// The user does not exist.
/// or
/// The paratext project does not exist.
/// </exception>
/// <exception cref="InvalidOperationException"></exception>
public async Task <string> CreateResourceProjectAsync(string curUserId, string paratextId)
{
Attempt <UserSecret> userSecretAttempt = await _userSecrets.TryGetAsync(curUserId);
if (!userSecretAttempt.TryResult(out UserSecret userSecret))
{
throw new DataNotFoundException("The user does not exist.");
}
// We check projects first, in case it is a project
IReadOnlyList <ParatextProject> ptProjects = await _paratextService.GetProjectsAsync(userSecret);
ParatextProject ptProject = ptProjects.SingleOrDefault(p => p.ParatextId == paratextId);
if (ptProject == null)
{
// If it is not a project, see if there is a matching resource
IReadOnlyList <ParatextResource> resources = await this._paratextService.GetResourcesAsync(curUserId);
ptProject = resources.SingleOrDefault(r => r.ParatextId == paratextId);
if (ptProject == null)
{
throw new DataNotFoundException("The paratext project or resource does not exist.");
}
}
return(await CreateResourceProjectInternalAsync(curUserId, ptProject));
}
private async Task DeleteBookAsync(Connection conn, TranslateProject project, TranslateDocumentSet docSet,
ParatextProject paratextProject, string docType, string bookId)
{
string projectPath = GetProjectPath(project, paratextProject);
File.Delete(GetBookTextFileName(projectPath, bookId));
Document <Delta> doc = GetShareDBDocument(conn, project, docSet, docType);
await doc.FetchAsync();
await doc.DeleteAsync();
}
private IEnumerable <string> GetBooksToDelete(TranslateProject project, ParatextProject paratextProject,
IEnumerable <string> books)
{
string projectPath = GetProjectPath(project, paratextProject);
var booksToDelete = new HashSet <string>(Directory.Exists(projectPath)
? Directory.EnumerateFiles(projectPath).Select(Path.GetFileNameWithoutExtension)
: Enumerable.Empty <string>());
booksToDelete.ExceptWith(books);
return(booksToDelete);
}
private async Task CloneBookAsync(User user, ParatextProject paratextProject, string fileName,
string bookId, Document <Delta> doc)
{
string bookText = await _paratextService.GetBookTextAsync(user, paratextProject.Id, bookId);
var bookTextElem = XElement.Parse(bookText);
Delta delta = _deltaUsxMapper.ToDelta(paratextProject.Id, bookTextElem.Element("usx"));
await doc.CreateAsync(delta);
await SaveBookTextAsync(bookTextElem, fileName);
}
private async Task SyncBookAsync(User user, ParatextProject paratextProject, string fileName,
string bookId, Document <Delta> doc)
{
await doc.FetchAsync();
XElement bookTextElem = await LoadBookTextAsync(fileName);
XElement oldUsxElem = bookTextElem.Element("usx");
if (oldUsxElem == null)
{
throw new InvalidOperationException("Invalid USX data, missing 'usx' element.");
}
XElement bookElem = oldUsxElem.Element("book");
if (bookElem == null)
{
throw new InvalidOperationException("Invalid USX data, missing 'book' element.");
}
XElement newUsxElem = _deltaUsxMapper.ToUsx((string)oldUsxElem.Attribute("version"),
(string)bookElem.Attribute("code"), (string)bookElem, doc.Data);
var revision = (string)bookTextElem.Attribute("revision");
string bookText;
if (XNode.DeepEquals(oldUsxElem, newUsxElem))
{
bookText = await _paratextService.GetBookTextAsync(user, paratextProject.Id, bookId);
}
else
{
bookText = await _paratextService.UpdateBookTextAsync(user, paratextProject.Id, bookId, revision,
newUsxElem.ToString());
}
bookTextElem = XElement.Parse(bookText);
Delta delta = _deltaUsxMapper.ToDelta(paratextProject.Id, bookTextElem.Element("usx"));
Delta diffDelta = doc.Data.Diff(delta);
await doc.SubmitOpAsync(diffDelta);
await SaveBookTextAsync(bookTextElem, fileName);
}
/// <summary>
/// Ensure the target project repository exists on the local SF server, cloning if necessary.
/// </summary>
private void EnsureProjectReposExists(UserSecret userSecret, ParatextProject target,
IInternetSharedRepositorySource repositorySource)
{
string username = GetParatextUsername(userSecret);
bool targetNeedsCloned =
ScrTextCollection.FindById(username, target.ParatextId) == null;
if (target is ParatextResource resource)
{
// If the target is a resource, install it
InstallResource(resource, target.ParatextId, targetNeedsCloned);
}
else if (targetNeedsCloned)
{
SharedRepository targetRepo = new SharedRepository(target.ShortName, HexId.FromStr(target.ParatextId),
RepositoryType.Shared);
CloneProjectRepo(repositorySource, target.ParatextId, targetRepo);
}
}
private async Task SendReceiveBookAsync(User user, Connection conn, TranslateProject project,
TranslateDocumentSet docSet, ParatextProject paratextProject, string docType, string bookId)
{
string projectPath = GetProjectPath(project, paratextProject);
if (!Directory.Exists(projectPath))
{
Directory.CreateDirectory(projectPath);
}
Document <Delta> doc = GetShareDBDocument(conn, project, docSet, docType);
string fileName = GetBookTextFileName(projectPath, bookId);
if (File.Exists(fileName))
{
await SyncBookAsync(user, paratextProject, fileName, bookId, doc);
}
else
{
await CloneBookAsync(user, paratextProject, fileName, bookId, doc);
}
}
/// <summary>
/// Returns SF project id of created project.
/// </summary>
public async Task <string> CreateProjectAsync(string curUserId, SFProjectCreateSettings settings)
{
Attempt <UserSecret> userSecretAttempt = await _userSecrets.TryGetAsync(curUserId);
if (!userSecretAttempt.TryResult(out UserSecret userSecret))
{
throw new DataNotFoundException("The user does not exist.");
}
IReadOnlyList <ParatextProject> ptProjects = await _paratextService.GetProjectsAsync(userSecret);
ParatextProject ptProject = ptProjects.SingleOrDefault(p => p.ParatextId == settings.ParatextId);
if (ptProject == null)
{
throw new DataNotFoundException("The paratext project does not exist.");
}
var project = new SFProject
{
ParatextId = settings.ParatextId,
Name = ptProject.Name,
ShortName = ptProject.ShortName,
WritingSystem = new WritingSystem {
Tag = ptProject.LanguageTag
},
TranslateConfig = new TranslateConfig
{
TranslationSuggestionsEnabled = settings.TranslationSuggestionsEnabled
},
CheckingConfig = new CheckingConfig
{
CheckingEnabled = settings.CheckingEnabled
}
};
Attempt <string> attempt = await TryGetProjectRoleAsync(project, curUserId);
if (!attempt.TryResult(out string projectRole) || projectRole != SFProjectRole.Administrator)
{
throw new ForbiddenException();
}
string projectId = ObjectId.GenerateNewId().ToString();
using (IConnection conn = await RealtimeService.ConnectAsync(curUserId))
{
if (this.RealtimeService.QuerySnapshots <SFProject>().Any(
(SFProject sfProject) => sfProject.ParatextId == project.ParatextId))
{
throw new InvalidOperationException(ErrorAlreadyConnectedKey);
}
IDocument <SFProject> projectDoc = await conn.CreateAsync <SFProject>(projectId, project);
await ProjectSecrets.InsertAsync(new SFProjectSecret { Id = projectDoc.Id });
IDocument <User> userDoc = await conn.FetchAsync <User>(curUserId);
await AddUserToProjectAsync(conn, projectDoc, userDoc, SFProjectRole.Administrator, false);
// Add the source after the project has been created
// This will make the source project appear after the target, if it needs to be created
if (settings.SourceParatextId != null && settings.SourceParatextId != settings.ParatextId)
{
TranslateSource source = await this.GetTranslateSourceAsync(
curUserId, userSecret, settings.SourceParatextId, ptProjects);
await projectDoc.SubmitJson0OpAsync(op =>
{
UpdateSetting(op, p => p.TranslateConfig.Source, source);
});
}
if (projectDoc.Data.TranslateConfig.TranslationSuggestionsEnabled)
{
var machineProject = new MachineProject
{
Id = projectDoc.Id,
SourceLanguageTag = projectDoc.Data.TranslateConfig.Source.WritingSystem.Tag,
TargetLanguageTag = projectDoc.Data.WritingSystem.Tag
};
await _engineService.AddProjectAsync(machineProject);
}
}
await _syncService.SyncAsync(curUserId, projectId, true);
return(projectId);
}
private string GetProjectPath(TranslateProject project, ParatextProject paratextProject)
{
return(Path.Combine(_options.Value.TranslateDir, project.Id, paratextProject.Id));
}
public async Task RunAsync(PerformContext context, IJobCancellationToken cancellationToken, string userId,
string jobId)
{
SendReceiveJob job = await _jobRepo.UpdateAsync(j => j.Id == jobId, u => u
.Set(j => j.BackgroundJobId, context.BackgroundJob.Id)
.Set(j => j.State, SendReceiveJob.SyncingState));
if (job == null)
{
return;
}
try
{
SendReceiveOptions options = _options.Value;
if ((await _userRepo.TryGetAsync(userId)).TryResult(out User user))
{
if ((await _projectRepo.TryGetAsync(job.ProjectRef)).TryResult(out TranslateProject project))
{
if (!Directory.Exists(options.TranslateDir))
{
Directory.CreateDirectory(options.TranslateDir);
}
IRepository <TranslateDocumentSet> docSetRepo = _docSetRepoFactory.Create(project);
using (var conn = new Connection(new Uri(options.ShareDBUrl)))
{
await conn.ConnectAsync();
ParatextProject sourceParatextProject = project.Config.Source.ParatextProject;
IReadOnlyList <string> sourceBooks = await _paratextService.GetBooksAsync(user,
sourceParatextProject.Id);
ParatextProject targetParatextProject = project.Config.Target.ParatextProject;
IReadOnlyList <string> targetBooks = await _paratextService.GetBooksAsync(user,
targetParatextProject.Id);
var booksToSendReceive = new HashSet <string>();
booksToSendReceive.UnionWith(sourceBooks);
booksToSendReceive.IntersectWith(targetBooks);
var booksToDelete = new HashSet <string>();
booksToDelete.UnionWith(GetBooksToDelete(project, sourceParatextProject, sourceBooks));
booksToDelete.UnionWith(GetBooksToDelete(project, targetParatextProject, targetBooks));
int step = 0;
int stepCount = booksToSendReceive.Count + booksToDelete.Count;
foreach (string bookId in booksToSendReceive)
{
if (!BookNames.TryGetValue(bookId, out string name))
{
name = bookId;
}
TranslateDocumentSet docSet = await docSetRepo.UpdateAsync(ds => ds.BookId == bookId,
u => u.SetOnInsert(ds => ds.Name, name)
.SetOnInsert(ds => ds.BookId, bookId)
.Set(ds => ds.IsDeleted, false), true);
await SendReceiveBookAsync(user, conn, project, docSet, sourceParatextProject, "source",
bookId);
await SendReceiveBookAsync(user, conn, project, docSet, targetParatextProject, "target",
bookId);
step++;
job = await UpdateProgress(job, step, stepCount);
}
foreach (string bookId in booksToDelete)
{
TranslateDocumentSet docSet = await docSetRepo.UpdateAsync(ds => ds.BookId == bookId,
u => u.Set(ds => ds.IsDeleted, true));
await DeleteBookAsync(conn, project, docSet, sourceParatextProject, "source", bookId);
await DeleteBookAsync(conn, project, docSet, targetParatextProject, "target", bookId);
step++;
job = await UpdateProgress(job, step, stepCount);
}
await conn.CloseAsync();
}
job = await _jobRepo.UpdateAsync(job, u => u
.Set(j => j.State, SendReceiveJob.IdleState)
.Unset(j => j.BackgroundJobId));
await _projectRepo.UpdateAsync(project,
u => u.Set(p => p.LastSyncedDate, job.DateModified));
}
}
}
catch (Exception e)
{
_logger.LogError(e, "Error occurred while executing Paratext S/R job '{Job}'", job.Id);
await _jobRepo.UpdateAsync(job, u => u
.Set(j => j.State, SendReceiveJob.HoldState)
.Unset(j => j.BackgroundJobId));
}
}
