public async Task <List <Chapter> > SyncBookUsxAsync(TextInfo text, TextType textType, string paratextId, string fileName, bool isReadOnly, ISet <int> chaptersToInclude) { if (text.Chapters.Count < 1) { // The SF DB is corrupt. Also, the project doc in the SF DB has no record of any chapters that // we could synchronize. SortedList <int, IDocument <TextData> > allTextDocsForBook = await PessimisticallyFetchTextDocsAsync(text, textType); if (allTextDocsForBook.Count < 1) { // We don't have any chapter text docs for the book in the SF DB. _logger.LogWarning("SyncBookUsxAsync() detected a corrupt SF DB for project with paratext id " + $"{paratextId}, for book {text.BookNum}, TextType {textType}, because the TextInfo has an " + $"invalid chapter count of {text.Chapters.Count}. There aren't any chapter text docs for " + $"this book in the DB anyway though. Returning null to skip fetching or syncing this book."); return(null); } else { // We have chapter text docs for the book in the SF DB that the project doc did not know were there. // Throw an error and there may need to be a manual investigation to know how to sync this project. throw new Exception("SyncBookUsxAsync() stopped because there are chapter text docs for a " + $"project's book that are not known about by the project doc. And the project doc has an " + $"invalid description of book chapters. TextInfo booknum is {text.BookNum}. " + $"Text type is {textType}. Paratext project id is {paratextId}. The project TextInfo knows " + $"about {text.Chapters.Count} chapters, but in the DB there are {allTextDocsForBook.Count} " + "chapters for the book."); } } SortedList <int, IDocument <TextData> > dbChapterDocs = await FetchTextDocsAsync(text, textType); string bookId = Canon.BookNumberToId(text.BookNum); string ptBookText = await FetchFromAndUpdateParatextAsync(text, paratextId, fileName, isReadOnly, bookId, dbChapterDocs); if (ptBookText == null) { return(null); } await UpdateProgress(); XElement bookTextElem = ParseText(ptBookText); var usxDoc = new XDocument(bookTextElem.Element("usx")); Dictionary <int, ChapterDelta> incomingChapters = _deltaUsxMapper.ToChapterDeltas(usxDoc) .ToDictionary(cd => cd.Number); // Set SF DB to snapshot from Paratext. List <Chapter> chapters = await ChangeDbToNewSnapshotAsync(text, textType, chaptersToInclude, dbChapterDocs, incomingChapters); // Save to disk await SaveXmlFileAsync(bookTextElem, fileName); await UpdateProgress(); return(chapters); }
private async Task <List <Chapter> > CloneBookUsxAsync(TextInfo text, TextType textType, string paratextId, string fileName, ISet <int> chaptersToInclude) { // Remove any stale text_data records that may be in the way. await DeleteAllTextDocsForBookAsync(text, textType); string bookText = await _paratextService.GetBookTextAsync(_userSecret, paratextId, Canon.BookNumberToId(text.BookNum)); var bookTextElem = ParseText(bookText); await UpdateProgress(); var usxDoc = new XDocument(bookTextElem.Element("usx")); Dictionary <int, ChapterDelta> deltas = _deltaUsxMapper.ToChapterDeltas(usxDoc) .ToDictionary(cd => cd.Number); var tasks = new List <Task>(); var chapters = new List <Chapter>(); foreach (KeyValuePair <int, ChapterDelta> kvp in deltas) { if (chaptersToInclude != null && !chaptersToInclude.Contains(kvp.Key)) { continue; } async Task createText(int chapterNum, Delta delta) { IDocument <TextData> textDataDoc = GetTextDoc(text, chapterNum, textType); await textDataDoc.FetchAsync(); if (textDataDoc.IsLoaded) { Console.WriteLine($"CloneBookUsxAsync: Going to delete text doc before re-creating it. " + $"FYI that it and its contents are: textinfo booknum {text.BookNum}, " + $"chapter count {text.Chapters.Count}, has source {text.HasSource}, " + $"int chapterNum: {chapterNum}, text type: {textType}, paratext project id {paratextId}. " + $"Contents: {textDataDoc.Data.ToString()} END_CONTENTS."); await textDataDoc.DeleteAsync(); } await textDataDoc.CreateAsync(new TextData(delta)); } tasks.Add(createText(kvp.Key, kvp.Value.Delta)); chapters.Add(new Chapter { Number = kvp.Key, LastVerse = kvp.Value.LastVerse, IsValid = kvp.Value.IsValid }); } await Task.WhenAll(tasks); await SaveXmlFileAsync(bookTextElem, fileName); await UpdateProgress(); return(chapters); }
public void GetUpdatedBookInfo() { CheckingStatuses checkingStatusData; try { checkingStatusData = CheckingStatuses.Get(UnderlyingScrText); checkingStatusData.CancelChanges(); // This forces it to reload from disk. } catch (Exception e) { throw new ApplicationException($"Unexpected error retrieving the checking status data for {kParatextProgramName} project: {ProjectId}", e); } foreach (var bookNum in CanonicalBookNumbersInProject) { var code = Canon.BookNumberToId(bookNum); if (!Canon.IsBookOTNT(bookNum)) { m_bookInfo.Add(bookNum, code, ParatextProjectBookInfo.BookState.ExcludedNonCanonical); continue; } var failedChecks = new List <String>(m_requiredChecks.Count); foreach (var check in m_requiredChecks) { CheckingStatus status; try { status = checkingStatusData.GetCheckingStatus(code, check); } catch (Exception e) { throw new ApplicationException($"Unexpected error retrieving the {check} check status for {code} in {kParatextProgramName} project: {ProjectId}", e); } if (status == null || !status.Successful) { failedChecks.Add(check); } } if (failedChecks.Any()) { m_bookInfo.Add(bookNum, code, ParatextProjectBookInfo.BookState.FailedCheck, failedChecks); } else { m_bookInfo.Add(bookNum, code, ParatextProjectBookInfo.BookState.NoProblem); } } }
private async Task UpdateNotesData(TextInfo text, List <Chapter> newChapters) { IReadOnlyList <IDocument <Question> > allQuestionDocs = await FetchQuestionDocsAsync(text); // handle deletion of chapters var chapterNums = new HashSet <int>(newChapters.Select(c => c.Number)); var tasks = new List <Task>(); foreach (IDocument <Question> questionDoc in allQuestionDocs) { if (!chapterNums.Contains(questionDoc.Data.VerseRef.ChapterNum)) { tasks.Add(questionDoc.DeleteAsync()); } } await Task.WhenAll(tasks); if (CheckingEnabled) { XElement oldNotesElem; string oldNotesText = await _paratextService.GetNotesAsync(_userSecret, _projectDoc.Data.ParatextId, Canon.BookNumberToId(text.BookNum)); if (oldNotesText != "") { oldNotesElem = ParseText(oldNotesText); } else { oldNotesElem = new XElement("notes", new XAttribute("version", "1.1")); } XElement notesElem = await _notesMapper.GetNotesChangelistAsync(oldNotesElem, allQuestionDocs); if (notesElem.Elements("thread").Any()) { await _paratextService.UpdateNotesAsync(_userSecret, _projectDoc.Data.ParatextId, notesElem.ToString()); } await UpdateProgress(); } }
private async Task <List <Chapter> > SyncOrCloneBookUsxAsync(TextInfo text, TextType textType, string paratextId, bool isReadOnly, ISet <int> chaptersToInclude = null) { string projectPath = GetProjectPath(textType); if (!_fileSystemService.DirectoryExists(projectPath)) { _fileSystemService.CreateDirectory(projectPath); } string fileName = GetUsxFileName(projectPath, text.BookNum); bool fileExists = _fileSystemService.FileExists(fileName); if (fileExists && text.Chapters.Count < 1) { Console.WriteLine($"SyncOrCloneBookUsxAsync: Warning: When processing textinfo booknum {text.BookNum}, " + $"chapters {text.Chapters.Count}, texttype {textType}, for paratext project id {paratextId}, " + $"the text chapter count was 0 but there was already a file at {fileName}. Perhaps indicating " + $"a prior failed clone. Going to try cloning it again, rather than syncing."); } if (fileExists && text.Chapters.Count > 0) { return(await SyncBookUsxAsync(text, textType, paratextId, fileName, isReadOnly, chaptersToInclude)); } else { try { return(await CloneBookUsxAsync(text, textType, paratextId, fileName, chaptersToInclude)); } catch (Exception) { string bookName = Canon.BookNumberToId(text.BookNum); _logger.LogWarning($"Failed to clone a book: {bookName}. Skipping..."); // We can skip cloning new books to this project folder (they get cloned // with the whole project in the following clone all migration step) return(null); } } }
public static string GetTextDocId(string projectId, int book, int chapter) { return($"{projectId}:{Canon.BookNumberToId(book)}:{chapter}:target"); }
private static string GetUsxFileName(string projectPath, int bookNum) { return(Path.Combine(projectPath, Canon.BookNumberToId(bookNum) + ".xml")); }