public async Task <List <Chapter> > SyncBookUsxAsync(TextInfo text, TextType textType, string paratextId,
string fileName, bool isReadOnly, ISet <int> chaptersToInclude)
{
if (text.Chapters.Count < 1)
{
// The SF DB is corrupt. Also, the project doc in the SF DB has no record of any chapters that
// we could synchronize.
SortedList <int, IDocument <TextData> > allTextDocsForBook =
await PessimisticallyFetchTextDocsAsync(text, textType);
if (allTextDocsForBook.Count < 1)
{
// We don't have any chapter text docs for the book in the SF DB.
_logger.LogWarning("SyncBookUsxAsync() detected a corrupt SF DB for project with paratext id "
+ $"{paratextId}, for book {text.BookNum}, TextType {textType}, because the TextInfo has an "
+ $"invalid chapter count of {text.Chapters.Count}. There aren't any chapter text docs for "
+ $"this book in the DB anyway though. Returning null to skip fetching or syncing this book.");
return(null);
}
else
{
// We have chapter text docs for the book in the SF DB that the project doc did not know were there.
// Throw an error and there may need to be a manual investigation to know how to sync this project.
throw new Exception("SyncBookUsxAsync() stopped because there are chapter text docs for a "
+ $"project's book that are not known about by the project doc. And the project doc has an "
+ $"invalid description of book chapters. TextInfo booknum is {text.BookNum}. "
+ $"Text type is {textType}. Paratext project id is {paratextId}. The project TextInfo knows "
+ $"about {text.Chapters.Count} chapters, but in the DB there are {allTextDocsForBook.Count} "
+ "chapters for the book.");
}
}
SortedList <int, IDocument <TextData> > dbChapterDocs = await FetchTextDocsAsync(text, textType);
string bookId = Canon.BookNumberToId(text.BookNum);
string ptBookText = await FetchFromAndUpdateParatextAsync(text,
paratextId, fileName, isReadOnly, bookId, dbChapterDocs);
if (ptBookText == null)
{
return(null);
}
await UpdateProgress();
XElement bookTextElem = ParseText(ptBookText);
var usxDoc = new XDocument(bookTextElem.Element("usx"));
Dictionary <int, ChapterDelta> incomingChapters = _deltaUsxMapper.ToChapterDeltas(usxDoc)
.ToDictionary(cd => cd.Number);
// Set SF DB to snapshot from Paratext.
List <Chapter> chapters = await ChangeDbToNewSnapshotAsync(text,
textType, chaptersToInclude, dbChapterDocs, incomingChapters);
// Save to disk
await SaveXmlFileAsync(bookTextElem, fileName);
await UpdateProgress();
return(chapters);
}
private async Task <List <Chapter> > CloneBookUsxAsync(TextInfo text, TextType textType, string paratextId,
string fileName, ISet <int> chaptersToInclude)
{
// Remove any stale text_data records that may be in the way.
await DeleteAllTextDocsForBookAsync(text, textType);
string bookText = await _paratextService.GetBookTextAsync(_userSecret, paratextId,
Canon.BookNumberToId(text.BookNum));
var bookTextElem = ParseText(bookText);
await UpdateProgress();
var usxDoc = new XDocument(bookTextElem.Element("usx"));
Dictionary <int, ChapterDelta> deltas = _deltaUsxMapper.ToChapterDeltas(usxDoc)
.ToDictionary(cd => cd.Number);
var tasks = new List <Task>();
var chapters = new List <Chapter>();
foreach (KeyValuePair <int, ChapterDelta> kvp in deltas)
{
if (chaptersToInclude != null && !chaptersToInclude.Contains(kvp.Key))
{
continue;
}
async Task createText(int chapterNum, Delta delta)
{
IDocument <TextData> textDataDoc = GetTextDoc(text, chapterNum, textType);
await textDataDoc.FetchAsync();
if (textDataDoc.IsLoaded)
{
Console.WriteLine($"CloneBookUsxAsync: Going to delete text doc before re-creating it. "
+ $"FYI that it and its contents are: textinfo booknum {text.BookNum}, "
+ $"chapter count {text.Chapters.Count}, has source {text.HasSource}, "
+ $"int chapterNum: {chapterNum}, text type: {textType}, paratext project id {paratextId}. "
+ $"Contents: {textDataDoc.Data.ToString()} END_CONTENTS.");
await textDataDoc.DeleteAsync();
}
await textDataDoc.CreateAsync(new TextData(delta));
}
tasks.Add(createText(kvp.Key, kvp.Value.Delta));
chapters.Add(new Chapter
{
Number = kvp.Key,
LastVerse = kvp.Value.LastVerse,
IsValid = kvp.Value.IsValid
});
}
await Task.WhenAll(tasks);
await SaveXmlFileAsync(bookTextElem, fileName);
await UpdateProgress();
return(chapters);
}
public void GetUpdatedBookInfo()
{
CheckingStatuses checkingStatusData;
try
{
checkingStatusData = CheckingStatuses.Get(UnderlyingScrText);
checkingStatusData.CancelChanges(); // This forces it to reload from disk.
}
catch (Exception e)
{
throw new ApplicationException($"Unexpected error retrieving the checking status data for {kParatextProgramName} project: {ProjectId}", e);
}
foreach (var bookNum in CanonicalBookNumbersInProject)
{
var code = Canon.BookNumberToId(bookNum);
if (!Canon.IsBookOTNT(bookNum))
{
m_bookInfo.Add(bookNum, code, ParatextProjectBookInfo.BookState.ExcludedNonCanonical);
continue;
}
var failedChecks = new List <String>(m_requiredChecks.Count);
foreach (var check in m_requiredChecks)
{
CheckingStatus status;
try
{
status = checkingStatusData.GetCheckingStatus(code, check);
}
catch (Exception e)
{
throw new ApplicationException($"Unexpected error retrieving the {check} check status for {code} in {kParatextProgramName} project: {ProjectId}", e);
}
if (status == null || !status.Successful)
{
failedChecks.Add(check);
}
}
if (failedChecks.Any())
{
m_bookInfo.Add(bookNum, code, ParatextProjectBookInfo.BookState.FailedCheck, failedChecks);
}
else
{
m_bookInfo.Add(bookNum, code, ParatextProjectBookInfo.BookState.NoProblem);
}
}
}
private async Task UpdateNotesData(TextInfo text, List <Chapter> newChapters)
{
IReadOnlyList <IDocument <Question> > allQuestionDocs = await FetchQuestionDocsAsync(text);
// handle deletion of chapters
var chapterNums = new HashSet <int>(newChapters.Select(c => c.Number));
var tasks = new List <Task>();
foreach (IDocument <Question> questionDoc in allQuestionDocs)
{
if (!chapterNums.Contains(questionDoc.Data.VerseRef.ChapterNum))
{
tasks.Add(questionDoc.DeleteAsync());
}
}
await Task.WhenAll(tasks);
if (CheckingEnabled)
{
XElement oldNotesElem;
string oldNotesText = await _paratextService.GetNotesAsync(_userSecret, _projectDoc.Data.ParatextId,
Canon.BookNumberToId(text.BookNum));
if (oldNotesText != "")
{
oldNotesElem = ParseText(oldNotesText);
}
else
{
oldNotesElem = new XElement("notes", new XAttribute("version", "1.1"));
}
XElement notesElem = await _notesMapper.GetNotesChangelistAsync(oldNotesElem, allQuestionDocs);
if (notesElem.Elements("thread").Any())
{
await _paratextService.UpdateNotesAsync(_userSecret, _projectDoc.Data.ParatextId,
notesElem.ToString());
}
await UpdateProgress();
}
}
private async Task <List <Chapter> > SyncOrCloneBookUsxAsync(TextInfo text, TextType textType, string paratextId,
bool isReadOnly, ISet <int> chaptersToInclude = null)
{
string projectPath = GetProjectPath(textType);
if (!_fileSystemService.DirectoryExists(projectPath))
{
_fileSystemService.CreateDirectory(projectPath);
}
string fileName = GetUsxFileName(projectPath, text.BookNum);
bool fileExists = _fileSystemService.FileExists(fileName);
if (fileExists && text.Chapters.Count < 1)
{
Console.WriteLine($"SyncOrCloneBookUsxAsync: Warning: When processing textinfo booknum {text.BookNum}, "
+ $"chapters {text.Chapters.Count}, texttype {textType}, for paratext project id {paratextId}, "
+ $"the text chapter count was 0 but there was already a file at {fileName}. Perhaps indicating "
+ $"a prior failed clone. Going to try cloning it again, rather than syncing.");
}
if (fileExists && text.Chapters.Count > 0)
{
return(await SyncBookUsxAsync(text, textType, paratextId, fileName, isReadOnly, chaptersToInclude));
}
else
{
try
{
return(await CloneBookUsxAsync(text, textType, paratextId, fileName, chaptersToInclude));
}
catch (Exception)
{
string bookName = Canon.BookNumberToId(text.BookNum);
_logger.LogWarning($"Failed to clone a book: {bookName}. Skipping...");
// We can skip cloning new books to this project folder (they get cloned
// with the whole project in the following clone all migration step)
return(null);
}
}
}
public static string GetTextDocId(string projectId, int book, int chapter)
{
return($"{projectId}:{Canon.BookNumberToId(book)}:{chapter}:target");
}
private static string GetUsxFileName(string projectPath, int bookNum)
{
return(Path.Combine(projectPath, Canon.BookNumberToId(bookNum) + ".xml"));
}