public void TestMakeFriendlyForPathFallback()
{
const string fallback = "Lift";
const string nonEmpty = "qwerty";
Assert.That(Sanitization.MakeFriendlyForPath(""), Is.EqualTo(""));
Assert.That(Sanitization.MakeFriendlyForPath("", fallback), Is.EqualTo(fallback));
Assert.That(Sanitization.MakeFriendlyForPath(nonEmpty, fallback), Is.EqualTo(nonEmpty));
}
public async Task TestDeletedWordsExportToLift()
{
var word = Util.RandomWord(_projId);
var secondWord = Util.RandomWord(_projId);
var wordToDelete = Util.RandomWord(_projId);
var wordToUpdate = _wordRepo.Create(word).Result;
wordToDelete = _wordRepo.Create(wordToDelete).Result;
// Create untouched word.
_ = _wordRepo.Create(secondWord).Result;
word.Id = "";
word.Vernacular = "updated";
await _wordService.Update(_projId, wordToUpdate.Id, word);
await _wordService.DeleteFrontierWord(_projId, wordToDelete.Id);
_liftController.ExportLiftFile(_projId, UserId).Wait();
var result = (FileStreamResult)_liftController.DownloadLiftFile(_projId, UserId).Result;
Assert.NotNull(result);
// Read contents.
byte[] contents;
await using (var fileStream = result.FileStream)
{
contents = ReadAllBytes(fileStream);
}
// Write LiftFile contents to a temporary directory.
var extractedExportDir = ExtractZipFileContents(contents);
var sanitizedProjName = Sanitization.MakeFriendlyForPath(ProjName, "Lift");
var exportPath = Path.Combine(
extractedExportDir, sanitizedProjName, sanitizedProjName + ".lift");
var text = await File.ReadAllTextAsync(exportPath, Encoding.UTF8);
// TODO: Add SIL or other XML assertion library and verify with xpath that the correct entries are
// kept vs deleted
// Make sure we exported 2 live and one dead entry
Assert.That(Regex.Matches(text, "<entry").Count, Is.EqualTo(3));
// There is only one deleted word
Assert.That(text.IndexOf("dateDeleted"), Is.EqualTo(text.LastIndexOf("dateDeleted")));
// Delete the export
await _liftController.DeleteLiftFile(UserId);
var notFoundResult = _liftController.DownloadLiftFile(_projId, UserId).Result;
Assert.That(notFoundResult is NotFoundObjectResult);
}
public void TestMakeFriendlyForPath(List <string> nameName)
{
Assert.That(Sanitization.MakeFriendlyForPath(nameName[0]), Is.EqualTo(nameName[1]));
}
/// <summary> Exports information from a project to a lift package zip </summary>
/// <exception cref="MissingProjectException"> If Project does not exist. </exception>
/// <returns> Path to compressed zip file containing export. </returns>
public async Task <string> LiftExport(
string projectId, IWordRepository wordRepo, IProjectRepository projRepo)
{
// Validate project exists.
var proj = await projRepo.GetProject(projectId);
if (proj is null)
{
throw new MissingProjectException($"Project does not exist: {projectId}");
}
var vernacularBcp47 = proj.VernacularWritingSystem.Bcp47;
// Generate the zip dir.
var exportDir = FileStorage.GenerateLiftExportDirPath(projectId);
var liftExportDir = Path.Combine(exportDir, "LiftExport");
if (Directory.Exists(liftExportDir))
{
Directory.Delete(liftExportDir, true);
}
var projNameAsPath = Sanitization.MakeFriendlyForPath(proj.Name, "Lift");
var zipDir = Path.Combine(liftExportDir, projNameAsPath);
Directory.CreateDirectory(zipDir);
// Add audio dir inside zip dir.
var audioDir = Path.Combine(zipDir, "audio");
Directory.CreateDirectory(audioDir);
var liftPath = Path.Combine(zipDir, projNameAsPath + ".lift");
// noBOM will work with PrinceXML
using var liftWriter = new CombineLiftWriter(liftPath, ByteOrderStyle.BOM);
var rangesDest = Path.Combine(zipDir, projNameAsPath + ".lift-ranges");
// Write header of lift document.
var header =
$@"
<ranges>
<range id = ""semantic-domain-ddp4"" href = ""{rangesDest}""/>
</ranges>
<fields>
<field tag = ""Plural"">
<form lang = ""en""><text></text></form>
<form lang = ""qaa-x-spec""><text> Class = LexEntry; Type = String; WsSelector = kwsVern </text></form>
</field>
</fields>
";
liftWriter.WriteHeader(header);
// Write out every word with all of its information
var allWords = await wordRepo.GetAllWords(projectId);
var frontier = await wordRepo.GetFrontier(projectId);
var activeWords = frontier.Where(
x => x.Senses.Any(s => s.Accessibility == State.Active)).ToList();
// All words in the frontier with any senses are considered current.
// The Combine does not import senseless entries and the interface is supposed to prevent creating them.
// So the the words found in allWords with no matching guid in activeWords are exported as 'deleted'.
var deletedWords = allWords.Where(
x => activeWords.All(w => w.Guid != x.Guid)).DistinctBy(w => w.Guid).ToList();
foreach (var wordEntry in activeWords)
{
var entry = new LexEntry(MakeSafeXmlAttribute(wordEntry.Vernacular), wordEntry.Guid);
if (DateTime.TryParse(wordEntry.Created, out var createdTime))
{
entry.CreationTime = createdTime;
}
if (DateTime.TryParse(wordEntry.Modified, out var modifiedTime))
{
entry.ModificationTime = modifiedTime;
}
AddNote(entry, wordEntry);
AddVern(entry, wordEntry, vernacularBcp47);
AddSenses(entry, wordEntry);
AddAudio(entry, wordEntry, audioDir, projectId);
liftWriter.Add(entry);
}
foreach (var wordEntry in deletedWords)
{
var entry = new LexEntry(MakeSafeXmlAttribute(wordEntry.Vernacular), wordEntry.Guid);
AddNote(entry, wordEntry);
AddVern(entry, wordEntry, vernacularBcp47);
AddSenses(entry, wordEntry);
AddAudio(entry, wordEntry, audioDir, projectId);
liftWriter.AddDeletedEntry(entry);
}
liftWriter.End();
// Export semantic domains to lift-ranges
var extractedPathToImport = FileStorage.GenerateImportExtractedLocationDirPath(projectId, false);
string?firstImportDir = null;
if (Directory.Exists(extractedPathToImport))
{
// TODO: Should an error be raised if this returns null?
firstImportDir = Directory.GetDirectories(extractedPathToImport).Select(
Path.GetFileName).ToList().Single();
}
var importLiftDir = firstImportDir ?? "";
var rangesSrc = Path.Combine(extractedPathToImport, importLiftDir, $"{importLiftDir}.lift-ranges");
// If there are no new semantic domains, and the old lift-ranges file is still around, just copy it
if (proj.SemanticDomains.Count == 0 && File.Exists(rangesSrc))
{
File.Copy(rangesSrc, rangesDest, true);
}
else // Make a new lift-ranges file
{
using var liftRangesWriter = XmlWriter.Create(rangesDest, new XmlWriterSettings
{
Indent = true,
NewLineOnAttributes = true,
Async = true
});
await liftRangesWriter.WriteStartDocumentAsync();
liftRangesWriter.WriteStartElement("lift-ranges");
liftRangesWriter.WriteStartElement("range");
liftRangesWriter.WriteAttributeString("id", "semantic-domain-ddp4");
// Pull from resources file with all English semantic domains
var assembly = typeof(LiftService).GetTypeInfo().Assembly;
const string semDomListFile = "BackendFramework.Data.sdList.txt";
var resource = assembly.GetManifestResourceStream(semDomListFile);
if (resource is null)
{
throw new Exception($"Unable to load semantic domain list: {semDomListFile}");
}
string sdList;
using (var reader = new StreamReader(resource, Encoding.UTF8))
{
sdList = await reader.ReadToEndAsync();
}
var sdLines = sdList.Split(Environment.NewLine);
foreach (var line in sdLines)
{
if (line != "")
{
var items = line.Split("`");
WriteRangeElement(liftRangesWriter, items[0], items[1], items[2], items[3]);
}
}
// Pull from new semantic domains in project
foreach (var sd in proj.SemanticDomains)
{
WriteRangeElement(liftRangesWriter, sd.Id, Guid.NewGuid().ToString(), sd.Name, sd.Description);
}
await liftRangesWriter.WriteEndElementAsync(); //end semantic-domain-ddp4 range
await liftRangesWriter.WriteEndElementAsync(); //end lift-ranges
await liftRangesWriter.WriteEndDocumentAsync();
await liftRangesWriter.FlushAsync();
liftRangesWriter.Close();
}
// Export character set to ldml.
var ldmlDir = Path.Combine(zipDir, "WritingSystems");
Directory.CreateDirectory(ldmlDir);
if (vernacularBcp47 != "")
{
var validChars = proj.ValidCharacters;
LdmlExport(ldmlDir, vernacularBcp47, validChars);
}
// Compress everything.
var destinationFileName = Path.Combine(exportDir,
Path.Combine($"LiftExportCompressed-{proj.Id}_{DateTime.Now:yyyy-MM-dd_hh-mm-ss}.zip"));
var zipParentDir = Path.GetDirectoryName(zipDir);
if (zipParentDir is null)
{
throw new Exception($"Unable to find parent dir of: {zipDir}");
}
ZipFile.CreateFromDirectory(zipParentDir, destinationFileName);
// Clean up the temporary folder structure that was compressed.
Directory.Delete(liftExportDir, true);
return(destinationFileName);
}
public void TestRoundtrip(RoundTripObj roundTripObj)
{
// This test assumes you have the starting .zip (Filename) included in your project files.
var pathToStartZip = Path.Combine(Util.AssetsDir, roundTripObj.Filename);
Assert.IsTrue(File.Exists(pathToStartZip));
// Roundtrip Part 1
// Init the project the .zip info is added to.
var proj1 = Util.RandomProject();
proj1.VernacularWritingSystem.Bcp47 = roundTripObj.Language;
proj1 = _projRepo.Create(proj1).Result;
// Upload the zip file.
// Generate api parameter with filestream.
using (var stream = File.OpenRead(pathToStartZip))
{
var fileUpload = InitFile(stream, roundTripObj.Filename);
// Make api call.
var result = _liftController.UploadLiftFile(proj1 !.Id, fileUpload).Result;
Assert.That(result is OkObjectResult);
}
proj1 = _projRepo.GetProject(proj1.Id).Result;
if (proj1 is null)
{
Assert.Fail();
return;
}
Assert.That(proj1.LiftImported);
var allWords = _wordRepo.GetAllWords(proj1.Id).Result;
Assert.AreEqual(allWords.Count, roundTripObj.NumOfWords);
// We are currently only testing guids on the single-entry data sets.
if (roundTripObj.EntryGuid != "" && allWords.Count == 1)
{
Assert.AreEqual(allWords[0].Guid.ToString(), roundTripObj.EntryGuid);
if (roundTripObj.SenseGuid != "")
{
Assert.AreEqual(allWords[0].Senses[0].Guid.ToString(), roundTripObj.SenseGuid);
}
}
// Export.
var exportedFilePath = _liftController.CreateLiftExport(proj1.Id).Result;
var exportedDirectory = FileOperations.ExtractZipFile(exportedFilePath, null, false);
// Assert the file was created with desired hierarchy.
Assert.That(Directory.Exists(exportedDirectory));
var sanitizedProjName = Sanitization.MakeFriendlyForPath(proj1.Name, "Lift");
var exportedProjDir = Path.Combine(exportedDirectory, sanitizedProjName);
Assert.That(Directory.Exists(Path.Combine(exportedProjDir, "audio")));
foreach (var audioFile in roundTripObj.AudioFiles)
{
Assert.That(File.Exists(Path.Combine(exportedProjDir, "audio", audioFile)));
}
Assert.That(Directory.Exists(Path.Combine(exportedProjDir, "WritingSystems")));
Assert.That(File.Exists(Path.Combine(
exportedProjDir, "WritingSystems", roundTripObj.Language + ".ldml")));
Assert.That(File.Exists(Path.Combine(exportedProjDir, sanitizedProjName + ".lift")));
Directory.Delete(exportedDirectory, true);
// Clean up.
_wordRepo.DeleteAllWords(proj1.Id);
// Roundtrip Part 2
// Init the project the .zip info is added to.
var proj2 = Util.RandomProject();
proj2.VernacularWritingSystem.Bcp47 = roundTripObj.Language;
proj2 = _projRepo.Create(proj2).Result;
// Upload the exported words again.
// Generate api parameter with filestream.
using (var fstream = File.OpenRead(exportedFilePath))
{
var fileUpload = InitFile(fstream, roundTripObj.Filename);
// Make api call.
var result2 = _liftController.UploadLiftFile(proj2 !.Id, fileUpload).Result;
Assert.That(result2 is OkObjectResult);
}
proj2 = _projRepo.GetProject(proj2.Id).Result;
if (proj2 is null)
{
Assert.Fail();
return;
}
// Clean up zip file.
File.Delete(exportedFilePath);
allWords = _wordRepo.GetAllWords(proj2.Id).Result;
Assert.That(allWords, Has.Count.EqualTo(roundTripObj.NumOfWords));
// We are currently only testing guids on the single-entry data sets.
if (roundTripObj.EntryGuid != "" && allWords.Count == 1)
{
Assert.AreEqual(allWords[0].Guid.ToString(), roundTripObj.EntryGuid);
if (roundTripObj.SenseGuid != "")
{
Assert.AreEqual(allWords[0].Senses[0].Guid.ToString(), roundTripObj.SenseGuid);
}
}
// Export.
exportedFilePath = _liftController.CreateLiftExport(proj2.Id).Result;
exportedDirectory = FileOperations.ExtractZipFile(exportedFilePath, null);
// Assert the file was created with desired hierarchy.
Assert.That(Directory.Exists(exportedDirectory));
sanitizedProjName = Sanitization.MakeFriendlyForPath(proj2.Name, "Lift");
exportedProjDir = Path.Combine(exportedDirectory, sanitizedProjName);
Assert.That(Directory.Exists(Path.Combine(exportedProjDir, "audio")));
foreach (var audioFile in roundTripObj.AudioFiles)
{
var path = Path.Combine(exportedProjDir, "audio", audioFile);
Assert.That(File.Exists(path),
$"The file {audioFile} can not be found at this path: {path}");
}
Assert.That(Directory.Exists(Path.Combine(exportedProjDir, "WritingSystems")));
Assert.That(File.Exists(Path.Combine(
exportedProjDir, "WritingSystems", roundTripObj.Language + ".ldml")));
Assert.That(File.Exists(Path.Combine(exportedProjDir, sanitizedProjName + ".lift")));
Directory.Delete(exportedDirectory, true);
// Clean up.
_wordRepo.DeleteAllWords(proj2.Id);
foreach (var project in new List <Project> {
proj1, proj2
})
{
_projRepo.Delete(project.Id);
}
}
