public IActionResult DownloadAudioFile(string projectId, string wordId, string fileName)
{
// if we require authorization and authentication for audio files, the frontend cannot just use the api
// endpoint as the src
//if (!_permissionService.IsProjectAuthorized("1", HttpContext))
//{
// return Forbid();
//}
// Sanitize user input
if (!Sanitization.SanitizeId(projectId) || !Sanitization.SanitizeId(wordId) ||
!Sanitization.SanitizeFileName(fileName))
{
return(new UnsupportedMediaTypeResult());
}
var filePath = FileStorage.GenerateAudioFilePath(projectId, fileName);
var file = System.IO.File.OpenRead(filePath);
if (file is null)
{
return(BadRequest("The file does not exist."));
}
return(File(file, "application/octet-stream"));
}
public void TestMakeFriendlyForPathFallback()
{
const string fallback = "Lift";
const string nonEmpty = "qwerty";
Assert.That(Sanitization.MakeFriendlyForPath(""), Is.EqualTo(""));
Assert.That(Sanitization.MakeFriendlyForPath("", fallback), Is.EqualTo(fallback));
Assert.That(Sanitization.MakeFriendlyForPath(nonEmpty, fallback), Is.EqualTo(nonEmpty));
}
// These internal methods are extracted for unit testing
internal async Task <IActionResult> ExportLiftFile(string projectId, string userId)
{
if (!await _permissionService.HasProjectPermission(HttpContext, Permission.ImportExport))
{
return(Forbid());
}
// Sanitize projectId
if (!Sanitization.SanitizeId(projectId))
{
return(new UnsupportedMediaTypeResult());
}
// Ensure project exists
var proj = await _projRepo.GetProject(projectId);
if (proj is null)
{
return(NotFound(projectId));
}
// Check if another export started
if (_liftService.IsExportInProgress(userId))
{
return(Conflict());
}
// Store in-progress status for the export
_liftService.SetExportInProgress(userId, true);
try
{
// Ensure project has words
var words = await _wordRepo.GetAllWords(projectId);
if (words.Count == 0)
{
_liftService.SetExportInProgress(userId, false);
return(BadRequest("No words to export."));
}
// Export the data to a zip, read into memory, and delete zip
var exportedFilepath = await CreateLiftExport(projectId);
// Store the temporary path to the exported file for user to download later.
_liftService.StoreExport(userId, exportedFilepath);
await _notifyService.Clients.All.SendAsync("DownloadReady", userId);
return(Ok(projectId));
}
catch
{
_liftService.SetExportInProgress(userId, false);
throw;
}
}
public async Task TestDeletedWordsExportToLift()
{
var word = Util.RandomWord(_projId);
var secondWord = Util.RandomWord(_projId);
var wordToDelete = Util.RandomWord(_projId);
var wordToUpdate = _wordRepo.Create(word).Result;
wordToDelete = _wordRepo.Create(wordToDelete).Result;
// Create untouched word.
_ = _wordRepo.Create(secondWord).Result;
word.Id = "";
word.Vernacular = "updated";
await _wordService.Update(_projId, wordToUpdate.Id, word);
await _wordService.DeleteFrontierWord(_projId, wordToDelete.Id);
_liftController.ExportLiftFile(_projId, UserId).Wait();
var result = (FileStreamResult)_liftController.DownloadLiftFile(_projId, UserId).Result;
Assert.NotNull(result);
// Read contents.
byte[] contents;
await using (var fileStream = result.FileStream)
{
contents = ReadAllBytes(fileStream);
}
// Write LiftFile contents to a temporary directory.
var extractedExportDir = ExtractZipFileContents(contents);
var sanitizedProjName = Sanitization.MakeFriendlyForPath(ProjName, "Lift");
var exportPath = Path.Combine(
extractedExportDir, sanitizedProjName, sanitizedProjName + ".lift");
var text = await File.ReadAllTextAsync(exportPath, Encoding.UTF8);
// TODO: Add SIL or other XML assertion library and verify with xpath that the correct entries are
// kept vs deleted
// Make sure we exported 2 live and one dead entry
Assert.That(Regex.Matches(text, "<entry").Count, Is.EqualTo(3));
// There is only one deleted word
Assert.That(text.IndexOf("dateDeleted"), Is.EqualTo(text.LastIndexOf("dateDeleted")));
// Delete the export
await _liftController.DeleteLiftFile(UserId);
var notFoundResult = _liftController.DownloadLiftFile(_projId, UserId).Result;
Assert.That(notFoundResult is NotFoundObjectResult);
}
public async Task <IActionResult> UploadAudioFile(string projectId, string wordId,
[FromForm] FileUpload fileUpload)
{
if (!await _permissionService.HasProjectPermission(HttpContext, Permission.WordEntry))
{
return(Forbid());
}
// sanitize user input
if (!Sanitization.SanitizeId(projectId) || !Sanitization.SanitizeId(wordId))
{
return(new UnsupportedMediaTypeResult());
}
var file = fileUpload.File;
if (file is null)
{
return(BadRequest("Null File"));
}
// Ensure file is not empty
if (file.Length == 0)
{
return(BadRequest("Empty File"));
}
// This path should be unique even though it is only based on the Word ID because currently, a new
// Word is created each time an audio file is uploaded.
fileUpload.FilePath = FileStorage.GenerateAudioFilePathForWord(projectId, wordId);
// Copy the file data to a new local file
await using (var fs = new FileStream(fileUpload.FilePath, FileMode.Create))
{
await file.CopyToAsync(fs);
}
// Add the relative path to the audio field
var word = await _wordRepo.GetWord(projectId, wordId);
if (word is null)
{
return(NotFound(wordId));
}
word.Audio.Add(Path.GetFileName(fileUpload.FilePath));
// Update the word with new audio file
await _wordService.Update(projectId, wordId, word);
return(Ok(word.Id));
}
public async Task <IActionResult> CanUploadLift(string projectId)
{
if (!await _permissionService.HasProjectPermission(HttpContext, Permission.ImportExport))
{
return(Forbid());
}
// Sanitize user input
if (!Sanitization.SanitizeId(projectId))
{
return(new UnsupportedMediaTypeResult());
}
return(Ok(await _projRepo.CanImportLift(projectId)));
}
public async Task <IActionResult> DeleteAudioFile(string projectId, string wordId, string fileName)
{
if (!await _permissionService.HasProjectPermission(HttpContext, Permission.WordEntry))
{
return(Forbid());
}
// sanitize user input
if (!Sanitization.SanitizeId(projectId) || !Sanitization.SanitizeId(wordId))
{
return(new UnsupportedMediaTypeResult());
}
var newWord = await _wordService.Delete(projectId, wordId, fileName);
if (newWord is not null)
{
return(Ok(newWord.Id));
}
return(NotFound("The project was found, but the word audio was not deleted"));
}
public void TestInvalidFileNames(string fileName)
{
Assert.False(Sanitization.SanitizeFileName(fileName));
}
public void TestValidFileNames(string fileName)
{
Assert.That(Sanitization.SanitizeFileName(fileName));
}
public void TestInvalidIds(string id)
{
Assert.False(Sanitization.SanitizeId(id));
}
public void TestValidIds(string id)
{
Assert.That(Sanitization.SanitizeId(id));
}
public void TestMakeFriendlyForPath(List <string> nameName)
{
Assert.That(Sanitization.MakeFriendlyForPath(nameName[0]), Is.EqualTo(nameName[1]));
}
/// <summary> Exports information from a project to a lift package zip </summary>
/// <exception cref="MissingProjectException"> If Project does not exist. </exception>
/// <returns> Path to compressed zip file containing export. </returns>
public async Task <string> LiftExport(
string projectId, IWordRepository wordRepo, IProjectRepository projRepo)
{
// Validate project exists.
var proj = await projRepo.GetProject(projectId);
if (proj is null)
{
throw new MissingProjectException($"Project does not exist: {projectId}");
}
var vernacularBcp47 = proj.VernacularWritingSystem.Bcp47;
// Generate the zip dir.
var exportDir = FileStorage.GenerateLiftExportDirPath(projectId);
var liftExportDir = Path.Combine(exportDir, "LiftExport");
if (Directory.Exists(liftExportDir))
{
Directory.Delete(liftExportDir, true);
}
var projNameAsPath = Sanitization.MakeFriendlyForPath(proj.Name, "Lift");
var zipDir = Path.Combine(liftExportDir, projNameAsPath);
Directory.CreateDirectory(zipDir);
// Add audio dir inside zip dir.
var audioDir = Path.Combine(zipDir, "audio");
Directory.CreateDirectory(audioDir);
var liftPath = Path.Combine(zipDir, projNameAsPath + ".lift");
// noBOM will work with PrinceXML
using var liftWriter = new CombineLiftWriter(liftPath, ByteOrderStyle.BOM);
var rangesDest = Path.Combine(zipDir, projNameAsPath + ".lift-ranges");
// Write header of lift document.
var header =
$@"
<ranges>
<range id = ""semantic-domain-ddp4"" href = ""{rangesDest}""/>
</ranges>
<fields>
<field tag = ""Plural"">
<form lang = ""en""><text></text></form>
<form lang = ""qaa-x-spec""><text> Class = LexEntry; Type = String; WsSelector = kwsVern </text></form>
</field>
</fields>
";
liftWriter.WriteHeader(header);
// Write out every word with all of its information
var allWords = await wordRepo.GetAllWords(projectId);
var frontier = await wordRepo.GetFrontier(projectId);
var activeWords = frontier.Where(
x => x.Senses.Any(s => s.Accessibility == State.Active)).ToList();
// All words in the frontier with any senses are considered current.
// The Combine does not import senseless entries and the interface is supposed to prevent creating them.
// So the the words found in allWords with no matching guid in activeWords are exported as 'deleted'.
var deletedWords = allWords.Where(
x => activeWords.All(w => w.Guid != x.Guid)).DistinctBy(w => w.Guid).ToList();
foreach (var wordEntry in activeWords)
{
var entry = new LexEntry(MakeSafeXmlAttribute(wordEntry.Vernacular), wordEntry.Guid);
if (DateTime.TryParse(wordEntry.Created, out var createdTime))
{
entry.CreationTime = createdTime;
}
if (DateTime.TryParse(wordEntry.Modified, out var modifiedTime))
{
entry.ModificationTime = modifiedTime;
}
AddNote(entry, wordEntry);
AddVern(entry, wordEntry, vernacularBcp47);
AddSenses(entry, wordEntry);
AddAudio(entry, wordEntry, audioDir, projectId);
liftWriter.Add(entry);
}
foreach (var wordEntry in deletedWords)
{
var entry = new LexEntry(MakeSafeXmlAttribute(wordEntry.Vernacular), wordEntry.Guid);
AddNote(entry, wordEntry);
AddVern(entry, wordEntry, vernacularBcp47);
AddSenses(entry, wordEntry);
AddAudio(entry, wordEntry, audioDir, projectId);
liftWriter.AddDeletedEntry(entry);
}
liftWriter.End();
// Export semantic domains to lift-ranges
var extractedPathToImport = FileStorage.GenerateImportExtractedLocationDirPath(projectId, false);
string?firstImportDir = null;
if (Directory.Exists(extractedPathToImport))
{
// TODO: Should an error be raised if this returns null?
firstImportDir = Directory.GetDirectories(extractedPathToImport).Select(
Path.GetFileName).ToList().Single();
}
var importLiftDir = firstImportDir ?? "";
var rangesSrc = Path.Combine(extractedPathToImport, importLiftDir, $"{importLiftDir}.lift-ranges");
// If there are no new semantic domains, and the old lift-ranges file is still around, just copy it
if (proj.SemanticDomains.Count == 0 && File.Exists(rangesSrc))
{
File.Copy(rangesSrc, rangesDest, true);
}
else // Make a new lift-ranges file
{
using var liftRangesWriter = XmlWriter.Create(rangesDest, new XmlWriterSettings
{
Indent = true,
NewLineOnAttributes = true,
Async = true
});
await liftRangesWriter.WriteStartDocumentAsync();
liftRangesWriter.WriteStartElement("lift-ranges");
liftRangesWriter.WriteStartElement("range");
liftRangesWriter.WriteAttributeString("id", "semantic-domain-ddp4");
// Pull from resources file with all English semantic domains
var assembly = typeof(LiftService).GetTypeInfo().Assembly;
const string semDomListFile = "BackendFramework.Data.sdList.txt";
var resource = assembly.GetManifestResourceStream(semDomListFile);
if (resource is null)
{
throw new Exception($"Unable to load semantic domain list: {semDomListFile}");
}
string sdList;
using (var reader = new StreamReader(resource, Encoding.UTF8))
{
sdList = await reader.ReadToEndAsync();
}
var sdLines = sdList.Split(Environment.NewLine);
foreach (var line in sdLines)
{
if (line != "")
{
var items = line.Split("`");
WriteRangeElement(liftRangesWriter, items[0], items[1], items[2], items[3]);
}
}
// Pull from new semantic domains in project
foreach (var sd in proj.SemanticDomains)
{
WriteRangeElement(liftRangesWriter, sd.Id, Guid.NewGuid().ToString(), sd.Name, sd.Description);
}
await liftRangesWriter.WriteEndElementAsync(); //end semantic-domain-ddp4 range
await liftRangesWriter.WriteEndElementAsync(); //end lift-ranges
await liftRangesWriter.WriteEndDocumentAsync();
await liftRangesWriter.FlushAsync();
liftRangesWriter.Close();
}
// Export character set to ldml.
var ldmlDir = Path.Combine(zipDir, "WritingSystems");
Directory.CreateDirectory(ldmlDir);
if (vernacularBcp47 != "")
{
var validChars = proj.ValidCharacters;
LdmlExport(ldmlDir, vernacularBcp47, validChars);
}
// Compress everything.
var destinationFileName = Path.Combine(exportDir,
Path.Combine($"LiftExportCompressed-{proj.Id}_{DateTime.Now:yyyy-MM-dd_hh-mm-ss}.zip"));
var zipParentDir = Path.GetDirectoryName(zipDir);
if (zipParentDir is null)
{
throw new Exception($"Unable to find parent dir of: {zipDir}");
}
ZipFile.CreateFromDirectory(zipParentDir, destinationFileName);
// Clean up the temporary folder structure that was compressed.
Directory.Delete(liftExportDir, true);
return(destinationFileName);
}
public void TestRoundtrip(RoundTripObj roundTripObj)
{
// This test assumes you have the starting .zip (Filename) included in your project files.
var pathToStartZip = Path.Combine(Util.AssetsDir, roundTripObj.Filename);
Assert.IsTrue(File.Exists(pathToStartZip));
// Roundtrip Part 1
// Init the project the .zip info is added to.
var proj1 = Util.RandomProject();
proj1.VernacularWritingSystem.Bcp47 = roundTripObj.Language;
proj1 = _projRepo.Create(proj1).Result;
// Upload the zip file.
// Generate api parameter with filestream.
using (var stream = File.OpenRead(pathToStartZip))
{
var fileUpload = InitFile(stream, roundTripObj.Filename);
// Make api call.
var result = _liftController.UploadLiftFile(proj1 !.Id, fileUpload).Result;
Assert.That(result is OkObjectResult);
}
proj1 = _projRepo.GetProject(proj1.Id).Result;
if (proj1 is null)
{
Assert.Fail();
return;
}
Assert.That(proj1.LiftImported);
var allWords = _wordRepo.GetAllWords(proj1.Id).Result;
Assert.AreEqual(allWords.Count, roundTripObj.NumOfWords);
// We are currently only testing guids on the single-entry data sets.
if (roundTripObj.EntryGuid != "" && allWords.Count == 1)
{
Assert.AreEqual(allWords[0].Guid.ToString(), roundTripObj.EntryGuid);
if (roundTripObj.SenseGuid != "")
{
Assert.AreEqual(allWords[0].Senses[0].Guid.ToString(), roundTripObj.SenseGuid);
}
}
// Export.
var exportedFilePath = _liftController.CreateLiftExport(proj1.Id).Result;
var exportedDirectory = FileOperations.ExtractZipFile(exportedFilePath, null, false);
// Assert the file was created with desired hierarchy.
Assert.That(Directory.Exists(exportedDirectory));
var sanitizedProjName = Sanitization.MakeFriendlyForPath(proj1.Name, "Lift");
var exportedProjDir = Path.Combine(exportedDirectory, sanitizedProjName);
Assert.That(Directory.Exists(Path.Combine(exportedProjDir, "audio")));
foreach (var audioFile in roundTripObj.AudioFiles)
{
Assert.That(File.Exists(Path.Combine(exportedProjDir, "audio", audioFile)));
}
Assert.That(Directory.Exists(Path.Combine(exportedProjDir, "WritingSystems")));
Assert.That(File.Exists(Path.Combine(
exportedProjDir, "WritingSystems", roundTripObj.Language + ".ldml")));
Assert.That(File.Exists(Path.Combine(exportedProjDir, sanitizedProjName + ".lift")));
Directory.Delete(exportedDirectory, true);
// Clean up.
_wordRepo.DeleteAllWords(proj1.Id);
// Roundtrip Part 2
// Init the project the .zip info is added to.
var proj2 = Util.RandomProject();
proj2.VernacularWritingSystem.Bcp47 = roundTripObj.Language;
proj2 = _projRepo.Create(proj2).Result;
// Upload the exported words again.
// Generate api parameter with filestream.
using (var fstream = File.OpenRead(exportedFilePath))
{
var fileUpload = InitFile(fstream, roundTripObj.Filename);
// Make api call.
var result2 = _liftController.UploadLiftFile(proj2 !.Id, fileUpload).Result;
Assert.That(result2 is OkObjectResult);
}
proj2 = _projRepo.GetProject(proj2.Id).Result;
if (proj2 is null)
{
Assert.Fail();
return;
}
// Clean up zip file.
File.Delete(exportedFilePath);
allWords = _wordRepo.GetAllWords(proj2.Id).Result;
Assert.That(allWords, Has.Count.EqualTo(roundTripObj.NumOfWords));
// We are currently only testing guids on the single-entry data sets.
if (roundTripObj.EntryGuid != "" && allWords.Count == 1)
{
Assert.AreEqual(allWords[0].Guid.ToString(), roundTripObj.EntryGuid);
if (roundTripObj.SenseGuid != "")
{
Assert.AreEqual(allWords[0].Senses[0].Guid.ToString(), roundTripObj.SenseGuid);
}
}
// Export.
exportedFilePath = _liftController.CreateLiftExport(proj2.Id).Result;
exportedDirectory = FileOperations.ExtractZipFile(exportedFilePath, null);
// Assert the file was created with desired hierarchy.
Assert.That(Directory.Exists(exportedDirectory));
sanitizedProjName = Sanitization.MakeFriendlyForPath(proj2.Name, "Lift");
exportedProjDir = Path.Combine(exportedDirectory, sanitizedProjName);
Assert.That(Directory.Exists(Path.Combine(exportedProjDir, "audio")));
foreach (var audioFile in roundTripObj.AudioFiles)
{
var path = Path.Combine(exportedProjDir, "audio", audioFile);
Assert.That(File.Exists(path),
$"The file {audioFile} can not be found at this path: {path}");
}
Assert.That(Directory.Exists(Path.Combine(exportedProjDir, "WritingSystems")));
Assert.That(File.Exists(Path.Combine(
exportedProjDir, "WritingSystems", roundTripObj.Language + ".ldml")));
Assert.That(File.Exists(Path.Combine(exportedProjDir, sanitizedProjName + ".lift")));
Directory.Delete(exportedDirectory, true);
// Clean up.
_wordRepo.DeleteAllWords(proj2.Id);
foreach (var project in new List <Project> {
proj1, proj2
})
{
_projRepo.Delete(project.Id);
}
}
[RequestSizeLimit(250_000_000)] // 250MB.
public async Task <IActionResult> UploadLiftFile(string projectId, [FromForm] FileUpload fileUpload)
{
if (!await _permissionService.HasProjectPermission(HttpContext, Permission.ImportExport))
{
return(Forbid());
}
// Sanitize projectId
if (!Sanitization.SanitizeId(projectId))
{
return(new UnsupportedMediaTypeResult());
}
// Ensure Lift file has not already been imported.
if (!await _projRepo.CanImportLift(projectId))
{
return(BadRequest("A Lift file has already been uploaded."));
}
var liftStoragePath = FileStorage.GenerateLiftImportDirPath(projectId);
// Clear out any files left by a failed import
RobustIO.DeleteDirectoryAndContents(liftStoragePath);
var file = fileUpload.File;
if (file is null)
{
return(BadRequest("Null File"));
}
// Ensure file is not empty
if (file.Length == 0)
{
return(BadRequest("Empty File"));
}
// Copy zip file data to a new temporary file
fileUpload.FilePath = Path.GetTempFileName();
await using (var fs = new FileStream(fileUpload.FilePath, FileMode.OpenOrCreate))
{
await file.CopyToAsync(fs);
}
// Make temporary destination for extracted files
var extractDir = FileOperations.GetRandomTempDir();
// Extract the zip to new created directory.
FileOperations.ExtractZipFile(fileUpload.FilePath, extractDir, true);
// Check number of directories extracted
var directoriesExtracted = Directory.GetDirectories(extractDir);
var extractedDirPath = "";
switch (directoriesExtracted.Length)
{
// If there was one directory, we're good
case 1:
{
extractedDirPath = directoriesExtracted.First();
break;
}
// If there were two, and there was a __MACOSX directory, ignore it
case 2:
{
var numDirs = 0;
foreach (var dir in directoriesExtracted)
{
if (dir.EndsWith("__MACOSX"))
{
Directory.Delete(dir, true);
}
else // This directory probably matters
{
extractedDirPath = dir;
numDirs++;
}
}
// Both directories seemed important
if (numDirs == 2)
{
return(BadRequest("Your zip file should have one directory."));
}
break;
}
// There were 0 or more than 2 directories
default:
{
return(BadRequest(
"Your zip file structure has the wrong number of directories."));
}
}
// Copy the extracted contents into the persistent storage location for the project.
FileOperations.CopyDirectory(extractedDirPath, liftStoragePath);
Directory.Delete(extractDir, true);
// Search for the lift file within the extracted files
var extractedLiftNameArr = Directory.GetFiles(liftStoragePath);
var extractedLiftPath = Array.FindAll(extractedLiftNameArr, x => x.EndsWith(".lift"));
if (extractedLiftPath.Length > 1)
{
return(BadRequest("More than one .lift file detected."));
}
if (extractedLiftPath.Length == 0)
{
return(BadRequest("No lift files detected."));
}
int liftParseResult;
// Sets the projectId of our parser to add words to that project
var liftMerger = _liftService.GetLiftImporterExporter(projectId, _wordRepo);
try
{
// Add character set to project from ldml file
var proj = await _projRepo.GetProject(projectId);
if (proj is null)
{
return(NotFound(projectId));
}
_liftService.LdmlImport(
Path.Combine(liftStoragePath, "WritingSystems"),
proj.VernacularWritingSystem.Bcp47, _projRepo, proj);
var parser = new LiftParser <LiftObject, LiftEntry, LiftSense, LiftExample>(liftMerger);
// Import words from lift file
liftParseResult = parser.ReadLiftFile(extractedLiftPath.FirstOrDefault());
await liftMerger.SaveImportEntries();
}
catch (Exception e)
{
_logger.LogError(e, $"Error importing lift file {fileUpload.Name} into project {projectId}.");
return(BadRequest("Error processing the lift data. Contact support for help."));
}
// Store that we have imported Lift data already for this project to signal the frontend
// not to attempt to import again.
var project = await _projRepo.GetProject(projectId);
if (project is null)
{
return(NotFound(projectId));
}
project.LiftImported = true;
await _projRepo.Update(projectId, project);
return(Ok(liftParseResult));
}
