private void parser_SetTotalNumberSteps(object sender,
LiftParser
<PalasoDataObject, LexEntry, LexSense,
LexExampleSentence> .StepsArgs e)
{
_progressState.TotalNumberOfSteps = e.Steps;
}
private void parser_ParsingWarning(
object sender,
LiftParser <PalasoDataObject, LexEntry, LexSense, LexExampleSentence> .ErrorArgs e
)
{
_progressState.ExceptionThatWasEncountered = e.Exception;
}
private void ImportCustomFields(string liftPathname)
{
if (string.IsNullOrEmpty(liftPathname))
{
return;
}
NonUndoableUnitOfWorkHelper.DoSomehow(_cache.ActionHandlerAccessor, () =>
{
string sFilename;
var fMigrationNeeded = Migrator.IsMigrationNeeded(liftPathname);
if (fMigrationNeeded)
{
var sOldVersion = Lift.Validation.Validator.GetLiftVersion(liftPathname);
sFilename = Migrator.MigrateToLatestVersion(liftPathname);
}
else
{
sFilename = liftPathname;
}
var flexImporter = new FlexLiftMerger(_cache, FlexLiftMerger.MergeStyle.MsKeepOnlyNew, true);
var parser = new LiftParser <LiftObject, CmLiftEntry, CmLiftSense, CmLiftExample>(flexImporter);
flexImporter.LiftFile = liftPathname;
var liftRangesFile = liftPathname + "-ranges";
if (File.Exists(liftRangesFile))
{
flexImporter.LoadLiftRanges(liftRangesFile);
}
parser.ReadLiftFile(sFilename);
});
}
static void Main(string[] args)
{
String line;
try
{
//Import the LIFT file and ranges file.
//m_progressDlg.Message = LexTextControls.ksLoadingVariousLists;
//var flexImporter = new MongoLiftMerger(cache, Import, chkTrustModTimes.Checked);
var flexImporter = new LiftToCombine();
var parser = new LiftParser <LiftObject, LiftEntry, LiftSense, LiftExample>(flexImporter);
parser.SetTotalNumberSteps += parser_SetTotalNumberSteps;
parser.SetStepsCompleted += parser_SetStepsCompleted;
parser.SetProgressMessage += parser_SetProgressMessage;
// flexImporter.LiftFile = sTempOrigFile;
//Before imporing the LIFT files ensure the LDML (language definition files) have the correct writing system codes.
//flexImporter.LdmlFilesMigration(sLIFTtempFolder, sFilename, sTempOrigFile + "-ranges");
//Import the Ranges file.
//flexImporter.LoadLiftRanges(sTempOrigFile + "-ranges"); // temporary (?) fix for FWR-3869.
//Import the LIFT data file.
int cEntries = parser.ReadLiftFile("C:\\Users\\FullerM\\Documents\\LiftTest\\LiftTest\\testingdata\\testingdata.xml");
}
catch (Exception e)
{
Console.WriteLine("Exception: " + e.Message);
}
finally
{
Console.WriteLine("Executing finally block.");
}
Console.ReadLine(); //Pause
}
/// <summary>
/// Import the LIFT file into FieldWorks.
/// </summary>
/// <returns>the name of the exported LIFT file if successful, or null if an error occurs.</returns>
/// <remarks>
/// This method is called in a thread, during the export process.
/// </remarks>
private object ImportLiftLexicon(IProgress progressDialog, params object[] parameters)
{
var liftPathname = parameters[0].ToString();
var mergeStyle = (FlexLiftMerger.MergeStyle)parameters[1];
// If we use true while importing changes from repo it will fail to copy any pix/aud files that have changed.
var fTrustModTimes = mergeStyle != FlexLiftMerger.MergeStyle.MsKeepOnlyNew;
if (_progressDlg == null)
{
_progressDlg = progressDialog;
}
progressDialog.Minimum = 0;
progressDialog.Maximum = 100;
progressDialog.Position = 0;
string sLogFile = null;
NonUndoableUnitOfWorkHelper.Do(Cache.ActionHandlerAccessor, () =>
{
string sFilename;
var fMigrationNeeded = Migrator.IsMigrationNeeded(liftPathname);
if (fMigrationNeeded)
{
var sOldVersion = Palaso.Lift.Validation.Validator.GetLiftVersion(liftPathname);
progressDialog.Message = String.Format(ResourceHelper.GetResourceString("kstidLiftVersionMigration"),
sOldVersion, Palaso.Lift.Validation.Validator.LiftVersion);
sFilename = Migrator.MigrateToLatestVersion(liftPathname);
}
else
{
sFilename = liftPathname;
}
progressDialog.Message = ResourceHelper.GetResourceString("kstidLoadingListInfo");
var flexImporter = new FlexLiftMerger(Cache, mergeStyle, fTrustModTimes);
var parser = new LiftParser <LiftObject, CmLiftEntry, CmLiftSense, CmLiftExample>(flexImporter);
parser.SetTotalNumberSteps += ParserSetTotalNumberSteps;
parser.SetStepsCompleted += ParserSetStepsCompleted;
parser.SetProgressMessage += ParserSetProgressMessage;
flexImporter.LiftFile = liftPathname;
flexImporter.LoadLiftRanges(liftPathname + "-ranges");
var cEntries = parser.ReadLiftFile(sFilename);
if (fMigrationNeeded)
{
// Try to move the migrated file to the temp directory, even if a copy of it
// already exists there.
var sTempMigrated = Path.Combine(Path.GetTempPath(),
Path.ChangeExtension(Path.GetFileName(sFilename), "." + Palaso.Lift.Validation.Validator.LiftVersion + ".lift"));
if (File.Exists(sTempMigrated))
{
File.Delete(sTempMigrated);
}
File.Move(sFilename, sTempMigrated);
}
progressDialog.Message = ResourceHelper.GetResourceString("kstidFixingRelationLinks");
flexImporter.ProcessPendingRelations(progressDialog);
sLogFile = flexImporter.DisplayNewListItems(liftPathname, cEntries);
});
return(sLogFile);
}
void parser_SetProgressMessage(object sender, LiftParser <LiftObject, CmLiftEntry, CmLiftSense, CmLiftExample> .MessageArgs e)
{
if (m_progressDlg != null)
{
m_progressDlg.Message = e.Message;
}
}
void parser_SetStepsCompleted(object sender, LiftParser <LiftObject, CmLiftEntry, CmLiftSense, CmLiftExample> .ProgressEventArgs e)
{
if (m_progressDlg != null)
{
int nMin = m_progressDlg.Minimum;
int nMax = m_progressDlg.Maximum;
Debug.Assert(nMin < nMax);
if (nMin >= nMax)
{
nMax = nMin + 1;
}
int n = e.Progress;
if (n < nMin)
{
n = nMin;
}
if (n > nMax)
{
while (n > nMax)
{
n = nMin + (n - nMax);
}
}
m_progressDlg.Position = n;
e.Cancel = m_progressDlg.Canceled;
}
}
void parser_SetTotalNumberSteps(object sender, LiftParser <LiftObject, CmLiftEntry, CmLiftSense, CmLiftExample> .StepsArgs e)
{
if (m_progressDlg != null)
{
m_progressDlg.Minimum = 0;
m_progressDlg.Maximum = e.Steps;
}
}
private void parser_SetStepsCompleted(object sender,
LiftParser
<PalasoDataObject, LexEntry, LexSense,
LexExampleSentence> .ProgressEventArgs e)
{
_progressState.NumberOfStepsCompleted = e.Progress;
e.Cancel = _progressState.Cancel;
}
public void Setup()
{
//_parsingErrors = new List<Exception>();
_doc = new XmlDocument();
//_doc.DocumentElement.SetAttribute("xmlns:flex", "http://fieldworks.sil.org");
_mocks = new Mockery();
_merger = _mocks.NewMock<ILexiconMerger<DummyBase, Dummy, Dummy, Dummy>>();
_parser = new LiftParser<DummyBase, Dummy, Dummy, Dummy>(_merger);
_parsingWarnings = new List<LiftParser<DummyBase, Dummy, Dummy, Dummy>.ErrorArgs>();
_parser.ParsingWarning += OnParsingWarning;
}
void ParserSetProgressMessage(object sender, LiftParser <LiftObject, CmLiftEntry, CmLiftSense, CmLiftExample> .MessageArgs e)
{
_progressDlg.Position = 0;
_progressDlg.Message = e.Message;
}
public void Read(string filePath, MemoryDataMapper <LexEntry> dataMapper)
{
const string status = "Loading entries";
Logger.WriteEvent(status);
_progressState.StatusLabel = status;
using (LexEntryFromLiftBuilder builder = new LexEntryFromLiftBuilder(dataMapper, _semanticDomainsList))
{
builder.AfterEntryRead += new EventHandler <LexEntryFromLiftBuilder.EntryEvent>(OnAfterEntryRead);
builder.ExpectedOptionTraits = _idsOfSingleOptionFields;
LiftParser <PalasoDataObject, LexEntry, LexSense, LexExampleSentence> parser =
new LiftParser <PalasoDataObject, LexEntry, LexSense, LexExampleSentence>(
builder);
parser.SetTotalNumberSteps += parser_SetTotalNumberSteps;
parser.SetStepsCompleted += parser_SetStepsCompleted;
parser.ParsingWarning += parser_ParsingWarning;
try
{
parser.ReadLiftFile(filePath);
if (_progressState.ExceptionThatWasEncountered != null)
{
throw _progressState.ExceptionThatWasEncountered;
}
}
// catch (LiftFormatException)
// {
// throw;
// }
catch (Exception)
{
_progressState.StatusLabel = "Looking for error in file...";
//our parser failed. Hopefully, because of bad lift. Validate it now to
//see if that's the problem.
Validator.CheckLiftWithPossibleThrow(filePath);
//if it got past that, ok, send along the error the parser encountered.
throw;
}
}
}
/// <summary>
/// Import from a LIFT file.
/// </summary>
/// <param name="progressDlg">The progress dialog.</param>
/// <param name="parameters">The parameters: 1) filename</param>
/// <returns></returns>
private object ImportLIFT(IThreadedProgress progressDlg, params object[] parameters)
{
m_progressDlg = progressDlg;
Debug.Assert(parameters.Length == 1);
string sOrigFile = (string)parameters[0];
try
{
// Create a temporary directory %temp%\TempForLIFTImport. Migrate as necessary and import from this
// directory. Directory is left after import is done in case it is needed, but will be deleted next time
// if it exists.
var sLIFTfolder = Path.GetDirectoryName(sOrigFile);
var sLIFTtempFolder = Path.Combine(Path.GetTempPath(), "TempForLIFTImport");
if (Directory.Exists(sLIFTtempFolder) == true)
{
Directory.Delete(sLIFTtempFolder, true);
}
LdmlFileBackup.CopyDirectory(sLIFTfolder, sLIFTtempFolder);
// Older LIFT files had ldml files in root directory. If found, move them to WritingSystem folder.
if (Directory.GetFiles(sLIFTtempFolder, "*.ldml").Length > 0)
{
var sWritingSystems = Path.Combine(sLIFTtempFolder, "WritingSystems");
if (Directory.Exists(sWritingSystems) == false)
{
Directory.CreateDirectory(sWritingSystems);
}
foreach (string filePath in Directory.GetFiles(sLIFTtempFolder, "*.ldml"))
{
string file = Path.GetFileName(filePath);
if (!File.Exists(Path.Combine(sWritingSystems, file)))
{
File.Move(filePath, Path.Combine(sWritingSystems, file));
}
}
}
var sTempOrigFile = Path.Combine(sLIFTtempFolder, sOrigFile.Substring(sLIFTfolder.Length + 1));
string sFilename;
//Do a LIFT Migration to the current version of LIFT if it is needed.
bool fMigrationNeeded = Migrator.IsMigrationNeeded(sTempOrigFile);
if (fMigrationNeeded)
{
string sOldVersion = Validator.GetLiftVersion(sTempOrigFile);
m_progressDlg.Message = String.Format(LexTextControls.ksMigratingLiftFile,
sOldVersion, Validator.LiftVersion);
sFilename = Migrator.MigrateToLatestVersion(sTempOrigFile);
}
else
{
sFilename = sTempOrigFile;
}
//Validate the LIFT file.
if (!Validate(sFilename, sTempOrigFile))
{
return(null);
}
//Import the LIFT file and ranges file.
m_progressDlg.Message = LexTextControls.ksLoadingVariousLists;
var flexImporter = new FlexLiftMerger(m_cache, m_msImport, m_chkTrustModTimes.Checked);
var parser = new LiftParser <LiftObject, CmLiftEntry, CmLiftSense, CmLiftExample>(flexImporter);
parser.SetTotalNumberSteps += parser_SetTotalNumberSteps;
parser.SetStepsCompleted += parser_SetStepsCompleted;
parser.SetProgressMessage += parser_SetProgressMessage;
flexImporter.LiftFile = sTempOrigFile;
//Before imporing the LIFT files ensure the LDML (language definition files) have the correct writing system codes.
flexImporter.LdmlFilesMigration(sLIFTtempFolder, sFilename, sTempOrigFile + "-ranges");
//Import the Ranges file.
flexImporter.LoadLiftRanges(sTempOrigFile + "-ranges"); // temporary (?) fix for FWR-3869.
//Import the LIFT data file.
int cEntries = parser.ReadLiftFile(sFilename);
if (fMigrationNeeded)
{
// Try to move the migrated file to the temp directory, even if a copy of it
// already exists there.
string sTempMigrated = Path.Combine(Path.GetTempPath(),
Path.ChangeExtension(Path.GetFileName(sFilename), "." + Validator.LiftVersion + FwFileExtensions.ksLexiconInterchangeFormat));
if (File.Exists(sTempMigrated))
{
File.Delete(sTempMigrated);
}
File.Move(sFilename, sTempMigrated);
}
flexImporter.ProcessPendingRelations(m_progressDlg);
return(flexImporter.DisplayNewListItems(sOrigFile, cEntries));
}
catch (Exception error)
{
string sMsg = String.Format(LexTextControls.ksLIFTImportProblem,
sOrigFile, error.Message);
try
{
StringBuilder bldr = new StringBuilder();
// leave in English for programmer's sake...
bldr.AppendFormat("Something went wrong while FieldWorks was attempting to import {0}.",
sOrigFile);
bldr.AppendLine();
bldr.AppendLine(error.Message);
bldr.AppendLine();
bldr.AppendLine(error.StackTrace);
if (Thread.CurrentThread.GetApartmentState() == ApartmentState.STA)
{
ClipboardUtils.SetDataObject(bldr.ToString(), true);
}
else
{
progressDlg.SynchronizeInvoke.Invoke(() => ClipboardUtils.SetDataObject(bldr.ToString(), true));
}
SIL.Utils.Logger.WriteEvent(bldr.ToString());
}
catch
{
}
MessageBox.Show(sMsg, LexTextControls.ksProblemImporting,
MessageBoxButtons.OK, MessageBoxIcon.Warning);
return(null);
}
}
void ParserSetStepsCompleted(object sender, LiftParser <LiftObject, CmLiftEntry, CmLiftSense, CmLiftExample> .ProgressEventArgs e)
{
var nMax = _progressDlg.Maximum;
_progressDlg.Position = e.Progress > nMax ? e.Progress % nMax : e.Progress;
}
void parser_SetTotalNumberSteps(object sender, LiftParser<LiftObject, CmLiftEntry, CmLiftSense, CmLiftExample>.StepsArgs e)
{
if (m_progressDlg != null)
{
m_progressDlg.Minimum = 0;
m_progressDlg.Maximum = e.Steps;
}
}
void parser_SetStepsCompleted(object sender, LiftParser<LiftObject, CmLiftEntry, CmLiftSense, CmLiftExample>.ProgressEventArgs e)
{
if (m_progressDlg != null)
{
int nMin = m_progressDlg.Minimum;
int nMax = m_progressDlg.Maximum;
Debug.Assert(nMin < nMax);
if (nMin >= nMax)
nMax = nMin + 1;
int n = e.Progress;
if (n < nMin)
{
n = nMin;
}
if (n > nMax)
{
while (n > nMax)
n = nMin + (n - nMax);
}
m_progressDlg.Position = n;
e.Cancel = m_progressDlg.Canceled;
}
}
/// <summary>
/// Import from a LIFT file.
/// </summary>
/// <param name="progressDlg">The progress dialog.</param>
/// <param name="parameters">The parameters: 1) filename</param>
/// <returns></returns>
private object ImportLIFT(IThreadedProgress progressDlg, params object[] parameters)
{
m_progressDlg = progressDlg;
Debug.Assert(parameters.Length == 1);
string sOrigFile = (string)parameters[0];
try
{
// Create a temporary directory %temp%\TempForLIFTImport. Migrate as necessary and import from this
// directory. Directory is left after import is done in case it is needed, but will be deleted next time
// if it exists.
var sLIFTfolder = Path.GetDirectoryName(sOrigFile);
var sLIFTtempFolder = Path.Combine(Path.GetTempPath(), "TempForLIFTImport");
if (Directory.Exists(sLIFTtempFolder) == true)
Directory.Delete(sLIFTtempFolder, true);
LdmlFileBackup.CopyDirectory(sLIFTfolder, sLIFTtempFolder);
// Older LIFT files had ldml files in root directory. If found, move them to WritingSystem folder.
if (Directory.GetFiles(sLIFTtempFolder, "*.ldml").Length > 0)
{
var sWritingSystems = Path.Combine(sLIFTtempFolder, "WritingSystems");
if (Directory.Exists(sWritingSystems) == false)
Directory.CreateDirectory(sWritingSystems);
foreach (string filePath in Directory.GetFiles(sLIFTtempFolder, "*.ldml"))
{
string file = Path.GetFileName(filePath);
if (!File.Exists(Path.Combine(sWritingSystems, file)))
File.Move(filePath, Path.Combine(sWritingSystems, file));
}
}
var sTempOrigFile = Path.Combine(sLIFTtempFolder, sOrigFile.Substring(sLIFTfolder.Length + 1));
string sFilename;
//Do a LIFT Migration to the current version of LIFT if it is needed.
bool fMigrationNeeded = Migrator.IsMigrationNeeded(sTempOrigFile);
if (fMigrationNeeded)
{
string sOldVersion = Validator.GetLiftVersion(sTempOrigFile);
m_progressDlg.Message = String.Format(LexTextControls.ksMigratingLiftFile,
sOldVersion, Validator.LiftVersion);
sFilename = Migrator.MigrateToLatestVersion(sTempOrigFile);
}
else
{
sFilename = sTempOrigFile;
}
//Validate the LIFT file.
if (!Validate(sFilename, sTempOrigFile))
return null;
//Import the LIFT file and ranges file.
m_progressDlg.Message = LexTextControls.ksLoadingVariousLists;
var flexImporter = new FlexLiftMerger(m_cache, m_msImport, m_chkTrustModTimes.Checked);
var parser = new LiftParser<LiftObject, CmLiftEntry, CmLiftSense, CmLiftExample>(flexImporter);
parser.SetTotalNumberSteps += parser_SetTotalNumberSteps;
parser.SetStepsCompleted += parser_SetStepsCompleted;
parser.SetProgressMessage += parser_SetProgressMessage;
flexImporter.LiftFile = sTempOrigFile;
//Before imporing the LIFT files ensure the LDML (language definition files) have the correct writing system codes.
flexImporter.LdmlFilesMigration(sLIFTtempFolder, sFilename, sTempOrigFile + "-ranges");
//Import the Ranges file.
flexImporter.LoadLiftRanges(sTempOrigFile + "-ranges"); // temporary (?) fix for FWR-3869.
//Import the LIFT data file.
int cEntries = parser.ReadLiftFile(sFilename);
if (fMigrationNeeded)
{
// Try to move the migrated file to the temp directory, even if a copy of it
// already exists there.
string sTempMigrated = Path.Combine(Path.GetTempPath(),
Path.ChangeExtension(Path.GetFileName(sFilename), "." + Validator.LiftVersion + FwFileExtensions.ksLexiconInterchangeFormat));
if (File.Exists(sTempMigrated))
File.Delete(sTempMigrated);
File.Move(sFilename, sTempMigrated);
}
flexImporter.ProcessPendingRelations(m_progressDlg);
return flexImporter.DisplayNewListItems(sOrigFile, cEntries);
}
catch (Exception error)
{
string sMsg = String.Format(LexTextControls.ksLIFTImportProblem,
sOrigFile, error.Message);
try
{
StringBuilder bldr = new StringBuilder();
// leave in English for programmer's sake...
bldr.AppendFormat("Something went wrong while FieldWorks was attempting to import {0}.",
sOrigFile);
bldr.AppendLine();
bldr.AppendLine(error.Message);
bldr.AppendLine();
bldr.AppendLine(error.StackTrace);
if (Thread.CurrentThread.GetApartmentState() == ApartmentState.STA)
ClipboardUtils.SetDataObject(bldr.ToString(), true);
else
progressDlg.SynchronizeInvoke.Invoke(() => ClipboardUtils.SetDataObject(bldr.ToString(), true));
SIL.Utils.Logger.WriteEvent(bldr.ToString());
}
catch
{
}
MessageBox.Show(sMsg, LexTextControls.ksProblemImporting,
MessageBoxButtons.OK, MessageBoxIcon.Warning);
return null;
}
}
void parser_SetProgressMessage(object sender, LiftParser<LiftObject, CmLiftEntry, CmLiftSense, CmLiftExample>.MessageArgs e)
{
if (m_progressDlg != null)
m_progressDlg.Message = e.Message;
}
public void TestLDMLMigration()
{
var projectFolder = Path.GetTempPath();
var testLiftDataSource = Path.Combine(FwDirectoryFinder.SourceDirectory,
"LexText/LexTextControls/LexTextControlsTests/LDML-11723");
var testLiftDataPath = Path.Combine(FwDirectoryFinder.SourceDirectory,
"LexText/LexTextControls/LexTextControlsTests/LDML-11723-test");
var sLiftDataFile = Path.Combine(testLiftDataPath, "LDML-11723.lift");
var sLiftRangesFile = Path.Combine(testLiftDataPath, "LDML-11723.lift-ranges");
var sWSfilesPath = Path.Combine(testLiftDataPath, "WritingSystems");
var enLdml = Path.Combine(sWSfilesPath, "en.ldml");
var sehLdml = Path.Combine(sWSfilesPath, "seh.ldml");
var esLdml = Path.Combine(sWSfilesPath, "es.ldml");
var xkalLdml = Path.Combine(sWSfilesPath, "x-kal.ldml");
var qaaxkalLdml = Path.Combine(sWSfilesPath, "qaa-x-kal.ldml");
var qaaIPAxkalLdml = Path.Combine(sWSfilesPath, "qaa-fonipa-x-kal.ldml");
var qaaPhonemicxkalLdml = Path.Combine(sWSfilesPath, "qaa-fonipa-x-kal-emic.ldml");
LdmlFileBackup.CopyDirectory(testLiftDataSource, testLiftDataPath);
//Make all files writable
// don't want to copy readonly property.
File.SetAttributes(sLiftDataFile, FileAttributes.Normal);
File.SetAttributes(sLiftRangesFile, FileAttributes.Normal);
File.SetAttributes(enLdml, FileAttributes.Normal);
File.SetAttributes(sehLdml, FileAttributes.Normal);
File.SetAttributes(esLdml, FileAttributes.Normal);
File.SetAttributes(xkalLdml, FileAttributes.Normal);
File.SetAttributes(qaaIPAxkalLdml, FileAttributes.Normal);
File.SetAttributes(qaaPhonemicxkalLdml, FileAttributes.Normal);
var flexImporter = new FlexLiftMerger(Cache, FlexLiftMerger.MergeStyle.MsKeepBoth, true);
var parser = new LiftParser<LiftObject, CmLiftEntry, CmLiftSense, CmLiftExample>(flexImporter);
//Mirgrate the LDML files and lang names
flexImporter.LdmlFilesMigration(testLiftDataPath, sLiftDataFile, sLiftRangesFile);
//Verify the migration worked
// Verify the data file exists
Assert.That(File.Exists(sLiftDataFile));
// Verify the ranges file exists
Assert.That(File.Exists(sLiftRangesFile));
// Verify that en.ldml is unchanged.
Assert.That(File.Exists(enLdml));
// Verify that seh.ldml is unchanged.
Assert.That(File.Exists(sehLdml));
// Verify that es.ldml is unchanged.
Assert.That(File.Exists(esLdml));
// Verify that qaa-fonipa-x-kal.ldml is unchanged.
Assert.That(File.Exists(qaaIPAxkalLdml));
// Verify that qaa-fonipa-x-kal-emic.ldml is unchanged.
Assert.That(File.Exists(qaaPhonemicxkalLdml));
// Verify that x-kal.ldml no longer exists
Assert.That(!File.Exists(xkalLdml));
// Verify that x-kal.ldml is renamed to qaa-x-kal and content changed
Assert.That(File.Exists(qaaxkalLdml));
//Verify qaa-x-kal.ldml file has correct changes in it.
VerifyKalabaLdmlFile(qaaxkalLdml);
//Verify LDML 11723.lift file has correct changes in it.
VerifyLiftDataFile(sLiftDataFile);
//Verify LDML 11723.lift file has correct changes in it.
VerifyLiftRangesFile(sLiftRangesFile);
//Delete the files that were converted to the new lang names.
LdmlFileBackup.DeleteDirectory(testLiftDataPath);
}
public async Task <IActionResult> UploadLiftFile(string projectId, [FromForm] FileUpload fileUpload)
{
if (!_permissionService.HasProjectPermission(Permission.ImportExport, HttpContext))
{
return(new ForbidResult());
}
// Ensure project exists
var project = _projectService.GetProject(projectId);
if (project == null)
{
return(new NotFoundObjectResult(projectId));
}
var file = fileUpload.File;
// Ensure file is not empty
if (file.Length == 0)
{
return(new BadRequestObjectResult("Empty File"));
}
// Get path to where we will copy the zip file
var util = new Utilities();
fileUpload.FilePath = util.GenerateFilePath(
FileType.Zip,
false,
"Compressed-Upload-" + string.Format("{0:yyyy-MM-dd_hh-mm-ss-fff}", DateTime.Now),
Path.Combine(projectId, "Import"));
// Copy file data to a new local file
using (var fs = new FileStream(fileUpload.FilePath, FileMode.OpenOrCreate))
{
await file.CopyToAsync(fs);
}
// Make destination for extracted files
var zipDest = Path.GetDirectoryName(fileUpload.FilePath);
Directory.CreateDirectory(zipDest);
if (Directory.Exists(Path.Combine(zipDest, "ExtractedLocation")))
{
return(new BadRequestObjectResult("A file has already been uploaded"));
}
// Extract the zip to new directory
var extractDir = Path.Combine(zipDest, "ExtractedLocation");
Directory.CreateDirectory(extractDir);
ZipFile.ExtractToDirectory(fileUpload.FilePath, extractDir);
// Check number of directories extracted
var directoriesExtracted = Directory.GetDirectories(extractDir);
var extractedDirPath = "";
// If there was one directory, we're good
if (directoriesExtracted.Length == 1)
{
extractedDirPath = directoriesExtracted.FirstOrDefault();
}
// If there were two, and there was a __MACOSX directory, ignore it
else if (directoriesExtracted.Length == 2)
{
var numDirs = 0;
foreach (var dir in directoriesExtracted)
{
if (dir.EndsWith("__MACOSX"))
{
Directory.Delete(dir, true);
}
else // This directory probably matters
{
extractedDirPath = dir;
numDirs++;
}
}
// Both directories seemed important
if (numDirs == 2)
{
return(new BadRequestObjectResult("Your zip file should have one directory"));
}
}
else // There were 0 or more than 2 directories
{
return(new BadRequestObjectResult("Your zip file structure has the wrong number of directories"));
}
// Search for the lift file within the extracted files
var extractedLiftNameArr = Directory.GetFiles(extractedDirPath);
var extractedLiftPath = Array.FindAll(extractedLiftNameArr, x => x.EndsWith(".lift"));
if (extractedLiftPath.Length > 1)
{
return(new BadRequestObjectResult("More than one .lift file detected"));
}
else if (extractedLiftPath.Length == 0)
{
return(new BadRequestObjectResult("No lift files detected"));
}
try
{
// Sets the projectId of our parser to add words to that project
_liftService.SetProject(projectId);
var parser = new LiftParser <LiftObject, LiftEntry, LiftSense, LiftExample>(_liftService);
// Import words from lift file
var resp = parser.ReadLiftFile(extractedLiftPath.FirstOrDefault());
// Add character set to project from ldml file
var proj = _projectService.GetProject(projectId).Result;
_liftService.LdmlImport(
Path.Combine(extractedDirPath, "WritingSystems"), proj.VernacularWritingSystem);
return(new ObjectResult(resp));
}
// If anything wrong happened, it's probably something wrong with the file itself
catch (Exception)
{
return(new UnsupportedMediaTypeResult());
}
}
void ParserSetTotalNumberSteps(object sender, LiftParser <LiftObject, CmLiftEntry, CmLiftSense, CmLiftExample> .StepsArgs e)
{
_progressDlg.Maximum = e.Steps;
_progressDlg.Position = 0;
}
private static void parser_SetProgressMessage(object sender, LiftParser <LiftObject, LiftEntry, LiftSense, LiftExample> .MessageArgs e)
{
//throw new NotImplementedException();
}
/// <summary>
/// Import the LIFT file into FieldWorks.
/// </summary>
/// <returns>the name of the exported LIFT file if successful, or null if an error occurs.</returns>
/// <remarks>
/// This method is called in a thread, during the export process.
/// </remarks>
private object ImportLiftLexicon(IProgress progressDialog, params object[] parameters)
{
var liftPathname = parameters[0].ToString();
var mergeStyle = (FlexLiftMerger.MergeStyle)parameters[1];
// If we use true while importing changes from repo it will fail to copy any pix/aud files that have changed.
var fTrustModTimes = mergeStyle != FlexLiftMerger.MergeStyle.MsKeepOnlyNew;
if (_progressDlg == null)
_progressDlg = progressDialog;
progressDialog.Minimum = 0;
progressDialog.Maximum = 100;
progressDialog.Position = 0;
string sLogFile = null;
if (File.Exists(LiftNotesPath))
{
using (var reader = new StreamReader(LiftNotesPath, Encoding.UTF8))
using (var writer = new StreamWriter(FlexNotesPath, false, Encoding.UTF8))
{
ConvertLiftNotesToFlex(reader, writer, Path.GetFileName(_liftPathname));
}
}
NonUndoableUnitOfWorkHelper.Do(Cache.ActionHandlerAccessor, () =>
{
string sFilename;
var fMigrationNeeded = Migrator.IsMigrationNeeded(liftPathname);
if (fMigrationNeeded)
{
var sOldVersion = Palaso.Lift.Validation.Validator.GetLiftVersion(liftPathname);
progressDialog.Message = String.Format(ResourceHelper.GetResourceString("kstidLiftVersionMigration"),
sOldVersion, Palaso.Lift.Validation.Validator.LiftVersion);
sFilename = Migrator.MigrateToLatestVersion(liftPathname);
}
else
{
sFilename = liftPathname;
}
progressDialog.Message = ResourceHelper.GetResourceString("kstidLoadingListInfo");
var flexImporter = new FlexLiftMerger(Cache, mergeStyle, fTrustModTimes);
var parser = new LiftParser<LiftObject, CmLiftEntry, CmLiftSense, CmLiftExample>(flexImporter);
parser.SetTotalNumberSteps += ParserSetTotalNumberSteps;
parser.SetStepsCompleted += ParserSetStepsCompleted;
parser.SetProgressMessage += ParserSetProgressMessage;
flexImporter.LiftFile = liftPathname;
flexImporter.LoadLiftRanges(liftPathname + "-ranges");
var cEntries = parser.ReadLiftFile(sFilename);
if (fMigrationNeeded)
{
// Try to move the migrated file to the temp directory, even if a copy of it
// already exists there.
var sTempMigrated = Path.Combine(Path.GetTempPath(),
Path.ChangeExtension(Path.GetFileName(sFilename), "." + Palaso.Lift.Validation.Validator.LiftVersion + ".lift"));
if (File.Exists(sTempMigrated))
File.Delete(sTempMigrated);
File.Move(sFilename, sTempMigrated);
}
progressDialog.Message = ResourceHelper.GetResourceString("kstidFixingRelationLinks");
flexImporter.ProcessPendingRelations(progressDialog);
sLogFile = flexImporter.DisplayNewListItems(liftPathname, cEntries);
});
return sLogFile;
}
private static void parser_SetTotalNumberSteps(object sender, LiftParser <LiftObject, LiftEntry, LiftSense, LiftExample> .StepsArgs e)
{
//throw new NotImplementedException();
}
private static void parser_SetStepsCompleted(object sender, LiftParser <LiftObject, LiftEntry, LiftSense, LiftExample> .ProgressEventArgs e)
{
// throw new NotImplementedException();
}
void ParserSetTotalNumberSteps(object sender, LiftParser<LiftObject, CmLiftEntry, CmLiftSense, CmLiftExample>.StepsArgs e)
{
_progressDlg.Maximum = e.Steps;
_progressDlg.Position = 0;
}
[RequestSizeLimit(250_000_000)] // 250MB.
public async Task <IActionResult> UploadLiftFile(string projectId, [FromForm] FileUpload fileUpload)
{
if (!await _permissionService.HasProjectPermission(HttpContext, Permission.ImportExport))
{
return(Forbid());
}
// Sanitize projectId
if (!Sanitization.SanitizeId(projectId))
{
return(new UnsupportedMediaTypeResult());
}
// Ensure Lift file has not already been imported.
if (!await _projRepo.CanImportLift(projectId))
{
return(BadRequest("A Lift file has already been uploaded."));
}
var liftStoragePath = FileStorage.GenerateLiftImportDirPath(projectId);
// Clear out any files left by a failed import
RobustIO.DeleteDirectoryAndContents(liftStoragePath);
var file = fileUpload.File;
if (file is null)
{
return(BadRequest("Null File"));
}
// Ensure file is not empty
if (file.Length == 0)
{
return(BadRequest("Empty File"));
}
// Copy zip file data to a new temporary file
fileUpload.FilePath = Path.GetTempFileName();
await using (var fs = new FileStream(fileUpload.FilePath, FileMode.OpenOrCreate))
{
await file.CopyToAsync(fs);
}
// Make temporary destination for extracted files
var extractDir = FileOperations.GetRandomTempDir();
// Extract the zip to new created directory.
FileOperations.ExtractZipFile(fileUpload.FilePath, extractDir, true);
// Check number of directories extracted
var directoriesExtracted = Directory.GetDirectories(extractDir);
var extractedDirPath = "";
switch (directoriesExtracted.Length)
{
// If there was one directory, we're good
case 1:
{
extractedDirPath = directoriesExtracted.First();
break;
}
// If there were two, and there was a __MACOSX directory, ignore it
case 2:
{
var numDirs = 0;
foreach (var dir in directoriesExtracted)
{
if (dir.EndsWith("__MACOSX"))
{
Directory.Delete(dir, true);
}
else // This directory probably matters
{
extractedDirPath = dir;
numDirs++;
}
}
// Both directories seemed important
if (numDirs == 2)
{
return(BadRequest("Your zip file should have one directory."));
}
break;
}
// There were 0 or more than 2 directories
default:
{
return(BadRequest(
"Your zip file structure has the wrong number of directories."));
}
}
// Copy the extracted contents into the persistent storage location for the project.
FileOperations.CopyDirectory(extractedDirPath, liftStoragePath);
Directory.Delete(extractDir, true);
// Search for the lift file within the extracted files
var extractedLiftNameArr = Directory.GetFiles(liftStoragePath);
var extractedLiftPath = Array.FindAll(extractedLiftNameArr, x => x.EndsWith(".lift"));
if (extractedLiftPath.Length > 1)
{
return(BadRequest("More than one .lift file detected."));
}
if (extractedLiftPath.Length == 0)
{
return(BadRequest("No lift files detected."));
}
int liftParseResult;
// Sets the projectId of our parser to add words to that project
var liftMerger = _liftService.GetLiftImporterExporter(projectId, _wordRepo);
try
{
// Add character set to project from ldml file
var proj = await _projRepo.GetProject(projectId);
if (proj is null)
{
return(NotFound(projectId));
}
_liftService.LdmlImport(
Path.Combine(liftStoragePath, "WritingSystems"),
proj.VernacularWritingSystem.Bcp47, _projRepo, proj);
var parser = new LiftParser <LiftObject, LiftEntry, LiftSense, LiftExample>(liftMerger);
// Import words from lift file
liftParseResult = parser.ReadLiftFile(extractedLiftPath.FirstOrDefault());
await liftMerger.SaveImportEntries();
}
catch (Exception e)
{
_logger.LogError(e, $"Error importing lift file {fileUpload.Name} into project {projectId}.");
return(BadRequest("Error processing the lift data. Contact support for help."));
}
// Store that we have imported Lift data already for this project to signal the frontend
// not to attempt to import again.
var project = await _projRepo.GetProject(projectId);
if (project is null)
{
return(NotFound(projectId));
}
project.LiftImported = true;
await _projRepo.Update(projectId, project);
return(Ok(liftParseResult));
}
void ParserSetProgressMessage(object sender, LiftParser<LiftObject, CmLiftEntry, CmLiftSense, CmLiftExample>.MessageArgs e)
{
_progressDlg.Position = 0;
_progressDlg.Message = e.Message;
}
void ParserSetStepsCompleted(object sender, LiftParser<LiftObject, CmLiftEntry, CmLiftSense, CmLiftExample>.ProgressEventArgs e)
{
var nMax = _progressDlg.Maximum;
_progressDlg.Position = e.Progress > nMax ? e.Progress % nMax : e.Progress;
}
void OnParsingWarning(object sender, LiftParser<DummyBase, Dummy, Dummy, Dummy>.ErrorArgs e)
{
_parsingWarnings.Add(e);
}
private string TryImport(string sOrigFile, string sOrigRangesFile, FlexLiftMerger.MergeStyle mergeStyle, int expectedCount, bool trustModificationTimes = true)
{
string logfile = null;
IProgress progressDlg = new DummyProgressDlg();
var fMigrationNeeded = Migrator.IsMigrationNeeded(sOrigFile);
var sFilename = fMigrationNeeded
? Migrator.MigrateToLatestVersion(sOrigFile)
: sOrigFile;
var flexImporter = new FlexLiftMerger(Cache, mergeStyle, trustModificationTimes);
var parser = new LiftParser<LiftObject, CmLiftEntry, CmLiftSense, CmLiftExample>(flexImporter);
flexImporter.LiftFile = sOrigFile;
//The following are the calls to import the Ranges file and then the Data file.
if (!String.IsNullOrEmpty(sOrigRangesFile))
flexImporter.LoadLiftRanges(sOrigRangesFile);
var cEntries = parser.ReadLiftFile(sFilename);
Assert.AreEqual(expectedCount, cEntries);
if (fMigrationNeeded)
File.Delete(sFilename);
flexImporter.ProcessPendingRelations(progressDlg);
logfile = flexImporter.DisplayNewListItems(sOrigFile, cEntries);
return logfile;
}
