public void NotFwDataFileForBreakupShouldThrow()
{
using (var tempFile = new TempFile(""))
{
var pathname = tempFile.Path;
Assert.Throws <ApplicationException>(() => FLExProjectSplitter.PushHumptyOffTheWall(new NullProgress(), pathname));
}
}
/// <summary>
/// Allow the client to do something right after a merge, but before the merge is committed.
/// </summary>
/// <remarks>This method is not be called at all, if there was no merging.</remarks>
public void PrepareForPostMergeCommit(IProgress progress)
{
RestoreProjectFile(progress);
progress.WriteMessage("Checking project for merge problems");
if (RunFixFwData(progress))
{
progress.WriteMessage("Ran fix-up utility after merge.");
FLExProjectSplitter.PushHumptyOffTheWall(progress, _writeVerbose, _fwdataPathname);
}
}
/// <summary>
/// Allow the client to do something right before the initial local commit.
/// </summary>
public void PrepareForInitialCommit(IProgress progress)
{
if (!_needToNestMainFile)
{
return; // Only nest it one time.
}
progress.WriteMessage("Split up project file: {0}", ProjectFilename);
FLExProjectSplitter.PushHumptyOffTheWall(progress, _writeVerbose, _fwdataPathname);
progress.WriteMessage("Finished splitting up project file: {0}", ProjectFilename);
_needToNestMainFile = false;
}
public void UserCancelledBreakupShouldThrow()
{
using (var tempFile = TempFile.WithFilename("foo" + Utilities.FwXmlExtension))
{
var progress = new NullProgress
{
CancelRequested = true
};
var pathname = tempFile.Path;
Assert.Throws <UserCancelledException>(() => FLExProjectSplitter.PushHumptyOffTheWall(progress, pathname));
}
}
private MetadataCache GetFreshMdc()
{
var mdc = MetadataCache.TestOnlyNewCache;
var modelVersionPathname = Path.Combine(_workingDir, SharedConstants.ModelVersionFilename);
if (!File.Exists(modelVersionPathname))
{
FLExProjectSplitter.WriteVersionFile(_srcFwdataPathname);
using (var fastSplitter = new FastXmlElementSplitter(_srcFwdataPathname))
{
bool foundOptionalFirstElement;
// NB: The main input file *does* have to deal with the optional first element.
foreach (var record in fastSplitter.GetSecondLevelElementBytes(SharedConstants.AdditionalFieldsTag, SharedConstants.RtTag, out foundOptionalFirstElement))
{
if (foundOptionalFirstElement)
{
// 2. Write custom properties file with custom properties.
FileWriterService.WriteCustomPropertyFile(mdc, _workingDir, record);
}
else
{
// Write empty custom properties file.
FileWriterService.WriteCustomPropertyFile(Path.Combine(_workingDir, SharedConstants.CustomPropertiesFilename), null);
}
break;
}
}
}
var modelData = File.ReadAllText(modelVersionPathname);
mdc.UpgradeToVersion(Int32.Parse(modelData.Split(new[] { "{", ":", "}" }, StringSplitOptions.RemoveEmptyEntries)[1]));
var customPropPathname = Path.Combine(_workingDir, SharedConstants.CustomPropertiesFilename);
mdc.AddCustomPropInfo(new MergeOrder(
customPropPathname, customPropPathname, customPropPathname,
new MergeSituation(customPropPathname, "", "", "", "", MergeOrder.ConflictHandlingModeChoices.WeWin)));
return(mdc);
}
internal static void WriteNestedDomainData(IProgress progress, bool writeVerbose, string rootDir,
IDictionary <string, XElement> wellUsedElements,
IDictionary <string, SortedDictionary <string, byte[]> > classData,
Dictionary <string, string> guidToClassMapping)
{
var anthropologyBaseDir = Path.Combine(rootDir, SharedConstants.Anthropology);
if (!Directory.Exists(anthropologyBaseDir))
{
Directory.CreateDirectory(anthropologyBaseDir);
}
FLExProjectSplitter.CheckForUserCancelRequested(progress);
if (writeVerbose)
{
progress.WriteVerbose("Writing the anthropology data....");
}
else
{
progress.WriteMessage("Writing the anthropology data....");
}
AnthropologyBoundedContextService.NestContext(anthropologyBaseDir, wellUsedElements, classData, guidToClassMapping);
}
internal static void WriteNestedDomainData(IProgress progress, bool writeVerbose, string rootDir,
IDictionary <string, XElement> wellUsedElements,
IDictionary <string, SortedDictionary <string, byte[]> > classData,
Dictionary <string, string> guidToClassMapping)
{
var generalBaseDir = Path.Combine(rootDir, SharedConstants.General);
if (!Directory.Exists(generalBaseDir))
{
Directory.CreateDirectory(generalBaseDir);
}
FLExProjectSplitter.CheckForUserCancelRequested(progress);
if (writeVerbose)
{
progress.WriteVerbose("Writing the general data....");
progress.WriteVerbose("Writing user-defined list data....");
}
else
{
progress.WriteMessage("Writing the general data....");
progress.WriteMessage("Writing user-defined list data....");
}
UserDefinedListsBoundedContextService.NestContext(generalBaseDir, wellUsedElements, classData, guidToClassMapping);
FLExProjectSplitter.CheckForUserCancelRequested(progress);
if (writeVerbose)
{
progress.WriteVerbose("Writing language project data....");
}
else
{
progress.WriteMessage("Writing language project data....");
}
GeneralDomainBoundedContext.NestContext(generalBaseDir, wellUsedElements, classData, guidToClassMapping);
}
internal static void WriteNestedDomainData(IProgress progress, bool writeVerbose, string rootDir,
IDictionary <string, XElement> wellUsedElements,
IDictionary <string, SortedDictionary <string, byte[]> > classData,
Dictionary <string, string> guidToClassMapping)
{
var linguisticsBaseDir = Path.Combine(rootDir, SharedConstants.Linguistics);
if (!Directory.Exists(linguisticsBaseDir))
{
Directory.CreateDirectory(linguisticsBaseDir);
}
FLExProjectSplitter.CheckForUserCancelRequested(progress);
if (writeVerbose)
{
progress.WriteVerbose("Writing the linguistics data....");
progress.WriteVerbose("Writing reversal data....");
}
else
{
progress.WriteMessage("Writing the linguistics data....");
progress.WriteMessage("Writing reversal data....");
}
ReversalBoundedContextService.NestContext(linguisticsBaseDir, wellUsedElements, classData, guidToClassMapping);
FLExProjectSplitter.CheckForUserCancelRequested(progress);
if (writeVerbose)
{
progress.WriteVerbose("Writing morphology and syntax data....");
}
else
{
progress.WriteMessage("Writing morphology and syntax data....");
}
MorphologyAndSyntaxBoundedContextService.NestContext(linguisticsBaseDir, wellUsedElements, classData, guidToClassMapping);
// Both ReversalBoundedContextService and MorphologyAndSyntaxBoundedContextService abscond with some stuff owned by LexDb. :-(
FLExProjectSplitter.CheckForUserCancelRequested(progress);
if (writeVerbose)
{
progress.WriteVerbose("Writing lexical data....");
}
else
{
progress.WriteMessage("Writing lexical data....");
}
LexiconBoundedContextService.NestContext(linguisticsBaseDir, wellUsedElements, classData, guidToClassMapping);
FLExProjectSplitter.CheckForUserCancelRequested(progress);
if (writeVerbose)
{
progress.WriteVerbose("Writing text corpus data....");
}
else
{
progress.WriteMessage("Writing text corpus data....");
}
TextCorpusBoundedContextService.NestContext(linguisticsBaseDir, wellUsedElements, classData, guidToClassMapping);
FLExProjectSplitter.CheckForUserCancelRequested(progress);
if (writeVerbose)
{
progress.WriteVerbose("Writing wordform and punctuation data....");
}
else
{
progress.WriteMessage("Writing wordform and punctuation data....");
}
WordformInventoryBoundedContextService.NestContext(linguisticsBaseDir, wellUsedElements, classData, guidToClassMapping);
FLExProjectSplitter.CheckForUserCancelRequested(progress);
if (writeVerbose)
{
progress.WriteVerbose("Writing discourse data....");
}
else
{
progress.WriteMessage("Writing discourse data....");
}
FLExProjectSplitter.CheckForUserCancelRequested(progress);
DiscourseAnalysisBoundedContextService.NestContext(linguisticsBaseDir, wellUsedElements, classData, guidToClassMapping);
if (writeVerbose)
{
progress.WriteVerbose("Writing phonology data....");
}
else
{
progress.WriteMessage("Writing phonology data....");
}
PhonologyBoundedContextService.NestContext(linguisticsBaseDir, wellUsedElements, classData, guidToClassMapping);
}
private void Verify(Stopwatch verifyTimer, StringBuilder sb)
{
GC.Collect(2, GCCollectionMode.Forced);
verifyTimer.Start();
GetFreshMdc(); // Want it fresh.
var origData = new Dictionary <string, byte[]>(StringComparer.InvariantCultureIgnoreCase);
using (var fastSplitterOrig = new FastXmlElementSplitter(_srcFwdataPathname + ".orig"))
{
var foundOrigOptionalFirstElement = false;
var testedforExistanceOfOrigOptionalFirstElement = false;
foreach (var origRecord in fastSplitterOrig.GetSecondLevelElementBytes(SharedConstants.AdditionalFieldsTag, SharedConstants.RtTag))
{
if (!testedforExistanceOfOrigOptionalFirstElement)
{
foundOrigOptionalFirstElement = FLExProjectSplitter.IsOptionalFirstElement(origRecord);
testedforExistanceOfOrigOptionalFirstElement = true;
}
if (foundOrigOptionalFirstElement)
{
origData.Add(SharedConstants.AdditionalFieldsTag, origRecord);
foundOrigOptionalFirstElement = false;
continue;
}
origData.Add(XmlUtils.GetAttributes(origRecord, new HashSet <string> {
SharedConstants.GuidStr
})[SharedConstants.GuidStr].ToLowerInvariant(), origRecord);
}
}
verifyTimer.Stop();
GC.Collect(2, GCCollectionMode.Forced);
verifyTimer.Start();
using (var fastSplitterNew = new FastXmlElementSplitter(_srcFwdataPathname))
{
// NB: The main input file *does* have to deal with the optional first element.
//var counter = 0;
var foundNewOptionalFirstElement = false;
var testedforExistanceOfNewOptionalFirstElement = false;
foreach (var newRecordAsBytes in fastSplitterNew.GetSecondLevelElementBytes(SharedConstants.AdditionalFieldsTag, SharedConstants.RtTag))
{
if (!testedforExistanceOfNewOptionalFirstElement)
{
foundNewOptionalFirstElement = FLExProjectSplitter.IsOptionalFirstElement(newRecordAsBytes);
testedforExistanceOfNewOptionalFirstElement = true;
}
var newRecCopyAsBytes = newRecordAsBytes;
byte[] origRecAsBytes;
string srcGuid = null;
if (foundNewOptionalFirstElement)
{
origRecAsBytes = origData[SharedConstants.AdditionalFieldsTag];
origData.Remove(SharedConstants.AdditionalFieldsTag);
foundNewOptionalFirstElement = false;
}
else
{
var attrValues = XmlUtils.GetAttributes(newRecordAsBytes, new HashSet <string> {
SharedConstants.GuidStr, SharedConstants.Class
});
srcGuid = attrValues[SharedConstants.GuidStr];
origRecAsBytes = origData[srcGuid];
origData.Remove(srcGuid);
if (attrValues[SharedConstants.Class] == "WfiWordform")
{
var wfElement = Utilities.CreateFromBytes(origRecAsBytes);
var csProp = wfElement.Element("Checksum");
if (csProp != null)
{
csProp.Attribute(SharedConstants.Val).Value = "0";
origRecAsBytes = SharedConstants.Utf8.GetBytes(wfElement.ToString());
}
}
}
//if (counter == 1000)
//{
// verifyTimer.Stop();
// GC.Collect(2, GCCollectionMode.Forced);
// verifyTimer.Start();
// counter = 0;
//}
//else
//{
// counter++;
//}
// Way too slow, since it has to always make the XmlNodes.
// Just feeding strings to XmlUtilities.AreXmlElementsEqual is faster,
// since it skips making them, if the strings are the same.
//var origNode = CreateXmlNodeFromBytes(origRecAsBytes);
//var newNode = CreateXmlNodeFromBytes(newRecCopyAsBytes);
//if (XmlUtilities.AreXmlElementsEqual(origNode, newNode))
// continue;
//if (srcGuid == null)
//{
// WriteProblemDataFile(Path.Combine(_workingDir, "CustomProperties-SRC.txt"), origNode);
// WriteProblemDataFile(Path.Combine(_workingDir, srcGuid + "CustomProperties-TRG.txt"), newNode);
// sb.Append("Main src and trg custom properties are different in the resulting xml.");
//}
//else
//{
// WriteProblemDataFile(Path.Combine(_workingDir, srcGuid + "-SRC.txt"), origNode);
// WriteProblemDataFile(Path.Combine(_workingDir, srcGuid + "-TRG.txt"), newNode);
// sb.AppendFormat("Main src and trg object with guid '{0}' are different in the resulting xml.", srcGuid);
//}
//if (XmlUtilities.AreXmlElementsEqual(SharedConstants.Utf8.GetString(origRecAsBytes), SharedConstants.Utf8.GetString(newRecCopyAsBytes)))
// continue;
if (XmlUtilities.AreXmlElementsEqual(origRecAsBytes, newRecCopyAsBytes))
{
continue;
}
if (srcGuid == null)
{
WriteProblemDataFile(Path.Combine(_workingDir, "CustomProperties-SRC.txt"), origRecAsBytes);
WriteProblemDataFile(Path.Combine(_workingDir, srcGuid + "CustomProperties-TRG.txt"), newRecCopyAsBytes);
sb.Append("Main src and trg custom properties are different in the resulting xml.");
}
else
{
WriteProblemDataFile(Path.Combine(_workingDir, srcGuid + "-SRC.txt"), origRecAsBytes);
WriteProblemDataFile(Path.Combine(_workingDir, srcGuid + "-TRG.txt"), newRecCopyAsBytes);
sb.AppendFormat("Main src and trg object with guid '{0}' are different in the resulting xml.", srcGuid);
}
sb.AppendLine();
}
}
if (origData.Count > 0)
{
sb.AppendFormat("Hmm, there are {0} more <rt> elements in the original than in the rebuilt fwdata file.", origData.Count);
sb.AppendLine();
foreach (var attrs in origData.Values.Select(byteData => XmlUtils.GetAttributes(byteData, new HashSet <string> {
SharedConstants.GuidStr, SharedConstants.Class
})))
{
sb.AppendFormat("\t\t'{0}' of class '{1}' is not in rebuilt file.", attrs[SharedConstants.GuidStr], attrs[SharedConstants.Class]);
sb.AppendLine();
}
}
verifyTimer.Stop();
}
private void RoundTripData(Stopwatch breakupTimer, Stopwatch restoreTimer, Stopwatch ambiguousTimer, StringBuilder sbValidation)
{
File.Copy(_srcFwdataPathname, _srcFwdataPathname + ".orig", true); // Keep it safe.
GetFreshMdc(); // Want it fresh.
breakupTimer.Start();
FLExProjectSplitter.PushHumptyOffTheWall(new NullProgress(), _srcFwdataPathname);
breakupTimer.Stop();
GC.Collect(2, GCCollectionMode.Forced);
if (_cbCheckAmbiguousElements.Checked)
{
var allDataFiles = new HashSet <string>();
var currentDir = Path.Combine(_workingDir, "Linguistics");
if (Directory.Exists(currentDir))
{
allDataFiles.UnionWith(from pathname in Directory.GetFiles(currentDir, "*.*", SearchOption.AllDirectories)
where !pathname.ToLowerInvariant().EndsWith("chorusnotes")
select pathname);
}
currentDir = Path.Combine(_workingDir, "Anthropology");
if (Directory.Exists(currentDir))
{
allDataFiles.UnionWith(from pathname in Directory.GetFiles(currentDir, "*.*", SearchOption.AllDirectories)
where !pathname.ToLowerInvariant().EndsWith("chorusnotes")
select pathname);
}
currentDir = Path.Combine(_workingDir, "Other");
if (Directory.Exists(currentDir))
{
allDataFiles.UnionWith(
from pathname in Directory.GetFiles(currentDir, "*.*", SearchOption.AllDirectories)
where !pathname.ToLowerInvariant().EndsWith("chorusnotes")
select pathname);
}
currentDir = Path.Combine(_workingDir, "General");
if (Directory.Exists(currentDir))
{
allDataFiles.UnionWith(from pathname in Directory.GetFiles(currentDir, "*.*", SearchOption.AllDirectories)
where !pathname.ToLowerInvariant().EndsWith("chorusnotes")
select pathname);
}
var mergeOrder = new MergeOrder(null, null, null, new NullMergeSituation())
{
EventListener = new ChangeAndConflictAccumulator()
};
var merger = FieldWorksMergeServices.CreateXmlMergerForFieldWorksData(mergeOrder, MetadataCache.MdCache);
ambiguousTimer.Start();
foreach (var dataFile in allDataFiles)
{
var extension = Path.GetExtension(dataFile).Substring(1);
string optionalElementName = null;
string mainRecordName = null;
switch (extension)
{
case SharedConstants.Style:
mainRecordName = SharedConstants.StStyle;
break;
case SharedConstants.List:
mainRecordName = SharedConstants.CmPossibilityList;
break;
case SharedConstants.langproj:
mainRecordName = SharedConstants.LangProject;
break;
case SharedConstants.Annotation:
mainRecordName = SharedConstants.CmAnnotation;
break;
case SharedConstants.Filter:
mainRecordName = SharedConstants.CmFilter;
break;
case SharedConstants.orderings:
mainRecordName = SharedConstants.VirtualOrdering;
break;
case SharedConstants.pictures:
mainRecordName = SharedConstants.CmPicture;
break;
case SharedConstants.ArchivedDraft:
mainRecordName = SharedConstants.ScrDraft;
break;
case SharedConstants.ImportSetting:
mainRecordName = SharedConstants.ScrImportSet;
break;
case SharedConstants.Srs:
mainRecordName = SharedConstants.ScrRefSystem;
break;
case SharedConstants.Trans:
mainRecordName = SharedConstants.Scripture;
break;
case SharedConstants.bookannotations:
mainRecordName = SharedConstants.ScrBookAnnotations;
break;
case SharedConstants.book:
mainRecordName = SharedConstants.ScrBook;
break;
case SharedConstants.Ntbk:
optionalElementName = SharedConstants.Header;
mainRecordName = SharedConstants.RnGenericRec;
break;
case SharedConstants.Reversal:
optionalElementName = SharedConstants.Header;
mainRecordName = SharedConstants.ReversalIndexEntry;
break;
case SharedConstants.Lexdb:
optionalElementName = SharedConstants.Header;
mainRecordName = SharedConstants.LexEntry;
break;
case SharedConstants.TextInCorpus:
mainRecordName = SharedConstants.Text;
break;
case SharedConstants.Inventory:
optionalElementName = SharedConstants.Header;
mainRecordName = SharedConstants.WfiWordform;
break;
case SharedConstants.DiscourseExt:
optionalElementName = SharedConstants.Header;
mainRecordName = SharedConstants.DsChart;
break;
case SharedConstants.Featsys:
mainRecordName = SharedConstants.FsFeatureSystem;
break;
case SharedConstants.Phondata:
mainRecordName = SharedConstants.PhPhonData;
break;
case SharedConstants.Morphdata:
mainRecordName = SharedConstants.MoMorphData;
break;
case SharedConstants.Agents:
mainRecordName = SharedConstants.CmAgent;
break;
}
using (var fastSplitter = new FastXmlElementSplitter(dataFile))
{
bool foundOptionalFirstElement;
foreach (var record in fastSplitter.GetSecondLevelElementBytes(optionalElementName, mainRecordName, out foundOptionalFirstElement))
{
XmlMergeService.RemoveAmbiguousChildren(merger.EventListener, merger.MergeStrategies, CreateXmlNodeFromBytes(record));
}
}
}
ambiguousTimer.Stop();
foreach (var warning in ((ChangeAndConflictAccumulator)merger.EventListener).Warnings)
{
sbValidation.AppendLine(warning.Description);
sbValidation.AppendLine();
sbValidation.AppendLine(warning.HtmlDetails);
sbValidation.AppendLine();
}
GC.Collect(2, GCCollectionMode.Forced);
}
restoreTimer.Start();
FLExProjectUnifier.PutHumptyTogetherAgain(new NullProgress(), _srcFwdataPathname);
restoreTimer.Stop();
GC.Collect(2, GCCollectionMode.Forced);
}
public void NonExistingFileForBreakupShouldThrow()
{
Assert.Throws <ApplicationException>(() => FLExProjectSplitter.PushHumptyOffTheWall(new NullProgress(), "Bogus" + Utilities.FwXmlExtension));
}
public void EmptyPathnameForBreakupShouldThrow()
{
Assert.Throws <ApplicationException>(() => FLExProjectSplitter.PushHumptyOffTheWall(new NullProgress(), ""));
}
internal static void WriteNestedDomainData(IProgress progress, bool writeVerbose, string rootDir,
IDictionary <string, XElement> wellUsedElements,
IDictionary <string, SortedDictionary <string, byte[]> > classData,
Dictionary <string, string> guidToClassMapping)
{
/*
* BC 1. ScrRefSystem (no owner)
* Books prop owns seq of ScrBookRef (which has all basic props).
* No other props.
* [Put all in one file in a subfolder of Scripture?]
* [[Nesting]]
*
* BC 2. CheckLists prop on LP that holds col of CmPossibilityList items.
* Each CmPossibilityList will hold ChkTerm (called ChkItem in model file) objects (derived from CmPossibility)
* [Store each list in a file. Put all lists in subfolder.]
* [[Nesting]]
*
* BC 3. Scripture (owned by LP)
* Leave in:
* Resources prop owns col of CmResource. [Leave.]
* Extract:
* BC 4. ArchivedDrafts prop owns col of ScrDraft. [Each ScrDraft goes in its own file. Archived stuff goes into subfolder of Scripture.]
* BC 5. Styles props owns col of StStyle. [Put styles in subfolder and one for each style.]
* BC 6. ImportSettings prop owns col of ScrImportSet. [Put sets in subfolder and one for each set.]
* BC 7. NoteCategories prop owns one CmPossibilityList [Put list in its own file.]
* BC 8. ScriptureBooks prop owns seq of ScrBook. [Put each book in its own folder (named for its cononical order number).]
* BC 9. BookAnnotations prop owns seq of ScrBookAnnotations. [Put each ScrBookAnnotations in corresponding subfolder along with optional book.]
*/
var scriptureBaseDir = Path.Combine(rootDir, SharedConstants.Other);
if (!Directory.Exists(scriptureBaseDir))
{
Directory.CreateDirectory(scriptureBaseDir);
}
FLExProjectSplitter.CheckForUserCancelRequested(progress);
if (writeVerbose)
{
progress.WriteVerbose("Writing the other data....");
}
else
{
progress.WriteMessage("Writing the other data....");
}
ScriptureReferenceSystemBoundedContextService.NestContext(scriptureBaseDir, classData, guidToClassMapping);
var langProj = wellUsedElements[SharedConstants.LangProject];
FLExProjectSplitter.CheckForUserCancelRequested(progress);
ScriptureCheckListsBoundedContextService.NestContext(langProj, scriptureBaseDir, classData, guidToClassMapping);
// These are intentionally out of order from the above numbering scheme.
var scrAsBytes = classData[SharedConstants.Scripture].Values.FirstOrDefault();
// // Lela Teli-3 has null.
if (scrAsBytes != null)
{
var scripture = Utilities.CreateFromBytes(scrAsBytes);
FLExProjectSplitter.CheckForUserCancelRequested(progress);
ArchivedDraftsBoundedContextService.NestContext(scripture.Element(SharedConstants.ArchivedDrafts), scriptureBaseDir, classData, guidToClassMapping);
FLExProjectSplitter.CheckForUserCancelRequested(progress);
ScriptureStylesBoundedContextService.NestContext(scripture.Element(SharedConstants.Styles), scriptureBaseDir, classData, guidToClassMapping);
FLExProjectSplitter.CheckForUserCancelRequested(progress);
ImportSettingsBoundedContextService.NestContext(scripture.Element(SharedConstants.ImportSettings), scriptureBaseDir, classData, guidToClassMapping);
FLExProjectSplitter.CheckForUserCancelRequested(progress);
ScriptureBoundedContextService.NestContext(langProj, scripture, scriptureBaseDir, classData, guidToClassMapping);
}
RemoveFolderIfEmpty(scriptureBaseDir);
}
