internal static void SortEntireFile(Dictionary<string, Dictionary<string, HashSet<string>>> sortableProperties, XmlWriter writer, string pathname)
{
var readerSettings = new XmlReaderSettings { IgnoreWhitespace = true };
// Step 2: Sort and rewrite file.
using (var fastSplitter = new FastXmlElementSplitter(pathname))
{
var sorter = new BigDataSorter();
bool foundOptionalFirstElement;
foreach (var record in fastSplitter.GetSecondLevelElementStrings(OptionalFirstElementTag, StartTag, out foundOptionalFirstElement))
{
if (foundOptionalFirstElement)
{
// Step 2A: Write out custom property declaration(s).
WriteElement(writer, SortCustomPropertiesRecord(record));
foundOptionalFirstElement = false;
}
else
{
// Step 2B: Sort main CmObject record.
var sortedMainObject = SortMainElement(sortableProperties, record);
sorter.Add(sortedMainObject.Attribute("guid").Value, Utf8.GetBytes(sortedMainObject.ToString()));
}
}
sorter.WriteResults(val => WriteElement(writer, readerSettings, val));
}
}
public void Empty_String_Parameter_Throws()
{
using (var tempFile = new TempFile(""))
{
using (var reader = new FastXmlElementSplitter(tempFile.Path))
{
// ToList is needed to make the enumeration evaluate.
Assert.Throws<ArgumentException>(() => reader.GetSecondLevelElementBytes("").ToList());
}
}
}
public MakeRecordDictionary(IDictionary<string, byte[]> dictionary, string pathname,
string firstElementMarker,
string recordStartingTag, string identifierAttribute)
{
_dictionary = dictionary;
_firstElementTag = firstElementMarker; // May be null, which is fine.
_recordStartingTag = recordStartingTag;
_elementSplitter = new FastXmlElementSplitter(pathname);
_utf8 = Encoding.UTF8;
_identifierWithDoubleQuote = _utf8.GetBytes(identifierAttribute + "=\"");
_identifierWithSingleQuote = _utf8.GetBytes(identifierAttribute + "='");
}
private void LoadDataFile(object sender, EventArgs e)
{
try
{
string selectedPathname = null;
using (var fileDlg = new OpenFileDialog())
{
if (fileDlg.ShowDialog(this) == DialogResult.OK)
selectedPathname = fileDlg.FileName;
}
if (!string.IsNullOrEmpty(selectedPathname))
{
var extension = Path.GetExtension(selectedPathname).ToLowerInvariant();
string firstElementMarker = null;
string recordMarker = null;
switch (extension)
{
case ".lift":
firstElementMarker = "header";
recordMarker = "entry";
break;
case ".chorusnotes":
recordMarker = "annotation";
break;
case ".fwdata":
firstElementMarker = "AdditionalFields";
recordMarker = "rt";
break;
}
using (var splitter = new FastXmlElementSplitter(selectedPathname))
{
bool foundOptionalFirstElement;
var results = splitter.GetSecondLevelElementBytes(firstElementMarker, recordMarker, out foundOptionalFirstElement);
Console.WriteLine("Records: " + results.Count());
}
}
Close();
}
catch (Exception err)
{
var msg = err.Message;
Console.WriteLine(msg);
throw;
}
}
public void No_Records_With_Children_Is_Fine()
{
const string noRecordsInput =
@"<?xml version='1.0' encoding='utf-8'?>
<classdata>
</classdata>";
var tempPathname = Path.GetTempFileName();
var goodXmlPathname = Path.ChangeExtension(tempPathname, ".ClassData");
File.Delete(tempPathname);
try
{
File.WriteAllText(goodXmlPathname, noRecordsInput, Encoding.UTF8);
using (var reader = new FastXmlElementSplitter(goodXmlPathname))
{
Assert.AreEqual(0, reader.GetSecondLevelElementBytes("rt").Count());
}
}
finally
{
File.Delete(goodXmlPathname);
}
}
public void Not_Xml_Throws()
{
const string noRecordsInput = "Some random text file.";
var tempPathname = Path.GetTempFileName();
var goodPathname = Path.ChangeExtension(tempPathname, ".txt");
File.Delete(tempPathname);
try
{
File.WriteAllText(goodPathname, noRecordsInput, Encoding.UTF8);
using (var reader = new FastXmlElementSplitter(goodPathname))
{
// An earlier version was expected to throw XmlException. But we aren't parsing XML well enough to do that confidently.
// Note: the ToList is needed to force the enumeration to enumerate.
Assert.Throws<ArgumentException>(() => reader.GetSecondLevelElementBytes("rt").ToList());
}
}
finally
{
File.Delete(goodPathname);
}
}
private static void TokenizeFile(MetadataCache mdc, string srcFwdataPathname, Dictionary<string, SortedDictionary<string, byte[]>> unownedObjects, Dictionary<string, SortedDictionary<string, byte[]>> classData, Dictionary<string, string> guidToClassMapping)
{
using (var fastSplitter = new FastXmlElementSplitter(srcFwdataPathname))
{
bool foundOptionalFirstElement;
// NB: The main input file *does* have to deal with the optional first element.
foreach (var record in fastSplitter.GetSecondLevelElementBytes(SharedConstants.AdditionalFieldsTag, SharedConstants.RtTag, out foundOptionalFirstElement))
{
if (foundOptionalFirstElement)
{
// Cache custom prop file for later write.
var cpElement = DataSortingService.SortCustomPropertiesRecord(SharedConstants.Utf8.GetString(record));
// Add custom property info to MDC, since it may need to be sorted in the data files.
foreach (var propElement in cpElement.Elements(SharedConstants.CustomField))
{
var className = propElement.Attribute(SharedConstants.Class).Value;
var propName = propElement.Attribute(SharedConstants.Name).Value;
var typeAttr = propElement.Attribute("type");
var adjustedTypeValue = MetadataCache.AdjustedPropertyType(typeAttr.Value);
if (adjustedTypeValue != typeAttr.Value)
typeAttr.Value = adjustedTypeValue;
var customProp = new FdoPropertyInfo(
propName,
typeAttr.Value,
true);
mdc.AddCustomPropInfo(
className,
customProp);
}
mdc.ResetCaches();
//optionalFirstElement = Utf8.GetBytes(cpElement.ToString());
foundOptionalFirstElement = false;
}
else
{
CacheDataRecord(unownedObjects, classData, guidToClassMapping, record);
}
}
}
GC.Collect(2, GCCollectionMode.Forced);
}
private void Verify(Stopwatch verifyTimer, StringBuilder sb)
{
GC.Collect(2, GCCollectionMode.Forced);
verifyTimer.Start();
GetFreshMdc(); // Want it fresh.
var origData = new Dictionary<string, byte[]>(StringComparer.InvariantCultureIgnoreCase);
using (var fastSplitterOrig = new FastXmlElementSplitter(_srcFwdataPathname + ".orig"))
{
var foundOrigOptionalFirstElement = false;
var testedforExistanceOfOrigOptionalFirstElement = false;
foreach (var origRecord in fastSplitterOrig.GetSecondLevelElementBytes(SharedConstants.AdditionalFieldsTag, SharedConstants.RtTag))
{
if (!testedforExistanceOfOrigOptionalFirstElement)
{
foundOrigOptionalFirstElement = FLExProjectSplitter.IsOptionalFirstElement(origRecord);
testedforExistanceOfOrigOptionalFirstElement = true;
}
if (foundOrigOptionalFirstElement)
{
origData.Add(SharedConstants.AdditionalFieldsTag, origRecord);
foundOrigOptionalFirstElement = false;
continue;
}
origData.Add(XmlUtils.GetAttributes(origRecord, new HashSet<string> { SharedConstants.GuidStr })[SharedConstants.GuidStr].ToLowerInvariant(), origRecord);
}
}
verifyTimer.Stop();
GC.Collect(2, GCCollectionMode.Forced);
verifyTimer.Start();
using (var fastSplitterNew = new FastXmlElementSplitter(_srcFwdataPathname))
{
// NB: The main input file *does* have to deal with the optional first element.
//var counter = 0;
var foundNewOptionalFirstElement = false;
var testedforExistanceOfNewOptionalFirstElement = false;
foreach (var newRecordAsBytes in fastSplitterNew.GetSecondLevelElementBytes(SharedConstants.AdditionalFieldsTag, SharedConstants.RtTag))
{
if (!testedforExistanceOfNewOptionalFirstElement)
{
foundNewOptionalFirstElement = FLExProjectSplitter.IsOptionalFirstElement(newRecordAsBytes);
testedforExistanceOfNewOptionalFirstElement = true;
}
var newRecCopyAsBytes = newRecordAsBytes;
byte[] origRecAsBytes;
string srcGuid = null;
if (foundNewOptionalFirstElement)
{
origRecAsBytes = origData[SharedConstants.AdditionalFieldsTag];
origData.Remove(SharedConstants.AdditionalFieldsTag);
foundNewOptionalFirstElement = false;
}
else
{
var attrValues = XmlUtils.GetAttributes(newRecordAsBytes, new HashSet<string> { SharedConstants.GuidStr, SharedConstants.Class });
srcGuid = attrValues[SharedConstants.GuidStr];
origRecAsBytes = origData[srcGuid];
origData.Remove(srcGuid);
if (attrValues[SharedConstants.Class] == "WfiWordform")
{
var wfElement = Utilities.CreateFromBytes(origRecAsBytes);
var csProp = wfElement.Element("Checksum");
if (csProp != null)
{
csProp.Attribute(SharedConstants.Val).Value = "0";
origRecAsBytes = SharedConstants.Utf8.GetBytes(wfElement.ToString());
}
}
}
//if (counter == 1000)
//{
// verifyTimer.Stop();
// GC.Collect(2, GCCollectionMode.Forced);
// verifyTimer.Start();
// counter = 0;
//}
//else
//{
// counter++;
//}
// Way too slow, since it has to always make the XmlNodes.
// Just feeding strings to XmlUtilities.AreXmlElementsEqual is faster,
// since it skips making them, if the strings are the same.
//var origNode = CreateXmlNodeFromBytes(origRecAsBytes);
//var newNode = CreateXmlNodeFromBytes(newRecCopyAsBytes);
//if (XmlUtilities.AreXmlElementsEqual(origNode, newNode))
// continue;
//if (srcGuid == null)
//{
// WriteProblemDataFile(Path.Combine(_workingDir, "CustomProperties-SRC.txt"), origNode);
// WriteProblemDataFile(Path.Combine(_workingDir, srcGuid + "CustomProperties-TRG.txt"), newNode);
// sb.Append("Main src and trg custom properties are different in the resulting xml.");
//}
//else
//{
// WriteProblemDataFile(Path.Combine(_workingDir, srcGuid + "-SRC.txt"), origNode);
// WriteProblemDataFile(Path.Combine(_workingDir, srcGuid + "-TRG.txt"), newNode);
// sb.AppendFormat("Main src and trg object with guid '{0}' are different in the resulting xml.", srcGuid);
//}
//if (XmlUtilities.AreXmlElementsEqual(SharedConstants.Utf8.GetString(origRecAsBytes), SharedConstants.Utf8.GetString(newRecCopyAsBytes)))
// continue;
if (XmlUtilities.AreXmlElementsEqual(origRecAsBytes, newRecCopyAsBytes))
continue;
if (srcGuid == null)
{
WriteProblemDataFile(Path.Combine(_workingDir, "CustomProperties-SRC.txt"), origRecAsBytes);
WriteProblemDataFile(Path.Combine(_workingDir, srcGuid + "CustomProperties-TRG.txt"), newRecCopyAsBytes);
sb.Append("Main src and trg custom properties are different in the resulting xml.");
}
else
{
WriteProblemDataFile(Path.Combine(_workingDir, srcGuid + "-SRC.txt"), origRecAsBytes);
WriteProblemDataFile(Path.Combine(_workingDir, srcGuid + "-TRG.txt"), newRecCopyAsBytes);
sb.AppendFormat("Main src and trg object with guid '{0}' are different in the resulting xml.", srcGuid);
}
sb.AppendLine();
}
}
if (origData.Count > 0)
{
sb.AppendFormat("Hmm, there are {0} more <rt> elements in the original than in the rebuilt fwdata file.", origData.Count);
sb.AppendLine();
foreach (var attrs in origData.Values.Select(byteData => XmlUtils.GetAttributes(byteData, new HashSet<string> { SharedConstants.GuidStr, SharedConstants.Class })))
{
sb.AppendFormat("\t\t'{0}' of class '{1}' is not in rebuilt file.", attrs[SharedConstants.GuidStr], attrs[SharedConstants.Class]);
sb.AppendLine();
}
}
verifyTimer.Stop();
}
/// <summary>
/// Sort the provided lift file into canonical order.
///
/// The resulting sorted file will be in a canonical order for the attributes and elements
/// </summary>
/// <param name="liftPathname">The assumed main lift file in a folder.</param>
public static void SortLiftFile(string liftPathname)
{
Guard.AgainstNullOrEmptyString(liftPathname, "liftPathname");
Guard.Against(Path.GetExtension(liftPathname).ToLowerInvariant() != ".lift", "Unexpected file extension");
Guard.Against<FileNotFoundException>(!File.Exists(liftPathname), "Lift file does not exist.");
using (var tempFile = new TempFile(File.ReadAllText(liftPathname), Utf8))
{
var sortedRootAttributes = SortRootElementAttributes(tempFile.Path);
var sortedEntries = new SortedDictionary<string, XElement>(StringComparer.InvariantCultureIgnoreCase);
XElement header = null;
using (var splitter = new FastXmlElementSplitter(tempFile.Path))
{
bool hasHeader;
foreach (var record in splitter.GetSecondLevelElementStrings("header", "entry", out hasHeader))
{
XElement element = FixBadTextElements(record);
SortAttributes(element);
if (hasHeader)
{
hasHeader = false;
header = element;
SortHeader(header);
}
else
{
var guidKey = element.Attribute("guid").Value.ToLowerInvariant();
if (!sortedEntries.ContainsKey(guidKey))
{
SortEntry(element);
sortedEntries.Add(GetUniqueKey(sortedEntries.Keys, guidKey), element);
}
}
}
}
using (var writer = XmlWriter.Create(tempFile.Path, CanonicalXmlSettings.CreateXmlWriterSettings()))
{
writer.WriteStartDocument();
writer.WriteStartElement("lift");
foreach (var rootAttributeKvp in sortedRootAttributes)
{
var keyParts = rootAttributeKvp.Key.Split(':');
if (keyParts.Length > 1)
writer.WriteAttributeString(keyParts[0], keyParts[1], null, rootAttributeKvp.Value);
else
writer.WriteAttributeString(rootAttributeKvp.Key, rootAttributeKvp.Value);
}
if (header != null)
{
WriteElement(writer, header);
}
foreach (var entryElement in sortedEntries.Values)
{
WriteElement(writer, entryElement);
}
writer.WriteEndElement();
writer.WriteEndDocument();
}
File.Copy(tempFile.Path, liftPathname, true);
}
}
private static void ProcessContent(FastXmlElementSplitter fastXmlElementSplitter, int expectedCount, string firstElementMarker,
string recordMarker, Encoding enc)
{
bool foundOptionalFirstElement;
var elementBytes =
fastXmlElementSplitter.GetSecondLevelElementBytes(firstElementMarker, recordMarker, out foundOptionalFirstElement)
.ToList();
Assert.AreEqual(expectedCount, elementBytes.Count);
var elementStrings =
fastXmlElementSplitter.GetSecondLevelElementStrings(firstElementMarker, recordMarker,
out foundOptionalFirstElement).ToList();
Assert.AreEqual(expectedCount, elementStrings.Count);
for (var i = 0; i < elementStrings.Count; ++i)
{
var currentStr = elementStrings[i];
Assert.AreEqual(
currentStr,
enc.GetString(elementBytes[i]));
var el = XElement.Parse(currentStr);
}
}
// This test may be uncommented to try the splitter on some particular file which causes problems.
//[Test]
//public void SplitterParsesProblemFile()
//{
// using (var fastXmlElementSplitter = new FastXmlElementSplitter(@"D:\DownLoads\y.lift"))
// {
// bool foundOptionalFirstElement;
// fastXmlElementSplitter.GetSecondLevelElementBytes("header", "entry", out foundOptionalFirstElement)
// .ToList();
// }
//}
private static void CheckGoodFile(string hasRecordsInput, int expectedCount, string firstElementMarker,
string recordMarker, Encoding enc = null)
{
var goodPathname = Path.GetTempFileName();
try
{
if (enc == null)
enc = Encoding.UTF8;
File.WriteAllText(goodPathname, hasRecordsInput, enc);
using (var fastXmlElementSplitter = new FastXmlElementSplitter(goodPathname))
{
ProcessContent(fastXmlElementSplitter, expectedCount, firstElementMarker, recordMarker, enc);
}
using (var fastXmlElementSplitter = new FastXmlElementSplitter(File.ReadAllBytes(goodPathname)))
{
ProcessContent(fastXmlElementSplitter, expectedCount, firstElementMarker, recordMarker, enc);
}
}
finally
{
File.Delete(goodPathname);
}
}
private static void ProcessMultipleEntryContent(FastXmlElementSplitter fastXmlElementSplitter)
{
string[] expectedNames = new string[] { "Language", "FontName", "FontSize", "Analyses", "Entries" };
var elements = fastXmlElementSplitter.GetSecondLevelElements().ToList();
Assert.AreEqual(5, elements.Count);
for (var i = 0; i < elements.Count; ++i)
{
var curElement = elements[i];
Assert.AreEqual(curElement.Name, expectedNames[i]);
var el = XElement.Parse(curElement.BytesAsString);
if (i >= 3) // parse the sublist for Analyses and Entries
{
using (var splitter = new FastXmlElementSplitter(curElement.Bytes))
{
var subElements = splitter.GetSecondLevelElements().ToList();
Assert.AreEqual(2, subElements.Count);
Assert.IsFalse(subElements.Any(t => t.Name != "item"));
}
}
}
}
public void SplitterHandlesMultipleEntryTypes()
{
const string simulatedLexicalFile =
@"<Lexicon>
<Language>Takwane</Language>
<FontName>Tahoma</FontName>
<FontSize>9</FontSize>
<Analyses>
<item>
<string>ahibathiziwa</string>
<ArrayOfLexeme>
<Lexeme Type='Prefix' Form='ahi' Homograph='1' />
<Lexeme Type='Stem' Form='bathiz' Homograph='1' />
<Lexeme Type='Suffix' Form='iwa' Homograph='1' />
</ArrayOfLexeme>
</item>
<item>
<string>dhahaleele</string>
<ArrayOfLexeme>
<Lexeme Type='Prefix' Form='dha' Homograph='1' />
<Lexeme Type='Stem' Form='haleel' Homograph='1' />
<Lexeme Type='Suffix' Form='e' Homograph='1' />
</ArrayOfLexeme>
</item>
</Analyses>
<Entries>
<item>
<Lexeme Type='Word' Form='eehu' Homograph='1' />
<Entry>
<Sense Id='AnuoxmE2'>
<Gloss Language='English'>our</Gloss>
</Sense>
<Sense Id='LgBvPFTX'>
<Gloss Language='Portuguese'>nosso</Gloss>
</Sense>
</Entry>
</item>
<item>
<Lexeme Type='Word' Form='yesu' Homograph='1' />
<Entry>
<Sense Id='BBY1Mk/1'>
<Gloss Language='English'>jesus</Gloss>
</Sense>
<Sense Id='zV8exv2K'>
<Gloss Language='Portuguese'>jesus</Gloss>
</Sense>
</Entry>
</item>
</Entries>
</Lexicon>";
using (var tempFile = new TempFile())
{
File.WriteAllText(tempFile.Path, simulatedLexicalFile, Encoding.UTF8);
using (var fastXmlElementSplitter = new FastXmlElementSplitter(tempFile.Path))
{
ProcessMultipleEntryContent(fastXmlElementSplitter);
}
using (var fastXmlElementSplitter = new FastXmlElementSplitter(File.ReadAllBytes(tempFile.Path)))
{
ProcessMultipleEntryContent(fastXmlElementSplitter);
}
}
}
public void Not_Xml_Throws()
{
const string noRecordsInput = "Some random text file.";
using (var tempFile = TempFile.WithExtension(".txt"))
{
File.WriteAllText(tempFile.Path, noRecordsInput, Encoding.UTF8);
using (var reader = new FastXmlElementSplitter(tempFile.Path))
{
// An earlier version was expected to throw XmlException. But we aren't parsing XML well enough to do that confidently.
// Note: the ToList is needed to force the enumeration to enumerate.
Assert.Throws<ArgumentException>(() => reader.GetSecondLevelElementBytes("rt").ToList());
}
}
}
private static Dictionary<string, string> MakeRecordDictionary(IMergeEventListener mainMergeEventListener,
IMergeStrategy mergeStrategy,
string pathname,
bool removeAmbiguousChildren,
string firstElementMarker,
string recordStartingTag,
string identifierAttribute)
{
var records = new Dictionary<string, string>(EstimatedObjectCount(pathname),
StringComparer.InvariantCultureIgnoreCase);
using (var fastSplitter = new FastXmlElementSplitter(pathname))
{
bool foundOptionalFirstElement;
foreach (var record in fastSplitter.GetSecondLevelElementStrings(firstElementMarker, recordStartingTag, out foundOptionalFirstElement))
{
if (foundOptionalFirstElement)
{
var key = firstElementMarker.ToLowerInvariant();
if (records.ContainsKey(key))
{
mainMergeEventListener.WarningOccurred(
new MergeWarning(string.Format("{0}: There is more than one optional first element '{1}'", pathname, key)));
}
else
{
if (removeAmbiguousChildren)
{
var possiblyRevisedRecord = RemoveAmbiguousChildren(mainMergeEventListener, mergeStrategy.GetStrategies(), record);
records.Add(key, possiblyRevisedRecord);
}
else
{
records.Add(key, record);
}
}
foundOptionalFirstElement = false;
}
else
{
var attrValues = XmlUtils.GetAttributes(record, new HashSet<string> {"dateDeleted", identifierAttribute});
// Eat tombstones.
if (attrValues["dateDeleted"] != null)
continue;
var identifier = attrValues[identifierAttribute];
if (string.IsNullOrEmpty(identifierAttribute))
{
mainMergeEventListener.WarningOccurred(
new MergeWarning(string.Format("{0}: There was no identifier for the record", pathname)));
continue;
}
if (records.ContainsKey(identifier))
{
mainMergeEventListener.WarningOccurred(
new MergeWarning(string.Format("{0}: There is more than one element with the identifier '{1}'", pathname,
identifier)));
}
else
{
if (removeAmbiguousChildren)
{
var possiblyRevisedRecord = RemoveAmbiguousChildren(mainMergeEventListener, mergeStrategy.GetStrategies(), record);
records.Add(identifier, possiblyRevisedRecord);
}
else
{
records.Add(identifier, record);
}
}
}
}
}
return records;
}
public void Null_Parameter_Throws()
{
// review: I (CP) don't know that this is a sufficiently good method for determining the file - even in windows.
// In mono I had to make this the absolute path. 2011-01
#if MONO
using (var reader = new FastXmlElementSplitter(Assembly.GetExecutingAssembly().CodeBase.Replace(@"file://", null)))
#else
using (var reader = new FastXmlElementSplitter(Assembly.GetExecutingAssembly().CodeBase.Replace(@"file:///", null)))
#endif
{
// ToList is needed to make the enumerable evaluate.
Assert.Throws<ArgumentException>(() => reader.GetSecondLevelElementBytes(null).ToList());
}
}
// This test may be uncommented to try the splitter on some particular file which causes problems.
//[Test]
//public void SplitterParsesProblemFile()
//{
// using (var fastXmlElementSplitter = new FastXmlElementSplitter(@"D:\DownLoads\y.lift"))
// {
// bool foundOptionalFirstElement;
// fastXmlElementSplitter.GetSecondLevelElementBytes("header", "entry", out foundOptionalFirstElement)
// .ToList();
// }
//}
private static void CheckGoodFile(string hasRecordsInput, int expectedCount, string firstElementMarker,
string recordMarker, Encoding enc = null)
{
var goodPathname = Path.GetTempFileName();
try
{
if (enc == null)
enc = Encoding.UTF8;
File.WriteAllText(goodPathname, hasRecordsInput, enc);
using (var fastXmlElementSplitter = new FastXmlElementSplitter(goodPathname))
{
bool foundOptionalFirstElement;
var elementBytes =
fastXmlElementSplitter.GetSecondLevelElementBytes(firstElementMarker, recordMarker, out foundOptionalFirstElement)
.ToList();
Assert.AreEqual(expectedCount, elementBytes.Count);
var elementStrings =
fastXmlElementSplitter.GetSecondLevelElementStrings(firstElementMarker, recordMarker,
out foundOptionalFirstElement).ToList();
Assert.AreEqual(expectedCount, elementStrings.Count);
for (var i = 0; i < elementStrings.Count; ++i)
{
var currentStr = elementStrings[i];
Assert.AreEqual(
currentStr,
enc.GetString(elementBytes[i]));
var el = XElement.Parse(currentStr);
}
}
}
finally
{
File.Delete(goodPathname);
}
}
public void No_Records_Without_Children_Is_Fine()
{
const string noRecordsInput =
@"<?xml version='1.0' encoding='utf-8'?>
<classdata />";
using (var tempFile = TempFile.WithExtension(".ClassData"))
{
File.WriteAllText(tempFile.Path, noRecordsInput, Encoding.UTF8);
using (var reader = new FastXmlElementSplitter(tempFile.Path))
{
Assert.AreEqual(0, reader.GetSecondLevelElementBytes("rt").Count());
}
}
}
private void Dispose(bool disposing)
{
if (_isDisposed)
return; // Done already, so nothing left to do.
if (disposing)
{
if (_elementSplitter != null)
_elementSplitter.Dispose();
}
// Dispose unmanaged resources here, whether disposing is true or false.
// Main data members.
_elementSplitter = null;
_dictionary = null;
_isDisposed = true;
}
private MetadataCache GetFreshMdc()
{
var mdc = MetadataCache.TestOnlyNewCache;
var modelVersionPathname = Path.Combine(_workingDir, SharedConstants.ModelVersionFilename);
if (!File.Exists(modelVersionPathname))
{
FLExProjectSplitter.WriteVersionFile(_srcFwdataPathname);
using (var fastSplitter = new FastXmlElementSplitter(_srcFwdataPathname))
{
bool foundOptionalFirstElement;
// NB: The main input file *does* have to deal with the optional first element.
foreach (var record in fastSplitter.GetSecondLevelElementBytes(SharedConstants.AdditionalFieldsTag, SharedConstants.RtTag, out foundOptionalFirstElement))
{
if (foundOptionalFirstElement)
{
// 2. Write custom properties file with custom properties.
FileWriterService.WriteCustomPropertyFile(mdc, _workingDir, record);
}
else
{
// Write empty custom properties file.
FileWriterService.WriteCustomPropertyFile(Path.Combine(_workingDir, SharedConstants.CustomPropertiesFilename), null);
}
break;
}
}
}
var modelData = File.ReadAllText(modelVersionPathname);
mdc.UpgradeToVersion(Int32.Parse(modelData.Split(new[] { "{", ":", "}" }, StringSplitOptions.RemoveEmptyEntries)[1]));
var customPropPathname = Path.Combine(_workingDir, SharedConstants.CustomPropertiesFilename);
mdc.AddCustomPropInfo(new MergeOrder(
customPropPathname, customPropPathname, customPropPathname,
new MergeSituation(customPropPathname, "", "", "", "", MergeOrder.ConflictHandlingModeChoices.WeWin)));
return mdc;
}
private static Dictionary<string, string> WriteOrCacheProperties(string mainFilePathname,
Dictionary<string, SortedDictionary<string, byte[]>> classData,
Dictionary<string, XElement> wellUsedElements)
{
var pathRoot = Path.GetDirectoryName(mainFilePathname);
var mdc = MetadataCache.MdCache;
// Key is the guid of the object, and value is the class name.
var guidToClassMapping = new Dictionary<string, string>();
using (var fastSplitter = new FastXmlElementSplitter(mainFilePathname))
{
var haveWrittenCustomFile = false;
bool foundOptionalFirstElement;
// NB: The main input file *does* have to deal with the optional first element.
foreach (var record in fastSplitter.GetSecondLevelElementBytes(SharedConstants.AdditionalFieldsTag, SharedConstants.RtTag, out foundOptionalFirstElement))
{
if (foundOptionalFirstElement)
{
// 2. Write custom properties file with custom properties.
FileWriterService.WriteCustomPropertyFile(mdc, pathRoot, record);
foundOptionalFirstElement = false;
haveWrittenCustomFile = true;
}
else
{
CacheDataRecord(record, wellUsedElements, classData, guidToClassMapping);
}
}
if (!haveWrittenCustomFile)
{
// Write empty custom properties file.
FileWriterService.WriteCustomPropertyFile(Path.Combine(pathRoot, SharedConstants.CustomPropertiesFilename), null);
}
}
return guidToClassMapping;
}
private void RoundTripData(Stopwatch breakupTimer, Stopwatch restoreTimer, Stopwatch ambiguousTimer, StringBuilder sbValidation)
{
File.Copy(_srcFwdataPathname, _srcFwdataPathname + ".orig", true); // Keep it safe.
GetFreshMdc(); // Want it fresh.
breakupTimer.Start();
FLExProjectSplitter.PushHumptyOffTheWall(new NullProgress(), _srcFwdataPathname);
breakupTimer.Stop();
GC.Collect(2, GCCollectionMode.Forced);
if (_cbCheckAmbiguousElements.Checked)
{
var allDataFiles = new HashSet<string>();
var currentDir = Path.Combine(_workingDir, "Linguistics");
if (Directory.Exists(currentDir))
{
allDataFiles.UnionWith(from pathname in Directory.GetFiles(currentDir, "*.*", SearchOption.AllDirectories)
where !pathname.ToLowerInvariant().EndsWith("chorusnotes")
select pathname);
}
currentDir = Path.Combine(_workingDir, "Anthropology");
if (Directory.Exists(currentDir))
{
allDataFiles.UnionWith(from pathname in Directory.GetFiles(currentDir, "*.*", SearchOption.AllDirectories)
where !pathname.ToLowerInvariant().EndsWith("chorusnotes")
select pathname);
}
currentDir = Path.Combine(_workingDir, "Other");
if (Directory.Exists(currentDir))
{
allDataFiles.UnionWith(
from pathname in Directory.GetFiles(currentDir, "*.*", SearchOption.AllDirectories)
where !pathname.ToLowerInvariant().EndsWith("chorusnotes")
select pathname);
}
currentDir = Path.Combine(_workingDir, "General");
if (Directory.Exists(currentDir))
{
allDataFiles.UnionWith(from pathname in Directory.GetFiles(currentDir, "*.*", SearchOption.AllDirectories)
where !pathname.ToLowerInvariant().EndsWith("chorusnotes")
select pathname);
}
var mergeOrder = new MergeOrder(null, null, null, new NullMergeSituation())
{
EventListener = new ChangeAndConflictAccumulator()
};
var merger = FieldWorksMergeServices.CreateXmlMergerForFieldWorksData(mergeOrder, MetadataCache.MdCache);
ambiguousTimer.Start();
foreach (var dataFile in allDataFiles)
{
var extension = Path.GetExtension(dataFile).Substring(1);
string optionalElementName = null;
string mainRecordName = null;
switch (extension)
{
case SharedConstants.Style:
mainRecordName = SharedConstants.StStyle;
break;
case SharedConstants.List:
mainRecordName = SharedConstants.CmPossibilityList;
break;
case SharedConstants.langproj:
mainRecordName = SharedConstants.LangProject;
break;
case SharedConstants.Annotation:
mainRecordName = SharedConstants.CmAnnotation;
break;
case SharedConstants.Filter:
mainRecordName = SharedConstants.CmFilter;
break;
case SharedConstants.orderings:
mainRecordName = SharedConstants.VirtualOrdering;
break;
case SharedConstants.pictures:
mainRecordName = SharedConstants.CmPicture;
break;
case SharedConstants.ArchivedDraft:
mainRecordName = SharedConstants.ScrDraft;
break;
case SharedConstants.ImportSetting:
mainRecordName = SharedConstants.ScrImportSet;
break;
case SharedConstants.Srs:
mainRecordName = SharedConstants.ScrRefSystem;
break;
case SharedConstants.Trans:
mainRecordName = SharedConstants.Scripture;
break;
case SharedConstants.bookannotations:
mainRecordName = SharedConstants.ScrBookAnnotations;
break;
case SharedConstants.book:
mainRecordName = SharedConstants.ScrBook;
break;
case SharedConstants.Ntbk:
optionalElementName = SharedConstants.Header;
mainRecordName = SharedConstants.RnGenericRec;
break;
case SharedConstants.Reversal:
optionalElementName = SharedConstants.Header;
mainRecordName = SharedConstants.ReversalIndexEntry;
break;
case SharedConstants.Lexdb:
optionalElementName = SharedConstants.Header;
mainRecordName = SharedConstants.LexEntry;
break;
case SharedConstants.TextInCorpus:
mainRecordName = SharedConstants.Text;
break;
case SharedConstants.Inventory:
optionalElementName = SharedConstants.Header;
mainRecordName = SharedConstants.WfiWordform;
break;
case SharedConstants.DiscourseExt:
optionalElementName = SharedConstants.Header;
mainRecordName = SharedConstants.DsChart;
break;
case SharedConstants.Featsys:
mainRecordName = SharedConstants.FsFeatureSystem;
break;
case SharedConstants.Phondata:
mainRecordName = SharedConstants.PhPhonData;
break;
case SharedConstants.Morphdata:
mainRecordName = SharedConstants.MoMorphData;
break;
case SharedConstants.Agents:
mainRecordName = SharedConstants.CmAgent;
break;
}
using (var fastSplitter = new FastXmlElementSplitter(dataFile))
{
bool foundOptionalFirstElement;
foreach (var record in fastSplitter.GetSecondLevelElementBytes(optionalElementName, mainRecordName, out foundOptionalFirstElement))
{
XmlMergeService.RemoveAmbiguousChildren(merger.EventListener, merger.MergeStrategies, CreateXmlNodeFromBytes(record));
}
}
}
ambiguousTimer.Stop();
foreach (var warning in ((ChangeAndConflictAccumulator)merger.EventListener).Warnings)
{
sbValidation.AppendLine(warning.Description);
sbValidation.AppendLine();
sbValidation.AppendLine(warning.HtmlDetails);
sbValidation.AppendLine();
}
GC.Collect(2, GCCollectionMode.Forced);
}
restoreTimer.Start();
FLExProjectUnifier.PutHumptyTogetherAgain(new NullProgress(), _srcFwdataPathname);
restoreTimer.Stop();
GC.Collect(2, GCCollectionMode.Forced);
}
