// Private Methods
private void Read(BinaryReader reader)
{
Stream stream = reader.BaseStream;
int posFileStart = (int)reader.GetPosition();
Spr0Header header = stream.ReadStructure <Spr0Header>();
stream.Seek(posFileStart + header.texturePointerTableOffset, SeekOrigin.Begin);
TypePointerTable[] texturePointerTable = stream.ReadStructures <TypePointerTable>(header.numTextures);
stream.Seek(posFileStart + header.keyFramePointerTableOffset, SeekOrigin.Begin);
TypePointerTable[] keyFramePointerTable = stream.ReadStructures <TypePointerTable>(header.numKeyFrames);
mTextures = new List <TmxFile>(header.numTextures);
for (int i = 0; i < header.numTextures; i++)
{
stream.Seek(posFileStart + texturePointerTable[i].offset, SeekOrigin.Begin);
mTextures.Add(TmxFile.Load(stream, true));
}
mKeyFrames = new List <SprSprite>(header.numKeyFrames);
for (int i = 0; i < header.numKeyFrames; i++)
{
stream.Seek(posFileStart + keyFramePointerTable[i].offset, SeekOrigin.Begin);
mKeyFrames.Add(new SprSprite(reader));
}
}
// Private Methods
private void InternalRead(BinaryReader reader)
{
int posFileStart = (int)reader.BaseStream.Position;
short flag = reader.ReadInt16();
short userId = reader.ReadInt16();
int length = reader.ReadInt32();
string tag = reader.ReadCString(4);
int unused = reader.ReadInt32();
if (tag != TAG)
{
throw new InvalidDataException();
}
int numTextures = reader.ReadInt32();
int[] texturePointerTable = reader.ReadInt32Array(numTextures);
mTextures = new List <TmxFile>(numTextures);
for (int i = 0; i < numTextures; i++)
{
reader.BaseStream.Seek(posFileStart + texturePointerTable[i], SeekOrigin.Begin);
mTextures.Add(TmxFile.Load(reader.BaseStream, true));
}
}
public ImportStudioTmStep(TranslationMemoryReference outputTranslationMemory, string name, TmxFile tmxFile)
: base(String.Format("Import {0}", name))
{
_outputTranslationMemory = outputTranslationMemory;
_name = name;
_tmxFile = tmxFile;
}
public void Parse_TmxFileNoTranslationUnits_To_TmxFileObject_Test()
{
// Assign
AbstractTmxParser parser = new TmxParser();
TmxFile expected = MockObjects.VALID_en_TmxFile;
// Act
TmxFile result = parser.ParseFile(MockObjects.NO_TRANSLATION_UNITS_TmxFilePath);
// Assert
// -- Assert FileInfo
Assert.AreEqual(MockObjects.NO_TRANSLATION_UNITS_TmxFilePath, result.FileInfo);
// -- Assert Language code
Assert.AreEqual("Unkown", result.LanguageCode);
// -- Assert Data
// ---- Assert version
Assert.AreEqual(expected.Data.Version, result.Data.Version);
// ---- Assert header
Assert.AreEqual(expected.Data.Header.CreationToolVersion, result.Data.Header.CreationToolVersion);
Assert.AreEqual(expected.Data.Header.DataType, result.Data.Header.DataType);
Assert.AreEqual(expected.Data.Header.SegmentType, result.Data.Header.SegmentType);
Assert.AreEqual(expected.Data.Header.AdminLanguage, result.Data.Header.AdminLanguage);
Assert.AreEqual(expected.Data.Header.SourceLanguage, result.Data.Header.SourceLanguage);
Assert.AreEqual(expected.Data.Header.OriginalTranslationMemoryFormat, result.Data.Header.OriginalTranslationMemoryFormat);
Assert.AreEqual(expected.Data.Header.CreationTool, result.Data.Header.CreationTool);
// ---- Assert body
// ------ Assert TranslationUnits
Assert.AreEqual(0, result.Data.Body.TranslationUnits.Count);
}
public void ValidAll_Commit_Test()
{
// Assign
ITmxContext context = new TmxContext(MockObjects.VALID_TestFilesPath);
bool pass = context.Add(MockObjects.VALID_en_Add_ContextTmxFile);
Assert.IsTrue(pass, "Context Add method is broken");
TmxFile update = (from tmx in context.Get("en")
where tmx.FileInfo.FullName == MockObjects.VALID_Update_TmxFilePath.FullName
select tmx).First <TmxFile>();
pass = context.Update(update);
Assert.IsTrue(pass, "Context Update method is broken");
TmxFile delete = (from tmx in context.Get("en")
where tmx.FileInfo.FullName == MockObjects.VALID_Delete_TmxFilePath.FullName
select tmx).First <TmxFile>();
pass = context.Update(delete);
Assert.IsTrue(pass, "Context Update method is broken");
// Act
int result = context.SaveChanges();
// Assert
Assert.AreEqual(3, result);
Assert.IsTrue(File.Exists(MockObjects.VALID_en_Add_ContextTmxFilePath.FullName));
Assert.IsTrue(File.Exists(MockObjects.VALID_Delete_TmxFilePath.FullName));
}
public CleanWorkbenchTmxStep(TmxFile inputTmxFile, TmxFile outputTmxFile, int?maxTusProcessed)
: base(String.Format("Clean {0}", Path.GetFileName(inputTmxFile.FilePath)))
{
_inputTmxFile = inputTmxFile;
_outputTmxFile = outputTmxFile;
_maxTusProcessed = maxTusProcessed;
}
internal override TmxFile ParseFile(FileInfo file)
{
if (file.Exists && file.Extension == ".tmx")
{
TmxFile tmxFile = new TmxFile();
// Allocate file information
tmxFile.FileInfo = file;
tmxFile.Id = Guid.NewGuid();
// Assign file contents
tmxFile.Data = Parse(file);
// Assign language code
tmxFile.LanguageCode =
tmxFile.Data.Body.TranslationUnits.Find(tuv => tuv.TranslationUnitVariants.Count > 0) != null
?
tmxFile.Data
.Body
.TranslationUnits
.First<TranslationUnit>(tu => tu.TranslationUnitVariants.Count > 0)
.TranslationUnitVariants
.First<TranslationUnitVariant>()
.Language
:
"Unkown"
;
return tmxFile;
} else
{
// Return that file does not exist
return null;
}
}
public void Parse_EmptyTmxFile_To_TmxFileObject_Test()
{
// Assign
AbstractTmxParser parser = new TmxParser();
// Act
TmxFile result = parser.ParseFile(MockObjects.EMPTY_TmxFilePath);
// Assert
Assert.IsNull(result);
}
public void Parse_NonExistTmxFile_To_TmxFileObject_Test()
{
// Assign
AbstractTmxParser parser = new TmxParser();
// Act
TmxFile result = parser.ParseFile(MockObjects.NonEXIST_TmxFilePath);
// Assert
Assert.IsNull(result);
}
public void Parse_InvalidTmxFile_To_TmxFileObject_Test()
{
// Assign
AbstractTmxParser parser = new TmxParser();
// Act
TmxFile result = parser.ParseFile(MockObjects.INVALID_TmxFilePath);
// Assert
Assert.IsNull(result);
}
internal override bool Delete(TmxFile file)
{
if (file.FileInfo.Exists)
{
File.Delete(file.FileInfo.FullName);
return true;
} else
{
return false;
}
}
internal override bool Delete(TmxFile file)
{
if (file.FileInfo.Exists)
{
File.Delete(file.FileInfo.FullName);
return(true);
}
else
{
return(false);
}
}
public void Parse_Valid_File_Test()
{
// Assign
LanguageRepository_Accessor parser = new LanguageRepository_Accessor();
TmxFile expected = MockObjects.VALID_en_TmxFile;
// Act
TmxFile result = parser.ParseFile(MockObjects.VALID_en_TmxFilePath);
// Assert
// -- Assert FileInfo
Assert.AreEqual(expected.FileInfo, result.FileInfo);
// -- Assert Language code
Assert.AreEqual(expected.LanguageCode, result.LanguageCode);
Assert.AreNotEqual("Unkown", result.LanguageCode);
// -- Assert Data
// ---- Assert version
Assert.AreEqual(expected.Data.Version, result.Data.Version);
// ---- Assert header
Assert.AreEqual(expected.Data.Header.CreationToolVersion, result.Data.Header.CreationToolVersion);
Assert.AreEqual(expected.Data.Header.DataType, result.Data.Header.DataType);
Assert.AreEqual(expected.Data.Header.SegmentType, result.Data.Header.SegmentType);
Assert.AreEqual(expected.Data.Header.AdminLanguage, result.Data.Header.AdminLanguage);
Assert.AreEqual(expected.Data.Header.SourceLanguage, result.Data.Header.SourceLanguage);
Assert.AreEqual(expected.Data.Header.OriginalTranslationMemoryFormat, result.Data.Header.OriginalTranslationMemoryFormat);
Assert.AreEqual(expected.Data.Header.CreationTool, result.Data.Header.CreationTool);
// ---- Assert body
// ------ Assert TranslationUnits
foreach (TranslationUnit expectedTU in expected.Data.Body.TranslationUnits)
{
TranslationUnit unit = result.Data.Body.TranslationUnits
.First <TranslationUnit>(
tu => tu.TranslationUnitId == expectedTU.TranslationUnitId
);
Assert.IsNotNull(unit);
Assert.AreEqual(expectedTU.TranslationUnitId, unit.TranslationUnitId);
// ------ Assert TranslationUnitVariant
foreach (TranslationUnitVariant expectedTUV in expectedTU.TranslationUnitVariants)
{
TranslationUnitVariant unitVar = unit.TranslationUnitVariants
.First <TranslationUnitVariant>(
tuv => tuv.Segment == expectedTUV.Segment
);
Assert.IsNotNull(unitVar);
Assert.AreEqual(expectedTUV.Language, unitVar.Language);
Assert.AreEqual(expectedTUV.Segment, unitVar.Segment);
}
}
}
/// <summary>
/// Add a tmx file to the language repository
/// </summary>
/// <param name="tmxFile">Tmx file object</param>
/// <returns>Sucessful</returns>
public bool Add(TmxFile tmxFile)
{
// Set modifiers
tmxFile.isNew = true;
tmxFile.isModified = false;
tmxFile.isDeleted = false;
tmxFile.Id = Guid.NewGuid();
// Check language is in history
foreach (List<TmxFile> files in History)
{
if (files.First<TmxFile>().LanguageCode == tmxFile.LanguageCode)
{
// Add files to list in history
if (!tmxFile.FileInfo.Directory.Exists)
{
tmxFile.FileInfo = new FileInfo(Path.Combine(Root.FullName, tmxFile.LanguageCode, tmxFile.FileInfo.Name));
}
files.Add(tmxFile);
return true;
}
}
List<TmxFile> repository = Get(tmxFile.LanguageCode);
if (repository.Count != 0)
{
// New list will have been added to history so recall Add
return Add(tmxFile);
} else
{
// Create new repository folder
if (!Directory.Exists(tmxFile.FileInfo.Directory.FullName))
{
string path = Path.Combine(Root.FullName, tmxFile.LanguageCode);
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
tmxFile.FileInfo = new FileInfo(Path.Combine(path, tmxFile.FileInfo.Name));
}
History.Add(new List<TmxFile>()
{
tmxFile
});
return true;
}
}
private void Serialize(TmxFile file)
{
XmlSerializer serializer = new XmlSerializer(typeof(Tmx));
using (StreamWriter writer = new StreamWriter(file.FileInfo.FullName))
{
using (XmlWriter xwriter = XmlWriter.Create(writer, new XmlWriterSettings()
{
Indent = true, Encoding = Encoding.UTF8
}))
{
xwriter.WriteDocType("tmx", "SYSTEM", "tmx14.dtd", null);
serializer.Serialize(xwriter, file.Data);
}
}
}
internal override bool Update(TmxFile file)
{
try
{
if (file.FileInfo.Exists && file.Data.Body != null)
{
Serialize(file);
return true;
} else
{
return false;
}
} catch
{
return false;
}
}
public void Valid_Update_Test()
{
// Assign
TmxContext_Accessor context = new TmxContext_Accessor(MockObjects.VALID_TestFilesPath);
TmxFile current = (from tmx in context.Get("en")
where tmx.FileInfo.FullName == MockObjects.VALID_Update_TmxFilePath.FullName
select tmx).First <TmxFile>();
// Act
bool result = context.Update(current);
// Assert
Assert.IsTrue(result);
Assert.IsTrue(current.isModified);
Assert.IsFalse(current.isDeleted);
Assert.IsFalse(current.isNew);
}
internal override bool Add(TmxFile file)
{
try
{
if (file.FileInfo.Directory.Exists && file.FileInfo.Extension == ".tmx" && file.Data.Body != null)
{
Serialize(file);
return true;
} else
{
return false;
}
} catch
{
return false;
}
}
public void Parse_TmxFileNoSegments_To_TmxFileObject_Test()
{
// Assign
AbstractTmxParser parser = new TmxParser();
TmxFile expected = MockObjects.VALID_en_TmxFile;
// Act
TmxFile result = parser.ParseFile(MockObjects.NO_SEGMENTS_TmxFilePath);
// Assert
// -- Assert FileInfo
Assert.AreEqual(MockObjects.NO_SEGMENTS_TmxFilePath, result.FileInfo);
// -- Assert Language code
Assert.AreEqual(expected.LanguageCode, result.LanguageCode);
Assert.AreNotEqual("Unkown", result.LanguageCode);
// -- Assert Data
// ---- Assert version
Assert.AreEqual(expected.Data.Version, result.Data.Version);
// ---- Assert header
Assert.AreEqual(expected.Data.Header.CreationToolVersion, result.Data.Header.CreationToolVersion);
Assert.AreEqual(expected.Data.Header.DataType, result.Data.Header.DataType);
Assert.AreEqual(expected.Data.Header.SegmentType, result.Data.Header.SegmentType);
Assert.AreEqual(expected.Data.Header.AdminLanguage, result.Data.Header.AdminLanguage);
Assert.AreEqual(expected.Data.Header.SourceLanguage, result.Data.Header.SourceLanguage);
Assert.AreEqual(expected.Data.Header.OriginalTranslationMemoryFormat, result.Data.Header.OriginalTranslationMemoryFormat);
Assert.AreEqual(expected.Data.Header.CreationTool, result.Data.Header.CreationTool);
// ---- Assert body
// ------ Assert TranslationUnits
foreach (TranslationUnit expectedTU in expected.Data.Body.TranslationUnits)
{
TranslationUnit unit = result.Data.Body.TranslationUnits
.First <TranslationUnit>(
tu => tu.TranslationUnitId == expectedTU.TranslationUnitId
);
Assert.IsNotNull(unit);
Assert.AreEqual(expectedTU.TranslationUnitId, unit.TranslationUnitId);
// ------ Assert TranslationUnitVariant
foreach (TranslationUnitVariant tuv in unit.TranslationUnitVariants)
{
Assert.AreEqual("en", tuv.Language);
Assert.IsNull(tuv.Segment);
}
}
}
public void NewLanguageCode_Add_Test()
{
// Assign
ITmxContext context = new TmxContext(MockObjects.VALID_LanguageRepositoryPath);
int count = context.Get("la").Count;
TmxFile addFile = MockObjects.NewLanguageCode_la_Add_ContextTmxFile;
// Act
bool result = context.Add(addFile);
// Assert
Assert.IsTrue(result);
List <TmxFile> items = context.Get("la");
Assert.AreEqual(count + 1, items.Count);
Assert.AreEqual(MockObjects.NewLanguageCode_la_Add_ContextTmxFilePath_Result.FullName, addFile.FileInfo.FullName);
}
public void ValidNew_Update_Test()
{
// Assign
TmxContext_Accessor context = new TmxContext_Accessor(MockObjects.VALID_TestFilesPath);
TmxFile newTmx = MockObjects.VALID_en_Add_ContextTmxFile;
bool result = context.Add(newTmx);
Assert.IsTrue(result, "Context Add method must be broken.");
// Act
result = context.Update(newTmx);
// Assert
Assert.IsTrue(result);
Assert.IsFalse(newTmx.isModified);
Assert.IsFalse(newTmx.isDeleted);
Assert.IsTrue(newTmx.isNew);
}
public void Invalid_Delete_Test()
{
// Assign
TmxContext_Accessor context = new TmxContext_Accessor(MockObjects.VALID_TestFilesPath);
int count = context.Get("en").Count;
TmxFile fake = new TmxFile();
Guid id = fake.Id;
// Act
bool result = context.Remove(fake);
// Assert
Assert.IsFalse(result);
Assert.IsFalse(fake.isModified);
Assert.IsFalse(fake.isDeleted);
Assert.IsFalse(fake.isNew);
Assert.AreEqual(count, context.Get("en").Count);
}
internal override bool Add(TmxFile file)
{
try
{
if (file.FileInfo.Directory.Exists && file.FileInfo.Extension == ".tmx" && file.Data.Body != null)
{
Serialize(file);
return(true);
}
else
{
return(false);
}
} catch
{
return(false);
}
}
public void Invalid_Delete_Test()
{
// Assign
TmxContext_Accessor context = new TmxContext_Accessor(MockObjects.VALID_TestFilesPath);
int count = context.Get("en").Count;
TmxFile fake = new TmxFile();
Guid id = fake.Id;
// Act
bool result = context.Remove(fake);
// Assert
Assert.IsFalse(result);
Assert.IsFalse(fake.isModified);
Assert.IsFalse(fake.isDeleted);
Assert.IsFalse(fake.isNew);
Assert.AreEqual(count, context.Get("en").Count);
}
internal override bool Update(TmxFile file)
{
try
{
if (file.FileInfo.Exists && file.Data.Body != null)
{
Serialize(file);
return(true);
}
else
{
return(false);
}
} catch
{
return(false);
}
}
public void Valid_Delete_Test()
{
// Assign
TmxContext_Accessor context = new TmxContext_Accessor(MockObjects.VALID_TestFilesPath);
int count = context.Get("en").Count;
TmxFile current = (from tmx in context.Get("en")
where tmx.FileInfo.FullName == MockObjects.VALID_Delete_TmxFilePath.FullName
select tmx).First <TmxFile>();
Guid id = current.Id;
// Act
bool result = context.Remove(current);
// Assert
Assert.IsTrue(result);
Assert.IsFalse(current.isModified);
Assert.IsTrue(current.isDeleted);
Assert.IsFalse(current.isNew);
Assert.AreEqual(count - 1, context.Get("en").Count);
}
public void Update_Valid_TmxFile_Test()
{
// Assign
AbstractLanguageRepository repository = new LanguageRepository();
TmxFile current = (from tmx in repository.Get(MockObjects.VALID_TestFilesPath)
where tmx.FileInfo.FullName == MockObjects.VALID_Update_TmxFilePath.FullName
select tmx).First <TmxFile>();
// Act
bool result = repository.Update(MockObjects.VALID_Update_TmxFile);
// Assert
Assert.IsTrue(result);
TmxFile updated = (from tmx in repository.Get(MockObjects.VALID_TestFilesPath)
where tmx.FileInfo.FullName == MockObjects.VALID_Update_TmxFilePath.FullName
select tmx).First <TmxFile>();
Assert.AreNotEqual(current, updated);
Assert.AreNotEqual(current.Data.Body.TranslationUnits.Count, updated.Data.Body.TranslationUnits.Count);
}
private void ExtractTMX(string file)
{
string output = txtBox_OutputDir.Text;
if (chkBox_KeepFolderStructure.Checked)
{
output += Path.DirectorySeparatorChar + Path.GetDirectoryName(file).Substring(txtBox_SearchDir.Text.Length);
}
if (chkBox_NameAfterContainers.Checked)
{
output += Path.DirectorySeparatorChar + Path.GetFileName(file) + "_extracted";
}
Directory.CreateDirectory(output);
TmxFile tmx = new TmxFile(file);
var bitmap = tmx.GetBitmap();
bitmap.Save(output + Path.DirectorySeparatorChar + Path.ChangeExtension(Path.GetFileName(file), "png"));
txtBox_Log.Text += $"Saving texture: {Path.GetFileName(file)}\n";
}
internal override TmxFile ParseFile(FileInfo file)
{
if (file.Exists && file.Extension == ".tmx")
{
TmxFile tmxFile = new TmxFile();
// Allocate file information
tmxFile.FileInfo = file;
tmxFile.Id = Guid.NewGuid();
// Assign file contents
tmxFile.Data = Parse(file);
// Assign language code
tmxFile.LanguageCode =
tmxFile.Data.Body.TranslationUnits.Find(tuv => tuv.TranslationUnitVariants.Count > 0) != null
?
tmxFile.Data
.Body
.TranslationUnits
.First <TranslationUnit>(tu => tu.TranslationUnitVariants.Count > 0)
.TranslationUnitVariants
.First <TranslationUnitVariant>()
.Language
:
"Unkown"
;
return(tmxFile);
}
else
{
// Return that file does not exist
return(null);
}
}
internal abstract bool Delete(TmxFile file);
abstract internal bool Update(TmxFile file);
public InputTmxFile(TmxFile tmxFile)
{
TmxFile = tmxFile;
}
abstract internal bool Delete(TmxFile file);
internal abstract bool Update(TmxFile file);
public ExportStudioTmStep(FileBasedTranslationMemory translationMemory, TmxFile exportTmxFile)
: base(String.Format("Exporting translation memory {0}", translationMemory.Name))
{
_translationMemory = translationMemory;
_exportTmxFile = exportTmxFile;
}
internal abstract bool Add(TmxFile file);
/// <summary>
/// The execute method.
/// </summary>
/// <returns>
/// The <see cref="bool" />.
/// </returns>
public override bool Execute()
{
string SntFileName = Path.GetTempPath() + "_TmxUpload.snt";
string uservalue = user.ValueString;
if (uservalue == string.Empty)
{
uservalue = "TmxUpload";
}
string ratingvalue = rating.ValueString;
if (ratingvalue == string.Empty)
{
ratingvalue = "6";
}
TmxFile TmxIn = new TmxFile(this.TmxDocument.ValueString);
string[] sntFilenames = TmxIn.WriteToSNTFiles(SntFileName);
if (sntFilenames.Length != 2)
{
Logger.WriteLine(LogLevel.Error, "More than 2 languages in the TMX file. Must have exactly 2.");
deleteSNTfiles(sntFilenames);
return(false);
}
TranslationMemory TM = new TranslationMemory();
TM.sourceLangID = this.sourceLanguage.ValueString.ToLowerInvariant();
TM.targetLangID = this.targetLanguage.ValueString.ToLowerInvariant();
// Read langauge names from Tmx
string TmxSourceLanguage = Path.GetFileNameWithoutExtension(sntFilenames[0]);
TmxSourceLanguage = TmxSourceLanguage.Substring(TmxSourceLanguage.LastIndexOf('_') + 1).ToLowerInvariant();
string TmxTargetLanguage = Path.GetFileNameWithoutExtension(sntFilenames[1]);
TmxTargetLanguage = TmxTargetLanguage.Substring(TmxTargetLanguage.LastIndexOf('_') + 1).ToLowerInvariant();
if (TmxSourceLanguage.Substring(0, 2) != TM.sourceLangID)
{
Logger.WriteLine(LogLevel.Error, "Source language mismatch between command line {0} and TMX language {1}. Please edit TmxLangMap.csv to fix. Aborting.", TM.sourceLangID, TmxSourceLanguage);
deleteSNTfiles(sntFilenames);
return(false);
}
if (TmxTargetLanguage.Substring(0, 2) != TM.targetLangID)
{
Logger.WriteLine(LogLevel.Error, "Target language mismatch between command line {0} and TMX language {1}. Please edit TmxLangMap.csv to fix. Aborting.", TM.targetLangID, TmxTargetLanguage);
deleteSNTfiles(sntFilenames);
return(false);
}
string[] sntSource = File.ReadAllLines(sntFilenames[0]);
string[] sntTarget = File.ReadAllLines(sntFilenames[1]);
if (sntSource.Length != sntTarget.Length)
{
Logger.WriteLine(LogLevel.Error, "Unequal number of segments. The TMX must have the same number of segments in the two given languages.");
deleteSNTfiles(sntFilenames);
return(false);
}
Logger.WriteLine(LogLevel.None, "{0} translation units read.", sntSource.Length);
TmxWriter ErrorTmx = new TmxWriter(Path.GetFileNameWithoutExtension(this.TmxDocument.ValueString) + ".errors." + TmxSourceLanguage + "_" + TmxTargetLanguage + "." + DateTime.Now.ToString("yyyyMMddThhmmssZ") + ".tmx", TmxSourceLanguage, TmxTargetLanguage);
//Load into TM and perform error check on each line.
int ratioViolationCount = 0; //counts number of ratio violations
int sntCountViolationCount = 0; //counts number of unequal sentence count violation.
for (int sntLineIndex = 0; sntLineIndex < sntSource.Length; sntLineIndex++)
{
//show a progress message.
if ((sntLineIndex % 10) == 0)
{
Logger.WriteLine(LogLevel.Debug, "{0} of {1} sentences aligned and error checked.", sntLineIndex, sntSource.Length);
}
//Length discrepancy check
float ratio = Math.Abs(sntSource[sntLineIndex].Length / sntTarget[sntLineIndex].Length);
if ((ratio > 3) && ((sntSource.Length > 15) || (sntTarget.Length > 15))) //skip the segment, and add to error.tmx
{
Logger.WriteLine(LogLevel.Debug, "Length ratio exceeded. Segment skipped: {0}", sntSource[sntLineIndex].Substring(0, (60 < sntSource[sntLineIndex].Length)?60:sntSource[sntLineIndex].Length));
ratioViolationCount++;
ErrorTmx.TmxWriteSegment(sntSource[sntLineIndex], sntTarget[sntLineIndex], TmxSourceLanguage, TmxTargetLanguage, TmxWriter.TUError.lengthratio);
if ((ratioViolationCount / sntSource.Length) > 0.10)
{
Logger.WriteLine(LogLevel.Error, "Length ratio exceeded for 10% of segments. Probably not a translation. Aborting.");
deleteSNTfiles(sntFilenames);
return(false);
}
continue;
}
//TODO: special handling of bpt/ept
sntSource[sntLineIndex] = System.Net.WebUtility.HtmlDecode(sntSource[sntLineIndex]);
sntTarget[sntLineIndex] = System.Net.WebUtility.HtmlDecode(sntTarget[sntLineIndex]);
//throw away segments with tags
if ((sntSource[sntLineIndex].Contains("<") && sntSource[sntLineIndex].Contains(">")) && (sntTarget[sntLineIndex].Contains("<") && sntTarget[sntLineIndex].Contains(">")))
{
Logger.WriteLine(LogLevel.Debug, "Tagged segment. Segment skipped: {0}", sntSource[sntLineIndex].Substring(0, (60 < sntSource[sntLineIndex].Length) ? 60 : sntSource[sntLineIndex].Length));
ErrorTmx.TmxWriteSegment(sntSource[sntLineIndex], sntTarget[sntLineIndex], TmxSourceLanguage, TmxTargetLanguage, TmxWriter.TUError.tagging);
continue;
}
//Encode the remaining <>&
sntSource[sntLineIndex] = System.Net.WebUtility.HtmlEncode(sntSource[sntLineIndex]);
sntTarget[sntLineIndex] = System.Net.WebUtility.HtmlEncode(sntTarget[sntLineIndex]);
int[] sourceSentLengths = TranslationServiceFacade.BreakSentences(sntSource[sntLineIndex], TM.sourceLangID);
int[] targetSentLengths = TranslationServiceFacade.BreakSentences(sntTarget[sntLineIndex], TM.targetLangID);
//unequal sentence count violation check
if (sourceSentLengths.Length != targetSentLengths.Length)
{
sntCountViolationCount++;
Logger.WriteLine(LogLevel.Debug, "Unequal number of sentences in segment. Segment skipped: {0}", sntSource[sntLineIndex].Substring(0, (60 < sntSource[sntLineIndex].Length)?60:sntSource[sntLineIndex].Length));
ErrorTmx.TmxWriteSegment(sntSource[sntLineIndex], sntTarget[sntLineIndex], TmxSourceLanguage, TmxTargetLanguage, TmxWriter.TUError.sentencecountmismatch);
if ((sntCountViolationCount / sntSource.Length) > 0.10)
{
Logger.WriteLine(LogLevel.Error, "Unequal sentence count exceeded for 10% of segments. Probably not a translation. Aborting.");
deleteSNTfiles(sntFilenames);
return(false);
}
continue;
}
//Split multiple sentences
int startIndexSrc = 0;
int startIndexTgt = 0;
for (int j = 0; j < sourceSentLengths.Length; j++)
{
TranslationUnit TU = new TranslationUnit();
TU.strSource = sntSource[sntLineIndex].Substring(startIndexSrc, sourceSentLengths[j]);
TU.strTarget = sntTarget[sntLineIndex].Substring(startIndexTgt, targetSentLengths[j]);
startIndexSrc = sourceSentLengths[j];
startIndexTgt = targetSentLengths[j];
TU.rating = int.Parse(ratingvalue);
TU.user = uservalue.ToUpperInvariant();
TM.Add(TU);
}
}
ErrorTmx.Dispose();
//Add the whole TM list to CTF, if a CTF write was requested.
if (boolWrite.ValueString.ToLowerInvariant() == "true")
{
int SentenceCount = 0;
foreach (TranslationUnit TU in TM)
{
TranslationServiceFacade.AddTranslation(TU.strSource, TU.strTarget, TM.sourceLangID, TM.targetLangID, TU.rating, TU.user);
if ((SentenceCount % 10) == 0)
{
Logger.WriteLine(LogLevel.Debug, "{0} of {1} sentences written. Continuing...", SentenceCount, sntSource.Length);
}
//Do not change the sleep time. This is slow and needs to be slow - the AddTranslation method is designed for interactive use.
Thread.Sleep(500);
SentenceCount++;
}
Logger.WriteLine(LogLevel.Msg, "{0} sentences written to CTF. Write complete. ", SentenceCount);
}
else
{
//Just list the entire TM on screen.
foreach (TranslationUnit TU in TM)
{
Logger.WriteLine(LogLevel.None, "{0} || {1}", TU.strSource, TU.strTarget);
}
}
return(true);
}
/// <summary>
/// The execute method.
/// </summary>
/// <returns>
/// The <see cref="bool" />.
/// </returns>
public override bool Execute()
{
string SntFileName = Path.GetTempPath() + "_TmxUpload.snt";
string uservalue = user.ValueString;
if (uservalue == string.Empty)
{
uservalue = "TmxUpload";
}
string ratingvalue = rating.ValueString;
if (ratingvalue == string.Empty)
{
ratingvalue = "6";
}
if (!File.Exists(TmxDocument.ValueString))
{
Logger.WriteLine(LogLevel.Error, "File {0} does not exist.", TmxDocument.ValueString);
return(false);
}
TmxFile TmxIn = new TmxFile(this.TmxDocument.ValueString);
string[] sntFilenames = TmxIn.WriteToSNTFiles(SntFileName);
if (sntFilenames != null)
{
if (sntFilenames.Length != 2)
{
Logger.WriteLine(LogLevel.Error, "More than 2 languages in the TMX file. Must have exactly 2.");
deleteSNTfiles(sntFilenames);
return(false);
}
}
else
{
Logger.WriteLine(LogLevel.Error, "Not a TMX file. Aborting.");
return(false);
}
TranslationMemory TM = new TranslationMemory();
TM.sourceLangID = this.sourceLanguage.ValueString.ToLowerInvariant();
TM.targetLangID = this.targetLanguage.ValueString.ToLowerInvariant();
// Read language names from Tmx
string TmxSourceLanguage = Path.GetFileNameWithoutExtension(sntFilenames[0]);
TmxSourceLanguage = TmxSourceLanguage.Substring(TmxSourceLanguage.LastIndexOf('_') + 1).ToLowerInvariant();
string TmxTargetLanguage = Path.GetFileNameWithoutExtension(sntFilenames[1]);
TmxTargetLanguage = TmxTargetLanguage.Substring(TmxTargetLanguage.LastIndexOf('_') + 1).ToLowerInvariant();
if (TmxSourceLanguage.Substring(0, 2) != TM.sourceLangID)
{
Logger.WriteLine(LogLevel.Error, "Source language mismatch between command line {0} and TMX language {1}. Please edit TmxLangMap.csv to fix. Aborting.", TM.sourceLangID, TmxSourceLanguage);
deleteSNTfiles(sntFilenames);
return(false);
}
if (TmxTargetLanguage.Substring(0, 2) != TM.targetLangID)
{
Logger.WriteLine(LogLevel.Error, "Target language mismatch between command line {0} and TMX language {1}. Please edit TmxLangMap.csv to fix. Aborting.", TM.targetLangID, TmxTargetLanguage);
deleteSNTfiles(sntFilenames);
return(false);
}
TranslationMemory TMErrors = new TranslationMemory();
TMErrors.sourceLangID = this.sourceLanguage.ValueString.ToLowerInvariant();
TMErrors.targetLangID = this.targetLanguage.ValueString.ToLowerInvariant();
string[] sntSource = File.ReadAllLines(sntFilenames[0]);
string[] sntTarget = File.ReadAllLines(sntFilenames[1]);
if (sntSource.Length != sntTarget.Length)
{
Logger.WriteLine(LogLevel.Error, "Unequal number of segments. The TMX must have the same number of segments in the two given languages.");
deleteSNTfiles(sntFilenames);
return(false);
}
TmxWriter ErrorTmx = new TmxWriter(Path.GetFileNameWithoutExtension(this.TmxDocument.ValueString) + ".errors." + TmxSourceLanguage + "_" + TmxTargetLanguage + "." + DateTime.Now.ToString("yyyyMMddThhmmssZ") + ".tmx", TmxSourceLanguage, TmxTargetLanguage, false);
//Load into TM without error checks. The AddTranslationSegments method performs error checks.
for (int sntLineIndex = 0; sntLineIndex < sntSource.Length; sntLineIndex++)
{
TranslationUnit TU = new TranslationUnit(sntSource[sntLineIndex], sntTarget[sntLineIndex], Convert.ToInt16(ratingvalue), uservalue, "", TUStatus.good, "");
TM.Add(TU);
}
Logger.WriteLine(LogLevel.None, "{0} translation units read.", sntSource.Length);
bool AddToCTF = false;
if (boolWrite.ValueString.ToLowerInvariant() == "true")
{
AddToCTF = true;
}
TMFunctions.AddTranslationSegmentsResponse ATSResponse = TMFunctions.AddTranslationSegments(TM, TMErrors, TM.sourceLangID, TM.targetLangID, Convert.ToInt16(ratingvalue), uservalue, AddToCTF);
Logger.WriteLine(LogLevel.Msg, "Error segments: {0}", ATSResponse.errorsegments);
if (AddToCTF)
{
Logger.WriteLine(LogLevel.Msg, "{0} sentences written to CTF. Write complete. ", ATSResponse.sentencesadded);
}
else
{
Logger.WriteLine(LogLevel.None, "Nothing written to CTF.");
}
return(true);
}
private void Serialize(TmxFile file)
{
XmlSerializer serializer = new XmlSerializer(typeof(Tmx));
using (StreamWriter writer = new StreamWriter(file.FileInfo.FullName))
{
using (XmlWriter xwriter = XmlWriter.Create(writer, new XmlWriterSettings() { Indent = true, Encoding = Encoding.UTF8 }))
{
xwriter.WriteDocType("tmx", "SYSTEM", "tmx14.dtd", null);
serializer.Serialize(xwriter, file.Data);
}
}
}
/// <summary>
/// Update a tmx file
/// </summary>
/// <param name="tmxFile">Tmx file to update</param>
/// <returns>Sucessful</returns>
public bool Update(TmxFile tmxFile)
{
foreach (List<TmxFile> files in History)
{
for (int i = 0; i < files.Count; i++)
{
if (files[i].Id == tmxFile.Id)
{
// Set modifiers
tmxFile.isModified = tmxFile.isDeleted || tmxFile.isNew ? false : true;
// Replace file in history
files[i] = tmxFile;
return true;
}
}
}
return false;
}
/// <summary>
/// The execute method.
/// </summary>
/// <returns>
/// The <see cref="bool" />.
/// </returns>
public override bool Execute()
{
string SntFileName = Path.GetTempPath() + "_TmxUpload.snt";
string uservalue = user.ValueString;
if (uservalue == string.Empty) uservalue = "TmxUpload";
string ratingvalue = rating.ValueString;
if (ratingvalue == string.Empty) ratingvalue = "6";
TmxFile TmxIn = new TmxFile(this.TmxDocument.ValueString);
string[] sntFilenames = TmxIn.WriteToSNTFiles(SntFileName);
if (sntFilenames.Length != 2) {
Logger.WriteLine(LogLevel.Error, "More than 2 languages in the TMX file. Must have exactly 2.");
deleteSNTfiles(sntFilenames);
return false;
}
TranslationMemory TM = new TranslationMemory();
TM.sourceLangID = this.sourceLanguage.ValueString.ToLowerInvariant();
TM.targetLangID = this.targetLanguage.ValueString.ToLowerInvariant();
// Read langauge names from Tmx
string TmxSourceLanguage = Path.GetFileNameWithoutExtension(sntFilenames[0]);
TmxSourceLanguage = TmxSourceLanguage.Substring(TmxSourceLanguage.LastIndexOf('_') + 1).ToLowerInvariant();
string TmxTargetLanguage = Path.GetFileNameWithoutExtension(sntFilenames[1]);
TmxTargetLanguage = TmxTargetLanguage.Substring(TmxTargetLanguage.LastIndexOf('_') + 1).ToLowerInvariant();
if (TmxSourceLanguage.Substring(0, 2) != TM.sourceLangID)
{
Logger.WriteLine(LogLevel.Error, "Source language mismatch between command line {0} and TMX language {1}. Please edit TmxLangMap.csv to fix. Aborting.", TM.sourceLangID, TmxSourceLanguage);
deleteSNTfiles(sntFilenames);
return false;
}
if (TmxTargetLanguage.Substring(0, 2) != TM.targetLangID)
{
Logger.WriteLine(LogLevel.Error, "Target language mismatch between command line {0} and TMX language {1}. Please edit TmxLangMap.csv to fix. Aborting.", TM.targetLangID, TmxTargetLanguage);
deleteSNTfiles(sntFilenames);
return false;
}
string[] sntSource = File.ReadAllLines(sntFilenames[0]);
string[] sntTarget = File.ReadAllLines(sntFilenames[1]);
if (sntSource.Length != sntTarget.Length){
Logger.WriteLine(LogLevel.Error, "Unequal number of segments. The TMX must have the same number of segments in the two given languages.");
deleteSNTfiles(sntFilenames);
return false;
}
Logger.WriteLine(LogLevel.None, "{0} translation units read.", sntSource.Length);
TmxWriter ErrorTmx = new TmxWriter(Path.GetFileNameWithoutExtension(this.TmxDocument.ValueString) + ".errors." + TmxSourceLanguage + "_" + TmxTargetLanguage + "." + DateTime.Now.ToString("yyyyMMddThhmmssZ") + ".tmx", TmxSourceLanguage, TmxTargetLanguage);
//Load into TM and perform error check on each line.
int ratioViolationCount = 0; //counts number of ratio violations
int sntCountViolationCount = 0; //counts number of unequal sentence count violation.
for (int sntLineIndex = 0; sntLineIndex < sntSource.Length; sntLineIndex++)
{
//show a progress message.
if ((sntLineIndex % 10) == 0) Logger.WriteLine(LogLevel.Debug, "{0} of {1} sentences aligned and error checked.", sntLineIndex, sntSource.Length);
//Length discrepancy check
float ratio = Math.Abs(sntSource[sntLineIndex].Length / sntTarget[sntLineIndex].Length);
if ((ratio > 3) && ((sntSource.Length > 15) || (sntTarget.Length > 15))) //skip the segment, and add to error.tmx
{
Logger.WriteLine(LogLevel.Debug, "Length ratio exceeded. Segment skipped: {0}", sntSource[sntLineIndex].Substring(0, (60<sntSource[sntLineIndex].Length)?60:sntSource[sntLineIndex].Length));
ratioViolationCount++;
ErrorTmx.TmxWriteSegment(sntSource[sntLineIndex], sntTarget[sntLineIndex], TmxSourceLanguage, TmxTargetLanguage, TmxWriter.TUError.lengthratio);
if ((ratioViolationCount / sntSource.Length) > 0.10)
{
Logger.WriteLine(LogLevel.Error, "Length ratio exceeded for 10% of segments. Probably not a translation. Aborting.");
deleteSNTfiles(sntFilenames);
return false;
}
continue;
}
//TODO: special handling of bpt/ept
sntSource[sntLineIndex] = System.Net.WebUtility.HtmlDecode(sntSource[sntLineIndex]);
sntTarget[sntLineIndex] = System.Net.WebUtility.HtmlDecode(sntTarget[sntLineIndex]);
//throw away segments with tags
if ((sntSource[sntLineIndex].Contains("<") && sntSource[sntLineIndex].Contains(">")) && (sntTarget[sntLineIndex].Contains("<") && sntTarget[sntLineIndex].Contains(">")))
{
Logger.WriteLine(LogLevel.Debug, "Tagged segment. Segment skipped: {0}", sntSource[sntLineIndex].Substring(0, (60 < sntSource[sntLineIndex].Length) ? 60 : sntSource[sntLineIndex].Length));
ErrorTmx.TmxWriteSegment(sntSource[sntLineIndex], sntTarget[sntLineIndex], TmxSourceLanguage, TmxTargetLanguage, TmxWriter.TUError.tagging);
continue;
}
//Encode the remaining <>&
sntSource[sntLineIndex] = System.Net.WebUtility.HtmlEncode(sntSource[sntLineIndex]);
sntTarget[sntLineIndex] = System.Net.WebUtility.HtmlEncode(sntTarget[sntLineIndex]);
int[] sourceSentLengths = TranslationServiceFacade.BreakSentences(sntSource[sntLineIndex], TM.sourceLangID);
int[] targetSentLengths = TranslationServiceFacade.BreakSentences(sntTarget[sntLineIndex], TM.targetLangID);
//unequal sentence count violation check
if (sourceSentLengths.Length != targetSentLengths.Length)
{
sntCountViolationCount++;
Logger.WriteLine(LogLevel.Debug, "Unequal number of sentences in segment. Segment skipped: {0}", sntSource[sntLineIndex].Substring(0, (60<sntSource[sntLineIndex].Length)?60:sntSource[sntLineIndex].Length));
ErrorTmx.TmxWriteSegment(sntSource[sntLineIndex], sntTarget[sntLineIndex], TmxSourceLanguage, TmxTargetLanguage, TmxWriter.TUError.sentencecountmismatch);
if ((sntCountViolationCount / sntSource.Length) > 0.10)
{
Logger.WriteLine(LogLevel.Error, "Unequal sentence count exceeded for 10% of segments. Probably not a translation. Aborting.");
deleteSNTfiles(sntFilenames);
return false;
}
continue;
}
//Split multiple sentences
int startIndexSrc = 0;
int startIndexTgt = 0;
for (int j = 0; j < sourceSentLengths.Length; j++ )
{
TranslationUnit TU = new TranslationUnit();
TU.strSource = sntSource[sntLineIndex].Substring(startIndexSrc, sourceSentLengths[j]);
TU.strTarget = sntTarget[sntLineIndex].Substring(startIndexTgt, targetSentLengths[j]);
startIndexSrc = sourceSentLengths[j];
startIndexTgt = targetSentLengths[j];
TU.rating = int.Parse(ratingvalue);
TU.user = uservalue.ToUpperInvariant();
TM.Add(TU);
}
}
ErrorTmx.Dispose();
//Add the whole TM list to CTF, if a CTF write was requested.
if (boolWrite.ValueString.ToLowerInvariant() == "true")
{
int SentenceCount = 0;
foreach (TranslationUnit TU in TM){
TranslationServiceFacade.AddTranslation(TU.strSource, TU.strTarget, TM.sourceLangID, TM.targetLangID, TU.rating, TU.user);
if ((SentenceCount % 10) == 0) Logger.WriteLine(LogLevel.Debug, "{0} of {1} sentences written. Continuing...", SentenceCount, sntSource.Length);
//Do not change the sleep time. This is slow and needs to be slow - the AddTranslation method is designed for interactive use.
Thread.Sleep(500);
SentenceCount++;
}
Logger.WriteLine(LogLevel.Msg, "{0} sentences written to CTF. Write complete. ", SentenceCount);
}
else
{
//Just list the entire TM on screen.
foreach (TranslationUnit TU in TM)
{
Logger.WriteLine(LogLevel.None, "{0} || {1}", TU.strSource, TU.strTarget);
}
}
return true;
}
public StripWorkbenchTusStep(string name, TmxFile inputTmxFile, TmxFile outputTmxFile) : base(String.Format("Identify Workbench TUs in {0}", name))
{
_inputTmxFile = inputTmxFile;
_outputTmxFile = outputTmxFile;
}
