/// <summary> /// The execute method. /// </summary> /// <returns> /// The <see cref="bool" />. /// </returns> public override bool Execute() { string SntFileName = Path.GetTempPath() + "_TmxUpload.snt"; string uservalue = user.ValueString; if (uservalue == string.Empty) uservalue = "TmxUpload"; string ratingvalue = rating.ValueString; if (ratingvalue == string.Empty) ratingvalue = "6"; TmxFile TmxIn = new TmxFile(this.TmxDocument.ValueString); string[] sntFilenames = TmxIn.WriteToSNTFiles(SntFileName); if (sntFilenames.Length != 2) { Logger.WriteLine(LogLevel.Error, "More than 2 languages in the TMX file. Must have exactly 2."); deleteSNTfiles(sntFilenames); return false; } TranslationMemory TM = new TranslationMemory(); TM.sourceLangID = this.sourceLanguage.ValueString.ToLowerInvariant(); TM.targetLangID = this.targetLanguage.ValueString.ToLowerInvariant(); // Read langauge names from Tmx string TmxSourceLanguage = Path.GetFileNameWithoutExtension(sntFilenames[0]); TmxSourceLanguage = TmxSourceLanguage.Substring(TmxSourceLanguage.LastIndexOf('_') + 1).ToLowerInvariant(); string TmxTargetLanguage = Path.GetFileNameWithoutExtension(sntFilenames[1]); TmxTargetLanguage = TmxTargetLanguage.Substring(TmxTargetLanguage.LastIndexOf('_') + 1).ToLowerInvariant(); if (TmxSourceLanguage.Substring(0, 2) != TM.sourceLangID) { Logger.WriteLine(LogLevel.Error, "Source language mismatch between command line {0} and TMX language {1}. Please edit TmxLangMap.csv to fix. Aborting.", TM.sourceLangID, TmxSourceLanguage); deleteSNTfiles(sntFilenames); return false; } if (TmxTargetLanguage.Substring(0, 2) != TM.targetLangID) { Logger.WriteLine(LogLevel.Error, "Target language mismatch between command line {0} and TMX language {1}. Please edit TmxLangMap.csv to fix. Aborting.", TM.targetLangID, TmxTargetLanguage); deleteSNTfiles(sntFilenames); return false; } string[] sntSource = File.ReadAllLines(sntFilenames[0]); string[] sntTarget = File.ReadAllLines(sntFilenames[1]); if (sntSource.Length != sntTarget.Length){ Logger.WriteLine(LogLevel.Error, "Unequal number of segments. The TMX must have the same number of segments in the two given languages."); deleteSNTfiles(sntFilenames); return false; } Logger.WriteLine(LogLevel.None, "{0} translation units read.", sntSource.Length); TmxWriter ErrorTmx = new TmxWriter(Path.GetFileNameWithoutExtension(this.TmxDocument.ValueString) + ".errors." + TmxSourceLanguage + "_" + TmxTargetLanguage + "." + DateTime.Now.ToString("yyyyMMddThhmmssZ") + ".tmx", TmxSourceLanguage, TmxTargetLanguage); //Load into TM and perform error check on each line. int ratioViolationCount = 0; //counts number of ratio violations int sntCountViolationCount = 0; //counts number of unequal sentence count violation. for (int sntLineIndex = 0; sntLineIndex < sntSource.Length; sntLineIndex++) { //show a progress message. if ((sntLineIndex % 10) == 0) Logger.WriteLine(LogLevel.Debug, "{0} of {1} sentences aligned and error checked.", sntLineIndex, sntSource.Length); //Length discrepancy check float ratio = Math.Abs(sntSource[sntLineIndex].Length / sntTarget[sntLineIndex].Length); if ((ratio > 3) && ((sntSource.Length > 15) || (sntTarget.Length > 15))) //skip the segment, and add to error.tmx { Logger.WriteLine(LogLevel.Debug, "Length ratio exceeded. Segment skipped: {0}", sntSource[sntLineIndex].Substring(0, (60<sntSource[sntLineIndex].Length)?60:sntSource[sntLineIndex].Length)); ratioViolationCount++; ErrorTmx.TmxWriteSegment(sntSource[sntLineIndex], sntTarget[sntLineIndex], TmxSourceLanguage, TmxTargetLanguage, TmxWriter.TUError.lengthratio); if ((ratioViolationCount / sntSource.Length) > 0.10) { Logger.WriteLine(LogLevel.Error, "Length ratio exceeded for 10% of segments. Probably not a translation. Aborting."); deleteSNTfiles(sntFilenames); return false; } continue; } //TODO: special handling of bpt/ept sntSource[sntLineIndex] = System.Net.WebUtility.HtmlDecode(sntSource[sntLineIndex]); sntTarget[sntLineIndex] = System.Net.WebUtility.HtmlDecode(sntTarget[sntLineIndex]); //throw away segments with tags if ((sntSource[sntLineIndex].Contains("<") && sntSource[sntLineIndex].Contains(">")) && (sntTarget[sntLineIndex].Contains("<") && sntTarget[sntLineIndex].Contains(">"))) { Logger.WriteLine(LogLevel.Debug, "Tagged segment. Segment skipped: {0}", sntSource[sntLineIndex].Substring(0, (60 < sntSource[sntLineIndex].Length) ? 60 : sntSource[sntLineIndex].Length)); ErrorTmx.TmxWriteSegment(sntSource[sntLineIndex], sntTarget[sntLineIndex], TmxSourceLanguage, TmxTargetLanguage, TmxWriter.TUError.tagging); continue; } //Encode the remaining <>& sntSource[sntLineIndex] = System.Net.WebUtility.HtmlEncode(sntSource[sntLineIndex]); sntTarget[sntLineIndex] = System.Net.WebUtility.HtmlEncode(sntTarget[sntLineIndex]); int[] sourceSentLengths = TranslationServiceFacade.BreakSentences(sntSource[sntLineIndex], TM.sourceLangID); int[] targetSentLengths = TranslationServiceFacade.BreakSentences(sntTarget[sntLineIndex], TM.targetLangID); //unequal sentence count violation check if (sourceSentLengths.Length != targetSentLengths.Length) { sntCountViolationCount++; Logger.WriteLine(LogLevel.Debug, "Unequal number of sentences in segment. Segment skipped: {0}", sntSource[sntLineIndex].Substring(0, (60<sntSource[sntLineIndex].Length)?60:sntSource[sntLineIndex].Length)); ErrorTmx.TmxWriteSegment(sntSource[sntLineIndex], sntTarget[sntLineIndex], TmxSourceLanguage, TmxTargetLanguage, TmxWriter.TUError.sentencecountmismatch); if ((sntCountViolationCount / sntSource.Length) > 0.10) { Logger.WriteLine(LogLevel.Error, "Unequal sentence count exceeded for 10% of segments. Probably not a translation. Aborting."); deleteSNTfiles(sntFilenames); return false; } continue; } //Split multiple sentences int startIndexSrc = 0; int startIndexTgt = 0; for (int j = 0; j < sourceSentLengths.Length; j++ ) { TranslationUnit TU = new TranslationUnit(); TU.strSource = sntSource[sntLineIndex].Substring(startIndexSrc, sourceSentLengths[j]); TU.strTarget = sntTarget[sntLineIndex].Substring(startIndexTgt, targetSentLengths[j]); startIndexSrc = sourceSentLengths[j]; startIndexTgt = targetSentLengths[j]; TU.rating = int.Parse(ratingvalue); TU.user = uservalue.ToUpperInvariant(); TM.Add(TU); } } ErrorTmx.Dispose(); //Add the whole TM list to CTF, if a CTF write was requested. if (boolWrite.ValueString.ToLowerInvariant() == "true") { int SentenceCount = 0; foreach (TranslationUnit TU in TM){ TranslationServiceFacade.AddTranslation(TU.strSource, TU.strTarget, TM.sourceLangID, TM.targetLangID, TU.rating, TU.user); if ((SentenceCount % 10) == 0) Logger.WriteLine(LogLevel.Debug, "{0} of {1} sentences written. Continuing...", SentenceCount, sntSource.Length); //Do not change the sleep time. This is slow and needs to be slow - the AddTranslation method is designed for interactive use. Thread.Sleep(500); SentenceCount++; } Logger.WriteLine(LogLevel.Msg, "{0} sentences written to CTF. Write complete. ", SentenceCount); } else { //Just list the entire TM on screen. foreach (TranslationUnit TU in TM) { Logger.WriteLine(LogLevel.None, "{0} || {1}", TU.strSource, TU.strTarget); } } return true; }
/// <summary> /// The execute method. /// </summary> /// <returns> /// The <see cref="bool" />. /// </returns> public override bool Execute() { string SntFileName = Path.GetTempPath() + "_TmxUpload.snt"; string uservalue = user.ValueString; if (uservalue == string.Empty) { uservalue = "TmxUpload"; } string ratingvalue = rating.ValueString; if (ratingvalue == string.Empty) { ratingvalue = "6"; } TmxFile TmxIn = new TmxFile(this.TmxDocument.ValueString); string[] sntFilenames = TmxIn.WriteToSNTFiles(SntFileName); if (sntFilenames.Length != 2) { Logger.WriteLine(LogLevel.Error, "More than 2 languages in the TMX file. Must have exactly 2."); deleteSNTfiles(sntFilenames); return(false); } TranslationMemory TM = new TranslationMemory(); TM.sourceLangID = this.sourceLanguage.ValueString.ToLowerInvariant(); TM.targetLangID = this.targetLanguage.ValueString.ToLowerInvariant(); // Read langauge names from Tmx string TmxSourceLanguage = Path.GetFileNameWithoutExtension(sntFilenames[0]); TmxSourceLanguage = TmxSourceLanguage.Substring(TmxSourceLanguage.LastIndexOf('_') + 1).ToLowerInvariant(); string TmxTargetLanguage = Path.GetFileNameWithoutExtension(sntFilenames[1]); TmxTargetLanguage = TmxTargetLanguage.Substring(TmxTargetLanguage.LastIndexOf('_') + 1).ToLowerInvariant(); if (TmxSourceLanguage.Substring(0, 2) != TM.sourceLangID) { Logger.WriteLine(LogLevel.Error, "Source language mismatch between command line {0} and TMX language {1}. Please edit TmxLangMap.csv to fix. Aborting.", TM.sourceLangID, TmxSourceLanguage); deleteSNTfiles(sntFilenames); return(false); } if (TmxTargetLanguage.Substring(0, 2) != TM.targetLangID) { Logger.WriteLine(LogLevel.Error, "Target language mismatch between command line {0} and TMX language {1}. Please edit TmxLangMap.csv to fix. Aborting.", TM.targetLangID, TmxTargetLanguage); deleteSNTfiles(sntFilenames); return(false); } string[] sntSource = File.ReadAllLines(sntFilenames[0]); string[] sntTarget = File.ReadAllLines(sntFilenames[1]); if (sntSource.Length != sntTarget.Length) { Logger.WriteLine(LogLevel.Error, "Unequal number of segments. The TMX must have the same number of segments in the two given languages."); deleteSNTfiles(sntFilenames); return(false); } Logger.WriteLine(LogLevel.None, "{0} translation units read.", sntSource.Length); TmxWriter ErrorTmx = new TmxWriter(Path.GetFileNameWithoutExtension(this.TmxDocument.ValueString) + ".errors." + TmxSourceLanguage + "_" + TmxTargetLanguage + "." + DateTime.Now.ToString("yyyyMMddThhmmssZ") + ".tmx", TmxSourceLanguage, TmxTargetLanguage); //Load into TM and perform error check on each line. int ratioViolationCount = 0; //counts number of ratio violations int sntCountViolationCount = 0; //counts number of unequal sentence count violation. for (int sntLineIndex = 0; sntLineIndex < sntSource.Length; sntLineIndex++) { //show a progress message. if ((sntLineIndex % 10) == 0) { Logger.WriteLine(LogLevel.Debug, "{0} of {1} sentences aligned and error checked.", sntLineIndex, sntSource.Length); } //Length discrepancy check float ratio = Math.Abs(sntSource[sntLineIndex].Length / sntTarget[sntLineIndex].Length); if ((ratio > 3) && ((sntSource.Length > 15) || (sntTarget.Length > 15))) //skip the segment, and add to error.tmx { Logger.WriteLine(LogLevel.Debug, "Length ratio exceeded. Segment skipped: {0}", sntSource[sntLineIndex].Substring(0, (60 < sntSource[sntLineIndex].Length)?60:sntSource[sntLineIndex].Length)); ratioViolationCount++; ErrorTmx.TmxWriteSegment(sntSource[sntLineIndex], sntTarget[sntLineIndex], TmxSourceLanguage, TmxTargetLanguage, TmxWriter.TUError.lengthratio); if ((ratioViolationCount / sntSource.Length) > 0.10) { Logger.WriteLine(LogLevel.Error, "Length ratio exceeded for 10% of segments. Probably not a translation. Aborting."); deleteSNTfiles(sntFilenames); return(false); } continue; } //TODO: special handling of bpt/ept sntSource[sntLineIndex] = System.Net.WebUtility.HtmlDecode(sntSource[sntLineIndex]); sntTarget[sntLineIndex] = System.Net.WebUtility.HtmlDecode(sntTarget[sntLineIndex]); //throw away segments with tags if ((sntSource[sntLineIndex].Contains("<") && sntSource[sntLineIndex].Contains(">")) && (sntTarget[sntLineIndex].Contains("<") && sntTarget[sntLineIndex].Contains(">"))) { Logger.WriteLine(LogLevel.Debug, "Tagged segment. Segment skipped: {0}", sntSource[sntLineIndex].Substring(0, (60 < sntSource[sntLineIndex].Length) ? 60 : sntSource[sntLineIndex].Length)); ErrorTmx.TmxWriteSegment(sntSource[sntLineIndex], sntTarget[sntLineIndex], TmxSourceLanguage, TmxTargetLanguage, TmxWriter.TUError.tagging); continue; } //Encode the remaining <>& sntSource[sntLineIndex] = System.Net.WebUtility.HtmlEncode(sntSource[sntLineIndex]); sntTarget[sntLineIndex] = System.Net.WebUtility.HtmlEncode(sntTarget[sntLineIndex]); int[] sourceSentLengths = TranslationServiceFacade.BreakSentences(sntSource[sntLineIndex], TM.sourceLangID); int[] targetSentLengths = TranslationServiceFacade.BreakSentences(sntTarget[sntLineIndex], TM.targetLangID); //unequal sentence count violation check if (sourceSentLengths.Length != targetSentLengths.Length) { sntCountViolationCount++; Logger.WriteLine(LogLevel.Debug, "Unequal number of sentences in segment. Segment skipped: {0}", sntSource[sntLineIndex].Substring(0, (60 < sntSource[sntLineIndex].Length)?60:sntSource[sntLineIndex].Length)); ErrorTmx.TmxWriteSegment(sntSource[sntLineIndex], sntTarget[sntLineIndex], TmxSourceLanguage, TmxTargetLanguage, TmxWriter.TUError.sentencecountmismatch); if ((sntCountViolationCount / sntSource.Length) > 0.10) { Logger.WriteLine(LogLevel.Error, "Unequal sentence count exceeded for 10% of segments. Probably not a translation. Aborting."); deleteSNTfiles(sntFilenames); return(false); } continue; } //Split multiple sentences int startIndexSrc = 0; int startIndexTgt = 0; for (int j = 0; j < sourceSentLengths.Length; j++) { TranslationUnit TU = new TranslationUnit(); TU.strSource = sntSource[sntLineIndex].Substring(startIndexSrc, sourceSentLengths[j]); TU.strTarget = sntTarget[sntLineIndex].Substring(startIndexTgt, targetSentLengths[j]); startIndexSrc = sourceSentLengths[j]; startIndexTgt = targetSentLengths[j]; TU.rating = int.Parse(ratingvalue); TU.user = uservalue.ToUpperInvariant(); TM.Add(TU); } } ErrorTmx.Dispose(); //Add the whole TM list to CTF, if a CTF write was requested. if (boolWrite.ValueString.ToLowerInvariant() == "true") { int SentenceCount = 0; foreach (TranslationUnit TU in TM) { TranslationServiceFacade.AddTranslation(TU.strSource, TU.strTarget, TM.sourceLangID, TM.targetLangID, TU.rating, TU.user); if ((SentenceCount % 10) == 0) { Logger.WriteLine(LogLevel.Debug, "{0} of {1} sentences written. Continuing...", SentenceCount, sntSource.Length); } //Do not change the sleep time. This is slow and needs to be slow - the AddTranslation method is designed for interactive use. Thread.Sleep(500); SentenceCount++; } Logger.WriteLine(LogLevel.Msg, "{0} sentences written to CTF. Write complete. ", SentenceCount); } else { //Just list the entire TM on screen. foreach (TranslationUnit TU in TM) { Logger.WriteLine(LogLevel.None, "{0} || {1}", TU.strSource, TU.strTarget); } } return(true); }
/// <summary> /// The execute method. /// </summary> /// <returns> /// The <see cref="bool" />. /// </returns> public override bool Execute() { string SntFileName = Path.GetTempPath() + "_TmxUpload.snt"; string uservalue = user.ValueString; if (uservalue == string.Empty) { uservalue = "TmxUpload"; } string ratingvalue = rating.ValueString; if (ratingvalue == string.Empty) { ratingvalue = "6"; } if (!File.Exists(TmxDocument.ValueString)) { Logger.WriteLine(LogLevel.Error, "File {0} does not exist.", TmxDocument.ValueString); return(false); } TmxFile TmxIn = new TmxFile(this.TmxDocument.ValueString); string[] sntFilenames = TmxIn.WriteToSNTFiles(SntFileName); if (sntFilenames != null) { if (sntFilenames.Length != 2) { Logger.WriteLine(LogLevel.Error, "More than 2 languages in the TMX file. Must have exactly 2."); deleteSNTfiles(sntFilenames); return(false); } } else { Logger.WriteLine(LogLevel.Error, "Not a TMX file. Aborting."); return(false); } TranslationMemory TM = new TranslationMemory(); TM.sourceLangID = this.sourceLanguage.ValueString.ToLowerInvariant(); TM.targetLangID = this.targetLanguage.ValueString.ToLowerInvariant(); // Read language names from Tmx string TmxSourceLanguage = Path.GetFileNameWithoutExtension(sntFilenames[0]); TmxSourceLanguage = TmxSourceLanguage.Substring(TmxSourceLanguage.LastIndexOf('_') + 1).ToLowerInvariant(); string TmxTargetLanguage = Path.GetFileNameWithoutExtension(sntFilenames[1]); TmxTargetLanguage = TmxTargetLanguage.Substring(TmxTargetLanguage.LastIndexOf('_') + 1).ToLowerInvariant(); if (TmxSourceLanguage.Substring(0, 2) != TM.sourceLangID) { Logger.WriteLine(LogLevel.Error, "Source language mismatch between command line {0} and TMX language {1}. Please edit TmxLangMap.csv to fix. Aborting.", TM.sourceLangID, TmxSourceLanguage); deleteSNTfiles(sntFilenames); return(false); } if (TmxTargetLanguage.Substring(0, 2) != TM.targetLangID) { Logger.WriteLine(LogLevel.Error, "Target language mismatch between command line {0} and TMX language {1}. Please edit TmxLangMap.csv to fix. Aborting.", TM.targetLangID, TmxTargetLanguage); deleteSNTfiles(sntFilenames); return(false); } TranslationMemory TMErrors = new TranslationMemory(); TMErrors.sourceLangID = this.sourceLanguage.ValueString.ToLowerInvariant(); TMErrors.targetLangID = this.targetLanguage.ValueString.ToLowerInvariant(); string[] sntSource = File.ReadAllLines(sntFilenames[0]); string[] sntTarget = File.ReadAllLines(sntFilenames[1]); if (sntSource.Length != sntTarget.Length) { Logger.WriteLine(LogLevel.Error, "Unequal number of segments. The TMX must have the same number of segments in the two given languages."); deleteSNTfiles(sntFilenames); return(false); } TmxWriter ErrorTmx = new TmxWriter(Path.GetFileNameWithoutExtension(this.TmxDocument.ValueString) + ".errors." + TmxSourceLanguage + "_" + TmxTargetLanguage + "." + DateTime.Now.ToString("yyyyMMddThhmmssZ") + ".tmx", TmxSourceLanguage, TmxTargetLanguage, false); //Load into TM without error checks. The AddTranslationSegments method performs error checks. for (int sntLineIndex = 0; sntLineIndex < sntSource.Length; sntLineIndex++) { TranslationUnit TU = new TranslationUnit(sntSource[sntLineIndex], sntTarget[sntLineIndex], Convert.ToInt16(ratingvalue), uservalue, "", TUStatus.good, ""); TM.Add(TU); } Logger.WriteLine(LogLevel.None, "{0} translation units read.", sntSource.Length); bool AddToCTF = false; if (boolWrite.ValueString.ToLowerInvariant() == "true") { AddToCTF = true; } TMFunctions.AddTranslationSegmentsResponse ATSResponse = TMFunctions.AddTranslationSegments(TM, TMErrors, TM.sourceLangID, TM.targetLangID, Convert.ToInt16(ratingvalue), uservalue, AddToCTF); Logger.WriteLine(LogLevel.Msg, "Error segments: {0}", ATSResponse.errorsegments); if (AddToCTF) { Logger.WriteLine(LogLevel.Msg, "{0} sentences written to CTF. Write complete. ", ATSResponse.sentencesadded); } else { Logger.WriteLine(LogLevel.None, "Nothing written to CTF."); } return(true); }