private void FetchServiceData(string host, string port, string modeltag)
{
StringBuilder connectionResult = new StringBuilder();
try
{
var serviceLanguagePairs = FiskmöMTServiceHelper.ListSupportedLanguages(host, port);
IEnumerable <string> modelTagLanguagePairs;
if (this.LanguagePairs != null)
{
var projectLanguagePairsWithMt = serviceLanguagePairs.Intersect(this.LanguagePairs);
modelTagLanguagePairs = projectLanguagePairsWithMt;
if (projectLanguagePairsWithMt.Count() == 0)
{
connectionResult.Append("No MT models available for the language pairs of the project");
}
else if (this.LanguagePairs.Count == projectLanguagePairsWithMt.Count())
{
connectionResult.Append("MT models available for all the language pairs of the project");
}
else
{
connectionResult.Append($"MT models available for some of the language pairs of the project: {String.Join(", ", projectLanguagePairsWithMt)}");
}
//Get the detailed status for each project language pair
foreach (var pair in this.LanguagePairs)
{
connectionResult.Append(Environment.NewLine);
var sourceTarget = pair.Split('-');
connectionResult.Append(FiskmöMTServiceHelper.CheckModelStatus(host, port, sourceTarget[0], sourceTarget[1], modeltag));
}
}
else
{
//This options is used with the batch task, where there's no easy way of getting
//the project language pairs, so all pairs are assumed.
modelTagLanguagePairs = serviceLanguagePairs;
connectionResult.Append($"MT models available for following language pairs: {String.Join(", ", serviceLanguagePairs)}");
}
//Get a list of model tags that are supported for these language pairs
List <string> modelTags = new List <string>();
foreach (var languagePair in modelTagLanguagePairs)
{
modelTags.AddRange(FiskmöMTServiceHelper.GetLanguagePairModelTags(host, port, languagePair.ToString()));
}
this.NoConnection = false;
Dispatcher.Invoke(() => UpdateModelTags(modelTags, modeltag));
}
catch (Exception ex) when(ex is EndpointNotFoundException || ex is CommunicationObjectFaultedException)
{
connectionResult.Append($"No connection to Fiskmö MT service at {host}:{port}.");
this.NoConnection = true;
}
Dispatcher.Invoke(() => this.ConnectionStatus = connectionResult.ToString());
}
//This function starts translating all segments in the document once the document is opened,
//so that the translator won't have to wait for the translation to finish when opening a segment.
//Note that Studio contains a feature called LookAhead which attempts to do a similar thing, but
//this feature appears to be buggy with TMs etc., so it's better to rely on a custom caching system.
private static void TranslateDocumentSegments(Document doc, LanguageDirection langPair, FiskmoOptions options)
{
var visitor = new FiskmoMarkupDataVisitor();
EditorController editorController = SdlTradosStudio.Application.GetController <EditorController>();
foreach (var segmentPair in doc.SegmentPairs)
{
if (segmentPair.Properties.ConfirmationLevel == Sdl.Core.Globalization.ConfirmationLevel.Unspecified)
{
visitor.Reset();
segmentPair.Source.AcceptVisitor(visitor);
var sourceText = visitor.PlainText;
var sourceCode = langPair.SourceLanguage.CultureInfo.TwoLetterISOLanguageName;
var targetCode = langPair.TargetLanguage.CultureInfo.TwoLetterISOLanguageName;
var langpair = $"{sourceCode}-{targetCode}";
//This will generate the translation and cache it for later use
FiskmöMTServiceHelper.Translate(options, sourceText, sourceCode, targetCode, options.modelTag);
}
}
processedDocuments[langPair].Add(doc);
}
public static string GetTokenCode(FiskmoOptions options)
{
return(FiskmöMTServiceHelper.GetTokenCode(options.mtServiceAddress, options.mtServicePort));
}
public static List <string> GetLanguagePairModelTags(FiskmoOptions options, string languagePair)
{
return(FiskmöMTServiceHelper.GetLanguagePairModelTags(options.mtServiceAddress, options.mtServicePort, languagePair));
}
public override void TaskComplete()
{
var projectInfo = this.Project.GetProjectInfo();
var projectGuid = projectInfo.Id;
var sourceCode = projectInfo.SourceLanguage.CultureInfo.TwoLetterISOLanguageName;
if (settings.AddFiskmoProvider)
{
//Add Fiskmö MT provider to the project
var tpConfig = this.Project.GetTranslationProviderConfiguration();
//Don't add another Fiskmo provider
if (!tpConfig.Entries.Any(x => x.MainTranslationProvider.Uri.Scheme == FiskmoProvider.FiskmoTranslationProviderScheme))
{
var fiskmoRef = new TranslationProviderReference(fiskmoOptions.Uri);
tpConfig.Entries.Add(
new TranslationProviderCascadeEntry(fiskmoRef, false, true, false));
this.Project.UpdateTranslationProviderConfiguration(tpConfig);
}
}
if (settings.Finetune)
{
foreach (var targetLang in projectInfo.TargetLanguages)
{
var targetCode = targetLang.CultureInfo.TwoLetterISOLanguageName;
//Remove duplicates
var uniqueProjectTranslations = this.ProjectTranslations[targetLang].Distinct().ToList();
List <string> uniqueNewSegments = new List <string>();
if (this.settings.BatchTranslate)
{
uniqueNewSegments = this.ProjectNewSegments[targetLang].Distinct().ToList();
}
if (this.settings.ExtractFuzzies)
{
var fuzzies = this.ProcessFuzzies(this.ProjectFuzzies[targetLang]);
uniqueProjectTranslations.AddRange(fuzzies);
}
if (uniqueProjectTranslations.Count < Int32.Parse(FiskmoTpSettings.Default.FinetuningMinSentencePairs))
{
throw new Exception(
$"Not enough sentence pairs for fine-tuning. Found {uniqueProjectTranslations.Count}, minimum is {FiskmoTpSettings.Default.FinetuningMinSentencePairs}");
}
//Send the tuning set to MT service
var result = FiskmöMTServiceHelper.Customize(
this.fiskmoOptions.mtServiceAddress,
this.fiskmoOptions.mtServicePort,
uniqueProjectTranslations,
uniqueNewSegments,
sourceCode,
targetCode,
this.fiskmoOptions.modelTag,
this.settings.IncludePlaceholderTags,
this.settings.IncludeTagPairs);
switch (result)
{
case "fine-tuning already in process":
throw new Exception("MT engine is currently batch translating or fine-tuning, wait for previous job to finish (or cancel it by restarting MT engine).");
default:
break;
}
}
}
//Send the new segments to MT engine for pretranslation.
//If finetuning is selected, the new segments are translated after
//customization finished, so this is only for BatchTranslateOnly
if (settings.BatchTranslate == true && settings.Finetune == false)
{
foreach (var targetLang in projectInfo.TargetLanguages)
{
var targetCode = targetLang.CultureInfo.TwoLetterISOLanguageName;
var uniqueNewSegments = this.ProjectNewSegments[targetLang].Distinct().ToList();
//Send the new segments to MT service
var result = FiskmöMTServiceHelper.PreTranslateBatch(fiskmoOptions.mtServiceAddress, fiskmoOptions.mtServicePort, uniqueNewSegments, sourceCode, targetCode, fiskmoOptions.modelTag);
switch (result)
{
case "batch translation or customization already in process":
throw new Exception("MT engine is currently batch translating or fine-tuning, wait for previous job to finish (or cancel it by restarting MT engine).");
default:
break;
}
}
}
}
private List <SearchResult> GenerateSystemResult(string sourceText, SearchMode mode, Segment segment, string sourceCode, string targetCode)
{
List <SearchResult> systemResults = new List <SearchResult>();
string translatedSentence = FiskmöMTServiceHelper.Translate(this._options, sourceText, sourceCode, targetCode, this._options.modelTag);
if (String.IsNullOrEmpty(translatedSentence))
{
return(systemResults);
}
// Look up the currently selected segment in the collection (normal segment lookup).
if (mode == SearchMode.FullSearch || mode == SearchMode.NormalSearch)
{
Segment translation = new Segment(_languageDirection.TargetCulture);
if (_visitor.Placeholders.Any() || _visitor.TagStarts.Any() || _visitor.TagEnds.Any())
{
var split = Regex.Split(translatedSentence, @"\b(PLACEHOLDER|TAGPAIRSTART ?| ?TAGPAIREND)\b");
//Tag starts and ends must match, so need a stack to keep track of what tags
//have been applied
var tagStack = new Stack <Tag>();
foreach (var part in split)
{
//Remove potential spaces from after TAGPAIRSTARTS and before TAGPAIREND
var normalpart = part.Replace("TAGPAIRSTART ", "TAGPAIRSTART");
normalpart = normalpart.Replace(" TAGPAIREND", "TAGPAIREND");
switch (normalpart)
{
case "PLACEHOLDER":
if (_visitor.Placeholders.Count != 0)
{
translation.Add(_visitor.Placeholders.Dequeue());
}
break;
case "TAGPAIRSTART":
if (_visitor.TagStarts.Count != 0)
{
var startTag = _visitor.TagStarts.Dequeue();
tagStack.Push(startTag);
translation.Add(startTag);
}
break;
case "TAGPAIREND":
if (tagStack.Count != 0)
{
var correspondingStartTag = tagStack.Pop();
var endTag = _visitor.TagEnds[correspondingStartTag.TagID];
translation.Add(endTag);
}
break;
default:
translation.Add(part);
break;
}
}
//Insert missing end tags
foreach (var excessStartTag in tagStack)
{
var nonEndedTagIndex = translation.Elements.IndexOf(excessStartTag);
var endTag = _visitor.TagEnds[excessStartTag.TagID];
translation.Elements.Insert(nonEndedTagIndex + 1, endTag);
}
}
else
{
translation.Add(translatedSentence);
}
systemResults.Add(CreateSearchResult(segment, translation, segment.HasTags, "fiskmö"));
}
return(systemResults);
}
