/// <summary>
/// Create language data file.
/// </summary>
/// <param name="fileName">Language data file name.</param>
/// <param name="domain">Domain.</param>
/// <returns>Errors.</returns>
public ErrorSet CombineDataFile(string fileName, string domain)
{
if (string.IsNullOrEmpty(domain) || string.IsNullOrEmpty(domain.Trim()))
{
domain = DomainItem.GeneralDomain;
}
ErrorSet errorSet = new ErrorSet();
if (domain.Equals(DomainItem.GeneralDomain, StringComparison.OrdinalIgnoreCase))
{
errorSet = EnsureNecessaryData(this._moduleDataSet);
}
else if (this._moduleDataSet.Count == 0)
{
errorSet.Add(new Error(DataCompilerError.DomainDataMissing, domain));
}
if (!errorSet.Contains(ErrorSeverity.MustFix))
{
using (LangDataFile langDataFile = new LangDataFile())
{
// Set file property
FileProperty fileProperty = new FileProperty();
fileProperty.Version = 1;
fileProperty.Build = 0;
fileProperty.LangID = (uint)_language;
langDataFile.FileProperties = fileProperty;
ArrayList sortedDataObjects = new ArrayList();
foreach (KeyValuePair<string, LangDataObject> obj in _moduleDataSet)
{
sortedDataObjects.Add(obj);
}
sortedDataObjects.Sort(new CompareLangDataObject());
// Set data objects
foreach (KeyValuePair<string, LangDataObject> obj in sortedDataObjects)
{
if (obj.Value.Data == null)
{
continue;
}
langDataFile.AddDataObject(obj.Value);
string message = Helper.NeutralFormat("Added {{{0}}} ({1}) data.",
obj.Value.Token.ToString(), obj.Key);
errorSet.Add(new Error(DataCompilerError.CompilingLog, message));
}
// Save as binary file
Helper.EnsureFolderExistForFile(fileName);
langDataFile.Save(fileName);
}
}
return errorSet;
}
/// <summary>
/// Merge compiling error into main error set.
/// </summary>
/// <param name="mainErrorSet">Main error set.</param>
/// <param name="subErrorSet">Sub error set.</param>
/// <param name="dataName">Data name.</param>
private static void MergeDependencyError(ErrorSet mainErrorSet, ErrorSet subErrorSet, string dataName)
{
if (mainErrorSet == null)
{
throw new ArgumentNullException("mainErrorSet");
}
if (subErrorSet == null)
{
throw new ArgumentNullException("subErrorSet");
}
if (string.IsNullOrEmpty(dataName))
{
throw new ArgumentNullException("dataName");
}
if (subErrorSet.Contains(ErrorSeverity.MustFix))
{
mainErrorSet.Add(DataCompilerError.DependenciesNotValid, dataName);
}
foreach (Error error in subErrorSet.Errors)
{
if (error.Severity == ErrorSeverity.MustFix)
{
mainErrorSet.Add(DataCompilerError.CompilingLogWithError, dataName, error.ToString());
}
else if (error.Severity == ErrorSeverity.Warning)
{
mainErrorSet.Add(DataCompilerError.CompilingLogWithWarning, dataName, error.ToString());
}
else if (error.Severity == ErrorSeverity.NoError)
{
mainErrorSet.Add(DataCompilerError.CompilingLogWithDataName, dataName, error.ToString());
}
}
}
/// <summary>
/// Char table compiler.
/// </summary>
/// <param name="outputStream">Output Stream.</param>
/// <returns>ErrorSet.</returns>
private ErrorSet CompileCharTable(Stream outputStream)
{
ErrorSet errorSet = new ErrorSet();
try
{
CharTable charTable = (CharTable)GetObject(RawDataName.CharTable, errorSet);
ChartableValidator charTableValidator = new ChartableValidator();
Microsoft.Tts.Offline.Core.Lexicon lexicon = (Microsoft.Tts.Offline.Core.Lexicon)GetObject(RawDataName.Lexicon, errorSet);
TtsPhoneSet phoneSet = (TtsPhoneSet)GetObject(RawDataName.PhoneSet, errorSet);
if (!errorSet.Contains(ErrorSeverity.MustFix))
{
charTableValidator.Lexicon = lexicon;
charTableValidator.PhoneSet = phoneSet;
charTableValidator.EnsureInitialized();
if (charTable.Language != charTableValidator.Language)
{
throw new InvalidDataException("chartable language should match with lexicon or phoneset");
}
ErrorSet charTableErrors = charTableValidator.Validate(charTable, false, null);
foreach (Error error in charTableErrors.Errors)
{
if (error.Severity == ErrorSeverity.MustFix)
{
errorSet.Add(DataCompilerError.CompilingLogWithError,
RawDataName.CharTable, error.ToString());
}
else
{
errorSet.Add(DataCompilerError.CompilingLogWithWarning,
RawDataName.CharTable, error.ToString());
}
}
errorSet.Merge(CharTableCompiler.Compile(charTable, phoneSet, outputStream));
}
}
catch (XmlException e)
{
errorSet.Add(DataCompilerError.RawDataError, e.Message);
}
return errorSet;
}
public ErrorSet Build(string moduleDataName, Stream outputStream, bool isEnableValidate, string formatGuid)
{
////#region Check arguments
if (string.IsNullOrEmpty(moduleDataName))
{
throw new ArgumentNullException("dataName");
}
if (outputStream == null)
{
throw new ArgumentNullException("outputStream");
}
////#endregion
ErrorSet errorSet = new ErrorSet();
ErrorSet subErrorSet = new ErrorSet();
try
{
switch (moduleDataName)
{
case ModuleDataName.PhoneSet:
TtsPhoneSet phoneSet = (TtsPhoneSet)GetObject(RawDataName.PhoneSet, errorSet);
if (!errorSet.Contains(ErrorSeverity.MustFix))
{
errorSet.Merge(PhoneSetCompiler.Compile(phoneSet, outputStream));
}
break;
case ModuleDataName.BackendPhoneSet:
phoneSet = (TtsPhoneSet)GetObject(RawDataName.BackendPhoneSet, errorSet);
if (!errorSet.Contains(ErrorSeverity.MustFix))
{
errorSet.Merge(PhoneSetCompiler.Compile(phoneSet, outputStream));
}
break;
case ModuleDataName.PosSet:
TtsPosSet posSet = (TtsPosSet)GetObject(RawDataName.PosSet, errorSet);
if (!errorSet.Contains(ErrorSeverity.MustFix))
{
errorSet.Merge(PosSetCompiler.Compile(posSet, outputStream));
}
break;
case ModuleDataName.PosTaggerPos:
LexicalAttributeSchema schema = (LexicalAttributeSchema)GetObject(
RawDataName.LexicalAttributeSchema, subErrorSet);
MergeDependencyError(errorSet, subErrorSet, _schemaFullName);
if (!subErrorSet.Contains(ErrorSeverity.MustFix))
{
TtsPosSet postaggingPosSet = TtsPosSet.LoadPosTaggingPosFromSchema(schema);
errorSet.Merge(PosSetCompiler.CompilePosTaggerPos(postaggingPosSet, outputStream));
}
break;
case ModuleDataName.Lexicon:
errorSet = CompileLexicon(outputStream);
break;
case ModuleDataName.CharTable:
ErrorSet charTableErrorSet = CompileCharTable(outputStream);
if (!isEnableValidate)
{
foreach (Error error in charTableErrorSet.Errors)
{
error.Severity = ErrorSeverity.Warning;
}
}
errorSet.Merge(charTableErrorSet);
break;
case ModuleDataName.SentenceSeparator:
string sentSepDataDir = _dataHandlerList.Datas[RawDataName.SentenceSeparatorDataPath].Path;
Collection<string> compiledSentenceSeparatorFiles = new Collection<string>();
errorSet = SentenceSeparatorCompiler.Compile(sentSepDataDir, outputStream, compiledSentenceSeparatorFiles);
if (errorSet.GetSeverityCount(ErrorSeverity.MustFix) == 0 &&
compiledSentenceSeparatorFiles.Count > 0)
{
errorSet.Add(ReportCompiledFiles("sentence separator", compiledSentenceSeparatorFiles));
}
break;
case ModuleDataName.WordBreaker:
{
System.IO.MemoryStream memStream = new MemoryStream();
string wordBreakerDataDir = _dataHandlerList.Datas[RawDataName.WordBreakerDataPath].Path;
Collection<string> compiledWordBreakerFiles = new Collection<string>();
errorSet = WordBreakerCompiler.Compile(wordBreakerDataDir, outputStream, compiledWordBreakerFiles, formatGuid);
if (errorSet.GetSeverityCount(ErrorSeverity.MustFix) == 0 && compiledWordBreakerFiles.Count > 0)
{
errorSet.Add(ReportCompiledFiles("word breaker", compiledWordBreakerFiles));
}
}
break;
case ModuleDataName.PostWordBreaker:
string postWordBreakerFilePath = _dataHandlerList.Datas[RawDataName.PostWordBreaker].Path;
errorSet = PostWordBreakerCompiler.Compile(postWordBreakerFilePath, outputStream);
break;
case ModuleDataName.ChineseTone:
string chineseToneFilePath = _dataHandlerList.Datas[RawDataName.ChineseTone].Path;
errorSet = ChineseToneCompiler.Compile(chineseToneFilePath, outputStream);
break;
case ModuleDataName.AcronymDisambiguation:
{
string acronymDisambiguationDataDir = _dataHandlerList.Datas[RawDataName.AcronymDisambiguation].Path;
Collection<string> compiledFiles = new Collection<string>();
errorSet = CrfModelCompiler.Compile(acronymDisambiguationDataDir, outputStream, compiledFiles, _language);
if (errorSet.GetSeverityCount(ErrorSeverity.MustFix) == 0 &&
compiledFiles.Count > 0)
{
errorSet.Add(ReportCompiledFiles("AcronymDisambiguation", compiledFiles));
}
}
break;
case ModuleDataName.NEDisambiguation:
{
string strNeDisambiguationDataDir = _dataHandlerList.Datas[RawDataName.NEDisambiguation].Path;
Collection<string> compiledFiles = new Collection<string>();
errorSet = CrfModelCompiler.Compile(strNeDisambiguationDataDir, outputStream, compiledFiles, _language);
if (errorSet.GetSeverityCount(ErrorSeverity.MustFix) == 0 &&
compiledFiles.Count > 0)
{
errorSet.Add(ReportCompiledFiles("NEDisambiguation", compiledFiles));
}
}
break;
case ModuleDataName.SyllabifyRule:
string syllabifyRuleFilePath = _dataHandlerList.Datas[RawDataName.SyllabifyRule].Path;
phoneSet = (TtsPhoneSet)GetObject(RawDataName.PhoneSet, subErrorSet);
MergeDependencyError(errorSet, subErrorSet, RawDataName.PhoneSet);
if (!subErrorSet.Contains(ErrorSeverity.MustFix))
{
errorSet.Merge(SyllabifyRuleCompiler.Compile(syllabifyRuleFilePath,
phoneSet, outputStream));
}
break;
case ModuleDataName.UnitGenerator:
string truncRuleFilePath = _dataHandlerList.Datas[RawDataName.TruncateRule].Path;
phoneSet = (TtsPhoneSet)GetObject(RawDataName.PhoneSet, subErrorSet);
MergeDependencyError(errorSet, subErrorSet, RawDataName.PhoneSet);
if (!subErrorSet.Contains(ErrorSeverity.MustFix))
{
errorSet.Merge(UnitGeneratorDataCompiler.Compile(truncRuleFilePath,
phoneSet, outputStream));
}
break;
case ModuleDataName.PolyphoneRule:
string generalRuleFilePath = _dataHandlerList.Datas[RawDataName.PolyphoneRule].Path;
phoneSet = (TtsPhoneSet)GetObject(RawDataName.PhoneSet, errorSet);
PolyphonyRuleFile polyRuleFile = new PolyphonyRuleFile();
ErrorSet polyErrorSet = polyRuleFile.Load(generalRuleFilePath, phoneSet);
if (!isEnableValidate)
{
foreach (Error error in polyErrorSet.Errors)
{
error.Severity = ErrorSeverity.Warning;
}
}
errorSet.Merge(polyErrorSet);
errorSet.Merge(CompileGeneralRule(generalRuleFilePath, outputStream));
break;
case ModuleDataName.BoundaryPronChangeRule:
generalRuleFilePath = _dataHandlerList.Datas[RawDataName.BoundaryPronChangeRule].Path;
errorSet = CompileGeneralRule(generalRuleFilePath, outputStream);
break;
case ModuleDataName.SentenceDetector:
generalRuleFilePath = _dataHandlerList.Datas[RawDataName.SentenceDetectRule].Path;
RuleFile ruleFile = new RuleFile();
List<string> dupKeys = ruleFile.GetDupKeys(generalRuleFilePath);
if (dupKeys.Count > 0)
{
foreach (string key in dupKeys)
{
errorSet.Add(new Error(DataCompilerError.DuplicateItemKey, key));
}
}
else
{
errorSet = CompileGeneralRule(generalRuleFilePath, outputStream);
}
break;
case ModuleDataName.QuotationMarkTable:
QuotationMarkTable quoteTable = QuotationMarkTable.Read(_dataHandlerList.Datas[RawDataName.QuotationMarkTable].Path);
errorSet = QuotationMarkCompiler.Compile(quoteTable, outputStream);
break;
case ModuleDataName.ParallelStructTable:
schema = (LexicalAttributeSchema)GetObject(
RawDataName.LexicalAttributeSchema, subErrorSet);
MergeDependencyError(errorSet, subErrorSet, _schemaFullName);
if (!subErrorSet.Contains(ErrorSeverity.MustFix))
{
TtsPosSet postaggingPosSet = TtsPosSet.LoadPosTaggingPosFromSchema(schema);
ParallelStructTable parallelStructTable = ParallelStructTable.Read(_dataHandlerList.Datas[RawDataName.ParallelStructTable].Path);
if (postaggingPosSet != null)
{
errorSet = ParallelStructCompiler.Compile(parallelStructTable, postaggingPosSet, outputStream);
}
}
break;
case ModuleDataName.WordFeatureSuffixTable:
schema = (LexicalAttributeSchema)GetObject(
RawDataName.LexicalAttributeSchema, subErrorSet);
MergeDependencyError(errorSet, subErrorSet, _schemaFullName);
if (!subErrorSet.Contains(ErrorSeverity.MustFix))
{
WordFeatureSuffixTable suffixTable = WordFeatureSuffixTable.Read(_dataHandlerList.Datas[RawDataName.WordFeatureSuffixTable].Path);
errorSet = WordFeatureSuffixCompiler.Compile(suffixTable, outputStream);
}
break;
case ModuleDataName.LtsRule:
string ltsRuleDataPath = _dataHandlerList.Datas[RawDataName.LtsRuleDataPath].Path;
errorSet = CompileLtsRule(ltsRuleDataPath, outputStream);
break;
case ModuleDataName.PhoneEventData:
PhoneConverterWrapper pcw = null;
// Check if the language has phone mapping data.
if (_moduleDataSet.ContainsKey(ModuleDataName.PhoneMappingRule))
{
// Check phone mapping binary data dependency.
if (_moduleDataSet[ModuleDataName.PhoneMappingRule].Data != null)
{
pcw = new PhoneConverterWrapper(_language, _moduleDataSet[ModuleDataName.PhoneMappingRule].Data);
}
else
{
errorSet.Add(DataCompilerError.DependenciesNotValid, "Please make sure that PhoneMappingRule has been compiled before PhoneEvent");
}
}
if (errorSet.GetSeverityCount(ErrorSeverity.MustFix) == 0)
{
phoneSet = (TtsPhoneSet)GetObject(RawDataName.PhoneSet, errorSet);
errorSet = PhoneEventCompiler.Compile(phoneSet, pcw, outputStream);
}
break;
case ModuleDataName.PosRule:
string lexicalRuleFilePath = _dataHandlerList.Datas[RawDataName.PosLexicalRule].Path;
string contextualRuleFilePath = _dataHandlerList.Datas[RawDataName.PosContextualRule].Path;
schema = (LexicalAttributeSchema)GetObject(
RawDataName.LexicalAttributeSchema, subErrorSet);
MergeDependencyError(errorSet, subErrorSet, _schemaFullName);
if (!subErrorSet.Contains(ErrorSeverity.MustFix))
{
posSet = TtsPosSet.LoadPosTaggingPosFromSchema(schema);
string posSetFilePath = Helper.GetTempFileName();
posSet.Save(posSetFilePath, Encoding.Unicode);
errorSet.Merge(CompilePosRule(lexicalRuleFilePath, contextualRuleFilePath,
posSetFilePath, outputStream));
File.Delete(posSetFilePath);
}
break;
case ModuleDataName.TnRule:
{
string tnmlRuleFilePath = _dataHandlerList.Datas[moduleDataName].Path;
string schemaFilePath = _dataHandlerList.Datas[RawDataName.LexicalAttributeSchema].Path;
errorSet = CompileTnml(tnmlRuleFilePath, schemaFilePath, outputStream, true);
}
break;
case ModuleDataName.FstNERule:
{
string fstNERuleFilePath = _dataHandlerList.Datas[moduleDataName].Path;
errorSet = CompileFstNE(fstNERuleFilePath, outputStream);
}
break;
case ModuleDataName.CompoundRule:
{
string tnmlRuleFilePath = _dataHandlerList.Datas[moduleDataName].Path;
string schemaFilePath = _dataHandlerList.Datas[RawDataName.LexicalAttributeSchema].Path;
phoneSet = (TtsPhoneSet)GetObject(RawDataName.PhoneSet, errorSet);
ErrorSet compundRuleError = DataFileValidator.ValidateCompoundRule(
_dataHandlerList.Datas[moduleDataName].Path, phoneSet);
if (!isEnableValidate)
{
foreach (Error error in compundRuleError.Errors)
{
error.Severity = ErrorSeverity.Warning;
}
}
errorSet.Merge(compundRuleError);
errorSet.Merge(CompileTnml(tnmlRuleFilePath, schemaFilePath, outputStream, false));
}
break;
case ModuleDataName.PhoneMappingRule:
case ModuleDataName.BackendPhoneMappingRule:
case ModuleDataName.FrontendBackendPhoneMappingRule:
case ModuleDataName.MixLingualPOSConverterData:
{
string tnmlRuleFilePath = _dataHandlerList.Datas[moduleDataName].Path;
string schemaFilePath = _dataHandlerList.Datas[RawDataName.LexicalAttributeSchema].Path;
errorSet = CompileTnml(tnmlRuleFilePath, schemaFilePath, outputStream, false);
}
break;
case ModuleDataName.ForeignLtsCollection:
errorSet = CompileForeignLtsCollection(_dataHandlerList.Datas[moduleDataName].Path, outputStream);
break;
case ModuleDataName.PolyphonyModel:
{
string polyphonyModelDataDir = _dataHandlerList.Datas[RawDataName.PolyphonyModel].Path;
Collection<string> compiledFiles = new Collection<string>();
errorSet = CrfModelCompiler.Compile(polyphonyModelDataDir, outputStream, compiledFiles, _language);
if (errorSet.GetSeverityCount(ErrorSeverity.MustFix) == 0 &&
compiledFiles.Count > 0)
{
errorSet.Add(ReportCompiledFiles("PolyphonyModel", compiledFiles));
}
}
break;
case ModuleDataName.RNNPolyphonyModel:
{
string polyphonyModelDataPath = _dataHandlerList.Datas[RawDataName.RNNPolyphonyModel].Path;
Collection<string> compiledFiles = new Collection<string>();
errorSet = RNNModelCompiler.Compile(polyphonyModelDataPath, outputStream, compiledFiles);
if (errorSet.GetSeverityCount(ErrorSeverity.MustFix) == 0 &&
compiledFiles.Count > 0)
{
errorSet.Add(ReportCompiledFiles("PolyphonyModel", compiledFiles));
}
}
break;
default:
errorSet.Add(DataCompilerError.InvalidModuleData, moduleDataName);
break;
}
}
catch (Exception ex)
{
Type exceptionType = ex.GetType();
if (exceptionType.Equals(typeof(FileNotFoundException)) ||
exceptionType.Equals(typeof(ArgumentNullException)) ||
exceptionType.Equals(typeof(XmlException)) ||
exceptionType.Equals(typeof(InvalidDataException)))
{
errorSet.Add(DataCompilerError.RawDataNotFound, moduleDataName,
Helper.BuildExceptionMessage(ex));
}
else
{
throw;
}
}
return errorSet;
}
/// <summary>
/// Handle the command line and generate the output.
/// </summary>
/// <param name="lexicalRuleFilePath">Path of POS Lexical rule.</param>
/// <param name="contextualRuleFilePath">Path of POS Contectual Rule.</param>
/// <param name="posSetFilePath">Path of POS set.</param>
/// <param name="outputStream">Output stream.</param>
/// <returns>ErrorSet.</returns>
private ErrorSet CompilePosRule(string lexicalRuleFilePath, string contextualRuleFilePath,
string posSetFilePath, Stream outputStream)
{
ErrorSet errorSet = new ErrorSet();
string toolFileName = ToolName.PosRuleCompiler;
string binaryPosRulePath = Helper.GetTempFileName();
string compilingArguments = Helper.NeutralFormat("\"{0}\" \"{1}\" \"{2}\" \"{3}\"",
lexicalRuleFilePath, contextualRuleFilePath, posSetFilePath, binaryPosRulePath);
string toolPath = Path.Combine(ToolDir, toolFileName);
CheckToolExists(toolPath, errorSet);
if (!File.Exists(lexicalRuleFilePath))
{
errorSet.Add(DataCompilerError.RawDataNotFound, RawDataName.PosLexicalRule, lexicalRuleFilePath);
}
if (!File.Exists(contextualRuleFilePath))
{
errorSet.Add(DataCompilerError.RawDataNotFound, RawDataName.PosContextualRule, contextualRuleFilePath);
}
if (!File.Exists(posSetFilePath))
{
errorSet.Add(DataCompilerError.RawDataNotFound, RawDataName.PosSet, posSetFilePath);
}
if (!errorSet.Contains(ErrorSeverity.MustFix))
{
HandleCommandLine(ModuleDataName.PosRule, toolPath, compilingArguments,
binaryPosRulePath, outputStream, errorSet);
}
return errorSet;
}
/// <summary>
/// General Rule Compiler.
/// </summary>
/// <param name="txtPath">Path of txt formatted general rule.</param>
/// <param name="outputStream">Output stream.</param>
/// <returns>ErrorSet.</returns>
private ErrorSet CompileGeneralRule(string txtPath, Stream outputStream)
{
ErrorSet errorSet = new ErrorSet();
string toolFileName = ToolName.RuleCompiler;
string binaryRulePath = Helper.GetTempFileName();
string compilingArguments = Helper.NeutralFormat("\"{0}\" \"{1}\"", txtPath, binaryRulePath);
string toolPath = Path.Combine(ToolDir, toolFileName);
const string DataName = "General Rule";
CheckToolExists(toolPath, errorSet);
if (!File.Exists(txtPath))
{
errorSet.Add(DataCompilerError.RawDataNotFound, DataName, txtPath);
}
if (!errorSet.Contains(ErrorSeverity.MustFix))
{
HandleCommandLine(DataName, toolPath, compilingArguments,
binaryRulePath, outputStream, errorSet);
}
return errorSet;
}
/// <summary>
/// FstNE rule compiler.
/// </summary>
/// <param name="fstNERuleFilePath">Path of FstNE rule.</param>
/// <param name="outputStream">Output stream.</param>
/// <returns>ErrorSet.</returns>
private ErrorSet CompileFstNE(string fstNERuleFilePath, Stream outputStream)
{
ErrorSet errorSet = new ErrorSet();
string toolFileName = ToolName.FstNECompiler;
string binaryFstNERulePath = Helper.GetTempFileName();
string compilingArguments = Helper.NeutralFormat(
"-lang {0} -intnml \"{1}\" -outfst \"{2}\"",
Localor.LanguageToString(_language), fstNERuleFilePath, binaryFstNERulePath);
string toolPath = Path.Combine(ToolDir, toolFileName);
CheckToolExists(toolPath, errorSet);
if (!File.Exists(fstNERuleFilePath))
{
errorSet.Add(DataCompilerError.RawDataNotFound, "FstNE rule", fstNERuleFilePath);
}
if (!errorSet.Contains(ErrorSeverity.MustFix))
{
HandleCommandLine("FstNE rule", toolPath, compilingArguments,
binaryFstNERulePath, outputStream, errorSet);
}
return errorSet;
}
/// <summary>
/// Tnml rule compiler.
/// </summary>
/// <param name="tnmlRuleFilePath">Path of Tnml rule.</param>
/// <param name="schemaFilePath">Path of Lexical Attribute Schema.</param>
/// <param name="outputStream">Output stream.</param>
/// <param name="isTNRule">Whether it's TN rule.</param>
/// <returns>ErrorSet.</returns>
private ErrorSet CompileTnml(string tnmlRuleFilePath, string schemaFilePath, Stream outputStream, bool isTNRule)
{
ErrorSet errorSet = new ErrorSet();
string toolFileName = ToolName.TnmlCompiler;
string binaryTnmlRulePath = Helper.GetTempFileName();
string compilingArguments = Helper.NeutralFormat(
"-lcid {0} -tnml \"{1}\" -schema \"{2}\" -tnbin \"{3}\"",
(uint)_language, tnmlRuleFilePath, schemaFilePath, binaryTnmlRulePath);
if (isTNRule)
{
compilingArguments += " -mode TTS -norulename FALSE";
}
string toolPath = Path.Combine(ToolDir, toolFileName);
CheckToolExists(toolPath, errorSet);
if (!File.Exists(tnmlRuleFilePath))
{
errorSet.Add(DataCompilerError.RawDataNotFound, "TNML rule", tnmlRuleFilePath);
}
if (!File.Exists(schemaFilePath))
{
errorSet.Add(DataCompilerError.RawDataNotFound, RawDataName.LexicalAttributeSchema, schemaFilePath);
}
if (!errorSet.Contains(ErrorSeverity.MustFix))
{
HandleCommandLine("TNML rule", toolPath, compilingArguments,
binaryTnmlRulePath, outputStream, errorSet);
}
return errorSet;
}
/// <summary>
/// Compile LTS rule.
/// </summary>
/// <param name="ltsDataDir">Directory of LTS data.</param>
/// <param name="outputStream">Output stream.</param>
/// <returns>Error Set.</returns>
private ErrorSet CompileLtsRule(string ltsDataDir, Stream outputStream)
{
ErrorSet errorSet = new ErrorSet();
string toolFileName = ToolName.LtsCompiler;
string letterSymPath = Path.Combine(ltsDataDir, "letter.sym");
string phoneSymPath = Path.Combine(ltsDataDir, "phone.sym");
string letterQPath = Path.Combine(ltsDataDir, "letter.q");
string phoneQPath = Path.Combine(ltsDataDir, "phone.q");
string ltsTreePath = Path.Combine(ltsDataDir, "tree.tree");
string trainSmpPath = Path.Combine(ltsDataDir, "train.smp");
CheckRawDataExists(letterSymPath, errorSet);
CheckRawDataExists(phoneSymPath, errorSet);
CheckRawDataExists(letterQPath, errorSet);
CheckRawDataExists(phoneQPath, errorSet);
CheckRawDataExists(ltsTreePath, errorSet);
CheckRawDataExists(trainSmpPath, errorSet);
string binaryLtsPath = Helper.GetTempFileName();
string compilingArguments = Helper.NeutralFormat(
"\"{0}\" \"{1}\" \"{2}\" \"{3}\" \"{4}\" {5} {6} {7} \"{8}\" \"{9}\"",
letterSymPath, phoneSymPath, letterQPath, phoneQPath, ltsTreePath,
0, 0, 0.00000001,
trainSmpPath, binaryLtsPath);
string toolPath = Path.Combine(ToolDir, toolFileName);
CheckToolExists(toolPath, errorSet);
if (!errorSet.Contains(ErrorSeverity.MustFix))
{
HandleCommandLine(ModuleDataName.LtsRule, toolPath, compilingArguments,
binaryLtsPath, outputStream, errorSet);
}
return errorSet;
}
private ErrorSet CompileLexicon(Stream outputStream)
{
if (outputStream == null)
{
throw new ArgumentNullException("outputStream");
}
ErrorSet errorSet = new ErrorSet();
ErrorSet subErrorSet = new ErrorSet();
LexicalAttributeSchema schema = (LexicalAttributeSchema)GetObject(
RawDataName.LexicalAttributeSchema, subErrorSet);
MergeDependencyError(errorSet, subErrorSet, _schemaFullName);
subErrorSet.Clear();
TtsPhoneSet phoneSet = (TtsPhoneSet)GetObject(RawDataName.PhoneSet, subErrorSet);
MergeDependencyError(errorSet, subErrorSet, RawDataName.PhoneSet);
if (!errorSet.Contains(ErrorSeverity.MustFix))
{
Microsoft.Tts.Offline.Core.Lexicon lexicon = (Microsoft.Tts.Offline.Core.Lexicon)GetObject(RawDataName.Lexicon, errorSet);
errorSet.Merge(lexicon.ErrorSet);
// Change to case insensitive lexicon
MemoryStream lexiconStream = new MemoryStream();
using (XmlWriter xmlWriter = XmlWriter.Create(lexiconStream))
{
Microsoft.Tts.Offline.Core.Lexicon.ContentControler lexiconControler =
new Microsoft.Tts.Offline.Core.Lexicon.ContentControler();
lexiconControler.IsCaseSensitive = true;
lexicon.Save(xmlWriter, lexiconControler);
}
lexiconStream.Seek(0, SeekOrigin.Begin);
Microsoft.Tts.Offline.Core.Lexicon caseInsensitiveLexicon = new Microsoft.Tts.Offline.Core.Lexicon();
using (StreamReader sr = new StreamReader(lexiconStream))
{
caseInsensitiveLexicon.Load(sr);
}
if (caseInsensitiveLexicon != null && !errorSet.Contains(ErrorSeverity.MustFix))
{
caseInsensitiveLexicon.LexicalAttributeSchema = schema;
caseInsensitiveLexicon.PhoneSet = phoneSet;
caseInsensitiveLexicon.Validate();
// Set severity of errors only in case-insensitive lexicon to NoError for they're not treated as real error
caseInsensitiveLexicon.ErrorSet.SetSeverity(ErrorSeverity.NoError);
string vendorLexiconPath = Helper.GetTempFileName();
caseInsensitiveLexicon.SaveToVendorLexicon(vendorLexiconPath);
string toolFileName = ToolName.BldVendor2;
string binaryLexiconPath = Helper.GetTempFileName();
string compilingArguments = Helper.NeutralFormat("-v {0} V2 \"{1}\" \"{2}\" \"{3}\" TTS",
(int)_language, _dataHandlerList.Datas[RawDataName.LexicalAttributeSchema].Path,
vendorLexiconPath, binaryLexiconPath);
string toolPath = Path.Combine(ToolDir, toolFileName);
CheckToolExists(toolPath, errorSet);
if (!errorSet.Contains(ErrorSeverity.MustFix))
{
HandleCommandLine(ModuleDataName.Lexicon, toolPath, compilingArguments,
binaryLexiconPath, outputStream, errorSet);
}
File.Delete(vendorLexiconPath);
errorSet.Merge(caseInsensitiveLexicon.ErrorSet);
}
else if (lexicon == null)
{
errorSet.Add(DataCompilerError.RawDataError, "Lexicon");
}
else
{
errorSet.Merge(caseInsensitiveLexicon.ErrorSet);
}
}
return errorSet;
}
/// <summary>
/// Compile the foreign LTS collection.
/// </summary>
/// <param name="configuration">Foreign LTS configuration.</param>
/// <param name="outputStream">Output steam.</param>
/// <returns>Error set.</returns>
private ErrorSet CompileForeignLtsCollection(string configuration, Stream outputStream)
{
ErrorSet errorSet = new ErrorSet();
// The configuration is written in
// "originLanguageA : phonesetA ; RuleA ; originLanguageB: phonesetB ; RuleB"
string[] phonesetLtsList = configuration.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
ushort count = Convert.ToUInt16(phonesetLtsList.Length / 2);
Offline.Language[] languages = new Offline.Language[count];
TtsPhoneSet[] phoneSets = new TtsPhoneSet[count];
string[] ltsPaths = new string[count];
// Load the phone sets
for (ushort i = 0; i < count; i++)
{
languages[i] = Offline.Language.Neutral;
string phoneSetPath = phonesetLtsList[i * 2].Trim();
int languageSeparatorIndex = phoneSetPath.IndexOf(":");
if (languageSeparatorIndex != -1)
{
string language = phoneSetPath.Substring(0, languageSeparatorIndex).Trim();
languages[i] = Localor.StringToLanguage(language);
phoneSetPath = phoneSetPath.Substring(languageSeparatorIndex + 1, phoneSetPath.Length - languageSeparatorIndex - 1).Trim();
}
if (!Path.IsPathRooted(phoneSetPath))
{
phoneSetPath = Path.Combine(_dataHandlerList.DataRoot, phoneSetPath);
}
phoneSets[i] = new TtsPhoneSet();
phoneSets[i].Load(phoneSetPath);
phoneSets[i].Validate();
if (languages[i] == Offline.Language.Neutral)
{
languages[i] = phoneSets[i].Language;
}
errorSet.Merge(phoneSets[i].ErrorSet);
if (phoneSets[i].ErrorSet.Contains(ErrorSeverity.MustFix))
{
phoneSets[i] = null;
}
else
{
ltsPaths[i] = phonesetLtsList[(i * 2) + 1].Trim();
if (!Path.IsPathRooted(ltsPaths[i]))
{
ltsPaths[i] = Path.Combine(_dataHandlerList.DataRoot, ltsPaths[i]);
}
}
}
if (!errorSet.Contains(ErrorSeverity.MustFix))
{
BinaryWriter bw = new BinaryWriter(outputStream);
{
bw.Write((ushort)count);
for (ushort i = 0; i < count; i++)
{
bw.Write((ushort)languages[i]);
bw.Write((ushort)phoneSets[i].Language);
}
// Write phone set offset
long phoneSetOffset = bw.BaseStream.Position;
for (byte i = 0; i < count; i++)
{
bw.Write((uint)0);
}
// Write LTS offset
long ltsOffset = bw.BaseStream.Position;
for (byte i = 0; i < count; i++)
{
bw.Write((uint)0);
}
// Write phone set
for (byte i = 0; i < count; i++)
{
long offset = bw.BaseStream.Position;
bw.BaseStream.Seek(phoneSetOffset, SeekOrigin.Begin);
if (offset > uint.MaxValue)
{
throw new InvalidDataException(Helper.NeutralFormat(
"Foreign LTS collection size exceeds the maximal size {0}", uint.MaxValue));
}
bw.Write((uint)offset);
phoneSetOffset += sizeof(uint);
bw.BaseStream.Seek(offset, SeekOrigin.Begin);
errorSet.Merge(PhoneSetCompiler.Compile(phoneSets[i], bw.BaseStream));
}
// Write LTS
for (byte i = 0; i < count; i++)
{
long offset = bw.BaseStream.Position;
bw.BaseStream.Seek(ltsOffset, SeekOrigin.Begin);
if (offset > uint.MaxValue)
{
throw new InvalidDataException(Helper.NeutralFormat(
"Foreign LTS collection size exceeds the maximal size {0}", uint.MaxValue));
}
bw.Write((uint)offset);
ltsOffset += sizeof(uint);
bw.BaseStream.Seek(offset, SeekOrigin.Begin);
LoadStream(ltsPaths[i], bw.BaseStream);
}
}
}
return errorSet;
}