// Value := '(' Expression ')' S
private State DoParseValue5Rule(State _state, List <Result> _outResults)
{
State _start = _state;
List <Result> results = new List <Result>();
_state = DoSequence(_state, results,
delegate(State s, List <Result> r) { return(DoParseLiteral(s, r, "(")); },
delegate(State s, List <Result> r) { return(DoParse(s, r, "Expression")); },
delegate(State s, List <Result> r) { return(DoParseLiteral(s, r, ")")); },
delegate(State s, List <Result> r) { return(DoParse(s, r, "S")); });
if (_state.Parsed)
{
Expression value = results.Count > 0 ? results[0].Value : default(Expression);
value = results[1].Value;
_outResults.Add(new Result(this, _start.Index, _state.Index - _start.Index, m_input, value));
}
else
{
string expected = null;
expected = "parenthesized expression";
if (expected != null)
{
_state = new State(_start.Index, false, ErrorSet.Combine(_start.Errors, new ErrorSet(_state.Errors.Index, expected)));
}
}
return(_state);
}
public ErrorSet(ErrorSet other) : this(bwapiPINVOKE.new_ErrorSet__SWIG_1(ErrorSet.getCPtr(other)), true)
{
if (bwapiPINVOKE.SWIGPendingException.Pending)
{
throw bwapiPINVOKE.SWIGPendingException.Retrieve();
}
}
// OrExpression := AndExpression ('or' SS AndExpression)+
private State DoParseOrExpression1Rule(State _state, List <Result> _outResults)
{
State _start = _state;
List <Result> results = new List <Result>();
_state = DoSequence(_state, results,
delegate(State s, List <Result> r) { return(DoParse(s, r, "AndExpression")); },
delegate(State s, List <Result> r) { return(DoRepetition(s, r, 1, 2147483647,
delegate(State s2, List <Result> r2) { return DoSequence(s2, r2,
delegate(State s3, List <Result> r3) { return DoParseLiteral(s3, r3, "or"); },
delegate(State s3, List <Result> r3) { return DoParse(s3, r3, "SS"); },
delegate(State s3, List <Result> r3) { return DoParse(s3, r3, "AndExpression"); }); })); });
if (_state.Parsed)
{
Predicate value = results.Count > 0 ? results[0].Value : default(Predicate);
value = new OrPredicate(from e in results where e.Value != null select e.Value);
_outResults.Add(new Result(this, _start.Index, _state.Index - _start.Index, m_input, value));
}
else
{
string expected;
expected = "or expression";
if (expected != null)
{
_state = new State(_start.Index, false, ErrorSet.Combine(_start.Errors, new ErrorSet(_state.Errors.Index, expected)));
}
}
return(_state);
}
// Literal := 'false' SS
private State DoParseLiteral3Rule(State _state, List <Result> _outResults)
{
State _start = _state;
List <Result> results = new List <Result>();
_state = DoSequence(_state, results,
delegate(State s, List <Result> r) { return(DoParseLiteral(s, r, "false")); },
delegate(State s, List <Result> r) { return(DoParse(s, r, "SS")); });
if (_state.Parsed)
{
Predicate value = results.Count > 0 ? results[0].Value : default(Predicate);
value = new BoolPredicate(false);
_outResults.Add(new Result(this, _start.Index, _state.Index - _start.Index, m_input, value));
}
else
{
string expected;
expected = "boolean literal";
if (expected != null)
{
_state = new State(_start.Index, false, ErrorSet.Combine(_start.Errors, new ErrorSet(_state.Errors.Index, expected)));
}
}
return(_state);
}
// Variable := VariablePrefix VariableSuffix* S
private State DoParseVariableRule(State _state, List <Result> _outResults)
{
State _start = _state;
List <Result> results = new List <Result>();
_state = DoSequence(_state, results,
delegate(State s, List <Result> r) { return(DoParse(s, r, "VariablePrefix")); },
delegate(State s, List <Result> r) { return(DoRepetition(s, r, 0, 2147483647,
delegate(State s2, List <Result> r2) { return DoParse(s2, r2, "VariableSuffix"); })); },
delegate(State s, List <Result> r) { return(DoParse(s, r, "S")); });
if (_state.Parsed)
{
Predicate value = results.Count > 0 ? results[0].Value : default(Predicate);
string text = m_input.Substring(_start.Index, _state.Index - _start.Index);
value = new VariablePredicate(text.Trim());
if (text != null)
{
_outResults.Add(new Result(this, _start.Index, _state.Index - _start.Index, m_input, value));
}
}
else
{
string expected;
expected = "variable";
if (expected != null)
{
_state = new State(_start.Index, false, ErrorSet.Combine(_start.Errors, new ErrorSet(_state.Errors.Index, expected)));
}
}
return(_state);
}
/// <summary>
/// Validate compound rule file.
/// </summary>
/// <param name="filePath">Compound rule file path.</param>
/// <param name="phoneset">TTS phone set.</param>
/// <returns>ErrorSet.</returns>
public static ErrorSet ValidateCompoundRule(string filePath, TtsPhoneSet phoneset)
{
// Validate parameter
if (string.IsNullOrEmpty(filePath))
{
throw new ArgumentNullException("filePath");
}
if (!File.Exists(filePath))
{
throw Helper.CreateException(typeof(FileNotFoundException), filePath);
}
ErrorSet errorSet = new ErrorSet();
using (XmlTextReader xmlTextReader = new XmlTextReader(filePath))
{
while (xmlTextReader.Read())
{
if (xmlTextReader.NodeType == XmlNodeType.Element &&
xmlTextReader.Name == "out")
{
if (xmlTextReader.Read() && xmlTextReader.NodeType == XmlNodeType.Text)
{
ValidateCompoundRuleNodePron(xmlTextReader.Value.Trim(),
phoneset, xmlTextReader.LineNumber, errorSet);
}
}
}
}
return errorSet;
}
// PrimaryExpression := 'excluded' SS '(' S Literal ')' S
private State DoParsePrimaryExpression3Rule(State _state, List <Result> _outResults)
{
State _start = _state;
List <Result> results = new List <Result>();
_state = DoSequence(_state, results,
delegate(State s, List <Result> r) { return(DoParseLiteral(s, r, "excluded")); },
delegate(State s, List <Result> r) { return(DoParse(s, r, "SS")); },
delegate(State s, List <Result> r) { return(DoParseLiteral(s, r, "(")); },
delegate(State s, List <Result> r) { return(DoParse(s, r, "S")); },
delegate(State s, List <Result> r) { return(DoParse(s, r, "Literal")); },
delegate(State s, List <Result> r) { return(DoParseLiteral(s, r, ")")); },
delegate(State s, List <Result> r) { return(DoParse(s, r, "S")); });
if (_state.Parsed)
{
Predicate value = results.Count > 0 ? results[0].Value : default(Predicate);
value = new ExcludedPredicate(results[2].Value);
_outResults.Add(new Result(this, _start.Index, _state.Index - _start.Index, m_input, value));
}
else
{
string expected;
expected = "excluded function";
if (expected != null)
{
_state = new State(_start.Index, false, ErrorSet.Combine(_start.Errors, new ErrorSet(_state.Errors.Index, expected)));
}
}
return(_state);
}
/// <summary>
/// Load stemmer file.
/// </summary>
/// <param name="stemmerFilePath">Stemmer file path.</param>
/// <param name="errorSet">Error set.</param>
/// <returns>Loaded stemmer items.</returns>
public static Dictionary<string, string> Read(string stemmerFilePath,
ErrorSet errorSet)
{
Dictionary<string, string> stemmer = new Dictionary<string, string>();
foreach (string line in Helper.FileLines(stemmerFilePath))
{
string[] items = line.Split(Delimitor.TabChars, StringSplitOptions.RemoveEmptyEntries);
if (items.Length < 2)
{
errorSet.Add(StemmerFileError.OneColumnLine, line, stemmerFilePath);
continue;
}
for (int i = 1; i < items.Length; i++)
{
if (stemmer.ContainsKey(items[i]))
{
// Skips this one if there already has it.
continue;
}
stemmer.Add(items[i], items[0]);
}
}
return stemmer;
}
public async Task ErrorSetIsReturned(ErrorSet errorSet)
{
var controller = new DummyController
{
Configuration = new HttpConfiguration(),
Request = new HttpRequestMessage()
};
controller.Request.Headers.Accept.Clear();
controller.Request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
var statusCode = HttpStatusCode.MethodNotAllowed;
var actionResult = new FailureWithErrorSetResult(statusCode, controller, errorSet);
var response = await actionResult.ExecuteAsync(new CancellationToken());
var responseSet = await response.Content.ReadAsAsync <ErrorSet>();
responseSet.Should().NotBeNull("because an error set should be present in the response");
responseSet.Errors.Should().NotBeNull("because there should be an error collection present in the response, even if empty");
var expectedErrors = errorSet.Errors.OrderBy(item => item.Code).ToList();
var responseErrors = responseSet.Errors.OrderBy(item => item.Code).ToList();
responseErrors.Should().HaveSameCount(expectedErrors, "because the number of errors in the result should match the provided error set")
.And.Equal(expectedErrors, (left, right) => ((left.Code == right.Code) && (left.Description == right.Description)), "because the provided error set should have been used in the response");
}
private State DoRepetition(State state, List <Result> results, int min, int max, ParseMethod method)
{
State start = state;
int count = 0;
while (count <= max)
{
State temp = method(state, results);
if (temp.Parsed && temp.Index > state.Index)
{
state = temp;
++count;
}
else
{
state = new State(state.Index, true, ErrorSet.Combine(state.Errors, temp.Errors));
break;
}
}
if (count < min || count > max)
{
state = new State(start.Index, false, ErrorSet.Combine(start.Errors, state.Errors));
}
return(state);
}
// Space := [ \t\r\n]
private State DoParseSpaceRule(State _state, List <Result> _outResults)
{
State _start = _state;
List <Result> results = new List <Result>();
_state = DoParseRange(_state, results, false, " \t\r\n", string.Empty, null, "[ \t\r\n]");
if (_state.Parsed)
{
Expression value = results.Count > 0 ? results[0].Value : default(Expression);
_outResults.Add(new Result(this, _start.Index, _state.Index - _start.Index, m_input, value));
}
else
{
string expected = null;
expected = "whitespace";
if (expected != null)
{
_state = new State(_start.Index, false, ErrorSet.Combine(_start.Errors, new ErrorSet(_state.Errors.Index, expected)));
}
}
return(_state);
}
/// <summary>
/// Compiles quotation mark table into binary stream.
/// </summary>
/// <param name="quoteTable">The instance of quotation mark table.</param>
/// <param name="outputStream">The instance of output binary stream.</param>
/// <returns>Any error found during the compilation.</returns>
public static ErrorSet Compile(QuotationMarkTable quoteTable, Stream outputStream)
{
if (quoteTable == null)
{
throw new ArgumentNullException("quoteTable");
}
if (outputStream == null)
{
throw new ArgumentNullException("outputStream");
}
ErrorSet errorSet = new ErrorSet();
BinaryWriter writer = new BinaryWriter(outputStream);
writer.Write((uint)quoteTable.Language);
writer.Write((uint)quoteTable.Items.Count);
foreach (var item in quoteTable.Items)
{
writer.Write((ushort)item.Left);
writer.Write((ushort)item.Right);
writer.Write((uint)item.Direct);
}
return errorSet;
}
public ErrorSetEnumerator(ErrorSet collection)
{
collectionRef = collection;
keyCollection = new global::System.Collections.Generic.List <Error>(collection.Values);
currentIndex = -1;
currentObject = null;
currentSize = collectionRef.Count;
}
/// <summary>
/// Load Lexicon Data object.
/// </summary>
/// <param name="errorSet">ErrorSet.</param>
/// <returns>Lexicon Data object.</returns>
internal override object LoadDataObject(ErrorSet errorSet)
{
Lexicon lexicon = new Lexicon(this.Language);
Lexicon.ContentControler lexiconControler = new Lexicon.ContentControler();
lexiconControler.IsCaseSensitive = true;
lexicon.Load(this.Path, lexiconControler);
return lexicon;
}
public void ConstructionAllowsAnErrorByComponents()
{
var expectedError = new Error("Code", "Path", "Description");
var set = new ErrorSet(expectedError.Code, expectedError.MemberPath, expectedError.Description);
set.Should().NotBeNull("because the construction should have been successful");
set.Errors.Should().HaveCount(1, "because the provided error should be the only one present.")
.And.Equal(new[] { expectedError }, (left, right) => ((left.Code == right.Code) && (left.Description == right.Description)), "because the provided error should have been used for the final error set");
}
/// <summary>
/// Creates a result that corresponds to HTTP 400 and response
/// body contains context around failure to present proper messaging.
/// </summary>
///
/// <param name="instance">The instance that this method was invoked on.</param>
/// <param name="errorSet">The set of errors associated with the response.</param>
/// <param name="contentLanguage">The ISO language-country code that represents the format of the human readable error descriptions. If not provided, US English will be assumed.</param>
/// <returns>The corresponding error scenario with details.</returns>
///
public static BadRequestWithErrorSetResult BadRequest(this ApiController instance,
ErrorSet errorSet,
string contentLanguage = null)
{
if (instance == null)
{
throw new ArgumentNullException(nameof(instance));
}
return(new BadRequestWithErrorSetResult(instance, errorSet, contentLanguage));
}
public AstHelper(ModuleBuilder moduleBuilder)
{
ModuleBuilder = moduleBuilder;
Expecting = new Expecting();
Functions = new FunctionScope();
Variables = new VariableScope();
Types = new TypeScope();
Errors = new ErrorSet();
ReturnScope = new ReturnScope();
}
private State DoParseLiteral(State state, List <Result> results, string literal)
{
State result;
if (string.Compare(m_input, state.Index, literal, 0, literal.Length) == 0)
{
results.Add(new Result(this, state.Index, literal.Length, m_input, default(string)));
result = new State(state.Index + literal.Length, true, state.Errors);
}
else
{
result = new State(state.Index, false, ErrorSet.Combine(state.Errors, new ErrorSet(state.Index, literal)));
}
return(result);
}
// Value := [0-9]+ '.' [0-9]+ (('e' / 'E') [0-9]+)? S
private State DoParseValue1Rule(State _state, List <Result> _outResults)
{
State _start = _state;
List <Result> results = new List <Result>();
_state = DoSequence(_state, results,
delegate(State s, List <Result> r) { return(DoRepetition(s, r, 1, 2147483647,
delegate(State s2, List <Result> r2) { return DoParseRange(s2, r2, false, string.Empty, "09", null, "[0-9]"); })); },
delegate(State s, List <Result> r) { return(DoParseLiteral(s, r, ".")); },
delegate(State s, List <Result> r) { return(DoRepetition(s, r, 1, 2147483647,
delegate(State s2, List <Result> r2) { return DoParseRange(s2, r2, false, string.Empty, "09", null, "[0-9]"); })); },
delegate(State s, List <Result> r) { return(DoRepetition(s, r, 0, 1,
delegate(State s2, List <Result> r2) { return DoSequence(s2, r2,
delegate(State s3, List <Result> r3) { return DoChoice(s3, r3,
delegate(State s4, List <Result> r4) { return DoParseLiteral(s4, r4, "e"); },
delegate(State s4, List <Result> r4) { return DoParseLiteral(s4, r4, "E"); }); },
delegate(State s3, List <Result> r3) { return DoRepetition(s3, r3, 1, 2147483647,
delegate(State s4, List <Result> r4) { return DoParseRange(s4, r4, false, string.Empty, "09", null, "[0-9]"); }); }); })); },
delegate(State s, List <Result> r) { return(DoParse(s, r, "S")); });
if (_state.Parsed)
{
Expression value = results.Count > 0 ? results[0].Value : default(Expression);
string text = m_input.Substring(_start.Index, _state.Index - _start.Index);
value = new FloatExpression(text.Trim());
if (text != null)
{
_outResults.Add(new Result(this, _start.Index, _state.Index - _start.Index, m_input, value));
}
}
else
{
string expected = null;
expected = "number";
if (expected != null)
{
_state = new State(_start.Index, false, ErrorSet.Combine(_start.Errors, new ErrorSet(_state.Errors.Index, expected)));
}
}
return(_state);
}
public void BadRquestWithErrorSet()
{
var controller = new DummyController
{
Configuration = new HttpConfiguration(),
Request = new HttpRequestMessage()
};
var error = new Error("test-errorCode", "test-memberPath", "test-description");
var errorSet = new ErrorSet(error);
var result = controller.BadRequest(errorSet);
result.Should().BeOfType <BadRequestWithErrorSetResult>().And.Should().NotBeNull("because a result should have been generated");
result.Content.Errors.Should().NotBeNull("because errors should have been returned.");
result.Content.Errors.Should().HaveCount(1);
result.Content.Errors.First().ShouldBeEquivalentTo(error);
}
private State DoParseLiteral(State state, List <Result> results, string literal)
{
int j = state.Index;
for (int i = 0; i < literal.Length; ++i)
{
if (char.ToLower(m_input[j + i]) != literal[i])
{
return(new State(state.Index, false, ErrorSet.Combine(state.Errors, new ErrorSet(state.Index, literal))));
}
}
int k = j + literal.Length;
results.Add(new Result(this, j, literal.Length, m_input, default(List <string>)));
state = new State(k, true, state.Errors);
return(state);
}
private State DoSequence(State state, List <Result> results, params ParseMethod[] methods)
{
State start = state;
int startResult = results.Count;
foreach (ParseMethod method in methods)
{
State temp = method(state, results);
if (temp.Parsed)
{
state = temp;
}
else
{
state = new State(start.Index, false, ErrorSet.Combine(start.Errors, temp.Errors));
results.RemoveRange(startResult, results.Count - startResult);
break;
}
}
return(state);
}
// Identifier := [$a-zA-Z] [a-zA-Z0-9]* S
private State DoParseIdentifierRule(State _state, List <Result> _outResults)
{
State _start = _state;
List <Result> results = new List <Result>();
_state = DoSequence(_state, results,
delegate(State s, List <Result> r) { return(DoParseRange(s, r, false, "$", "azAZ", null, "[$a-zA-Z]")); },
delegate(State s, List <Result> r) { return(DoRepetition(s, r, 0, 2147483647,
delegate(State s2, List <Result> r2) { return DoParseRange(s2, r2, false, string.Empty, "azAZ09", null, "[a-zA-Z0-9]"); })); },
delegate(State s, List <Result> r) { return(DoParse(s, r, "S")); });
if (_state.Parsed)
{
Expression value = results.Count > 0 ? results[0].Value : default(Expression);
string text = m_input.Substring(_start.Index, _state.Index - _start.Index);
value = new VariableExpression(text.Trim());
if (text != null)
{
_outResults.Add(new Result(this, _start.Index, _state.Index - _start.Index, m_input, value));
}
}
else
{
string expected = null;
expected = "variable";
if (expected != null)
{
_state = new State(_start.Index, false, ErrorSet.Combine(_start.Errors, new ErrorSet(_state.Errors.Index, expected)));
}
}
return(_state);
}
/// <summary>
/// Compiler.
/// </summary>
/// <param name="mapFileName">Path of phoneme mapping file.</param>
/// <param name="sourceAsId">Whether source phone is phone id, to converted into int.</param>
/// <param name="outputStream">Output Stream.</param>
/// <returns>ErrorSet.</returns>
public static ErrorSet Compile(string mapFileName, bool sourceAsId, Stream outputStream)
{
if (string.IsNullOrEmpty(mapFileName))
{
throw new ArgumentNullException("mapFileName");
}
if (outputStream == null)
{
throw new ArgumentNullException("outputStream");
}
ErrorSet errorSet = new ErrorSet();
PhonemeMap phonemeMap = new PhonemeMap();
phonemeMap.LoadXml(mapFileName);
// Convert SAPI phoneme string
if (sourceAsId)
{
ConvertSourcePhoneID(phonemeMap.Pairs);
}
phonemeMap.Sort();
// Write the binary mapping file
BinaryWriter writer = new BinaryWriter(outputStream);
{
// Write number of phoneme mapping tables
writer.Write(1);
byte[] data = phonemeMap.ToBytes();
writer.Write(data.Length);
writer.Write(data);
}
return errorSet;
}
/// <summary>
/// Compile the trunc rule into binary writer.
/// </summary>
/// <param name="truncRuleFileName">File path of trunc rule.</param>
/// <param name="phoneSet">Phone set.</param>
/// <param name="bw">Binary writer.</param>
/// <returns>Error.</returns>
private static ErrorSet CompTruncRuleData(string truncRuleFileName, TtsPhoneSet phoneSet, BinaryWriter bw)
{
// maximum truncate rule length is 5 phonmes currently
const int MaxTruncRuleLength = 5;
ErrorSet errorSet = new ErrorSet();
List<TruncateNucleusRule> rules = new List<TruncateNucleusRule>();
XmlDocument xmldoc = new XmlDocument();
xmldoc.Load(truncRuleFileName);
XmlNamespaceManager nm = new XmlNamespaceManager(xmldoc.NameTable);
nm.AddNamespace("tts", "http://schemas.microsoft.com/tts/toolsuite");
XmlNodeList nodeList = xmldoc.DocumentElement.SelectNodes(
"/tts:offline/tts:truncateRules/tts:truncateRule", nm);
if (nodeList != null)
{
foreach (XmlNode node in nodeList)
{
XmlNodeList phoneNodeList;
XmlElement xmlNode = node as XmlElement;
string side = xmlNode.GetAttribute("side");
int direction = 0;
if (side.Equals("Right", StringComparison.OrdinalIgnoreCase))
{
direction = 2; // TruncFromRight
}
else if (side.Equals("Left", StringComparison.OrdinalIgnoreCase))
{
direction = 1; // TruncFromLeft
}
else
{
errorSet.Add(UnitGeneratorDataCompilerError.WrongRuleSide,
side, xmlNode.InnerXml);
}
phoneNodeList = xmlNode.SelectNodes("tts:phone", nm);
if (phoneNodeList.Count > MaxTruncRuleLength)
{
errorSet.Add(UnitGeneratorDataCompilerError.RuleLengthExceeded,
MaxTruncRuleLength.ToString(CultureInfo.InvariantCulture), xmlNode.InnerXml);
}
else
{
int idx = 0;
short[] ids = new short[MaxTruncRuleLength + 1];
foreach (XmlNode phoneNode in phoneNodeList)
{
XmlElement xmlPhoneNode = phoneNode as XmlElement;
string phoneValue = xmlPhoneNode.GetAttribute("value");
Phone phone = phoneSet.GetPhone(phoneValue);
if (phone != null)
{
ids[idx++] = (short)phone.Id;
}
else
{
errorSet.Add(UnitGeneratorDataCompilerError.InvalidPhone, phoneValue);
}
}
ids[idx] = 0;
TruncateNucleusRule rule = new TruncateNucleusRule();
rule.Ids = ids;
rule.Direction = direction;
rules.Add(rule);
}
}
}
// write the data
bw.Write(rules.Count);
foreach (TruncateNucleusRule ci in rules)
{
bw.Write(ci.Direction);
for (int i = 0; i < ci.Ids.Length; i++)
{
bw.Write(BitConverter.GetBytes(ci.Ids[i]));
}
}
return errorSet;
}
/// <summary>
/// Compiler.
/// </summary>
/// <param name="truncRuleFileName">File path of trunc rule.</param>
/// <param name="phoneSet">Phone set.</param>
/// <param name="outputStream">Output Stream.</param>
/// <returns>ErrorSet.</returns>
public static ErrorSet Compile(string truncRuleFileName,
TtsPhoneSet phoneSet, Stream outputStream)
{
if (string.IsNullOrEmpty(truncRuleFileName))
{
throw new ArgumentNullException("truncRuleFileName");
}
// pauseLengthFileName could be null
if (phoneSet == null)
{
throw new ArgumentNullException("phoneSet");
}
if (outputStream == null)
{
throw new ArgumentNullException("outputStream");
}
ErrorSet errorSet = new ErrorSet();
phoneSet.Validate();
if (phoneSet.ErrorSet.Contains(ErrorSeverity.MustFix))
{
errorSet.Add(UnitGeneratorDataCompilerError.InvalidPhoneSet);
}
else
{
BinaryWriter bw = new BinaryWriter(outputStream);
{
errorSet.Merge(CompTruncRuleData(truncRuleFileName, phoneSet, bw));
}
}
return errorSet;
}
/// <summary>
/// Save the Lexicon into vendor 1 Lexicon for bldVendor1.
/// Should use Validate(PhoneSet, LexicalAttributeSchema) first.
/// Will skip the error data.
/// </summary>
/// <param name="vendorLexFileName">Name of vendor lexicon.</param>
public void SaveToVendor1Lex(string vendorLexFileName)
{
if (PhoneSet == null)
{
throw new ArgumentException("Please set Phone Set first " +
"and phone set should not be null");
}
if (PosSet == null)
{
throw new ArgumentException("Please set POS set first and POS set should not be null");
}
if (GetAllDomainsCountInfo().Count > 1)
{
throw new InvalidOperationException("This function should only work for domain lexicon " +
"and don't work for combined lexicon.");
}
if (!PosSet.Language.Equals(Language))
{
throw new InvalidDataException(Error.BuildMessage(CommonError.NotConsistentLanguage,
Language.ToString(), "lexicon",
PosSet.Language.ToString(), "POS Set"));
}
if (!PhoneSet.Language.Equals(Language))
{
throw new InvalidDataException(Error.BuildMessage(CommonError.NotConsistentLanguage,
Language.ToString(), "lexicon", PhoneSet.Language.ToString(), "phone set"));
}
using (TextWriter tw = new StreamWriter(vendorLexFileName, false, Encoding.Unicode))
{
foreach (string graphme in Items.Keys)
{
LexicalItem lexItem = Items[graphme];
if (!lexItem.Valid)
{
continue;
}
Debug.Assert(lexItem.Pronunciations.Count > 0);
int pronIndex = 0;
foreach (LexiconPronunciation lexPron in lexItem.Pronunciations)
{
if (!lexPron.Valid)
{
continue;
}
ErrorSet phoneConvertErrorSet = new ErrorSet();
string hexIds = Pronunciation.ConvertIntoHexIds(lexPron.Symbolic,
_ttsPhoneSet, phoneConvertErrorSet);
if (string.IsNullOrEmpty(hexIds))
{
continue;
}
tw.WriteLine("Word {0}", graphme);
tw.WriteLine("Pronunciation{0} {1}", pronIndex++, hexIds);
Debug.Assert(lexPron.Symbolic != null);
Debug.Assert(lexPron.Properties != null && lexPron.Properties.Count > 0);
int propertyIndex = 0;
foreach (LexiconItemProperty pr in lexPron.Properties)
{
if (!pr.Valid)
{
continue;
}
int posID = 0;
int genderID = 0;
int caseID = 0;
int numberID = 0;
Debug.Assert(pr.PartOfSpeech != null && !string.IsNullOrEmpty(pr.PartOfSpeech.Value));
if (pr.PartOfSpeech != null && !string.IsNullOrEmpty(pr.PartOfSpeech.Value))
{
posID = (int)PosSet.Items[pr.PartOfSpeech.Value];
}
if (pr.Case != null && !string.IsNullOrEmpty(pr.Case.Value))
{
ErrorSet.Merge(CaseItem.StringToId(pr.Case.Value, out caseID));
}
if (pr.Gender != null && !string.IsNullOrEmpty(pr.Gender.Value))
{
ErrorSet.Merge(GenderItem.StringToId(pr.Gender.Value, out genderID));
}
if (pr.Number != null && !string.IsNullOrEmpty(pr.Number.Value))
{
ErrorSet.Merge(NumberItem.StringToId(pr.Number.Value, out numberID));
}
// Currently we encode gender before POS field
// miscValue is of type of 32 bit
// low word (16 bit) is used for POS
// In high word, gender uses 3 bits
// case uses 4 bits and number uses 2 bits
int miscValue = posID + (genderID << 16) + (caseID << 19) + (numberID << 23);
tw.WriteLine("POS{0} {1}", propertyIndex++, miscValue);
}
}
}
}
}
public Lexicon[] SplitIntoDomainLexicons(SP.ServiceProvider sp, ErrorSet errorSet)
{
// Dictionary key="domain tag string", value="Lexicon instance"
Dictionary<string, Lexicon> domainLexicons = new Dictionary<string, Lexicon>();
Lexicon generalLexicon = new Lexicon(this.Language);
generalLexicon.Encoding = Encoding;
generalLexicon.DomainTag = DomainItem.GeneralDomain;
domainLexicons.Add(generalLexicon.DomainTag, generalLexicon);
foreach (KeyValuePair<string, LexicalItem> pair in this.Items)
{
Dictionary<string, LexicalItem> domainLexItems = pair.Value.SplitToDomainLexicalItems();
bool same = ArePronsSameForAllDomains(domainLexItems);
bool added = false;
bool isExpandedWords = false;
// check if has pronunciation is expaned.
// for chinese/japanese/korean have no expanded words, they needn't to check if is expanded word.
if (Language != Language.ZhCN &&
Language != Language.ZhHK &&
Language != Language.ZhTW &&
Language != Language.JaJP &&
Language != Language.KoKR)
{
foreach (LexiconPronunciation pron in pair.Value.Pronunciations)
{
if (LexicalItem.IsExpandedWord(pair.Value.Grapheme, pair.Value.Language, pron, sp))
{
isExpandedWords = true;
break;
}
}
}
// for zh-XX, if there is any english word, the word must not be regularly.
if ((Language == Language.ZhCN || Language == Language.ZhTW || Language == Language.ZhHK)
&& Helper.IsEnglishWord(pair.Value.Grapheme))
{
isExpandedWords = true;
}
// word is not expaneded
if (!isExpandedWords)
{
if (same || domainLexItems.Count == 1)
{
LexicalItem newLexItem = pair.Value.Clone();
newLexItem.CleanAllDomainTags();
FillDomainLexicalItem(domainLexicons, DomainItem.GeneralDomain, newLexItem);
added = true;
}
else
{
CheckGeneralPronExist(domainLexItems, errorSet, pair.Value.Grapheme);
}
}
if (!added)
{
foreach (KeyValuePair<string, LexicalItem> lexItemPair in domainLexItems)
{
FillDomainLexicalItem(domainLexicons, lexItemPair.Key, lexItemPair.Value);
}
}
}
return domainLexicons.Values.ToArray();
}
/// <summary>
/// Data validation.
/// </summary>
/// <param name="language">Language.</param>
/// <returns>Data error set found.</returns>
public ErrorSet Validate(Language language)
{
// Files existance validation
if (!Directory.Exists(Dir))
{
throw Helper.CreateException(typeof(DirectoryNotFoundException),
Dir);
}
if (!File.Exists(ScriptFilePath))
{
throw Helper.CreateException(typeof(FileNotFoundException),
ScriptFilePath);
}
if (!File.Exists(FileMapFilePath))
{
throw Helper.CreateException(typeof(FileNotFoundException),
FileMapFilePath);
}
if (!File.Exists(UnitFeatureFilePath))
{
throw Helper.CreateException(typeof(FileNotFoundException),
UnitFeatureFilePath);
}
ErrorSet errorSet = new ErrorSet();
ErrorSet subErrorSet =
FindUnmatchedSentences(ScriptFilePath, language, FileMapFilePath);
errorSet.Merge(subErrorSet);
subErrorSet = ValidateFeatureData(UnitFeatureFilePath, ScriptFilePath,
language);
errorSet.Merge(subErrorSet);
return errorSet;
}
/// <summary>
/// Check if the pronunciation for general domain exist.
/// We check general domain pronunciation to avoid the case that word have address domain pronunciation, but got LTSed in general domain.
/// </summary>
/// <param name="domainLexItems">Domain lexiconItems.</param>
/// <param name="errorSet">The errorSet.</param>
/// <param name="word">The current word.</param>
private void CheckGeneralPronExist(Dictionary<string, LexicalItem> domainLexItems,
ErrorSet errorSet, string word)
{
Helper.ThrowIfNull(domainLexItems);
Helper.ThrowIfNull(errorSet);
if (!domainLexItems.ContainsKey(DomainItem.GeneralDomain))
{
errorSet.Add(LexiconError.LackGeneralDomainPronError, word);
}
}
/// <summary>
/// Validate char table.
/// </summary>
/// <param name="table">Char table.</param>
/// <param name="shallow">Shallow validation.</param>
/// <param name="wordsNotInLexicon">WordsNotInLexicon.</param>
/// <returns>ErrorSet.</returns>
public ErrorSet Validate(CharTable table,
bool shallow, Collection<string> wordsNotInLexicon)
{
if (table == null)
{
throw new ArgumentNullException("table");
}
ErrorSet errorSet = new ErrorSet();
int upperCaseNumber = 0;
int lowerCaseNumber = 0;
int digitNumber = 0;
Collection<string> symbols = new Collection<string>();
foreach (CharElement charElement in table.CharList)
{
if (charElement.Type == CharElement.CharType.UpperCase)
{
upperCaseNumber++;
}
else if (charElement.Type == CharElement.CharType.LowerCase)
{
lowerCaseNumber++;
}
else if (charElement.Type == CharElement.CharType.Digit)
{
digitNumber++;
}
if (!symbols.Contains(charElement.Symbol))
{
symbols.Add(charElement.Symbol);
}
else
{
errorSet.Add(new Error(CharTableError.DuplicateSymbol,
charElement.Symbol));
}
if (!shallow)
{
ValidateCharElement(charElement, errorSet, wordsNotInLexicon);
}
}
if (upperCaseNumber != lowerCaseNumber)
{
errorSet.Add(new Error(CharTableError.MismatchUpperAndLower,
upperCaseNumber.ToString(CultureInfo.InvariantCulture),
lowerCaseNumber.ToString(CultureInfo.InvariantCulture)));
}
if (digitNumber != 10)
{
errorSet.Add(new Error(CharTableError.ErrorDigitCount));
}
return errorSet;
}
internal static global::System.Runtime.InteropServices.HandleRef getCPtr(ErrorSet obj)
{
return((obj == null) ? new global::System.Runtime.InteropServices.HandleRef(null, global::System.IntPtr.Zero) : obj.swigCPtr);
}
/// <summary>
/// ParsePolyCondition.
/// </summary>
/// <param name="expression">Expression.</param>
/// <param name="condition">Condition.</param>
/// <param name="errorSet">ErrorSet.</param>
private void ParsePolyCondition(string expression,
PolyphonyCondition condition, ErrorSet errorSet)
{
string subExpression = expression;
// If the value is string, then search operator before """
if (subExpression.IndexOf('"') > 0)
{
subExpression = subExpression.Substring(0, subExpression.IndexOf('"'));
}
foreach (string oper in _allOperators)
{
if (subExpression.IndexOf(oper) >= 0)
{
condition.Operator = oper;
break;
}
}
bool succeeded = true;
if (string.IsNullOrEmpty(condition.Operator))
{
errorSet.Add(PolyRuleError.MissingOperatorInCondition,
expression);
succeeded = false;
}
if (succeeded)
{
condition.Key = expression.Substring(0,
expression.IndexOf(condition.Operator)).Trim();
if (!_keyTypes.ContainsKey(condition.Key))
{
errorSet.Add(PolyRuleError.NotDeclearedConditionKey,
condition.Key, expression);
succeeded = false;
}
}
if (succeeded)
{
string valueExpression = expression.Substring(
expression.IndexOf(condition.Operator) + condition.Operator.Length).Trim();
if (_keyTypes[condition.Key] == KeyType.String)
{
Match match = Regex.Match(valueExpression, @"^""(.*)""$");
if (match.Success)
{
valueExpression = match.Groups[1].ToString();
}
else
{
errorSet.Add(PolyRuleError.InvalidConditionFormat,
expression);
succeeded = false;
}
}
else
{
int intValue;
if (!int.TryParse(valueExpression, out intValue))
{
errorSet.Add(PolyRuleError.InvalidConditionFormat,
expression);
succeeded = false;
}
}
condition.Value = valueExpression;
}
}
/// <summary>
/// Appends a normal word in the end of given utterance.
/// </summary>
/// <param name="utterance">
/// The given utterance.
/// </param>
/// <param name="scriptWord">
/// The script word.
/// </param>
/// <returns>
/// The phoneme count of the given word.
/// </returns>
/// <exception cref="InvalidDataException">
/// Exception.
/// </exception>
private int AppendNormalWord(TtsUtterance utterance, ScriptWord scriptWord)
{
TtsWord word = utterance.AppendNewWord();
word.LangId = (ushort)scriptWord.Language;
word.BreakLevel = (TtsBreakLevel)scriptWord.Break;
word.Emphasis = (TtsEmphasis)scriptWord.Emphasis;
word.WordText = scriptWord.Grapheme;
word.NETypeText = scriptWord.NETypeText;
word.WordRegularText = scriptWord.RegularText;
word.WordType = TtsWordType.WT_NORMAL;
word.AcousticDomain = DomainExtension.MapToEnum(scriptWord.AcousticDomainTag);
word.WordExpansion = scriptWord.Expansion;
word.ReadablePronunciation = scriptWord.Pronunciation;
if (!string.IsNullOrEmpty(scriptWord.Pronunciation))
{
word.PhoneIds = Phoneme.PronunciationToPhoneIds(Pronunciation.RemoveUnitBoundary(scriptWord.Pronunciation));
}
if (NeedPos)
{
// Checks pos.
if (string.IsNullOrEmpty(scriptWord.PosString))
{
throw new InvalidDataException(
Helper.NeutralFormat("No POS found in sentence \"{0}\" for word \"{1}\"",
scriptWord.Sentence.ScriptItem.Id, scriptWord.Grapheme));
}
// Sets pos value.
word.Pos = (ushort)PosSet.Items[scriptWord.PosString];
string taggingPos = PosSet.CategoryTaggingPOS[scriptWord.PosString];
word.POSTaggerPos = (ushort)PosSet.Items[taggingPos];
}
// Gets the normal phoneme count.
ErrorSet errorSet = new ErrorSet();
int count = scriptWord.GetNormalPhoneNames(PhoneSet, errorSet).Count;
if (errorSet.Count > 0)
{
throw new InvalidDataException(
Helper.NeutralFormat("Invalid phone found in sentence \"{0}\" for word \"{1}\"",
scriptWord.Sentence.ScriptItem.Id, scriptWord.Grapheme));
}
word.TextOffset = (uint)scriptWord.OffsetInString;
word.TextLength = (uint)scriptWord.LengthInString;
return count;
}
/// <summary>
/// TryParseKeyDeclear.
/// </summary>
/// <param name="line">Line.</param>
/// <param name="firstKeyString">FirstKeyString.</param>
/// <param name="errorSet">ErrorSet.</param>
/// <returns>Whether the line is key declear line.</returns>
private bool TryParseKeyDeclear(string line,
ref bool firstKeyString, ErrorSet errorSet)
{
bool isKeyDeclearLine = false;
// No need check key declear after finish parsing the declear part.
if (IsDeclearKey(line))
{
isKeyDeclearLine = true;
string keyName = string.Empty;
KeyType keyType = KeyType.String;
ParseDeclearKey(line, ref keyName, ref keyType);
if (_keyTypes.ContainsKey(keyName))
{
errorSet.Add(PolyRuleError.DuplicateKeyName, keyName);
}
else
{
_keyTypes.Add(keyName, keyType);
if (firstKeyString)
{
_keyString = keyName;
firstKeyString = false;
}
}
}
return isKeyDeclearLine;
}
/// <summary>
/// TryParseKeyLine.
/// </summary>
/// <param name="line">Line.</param>
/// <param name="polyphonyWord">PolyphonyWord.</param>
/// <param name="errorSet">ErrorSet.</param>
/// <param name="domain">Domain.</param>
/// <returns>Whether the line is key line.</returns>
private bool TryParseKeyLine(string line, ref PolyphonyRule polyphonyWord,
ErrorSet errorSet, string domain)
{
bool isKeyLine = false;
if (IsKeyLine(line))
{
isKeyLine = true;
if (polyphonyWord != null)
{
if (polyphonyWord.PolyphonyProns.Count == 0)
{
errorSet.Add(PolyRuleError.NoConditionForWord, polyphonyWord.Word);
}
else
{
_polyphonyWords.Add(polyphonyWord);
}
}
polyphonyWord = new PolyphonyRule();
polyphonyWord.Domain = domain;
int errorCountBeforeParsing = errorSet.Errors.Count;
string keyValue = ParseKeyValueLine(line, errorSet);
if (errorSet.Errors.Count == errorCountBeforeParsing)
{
polyphonyWord.Word = keyValue;
}
}
return isKeyLine;
}
/// <summary>
/// Initializes a new instance of the <see cref="BadRequestWithErrorSetResult"/> class.
/// </summary>
///
/// <param name="statusCode">The HTTP status code associated with this failure.</param>
/// <param name="controller">The controller that encountered this failure.</param>
/// <param name="errorSet">The set of errors that provide context about the failure to callers.</param>
/// <param name="contentLanguage">The ISO language-country code that represents the format of the human readable error descriptions. If not provided, US English will be assumed.</param>
///
public BadRequestWithErrorSetResult(ApiController controller,
ErrorSet errorSet,
string contentLanguage = null) : base(System.Net.HttpStatusCode.BadRequest, controller, errorSet, contentLanguage)
{
}
public SemanticException(ErrorSet errors)
{
_errors = errors;
}
public async Task <IHttpActionResult> FulfillOrderWebHook(string partner,
[FromBody] OrderFulfillmentMessage order)
{
// Verify that the order was sent as the request body.
if (order == null)
{
var errorSet = new ErrorSet(ErrorCode.ValueIsRequired, nameof(order), "The order body is required.");
try
{
var body = await this.Request.SafeReadContentAsStringAsync();
this.Log
.WithCorrelationId(this.Request.GetOrderFulfillmentCorrelationId())
.Information($"Response: {{Response}} { Environment.NewLine } Missing order detected for {{Controller}}::{{ActionMethod}}({{Partner}}) with Headers: [{{Headers}}] { Environment.NewLine } Body: {{RequestBody}} { Environment.NewLine } The following errors were observed: { Environment.NewLine }{{ErrorSet}}",
HttpStatusCode.BadRequest,
nameof(OrderSubmissionController),
nameof(FulfillOrderWebHook),
partner,
this.Request.Headers,
body,
errorSet);
}
catch
{
// Do nothing; logging is a non-critical operation that should not cause
// cascading failures.
}
return(this.BadRequest(errorSet));
}
// If the emulation data was provided and the caller is not priviledged, then reject the request.
var principal = this.User as ClaimsPrincipal;
if ((order.Emulation != null) &&
((principal == null) || (!principal.HasClaim(claim => claim.Type == CustomClaimTypes.MayAccessPriviledgedOperations))))
{
try
{
this.Log
.WithCorrelationId(this.Request.GetOrderFulfillmentCorrelationId())
.Warning($"Response: {{Response}} { Environment.NewLine } Unauthorized request detected for {{Controller}}::{{ActionMethod}}({{Partner}}) with Headers: [{{Headers}}] { Environment.NewLine } The caller does not have permission to perform a priviledged operation.",
HttpStatusCode.Forbidden,
nameof(OrderSubmissionController),
nameof(FulfillOrderWebHook),
partner,
this.Request.Headers);
}
catch
{
// Do nothing; logging is a non-critical operation that should not cause
// cascading failures.
}
return(this.Content <object>(HttpStatusCode.Forbidden, null));
}
var assets = order.LineItems
.SelectMany(item => item.Assets)
.Where(asset => asset != null)
.ToDictionary(asset => asset.Name, asset => this.ParseAssetLocationToUrl(asset.Location));
// Create the command to trigger order processing and place it on the command queue
var command = new ProcessOrder
{
OrderId = order.OrderRequestHeader.OrderId,
PartnerCode = partner,
Assets = assets,
Emulation = order.Emulation,
Id = Guid.NewGuid(),
CorrelationId = this.Request.GetOrderFulfillmentCorrelationId(),
OccurredTimeUtc = this.clock.GetCurrentInstant().ToDateTimeUtc(),
CurrentUser = principal?.Identity?.Name,
Sequence = 0
};
if (!(await this.processOrderCommandPublisher.TryPublishAsync(command)))
{
this.Log
.WithCorrelationId(this.Request.GetOrderFulfillmentCorrelationId())
.Error("Unable to publish the {CommandName} for {Partner}//{Order}", nameof(ProcessOrder), partner, command.OrderId);
return(this.ServiceUnavailable(this.CaclulateRetryAfter(this.config.ServiceUnavailableeRetryAfterInSeconds)));
}
// This can't be fire-and-forget, as completing the handler with the outstanding request causes an exception.
try
{
await this.eventPublisher.TryPublishAsync(command.CreateNewOrderEvent <OrderReceived>());
}
catch (Exception ex)
{
this.Log
.WithCorrelationId(this.Request.GetOrderFulfillmentCorrelationId())
.Error(ex, "The event {EventName} for {Partner}//{Order} could not be published. This is non-critical for fulfillment processing.", nameof(OrderReceived), partner, command.OrderId);
}
return(this.Accepted(new OrderFulfillmentAccepted(command.OrderId), this.CaclulateRetryAfter(this.config.OrderAcceptedRetryAfterInSeconds)));
}
private CFlatTree CreateParser(string input, ErrorSet errorSet)
{
var antlrStringStream = new ANTLRStringStream(input);
var lexter = new CFlatLexer(antlrStringStream);
var tokens = new CommonTokenStream(lexter);
var parser = new CFlatParser(tokens);
var tree = parser.prog().Tree;
var nodes = new CommonTreeNodeStream(tree);
var walker = new CFlatTree(nodes, errorSet);
return walker;
}
/// <summary>
/// Save pronunciations.
/// </summary>
/// <param name="tw">Text writer.</param>
/// <param name="graphme">Word graphme.</param>
/// <param name="lexItem">Lexicon items.</param>
private void SavePronunciation(TextWriter tw, string graphme, LexicalItem lexItem)
{
foreach (LexiconPronunciation lexPron in lexItem.Pronunciations)
{
if (!lexPron.Valid)
{
this.ErrorSet.Add(LexiconCompilerError.RemoveInvalidPronunciation, graphme, lexPron.Symbolic);
continue;
}
ErrorSet phoneConvertErrorSet = new ErrorSet();
string hexIds = Pronunciation.ConvertIntoHexIds(lexPron.Symbolic,
_ttsPhoneSet, phoneConvertErrorSet);
if (string.IsNullOrEmpty(hexIds))
{
continue;
}
string firstHalf = Helper.NeutralFormat("{0}\t{1}\t{2}",
graphme, hexIds, "sppos=noncontent");
Collection<string> attributeStringList = new Collection<string>();
Debug.Assert(lexPron.Symbolic != null);
Debug.Assert(lexPron.Properties != null && lexPron.Properties.Count > 0);
SaveProperty(graphme, lexPron, attributeStringList);
foreach (string attributeString in attributeStringList)
{
tw.WriteLine(firstHalf + attributeString);
}
}
}
public static ErrorSet allErrors() {
ErrorSet ret = new ErrorSet(bwapiPINVOKE.allErrors(), false);
return ret;
}
/// <summary>
/// Validation data alignment between feature file and script file.
/// </summary>
/// <param name="featureFile">Feature file.</param>
/// <param name="scriptFile">Script file.</param>
/// <param name="language">Language.</param>
/// <returns>Data error set found.</returns>
public static ErrorSet ValidateFeatureData(string featureFile,
string scriptFile, Language language)
{
ErrorSet errorSet = new ErrorSet();
TtsPhoneSet phoneSet = Localor.GetPhoneSet(language);
XmlScriptValidateSetting validateSetting = new XmlScriptValidateSetting(phoneSet, null);
XmlScriptFile script = XmlScriptFile.LoadWithValidation(scriptFile, validateSetting);
if (script.ErrorSet.Count > 0)
{
string message = string.Format(CultureInfo.InvariantCulture,
"{0} error(s) found in the script file [{1}]",
script.ErrorSet.Count, scriptFile);
throw new InvalidDataException(message);
}
XmlUnitFeatureFile unitFeatureFile = new XmlUnitFeatureFile(featureFile);
if (unitFeatureFile.Units.Count <= 0)
{
string message = string.Format(CultureInfo.InvariantCulture,
"Zero unit feature item in unit feature file {0}", featureFile);
errorSet.Add(VoiceFontError.OtherErrors, message);
throw new InvalidDataException(message);
}
if (unitFeatureFile.Language != language)
{
string message = string.Format(CultureInfo.InvariantCulture,
"Different lanuage\r\nScript File {0}: lang = {1}\r\n Feature File {2}: lang = {3}",
scriptFile, Localor.LanguageToString(language),
featureFile, Localor.LanguageToString(unitFeatureFile.Language));
throw new InvalidDataException(message);
}
foreach (string key in unitFeatureFile.Units.Keys)
{
UnitFeature unit = unitFeatureFile.Units[key];
string sid = unit.SentenceId;
int unitIndex = unit.Index;
string unitName = unit.Name;
if (unit.Index < 0)
{
string message = string.Format(CultureInfo.InvariantCulture,
"invalid unit index [{0}] found in feature file [{1}]. It should not be negative integer for unit indexing.",
unit.Index, featureFile);
errorSet.Add(VoiceFontError.OtherErrors, message);
continue;
}
try
{
if (!script.ItemDic.ContainsKey(unit.SentenceId))
{
string message = string.Format(CultureInfo.InvariantCulture,
"sentence id {0} in feature file [{1}] is not in script file [{2}]",
sid, featureFile, scriptFile);
errorSet.Add(ScriptError.OtherErrors, sid, message);
continue;
}
ScriptItem item = script.ItemDic[sid];
Phoneme phoneme = Localor.GetPhoneme(language);
SliceData sliceData = Localor.GetSliceData(language);
Collection<TtsUnit> itemUnits = item.GetUnits(phoneme, sliceData);
if (unitIndex >= itemUnits.Count)
{
string message = string.Format(CultureInfo.InvariantCulture,
"the {0}th unit [{1}] in sentence {2} of feature file [{3}] is out of range for sentence {2} in script file [{4}]",
unitIndex, unitName, sid, featureFile, scriptFile);
errorSet.Add(ScriptError.OtherErrors, sid, message);
continue;
}
TtsUnit ttsUnit = itemUnits[unitIndex];
string sliceName = ttsUnit.FullName.Replace(' ', '+');
if (sliceName != unitName)
{
string str1 = "the {0}th unit [{1}] in sentence {3} of feature file [{4}] ";
string str2 = "is not matched with {0}th unit [{2}] for sentence {3} in script file [{5}]";
string message = string.Format(CultureInfo.InvariantCulture,
str1 + str2,
unitIndex, unitName, sliceName, sid, featureFile, scriptFile);
errorSet.Add(ScriptError.OtherErrors, sid, message);
continue;
}
}
catch (InvalidDataException ide)
{
errorSet.Add(ScriptError.OtherErrors, sid, Helper.BuildExceptionMessage(ide));
}
}
return errorSet;
}
/// <summary>
/// Check expansion type.
/// </summary>
/// <param name="expansion">Expansion words.</param>
/// <param name="expansionType">Type string.</param>
/// <param name="errorSet">Errors.</param>
/// <param name="wordsNotInLexicon">WordsNotInLexicon.</param>
private void ValidateExpansion(string expansion, string expansionType,
ErrorSet errorSet, Collection<string> wordsNotInLexicon)
{
if (expansion == null)
{
throw new ArgumentNullException("expansion");
}
if (expansionType == null)
{
throw new ArgumentNullException("expansionType");
}
if (errorSet == null)
{
throw new ArgumentNullException("errorSet");
}
string[] arr = expansion.Split(new char[] { '\t', ' ' },
StringSplitOptions.RemoveEmptyEntries);
foreach (string word in arr)
{
// skip validating ExpansionWordNotInLexicon chartable error for English readout in zhXX
// because English word cannot be in lexicon of these languages
if ((Language == Language.ZhCN || Language == Language.ZhTW || Language == Language.ZhHK)
&& Helper.IsEnglishWord(word))
{
continue;
}
if (_lexicon.Lookup(word, true) == null)
{
errorSet.Add(new Error(CharTableError.ExpansionWordNotInLexicon,
word));
if (wordsNotInLexicon != null && !wordsNotInLexicon.Contains(word))
{
wordsNotInLexicon.Add(word);
}
}
}
}
/// <summary>
/// Get the object.
/// </summary>
/// <param name="errorSet">ErrorSet.</param>
/// <returns>Object.</returns>
public object GetObject(ErrorSet errorSet)
{
if (errorSet == null)
{
throw new ArgumentNullException("errorSet");
}
if (!_processedLoad && _object == null)
{
_processedLoad = true;
if (string.IsNullOrEmpty(this.Path))
{
errorSet.Add(DataCompilerError.PathNotInitialized, this.Name);
}
else if (!File.Exists(this.Path))
{
errorSet.Add(DataCompilerError.RawDataNotFound, this.Name,
this.Path);
}
else
{
_object = LoadDataObject(errorSet);
}
}
else if (_processedLoad && _object == null)
{
errorSet.Add(DataCompilerError.RawDataError, this.Name);
}
return _object;
}
/// <summary>
/// Validate char element.
/// </summary>
/// <param name="element">Char element.</param>
/// <param name="errorSet">Errors.</param>
/// <param name="wordsNotInLexicon">WordsNotInLexicon.</param>
public void ValidateCharElement(CharElement element,
ErrorSet errorSet, Collection<string> wordsNotInLexicon)
{
if (errorSet == null)
{
throw new ArgumentNullException("errors");
}
if (element == null)
{
throw new ArgumentNullException("element");
}
if (string.IsNullOrEmpty(element.Symbol))
{
errorSet.Add(new Error(CharTableError.EmptySymbol));
}
else
{
EnsureInitialized();
if (_lexicon.Lookup(element.Symbol, true) == null)
{
errorSet.Add(new Error(CharTableError.SymbolNotInLexicon,
element.Symbol));
}
if (!element.IsAlphabet())
{
// IsolatedExpansion should not empty for no-alphabet
if (!string.IsNullOrEmpty(element.IsolatedExpansion))
{
ValidateExpansion(element.IsolatedExpansion,
"IsolatedSymbolReadout", errorSet, wordsNotInLexicon);
}
else
{
errorSet.Add(new Error(CharTableError.EmptyIsolatedSymbol,
element.Symbol));
}
// ContextualSymbolReadout is optional for no-alphabet
if (!string.IsNullOrEmpty(element.ContextualExpansion))
{
ValidateExpansion(element.ContextualExpansion,
"ContextualSymbolReadout", errorSet, wordsNotInLexicon);
}
// No-alphabet should not have features.
if (element.Feature != CharTableCompiler.CharFeature.None)
{
errorSet.Add(new Error(CharTableError.NonAlphabetShouldNoFeatures,
element.Symbol));
}
// No-alphabet should not have pronunciation.
if (!string.IsNullOrEmpty(element.Pronunciation))
{
errorSet.Add(new Error(CharTableError.NonAlphabetShouldNoPronunciation,
element.Symbol));
}
}
else
{
if (string.IsNullOrEmpty(element.Pronunciation))
{
errorSet.Add(new Error(CharTableError.AlphabetNoPron,
element.Symbol));
}
else
{
ErrorSet pronErrorSet = Pronunciation.Validate(
element.Pronunciation, _phoneset);
foreach (Error error in pronErrorSet.Errors)
{
Error alphabetError = new Error(CharTableError.AlphabetInvalidPron,
error, element.Symbol, element.Pronunciation);
// Keep the same error level with pronunciation error.
alphabetError.Severity = error.Severity;
errorSet.Add(alphabetError);
}
}
if (element.Feature == CharTableCompiler.CharFeature.None)
{
errorSet.Add(new Error(CharTableError.AlphabetNoFeatures,
element.Symbol));
}
if (!string.IsNullOrEmpty(element.IsolatedExpansion) ||
!string.IsNullOrEmpty(element.ContextualExpansion))
{
errorSet.Add(new Error(CharTableError.AlphabetShouldNoExpansion,
element.Symbol));
}
}
}
}
/// <summary>
/// Load data object.
/// </summary>
/// <param name="errorSet">ErrorSet.</param>
/// <returns>Data object.</returns>
internal virtual object LoadDataObject(ErrorSet errorSet)
{
return null;
}
private ErrorSet GetErrors(string input)
{
var errorSet = new ErrorSet();
var parser = CreateParser(input, errorSet);
var @class = parser.prog();
@class.Compile(new CompilerContext(new ClrCodeGenerator("TestOut"), errorSet));
return errorSet;
}
/// <summary>
/// Load.
/// </summary>
/// <param name="filePath">FilePath.</param>
/// <param name="phoneSet">PhoneSet.</param>
/// <returns>ErrorSet.</returns>
public ErrorSet Load(string filePath, TtsPhoneSet phoneSet)
{
// This validation is needed by Fxcop checking parameters.
if (phoneSet == null)
{
phoneSet = null;
}
if (string.IsNullOrEmpty(filePath))
{
throw new ArgumentNullException("filePath");
}
if (!File.Exists(filePath))
{
throw Helper.CreateException(typeof(FileNotFoundException), filePath);
}
if (!Helper.IsUnicodeFile(filePath))
{
throw new InvalidDataException(Helper.NeutralFormat(
"Polyphony rule file [{0}] is not unicode.", filePath));
}
ErrorSet errorSet = new ErrorSet();
_keyTypes.Clear();
bool finishReadHead = false;
bool firstKeyString = true;
PolyphonyRule polyphonyWord = null;
int lineNum = 0;
string domain = DomainItem.GeneralDomain;
foreach (string line in Helper.FileLines(filePath, Encoding.Unicode, false))
{
lineNum++;
string trimedLine = line.Trim();
if (string.IsNullOrEmpty(trimedLine))
{
continue;
}
if (IsComment(trimedLine))
{
continue;
}
if (IsDomainTag(trimedLine))
{
ParseDomainKey(trimedLine, ref domain);
continue;
}
ErrorSet parseErrorSet = new ErrorSet();
if (!finishReadHead)
{
bool isKeyDeclear = TryParseKeyDeclear(trimedLine,
ref firstKeyString, parseErrorSet);
AddParseError(errorSet, lineNum, parseErrorSet);
if (isKeyDeclear)
{
continue;
}
else
{
finishReadHead = true;
}
}
PolyruleKeys.Instance.KeyTypes = _keyTypes;
parseErrorSet.Clear();
bool isKeyLine = TryParseKeyLine(trimedLine,
ref polyphonyWord, parseErrorSet, domain);
domain = DomainItem.GeneralDomain;
AddParseError(errorSet, lineNum, parseErrorSet);
if (isKeyLine)
{
continue;
}
parseErrorSet.Clear();
bool isConditionLine = TryParseConditionLine(trimedLine, phoneSet,
polyphonyWord, parseErrorSet);
AddParseError(errorSet, lineNum, parseErrorSet);
if (isConditionLine)
{
continue;
}
errorSet.Add(PolyRuleError.InvalidLineFormat,
lineNum.ToString(CultureInfo.InvariantCulture), trimedLine);
}
if (polyphonyWord != null)
{
_polyphonyWords.Add(polyphonyWord);
}
if (string.IsNullOrEmpty(_keyString))
{
errorSet.Add(PolyRuleError.MissPrimaryKey,
filePath);
}
errorSet.AddRange(CheckDupWordDefinitions());
foreach (PolyphonyRule rule in _polyphonyWords)
{
errorSet.AddRange(rule.CheckDupRuleConditions());
}
return errorSet;
}
/// <summary>
/// Find unmatching sentences between filemap file and script file
/// <param />
/// This function should be merged with that in forcedalignment into common library.
/// </summary>
/// <param name="scriptFilePath">The location of script file.</param>
/// <param name="language">Language of the script file.</param>
/// <param name="mapFilePath">The location of file fist map path.</param>
/// <returns>Unmatching sentence ids.</returns>
public static ErrorSet FindUnmatchedSentences(string scriptFilePath,
Language language, string mapFilePath)
{
ErrorSet errorSet = new ErrorSet();
TtsPhoneSet phoneSet = Localor.GetPhoneSet(language);
XmlScriptValidateSetting validateSetting = new XmlScriptValidateSetting(phoneSet, null);
XmlScriptFile script = XmlScriptFile.LoadWithValidation(scriptFilePath, validateSetting);
script.Remove(ScriptHelper.GetNeedDeleteItemIds(script.ErrorSet));
Dictionary<string, string> map = Microsoft.Tts.Offline.FileListMap.ReadAllData(mapFilePath);
errorSet.Merge(script.ErrorSet);
foreach (string sid in script.ItemDic.Keys)
{
if (!map.ContainsKey(sid))
{
string message = Helper.NeutralFormat(
"Script item {0} in script file but not in file list map file", sid);
errorSet.Add(ScriptError.OtherErrors, sid, message);
}
}
foreach (string sid in map.Keys)
{
if (!script.ItemDic.ContainsKey(sid))
{
string message = Helper.NeutralFormat(
"Script item {0} in file list map file but not in script file", sid);
errorSet.Add(ScriptError.OtherErrors, sid, message);
}
}
return errorSet;
}
/// <summary>
/// AddParseError.
/// </summary>
/// <param name="errorSet">ErrorSet.</param>
/// <param name="lineNum">LineNum.</param>
/// <param name="parseErrorSet">ParseErrorSet.</param>
private void AddParseError(ErrorSet errorSet, int lineNum, ErrorSet parseErrorSet)
{
foreach (Error parseError in parseErrorSet.Errors)
{
Error error = new Error(PolyRuleError.ParseError, parseError,
lineNum.ToString(CultureInfo.InvariantCulture));
// Keep the same severity with the original error severity.
error.Severity = parseError.Severity;
errorSet.Add(error);
}
}
/// <summary>
/// TryParseConditionLine.
/// </summary>
/// <param name="line">Line.</param>
/// <param name="phoneSet">PhoneSet.</param>
/// <param name="polyphonyWord">PolyphonyWord.</param>
/// <param name="errorSet">ErrorSet.</param>
/// <returns>Whether the line is condition line.</returns>
private bool TryParseConditionLine(string line, TtsPhoneSet phoneSet,
PolyphonyRule polyphonyWord, ErrorSet errorSet)
{
bool isConditionLine = false;
if (IsConditionLine(line))
{
isConditionLine = true;
if (polyphonyWord == null)
{
errorSet.Add(PolyRuleError.MissKeyValueLine, line);
}
errorSet.AddRange(ParseConditionLine(line, phoneSet, polyphonyWord));
}
return isConditionLine;
}
/// <summary>
/// Called before the action method that the attribute decorates is executed.
/// </summary>
///
/// <param name="actionContext">The context in which the action will be invoked.</param>
/// <param name="cancellationToken">A token which can be used to signal that the operation should be cancelled.</param>
///
/// <returns>The Task that represents the unit of work.</returns>
///
public override async Task OnActionExecutingAsync(HttpActionContext actionContext,
CancellationToken cancellationToken)
{
var locator = actionContext.Request.GetDependencyScope();
var failures = Enumerable.Empty <Error>();
// If there is an available message validator for the parameter passed to the controller, use it
// to validate the parameter.
foreach (var pair in actionContext.ActionArguments)
{
var value = pair.Value;
if (value == null)
{
continue;
}
var validatorType = ValidateMessageAttribute.BaseValidatorInterfaceType.MakeGenericType(value.GetType());
var validator = locator.GetService(validatorType) as IValidator;
if (validator != null)
{
failures = failures.Concat(validator.Validate(value)
.Select(error => new Error(error.Code, ValidateMessageAttribute.FormatMemberPath(pair.Key, error.MemberPath), error.Description)));
}
}
// If there were failures, then consider the request invalid and return the error set.
if (failures.Any())
{
var errorSet = new ErrorSet(failures);
actionContext.Response = actionContext.Request.CreateResponse(HttpStatusCode.BadRequest, errorSet);
try
{
var logger = locator.GetService(typeof(ILogger)) as ILogger;
if (logger != null)
{
var request = actionContext.Request;
var body = await request.SafeReadContentAsStringAsync();
logger.WithCorrelationId(actionContext.Request?.GetOrderFulfillmentCorrelationId())
.Information($"Response: {{Response}} { Environment.NewLine } Message validation failed for {{Route}} with Headers: [{{Headers}}] { Environment.NewLine } Body: {{RequestBody}}. { Environment.NewLine } The following errors were observed: { Environment.NewLine }{{ErrorSet}}",
HttpStatusCode.BadRequest,
request?.RequestUri,
request?.Headers,
body,
errorSet);
}
}
catch
{
// Do nothing; logging is a non-critical operation that should not cause
// cascading failures.
}
}
}
