private static MultipartToken CollectOfKind(IList <Token> tokens, TokenKind kind, int tokenCount, int startIndex) { var result = new MultipartToken(); int excess = 0; for (int i = startIndex; i < tokens.Count; i++) { var token = tokens[i]; if (token.Kind == kind || token.Kind == TokenKind.Keyword) { result.Add(token); if (result.Count == (tokenCount + excess)) { break; } } else if (token.Kind == TokenKind.Owner) { if (result.Count == 1) { result.Add(token); excess++; } } else { break; } } return(result); }
internal AttributeDefinition(MultipartToken key) : this(key, MultipartToken.FromToken(new Token { Content = "true", Index = -1, Kind = TokenKind.None })) { }
private static MultipartToken Create(TokenKind kind, string content, int index) { return(MultipartToken.FromToken(new Token { Content = content, Index = index, Kind = kind })); }
private AttributeDefinition FindNextAttributeDefinition(IList <Token> tokens, ref int index) { for (int i = index; i < tokens.Count - 1; i++) { var leftToken = MatchTokenAsGreedy(tokens, i); if (leftToken == null) { continue; } index = leftToken.Match.Max(x => x.Index); var rightToken = MatchTokenAsGreedy(tokens, index + 1); if (rightToken == null) { if (leftToken.Type == null) { return(new AttributeDefinition(leftToken.Match, MultipartToken.FromToken(new Token { Content = "true", Index = -1, Kind = TokenKind.None }))); } return(new AttributeDefinition(MultipartToken.FromToken(leftToken.Type), leftToken.Match)); } index = rightToken.Match.Max(x => x.Index); return(new AttributeDefinition(leftToken.Match, rightToken.Match)); } if (tokens.Count - index == 1 && tokens[index].Kind == TokenKind.None) { // Todo: Should we check if it's a verb?! return(new AttributeDefinition(MultipartToken.FromToken(tokens[index]))); } return(null); }
public QueryDefinition Parse(IList <Token> tokens) { var result = new QueryDefinition(); var lemma = tokens[0]; var startIndex = 1; bool found = false; // Note: Adjective comes first! if (tokens.Count > 1 && tokens[0].Kind == TokenKind.None) { //startIndex++; int index = 0; var attributeDefinition = FindNextAttributeDefinition(tokens, ref index); if (attributeDefinition != null) { result.Properties.Add(attributeDefinition); lemma = tokens[index + 1]; startIndex = Math.Max(attributeDefinition.Key.Max(x => x.Index), attributeDefinition.Value.Max(x => x.Index)) + 2; found = true; } if (found == false) { if (_properties.ContainsKey(tokens[0].Content.ToLowerInvariant())) { result.Properties.Add(new AttributeDefinition(MultipartToken.FromToken(new Token { Kind = TokenKind.None, Content = _properties[tokens[0].Content.ToLowerInvariant()] }), MultipartToken.FromToken(new Token { Kind = TokenKind.None, Content = tokens[0].Content.ToLowerInvariant() }) )); } else { // Todo: Determine if it's a bool property or if it's non-bool property! result.Properties.Add(new AttributeDefinition(MultipartToken.FromToken(new Token { Kind = TokenKind.None, Content = tokens[0].Content }))); } } } // Note: Regular Parsing -> Propery - Value Pair! if (lemma.Kind == TokenKind.Keyword) { // Note: Now that we have a keyword find the next two tokens! If we can! var index = startIndex; var propertyValuePair = FindNextAttributeDefinition(tokens, ref index); result.Target = TokenNormalizer.CaseNormalizeLemma(lemma.Content).ToLowerInvariant(); while (propertyValuePair != null) { result.Properties.Add(propertyValuePair); // Todo: We should be counting, instead of always adding one!! index += 1; propertyValuePair = FindNextAttributeDefinition(tokens, ref index); } } return(result); }
private TokenMatch MatchTokenAsGreedy(IList <Token> tokens, int index) { foreach (var item in CollectAllOfKind(tokens, TokenKind.None, index).Concat(CollectAllOfKind2(tokens, TokenKind.None, index))) { if (item.IsEmpty()) { continue; } var s1 = string.Join(" ", item.Select(x => x.NormalizedContentFull)); var s2 = string.Join(" ", item.Select(x => x.NormalizedContentPartial)); var s3 = string.Join(" ", item.Select(x => x.NormalizedContentNumber)); var s4 = string.Join(" ", item.Select(x => x.Content)); if (_properties.ContainsKey(s1)) { var token = new Token { Content = _properties[s1], Index = -1, Kind = TokenKind.None }; return(new TokenMatch(item, token)); } if (_properties.ContainsKey(s2)) { var token = new Token { Content = _properties[s2], Index = -1, Kind = TokenKind.None }; return(new TokenMatch(item, token)); } if (_temporal.ContainsKey(s1)) { var token = new Token { Content = s1, Index = -1, Kind = TokenKind.None }; return(new TokenMatch(item, token)); } if (_temporal.ContainsKey(s2)) { var token = new Token { Content = s2, Index = -1, Kind = TokenKind.None }; return(new TokenMatch(item, token)); } // Todo: Rename! foreach (var rename in _temporal.Where(x => x.Value.Item1)) { var regex = new Regex(rename.Key); if (regex.IsMatch(s1)) { var group = regex.Match(s1).Groups["Number"]; var token = new Token { Content = s1, Index = -1, Kind = TokenKind.None }; item.Type = typeof(TimeSpan); item.Hint = rename.Value.Item2.Invoke(int.Parse(group.Value)); return(new TokenMatch(item, token)); } if (regex.IsMatch(s2)) { var group = regex.Match(s2).Groups["Number"]; var token = new Token { Content = s2, Index = -1, Kind = TokenKind.None, }; item.Type = typeof(TimeSpan); item.Hint = rename.Value.Item2.Invoke(int.Parse(group.Value)); return(new TokenMatch(item, token)); } if (regex.IsMatch(s3)) { var group = regex.Match(s3).Groups["Number"]; var token = new Token { Content = s3, Index = -1, Kind = TokenKind.None }; item.Type = typeof(TimeSpan); item.Hint = rename.Value.Item2.Invoke(int.Parse(group.Value)); return(new TokenMatch(item, token)); } if (regex.IsMatch(s4)) { var group = regex.Match(s4).Groups["Number"]; var token = new Token { Content = s4, Index = -1, Kind = TokenKind.None }; item.Type = typeof(TimeSpan); item.Hint = rename.Value.Item2.Invoke(int.Parse(group.Value)); return(new TokenMatch(item, token)); } } if (item.Count == 3 && item[1].Kind == TokenKind.Owner) { return(new TokenMatch(new MultipartToken { item[0], item[1], })); } } return(tokens[index].Kind == TokenKind.None ? new TokenMatch(MultipartToken.FromToken(tokens[index])) : null); }
internal TokenMatch(MultipartToken match) : this(match, null) { }
internal TokenMatch(MultipartToken match, Token type) { Match = match; Type = type; }
internal AttributeDefinition(MultipartToken key, MultipartToken value) { Key = key; Value = value; }