private Features GetFeatures(List <FeatureTemplate> list, Token token, IndexedSentence sentence, bool?local = null)
        {
            Features features = new Features(list.Count);

            foreach (FeatureTemplate template in list)
            {
                if (local == null || local == template.IsLocal)
                {
                    features.Add(template.Name, template.GetValue(token, sentence));
                }
            }

            return(features);
        }
        public void Tag(IEnumerable <Sentence> sentences)
        {
            if (perceptronMsd == null)
            {
                throw new InvalidOperationException("Model not loaded.");
            }

            Parallel.ForEach(sentences, (sentence, index) =>
            {
                IndexedSentence normalizedSentence =
                    new IndexedSentence((Reverse ? (sentence as IEnumerable <Token>).Reverse() : sentence).Select(t => Normalize(t)));

                foreach (Token token in normalizedSentence)
                {
                    Tag[] possibleTags = token.PossibleTags;
                    if (possibleTags == null || possibleTags.Length == 0)
                    {
                        possibleTags = perceptronMsd.Tags.ToArray();
                    }

                    if (possibleTags.Length == 1)
                    {
                        token.PredictedTag = possibleTags[0];
                    }
                    else
                    {
                        double?maxScoreMsd = null;
                        Tag bestMsd        = null;

                        Features localFeaturesTag = GetFeatures(featureTemplatesTag, token, normalizedSentence, true);

                        foreach (Tag tag in possibleTags)
                        {
                            Tag tagMsd         = new Tag(tag.Msd);
                            token.PredictedTag = tagMsd;

                            Features featuresTag = GetFeatures(featureTemplatesTag, token, normalizedSentence, false);
                            featuresTag.AddRange(localFeaturesTag);

                            double scoreMsd = perceptronMsd.Score(featuresTag, tagMsd);
                            if (maxScoreMsd == null || scoreMsd > maxScoreMsd.Value)
                            {
                                maxScoreMsd = scoreMsd;
                                bestMsd     = tagMsd;
                            }
                        }

                        double?maxScoreLemma = null;
                        string bestLemma     = null;

                        token.PredictedTag          = bestMsd;
                        Features localFeaturesLemma = GetFeatures(featureTemplatesLemma, token, normalizedSentence, true);

                        foreach (Tag tag in possibleTags)
                        {
                            if (tag.Msd != bestMsd.Msd || tag.Lemma == null)
                            {
                                continue;
                            }

                            token.PredictedTag     = new Tag(token.CorrectTag.Msd, tag.Lemma);
                            Features featuresLemma = GetFeatures(featureTemplatesLemma, token, normalizedSentence, false);
                            featuresLemma.AddRange(localFeaturesLemma);

                            double scoreLemma = perceptronLemma.Score(featuresLemma, tag.Lemma);
                            if (maxScoreLemma == null || scoreLemma > maxScoreLemma.Value)
                            {
                                maxScoreLemma = scoreLemma;
                                bestLemma     = tag.Lemma;
                            }
                        }

                        token.PredictedTag = new Tag(bestMsd.Msd, bestLemma);
                    }

                    int pos = normalizedSentence[token];
                    if (Reverse)
                    {
                        pos = normalizedSentence.Count - pos - 1;
                    }
                    Token original        = sentence[pos];
                    original.PredictedTag = token.PredictedTag;
                    Unnormalize(original);
                }
            });
        }
 public abstract string GetValue(Token token, IndexedSentence sentence);