private void TestPerInputNodeAttributes()
        {
            _markovChainString.Children.Clear();

            string inputString = "This string contains per node attributes.";

            MarkovChainInputString markovChainInputString = Preprocess(inputString, int.MaxValue);

            foreach (MarkovChainInputNodeString markovChainInputNodeString in markovChainInputString.Nodes)
            {
                if (markovChainInputNodeString.String.Contains("i"))
                {
                    markovChainInputNodeString.Attributes.Add("perNodeAttribute", 1);
                }
            }

            Dictionary <string, double> globalAttributes = new Dictionary <string, double>();

            globalAttributes.Add("globalAttribute1", 1);
            globalAttributes.Add("globalAttribute2", 1);

            AddMarkovChainString(markovChainInputString, false, globalAttributes);

            MarkovChainString markovChainString = GetMarkovChainString(inputString, false, false);
        }
        public MarkovChainInputString Preprocess(string input, int maximumNumberOfNodesToProcess)
        {
            MarkovChainInputString markovChainInputString = new MarkovChainInputString();

            markovChainInputString.Nodes = new List <MarkovChainInputNodeString>();

            foreach (string word in UserDefinedFunctions.ExtractWords(input, _extractText, _extractDistinctWords))
            {
                MarkovChainInputNodeString markovChainInputNodeString = new MarkovChainInputNodeString();

                markovChainInputNodeString.String     = word;
                markovChainInputNodeString.Attributes = new Dictionary <string, double>();

                markovChainInputString.Nodes.Add(markovChainInputNodeString);
            }

            markovChainInputString.Nodes = markovChainInputString.Nodes.Take(maximumNumberOfNodesToProcess).ToList();

            return(markovChainInputString);
        }
        public MarkovChainNodeString AddMarkovChainString(MarkovChainInputString markovChainInputString, bool addMarkovChainBreaks, Dictionary <string, double> attributes)
        {
            if (markovChainInputString == null || markovChainInputString.Nodes == null || markovChainInputString.Nodes.Count == 0 || string.IsNullOrEmpty(markovChainInputString.ToString()))
            {
                return(null);
            }

            MarkovChainNodeString parent = null;
            MarkovChainNodeString currentMarkovChainNodeString = null;
            bool   isFirstWord = true;
            string lastWord    = null;

            if (!_isCaseSensitive)
            {
                for (int i = 0; i < markovChainInputString.Nodes.Count; i++)
                {
                    markovChainInputString.Nodes[i].String = markovChainInputString.Nodes[i].String.ToLowerInvariant();
                }
            }

            int    index = 1;
            string path  = null;

            foreach (MarkovChainInputNodeString markovChainInputNodeString in markovChainInputString.Nodes)
            {
                string word2 = string.Intern(markovChainInputNodeString.String.Trim());
                lastWord = word2;
                path    += word2 + "\\";

                if (isFirstWord)
                {
                    isFirstWord = false;

                    if (!MarkovChainString.Children.ContainsKey(word2))
                    {
                        MarkovChainNodeString markovChainNodeString = new MarkovChainNodeString(typeof(MCStorageString), false, _onDiskDirectoryBasePath, path);

                        MarkovChainString.Children.Add(word2, markovChainNodeString);
                    }

                    MarkovChainString.ChainCount++;

                    ManageAttributes(MarkovChainString, attributes);

                    MarkovChainString.Update();

                    /**/

                    currentMarkovChainNodeString = MarkovChainString.Children[word2];
                    currentMarkovChainNodeString.ChainCount++;
                    currentMarkovChainNodeString.Children.Path   = path;
                    currentMarkovChainNodeString.IsInputBoundary = false;
                    currentMarkovChainNodeString.IsWordBoundary  = true;
                    //commented to allow the model to be serialiazed to JSON...  should be re-added once disk-backed storage is added...
                    //currentMarkovChainNodeString.Parent = MarkovChainString
                    currentMarkovChainNodeString.Index = index++;

                    ManageAttributes(currentMarkovChainNodeString, attributes);

                    MarkovChainString.Children[word2] = currentMarkovChainNodeString;
                    //currentMarkovChainNodeString.Update();

                    continue;
                }

                if (!currentMarkovChainNodeString.Children.ContainsKey(word2))
                {
                    MarkovChainNodeString markovChainNodeString = new MarkovChainNodeString(typeof(MCStorageString), false, _onDiskDirectoryBasePath, path);

                    currentMarkovChainNodeString.Children.Add(word2, markovChainNodeString);
                }

                parent = currentMarkovChainNodeString;
                currentMarkovChainNodeString      = currentMarkovChainNodeString.Children[word2];
                currentMarkovChainNodeString.Path = path;
                currentMarkovChainNodeString.ChainCount++;
                currentMarkovChainNodeString.Children.Path   = path;
                currentMarkovChainNodeString.IsInputBoundary = false;
                currentMarkovChainNodeString.IsWordBoundary  = true;
                //commented to allow the model to be serialiazed to JSON...  should be re-added once disk-backed storage is added...
                //currentMarkovChainNodeString.Parent = parent;
                currentMarkovChainNodeString.Index = index++;

                ManageAttributes(currentMarkovChainNodeString, attributes);

                if (addMarkovChainBreaks)
                {
                    //experimental...
                    if (!MarkovChainString.Children.ContainsKey(word2))
                    {
                        MarkovChainString.Children.Add(word2, currentMarkovChainNodeString);
                    }
                    else
                    {
                        ManageAttributes(currentMarkovChainNodeString, currentMarkovChainNodeString.Attributes);
                    }
                }

                parent.Children[word2] = currentMarkovChainNodeString;
                //currentMarkovChainNodeString.Update();
            }

            if (currentMarkovChainNodeString != null && parent != null && !string.IsNullOrEmpty(lastWord))
            {
                currentMarkovChainNodeString.IsInputBoundary = true;
                parent.Children[lastWord] = currentMarkovChainNodeString;
                //currentMarkovChainNodeString.Update();
            }

            return(MarkovChainString);
        }

        private void ManageAttributes(MarkovChainNodeString currentMarkovChainNodeString, Dictionary <string, double> attributes)
        {
            if (attributes != null)
            {
                if (currentMarkovChainNodeString.Attributes == null)
                {
                    currentMarkovChainNodeString.Attributes = new Dictionary <string, double>();
                }

                List <string> keys = attributes.Keys.ToList();

                foreach (string key in keys)
                {
                    if (!currentMarkovChainNodeString.Attributes.ContainsKey(key))
                    {
                        currentMarkovChainNodeString.Attributes.Add(key, attributes[key]);
                    }
                    else
                    {
                        currentMarkovChainNodeString.Attributes[key] += attributes[key];
                    }
                }
            }
        }

        public void PrepareMarkovChainString()
        {
            PrepareMarkovChainString(MarkovChainString);
        }

        private void PrepareMarkovChainString(MarkovChainNodeString markovChainNodeString)
        {
            markovChainNodeString.Children = new MarkovChainNodeStorageInMemoryDictionaryString(markovChainNodeString.Children.OrderByDescendingChainCount(), _onDiskDirectoryBasePath);

            foreach (KeyValuePair <string, MarkovChainNodeString> keyValuePair in markovChainNodeString.Children)
            {
                PrepareMarkovChainString(keyValuePair.Value);
            }
        }

        public MarkovChainString GetMarkovChainString(string input, bool decrementChainCount, bool alwaysContinueToEndOfChain)
        {
            if (string.IsNullOrEmpty(input))
            {
                return(null);
            }

            MarkovChainString markovChainString = new MarkovChainString();

            markovChainString.IsBrokenChain = false;
            markovChainString.Nodes         = new List <MarkovChainNodeString>();
            markovChainString.Input         = input;

            bool continueProcessing = true;

            if (MarkovChainString.Children.Count != 0)
            {
                MarkovChainNodeString startingNodeString = MarkovChainString;
                decimal chainCount      = 0;
                decimal chainCountTotal = 0;
                string  path            = null;

                StringBuilder chainString = new StringBuilder();

                if (!string.IsNullOrEmpty(input))
                {
                    if (!_isCaseSensitive)
                    {
                        input = input.ToLowerInvariant();
                    }

                    Func <string, bool> processWord = new Func <string, bool>(delegate(string word)
                    {
                        try
                        {
                            markovChainString.Nodes.Add(startingNodeString);

                            chainCount      += startingNodeString.Children[word].ChainCount;
                            chainCountTotal += startingNodeString.Children.Values.Sum(_ => _.ChainCount);

                            chainString.Append(word + " ");

                            startingNodeString = startingNodeString.Children[word];
                            if (decrementChainCount)
                            {
                                startingNodeString.ChainCount--;
                            }

                            startingNodeString.Path          = path;
                            startingNodeString.Children.Path = path;
                        }
                        catch (Exception)
                        {
                            return(false);
                        }

                        return(true);
                    });

                    List <string> words = UserDefinedFunctions.ExtractWords(input, _extractText, _extractDistinctWords).Cast <string>().ToList();

                    foreach (string word in words)
                    {
                        path += word + "\\";

                        if (startingNodeString.Children.ContainsKey(word))
                        {
                            processWord(word);
                        }
                        else if (MarkovChainString.Children.ContainsKey(word))
                        {
                            startingNodeString = MarkovChainString;

                            markovChainString.IsBrokenChain = true;

                            processWord(word);
                        }
                        else
                        {
                            if (startingNodeString == MarkovChainString)
                            {
                                return(markovChainString);
                            }

                            break;
                        }
                    }

                    if (markovChainString.Nodes.Count == words.Count())
                    {
                        continueProcessing = false;
                    }
                }

                /**/

                markovChainString.ChainCountKnown      = chainCount;
                markovChainString.ChainCountTotalKnown = chainCountTotal;

                if (continueProcessing || alwaysContinueToEndOfChain)
                {
                    KeyValuePair <string, MarkovChainNodeString> keyValuePair = startingNodeString.Children.OrderByDescendingChainCount().FirstOrDefault();

                    while (keyValuePair.Key != null && startingNodeString.Children.Values.Count != 0)
                    {
                        markovChainString.Nodes.Add(startingNodeString);

                        chainCount      += startingNodeString.Children[keyValuePair.Key].ChainCount;
                        chainCountTotal += startingNodeString.Children.Values.Sum(_ => _.ChainCount);

                        chainString.Append(keyValuePair.Key + " ");

                        startingNodeString = startingNodeString.Children[keyValuePair.Key];
                        if (decrementChainCount)
                        {
                            keyValuePair.Value.ChainCount--;
                        }

                        path += keyValuePair.Key + "\\";

                        startingNodeString.Path          = path;
                        startingNodeString.Children.Path = path;

                        keyValuePair = startingNodeString.Children.OrderByDescendingChainCount().FirstOrDefault();
                    }
                }

                markovChainString.String          = chainString.ToString().Trim();
                markovChainString.ChainCount      = chainCount;
                markovChainString.ChainCountTotal = chainCountTotal;
            }

            return(markovChainString);
        }