Exemplo n.º 1
0
        public virtual IList <SegToken> GetShortPath()
        {
            int current;
            int nodeCount             = ToCount;
            IList <PathNode> path     = new List <PathNode>();
            PathNode         zeroPath = new PathNode();

            zeroPath.Weight  = 0;
            zeroPath.PreNode = 0;
            path.Add(zeroPath);
            for (current = 1; current <= nodeCount; current++)
            {
                double weight;
                IList <SegTokenPair> edges = GetToList(current);

                double       minWeight = double.MaxValue;
                SegTokenPair minEdge   = null;
                foreach (SegTokenPair edge in edges)
                {
                    weight = edge.Weight;
                    PathNode preNode2 = path[edge.From];
                    if (preNode2.Weight + weight < minWeight)
                    {
                        minWeight = preNode2.Weight + weight;
                        minEdge   = edge;
                    }
                }
                PathNode newNode = new PathNode();
                newNode.Weight  = minWeight;
                newNode.PreNode = minEdge.From;
                path.Add(newNode);
            }

            // Calculate PathNodes
            int preNode, lastNode;

            lastNode = path.Count - 1;
            current  = lastNode;
            IList <int>      rpath      = new List <int>();
            IList <SegToken> resultPath = new List <SegToken>();

            rpath.Add(current);
            while (current != 0)
            {
                PathNode currentPathNode = path[current];
                preNode = currentPathNode.PreNode;
                rpath.Add(preNode);
                current = preNode;
            }
            for (int j = rpath.Count - 1; j >= 0; j--)
            {
                //int idInteger = rpath.get(j);
                //int id = idInteger.intValue();
                int      id = rpath[j];
                SegToken t  = segTokenList[id];
                resultPath.Add(t);
            }
            return(resultPath);
        }
Exemplo n.º 2
0
        /// <summary>
        /// <see cref="object.Equals(object)"/>
        /// </summary>
        public override bool Equals(object obj)
        {
            if (this == obj)
            {
                return(true);
            }
            if (obj == null)
            {
                return(false);
            }
            if (GetType() != obj.GetType())
            {
                return(false);
            }
            SegToken other = (SegToken)obj;

            if (!Arrays.Equals(CharArray, other.CharArray))
            {
                return(false);
            }
            if (EndOffset != other.EndOffset)
            {
                return(false);
            }
            if (Index != other.Index)
            {
                return(false);
            }
            if (StartOffset != other.StartOffset)
            {
                return(false);
            }
            if (Weight != other.Weight)
            {
                return(false);
            }
            if (WordType != other.WordType)
            {
                return(false);
            }
            return(true);
        }
Exemplo n.º 3
0
        /// <summary>
        /// Add a <see cref="SegToken"/> to the mapping, creating a new mapping at the token's startOffset if one does not exist.
        /// </summary>
        /// <param name="token">token <see cref="SegToken"/>.</param>
        public virtual void AddToken(SegToken token)
        {
            int s = token.StartOffset;

            if (!IsStartExist(s))
            {
                List <SegToken> newlist = new List <SegToken>();
                newlist.Add(token);
                tokenListTable[s] = newlist;
            }
            else
            {
                IList <SegToken> tokenList = tokenListTable[s];
                tokenList.Add(token);
            }
            if (s > maxStart)
            {
                maxStart = s;
            }
        }
Exemplo n.º 4
0
        private static readonly WordDictionary wordDict = WordDictionary.GetInstance(); // LUCENENET: marked readonly

        /// <summary>
        /// Create the <see cref="SegGraph"/> for a sentence.
        /// </summary>
        /// <param name="sentence">input sentence, without start and end markers</param>
        /// <returns><see cref="SegGraph"/> corresponding to the input sentence.</returns>
        private SegGraph CreateSegGraph(string sentence)
        {
            int i = 0, j;
            int length = sentence.Length;
            int foundIndex;

            CharType[]    charTypeArray = GetCharTypes(sentence);
            StringBuilder wordBuf       = new StringBuilder();
            SegToken      token;
            int           frequency; // the number of times word appears. // LUCENENET: IDE0059: Remove unnecessary value assignment
            bool          hasFullWidth;
            WordType      wordType;

            char[] charArray;

            SegGraph segGraph = new SegGraph();

            while (i < length)
            {
                hasFullWidth = false;
                switch (charTypeArray[i])
                {
                case CharType.SPACE_LIKE:
                    i++;
                    break;

                case CharType.HANZI:
                    j = i + 1;
                    //wordBuf.delete(0, wordBuf.length());
                    wordBuf.Remove(0, wordBuf.Length);
                    // It doesn't matter if a single Chinese character (Hanzi) can form a phrase or not,
                    // it will store that single Chinese character (Hanzi) in the SegGraph.  Otherwise, it will
                    // cause word division.
                    wordBuf.Append(sentence[i]);
                    charArray = new char[] { sentence[i] };
                    frequency = wordDict.GetFrequency(charArray);
                    token     = new SegToken(charArray, i, j, WordType.CHINESE_WORD,
                                             frequency);
                    segGraph.AddToken(token);

                    foundIndex = wordDict.GetPrefixMatch(charArray);
                    while (j <= length && foundIndex != -1)
                    {
                        if (wordDict.IsEqual(charArray, foundIndex) && charArray.Length > 1)
                        {
                            // It is the phrase we are looking for; In other words, we have found a phrase SegToken
                            // from i to j.  It is not a monosyllabic word (single word).
                            frequency = wordDict.GetFrequency(charArray);
                            token     = new SegToken(charArray, i, j, WordType.CHINESE_WORD,
                                                     frequency);
                            segGraph.AddToken(token);
                        }

                        while (j < length && charTypeArray[j] == CharType.SPACE_LIKE)
                        {
                            j++;
                        }

                        if (j < length && charTypeArray[j] == CharType.HANZI)
                        {
                            wordBuf.Append(sentence[j]);
                            charArray = new char[wordBuf.Length];
                            //wordBuf.GetChars(0, charArray.Length, charArray, 0);
                            wordBuf.CopyTo(0, charArray, 0, charArray.Length);
                            // idArray has been found (foundWordIndex!=-1) as a prefix before.
                            // Therefore, idArray after it has been lengthened can only appear after foundWordIndex.
                            // So start searching after foundWordIndex.
                            foundIndex = wordDict.GetPrefixMatch(charArray, foundIndex);
                            j++;
                        }
                        else
                        {
                            break;
                        }
                    }
                    i++;
                    break;

                case CharType.FULLWIDTH_LETTER:
                    hasFullWidth = true;     /* intentional fallthrough */

                    j = i + 1;
                    while (j < length &&
                           (charTypeArray[j] == CharType.LETTER || charTypeArray[j] == CharType.FULLWIDTH_LETTER))
                    {
                        if (charTypeArray[j] == CharType.FULLWIDTH_LETTER)
                        {
                            hasFullWidth = true;
                        }
                        j++;
                    }
                    // Found a Token from i to j. Type is LETTER char string.
                    charArray = Utility.STRING_CHAR_ARRAY;
                    frequency = wordDict.GetFrequency(charArray);
                    wordType  = hasFullWidth ? WordType.FULLWIDTH_STRING : WordType.STRING;
                    token     = new SegToken(charArray, i, j, wordType, frequency);
                    segGraph.AddToken(token);
                    i = j;
                    break;

                case CharType.LETTER:
                    j = i + 1;
                    while (j < length &&
                           (charTypeArray[j] == CharType.LETTER || charTypeArray[j] == CharType.FULLWIDTH_LETTER))
                    {
                        if (charTypeArray[j] == CharType.FULLWIDTH_LETTER)
                        {
                            hasFullWidth = true;
                        }
                        j++;
                    }
                    // Found a Token from i to j. Type is LETTER char string.
                    charArray = Utility.STRING_CHAR_ARRAY;
                    frequency = wordDict.GetFrequency(charArray);
                    wordType  = hasFullWidth ? WordType.FULLWIDTH_STRING : WordType.STRING;
                    token     = new SegToken(charArray, i, j, wordType, frequency);
                    segGraph.AddToken(token);
                    i = j;
                    break;

                case CharType.FULLWIDTH_DIGIT:
                    hasFullWidth = true;     /* intentional fallthrough */

                    j = i + 1;
                    while (j < length &&
                           (charTypeArray[j] == CharType.DIGIT || charTypeArray[j] == CharType.FULLWIDTH_DIGIT))
                    {
                        if (charTypeArray[j] == CharType.FULLWIDTH_DIGIT)
                        {
                            hasFullWidth = true;
                        }
                        j++;
                    }
                    // Found a Token from i to j. Type is NUMBER char string.
                    charArray = Utility.NUMBER_CHAR_ARRAY;
                    frequency = wordDict.GetFrequency(charArray);
                    wordType  = hasFullWidth ? WordType.FULLWIDTH_NUMBER : WordType.NUMBER;
                    token     = new SegToken(charArray, i, j, wordType, frequency);
                    segGraph.AddToken(token);
                    i = j;
                    break;

                case CharType.DIGIT:
                    j = i + 1;
                    while (j < length &&
                           (charTypeArray[j] == CharType.DIGIT || charTypeArray[j] == CharType.FULLWIDTH_DIGIT))
                    {
                        if (charTypeArray[j] == CharType.FULLWIDTH_DIGIT)
                        {
                            hasFullWidth = true;
                        }
                        j++;
                    }
                    // Found a Token from i to j. Type is NUMBER char string.
                    charArray = Utility.NUMBER_CHAR_ARRAY;
                    frequency = wordDict.GetFrequency(charArray);
                    wordType  = hasFullWidth ? WordType.FULLWIDTH_NUMBER : WordType.NUMBER;
                    token     = new SegToken(charArray, i, j, wordType, frequency);
                    segGraph.AddToken(token);
                    i = j;
                    break;

                case CharType.DELIMITER:
                    j = i + 1;
                    // No need to search the weight for the punctuation.  Picking the highest frequency will work.
                    frequency = Utility.MAX_FREQUENCE;
                    charArray = new char[] { sentence[i] };
                    token     = new SegToken(charArray, i, j, WordType.DELIMITER, frequency);
                    segGraph.AddToken(token);
                    i = j;
                    break;

                default:
                    j = i + 1;
                    // Treat the unrecognized char symbol as unknown string.
                    // For example, any symbol not in GB2312 is treated as one of these.
                    charArray = Utility.STRING_CHAR_ARRAY;
                    frequency = wordDict.GetFrequency(charArray);
                    token     = new SegToken(charArray, i, j, WordType.STRING, frequency);
                    segGraph.AddToken(token);
                    i = j;
                    break;
                }
            }

            // Add two more Tokens: "beginning xx beginning"
            charArray = Utility.START_CHAR_ARRAY;
            frequency = wordDict.GetFrequency(charArray);
            token     = new SegToken(charArray, -1, 0, WordType.SENTENCE_BEGIN, frequency);
            segGraph.AddToken(token);

            // "end xx end"
            charArray = Utility.END_CHAR_ARRAY;
            frequency = wordDict.GetFrequency(charArray);
            token     = new SegToken(charArray, length, length + 1, WordType.SENTENCE_END,
                                     frequency);
            segGraph.AddToken(token);

            return(segGraph);
        }