Ejemplo n.º 1
0
        public bool Load(string sFilename)
        {
            bool isSuccess = true;
            ContextItem pCur = null, pPre = null;

            FileStream fileStream = null;
            BinaryReader binReader = null;

            try
            {
                fileStream = new FileStream(sFilename, FileMode.Open, FileAccess.Read);
                if (fileStream == null)
                    return false;

                binReader = new BinaryReader(fileStream, Encoding.GetEncoding("gb2312"));


                m_nTableLen = binReader.ReadInt32();   //write the table length
                m_pSymbolTable = new int[m_nTableLen]; //new buffer for symbol

                for (int i = 0; i < m_nTableLen; i++)  //write the symbol table
                    m_pSymbolTable[i] = binReader.ReadInt32();

                while (binReader.PeekChar() != -1)
                {
                    //Read the context 
                    pCur = new ContextItem();
                    pCur.next = null;
                    pCur.nKey = binReader.ReadInt32();
                    pCur.nTotalFreq = binReader.ReadInt32();

                    pCur.aTagFreq = new int[m_nTableLen];
                    for (int i = 0; i < m_nTableLen; i++)     //the every POS frequency
                        pCur.aTagFreq[i] = binReader.ReadInt32();


                    pCur.aContextArray = new int[m_nTableLen][];
                    for (int i = 0; i < m_nTableLen; i++)
                    {
                        pCur.aContextArray[i] = new int[m_nTableLen];
                        for (int j = 0; j < m_nTableLen; j++)
                            pCur.aContextArray[i][j] = binReader.ReadInt32();
                    }

                    if (pPre == null)
                        m_pContext = pCur;
                    else
                        pPre.next = pCur;

                    pPre = pCur;
                }
            }
            catch (Exception e)
            {
                Console.WriteLine(e.Message);
                isSuccess = false;
            }
            finally
            {
                if (binReader != null)
                    binReader.Close();

                if (fileStream != null)
                    fileStream.Close();
            }

            return isSuccess;
        }
Ejemplo n.º 2
0
 public void ReleaseContextStat()
 {
     m_pContext = null;
     m_pSymbolTable = null;
 }
Ejemplo n.º 3
0
        public bool Add(int nKey, int nPrevSymbol, int nCurSymbol, int nFrequency)
        {
            //Add the context symbol to the array
            ContextItem pRetItem, pNew;
            int nPrevIndex, nCurIndex;

            //Not get it
            if (!GetItem(nKey, out pRetItem))
            {
                pNew = new ContextItem();
                pNew.nKey = nKey;
                pNew.nTotalFreq = 0;
                pNew.next = null;
                pNew.aContextArray = new int[m_nTableLen][];
                pNew.aTagFreq = new int[m_nTableLen];
                for (int i = 0; i < m_nTableLen; i++)
                    pNew.aContextArray[i] = new int[m_nTableLen];

                if (pRetItem == null)
                    //Empty, the new item is head
                    m_pContext = pNew;
                else
                //Link the new item between pRetItem and its next item
                {
                    pNew.next = pRetItem.next;
                    pRetItem.next = pNew;
                }
                pRetItem = pNew;
            }

            nPrevIndex = Utility.BinarySearch(nPrevSymbol, m_pSymbolTable);
            if (nPrevSymbol > 256 && nPrevIndex == -1)
                //Not find, just for 'nx' and other uncommon POS
                nPrevIndex = Utility.BinarySearch(nPrevSymbol - nPrevSymbol % 256, m_pSymbolTable);

            nCurIndex = Utility.BinarySearch(nCurSymbol, m_pSymbolTable);

            if (nCurSymbol > 256 && nCurIndex == -1)
                //Not find, just for 'nx' and other uncommon POS
                nCurIndex = Utility.BinarySearch(nCurSymbol - nCurSymbol % 256, m_pSymbolTable);

            if (nPrevIndex == -1 || nCurIndex == -1)
                //error finding the symbol
                return false;

            //Add the frequency
            pRetItem.aContextArray[nPrevIndex][nCurIndex] += nFrequency;
            pRetItem.aTagFreq[nPrevIndex] += nFrequency;
            pRetItem.nTotalFreq += nFrequency;

            return true;
        }
Ejemplo n.º 4
0
        //=========================================================
        // 返回nKey为指定nKey的结点,如果没找到,则返回前一个结点
        //=========================================================
        public bool GetItem(int nKey, out ContextItem pItemRet)
        {
            ContextItem pCur = m_pContext, pPrev = null;
            if (nKey == 0 && m_pContext != null)
            {
                pItemRet = m_pContext;
                return true;
            }

            while (pCur != null && pCur.nKey < nKey)
            {
                //delete the context array
                pPrev = pCur;
                pCur = pCur.next;
            }

            if (pCur != null && pCur.nKey == nKey)
            {
                //find it and return the current item
                pItemRet = pCur;
                return true;
            }

            pItemRet = pPrev;
            return false;
        }
Ejemplo n.º 5
0
 public void SetTableLen(int nTableLen)
 {
     m_nTableLen = nTableLen;
     m_pSymbolTable = new int[nTableLen];
     m_pContext = null;
 }
Ejemplo n.º 6
0
        public bool Load(string sFilename)
        {
            bool        isSuccess = true;
            ContextItem pCur = null, pPre = null;

            FileStream   fileStream = null;
            BinaryReader binReader  = null;

            try
            {
                fileStream = new FileStream(sFilename, FileMode.Open, FileAccess.Read);
                if (fileStream == null)
                {
                    return(false);
                }

                binReader = new BinaryReader(fileStream, Encoding.GetEncoding("gb2312"));


                m_nTableLen    = binReader.ReadInt32(); //write the table length
                m_pSymbolTable = new int[m_nTableLen];  //new buffer for symbol

                for (int i = 0; i < m_nTableLen; i++)   //write the symbol table
                {
                    m_pSymbolTable[i] = binReader.ReadInt32();
                }

                while (binReader.PeekChar() != -1)
                {
                    //Read the context
                    pCur            = new ContextItem();
                    pCur.next       = null;
                    pCur.nKey       = binReader.ReadInt32();
                    pCur.nTotalFreq = binReader.ReadInt32();

                    pCur.aTagFreq = new int[m_nTableLen];
                    for (int i = 0; i < m_nTableLen; i++) //the every POS frequency
                    {
                        pCur.aTagFreq[i] = binReader.ReadInt32();
                    }


                    pCur.aContextArray = new int[m_nTableLen][];
                    for (int i = 0; i < m_nTableLen; i++)
                    {
                        pCur.aContextArray[i] = new int[m_nTableLen];
                        for (int j = 0; j < m_nTableLen; j++)
                        {
                            pCur.aContextArray[i][j] = binReader.ReadInt32();
                        }
                    }

                    if (pPre == null)
                    {
                        m_pContext = pCur;
                    }
                    else
                    {
                        pPre.next = pCur;
                    }

                    pPre = pCur;
                }
            }
            catch (Exception e)
            {
                Console.WriteLine(e.Message);
                isSuccess = false;
            }
            finally
            {
                if (binReader != null)
                {
                    binReader.Close();
                }

                if (fileStream != null)
                {
                    fileStream.Close();
                }
            }

            return(isSuccess);
        }
Ejemplo n.º 7
0
        public bool Save(string sFilename)
        {
            bool          isSuccess  = true;
            FileStream    outputFile = null;
            FileStream    logFile    = null;
            BinaryWriter  writer     = null;
            StreamWriter  sw         = null;
            StringBuilder sb         = new StringBuilder();

            try
            {
                outputFile = new FileStream(sFilename, FileMode.Create, FileAccess.Write);
                if (outputFile == null)
                {
                    return(false);
                }

                logFile = new FileStream(sFilename + ".shw", FileMode.Create, FileAccess.Write);
                if (logFile == null)
                {
                    outputFile.Close();
                    return(false);
                }

                writer = new BinaryWriter(outputFile, Encoding.GetEncoding("gb2312"));
                sw     = new StreamWriter(logFile);

                writer.Write(m_nTableLen);            //write the table length
                sb.Append(string.Format("Table Len={0}\r\nSymbol:\r\n", m_nTableLen));
                for (int i = 0; i < m_nTableLen; i++) //write the symbol table
                {
                    writer.Write(m_pSymbolTable[i]);
                    sb.Append(string.Format("{0} ", m_pSymbolTable[i]));
                }
                sb.Append("\r\n");

                ContextItem pCur = m_pContext;
                while (pCur != null)
                {
                    writer.Write(pCur.nKey);
                    writer.Write(pCur.nTotalFreq);
                    sb.Append(string.Format("nKey={0},Total frequency={1}:\r\n", pCur.nKey, pCur.nTotalFreq));

                    for (int i = 0; i < m_nTableLen; i++)
                    {
                        writer.Write(pCur.aTagFreq[i]);
                    }

                    //the every POS frequency
                    for (int i = 0; i < m_nTableLen; i++)
                    {
                        for (int j = 0; j < m_nTableLen; j++)
                        {
                            writer.Write(pCur.aContextArray[i][j]);
                        }

                        sb.Append(string.Format("No.{0,2}={1,3}: ", i, m_pSymbolTable[i]));
                        for (int j = 0; j < m_nTableLen; j++)
                        {
                            sb.Append(string.Format("{0,5} ", pCur.aContextArray[i][j]));
                        }

                        sb.Append(string.Format("total={0}:\r\n", pCur.aTagFreq[i]));
                    }
                    pCur = pCur.next;
                }

                sw.Write(sb.ToString());
            }
            catch
            {
                isSuccess = false;
            }
            finally
            {
                if (writer != null)
                {
                    writer.Close();
                }

                if (outputFile != null)
                {
                    outputFile.Close();
                }

                if (sw != null)
                {
                    sw.Close();
                }

                if (logFile != null)
                {
                    logFile.Close();
                }
            }

            return(isSuccess);
        }
Ejemplo n.º 8
0
        public bool Add(int nKey, int nPrevSymbol, int nCurSymbol, int nFrequency)
        {
            //Add the context symbol to the array
            ContextItem pRetItem, pNew;
            int         nPrevIndex, nCurIndex;

            //Not get it
            if (!GetItem(nKey, out pRetItem))
            {
                pNew               = new ContextItem();
                pNew.nKey          = nKey;
                pNew.nTotalFreq    = 0;
                pNew.next          = null;
                pNew.aContextArray = new int[m_nTableLen][];
                pNew.aTagFreq      = new int[m_nTableLen];
                for (int i = 0; i < m_nTableLen; i++)
                {
                    pNew.aContextArray[i] = new int[m_nTableLen];
                }

                if (pRetItem == null)
                {
                    //Empty, the new item is head
                    m_pContext = pNew;
                }
                else
                //Link the new item between pRetItem and its next item
                {
                    pNew.next     = pRetItem.next;
                    pRetItem.next = pNew;
                }
                pRetItem = pNew;
            }

            nPrevIndex = Utility.BinarySearch(nPrevSymbol, m_pSymbolTable);
            if (nPrevSymbol > 256 && nPrevIndex == -1)
            {
                //Not find, just for 'nx' and other uncommon POS
                nPrevIndex = Utility.BinarySearch(nPrevSymbol - nPrevSymbol % 256, m_pSymbolTable);
            }

            nCurIndex = Utility.BinarySearch(nCurSymbol, m_pSymbolTable);

            if (nCurSymbol > 256 && nCurIndex == -1)
            {
                //Not find, just for 'nx' and other uncommon POS
                nCurIndex = Utility.BinarySearch(nCurSymbol - nCurSymbol % 256, m_pSymbolTable);
            }

            if (nPrevIndex == -1 || nCurIndex == -1)
            {
                //error finding the symbol
                return(false);
            }

            //Add the frequency
            pRetItem.aContextArray[nPrevIndex][nCurIndex] += nFrequency;
            pRetItem.aTagFreq[nPrevIndex] += nFrequency;
            pRetItem.nTotalFreq           += nFrequency;

            return(true);
        }
Ejemplo n.º 9
0
 public void ReleaseContextStat()
 {
     m_pContext     = null;
     m_pSymbolTable = null;
 }
Ejemplo n.º 10
0
 public void SetTableLen(int nTableLen)
 {
     m_nTableLen    = nTableLen;
     m_pSymbolTable = new int[nTableLen];
     m_pContext     = null;
 }