Exemplo n.º 1
0
        /**
         * get the region with a int ip address with memory binary search algorithm
         *
         * @param   ip
         * @throws  IOException
         */
        public DataBlock memorySearch(long ip)
        {
            int blen = IndexBlock.getIndexBlockLength();

            if (dbBinStr == null)
            {
                dbBinStr = new byte[(int)raf.length()];
                raf.seek(0L);
                raf.readFully(dbBinStr, 0, dbBinStr.Length);

                //initialize the global vars
                firstIndexPtr    = Util.getIntLong(dbBinStr, 0);
                lastIndexPtr     = Util.getIntLong(dbBinStr, 4);
                totalIndexBlocks = (int)((lastIndexPtr - firstIndexPtr) / blen) + 1;
            }

            //search the index blocks to define the data
            int  l = 0, h = totalIndexBlocks;
            long sip, eip, dataptr = 0;

            while (l <= h)
            {
                int m = (l + h) >> 1;
                int p = (int)(firstIndexPtr + m * blen);

                sip = Util.getIntLong(dbBinStr, p);
                if (ip < sip)
                {
                    h = m - 1;
                }
                else
                {
                    eip = Util.getIntLong(dbBinStr, p + 4);
                    if (ip > eip)
                    {
                        l = m + 1;
                    }
                    else
                    {
                        dataptr = Util.getIntLong(dbBinStr, p + 8);
                        break;
                    }
                }
            }

            //not matched
            if (dataptr == 0)
            {
                return(null);
            }

            //get the data
            int dataLen = (int)((dataptr >> 24) & 0xFF);
            int dataPtr = (int)((dataptr & 0x00FFFFFF));
            int city_id = (int)Util.getIntLong(dbBinStr, dataPtr);
            //String region = new String(dbBinStr, dataPtr + 4, dataLen - 4, "UTF-8");
            var region = ReadString(dbBinStr, dataPtr + 4, dataLen - 4);

            return(new DataBlock(city_id, region, dataPtr));
        }
Exemplo n.º 2
0
        /**
         * construct method with self-define std ip2region bianry string support
         * Thanks to the issue from Wendal at https://gitee.com/lionsoul/ip2region/issues/IILFL
         *
         * @param   dbConfig
         * @param   dbBinStr
         */
        public DbSearcher(DbConfig dbConfig, byte[] dbBinStr)
        {
            this.dbConfig = dbConfig;
            this.dbBinStr = dbBinStr;

            firstIndexPtr    = Util.getIntLong(dbBinStr, 0);
            lastIndexPtr     = Util.getIntLong(dbBinStr, 4);
            totalIndexBlocks = (int)((lastIndexPtr - firstIndexPtr) / IndexBlock.getIndexBlockLength()) + 1;
        }
Exemplo n.º 3
0
        /**
         * internal method to add a new data block record
         *
         * @param	raf
         * @param	startIp
         * @param	endIp
         * @param	region data
         */
        private void addDataBlock(
            RandomAccessFile raf,
            String startIp, String endIp, String region)
        {
            byte[] data    = region.getBytes("UTF-8");
            int    dataPtr = 0;

            /*byte[] city = new byte[4];
             * int city_id = getCityId(region);
             * Util.writeIntLong(city, 0, city_id);
             * dataPtr = (int)raf.getFilePointer();
             * raf.write(city);
             * raf.write(data);*/

            //check region ptr pool first
            if (regionPtrPool.containsKey(region))
            {
                DataBlock dataBlock = regionPtrPool.get(region);
                dataPtr = dataBlock.getDataPtr();
                Console.WriteLine("dataPtr: " + dataPtr + ", region: " + region);
            }
            else
            {
                byte[] city    = new byte[4];
                int    city_id = getCityId(region);
                Util.writeIntLong(city, 0, city_id);
                dataPtr = (int)raf.getFilePointer();
                raf.write(city);
                raf.write(data);

                regionPtrPool.put(region, new DataBlock(city_id, region, dataPtr));
            }

            //add the data index blocks
            IndexBlock ib = new IndexBlock(
                Util.ip2long(startIp),
                Util.ip2long(endIp),
                dataPtr,
                data.Length + 4     //4 bytes for the city id
                );

            indexPool.add(ib);
        }
Exemplo n.º 4
0
        /**
         * make the Db file
         *
         * @param	dbFile target output file path
         * @throws IOException
         */
        public void make(String dbFile)
        {
            //check and load the gloabl region
            if (globalRegionFile != null)
            {
                Console.WriteLine("+-Try to load the global region data ...");
                StreamReader greader = new StreamReader((globalRegionFile));
                String       gline   = null;
                while (!greader.EndOfStream)
                {
                    gline = greader.ReadLine();// != null
                    if (String.IsNullOrWhiteSpace(gline))
                    {
                        continue;
                    }

                    String[] p = gline.Split(",");
                    if (p.Length != 5)
                    {
                        continue;
                    }

                    //push the mapping
                    globalRegionMap.put(p[2], Int32.Parse(p[0]));
                }

                greader.Close();
                Console.WriteLine("|--[Ok]");
            }

            //alloc the header size
            StreamReader     reader = new StreamReader(this.ipSrcFile);
            RandomAccessFile raf    = new RandomAccessFile(dbFile);

            //init the db file
            initDbFile(raf);
            Console.WriteLine("+-Db file initialized.");

            //analysis main loop
            Console.WriteLine("+-Try to write the data blocks ... ");
            String line = null;

            while (!reader.EndOfStream)
            {
                //line = reader.ReadLine()
                line = reader.ReadLine();
                if (String.IsNullOrWhiteSpace(line))
                {
                    continue;
                }
                line = line.Trim();
                if (line.Length == 0)
                {
                    continue;
                }
                if (line[0] == '#')
                {
                    continue;
                }

                //1. get the start ip
                int sIdx = 0, eIdx = 0;
                if ((eIdx = line.IndexOf('|', sIdx + 1)) == -1)
                {
                    continue;
                }
                String startIp = line.Substring(sIdx, eIdx);

                //2. get the end ip
                sIdx = eIdx + 1;
                if ((eIdx = line.IndexOf('|', sIdx + 1)) == -1)
                {
                    continue;
                }
                String endIp = line.Substring(sIdx, eIdx - sIdx);

                //3. get the region
                sIdx = eIdx + 1;
                String region = line.Substring(sIdx);

                Console.WriteLine("+-Try to process item " + line);
                addDataBlock(raf, startIp, endIp, region);
                Console.WriteLine("|--[Ok]");
            }
            Console.WriteLine("|--Data block flushed!");
            Console.WriteLine("|--Data file pointer: " + raf.getFilePointer() + "\n");

            //write the index bytes
            Console.WriteLine("+-Try to write index blocks ... ");

            //record the start block
            IndexBlock  indexBlock = null;
            HeaderBlock hb         = null;

            indexBlock = indexPool.First();
            long indexStartIp = indexBlock.getStartIp(),
                 indexStratPtr = raf.getFilePointer(), indexEndPtr;

            headerPool.add(new HeaderBlock(indexStartIp, (int)(indexStratPtr)));

            int blockLength = IndexBlock.getIndexBlockLength();
            int counter = 0, shotCounter = (dbConfig.getIndexBlockSize() / blockLength) - 1;

            //Iterator<IndexBlock> indexIt = indexPool.iterator();
            //while ( indexIt.hasNext() ) {
            foreach (var block in indexPool)
            {
                if (++counter >= shotCounter)
                {
                    hb = new HeaderBlock(
                        block.getStartIp(),
                        (int)raf.getFilePointer()
                        );

                    headerPool.add(hb);
                    counter = 0;
                }

                //write the buffer
                raf.write(block.getBytes());
            }

            //record the end block
            if (counter > 0)
            {
                indexBlock = indexPool.Last();
                hb         = new HeaderBlock(
                    indexBlock.getStartIp(),
                    ((int)raf.getFilePointer()) - IndexBlock.getIndexBlockLength()
                    );

                headerPool.add(hb);
            }

            indexEndPtr = raf.getFilePointer();
            Console.WriteLine("|--[Ok]");

            //write the super blocks
            Console.WriteLine("+-Try to write the super blocks ... ");
            raf.seek(0L);   //reset the file pointer
            byte[] superBuffer = new byte[8];
            Util.writeIntLong(superBuffer, 0, indexStratPtr);
            Util.writeIntLong(superBuffer, 4, indexEndPtr - blockLength);
            raf.write(superBuffer);
            Console.WriteLine("|--[Ok]");

            //write the header blocks
            Console.WriteLine("+-Try to write the header blocks ... ");
            //Iterator<HeaderBlock> headerIt = headerPool.iterator();
            //while ( headerIt.hasNext() ) {
            //	HeaderBlock headerBlock = headerIt.next();
            //	raf.write(headerBlock.getBytes());
            //}
            foreach (var headerBlock in headerPool)
            {
                raf.write(headerBlock.getBytes());
            }

            //write the copyright and the release timestamp info
            Console.WriteLine("+-Try to write the copyright and release date info ... ");
            raf.seek(raf.length());
            //Calendar cal = Calendar.getInstance();
            //SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd");
            var    date      = DateTime.Now;
            String copyright = "Created by lionsoul at " + date.ToString("yyyy/MM/dd");
            var    timespan  = new DateTimeOffset(date).ToUnixTimeSeconds();

            raf.write((int)(timespan)); //the unix timestamp
            raf.write(copyright.getBytes("UTF-8"));
            Console.WriteLine("|--[Ok]");

            reader.Close();
            raf.close();
        }
Exemplo n.º 5
0
        /**
         * get the region with a int ip address with binary search algorithm
         *
         * @param   ip
         * @throws  IOException
         */
        public DataBlock binarySearch(long ip)
        {
            int blen = IndexBlock.getIndexBlockLength();

            if (totalIndexBlocks == 0)
            {
                raf.seek(0L);
                byte[] superBytes = new byte[8];
                raf.readFully(superBytes, 0, superBytes.Length);
                //initialize the global vars
                firstIndexPtr    = Util.getIntLong(superBytes, 0);
                lastIndexPtr     = Util.getIntLong(superBytes, 4);
                totalIndexBlocks = (int)((lastIndexPtr - firstIndexPtr) / blen) + 1;
            }

            //search the index blocks to define the data
            int l = 0, h = totalIndexBlocks;

            byte[] buffer = new byte[blen];
            long   sip, eip, dataptr = 0;

            while (l <= h)
            {
                int m = (l + h) >> 1;
                raf.seek(firstIndexPtr + m * blen); //set the file pointer
                raf.readFully(buffer, 0, buffer.Length);
                sip = Util.getIntLong(buffer, 0);
                if (ip < sip)
                {
                    h = m - 1;
                }
                else
                {
                    eip = Util.getIntLong(buffer, 4);
                    if (ip > eip)
                    {
                        l = m + 1;
                    }
                    else
                    {
                        dataptr = Util.getIntLong(buffer, 8);
                        break;
                    }
                }
            }

            //not matched
            if (dataptr == 0)
            {
                return(null);
            }

            //get the data
            int dataLen = (int)((dataptr >> 24) & 0xFF);
            int dataPtr = (int)((dataptr & 0x00FFFFFF));

            raf.seek(dataPtr);
            byte[] data = new byte[dataLen];
            raf.readFully(data, 0, data.Length);

            int city_id = (int)Util.getIntLong(data, 0);
            //String region = new String(data, 4, data.Length - 4, "UTF-8");
            var region = ReadString(data, 4, data.Length - 4);

            return(new DataBlock(city_id, region, dataPtr));
        }
Exemplo n.º 6
0
        /**
         * get the region with a int ip address with b-tree algorithm
         *
         * @param   ip
         * @throws  IOException
         */
        public DataBlock btreeSearch(long ip)
        {
            //check and load the header
            if (HeaderSip == null)
            {
                raf.seek(8L); //pass the super block
                byte[] b = new byte[dbConfig.getTotalHeaderSize()];
                // byte[] b = new byte[4096];
                raf.readFully(b, 0, b.Length);

                //fill the header
                int len = b.Length >> 3, idx = 0; //b.lenght / 8
                HeaderSip = new long[len];
                HeaderPtr = new int [len];
                long startIp, xDataPtr;
                for (int i = 0; i < b.Length; i += 8)
                {
                    startIp  = Util.getIntLong(b, i);
                    xDataPtr = Util.getIntLong(b, i + 4);
                    if (xDataPtr == 0)
                    {
                        break;
                    }

                    HeaderSip[idx] = startIp;
                    HeaderPtr[idx] = (int)xDataPtr;
                    idx++;
                }

                headerLength = idx;
            }

            //1. define the index block with the binary search
            if (ip == HeaderSip[0])
            {
                return(getByIndexPtr(HeaderPtr[0]));
            }
            else if (ip == HeaderSip[headerLength - 1])
            {
                return(getByIndexPtr(HeaderPtr[headerLength - 1]));
            }

            int l = 0, h = headerLength, sptr = 0, eptr = 0;

            while (l <= h)
            {
                int m = (l + h) >> 1;

                //perfetc matched, just return it
                if (ip == HeaderSip[m])
                {
                    if (m > 0)
                    {
                        sptr = HeaderPtr[m - 1];
                        eptr = HeaderPtr[m];
                    }
                    else
                    {
                        sptr = HeaderPtr[m];
                        eptr = HeaderPtr[m + 1];
                    }

                    break;
                }

                //less then the middle value
                if (ip < HeaderSip[m])
                {
                    if (m == 0)
                    {
                        sptr = HeaderPtr[m];
                        eptr = HeaderPtr[m + 1];
                        break;
                    }
                    else if (ip > HeaderSip[m - 1])
                    {
                        sptr = HeaderPtr[m - 1];
                        eptr = HeaderPtr[m];
                        break;
                    }

                    h = m - 1;
                }
                else
                {
                    if (m == headerLength - 1)
                    {
                        sptr = HeaderPtr[m - 1];
                        eptr = HeaderPtr[m];
                        break;
                    }
                    else if (ip <= HeaderSip[m + 1])
                    {
                        sptr = HeaderPtr[m];
                        eptr = HeaderPtr[m + 1];
                        break;
                    }

                    l = m + 1;
                }
            }

            //match nothing just stop it
            if (sptr == 0)
            {
                return(null);
            }

            //2. search the index blocks to define the data
            int blockLen = eptr - sptr, blen = IndexBlock.getIndexBlockLength();

            byte[] iBuffer = new byte[blockLen + blen]; //include the right border block
            raf.seek(sptr);
            raf.readFully(iBuffer, 0, iBuffer.Length);

            l = 0;
            h = blockLen / blen;
            long sip, eip, dataptr = 0;

            while (l <= h)
            {
                int m = (l + h) >> 1;
                int p = m * blen;
                sip = Util.getIntLong(iBuffer, p);
                if (ip < sip)
                {
                    h = m - 1;
                }
                else
                {
                    eip = Util.getIntLong(iBuffer, p + 4);
                    if (ip > eip)
                    {
                        l = m + 1;
                    }
                    else
                    {
                        dataptr = Util.getIntLong(iBuffer, p + 8);
                        break;
                    }
                }
            }

            //not matched
            if (dataptr == 0)
            {
                return(null);
            }

            //3. get the data
            int dataLen = (int)((dataptr >> 24) & 0xFF);
            int dataPtr = (int)((dataptr & 0x00FFFFFF));

            raf.seek(dataPtr);
            byte[] data = new byte[dataLen];
            raf.readFully(data, 0, data.Length);

            int city_id = (int)Util.getIntLong(data, 0);
            //String region = new String(data, 4, data.Length - 4, "UTF-8");
            var region = ReadString(data, 4, data.Length - 4);

            return(new DataBlock(city_id, region, dataPtr));
        }