/** * get the region with a int ip address with memory binary search algorithm * * @param ip * @throws IOException */ public DataBlock memorySearch(long ip) { int blen = IndexBlock.getIndexBlockLength(); if (dbBinStr == null) { dbBinStr = new byte[(int)raf.length()]; raf.seek(0L); raf.readFully(dbBinStr, 0, dbBinStr.Length); //initialize the global vars firstIndexPtr = Util.getIntLong(dbBinStr, 0); lastIndexPtr = Util.getIntLong(dbBinStr, 4); totalIndexBlocks = (int)((lastIndexPtr - firstIndexPtr) / blen) + 1; } //search the index blocks to define the data int l = 0, h = totalIndexBlocks; long sip, eip, dataptr = 0; while (l <= h) { int m = (l + h) >> 1; int p = (int)(firstIndexPtr + m * blen); sip = Util.getIntLong(dbBinStr, p); if (ip < sip) { h = m - 1; } else { eip = Util.getIntLong(dbBinStr, p + 4); if (ip > eip) { l = m + 1; } else { dataptr = Util.getIntLong(dbBinStr, p + 8); break; } } } //not matched if (dataptr == 0) { return(null); } //get the data int dataLen = (int)((dataptr >> 24) & 0xFF); int dataPtr = (int)((dataptr & 0x00FFFFFF)); int city_id = (int)Util.getIntLong(dbBinStr, dataPtr); //String region = new String(dbBinStr, dataPtr + 4, dataLen - 4, "UTF-8"); var region = ReadString(dbBinStr, dataPtr + 4, dataLen - 4); return(new DataBlock(city_id, region, dataPtr)); }
/** * construct method with self-define std ip2region bianry string support * Thanks to the issue from Wendal at https://gitee.com/lionsoul/ip2region/issues/IILFL * * @param dbConfig * @param dbBinStr */ public DbSearcher(DbConfig dbConfig, byte[] dbBinStr) { this.dbConfig = dbConfig; this.dbBinStr = dbBinStr; firstIndexPtr = Util.getIntLong(dbBinStr, 0); lastIndexPtr = Util.getIntLong(dbBinStr, 4); totalIndexBlocks = (int)((lastIndexPtr - firstIndexPtr) / IndexBlock.getIndexBlockLength()) + 1; }
/** * make the Db file * * @param dbFile target output file path * @throws IOException */ public void make(String dbFile) { //check and load the gloabl region if (globalRegionFile != null) { Console.WriteLine("+-Try to load the global region data ..."); StreamReader greader = new StreamReader((globalRegionFile)); String gline = null; while (!greader.EndOfStream) { gline = greader.ReadLine();// != null if (String.IsNullOrWhiteSpace(gline)) { continue; } String[] p = gline.Split(","); if (p.Length != 5) { continue; } //push the mapping globalRegionMap.put(p[2], Int32.Parse(p[0])); } greader.Close(); Console.WriteLine("|--[Ok]"); } //alloc the header size StreamReader reader = new StreamReader(this.ipSrcFile); RandomAccessFile raf = new RandomAccessFile(dbFile); //init the db file initDbFile(raf); Console.WriteLine("+-Db file initialized."); //analysis main loop Console.WriteLine("+-Try to write the data blocks ... "); String line = null; while (!reader.EndOfStream) { //line = reader.ReadLine() line = reader.ReadLine(); if (String.IsNullOrWhiteSpace(line)) { continue; } line = line.Trim(); if (line.Length == 0) { continue; } if (line[0] == '#') { continue; } //1. get the start ip int sIdx = 0, eIdx = 0; if ((eIdx = line.IndexOf('|', sIdx + 1)) == -1) { continue; } String startIp = line.Substring(sIdx, eIdx); //2. get the end ip sIdx = eIdx + 1; if ((eIdx = line.IndexOf('|', sIdx + 1)) == -1) { continue; } String endIp = line.Substring(sIdx, eIdx - sIdx); //3. get the region sIdx = eIdx + 1; String region = line.Substring(sIdx); Console.WriteLine("+-Try to process item " + line); addDataBlock(raf, startIp, endIp, region); Console.WriteLine("|--[Ok]"); } Console.WriteLine("|--Data block flushed!"); Console.WriteLine("|--Data file pointer: " + raf.getFilePointer() + "\n"); //write the index bytes Console.WriteLine("+-Try to write index blocks ... "); //record the start block IndexBlock indexBlock = null; HeaderBlock hb = null; indexBlock = indexPool.First(); long indexStartIp = indexBlock.getStartIp(), indexStratPtr = raf.getFilePointer(), indexEndPtr; headerPool.add(new HeaderBlock(indexStartIp, (int)(indexStratPtr))); int blockLength = IndexBlock.getIndexBlockLength(); int counter = 0, shotCounter = (dbConfig.getIndexBlockSize() / blockLength) - 1; //Iterator<IndexBlock> indexIt = indexPool.iterator(); //while ( indexIt.hasNext() ) { foreach (var block in indexPool) { if (++counter >= shotCounter) { hb = new HeaderBlock( block.getStartIp(), (int)raf.getFilePointer() ); headerPool.add(hb); counter = 0; } //write the buffer raf.write(block.getBytes()); } //record the end block if (counter > 0) { indexBlock = indexPool.Last(); hb = new HeaderBlock( indexBlock.getStartIp(), ((int)raf.getFilePointer()) - IndexBlock.getIndexBlockLength() ); headerPool.add(hb); } indexEndPtr = raf.getFilePointer(); Console.WriteLine("|--[Ok]"); //write the super blocks Console.WriteLine("+-Try to write the super blocks ... "); raf.seek(0L); //reset the file pointer byte[] superBuffer = new byte[8]; Util.writeIntLong(superBuffer, 0, indexStratPtr); Util.writeIntLong(superBuffer, 4, indexEndPtr - blockLength); raf.write(superBuffer); Console.WriteLine("|--[Ok]"); //write the header blocks Console.WriteLine("+-Try to write the header blocks ... "); //Iterator<HeaderBlock> headerIt = headerPool.iterator(); //while ( headerIt.hasNext() ) { // HeaderBlock headerBlock = headerIt.next(); // raf.write(headerBlock.getBytes()); //} foreach (var headerBlock in headerPool) { raf.write(headerBlock.getBytes()); } //write the copyright and the release timestamp info Console.WriteLine("+-Try to write the copyright and release date info ... "); raf.seek(raf.length()); //Calendar cal = Calendar.getInstance(); //SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd"); var date = DateTime.Now; String copyright = "Created by lionsoul at " + date.ToString("yyyy/MM/dd"); var timespan = new DateTimeOffset(date).ToUnixTimeSeconds(); raf.write((int)(timespan)); //the unix timestamp raf.write(copyright.getBytes("UTF-8")); Console.WriteLine("|--[Ok]"); reader.Close(); raf.close(); }
/** * get the region with a int ip address with binary search algorithm * * @param ip * @throws IOException */ public DataBlock binarySearch(long ip) { int blen = IndexBlock.getIndexBlockLength(); if (totalIndexBlocks == 0) { raf.seek(0L); byte[] superBytes = new byte[8]; raf.readFully(superBytes, 0, superBytes.Length); //initialize the global vars firstIndexPtr = Util.getIntLong(superBytes, 0); lastIndexPtr = Util.getIntLong(superBytes, 4); totalIndexBlocks = (int)((lastIndexPtr - firstIndexPtr) / blen) + 1; } //search the index blocks to define the data int l = 0, h = totalIndexBlocks; byte[] buffer = new byte[blen]; long sip, eip, dataptr = 0; while (l <= h) { int m = (l + h) >> 1; raf.seek(firstIndexPtr + m * blen); //set the file pointer raf.readFully(buffer, 0, buffer.Length); sip = Util.getIntLong(buffer, 0); if (ip < sip) { h = m - 1; } else { eip = Util.getIntLong(buffer, 4); if (ip > eip) { l = m + 1; } else { dataptr = Util.getIntLong(buffer, 8); break; } } } //not matched if (dataptr == 0) { return(null); } //get the data int dataLen = (int)((dataptr >> 24) & 0xFF); int dataPtr = (int)((dataptr & 0x00FFFFFF)); raf.seek(dataPtr); byte[] data = new byte[dataLen]; raf.readFully(data, 0, data.Length); int city_id = (int)Util.getIntLong(data, 0); //String region = new String(data, 4, data.Length - 4, "UTF-8"); var region = ReadString(data, 4, data.Length - 4); return(new DataBlock(city_id, region, dataPtr)); }
/** * get the region with a int ip address with b-tree algorithm * * @param ip * @throws IOException */ public DataBlock btreeSearch(long ip) { //check and load the header if (HeaderSip == null) { raf.seek(8L); //pass the super block byte[] b = new byte[dbConfig.getTotalHeaderSize()]; // byte[] b = new byte[4096]; raf.readFully(b, 0, b.Length); //fill the header int len = b.Length >> 3, idx = 0; //b.lenght / 8 HeaderSip = new long[len]; HeaderPtr = new int [len]; long startIp, xDataPtr; for (int i = 0; i < b.Length; i += 8) { startIp = Util.getIntLong(b, i); xDataPtr = Util.getIntLong(b, i + 4); if (xDataPtr == 0) { break; } HeaderSip[idx] = startIp; HeaderPtr[idx] = (int)xDataPtr; idx++; } headerLength = idx; } //1. define the index block with the binary search if (ip == HeaderSip[0]) { return(getByIndexPtr(HeaderPtr[0])); } else if (ip == HeaderSip[headerLength - 1]) { return(getByIndexPtr(HeaderPtr[headerLength - 1])); } int l = 0, h = headerLength, sptr = 0, eptr = 0; while (l <= h) { int m = (l + h) >> 1; //perfetc matched, just return it if (ip == HeaderSip[m]) { if (m > 0) { sptr = HeaderPtr[m - 1]; eptr = HeaderPtr[m]; } else { sptr = HeaderPtr[m]; eptr = HeaderPtr[m + 1]; } break; } //less then the middle value if (ip < HeaderSip[m]) { if (m == 0) { sptr = HeaderPtr[m]; eptr = HeaderPtr[m + 1]; break; } else if (ip > HeaderSip[m - 1]) { sptr = HeaderPtr[m - 1]; eptr = HeaderPtr[m]; break; } h = m - 1; } else { if (m == headerLength - 1) { sptr = HeaderPtr[m - 1]; eptr = HeaderPtr[m]; break; } else if (ip <= HeaderSip[m + 1]) { sptr = HeaderPtr[m]; eptr = HeaderPtr[m + 1]; break; } l = m + 1; } } //match nothing just stop it if (sptr == 0) { return(null); } //2. search the index blocks to define the data int blockLen = eptr - sptr, blen = IndexBlock.getIndexBlockLength(); byte[] iBuffer = new byte[blockLen + blen]; //include the right border block raf.seek(sptr); raf.readFully(iBuffer, 0, iBuffer.Length); l = 0; h = blockLen / blen; long sip, eip, dataptr = 0; while (l <= h) { int m = (l + h) >> 1; int p = m * blen; sip = Util.getIntLong(iBuffer, p); if (ip < sip) { h = m - 1; } else { eip = Util.getIntLong(iBuffer, p + 4); if (ip > eip) { l = m + 1; } else { dataptr = Util.getIntLong(iBuffer, p + 8); break; } } } //not matched if (dataptr == 0) { return(null); } //3. get the data int dataLen = (int)((dataptr >> 24) & 0xFF); int dataPtr = (int)((dataptr & 0x00FFFFFF)); raf.seek(dataPtr); byte[] data = new byte[dataLen]; raf.readFully(data, 0, data.Length); int city_id = (int)Util.getIntLong(data, 0); //String region = new String(data, 4, data.Length - 4, "UTF-8"); var region = ReadString(data, 4, data.Length - 4); return(new DataBlock(city_id, region, dataPtr)); }