public bool AddCharacters(int threadid, char[] chararray, int coffset, int chararraylength) { //Console.Write(new string(chararray,coffset,chararraylength)); //Console.Write(" "); singleTPmemorymap map = TPmemorymap[threadid]; //make space for required number of characters int newoffset = map.charmap_offset + chararraylength; int max = newoffset / chunksize_characters + 1; if (max > map.charmap_segments) { for (int i = 0; i < (max - map.charmap_segments); i++) { int newseg = DequeueMemory(threadid, true); if (newseg == -1) { map.full = true; return(false); } map.charmap[map.charmap_segments + i] = newseg; } map.charmap_segments = max; } bool done_copying = false; int internaloffset = map.charmap_offset % chunksize_characters; int currentseg = map.charmap_offset / chunksize_characters; int remaininglength = chararraylength; while (!done_copying) { if (internaloffset + remaininglength > chunksize_characters) { Array.Copy(chararray, coffset, globalcharacters, map.charmap[currentseg] * chunksize_characters + internaloffset, chunksize_characters - internaloffset); coffset += chunksize_characters - internaloffset; remaininglength -= chunksize_characters - internaloffset; internaloffset = 0; currentseg++; } else { Array.Copy(chararray, coffset, globalcharacters, map.charmap[currentseg] * chunksize_characters + internaloffset, remaininglength); done_copying = true; } } //Console.WriteLine(new string(globalcharacters,0,5000)); return(true); }
public void GetChars(int threadid, int offset, int length, ref char[] chararray, ref int startindex) { //Console.WriteLine(offset); //Console.WriteLine(new string(globalcharacters, 0, 5000)); singleTPmemorymap map = TPmemorymap[threadid]; //get array bool done_copying = false; int internaloffset = offset % chunksize_characters; int currentseg = offset / chunksize_characters; int remaininglength = length; if (internaloffset + remaininglength <= chunksize_characters) { //character extract does not exceeds chunk boundary chararray = globalcharacters; startindex = map.charmap[currentseg] * chunksize_characters + internaloffset; //Console.Write(new string(globalcharacters, startindex, length)); //Console.Write(" "); return; } chararray = map.dummychars; startindex = 0; int coffset = 0; while (!done_copying) { if (internaloffset + remaininglength > chunksize_characters) { //character extract exceeds chunk boundary - need to copy everything to dummy Array.Copy(globalcharacters, map.charmap[currentseg] * chunksize_characters + internaloffset, chararray, coffset, chunksize_characters - internaloffset); coffset += chunksize_characters - internaloffset; remaininglength -= chunksize_characters - internaloffset; internaloffset = 0; currentseg++; } else { Array.Copy(globalcharacters, map.charmap[currentseg] * chunksize_characters + internaloffset, chararray, coffset, remaininglength); done_copying = true; } } }
public bool AddWikiConstruct(int threadid, char[] chararray, int offset, int chararraylength, int wikitype) { //TPmemorymap = new ConcurrentDictionary<int, singleTPmemorymap>(); is instantiated in MemoryManager constructor singleTPmemorymap map = TPmemorymap[threadid]; if (map.full) { return(false); } if (AddCharacters(threadid, chararray, offset, chararraylength) && AddShortsWiki(threadid, map.charmap_offset, chararraylength, wikitype)) { map.charmap_offset += chararraylength; map.wikicounter++; } else { memstats.overflow++; } return(true); }
public bool AddWord(int threadid, char[] chararray, int chararraylength, bool sticky, int division, bool stopword, bool isInt, int decodedInt) { singleTPmemorymap map = TPmemorymap[threadid]; if (map.full) { return(false); } if (isInt) { //Console.WriteLine(decodedInt); if (AddCharacters(threadid, chararray, 0, 0) && AddShortsWords(threadid, decodedInt, 0, division, sticky, stopword, true)) { map.charmap_offset += 0; map.wordcounter++; ++map.nonstopwordcounter; } else { memstats.overflow++; } } else { if (AddCharacters(threadid, chararray, 0, chararraylength) && AddShortsWords(threadid, TPmemorymap[threadid].charmap_offset, chararraylength, division, sticky, stopword, false)) { map.charmap_offset += chararraylength; map.wordcounter++; if (!stopword) { ++map.nonstopwordcounter; } } else { memstats.overflow++; } } return(true); }
public bool GetWikiConstruct(int threadid, int i, ref int wikitype, ref char[] wikiarray, ref int startindex, ref int length) { singleTPmemorymap map = TPmemorymap[threadid]; //copy shorts into dummyshort array bool done_copying = false; int internaloffset = i * 5 % chunksize_shorts; int currentseg = i * 5 / chunksize_shorts; int coffset = 0; int remaininglength = 5; while (!done_copying) { if (internaloffset + remaininglength > chunksize_shorts) { //Console.WriteLine(" break "); Array.Copy(globalshorts, map.intmap_wiki[currentseg] * chunksize_shorts + internaloffset, map.dummyshort, coffset, chunksize_shorts - internaloffset); coffset += chunksize_shorts - internaloffset; remaininglength -= chunksize_shorts - internaloffset; internaloffset = 0; currentseg++; } else { //Console.WriteLine(" nobreak "); Array.Copy(globalshorts, map.intmap_wiki[currentseg] * chunksize_shorts + internaloffset, map.dummyshort, coffset, remaininglength); done_copying = true; } } //extract information from dummyshort int offset = (((int)map.dummyshort[0]) << 16) + (int)map.dummyshort[1]; length = (((int)map.dummyshort[2]) << 16) + (int)map.dummyshort[3]; wikitype = map.dummyshort[4]; GetChars(threadid, offset, length, ref wikiarray, ref startindex); //Console.Write(offset); //Console.Write(" {0} {1} {2} ",startindex,length,wikiarray.ToString()); //Console.Write(new string(wikiarray, startindex, length)); //Console.Write(" "); return(true); }
public bool GetWord(int threadid, int i, ref char[] chararray, ref int startindex, ref int length, ref bool sticky, ref int division, ref bool stopword, ref bool isInt, ref int decodedInt) { singleTPmemorymap map = TPmemorymap[threadid]; //copy shorts into dummyshort array bool done_copying = false; int internaloffset = i * 5 % chunksize_shorts; int currentseg = i * 5 / chunksize_shorts; int coffset = 0; int remaininglength = 5; while (!done_copying) { if (internaloffset + remaininglength > chunksize_shorts) { Array.Copy(globalshorts, map.intmap_words[currentseg] * chunksize_shorts + internaloffset, map.dummyshort, coffset, chunksize_shorts - internaloffset); coffset += chunksize_shorts - internaloffset; remaininglength -= chunksize_shorts - internaloffset; internaloffset = 0; currentseg++; } else { Array.Copy(globalshorts, map.intmap_words[currentseg] * chunksize_shorts + internaloffset, map.dummyshort, coffset, remaininglength); done_copying = true; } } //Console.WriteLine("{0} {1} {2} {3} {4}", map.dummyshort[0], map.dummyshort[1], map.dummyshort[2], map.dummyshort[3], map.dummyshort[4]); //extract information from dummyshort int offset = (((int)map.dummyshort[0]) << 16) + (int)map.dummyshort[1]; length = (((int)map.dummyshort[2]) << 16) + (int)map.dummyshort[3]; //Console.WriteLine(length); division = map.dummyshort[4] % 256; if ((map.dummyshort[4] >> 8) % 2 == 1) { sticky = true; } else { sticky = false; } if ((map.dummyshort[4] >> 9) % 2 == 1) { stopword = true; } else { stopword = false; } if ((map.dummyshort[4] >> 10) % 2 == 1) { isInt = true; //Console.WriteLine(length); } else { isInt = false; } if (isInt) { decodedInt = offset; //Console.WriteLine(decodedInt); } else { GetChars(threadid, offset, length, ref chararray, ref startindex); } return(true); }
//5 short integers to code the wiki construct public bool AddShortsWiki(int threadid, int offset, int length, int type) { singleTPmemorymap map = TPmemorymap[threadid]; //make space for required number of characters int newoffset = map.wikicounter * 5 + 5; int max = newoffset / chunksize_shorts + 1; if (max > map.intmap_wiki_segments) { for (int i = 0; i < (max - map.intmap_wiki_segments); i++) { int newseg = DequeueMemory(threadid, false); if (newseg == -1) { map.full = true; return(false); } if (map.intmap_wiki_segments + i < map.intmap_wiki.Length) { map.intmap_wiki[map.intmap_wiki_segments + i] = newseg; } else { map.full = true; return(false); } } map.intmap_wiki_segments = max; } //fill dummy short map.dummyshort[0] = (ushort)(offset >> 16); map.dummyshort[1] = (ushort)(offset % (2 << 16)); map.dummyshort[2] = (ushort)(length >> 16); map.dummyshort[3] = (ushort)(length % (2 << 16)); map.dummyshort[4] = (ushort)type; //now copy dummyshort bool done_copying = false; int internaloffset = (map.wikicounter * 5) % chunksize_shorts; int currentseg = (map.wikicounter * 5) / chunksize_shorts; //Console.Write(map.intmap_wiki[currentseg]); //Console.Write(" "); int coffset = 0; int remaininglength = 5; while (!done_copying) { if (internaloffset + remaininglength > chunksize_shorts) { Array.Copy(map.dummyshort, coffset, globalshorts, map.intmap_wiki[currentseg] * chunksize_shorts + internaloffset, chunksize_shorts - internaloffset); coffset += chunksize_shorts - internaloffset; remaininglength -= chunksize_shorts - internaloffset; internaloffset = 0; currentseg++; } else { Array.Copy(map.dummyshort, coffset, globalshorts, map.intmap_wiki[currentseg] * chunksize_shorts + internaloffset, remaininglength); done_copying = true; } } return(true); }
//5 short integers to code the word integers public bool AddShortsWords(int threadid, int offset, int length, int division, bool sticky, bool stopword, bool isInt) { singleTPmemorymap map = TPmemorymap[threadid]; //make space for required number of characters int newoffset = map.wordcounter * 5 + 5; int max = newoffset / chunksize_shorts + 1; if (max > map.intmap_words_segments) { for (int i = 0; i < (max - map.intmap_words_segments); i++) { int newseg = DequeueMemory(threadid, false); if (newseg == -1) { map.full = true; return(false); } if (map.intmap_words_segments + i < map.intmap_words.Length) { map.intmap_words[map.intmap_words_segments + i] = newseg; } else { map.full = true; return(false); } } map.intmap_words_segments = max; } //fill dummy short map.dummyshort[0] = (ushort)(offset >> 16); map.dummyshort[1] = (ushort)(offset % (2 << 16)); map.dummyshort[2] = (ushort)(length >> 16); map.dummyshort[3] = (ushort)(length % (2 << 16)); int combined = division % 256; if (sticky) { combined = combined | (1 << 8); } if (stopword) { combined = combined | (1 << 9); } if (isInt) { combined = combined | (1 << 10); } map.dummyshort[4] = (ushort)combined; //now copy dummyshort bool done_copying = false; int internaloffset = (map.wordcounter * 5) % chunksize_shorts; int currentseg = (map.wordcounter * 5) / chunksize_shorts; int coffset = 0; int remaininglength = 5; while (!done_copying) { if (internaloffset + remaininglength > chunksize_shorts) { Array.Copy(map.dummyshort, coffset, globalshorts, map.intmap_words[currentseg] * chunksize_shorts + internaloffset, chunksize_shorts - internaloffset); coffset += chunksize_shorts - internaloffset; remaininglength -= chunksize_shorts - internaloffset; internaloffset = 0; currentseg++; } else { Array.Copy(map.dummyshort, coffset, globalshorts, map.intmap_words[currentseg] * chunksize_shorts + internaloffset, remaininglength); done_copying = true; } } return(true); }