public bool AddCharacters(int threadid, char[] chararray, int coffset, int chararraylength)
{
//Console.Write(new string(chararray,coffset,chararraylength));
//Console.Write(" ");
singleTPmemorymap map = TPmemorymap[threadid];
//make space for required number of characters
int newoffset = map.charmap_offset + chararraylength;
int max = newoffset / chunksize_characters + 1;
if (max > map.charmap_segments)
{
for (int i = 0; i < (max - map.charmap_segments); i++)
{
int newseg = DequeueMemory(threadid, true);
if (newseg == -1)
{
map.full = true;
return(false);
}
map.charmap[map.charmap_segments + i] = newseg;
}
map.charmap_segments = max;
}
bool done_copying = false;
int internaloffset = map.charmap_offset % chunksize_characters;
int currentseg = map.charmap_offset / chunksize_characters;
int remaininglength = chararraylength;
while (!done_copying)
{
if (internaloffset + remaininglength > chunksize_characters)
{
Array.Copy(chararray, coffset, globalcharacters, map.charmap[currentseg] * chunksize_characters + internaloffset, chunksize_characters - internaloffset);
coffset += chunksize_characters - internaloffset;
remaininglength -= chunksize_characters - internaloffset;
internaloffset = 0;
currentseg++;
}
else
{
Array.Copy(chararray, coffset, globalcharacters, map.charmap[currentseg] * chunksize_characters + internaloffset, remaininglength);
done_copying = true;
}
}
//Console.WriteLine(new string(globalcharacters,0,5000));
return(true);
}
public void GetChars(int threadid, int offset, int length, ref char[] chararray, ref int startindex)
{
//Console.WriteLine(offset);
//Console.WriteLine(new string(globalcharacters, 0, 5000));
singleTPmemorymap map = TPmemorymap[threadid];
//get array
bool done_copying = false;
int internaloffset = offset % chunksize_characters;
int currentseg = offset / chunksize_characters;
int remaininglength = length;
if (internaloffset + remaininglength <= chunksize_characters)
{
//character extract does not exceeds chunk boundary
chararray = globalcharacters;
startindex = map.charmap[currentseg] * chunksize_characters + internaloffset;
//Console.Write(new string(globalcharacters, startindex, length));
//Console.Write(" ");
return;
}
chararray = map.dummychars;
startindex = 0;
int coffset = 0;
while (!done_copying)
{
if (internaloffset + remaininglength > chunksize_characters)
{
//character extract exceeds chunk boundary - need to copy everything to dummy
Array.Copy(globalcharacters, map.charmap[currentseg] * chunksize_characters + internaloffset, chararray, coffset, chunksize_characters - internaloffset);
coffset += chunksize_characters - internaloffset;
remaininglength -= chunksize_characters - internaloffset;
internaloffset = 0;
currentseg++;
}
else
{
Array.Copy(globalcharacters, map.charmap[currentseg] * chunksize_characters + internaloffset, chararray, coffset, remaininglength);
done_copying = true;
}
}
}
public bool AddWikiConstruct(int threadid, char[] chararray, int offset, int chararraylength, int wikitype)
{
//TPmemorymap = new ConcurrentDictionary<int, singleTPmemorymap>(); is instantiated in MemoryManager constructor
singleTPmemorymap map = TPmemorymap[threadid];
if (map.full)
{
return(false);
}
if (AddCharacters(threadid, chararray, offset, chararraylength) && AddShortsWiki(threadid, map.charmap_offset, chararraylength, wikitype))
{
map.charmap_offset += chararraylength;
map.wikicounter++;
}
else
{
memstats.overflow++;
}
return(true);
}
public bool AddWord(int threadid, char[] chararray, int chararraylength, bool sticky, int division, bool stopword, bool isInt, int decodedInt)
{
singleTPmemorymap map = TPmemorymap[threadid];
if (map.full)
{
return(false);
}
if (isInt)
{
//Console.WriteLine(decodedInt);
if (AddCharacters(threadid, chararray, 0, 0) && AddShortsWords(threadid, decodedInt, 0, division, sticky, stopword, true))
{
map.charmap_offset += 0;
map.wordcounter++;
++map.nonstopwordcounter;
}
else
{
memstats.overflow++;
}
}
else
{
if (AddCharacters(threadid, chararray, 0, chararraylength) && AddShortsWords(threadid, TPmemorymap[threadid].charmap_offset, chararraylength, division, sticky, stopword, false))
{
map.charmap_offset += chararraylength;
map.wordcounter++;
if (!stopword)
{
++map.nonstopwordcounter;
}
}
else
{
memstats.overflow++;
}
}
return(true);
}
public bool GetWikiConstruct(int threadid, int i, ref int wikitype, ref char[] wikiarray, ref int startindex, ref int length)
{
singleTPmemorymap map = TPmemorymap[threadid];
//copy shorts into dummyshort array
bool done_copying = false;
int internaloffset = i * 5 % chunksize_shorts;
int currentseg = i * 5 / chunksize_shorts;
int coffset = 0;
int remaininglength = 5;
while (!done_copying)
{
if (internaloffset + remaininglength > chunksize_shorts)
{
//Console.WriteLine(" break ");
Array.Copy(globalshorts, map.intmap_wiki[currentseg] * chunksize_shorts + internaloffset, map.dummyshort, coffset, chunksize_shorts - internaloffset);
coffset += chunksize_shorts - internaloffset;
remaininglength -= chunksize_shorts - internaloffset;
internaloffset = 0;
currentseg++;
}
else
{
//Console.WriteLine(" nobreak ");
Array.Copy(globalshorts, map.intmap_wiki[currentseg] * chunksize_shorts + internaloffset, map.dummyshort, coffset, remaininglength);
done_copying = true;
}
}
//extract information from dummyshort
int offset = (((int)map.dummyshort[0]) << 16) + (int)map.dummyshort[1];
length = (((int)map.dummyshort[2]) << 16) + (int)map.dummyshort[3];
wikitype = map.dummyshort[4];
GetChars(threadid, offset, length, ref wikiarray, ref startindex);
//Console.Write(offset);
//Console.Write(" {0} {1} {2} ",startindex,length,wikiarray.ToString());
//Console.Write(new string(wikiarray, startindex, length));
//Console.Write(" ");
return(true);
}
public bool GetWord(int threadid, int i, ref char[] chararray, ref int startindex, ref int length, ref bool sticky, ref int division, ref bool stopword, ref bool isInt, ref int decodedInt)
{
singleTPmemorymap map = TPmemorymap[threadid];
//copy shorts into dummyshort array
bool done_copying = false;
int internaloffset = i * 5 % chunksize_shorts;
int currentseg = i * 5 / chunksize_shorts;
int coffset = 0;
int remaininglength = 5;
while (!done_copying)
{
if (internaloffset + remaininglength > chunksize_shorts)
{
Array.Copy(globalshorts, map.intmap_words[currentseg] * chunksize_shorts + internaloffset, map.dummyshort, coffset, chunksize_shorts - internaloffset);
coffset += chunksize_shorts - internaloffset;
remaininglength -= chunksize_shorts - internaloffset;
internaloffset = 0;
currentseg++;
}
else
{
Array.Copy(globalshorts, map.intmap_words[currentseg] * chunksize_shorts + internaloffset, map.dummyshort, coffset, remaininglength);
done_copying = true;
}
}
//Console.WriteLine("{0} {1} {2} {3} {4}", map.dummyshort[0], map.dummyshort[1], map.dummyshort[2], map.dummyshort[3], map.dummyshort[4]);
//extract information from dummyshort
int offset = (((int)map.dummyshort[0]) << 16) + (int)map.dummyshort[1];
length = (((int)map.dummyshort[2]) << 16) + (int)map.dummyshort[3];
//Console.WriteLine(length);
division = map.dummyshort[4] % 256;
if ((map.dummyshort[4] >> 8) % 2 == 1)
{
sticky = true;
}
else
{
sticky = false;
}
if ((map.dummyshort[4] >> 9) % 2 == 1)
{
stopword = true;
}
else
{
stopword = false;
}
if ((map.dummyshort[4] >> 10) % 2 == 1)
{
isInt = true;
//Console.WriteLine(length);
}
else
{
isInt = false;
}
if (isInt)
{
decodedInt = offset;
//Console.WriteLine(decodedInt);
}
else
{
GetChars(threadid, offset, length, ref chararray, ref startindex);
}
return(true);
}
//5 short integers to code the wiki construct
public bool AddShortsWiki(int threadid, int offset, int length, int type)
{
singleTPmemorymap map = TPmemorymap[threadid];
//make space for required number of characters
int newoffset = map.wikicounter * 5 + 5;
int max = newoffset / chunksize_shorts + 1;
if (max > map.intmap_wiki_segments)
{
for (int i = 0; i < (max - map.intmap_wiki_segments); i++)
{
int newseg = DequeueMemory(threadid, false);
if (newseg == -1)
{
map.full = true;
return(false);
}
if (map.intmap_wiki_segments + i < map.intmap_wiki.Length)
{
map.intmap_wiki[map.intmap_wiki_segments + i] = newseg;
}
else
{
map.full = true;
return(false);
}
}
map.intmap_wiki_segments = max;
}
//fill dummy short
map.dummyshort[0] = (ushort)(offset >> 16);
map.dummyshort[1] = (ushort)(offset % (2 << 16));
map.dummyshort[2] = (ushort)(length >> 16);
map.dummyshort[3] = (ushort)(length % (2 << 16));
map.dummyshort[4] = (ushort)type;
//now copy dummyshort
bool done_copying = false;
int internaloffset = (map.wikicounter * 5) % chunksize_shorts;
int currentseg = (map.wikicounter * 5) / chunksize_shorts;
//Console.Write(map.intmap_wiki[currentseg]);
//Console.Write(" ");
int coffset = 0;
int remaininglength = 5;
while (!done_copying)
{
if (internaloffset + remaininglength > chunksize_shorts)
{
Array.Copy(map.dummyshort, coffset, globalshorts, map.intmap_wiki[currentseg] * chunksize_shorts + internaloffset, chunksize_shorts - internaloffset);
coffset += chunksize_shorts - internaloffset;
remaininglength -= chunksize_shorts - internaloffset;
internaloffset = 0;
currentseg++;
}
else
{
Array.Copy(map.dummyshort, coffset, globalshorts, map.intmap_wiki[currentseg] * chunksize_shorts + internaloffset, remaininglength);
done_copying = true;
}
}
return(true);
}
//5 short integers to code the word integers
public bool AddShortsWords(int threadid, int offset, int length, int division, bool sticky, bool stopword, bool isInt)
{
singleTPmemorymap map = TPmemorymap[threadid];
//make space for required number of characters
int newoffset = map.wordcounter * 5 + 5;
int max = newoffset / chunksize_shorts + 1;
if (max > map.intmap_words_segments)
{
for (int i = 0; i < (max - map.intmap_words_segments); i++)
{
int newseg = DequeueMemory(threadid, false);
if (newseg == -1)
{
map.full = true;
return(false);
}
if (map.intmap_words_segments + i < map.intmap_words.Length)
{
map.intmap_words[map.intmap_words_segments + i] = newseg;
}
else
{
map.full = true;
return(false);
}
}
map.intmap_words_segments = max;
}
//fill dummy short
map.dummyshort[0] = (ushort)(offset >> 16);
map.dummyshort[1] = (ushort)(offset % (2 << 16));
map.dummyshort[2] = (ushort)(length >> 16);
map.dummyshort[3] = (ushort)(length % (2 << 16));
int combined = division % 256;
if (sticky)
{
combined = combined | (1 << 8);
}
if (stopword)
{
combined = combined | (1 << 9);
}
if (isInt)
{
combined = combined | (1 << 10);
}
map.dummyshort[4] = (ushort)combined;
//now copy dummyshort
bool done_copying = false;
int internaloffset = (map.wordcounter * 5) % chunksize_shorts;
int currentseg = (map.wordcounter * 5) / chunksize_shorts;
int coffset = 0;
int remaininglength = 5;
while (!done_copying)
{
if (internaloffset + remaininglength > chunksize_shorts)
{
Array.Copy(map.dummyshort, coffset, globalshorts, map.intmap_words[currentseg] * chunksize_shorts + internaloffset, chunksize_shorts - internaloffset);
coffset += chunksize_shorts - internaloffset;
remaininglength -= chunksize_shorts - internaloffset;
internaloffset = 0;
currentseg++;
}
else
{
Array.Copy(map.dummyshort, coffset, globalshorts, map.intmap_words[currentseg] * chunksize_shorts + internaloffset, remaininglength);
done_copying = true;
}
}
return(true);
}
