public void FlushBufferToDisk(List <SaxData> buf)
{
FileStream fs;
if (File.Exists(FileName))
{
fs = new FileStream(FileName, FileMode.Append);
}
else
{
fs = new FileStream(FileName, FileMode.Create);
}
using (BinaryWriter bw = new BinaryWriter(fs))
{
onDisk = true;
fileAccessCount++; // Increment the number of file access everytime that write on disk a set of ts
DiskCost.increaserandomcost();
// Read all entries into memory buffer
// memory stream ctor takes in buffer size, currently a poor estimate
using (MemoryStream ms = new MemoryStream(buf.Count * (Globals.SaxValuesByteLength)))
{
foreach (SaxData data in buf)
{
DiskCost.increasesequentialcost();
byte[] dataBytes = data.ToBytes();
ms.Write(dataBytes, 0, dataBytes.Length);
}
bw.Write(ms.ToArray());
}
}
fs.Close();
}
public void Insert(SaxData input)
{
string saxString = Sax.SaxDataRepToSaxStr(input, options.SaxOpts);
if (splitDepth == 0 && flush == false)
{
if (!buffer.ContainsKey(saxString))
{
buffer.Add(saxString, new List <SaxData>());
}
buffer[saxString].Add(input);
}
else
{
if (index.ContainsKey(saxString))
{
IndexEntry entry = index[saxString];
if (entry is TermEntry)// if terminal, then search path terminates here
{
TermEntry tentry = (TermEntry)entry;
string oldFileName = tentry.FileName;
if (SplitEntry(tentry) == false) // check bucket requires a split
{
tentry.InsertToBuffer(input);
}
else
{
List <SaxData> B = tentry.getbuffer();
if (B == null)
{
B = new List <SaxData>();
}
DiskCost.increasesavedcost(B.Count);
ushort[] newMask = this.options.MaskCopy;
ushort[] newSaxString = Sax.SaxStrToSaxVals(saxString);
string newName = "";
for (int i = 0; i < newMask.Length; i++)
{
newName = newName + newSaxString[i].ToString() + "." + newMask[i].ToString() + "_";
}
newName = newName.Substring(0, newName.Length - 1);
string[] files = Directory.GetFiles(WorkingFolder,
string.Concat(newName, "*.txt"));
//string[] files = Directory.GetFiles(tentry.FileName);
if (tentry.OnDisk == true)
{
Assert.AreEqual(files.Length, 1);
}
else
{
Assert.AreEqual(files.Length, 0);
}
byte[] temp;
int pos = 0;
long length = -1;
int bytesToRead = SaxData.ByteLength(typeof(DATAFORMAT));
foreach (string f in files)
{
using (BinaryReader br = new BinaryReader(new FileStream(f, FileMode.Open, FileAccess.Read)))
{
length = br.BaseStream.Length;
if (length != 0)
{
DiskCost.increaserandomcost();
}
if (Math.IEEERemainder(length, bytesToRead) != 0)
{
throw new ApplicationException("Math.IEEERemainder(br.BaseStream.Length, bytesToRead) != 0");
}
while (pos < length)
{
temp = br.ReadBytes(bytesToRead);
if (temp.Length != bytesToRead)
{
throw new ApplicationException("temp.Length != bytesToRead");
}
B.Add(SaxData.Parse <DATAFORMAT>(temp));
DiskCost.increasereadcost();
pos += bytesToRead;
}
}
File.Delete(f);
}
SplitEntry <DATAFORMAT> newSplit;
if (Globals.NewSplitPolicy)
{
newSplit = new SplitEntry <DATAFORMAT>(saxString, UpdateOptions(B), (byte)(1 + splitDepth));
}
else
{
newSplit = new SplitEntry <DATAFORMAT>(saxString, UpdateOptions(null), (byte)(1 + splitDepth));
}
newSplit.Insert(input);
foreach (SaxData S in B)
{
newSplit.Insert(S);
}
// update index entry from terminal to split
index[saxString] = newSplit;
}
}
else if (entry is SplitEntry <DATAFORMAT> ) // internal node
{
((SplitEntry <DATAFORMAT>)entry).Insert(input);
}
}
else // saxString has not been seen before, create new file and entry
{
ushort[] newMask = this.options.MaskCopy;
ushort[] newSaxString = Sax.SaxStrToSaxVals(saxString);
string newName = "";
for (int i = 0; i < newMask.Length; i++)
{
newName = newName + newSaxString[i].ToString() + "." + newMask[i].ToString() + "_";
}
newName = newName.Substring(0, newName.Length - 1);
string newfile = Path.Combine(WorkingFolder, string.Concat(newName, ".0.txt"));
TermEntry newEntry = new TermEntry(saxString, newfile);
newEntry.InsertToBuffer(input);
index.Add(saxString, newEntry);
}
}
}
