// Finds double line break boundaries (i.e. empty line between records). "\r" newline only!
// logStream - stream to find boundaries, seeked to start of record area
// offsetsStream - offsets written as 8-byte ints in order
public static void FindBoundaries(string logPath, Stream logStream, Stream offsetsStream, Stream level2Stream)
{
long start = logStream.Position - 2; // caller consumed boundary - restore it for search
long length = logStream.Length - start;
int procs = 1;//Environment.ProcessorCount;
Workspace[] workspaces = new Workspace[procs];
WaitHandle[] finished = new WaitHandle[procs];
for (int i = 0; i < procs; i++)
{
long start0 = start + ((i + 0) * length) / procs;
long start1 = start + ((i + 1) * length) / procs;
Debug.Assert((i < procs - 1) || (start1 == start + length));
workspaces[i] = new Workspace(start0, start1, logPath);
finished[i] = workspaces[i].finished;
}
#if false // Not supported on STA thread
WaitHandle.WaitAll(finished);
#else
for (int i = 0; i < procs; i++)
{
finished[i].WaitOne();
}
#endif
BitVector mergedLevel2 = new BitVector();
int mergedLevel2Index = 0;
byte[] buffer = new byte[Constants.BufferSize];
int l2EpochCount = 0;
for (int i = 0; i < procs; i++)
{
workspaces[i].offsetsStream1.Seek(0, SeekOrigin.Begin);
int c = -1;
while (c != 0)
{
int o = 0;
while ((buffer.Length - o != 0) && ((c = workspaces[i].offsetsStream1.Read(buffer, o, buffer.Length - o)) != 0))
{
o += c;
}
offsetsStream.Write(buffer, 0, o);
}
BitVector workspaceLevel2 = workspaces[i].level2;
for (int j = 0; j < workspaceLevel2.Count; j++)
{
bool l2 = workspaceLevel2[j];
mergedLevel2[mergedLevel2Index++] = l2;
if (l2)
{
l2EpochCount++;
}
}
}
offsetsStream.Seek(0, SeekOrigin.Begin);
offsetsStream.SetLength(offsetsStream.Length - EpochVirtualList.OffsetRecordLength); // remove terminating boundary
Debug.Assert(offsetsStream.Length % EpochVirtualList.OffsetRecordLength == 0);
int epochCount = (int)(offsetsStream.Length / EpochVirtualList.OffsetRecordLength);
mergedLevel2.Save(level2Stream);
for (int i = 0; i < procs; i++)
{
workspaces[i].Dispose();
}
if (DebugFindBoundaries)
{
int debugl2EpochCount = 0, debugEpochCount = 0;
using (Stream logStreamTest = new FileStream(logPath, FileMode.Open, FileAccess.Read, FileShare.Read, Constants.BufferSize))
{
using (TextReader reader = new StreamReader(logPath, Encoding.UTF8))
{
// skip header
string line;
while (!String.IsNullOrEmpty(line = reader.ReadLine()))
{
}
// epochs
while ((line = reader.ReadLine()) != null)
{
debugEpochCount++;
if (String.Equals(line, "l2"))
{
debugl2EpochCount++;
}
while (!String.IsNullOrEmpty(line = reader.ReadLine()))
{
}
}
}
}
Debug.Assert(debugEpochCount == epochCount);
Debug.Assert(debugl2EpochCount == l2EpochCount);
}
}
public static Top Read(string logPath, Stream logStream)
{
TopProxy topProxy = new TopProxy();
List <Definition> definitions = new List <Definition>();
bool level2;
long timerBasis;
int samplingRate;
int envelopeRate;
int concurrency;
string acceleratorPath = Accelerators.Schedule.QueryAcceleratorPath(logPath, logStream, AcceleratorVersion);
string level2AcceleratorPath = Accelerators.Events.QueryAcceleratorPath(logPath, logStream, AcceleratorVersion);
using (TextReader reader = new StreamReader2(logStream))
{
// always read header from log file
string line;
string[] parts;
line = reader.ReadLine();
parts = line.Split('\t');
Debug.Assert(parts[0] == "version");
int version = Int32.Parse(parts[1]);
Debug.Assert(version == 1);
line = reader.ReadLine();
parts = line.Split('\t');
Debug.Assert(parts[0] == "level");
level2 = Int64.Parse(parts[1]) > 1;
line = reader.ReadLine();
parts = line.Split('\t');
Debug.Assert(parts[0] == "tres");
timerBasis = Int64.Parse(parts[1]);
line = reader.ReadLine();
parts = line.Split('\t');
Debug.Assert(parts[0] == "srate");
samplingRate = Int32.Parse(parts[1]);
line = reader.ReadLine();
parts = line.Split('\t');
Debug.Assert(parts[0] == "erate");
envelopeRate = Int32.Parse(parts[1]);
line = reader.ReadLine();
parts = line.Split('\t');
Debug.Assert(parts[0] == "threads");
concurrency = Int32.Parse(parts[1]);
line = reader.ReadLine();
if (!String.Equals(line, ":"))
{
Debug.Assert(false);
}
while (!String.IsNullOrEmpty(line = reader.ReadLine()))
{
parts = line.Split('\t');
switch (parts[1])
{
default:
Debug.Assert(false);
throw new ArgumentException();
case "section":
definitions.Add(
new Definition(
topProxy,
Int32.Parse(parts[0]),
parts[2],
Kind.Effect,
-1));
break;
case "track":
definitions.Add(
new Definition(
topProxy,
Int32.Parse(parts[0]),
parts[3],
Kind.Track,
Int32.Parse(parts[2])));
break;
}
}
}
// create offsets table for epoch records
if (acceleratorPath == null)
{
acceleratorPath = Path.GetTempFileName();
level2AcceleratorPath = Path.GetTempFileName();
#if false
long end = stream.Length;
using (Stream acceleratorStream = new FileStream(acceleratorPath, FileMode.Create, FileAccess.ReadWrite, FileShare.None, Constants.BufferSize))
{
using (BinaryWriter acceleratorWriter = new BinaryWriter(acceleratorStream, Encoding.UTF8))
{
while (((StreamReader2)reader).Position < end)
{
acceleratorWriter.Write((long)((StreamReader2)reader).Position);
EpochResident.Read(reader, topProxy, definitions.Count);
}
}
}
#else
using (Stream acceleratorStream = new FileStream(acceleratorPath, FileMode.Create, FileAccess.ReadWrite, FileShare.None, Constants.BufferSize))
{
using (Stream level2Stream = new FileStream(level2AcceleratorPath, FileMode.Create, FileAccess.ReadWrite, FileShare.None, Constants.BufferSize))
{
FindBoundariesHelper.FindBoundaries(logPath, logStream, acceleratorStream, level2Stream);
}
}
#endif
Accelerators.Schedule.RecordAcceleratorPath(logPath, logStream, acceleratorPath, AcceleratorVersion);
Accelerators.Events.RecordAcceleratorPath(logPath, logStream, level2AcceleratorPath, AcceleratorVersion);
}
Stream acceleratorStream2 = new FileStream(acceleratorPath, FileMode.Open, FileAccess.Read, FileShare.Read, Constants.BufferSize);
BinaryReader acceleratorReader2 = new BinaryReader(acceleratorStream2, Encoding.UTF8);
BitVector level2Bits;
using (Stream level2Stream = new FileStream(level2AcceleratorPath, FileMode.Open, FileAccess.Read, FileShare.Read, Constants.BufferSize))
{
level2Bits = BitVector.Load(level2Stream);
}
IList <Epoch> epochs = new EpochVirtualList(
topProxy,
(int)(acceleratorStream2.Length / EpochVirtualList.OffsetRecordLength),
level2Bits);
//System.Windows.Forms.MessageBox.Show("Number of epochs: " + epochs.Count.ToString());
Top top = new Top(
logStream,
level2,
timerBasis,
samplingRate,
envelopeRate,
concurrency,
definitions,
acceleratorStream2,
acceleratorReader2,
epochs);
topProxy.Top = top;
return(top);
}
public EpochVirtualListEnumerator(TopProxy topProxy, int count, BitVector level2)
{
this.topProxy = topProxy;
this.count = count;
this.level2 = level2;
}
private void Generate()
{
if (bars != null)
{
return;
}
if (!top.Level2)
{
return;
}
#if false
string acceleratorPath = Accelerators.Events.QueryAcceleratorPath(logPath, top.LogStream, Top.AcceleratorVersion);
if (acceleratorPath == null)
{
evtf = new BitVector(top.Epochs.Count);
}
else
{
using (Stream stream = new FileStream(acceleratorPath, FileMode.Open, FileAccess.Read, FileShare.Read, Constants.BufferSize))
{
evtf = BitVector.Load(stream);
}
}
#endif
List <EventBar>[] bars2 = new List <EventBar> [top.Definitions.Count];
Dictionary <int, EventBar>[] bars3 = new Dictionary <int, EventBar> [top.Definitions.Count];
for (int j = 0; j < top.Definitions.Count; j++)
{
bars2[j] = new List <EventBar>();
bars3[j] = new Dictionary <int, EventBar>();
}
for (int i = 0; i < top.Epochs.Count; i++)
{
#if false
if (acceleratorPath != null)
{
if (!evtf[i])
{
continue;
}
}
bool found = false;
#endif
Epoch epoch = top.Epochs[i];
if (!epoch.Level2)
{
continue;
}
for (int j = 0; j < top.Definitions.Count; j++)
{
DatumEvent[] events = epoch.Data[j].Events;
foreach (DatumEvent e in events)
{
#if false
found = true;
#endif
switch (e.Tag)
{
default:
Debug.Assert(false);
break;
case "Start":
{
EventBar bar = new EventBar();
bar.start = i;
bar.seq = e.Seq;
bar.frameIndex = ((DatumEvent2)e).frameIndex;
bar.noteIndex = ((DatumEvent2)e).noteIndex;
bars2[j].Add(bar);
bars3[j].Add(e.Seq, bar);
}
break;
case "Stop":
{
EventBar bar = bars3[j][e.Seq];
bar.end = i;
}
break;
case "Restart":
{
EventBar bar = bars3[j][e.Seq];
EventBar.Tie tie = new EventBar.Tie();
tie.when = i;
tie.frameIndex = ((DatumEvent2)e).frameIndex;
tie.noteIndex = ((DatumEvent2)e).noteIndex;
int p = bar.ties.Length;
Array.Resize(ref bar.ties, p + 1);
bar.ties[p] = tie;
}
break;
case "SkipEnter":
case "SkipLeave":
{
EventBar bar = new EventBar();
bar.tag = e.Tag;
bar.start = i;
bar.end = i;
bar.seq = e.Seq;
bars2[j].Add(bar);
bars3[j].Add(e.Seq, bar);
}
break;
}
}
}
#if false
if ((acceleratorPath == null) && found)
{
evtf[i] = true;
}
#endif
}
#if false
if (acceleratorPath == null)
{
acceleratorPath = System.IO.Path.GetTempFileName();
using (Stream stream = new FileStream(acceleratorPath, FileMode.Create, FileAccess.Write, FileShare.None, Constants.BufferSize))
{
evtf.Save(stream);
}
Accelerators.Events.RecordAcceleratorPath(logPath, top.LogStream, acceleratorPath, Top.AcceleratorVersion);
}
#endif
bars = new EventBar[top.Definitions.Count][];
for (int j = 0; j < top.Definitions.Count; j++)
{
bars[j] = bars2[j].ToArray();
}
}
