/// <summary>
/// build index for an entire folder
/// </summary>
/// <param name="folder"></param>
public HistSimIndexPlay(string folder, int interval)
{
_folder = folder + "\\";
string[,] tfi = Util.TickFileIndex(folder, TikConst.WILDCARD_EXT);
// see if we have index
string fn;
HistSimIndex hsi;
_extdebug = tfi.GetLength(0) > 500;
edebug("checking for index for: " + tfi.GetLength(0) + "-sized simulation.");
if (HistSimIndex.HaveIndex(folder, tfi, interval, out fn))
{
edebug("index found.");
hsi = HistSimIndex.FromFile(fn);
hsi.GotDebug += new DebugDelegate(debug);
}
else if (HistSimIndex.BuildIndex(folder, tfi, out hsi, true, true, interval, debug))
{
edebug("Index built successfully. ");
}
else
{
debug("Error building index.");
return;
}
myhsi = hsi;
_reader.DoWork += new DoWorkEventHandler(_reader_DoWork);
_reader.WorkerSupportsCancellation = true;
hsip_avail = hsi.Playindex.Length;
checkindex();
}
/// <summary>
/// deserialize an index
/// </summary>
/// <param name="msg"></param>
/// <returns></returns>
public static HistSimIndex Deserialize(string msg)
{
HistSimIndex cw = null;
try
{
string xmsg = GZip.Uncompress(msg);
// prepare serializer
XmlSerializer xs = new XmlSerializer(typeof(HistSimIndex));
// read in message
StringReader fs = new StringReader(xmsg);
// deserialize message
cw = (HistSimIndex)xs.Deserialize(fs);
// close serializer
fs.Close();
// unpack toc
cw.unpackTOC();
}
#if DEBUG
catch (FileNotFoundException ex)
{
Console.WriteLine("fill not found error deserializing histsimindex: " + msg + " err: " + ex.Message + ex.StackTrace);
}
#else
catch (FileNotFoundException)
{
}
#endif
return(cw);
}
public static bool BuildIndex(string folder, string[,] tickfiles, out HistSimIndex hsi, bool start, bool saveindex, int interval, DebugDelegate debug)
{
hsi = new HistSimIndex(folder, tickfiles, interval);
if (debug != null)
{
hsi.GotDebug += new DebugDelegate(debug);
}
bool ok = true;
if (start)
{
hsi.Start();
}
if (start && saveindex)
{
bool save = ToFile(hsi, idxfold(folder), debug);
if (debug != null)
{
debug(save ? ("Saved index: " + folder) : "Saving index failed.");
}
ok &= save;
}
ok &= start ? hsi.isComplete : hsi.TOC.Length > 0;
return(ok);
}
public override bool Equals(object obj)
{
try
{
HistSimIndex hsi = (HistSimIndex)obj;
return(Equals(hsi));
}
catch { }
return(false);
}
static string indextoc(HistSimIndex hsi)
{
string data = hsi.Interval == 0 ? string.Empty : hsi.Interval.ToString();
for (int i = 0; i < hsi.TOC.GetLength(0); i++)
{
data += hsi.TOC[i, 0] + "=" + hsi.TOC[i, 1] + " ";
}
return(data);
}
public HistSimIndexPlay(HistSimIndex hsi, DebugDelegate deb)
{
_folder = hsi.Folder;
if (deb != null)
{
GotDebug += deb;
}
_reader.DoWork += new DoWorkEventHandler(_reader_DoWork);
_reader.WorkerSupportsCancellation = true;
myhsi = hsi;
hsip_avail = hsi.Playindex.Length;
checkindex();
}
public static bool BuildIndex(string folder, TickFileFilter tff, out HistSimIndex hsi, bool start, bool saveidx, int interval, DebugDelegate debug)
{
if (debug != null)
{
debug("getting tickfiles available, please wait...");
}
string[,] allows = Util.TickFileIndex(folder, TikConst.WILDCARD_EXT);
if (debug != null)
{
debug("found " + allows.Length + " tickfiles.");
}
return(BuildIndex(folder, allows, tff, out hsi, start, saveidx, interval, debug));
}
public HistSimIndexPlay(string folder, TickFileFilter tff, int interval, DebugDelegate deb)
{
if (deb != null)
{
GotDebug += deb;
}
_folder = folder + "\\";
// see if we have index
string fn;
HistSimIndex hsi;
debug("getting tickfiles, please wait...");
string[,] total = tff.bUseCSV ? ChimeraDataUtils.TickFileIndex(folder, "*_trades.csv", true)
: Util.TickFileIndex(folder, TikConst.WILDCARD_EXT, false);
debug("found " + total.GetLength(0).ToString("N0") + " tickfiles");
string[,] filtered = tff.AllowsIndexAndSize(total);
debug("post-filter: " + filtered.GetLength(0).ToString("N0") + ", checking for index...");
if (HistSimIndex.HaveIndex(folder, filtered, out fn, deb, tff.bUseCSV))
{
debug("index found, starting load: " + fn);
hsi = HistSimIndex.FromFile(fn, debug);
if (hsi == null)
{
debug("unable to load index.");
return;
}
hsi.GotDebug += new DebugDelegate(debug);
}
else if (HistSimIndex.BuildIndex(folder, filtered, out hsi, true, true, interval, debug, tff.bUseCSV))
{
debug("Index built successfully, ready to play.");
}
else
{
debug("Error building index...");
return;
}
_reader.DoWork += new DoWorkEventHandler(_reader_DoWork);
_reader.WorkerSupportsCancellation = true;
myhsi = hsi;
hsip_avail = hsi.Playindex.Length;
checkindex();
}
public static string Serialize(HistSimIndex hsi, DebugDelegate debug)
{
try
{
hsi.packTOC();
XmlSerializer xs = new XmlSerializer(typeof(HistSimIndex));
StringWriter fs = new StringWriter();
xs.Serialize(fs, hsi);
fs.Close();
string msg = fs.GetStringBuilder().ToString();
return GZip.Compress(msg);
}
catch (Exception ex)
{
if (debug != null)
debug(ex.Message + ex.StackTrace);
}
return string.Empty;
}
public static bool ToFile(HistSimIndex hsi, string path, DebugDelegate debug)
{
try
{
// get checksum of data
string md5 = MD5(indextoc(hsi));
// see if it's complete
if (!hsi.isComplete)
{
if (debug != null)
{
debug("Index not completed " + md5);
}
return(false);
}
// get index directory
string dir = Path.GetDirectoryName(path);
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
// get filepath
string filepath = dir + "\\" + md5 + ".txt";
// pack contents so serializable
hsi.packTOC();
XmlSerializer xs = new XmlSerializer(typeof(HistSimIndex));
Stream fs = new FileStream(filepath, FileMode.Create);
System.IO.Compression.GZipStream gs = new System.IO.Compression.GZipStream(fs, System.IO.Compression.CompressionMode.Compress);
xs.Serialize(gs, hsi);
gs.Close();
fs.Close();
}
catch (Exception ex)
{
if (debug != null)
{
debug(ex.Message + ex.StackTrace);
}
return(false);
}
return(true);
}
public HistSimIndexPlay(string[] dataset, string folder, int interval, DebugDelegate deb)
{
if (deb != null)
{
GotDebug += deb;
}
if (dataset.Length < 1)
{
debug("Empty data set, quitting...");
return;
}
_folder = folder;
// get index of set
string[,] tfi = GetTickIndex(dataset, false, folder);
_extdebug = tfi.GetLength(0) > 500;
// see if we have index
string fn;
HistSimIndex hsi;
edebug("checking for index");
if (HistSimIndex.HaveIndex(folder, tfi, interval, out fn))
{
edebug("index found.");
hsi = HistSimIndex.FromFile(fn);
hsi.GotDebug += new DebugDelegate(debug);
}
else if (HistSimIndex.BuildIndex(folder, tfi, out hsi, true, true, interval, debug))
{
edebug("Index built successfully");
}
else
{
debug("Error building index...");
return;
}
myhsi = hsi;
_reader.DoWork += new DoWorkEventHandler(_reader_DoWork);
_reader.WorkerSupportsCancellation = true;
hsip_avail = hsi.Playindex.Length;
checkindex();
}
public static string Serialize(HistSimIndex hsi, DebugDelegate debug)
{
try
{
hsi.packTOC();
XmlSerializer xs = new XmlSerializer(typeof(HistSimIndex));
StringWriter fs = new StringWriter();
xs.Serialize(fs, hsi);
fs.Close();
string msg = fs.GetStringBuilder().ToString();
return(GZip.Compress(msg));
}
catch (Exception ex)
{
if (debug != null)
{
debug(ex.Message + ex.StackTrace);
}
}
return(string.Empty);
}
/// <summary>
/// checks whether we already have an index built for a given data set.
/// if index is empty, deletes index.
/// </summary>
/// <param name="folder"></param>
/// <param name="tff"></param>
/// <param name="file"></param>
/// <returns></returns>
public static bool HaveIndex(string folder, string[,] tickindex, int interval, out string file)
{
HistSimIndex hsi;
if (BuildIndex(folder, tickindex, out hsi, false, false, interval, null))
{
string checksum = MD5(indextoc(hsi));
string fn = idxfold(folder) + checksum + ".txt";
if (File.Exists(fn))
{
file = fn;
hsi = HistSimIndex.FromFile(fn);
bool complete = hsi.isComplete && (hsi.Playindex.Length > 0);
if (!complete)
{
File.Delete(fn);
return(false);
}
return(true);
}
}
file = string.Empty;
return(false);
}
/// <summary>
/// whether two indicies are equivalent (index same files)
/// </summary>
/// <param name="idx"></param>
/// <returns></returns>
public bool Equals(HistSimIndex idx)
{
if (_contents.Length != idx.TOC.Length)
{
return(false);
}
bool equal = true;
for (int i = 0; i < idx.TOC.GetLength(0); i++)
{
// look for our match
bool match = false;
for (int j = 0; j < _contents.GetLength(0); j++)
{
if ((idx.TOC[i, 0] == _contents[j, 0]) && (idx.TOC[i, 1] == _contents[j, 1]))
{
match = true;
break;
}
}
equal &= match;
}
return(equal);
}
/// <summary>
/// build index for an entire folder
/// </summary>
/// <param name="folder"></param>
public HistSimIndexPlay(string folder, int interval)
{
_folder = folder+"\\";
string[,] tfi = Util.TickFileIndex(folder, TikConst.WILDCARD_EXT);
// see if we have index
string fn;
HistSimIndex hsi;
_extdebug = tfi.GetLength(0) > 500;
edebug("checking for index for: "+tfi.GetLength(0)+"-sized simulation.");
if (HistSimIndex.HaveIndex(folder,tfi,interval,out fn))
{
edebug("index found.");
hsi = HistSimIndex.FromFile(fn);
hsi.GotDebug += new DebugDelegate(debug);
}
else if (HistSimIndex.BuildIndex(folder,tfi,out hsi, true,true,interval,debug))
{
edebug("Index built successfully. ");
}
else
{
debug("Error building index.");
return;
}
myhsi = hsi;
_reader.DoWork += new DoWorkEventHandler(_reader_DoWork);
_reader.WorkerSupportsCancellation = true;
hsip_avail = hsi.Playindex.Length;
checkindex();
}
/// <summary>
/// gets index for set of tick files
/// </summary>
/// <param name="tickfiles"></param>
/// <param name="hsi"></param>
/// <param name="start">whether to build the index or just the TOC</param>
/// <returns></returns>
public static bool BuildIndex(string folder, string[,] tickfiles, out HistSimIndex hsi, bool start, bool bUseCSV)
{
return(BuildIndex(folder, tickfiles, out hsi, start, true, 0, null, bUseCSV));
}
public static bool BuildIndex(string folder, string[,] tickfiles, out HistSimIndex hsi, bool start, bool saveindex, int interval, DebugDelegate debug)
{
hsi = new HistSimIndex(folder,tickfiles,interval);
if (debug!=null)
hsi.GotDebug+=new DebugDelegate(debug);
bool ok = true;
if (start)
hsi.Start();
if (start && saveindex)
{
bool save = ToFile(hsi, idxfold(folder), debug);
if (debug != null)
debug(save ? ("Saved index: " + folder) : "Saving index failed.");
ok &= save;
}
ok &= start ? hsi.isComplete : hsi.TOC.Length > 0;
return ok;
}
public HistSimMemory(HistSimIndex hsi,DebugDelegate debug)
: base(hsi,debug)
{
}
public HistSimIndexPlay(HistSimIndex hsi, DebugDelegate deb)
{
_folder = hsi.Folder;
if (deb != null)
GotDebug += deb;
_reader.DoWork += new DoWorkEventHandler(_reader_DoWork);
_reader.WorkerSupportsCancellation = true;
myhsi = hsi;
hsip_avail = hsi.Playindex.Length;
checkindex();
}
/// <summary>
/// gets index for given set of tick files
/// </summary>
/// <param name="tickfiles"></param>
/// <param name="hsi"></param>
/// <returns></returns>
public static bool BuildIndex(string folder, string[,] tickfiles, out HistSimIndex hsi) { return BuildIndex(folder, tickfiles, out hsi, true); }
/// <summary>
/// gets index from set of tick files and starts indexing
/// </summary>
/// <param name="tickfiles"></param>
/// <param name="tff"></param>
/// <param name="hsi"></param>
/// <param name="start"></param>
/// <returns></returns>
public static bool BuildIndex(string folder, string[,] tickfiles, TickFileFilter tff, out HistSimIndex hsi) { return BuildIndex(folder, tickfiles, tff,out hsi,0, null); }
public static bool BuildIndex(string folder, string[,] tickfiles, out HistSimIndex hsi, bool start, bool saveindex) { return BuildIndex(folder, tickfiles, out hsi, start, saveindex, 0,null); }
public HistSimMemory(HistSimIndex hsi, DebugDelegate debug)
: base(hsi, debug)
{
}
/// <summary>
/// serialize index
/// </summary>
/// <param name="hsi"></param>
/// <returns></returns>
public static string Serialize(HistSimIndex hsi)
{
return(Serialize(hsi, null));
}
public static bool BuildIndex(string folder, string[,] tickfiles, out HistSimIndex hsi, bool start, bool saveindex)
{
return(BuildIndex(folder, tickfiles, out hsi, start, saveindex, 0, null));
}
/// <summary>
/// gets index for given set of tick files
/// </summary>
/// <param name="tickfiles"></param>
/// <param name="hsi"></param>
/// <returns></returns>
public static bool BuildIndex(string folder, string[,] tickfiles, out HistSimIndex hsi)
{
return(BuildIndex(folder, tickfiles, out hsi, true));
}
public static bool BuildIndex(string folder, string[,] tickfiles, TickFileFilter tff, out HistSimIndex hsi, int interval, DebugDelegate debug)
{
if (debug != null)
{
debug("filtering tickfiles...");
}
string[,] allows = tff.AllowsIndexAndSize(tickfiles);
if (debug != null)
{
debug("using " + allows.Length + " tickfiles post-filter.");
}
return(BuildIndex(folder, allows, out hsi, true, true, interval, debug));
}
/// <summary>
/// gets index from set of tick files and starts indexing
/// </summary>
/// <param name="tickfiles"></param>
/// <param name="tff"></param>
/// <param name="hsi"></param>
/// <param name="start"></param>
/// <returns></returns>
public static bool BuildIndex(string folder, string[,] tickfiles, TickFileFilter tff, out HistSimIndex hsi)
{
return(BuildIndex(folder, tickfiles, tff, out hsi, 0, null));
}
/// <summary>
/// serialize idnex as a file
/// </summary>
/// <param name="hsi"></param>
/// <param name="path"></param>
/// <returns></returns>
public static bool ToFile(HistSimIndex hsi, string path) { return ToFile(hsi, path, null); }
/// <summary>
/// get index from a set of tick files given a filter
/// </summary>
/// <param name="folder"></param>
/// <param name="tff"></param>
/// <param name="hsi"></param>
/// <returns></returns>
public static bool BuildIndex(string folder, TickFileFilter tff, out HistSimIndex hsi, bool start)
{
return(BuildIndex(folder, tff, out hsi, start, true, 0, null));
}
public HistSimMemory(HistSimIndex hsi)
: base(hsi, null)
{
}
public static bool BuildIndex(string folder, TickFileFilter tff, out HistSimIndex hsi,int interval,DebugDelegate debug)
{
if (debug != null)
debug("getting tickfiles available, please wait...");
string[,] allows = Util.TickFileIndex(folder, TikConst.WILDCARD_EXT);
if (debug != null)
debug("found " + allows.Length + " tickfiles.");
return BuildIndex(folder,allows, tff,out hsi, true,true,interval,debug);
}
/// <summary>
/// create histsimindex player from an index
/// </summary>
/// <param name="hsi"></param>
public HistSimIndexPlay(HistSimIndex hsi) : this(hsi, null)
{
}
static string indextoc(HistSimIndex hsi)
{
string data = hsi.Interval == 0 ? string.Empty : hsi.Interval.ToString();
for (int i = 0; i < hsi.TOC.GetLength(0); i++)
{
data += hsi.TOC[i,0]+"="+hsi.TOC[i,1] + " ";
}
return data;
}
public static bool BuildIndex(string folder, string[,] tickfiles, TickFileFilter tff, out HistSimIndex hsi, int interval, DebugDelegate debug)
{
if (debug != null)
debug("filtering tickfiles...");
string[,] allows = tff.AllowsIndexAndSize(tickfiles);
if (debug != null)
debug("using " + allows.Length + " tickfiles post-filter.");
return BuildIndex(folder,allows, out hsi, true,true,interval,debug);
}
/// <summary>
/// whether two indicies are equivalent (index same files)
/// </summary>
/// <param name="idx"></param>
/// <returns></returns>
public bool Equals(HistSimIndex idx)
{
if (_contents.Length != idx.TOC.Length) return false;
bool equal = true;
for (int i = 0; i < idx.TOC.GetLength(0); i++)
{
// look for our match
bool match = false;
for (int j = 0; j<_contents.GetLength(0); j++)
if ((idx.TOC[i,0] == _contents[j,0]) && (idx.TOC[i,1]==_contents[j,1]))
{
match = true;
break;
}
equal &= match;
}
return equal;
}
public HistSimMemory(HistSimIndex hsi)
: base(hsi,null)
{
}
/// <summary>
/// serialize idnex as a file
/// </summary>
/// <param name="hsi"></param>
/// <param name="path"></param>
/// <returns></returns>
public static bool ToFile(HistSimIndex hsi, string path)
{
return(ToFile(hsi, path, null));
}
/// <summary>
/// create histsimindex player from an index
/// </summary>
/// <param name="hsi"></param>
public HistSimIndexPlay(HistSimIndex hsi) : this(hsi, null) { }
/// <summary>
/// serialize index
/// </summary>
/// <param name="hsi"></param>
/// <returns></returns>
public static string Serialize(HistSimIndex hsi) { return Serialize(hsi, null); }
public HistSimIndexPlay(string folder, TickFileFilter tff,int interval,DebugDelegate deb)
{
if (deb!=null)
GotDebug+=deb;
_folder = folder + "\\";
// see if we have index
string fn;
HistSimIndex hsi;
debug("getting tickfiles, please wait...");
string[,] total = Util.TickFileIndex(folder, TikConst.WILDCARD_EXT,false);
debug("found " + total.GetLength(0).ToString("N0") + " tickfiles");
string[,] filtered = tff.AllowsIndexAndSize(total);
debug("post-filter: " + filtered.GetLength(0).ToString("N0")+", checking for index...");
if (HistSimIndex.HaveIndex(folder, filtered, out fn,deb))
{
debug("index found, starting load: "+fn);
hsi = HistSimIndex.FromFile(fn,debug);
if (hsi == null)
{
debug("unable to load index.");
return;
}
hsi.GotDebug+=new DebugDelegate(debug);
}
else if (HistSimIndex.BuildIndex(folder, filtered, out hsi, true,true,interval,debug))
{
debug("Index built successfully, ready to play.");
}
else
{
debug("Error building index...");
return;
}
_reader.DoWork += new DoWorkEventHandler(_reader_DoWork);
_reader.WorkerSupportsCancellation = true;
myhsi = hsi;
hsip_avail = hsi.Playindex.Length;
checkindex();
}
public static bool ToFile(HistSimIndex hsi, string path,DebugDelegate debug)
{
try
{
// get checksum of data
string md5 = MD5(indextoc(hsi));
// see if it's complete
if (!hsi.isComplete)
{
if (debug!=null)
debug("Index not completed "+md5);
return false;
}
// get index directory
string dir = Path.GetDirectoryName(path);
if (!Directory.Exists(dir))
Directory.CreateDirectory(dir);
// get filepath
string filepath = dir + "\\" + md5 + ".txt";
// pack contents so serializable
hsi.packTOC();
XmlSerializer xs = new XmlSerializer(typeof(HistSimIndex));
Stream fs = new FileStream(filepath, FileMode.Create);
System.IO.Compression.GZipStream gs = new System.IO.Compression.GZipStream(fs, System.IO.Compression.CompressionMode.Compress);
xs.Serialize(gs, hsi);
gs.Close();
fs.Close();
}
catch (Exception ex)
{
if (debug != null)
debug(ex.Message + ex.StackTrace);
return false;
}
return true;
}
public HistSimIndexPlay(string[] dataset,string folder, int interval, DebugDelegate deb)
{
if (deb != null)
GotDebug += deb;
if (dataset.Length < 1)
{
debug("Empty data set, quitting...");
return;
}
_folder = folder;
// get index of set
string[,] tfi = GetTickIndex(dataset,false,folder);
_extdebug = tfi.GetLength(0) > 500;
// see if we have index
string fn;
HistSimIndex hsi;
edebug("checking for index");
if (HistSimIndex.HaveIndex(folder, tfi,interval, out fn))
{
edebug("index found.");
hsi = HistSimIndex.FromFile(fn);
hsi.GotDebug += new DebugDelegate(debug);
}
else if (HistSimIndex.BuildIndex(folder, tfi, out hsi, true, true,interval, debug))
{
edebug("Index built successfully");
}
else
{
debug("Error building index...");
return;
}
myhsi = hsi;
_reader.DoWork += new DoWorkEventHandler(_reader_DoWork);
_reader.WorkerSupportsCancellation = true;
hsip_avail = hsi.Playindex.Length;
checkindex();
}
/// <summary>
/// get index from a set of tick files given a filter and starts indexing
/// </summary>
/// <param name="folder"></param>
/// <param name="tff"></param>
/// <param name="hsi"></param>
/// <returns></returns>
public static bool BuildIndex(string folder, TickFileFilter tff, out HistSimIndex hsi) { return BuildIndex(folder, tff, out hsi, true, true, 0,null); }
