/// <summary> /// build index for an entire folder /// </summary> /// <param name="folder"></param> public HistSimIndexPlay(string folder, int interval) { _folder = folder + "\\"; string[,] tfi = Util.TickFileIndex(folder, TikConst.WILDCARD_EXT); // see if we have index string fn; HistSimIndex hsi; _extdebug = tfi.GetLength(0) > 500; edebug("checking for index for: " + tfi.GetLength(0) + "-sized simulation."); if (HistSimIndex.HaveIndex(folder, tfi, interval, out fn)) { edebug("index found."); hsi = HistSimIndex.FromFile(fn); hsi.GotDebug += new DebugDelegate(debug); } else if (HistSimIndex.BuildIndex(folder, tfi, out hsi, true, true, interval, debug)) { edebug("Index built successfully. "); } else { debug("Error building index."); return; } myhsi = hsi; _reader.DoWork += new DoWorkEventHandler(_reader_DoWork); _reader.WorkerSupportsCancellation = true; hsip_avail = hsi.Playindex.Length; checkindex(); }
/// <summary> /// deserialize an index /// </summary> /// <param name="msg"></param> /// <returns></returns> public static HistSimIndex Deserialize(string msg) { HistSimIndex cw = null; try { string xmsg = GZip.Uncompress(msg); // prepare serializer XmlSerializer xs = new XmlSerializer(typeof(HistSimIndex)); // read in message StringReader fs = new StringReader(xmsg); // deserialize message cw = (HistSimIndex)xs.Deserialize(fs); // close serializer fs.Close(); // unpack toc cw.unpackTOC(); } #if DEBUG catch (FileNotFoundException ex) { Console.WriteLine("fill not found error deserializing histsimindex: " + msg + " err: " + ex.Message + ex.StackTrace); } #else catch (FileNotFoundException) { } #endif return(cw); }
public static bool BuildIndex(string folder, string[,] tickfiles, out HistSimIndex hsi, bool start, bool saveindex, int interval, DebugDelegate debug) { hsi = new HistSimIndex(folder, tickfiles, interval); if (debug != null) { hsi.GotDebug += new DebugDelegate(debug); } bool ok = true; if (start) { hsi.Start(); } if (start && saveindex) { bool save = ToFile(hsi, idxfold(folder), debug); if (debug != null) { debug(save ? ("Saved index: " + folder) : "Saving index failed."); } ok &= save; } ok &= start ? hsi.isComplete : hsi.TOC.Length > 0; return(ok); }
public override bool Equals(object obj) { try { HistSimIndex hsi = (HistSimIndex)obj; return(Equals(hsi)); } catch { } return(false); }
static string indextoc(HistSimIndex hsi) { string data = hsi.Interval == 0 ? string.Empty : hsi.Interval.ToString(); for (int i = 0; i < hsi.TOC.GetLength(0); i++) { data += hsi.TOC[i, 0] + "=" + hsi.TOC[i, 1] + " "; } return(data); }
public HistSimIndexPlay(HistSimIndex hsi, DebugDelegate deb) { _folder = hsi.Folder; if (deb != null) { GotDebug += deb; } _reader.DoWork += new DoWorkEventHandler(_reader_DoWork); _reader.WorkerSupportsCancellation = true; myhsi = hsi; hsip_avail = hsi.Playindex.Length; checkindex(); }
public static bool BuildIndex(string folder, TickFileFilter tff, out HistSimIndex hsi, bool start, bool saveidx, int interval, DebugDelegate debug) { if (debug != null) { debug("getting tickfiles available, please wait..."); } string[,] allows = Util.TickFileIndex(folder, TikConst.WILDCARD_EXT); if (debug != null) { debug("found " + allows.Length + " tickfiles."); } return(BuildIndex(folder, allows, tff, out hsi, start, saveidx, interval, debug)); }
public HistSimIndexPlay(string folder, TickFileFilter tff, int interval, DebugDelegate deb) { if (deb != null) { GotDebug += deb; } _folder = folder + "\\"; // see if we have index string fn; HistSimIndex hsi; debug("getting tickfiles, please wait..."); string[,] total = tff.bUseCSV ? ChimeraDataUtils.TickFileIndex(folder, "*_trades.csv", true) : Util.TickFileIndex(folder, TikConst.WILDCARD_EXT, false); debug("found " + total.GetLength(0).ToString("N0") + " tickfiles"); string[,] filtered = tff.AllowsIndexAndSize(total); debug("post-filter: " + filtered.GetLength(0).ToString("N0") + ", checking for index..."); if (HistSimIndex.HaveIndex(folder, filtered, out fn, deb, tff.bUseCSV)) { debug("index found, starting load: " + fn); hsi = HistSimIndex.FromFile(fn, debug); if (hsi == null) { debug("unable to load index."); return; } hsi.GotDebug += new DebugDelegate(debug); } else if (HistSimIndex.BuildIndex(folder, filtered, out hsi, true, true, interval, debug, tff.bUseCSV)) { debug("Index built successfully, ready to play."); } else { debug("Error building index..."); return; } _reader.DoWork += new DoWorkEventHandler(_reader_DoWork); _reader.WorkerSupportsCancellation = true; myhsi = hsi; hsip_avail = hsi.Playindex.Length; checkindex(); }
public static string Serialize(HistSimIndex hsi, DebugDelegate debug) { try { hsi.packTOC(); XmlSerializer xs = new XmlSerializer(typeof(HistSimIndex)); StringWriter fs = new StringWriter(); xs.Serialize(fs, hsi); fs.Close(); string msg = fs.GetStringBuilder().ToString(); return GZip.Compress(msg); } catch (Exception ex) { if (debug != null) debug(ex.Message + ex.StackTrace); } return string.Empty; }
public static bool ToFile(HistSimIndex hsi, string path, DebugDelegate debug) { try { // get checksum of data string md5 = MD5(indextoc(hsi)); // see if it's complete if (!hsi.isComplete) { if (debug != null) { debug("Index not completed " + md5); } return(false); } // get index directory string dir = Path.GetDirectoryName(path); if (!Directory.Exists(dir)) { Directory.CreateDirectory(dir); } // get filepath string filepath = dir + "\\" + md5 + ".txt"; // pack contents so serializable hsi.packTOC(); XmlSerializer xs = new XmlSerializer(typeof(HistSimIndex)); Stream fs = new FileStream(filepath, FileMode.Create); System.IO.Compression.GZipStream gs = new System.IO.Compression.GZipStream(fs, System.IO.Compression.CompressionMode.Compress); xs.Serialize(gs, hsi); gs.Close(); fs.Close(); } catch (Exception ex) { if (debug != null) { debug(ex.Message + ex.StackTrace); } return(false); } return(true); }
public HistSimIndexPlay(string[] dataset, string folder, int interval, DebugDelegate deb) { if (deb != null) { GotDebug += deb; } if (dataset.Length < 1) { debug("Empty data set, quitting..."); return; } _folder = folder; // get index of set string[,] tfi = GetTickIndex(dataset, false, folder); _extdebug = tfi.GetLength(0) > 500; // see if we have index string fn; HistSimIndex hsi; edebug("checking for index"); if (HistSimIndex.HaveIndex(folder, tfi, interval, out fn)) { edebug("index found."); hsi = HistSimIndex.FromFile(fn); hsi.GotDebug += new DebugDelegate(debug); } else if (HistSimIndex.BuildIndex(folder, tfi, out hsi, true, true, interval, debug)) { edebug("Index built successfully"); } else { debug("Error building index..."); return; } myhsi = hsi; _reader.DoWork += new DoWorkEventHandler(_reader_DoWork); _reader.WorkerSupportsCancellation = true; hsip_avail = hsi.Playindex.Length; checkindex(); }
public static string Serialize(HistSimIndex hsi, DebugDelegate debug) { try { hsi.packTOC(); XmlSerializer xs = new XmlSerializer(typeof(HistSimIndex)); StringWriter fs = new StringWriter(); xs.Serialize(fs, hsi); fs.Close(); string msg = fs.GetStringBuilder().ToString(); return(GZip.Compress(msg)); } catch (Exception ex) { if (debug != null) { debug(ex.Message + ex.StackTrace); } } return(string.Empty); }
/// <summary> /// checks whether we already have an index built for a given data set. /// if index is empty, deletes index. /// </summary> /// <param name="folder"></param> /// <param name="tff"></param> /// <param name="file"></param> /// <returns></returns> public static bool HaveIndex(string folder, string[,] tickindex, int interval, out string file) { HistSimIndex hsi; if (BuildIndex(folder, tickindex, out hsi, false, false, interval, null)) { string checksum = MD5(indextoc(hsi)); string fn = idxfold(folder) + checksum + ".txt"; if (File.Exists(fn)) { file = fn; hsi = HistSimIndex.FromFile(fn); bool complete = hsi.isComplete && (hsi.Playindex.Length > 0); if (!complete) { File.Delete(fn); return(false); } return(true); } } file = string.Empty; return(false); }
/// <summary> /// whether two indicies are equivalent (index same files) /// </summary> /// <param name="idx"></param> /// <returns></returns> public bool Equals(HistSimIndex idx) { if (_contents.Length != idx.TOC.Length) { return(false); } bool equal = true; for (int i = 0; i < idx.TOC.GetLength(0); i++) { // look for our match bool match = false; for (int j = 0; j < _contents.GetLength(0); j++) { if ((idx.TOC[i, 0] == _contents[j, 0]) && (idx.TOC[i, 1] == _contents[j, 1])) { match = true; break; } } equal &= match; } return(equal); }
/// <summary> /// build index for an entire folder /// </summary> /// <param name="folder"></param> public HistSimIndexPlay(string folder, int interval) { _folder = folder+"\\"; string[,] tfi = Util.TickFileIndex(folder, TikConst.WILDCARD_EXT); // see if we have index string fn; HistSimIndex hsi; _extdebug = tfi.GetLength(0) > 500; edebug("checking for index for: "+tfi.GetLength(0)+"-sized simulation."); if (HistSimIndex.HaveIndex(folder,tfi,interval,out fn)) { edebug("index found."); hsi = HistSimIndex.FromFile(fn); hsi.GotDebug += new DebugDelegate(debug); } else if (HistSimIndex.BuildIndex(folder,tfi,out hsi, true,true,interval,debug)) { edebug("Index built successfully. "); } else { debug("Error building index."); return; } myhsi = hsi; _reader.DoWork += new DoWorkEventHandler(_reader_DoWork); _reader.WorkerSupportsCancellation = true; hsip_avail = hsi.Playindex.Length; checkindex(); }
/// <summary> /// gets index for set of tick files /// </summary> /// <param name="tickfiles"></param> /// <param name="hsi"></param> /// <param name="start">whether to build the index or just the TOC</param> /// <returns></returns> public static bool BuildIndex(string folder, string[,] tickfiles, out HistSimIndex hsi, bool start, bool bUseCSV) { return(BuildIndex(folder, tickfiles, out hsi, start, true, 0, null, bUseCSV)); }
public static bool BuildIndex(string folder, string[,] tickfiles, out HistSimIndex hsi, bool start, bool saveindex, int interval, DebugDelegate debug) { hsi = new HistSimIndex(folder,tickfiles,interval); if (debug!=null) hsi.GotDebug+=new DebugDelegate(debug); bool ok = true; if (start) hsi.Start(); if (start && saveindex) { bool save = ToFile(hsi, idxfold(folder), debug); if (debug != null) debug(save ? ("Saved index: " + folder) : "Saving index failed."); ok &= save; } ok &= start ? hsi.isComplete : hsi.TOC.Length > 0; return ok; }
public HistSimMemory(HistSimIndex hsi,DebugDelegate debug) : base(hsi,debug) { }
public HistSimIndexPlay(HistSimIndex hsi, DebugDelegate deb) { _folder = hsi.Folder; if (deb != null) GotDebug += deb; _reader.DoWork += new DoWorkEventHandler(_reader_DoWork); _reader.WorkerSupportsCancellation = true; myhsi = hsi; hsip_avail = hsi.Playindex.Length; checkindex(); }
/// <summary> /// gets index for given set of tick files /// </summary> /// <param name="tickfiles"></param> /// <param name="hsi"></param> /// <returns></returns> public static bool BuildIndex(string folder, string[,] tickfiles, out HistSimIndex hsi) { return BuildIndex(folder, tickfiles, out hsi, true); }
/// <summary> /// gets index from set of tick files and starts indexing /// </summary> /// <param name="tickfiles"></param> /// <param name="tff"></param> /// <param name="hsi"></param> /// <param name="start"></param> /// <returns></returns> public static bool BuildIndex(string folder, string[,] tickfiles, TickFileFilter tff, out HistSimIndex hsi) { return BuildIndex(folder, tickfiles, tff,out hsi,0, null); }
public static bool BuildIndex(string folder, string[,] tickfiles, out HistSimIndex hsi, bool start, bool saveindex) { return BuildIndex(folder, tickfiles, out hsi, start, saveindex, 0,null); }
public HistSimMemory(HistSimIndex hsi, DebugDelegate debug) : base(hsi, debug) { }
/// <summary> /// serialize index /// </summary> /// <param name="hsi"></param> /// <returns></returns> public static string Serialize(HistSimIndex hsi) { return(Serialize(hsi, null)); }
public static bool BuildIndex(string folder, string[,] tickfiles, out HistSimIndex hsi, bool start, bool saveindex) { return(BuildIndex(folder, tickfiles, out hsi, start, saveindex, 0, null)); }
/// <summary> /// gets index for given set of tick files /// </summary> /// <param name="tickfiles"></param> /// <param name="hsi"></param> /// <returns></returns> public static bool BuildIndex(string folder, string[,] tickfiles, out HistSimIndex hsi) { return(BuildIndex(folder, tickfiles, out hsi, true)); }
public static bool BuildIndex(string folder, string[,] tickfiles, TickFileFilter tff, out HistSimIndex hsi, int interval, DebugDelegate debug) { if (debug != null) { debug("filtering tickfiles..."); } string[,] allows = tff.AllowsIndexAndSize(tickfiles); if (debug != null) { debug("using " + allows.Length + " tickfiles post-filter."); } return(BuildIndex(folder, allows, out hsi, true, true, interval, debug)); }
/// <summary> /// gets index from set of tick files and starts indexing /// </summary> /// <param name="tickfiles"></param> /// <param name="tff"></param> /// <param name="hsi"></param> /// <param name="start"></param> /// <returns></returns> public static bool BuildIndex(string folder, string[,] tickfiles, TickFileFilter tff, out HistSimIndex hsi) { return(BuildIndex(folder, tickfiles, tff, out hsi, 0, null)); }
/// <summary> /// serialize idnex as a file /// </summary> /// <param name="hsi"></param> /// <param name="path"></param> /// <returns></returns> public static bool ToFile(HistSimIndex hsi, string path) { return ToFile(hsi, path, null); }
/// <summary> /// get index from a set of tick files given a filter /// </summary> /// <param name="folder"></param> /// <param name="tff"></param> /// <param name="hsi"></param> /// <returns></returns> public static bool BuildIndex(string folder, TickFileFilter tff, out HistSimIndex hsi, bool start) { return(BuildIndex(folder, tff, out hsi, start, true, 0, null)); }
public HistSimMemory(HistSimIndex hsi) : base(hsi, null) { }
public static bool BuildIndex(string folder, TickFileFilter tff, out HistSimIndex hsi,int interval,DebugDelegate debug) { if (debug != null) debug("getting tickfiles available, please wait..."); string[,] allows = Util.TickFileIndex(folder, TikConst.WILDCARD_EXT); if (debug != null) debug("found " + allows.Length + " tickfiles."); return BuildIndex(folder,allows, tff,out hsi, true,true,interval,debug); }
/// <summary> /// create histsimindex player from an index /// </summary> /// <param name="hsi"></param> public HistSimIndexPlay(HistSimIndex hsi) : this(hsi, null) { }
static string indextoc(HistSimIndex hsi) { string data = hsi.Interval == 0 ? string.Empty : hsi.Interval.ToString(); for (int i = 0; i < hsi.TOC.GetLength(0); i++) { data += hsi.TOC[i,0]+"="+hsi.TOC[i,1] + " "; } return data; }
public static bool BuildIndex(string folder, string[,] tickfiles, TickFileFilter tff, out HistSimIndex hsi, int interval, DebugDelegate debug) { if (debug != null) debug("filtering tickfiles..."); string[,] allows = tff.AllowsIndexAndSize(tickfiles); if (debug != null) debug("using " + allows.Length + " tickfiles post-filter."); return BuildIndex(folder,allows, out hsi, true,true,interval,debug); }
/// <summary> /// whether two indicies are equivalent (index same files) /// </summary> /// <param name="idx"></param> /// <returns></returns> public bool Equals(HistSimIndex idx) { if (_contents.Length != idx.TOC.Length) return false; bool equal = true; for (int i = 0; i < idx.TOC.GetLength(0); i++) { // look for our match bool match = false; for (int j = 0; j<_contents.GetLength(0); j++) if ((idx.TOC[i,0] == _contents[j,0]) && (idx.TOC[i,1]==_contents[j,1])) { match = true; break; } equal &= match; } return equal; }
public HistSimMemory(HistSimIndex hsi) : base(hsi,null) { }
/// <summary> /// serialize idnex as a file /// </summary> /// <param name="hsi"></param> /// <param name="path"></param> /// <returns></returns> public static bool ToFile(HistSimIndex hsi, string path) { return(ToFile(hsi, path, null)); }
/// <summary> /// serialize index /// </summary> /// <param name="hsi"></param> /// <returns></returns> public static string Serialize(HistSimIndex hsi) { return Serialize(hsi, null); }
public HistSimIndexPlay(string folder, TickFileFilter tff,int interval,DebugDelegate deb) { if (deb!=null) GotDebug+=deb; _folder = folder + "\\"; // see if we have index string fn; HistSimIndex hsi; debug("getting tickfiles, please wait..."); string[,] total = Util.TickFileIndex(folder, TikConst.WILDCARD_EXT,false); debug("found " + total.GetLength(0).ToString("N0") + " tickfiles"); string[,] filtered = tff.AllowsIndexAndSize(total); debug("post-filter: " + filtered.GetLength(0).ToString("N0")+", checking for index..."); if (HistSimIndex.HaveIndex(folder, filtered, out fn,deb)) { debug("index found, starting load: "+fn); hsi = HistSimIndex.FromFile(fn,debug); if (hsi == null) { debug("unable to load index."); return; } hsi.GotDebug+=new DebugDelegate(debug); } else if (HistSimIndex.BuildIndex(folder, filtered, out hsi, true,true,interval,debug)) { debug("Index built successfully, ready to play."); } else { debug("Error building index..."); return; } _reader.DoWork += new DoWorkEventHandler(_reader_DoWork); _reader.WorkerSupportsCancellation = true; myhsi = hsi; hsip_avail = hsi.Playindex.Length; checkindex(); }
public static bool ToFile(HistSimIndex hsi, string path,DebugDelegate debug) { try { // get checksum of data string md5 = MD5(indextoc(hsi)); // see if it's complete if (!hsi.isComplete) { if (debug!=null) debug("Index not completed "+md5); return false; } // get index directory string dir = Path.GetDirectoryName(path); if (!Directory.Exists(dir)) Directory.CreateDirectory(dir); // get filepath string filepath = dir + "\\" + md5 + ".txt"; // pack contents so serializable hsi.packTOC(); XmlSerializer xs = new XmlSerializer(typeof(HistSimIndex)); Stream fs = new FileStream(filepath, FileMode.Create); System.IO.Compression.GZipStream gs = new System.IO.Compression.GZipStream(fs, System.IO.Compression.CompressionMode.Compress); xs.Serialize(gs, hsi); gs.Close(); fs.Close(); } catch (Exception ex) { if (debug != null) debug(ex.Message + ex.StackTrace); return false; } return true; }
public HistSimIndexPlay(string[] dataset,string folder, int interval, DebugDelegate deb) { if (deb != null) GotDebug += deb; if (dataset.Length < 1) { debug("Empty data set, quitting..."); return; } _folder = folder; // get index of set string[,] tfi = GetTickIndex(dataset,false,folder); _extdebug = tfi.GetLength(0) > 500; // see if we have index string fn; HistSimIndex hsi; edebug("checking for index"); if (HistSimIndex.HaveIndex(folder, tfi,interval, out fn)) { edebug("index found."); hsi = HistSimIndex.FromFile(fn); hsi.GotDebug += new DebugDelegate(debug); } else if (HistSimIndex.BuildIndex(folder, tfi, out hsi, true, true,interval, debug)) { edebug("Index built successfully"); } else { debug("Error building index..."); return; } myhsi = hsi; _reader.DoWork += new DoWorkEventHandler(_reader_DoWork); _reader.WorkerSupportsCancellation = true; hsip_avail = hsi.Playindex.Length; checkindex(); }
/// <summary> /// get index from a set of tick files given a filter and starts indexing /// </summary> /// <param name="folder"></param> /// <param name="tff"></param> /// <param name="hsi"></param> /// <returns></returns> public static bool BuildIndex(string folder, TickFileFilter tff, out HistSimIndex hsi) { return BuildIndex(folder, tff, out hsi, true, true, 0,null); }