/// <summary>
/// create file from ticks
/// </summary>
/// <param name="ticks"></param>
/// <param name="debs"></param>
/// <returns></returns>
public static bool TicksToFile(Tick[] ticks, DebugDelegate debs)
{
try
{
TikWriter tw = new TikWriter();
foreach (Tick k in ticks)
{
tw.newTick(k);
}
tw.Close();
if (debs != null)
{
debs(tw.RealSymbol + " saved " + tw.Count + " ticks to: " + tw.Filepath);
}
}
catch (Exception ex)
{
if (debs != null)
{
debs(ex.Message + ex.StackTrace);
}
return(false);
}
return(true);
}
public bool newTick(Tick t)
{
if ((t.symbol==null) || (t.symbol=="")) return false;
TikWriter tw;
if (filedict.TryGetValue(t.symbol, out tw))
{
try
{
tw.newTick(t);
}
catch (IOException) { return false; }
}
else
{
try
{
tw = new TikWriter(_path, t.symbol, t.date);
tw.newTick(t);
filedict.Add(t.symbol, tw);
}
catch (IOException) { return false; }
catch (Exception) { return false; }
}
return true;
}
public bool newTick(Tick t)
{
if ((t.symbol == null) || (t.symbol == ""))
{
return(false);
}
TikWriter tw;
if (filedict.TryGetValue(t.symbol, out tw))
{
try
{
tw.newTick(t);
}
catch (IOException) { return(false); }
}
else
{
try
{
tw = new TikWriter(_path, t.symbol, t.date);
tw.newTick(t);
filedict.Add(t.symbol, tw);
}
catch (IOException) { return(false); }
catch (Exception) { return(false); }
}
return(true);
}
/// <summary>
/// converts EPF files to tradelink tick files in current directory
/// </summary>
/// <param name="args"></param>
public static void EPF2TIK(string[] args)
{
// get a list of epf files
foreach (string file in args)
{
SecurityImpl sec = SecurityImpl.FromTIK(file);
sec.HistSource.gotTick += new TradeLink.API.TickDelegate(HistSource_gotTick);
_tw = new TikWriter(sec.Symbol);
while (sec.NextTick())
_tw.Close();
}
}
/// <summary>
/// converts EPF files to tradelink tick files in current directory
/// </summary>
/// <param name="args"></param>
public static void EPF2TIK(string[] args)
{
// get a list of epf files
foreach (string file in args)
{
SecurityImpl sec = SecurityImpl.FromTIK(file);
sec.HistSource.gotTick += new TradeLink.API.TickDelegate(HistSource_gotTick);
_tw = new TikWriter(sec.Symbol);
while (sec.NextTick())
{
_tw.Close();
}
}
}
/// <summary>
/// write header for tick file
/// </summary>
/// <param name="bw"></param>
/// <param name="realsymbol"></param>
/// <returns></returns>
public static bool Header(TikWriter bw, string realsymbol)
{
bw.OutStream = new FileStream(bw.Filepath, FileMode.Create, FileAccess.Write, FileShare.Read);
// version
bw.Write(TikConst.Version);
bw.Write(TikConst.FILECURRENTVERSION);
// full symbol name
bw.Write(realsymbol); //
// fields end
bw.Write(TikConst.StartData);
// flag header as created
bw._hasheader = true;
return(true);
}
public static bool TicksToFile(Tick[] ticks, DebugDelegate debs)
{
try
{
TikWriter tw = new TikWriter();
foreach (Tick k in ticks)
tw.newTick(k);
tw.Close();
if (debs != null)
debs(tw.RealSymbol + " saved " + tw.Count + " ticks to: " + tw.Filepath);
}
catch (Exception ex)
{
if (debs != null)
debs(ex.Message + ex.StackTrace);
return false;
}
return true;
}
bool convert(Converter con, string filename,int tradesize,string sym)
{
int bads = 0;
int thistotal = _ticksprocessed;
bool g = true;
// get output filename
string convertname = string.Empty;
// setup writing to output
TikWriter outfile = null;
// setup input file
StreamReader infile = null;
int _date = 0;
int cqgdecimalplaces = 2;
try
{
// open input file
switch (con)
{
case Converter.TradeStation:
infile = new StreamReader(filename);
// read in and ignore header of input file
infile.ReadLine();
break;
case Converter.eSignal_EPF:
infile = new StreamReader(filename);
// ignore header
SecurityImpl esec = eSigTick.InitEpf(infile);
_sym = esec.Symbol;
break;
case Converter.CQG:
infile = new StreamReader(filename);
cqgdecimalplaces = (int) this._cqgdecimalplacesinput.Value;
// no header
break;
case Converter.TradingPhysicsTnS:
case Converter.TradingPhysicsTV:
string file = System.IO.Path.GetFileName(filename);
string[] date_sym = file.Split('_');
string[] sym_ext = date_sym[1].Split('.');
string datestr = date_sym[0];
int.TryParse(datestr, out _date);
_sym = sym_ext[0];
infile = new StreamReader(filename);
infile.ReadLine();//discard header line
break;
case Converter.QCollector_eSignal:
infile = new StreamReader(filename);
// no header in file
break;
case Converter.MultiCharts:
// The symbol for data being imported is obtained from the filename
// Selected files for import must be named SYMBOL.ext - eg AAPL.txt, GOOG.txt
_sym = System.IO.Path.GetFileNameWithoutExtension(filename);
infile = new StreamReader(filename);
infile.ReadLine(); // ignore first line header of input file
break;
}
}
catch (Exception ex) { debug("error reading input header:" + ex.Message); g = false; }
// setup previous tick and current tick
Tick pk = new TickImpl();
Tick k = null;
do
{
try
{
// get next tick from the file
switch (con)
{
case Converter.CQG:
k = CQG.parseline(infile.ReadLine(), tradesize, cqgdecimalplaces );
break;
case Converter.eSignal_EPF:
k = eSigTick.FromStream(_sym, infile);
break;
case Converter.TradeStation:
k = TradeStation.parseline(infile.ReadLine(), sym, tradesize);
break;
case Converter.TradingPhysicsTnS:
k = TradingPhysicsTnS.parseline(infile.ReadLine(), _sym, _date);
break;
case Converter.TradingPhysicsTV:
k = TradingPhysicsTV.parseline(infile.ReadLine(), _sym, _date);
break;
case Converter.QCollector_eSignal:
k = QCollector.parseline(infile.ReadLine(), sym);
break;
case Converter.MultiCharts:
k = MultiCharts.parseline(infile.ReadLine(), _sym);
break;
}
}
catch (Exception ex) { bads++; continue; }
if (k == null)
{
debug("Invalid converter: " + con.ToString());
return false;
}
// bad tick
if (k.date == 0) { bads++; continue; }
// if dates don't match, we need to write new output file
if (k.date != pk.date)
{
try
{
// if the outfile was open previously, close it
if (outfile != null)
outfile.Close();
// get path from input
string path = Path.GetDirectoryName(filename) + "\\";
// setup new file
outfile = new TikWriter(path,k.symbol, k.date);
// report progress
progress((double)_ticksprocessed / _approxtotal);
}
catch (Exception ex) { debug(ex.Message); g = false; }
}
try
{
// write the tick
outfile.newTick(k);
// save this tick as previous tick
pk = k;
// count the tick as processed
_ticksprocessed++;
}
catch (Exception ex) { debug("error writing output tick: " + ex.Message); g = false; }
}
// keep going until input file is exhausted
while (!infile.EndOfStream);
// close output file
if (outfile == null)
{
debug("Tick file was never opened, likely that input file in wrong format.");
}
else
{
debug("Saved: " + outfile.Filepath);
outfile.Close();
}
// close input file
infile.Close();
// get percentage of good ticks
double goodratio = (_ticksprocessed - thistotal - bads) / (_ticksprocessed - (double)thistotal);
if (goodratio < GOODRATIO)
{
debug("Less than " + GOODRATIO.ToString("P0") + " good ticks");
g = false;
}
// return status
return g;
}
public static bool ChartSave(BarList bl, string path, int date)
{
try
{
Directory.CreateDirectory(path);
}
catch { return false; }
try
{
if (date == 0)
date = bl.RecentBar.Bardate;
string fn = TikWriter.SafeFilename(bl.Symbol, path, date);
if (System.IO.File.Exists(fn))
{
try { System.IO.File.Delete(fn); }
catch { return false; }
}
TikWriter tw = new TikWriter(path, bl.Symbol, date);
foreach (Bar b in bl)
{
Tick[] ticks = BarImpl.ToTick(b);
foreach (Tick k in ticks)
tw.newTick(k);
}
tw.Close();
return true;
}
catch { }
return false;
}
/// <summary>
/// build the index
/// </summary>
public void Start()
{
const int RATE = 30000;
int esttick = estsize() / 40;
double est = esttick/ RATE;
TimeSpan ts = new TimeSpan(0, 0, (int)est);
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
sw.Start();
debug("Starting creation of index... estimated: "+ts.ToString());
// start the workers
for (int i = 0; i < TOC.GetLength(0); i++)
{
// try provided path
string fn = TOC[i, 0];
// ensure file exists
if (File.Exists(fn))
readers.Add(new tickreader(i, fn, _binaryindex, Interval));
else
{
// try default path
fn = Folder + "\\" + Path.GetFileName(TOC[i, 0]);
if (File.Exists(fn))
readers.Add(new tickreader(i, fn, _binaryindex, Interval));
else
debug(fn + " not found.");
}
}
debug("Started "+readers.Count+" readers, waiting for completition");
// wait till they're done
_running = true;
while (_running)
{
System.Threading.Thread.Sleep(SLEEP);
bool busy = false;
for (int i = 0; i < readers.Count; i++)
busy |= readers[i].IsBusy;
_running = busy;
}
debug("Readers completed, assembling master index.");
_running = false;
// estimate size
int size = 0;
bool finished = true;
foreach (tickreader tr in readers)
{
finished &= tr.finished;
size += tr.count;
}
// append to single arrays
List<long> times = new List<long>(size);
List<int> index = new List<int>(size);
foreach (tickreader tr in readers)
{
times.AddRange(tr.datetimes);
index.AddRange(tr.index);
}
long[] ltimes = times.ToArray();
int[] iindex = index.ToArray();
// sort master array
Array.Sort(ltimes, iindex);
// save play index
_idx = iindex;
// save times
_times = ltimes;
// see if we're saving binary index
if (_binaryindex)
{
// fake symbol name
string sym = MD5(indextoc(this));
// ensure binary index folder exists
if (!Directory.Exists(IndexFolder))
Directory.CreateDirectory(IndexFolder);
// create file
TikWriter tw = new TikWriter(IndexFolder, sym, 0);
// get a filename
_binaryidx = tw.Filepath;
debug("Creating binary index: "+BinaryIndex);
// master tick list
Tick[] ktiks = new Tick[times.Count];
ltimes = times.ToArray();
int next = 0;
for (int i = 0; i < readers.Count; i++)
{
int len = readers[i].ticks.Count;
readers[i].ticks.CopyTo(0, ktiks, next, len);
next += len;
readers[i].ticks.Clear();
}
// sort ticks
Array.Sort(ltimes, ktiks);
// save ticks to binary file
int count = 0;
for (int i = 0; i < iindex.Length; i++)
{
// get tick
Tick k = ktiks[count++];
// save
tw.newTick(k);
}
tw.Close();
debug("binary index completed containing " + count + " ticks.");
}
// save finish state
_finished = finished;
// clean up readers
readers.Clear();
sw.Stop();
double actual = sw.Elapsed.TotalSeconds;
double rate = actual == 0 ? 0 : ltimes.Length / actual;
debug("Master index complete. actual: "+sw.Elapsed.ToString()+"("+rate.ToString("N0")+" ticks/sec)");
}
double writeandread(Tick[] data, int date, bool printperf)
{
// clear out the read buffer
readdata.Clear();
// keep track of time
double elapms;
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
// write new file from data
TikWriter tw = new TikWriter(PATH,data[0].symbol,date==0 ? data[0].date: date);
string FILE = tw.Filepath;
sw.Start();
foreach (Tick k in data)
tw.newTick(k);
sw.Stop();
tw.Close();
elapms = (double)sw.ElapsedMilliseconds;
if (printperf)
Console.WriteLine("write speed (ticks/sec): " + (data.Length / (elapms / 1000)).ToString("n0"));
// read file back in from file
TikReader tr = new TikReader(FILE);
tr.gotTick += new TickDelegate(tr_gotTick);
sw.Reset();
sw.Start();
while (tr.NextTick()) ;
sw.Stop();
tr.Close();
elapms = (double)sw.ElapsedMilliseconds;
if (printperf)
Console.WriteLine("read speed (ticks/sec): " + (data.Length/(elapms/1000)).ToString("n0"));
// remove file
removefile(FILE);
return elapms;
}
bool convert(string filename,int tradesize)
{
bool g = true;
// get output filename
string convertname = string.Empty;
// setup writing to output
TikWriter outfile = null;
// setup input file
StreamReader infile = null;
try
{
// open input file
infile = new StreamReader(filename);
}
catch (Exception ex) { debug("error reading input header:" + ex.Message); g = false; }
// setup previous tick
TickImpl pk = new TickImpl();
do
{
// get next tick from the file
TickImpl k = parseline(infile.ReadLine(), tradesize);
// if dates don't match, we need to write new output file
if (k.date != pk.date)
{
// if the outfile was open previously, close it
if (outfile != null) outfile.Close();
try
{
// get file name
string fn = _path+"//"+k.symbol + k.date + ".EPF";
// setup new file
outfile = new TikWriter(k.symbol, k.date);
// report progress
progress((double)_ticksprocessed / _approxtotal);
}
catch (Exception ex) { debug(ex.Message); g = false; }
}
try
{
// write the tick
outfile.newTick(k);
}
catch (Exception ex) { debug("error writing output tick: " + ex.Message); g = false; }
// save this tick as previous tick
pk = k;
// count the tick as processed
_ticksprocessed++;
}
// keep going until input file is exhausted
while (!infile.EndOfStream);
// close output file
outfile.Close();
// return status
return g;
}
bool CSV_DTOHLCV_convert(string filename, int tradesize, string sym)
{
int bads = 0;
int thistotal = _ticksprocessed;
bool g = true;
// get output filename
string convertname = string.Empty;
// setup writing to output
TikWriter outfile = null;
// setup input file
StreamReader infile = null;
try
{
// open input file
infile = new StreamReader(filename);
// read in and ignore header of input file
infile.ReadLine();
}
catch (Exception ex) { debug("error reading input header:" + ex.Message); g = false; }
// setup previous tick and current tick
Tick[] pk = { new TickImpl(), new TickImpl(), new TickImpl(), new TickImpl() };
Tick[] k = null; // this is a tick array instead of a simple tick
do
{
try
{
// get next tick from the file
k = CSV.parseline(infile.ReadLine(), sym);
}
catch (Exception ex)
{
debug("error reading line of: " + filename + " err: " + ex.Message + ex.StackTrace);
bads++;
continue;
}
if (k == null)
{
debug("Invalid converter: " + Converter.CSV_DTOHLCV.ToString());
return false;
}
// bad tick
if (k[0].date == 0) { bads++; continue; }
// if dates don't match, we need to write new output file; only checking the opening ticks' dates
if (k[0].date != pk[0].date)
{
try
{
// if the outfile was open previously, close it
if (outfile != null)
outfile.Close();
// get path from input
string path = Path.GetDirectoryName(filename) + "\\";
// setup new file
outfile = new TikWriter(path, k[0].symbol, k[0].date);
// report progress
progress((double)_ticksprocessed / _approxtotal);
}
catch (Exception ex) { debug(ex.Message); g = false; }
}
try
{
// write the tick
foreach (Tick t in k)
outfile.newTick(t);
// save this tick array as previous tick array
pk = k;
// count the tick as processed
_ticksprocessed += 4;
}
catch (Exception ex) { debug("error writing output tick: " + ex.Message); g = false; }
}
// keep going until input file is exhausted
while (!infile.EndOfStream);
// close output file
if (outfile == null)
{
debug("Tick file was never opened, likely that input file in wrong format.");
}
else
{
debug("Saved: " + outfile.Filepath);
outfile.Close();
}
// close input file
infile.Close();
// get percentage of good ticks
double goodratio = (_ticksprocessed - thistotal - bads) / (_ticksprocessed - (double)thistotal);
if (goodratio < GOODRATIO)
{
debug("Less than " + GOODRATIO.ToString("P0") + " good ticks");
g = false;
}
// return status
return g;
}
/// <summary>
/// build the index
/// </summary>
public void Start()
{
const int RATE = 30000;
int esttick = estsize() / 40;
double est = esttick / RATE;
TimeSpan ts = new TimeSpan(0, 0, (int)est);
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
sw.Start();
debug("Starting creation of index... estimated: " + ts.ToString());
// start the workers
for (int i = 0; i < TOC.GetLength(0); i++)
{
// try provided path
string fn = TOC[i, 0];
// ensure file exists
if (File.Exists(fn))
{
readers.Add(new tickreader(i, fn, _binaryindex, Interval));
}
else
{
// try default path
fn = Folder + "\\" + Path.GetFileName(TOC[i, 0]);
if (File.Exists(fn))
{
readers.Add(new tickreader(i, fn, _binaryindex, Interval));
}
else
{
debug(fn + " not found.");
}
}
}
debug("Started " + readers.Count + " readers, waiting for completition");
// wait till they're done
_running = true;
while (_running)
{
System.Threading.Thread.Sleep(SLEEP);
bool busy = false;
for (int i = 0; i < readers.Count; i++)
{
busy |= readers[i].IsBusy;
}
_running = busy;
}
debug("Readers completed, assembling master index.");
_running = false;
// estimate size
int size = 0;
bool finished = true;
foreach (tickreader tr in readers)
{
finished &= tr.finished;
size += tr.count;
}
// append to single arrays
List <long> times = new List <long>(size);
List <int> index = new List <int>(size);
foreach (tickreader tr in readers)
{
times.AddRange(tr.datetimes);
index.AddRange(tr.index);
}
long[] ltimes = times.ToArray();
int[] iindex = index.ToArray();
// sort master array
Array.Sort(ltimes, iindex);
// save play index
_idx = iindex;
// save times
_times = ltimes;
// see if we're saving binary index
if (_binaryindex)
{
// fake symbol name
string sym = MD5(indextoc(this));
// ensure binary index folder exists
if (!Directory.Exists(IndexFolder))
{
Directory.CreateDirectory(IndexFolder);
}
// create file
TikWriter tw = new TikWriter(IndexFolder, sym, 0);
// get a filename
_binaryidx = tw.Filepath;
debug("Creating binary index: " + BinaryIndex);
// master tick list
Tick[] ktiks = new Tick[times.Count];
ltimes = times.ToArray();
int next = 0;
for (int i = 0; i < readers.Count; i++)
{
int len = readers[i].ticks.Count;
readers[i].ticks.CopyTo(0, ktiks, next, len);
next += len;
readers[i].ticks.Clear();
}
// sort ticks
Array.Sort(ltimes, ktiks);
// save ticks to binary file
int count = 0;
for (int i = 0; i < iindex.Length; i++)
{
// get tick
Tick k = ktiks[count++];
// save
tw.newTick(k);
}
tw.Close();
debug("binary index completed containing " + count + " ticks.");
}
// save finish state
_finished = finished;
// clean up readers
readers.Clear();
sw.Stop();
double actual = sw.Elapsed.TotalSeconds;
double rate = actual == 0 ? 0 : ltimes.Length / actual;
debug("Master index complete. actual: " + sw.Elapsed.ToString() + "(" + rate.ToString("N0") + " ticks/sec)");
}
bool convert(Converter con, string filename,int tradesize)
{
int bads = 0;
int thistotal = _ticksprocessed;
bool g = true;
// get output filename
string convertname = string.Empty;
// setup writing to output
TikWriter outfile = null;
// setup input file
StreamReader infile = null;
try
{
// open input file
switch (con)
{
case Converter.TradeStation:
infile = new StreamReader(filename);
// read in and ignore header of input file
infile.ReadLine();
break;
case Converter.eSignal_EPF:
infile = new StreamReader(filename);
// ignore header
SecurityImpl esec = eSigTick.InitEpf(infile);
_sym = esec.Symbol;
break;
case Converter.CQG:
infile = new StreamReader(filename);
// no header
break;
}
}
catch (Exception ex) { debug("error reading input header:" + ex.Message); g = false; }
// setup previous tick and current tick
Tick pk = new TickImpl();
Tick k = null;
do
{
try
{
// get next tick from the file
switch (con)
{
case Converter.CQG:
k = CQG.parseline(infile.ReadLine(), tradesize);
break;
case Converter.eSignal_EPF:
k = eSigTick.FromStream(_sym, infile);
break;
case Converter.TradeStation:
k = TradeStation.parseline(infile.ReadLine(), _sym, tradesize);
break;
}
}
catch (Exception ex) { bads++; continue; }
if (k == null)
{
debug("Invalid converter: " + con.ToString());
return false;
}
// bad tick
if (k.date == 0) { bads++; continue; }
// if dates don't match, we need to write new output file
if (k.date != pk.date)
{
try
{
// if the outfile was open previously, close it
if (outfile != null)
outfile.Close();
// get path from input
string path = Path.GetDirectoryName(filename) + "\\";
// setup new file
outfile = new TikWriter(path,k.symbol, k.date);
// report progress
progress((double)_ticksprocessed / _approxtotal);
}
catch (Exception ex) { debug(ex.Message); g = false; }
}
try
{
// write the tick
outfile.newTick(k);
// save this tick as previous tick
pk = k;
// count the tick as processed
_ticksprocessed++;
}
catch (Exception ex) { debug("error writing output tick: " + ex.Message); g = false; }
}
// keep going until input file is exhausted
while (!infile.EndOfStream);
// close output file
outfile.Close();
// close input file
infile.Close();
// get percentage of good ticks
double goodratio = (_ticksprocessed - thistotal - bads) / (_ticksprocessed - (double)thistotal);
if (goodratio < GOODRATIO)
{
debug("Less than " + GOODRATIO.ToString("P0") + " good ticks");
g = false;
}
// return status
return g;
}
public static bool Header(TikWriter bw, string realsymbol)
{
bw.OutStream = new FileStream(bw.Filepath, FileMode.Create, FileAccess.Write, FileShare.Read);
// version
bw.Write(TikConst.Version);
bw.Write(TikConst.VERSION);
// full symbol name
bw.Write(realsymbol); //
// fields end
bw.Write(TikConst.StartData);
// flag header as created
bw._hasheader = true;
return true;
}
public bool newTick(Tick t)
{
if (_stopped)
{
return(false);
}
if ((t.symbol == null) || (t.symbol == ""))
{
return(false);
}
TikWriter tw;
// prepare last date of tick
int lastdate = 0;
// get last date
bool havedate = datedict.TryGetValue(t.symbol, out lastdate);
// if we don't have date, use present date
if (!havedate)
{
lastdate = t.date;
datedict.Add(t.symbol, t.date);
}
// see if we need a new day
bool samedate = lastdate == t.date;
// see if we have stream already
bool havestream = filedict.TryGetValue(t.symbol, out tw);
// if no changes, just save tick
if (samedate && havestream)
{
try
{
tw.newTick((TickImpl)t);
return(true);
}
catch (IOException) { return(false); }
}
else
{
try
{
// if new date, close stream
if (!samedate)
{
try
{
tw.Close();
}
catch (IOException) { }
}
// ensure file is writable
string fn = TikWriter.SafeFilename(t.symbol, _path, t.date);
if (TikUtil.IsFileWritetable(fn))
{
// open new stream
tw = new TikWriter(_path, t.symbol, t.date);
// save tick
tw.newTick((TickImpl)t);
// save stream
if (!havestream)
{
filedict.Add(t.symbol, tw);
}
else
{
filedict[t.symbol] = tw;
}
// save date if changed
if (!samedate)
{
datedict[t.symbol] = t.date;
}
}
}
catch (IOException) { return(false); }
catch (Exception) { return(false); }
}
return(false);
}