bool convert(string filename, int tradesize)
{
bool g = true;
// get output filename
string convertname = string.Empty;
// setup writing to output
TikWriter outfile = null;
// setup input file
StreamReader infile = null;
try
{
// open input file
infile = new StreamReader(filename);
}
catch (Exception ex) { debug("error reading input header:" + ex.Message); g = false; }
// setup previous tick
TickImpl pk = new TickImpl();
do
{
// get next tick from the file
TickImpl k = parseline(infile.ReadLine(), tradesize);
// if dates don't match, we need to write new output file
if (k.date != pk.date)
{
// if the outfile was open previously, close it
if (outfile != null)
{
outfile.Close();
}
try
{
// get file name
string fn = _path + "//" + k.symbol + k.date + ".EPF";
// setup new file
outfile = new TikWriter(k.symbol, k.date);
// report progress
progress((double)_ticksprocessed / _approxtotal);
}
catch (Exception ex) { debug(ex.Message); g = false; }
}
try
{
// write the tick
outfile.newTick(k);
}
catch (Exception ex) { debug("error writing output tick: " + ex.Message); g = false; }
// save this tick as previous tick
pk = k;
// count the tick as processed
_ticksprocessed++;
}
// keep going until input file is exhausted
while (!infile.EndOfStream);
// close output file
outfile.Close();
// return status
return(g);
}
double writeandread(Tick[] data, int date, bool printperf, bool clearreadbuffer)
{
// clear out the read buffer
if (clearreadbuffer)
{
readdata.Clear();
}
// keep track of time
double elapms;
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
// write new file from data
TikWriter tw = new TikWriter(PATH, data[0].symbol, date == 0 ? data[0].date: date);
string FILE = tw.Filepath;
sw.Start();
foreach (Tick k in data)
{
tw.newTick(k);
}
sw.Stop();
tw.Close();
elapms = (double)sw.ElapsedMilliseconds;
if (printperf)
{
Console.WriteLine("write speed (ticks/sec): " + (data.Length / (elapms / 1000)).ToString("n0"));
}
// read file back in from file
TikReader tr = new TikReader(FILE);
tr.gotTick += new TickDelegate(tr_gotTick);
sw.Reset();
sw.Start();
while (tr.NextTick())
{
;
}
sw.Stop();
tr.Close();
elapms = (double)sw.ElapsedMilliseconds;
if (printperf)
{
Console.WriteLine("read speed (ticks/sec): " + (data.Length / (elapms / 1000)).ToString("n0"));
}
// remove file
removefile(FILE);
return(elapms);
}
public void CreateRead()
{
readdata.Clear();
readdata2.Clear();
FILE = TikWriter.SafeFilename(SYM, PATH, DATE);
TestTikWriterReader.removefile(FILE);
{
Tick[] data = RandomTicks.GenerateSymbol(SYM, MAXTICKS);
TickArchiver ta = new TickArchiver(Environment.CurrentDirectory);
for (int i = 0; i < data.Length; i++)
{
data[i].date = DATE;
data[i].time = Util.DT2FT(DateTime.Now);
ta.newTick(data[i]);
}
ta.Stop();
// read file back in from file
TikReader tr = new TikReader(FILE);
tr.gotTick += new TickDelegate(tr_gotTick);
while (tr.NextTick())
{
;
}
// verify length
Assert.AreEqual(data.Length, readdata.Count);
// verify content
bool equal = true;
for (int i = 0; i < MAXTICKS; i++)
{
equal &= data[i].trade == readdata[i].trade;
}
tr.Close();
readdata.Clear();
Assert.IsTrue(equal, "ticks did not matched archive.");
TestTikWriterReader.removefile(FILE);
}
}
public void CreateRead()
{
_readdata.Clear();
_readdata2.Clear();
_file = TikWriter.SafeFilename(Sym, _path, Date);
TestTikWriterReader.Removefile(_file);
{
TickImpl[] data = RandomTicks.GenerateSymbol(Sym, Maxticks).Select(x => (TickImpl)x).ToArray();
TickArchiver ta = new TickArchiver(Environment.CurrentDirectory);
for (int i = 0; i < data.Length; i++)
{
data[i].Date = Date;
data[i].Time = Util.DT2FT(DateTime.Now);
ta.NewTick(data[i]);
}
ta.Stop();
// read file back in from file
TikReader tr = new TikReader(_file);
tr.GotTick += tr_gotTick;
while (tr.NextTick())
{
}
// verify length
Assert.Equal(data.Length, _readdata.Count);
// verify content
bool equal = true;
for (int i = 0; i < Maxticks; i++)
{
equal &= data[i].Trade == _readdata[i].Trade;
}
tr.Close();
_readdata.Clear();
Assert.True(equal, "ticks did not matched archive.");
TestTikWriterReader.Removefile(_file);
}
}
bool convert(Converter con, string filename, int tradesize, string sym)
{
int bads = 0;
int thistotal = _ticksprocessed;
bool g = true;
// get output filename
string convertname = string.Empty;
// setup writing to output
TikWriter outfile = null;
// setup input file
StreamReader infile = null;
int _date = 0;
int cqgdecimalplaces = 2;
try
{
// open input file
switch (con)
{
case Converter.TradeStation:
infile = new StreamReader(filename);
// read in and ignore header of input file
infile.ReadLine();
break;
case Converter.eSignal_EPF:
infile = new StreamReader(filename);
// ignore header
SecurityImpl esec = eSigTick.InitEpf(infile);
_sym = esec.Symbol;
break;
case Converter.CQG:
infile = new StreamReader(filename);
cqgdecimalplaces = (int)this._cqgdecimalplacesinput.Value;
// no header
break;
case Converter.TradingPhysicsTnS:
case Converter.TradingPhysicsTV:
string file = System.IO.Path.GetFileName(filename);
string[] date_sym = file.Split('_');
string[] sym_ext = date_sym[1].Split('.');
string datestr = date_sym[0];
int.TryParse(datestr, out _date);
_sym = sym_ext[0];
infile = new StreamReader(filename);
infile.ReadLine(); //discard header line
break;
case Converter.QCollector_eSignal:
infile = new StreamReader(filename);
// no header in file
break;
case Converter.MultiCharts:
// The symbol for data being imported is obtained from the filename
// Selected files for import must be named SYMBOL.ext - eg AAPL.txt, GOOG.txt
_sym = System.IO.Path.GetFileNameWithoutExtension(filename);
infile = new StreamReader(filename);
infile.ReadLine(); // ignore first line header of input file
break;
}
}
catch (Exception ex) { debug("error reading input header:" + ex.Message); g = false; }
// setup previous tick and current tick
Tick pk = new TickImpl();
Tick k = null;
do
{
try
{
// get next tick from the file
switch (con)
{
case Converter.CQG:
k = CQG.parseline(infile.ReadLine(), tradesize, cqgdecimalplaces);
break;
case Converter.eSignal_EPF:
k = eSigTick.FromStream(_sym, infile);
break;
case Converter.TradeStation:
k = TradeStation.parseline(infile.ReadLine(), sym);
break;
case Converter.TradingPhysicsTnS:
k = TradingPhysicsTnS.parseline(infile.ReadLine(), _sym, _date);
break;
case Converter.TradingPhysicsTV:
k = TradingPhysicsTV.parseline(infile.ReadLine(), _sym, _date);
break;
case Converter.QCollector_eSignal:
k = QCollector.parseline(infile.ReadLine(), sym);
break;
case Converter.MultiCharts:
k = MultiCharts.parseline(infile.ReadLine(), _sym);
break;
}
}
catch (Exception ex) { bads++; continue; }
if (k == null)
{
debug("Invalid converter: " + con.ToString());
return(false);
}
// bad tick
if (k.date == 0)
{
bads++; continue;
}
// if dates don't match, we need to write new output file
if (k.date != pk.date)
{
try
{
// if the outfile was open previously, close it
if (outfile != null)
{
outfile.Close();
}
// get path from input
string path = Path.GetDirectoryName(filename) + "\\";
// setup new file
outfile = new TikWriter(path, k.symbol, k.date);
// report progress
progress((double)_ticksprocessed / _approxtotal);
}
catch (Exception ex) { debug(ex.Message); g = false; }
}
try
{
// write the tick
outfile.newTick(k);
// save this tick as previous tick
pk = k;
// count the tick as processed
_ticksprocessed++;
}
catch (Exception ex) { debug("error writing output tick: " + ex.Message); g = false; }
}
// keep going until input file is exhausted
while (!infile.EndOfStream);
// close output file
if (outfile == null)
{
debug("Tick file was never opened, likely that input file in wrong format.");
}
else
{
debug("Saved: " + outfile.Filepath);
outfile.Close();
}
// close input file
infile.Close();
// get percentage of good ticks
double goodratio = (_ticksprocessed - thistotal - bads) / (_ticksprocessed - (double)thistotal);
if (goodratio < GOODRATIO)
{
debug("Less than " + GOODRATIO.ToString("P0") + " good ticks");
g = false;
}
// return status
return(g);
}
public void Multiday()
{
readdata.Clear();
readdata2.Clear();
int d = 20100223;
int t = 235900;
int t1 = 0;
const decimal p = 50;
int s = 100;
string FILE1 = TikWriter.SafeFilename(SYM, PATH, d);
TestTikWriterReader.removefile(FILE1);
string FILE2 = TikWriter.SafeFilename(SYM, PATH, d + 1);
TestTikWriterReader.removefile(FILE2);
Tick[] data = new Tick[]
{
TickImpl.NewTrade(SYM, d, t++, p, s, string.Empty),
TickImpl.NewTrade(SYM, d, t++, p, s, string.Empty),
TickImpl.NewTrade(SYM, d, t++, p, s, string.Empty),
TickImpl.NewTrade(SYM, d, t++, p, s, string.Empty),
TickImpl.NewTrade(SYM, d, t++, p, s, string.Empty),
// day two
TickImpl.NewTrade(SYM, ++d, t1++, p, s, string.Empty),
TickImpl.NewTrade(SYM, d, t1++, p, s, string.Empty),
TickImpl.NewTrade(SYM, d, t1++, p, s, string.Empty),
TickImpl.NewTrade(SYM, d, t1++, p, s, string.Empty),
TickImpl.NewTrade(SYM, d, t1++, p, s, string.Empty),
};
TickArchiver ta = new TickArchiver(Environment.CurrentDirectory);
for (int i = 0; i < data.Length; i++)
{
ta.newTick(data[i]);
}
ta.Stop();
// read file back in from files
if (System.IO.File.Exists(FILE1))
{
TikReader tr = new TikReader(FILE1);
tr.gotTick += new TickDelegate(tr_gotTick);
while (tr.NextTick())
{
;
}
tr.Close();
}
if (System.IO.File.Exists(FILE2))
{
TikReader tr2 = new TikReader(FILE2);
tr2.gotTick += new TickDelegate(tr2_gotTick);
while (tr2.NextTick())
{
;
}
tr2.Close();
}
// verify length
Assert.AreEqual(5, readdata2.Count);
Assert.AreEqual(5, readdata.Count);
TestTikWriterReader.removefile(FILE1);
TestTikWriterReader.removefile(FILE2);
}
public void Multiday()
{
_readdata.Clear();
_readdata2.Clear();
int d = 20100223;
int t = 235900;
int t1 = 0;
const decimal p = 50;
int s = 100;
string file1 = TikWriter.SafeFilename(Sym, _path, d);
TestTikWriterReader.Removefile(file1);
string file2 = TikWriter.SafeFilename(Sym, _path, d + 1);
TestTikWriterReader.Removefile(file2);
Tick[] data =
{
TickImpl.NewTrade(Sym, d, t++, p, s, string.Empty),
TickImpl.NewTrade(Sym, d, t++, p, s, string.Empty),
TickImpl.NewTrade(Sym, d, t++, p, s, string.Empty),
TickImpl.NewTrade(Sym, d, t++, p, s, string.Empty),
TickImpl.NewTrade(Sym, d, t++, p, s, string.Empty),
// day two
TickImpl.NewTrade(Sym, ++d, t1++, p, s, string.Empty),
TickImpl.NewTrade(Sym, d, t1++, p, s, string.Empty),
TickImpl.NewTrade(Sym, d, t1++, p, s, string.Empty),
TickImpl.NewTrade(Sym, d, t1++, p, s, string.Empty),
TickImpl.NewTrade(Sym, d, t1++, p, s, string.Empty),
};
TickArchiver ta = new TickArchiver(Environment.CurrentDirectory);
for (int i = 0; i < data.Length; i++)
{
ta.NewTick(data[i]);
}
ta.Stop();
// read file back in from files
if (System.IO.File.Exists(file1))
{
TikReader tr = new TikReader(file1);
tr.GotTick += tr_gotTick;
while (tr.NextTick())
{
;
}
tr.Close();
}
if (System.IO.File.Exists(file2))
{
TikReader tr2 = new TikReader(file2);
tr2.GotTick += tr2_gotTick;
while (tr2.NextTick())
{
;
}
tr2.Close();
}
// verify length
Assert.Equal(5, _readdata2.Count);
Assert.Equal(5, _readdata.Count);
TestTikWriterReader.Removefile(file1);
TestTikWriterReader.Removefile(file2);
}
bool CSV_DTOHLCV_convert(string filename, int tradesize, string sym)
{
int bads = 0;
int thistotal = _ticksprocessed;
bool g = true;
// get output filename
string convertname = string.Empty;
// setup writing to output
TikWriter outfile = null;
// setup input file
StreamReader infile = null;
try
{
// open input file
infile = new StreamReader(filename);
// read in and ignore header of input file
infile.ReadLine();
}
catch (Exception ex) { debug("error reading input header:" + ex.Message); g = false; }
// setup previous tick and current tick
Tick[] pk = { new TickImpl(), new TickImpl(), new TickImpl(), new TickImpl() };
Tick[] k = null; // this is a tick array instead of a simple tick
do
{
try
{
// get next tick from the file
k = CSV.parseline(infile.ReadLine(), sym);
}
catch (Exception ex)
{
debug("error reading line of: " + filename + " err: " + ex.Message + ex.StackTrace);
bads++;
continue;
}
if (k == null)
{
debug("Invalid converter: " + Converter.CSV_DTOHLCV.ToString());
return(false);
}
// bad tick
if (k[0].date == 0)
{
bads++; continue;
}
// if dates don't match, we need to write new output file; only checking the opening ticks' dates
if (k[0].date != pk[0].date)
{
try
{
// if the outfile was open previously, close it
if (outfile != null)
{
outfile.Close();
}
// get path from input
string path = Path.GetDirectoryName(filename) + "\\";
// setup new file
outfile = new TikWriter(path, k[0].symbol, k[0].date);
// report progress
progress((double)_ticksprocessed / _approxtotal);
}
catch (Exception ex) { debug(ex.Message); g = false; }
}
try
{
// write the tick
foreach (Tick t in k)
{
outfile.newTick(t);
}
// save this tick array as previous tick array
pk = k;
// count the tick as processed
_ticksprocessed += 4;
}
catch (Exception ex) { debug("error writing output tick: " + ex.Message); g = false; }
}
// keep going until input file is exhausted
while (!infile.EndOfStream);
// close output file
if (outfile == null)
{
debug("Tick file was never opened, likely that input file in wrong format.");
}
else
{
debug("Saved: " + outfile.Filepath);
outfile.Close();
}
// close input file
infile.Close();
// get percentage of good ticks
double goodratio = (_ticksprocessed - thistotal - bads) / (_ticksprocessed - (double)thistotal);
if (goodratio < GOODRATIO)
{
debug("Less than " + GOODRATIO.ToString("P0") + " good ticks");
g = false;
}
// return status
return(g);
}
bool convert(Converter con, string filename, int tradesize)
{
int bads = 0;
int thistotal = _ticksprocessed;
bool g = true;
// get output filename
string convertname = string.Empty;
// setup writing to output
TikWriter outfile = null;
// setup input file
StreamReader infile = null;
try
{
// open input file
switch (con)
{
case Converter.TradeStation:
infile = new StreamReader(filename);
// read in and ignore header of input file
infile.ReadLine();
break;
case Converter.eSignal_EPF:
infile = new StreamReader(filename);
// ignore header
SecurityImpl esec = eSigTick.InitEpf(infile);
_sym = esec.Symbol;
break;
case Converter.CQG:
infile = new StreamReader(filename);
// no header
break;
}
}
catch (Exception ex) { debug("error reading input header:" + ex.Message); g = false; }
// setup previous tick and current tick
Tick pk = new TickImpl();
Tick k = null;
do
{
try
{
// get next tick from the file
switch (con)
{
case Converter.CQG:
k = CQG.parseline(infile.ReadLine(), tradesize);
break;
case Converter.eSignal_EPF:
k = eSigTick.FromStream(_sym, infile);
break;
case Converter.TradeStation:
k = TradeStation.parseline(infile.ReadLine(), _sym, tradesize);
break;
}
}
catch (Exception ex) { bads++; continue; }
if (k == null)
{
debug("Invalid converter: " + con.ToString());
return(false);
}
// bad tick
if (k.date == 0)
{
bads++; continue;
}
// if dates don't match, we need to write new output file
if (k.date != pk.date)
{
try
{
// if the outfile was open previously, close it
if (outfile != null)
{
outfile.Close();
}
// get path from input
string path = Path.GetDirectoryName(filename) + "\\";
// setup new file
outfile = new TikWriter(path, k.symbol, k.date);
// report progress
progress((double)_ticksprocessed / _approxtotal);
}
catch (Exception ex) { debug(ex.Message); g = false; }
}
try
{
// write the tick
outfile.newTick(k);
// save this tick as previous tick
pk = k;
// count the tick as processed
_ticksprocessed++;
}
catch (Exception ex) { debug("error writing output tick: " + ex.Message); g = false; }
}
// keep going until input file is exhausted
while (!infile.EndOfStream);
// close output file
outfile.Close();
// close input file
infile.Close();
// get percentage of good ticks
double goodratio = (_ticksprocessed - thistotal - bads) / (_ticksprocessed - (double)thistotal);
if (goodratio < GOODRATIO)
{
debug("Less than " + GOODRATIO.ToString("P0") + " good ticks");
g = false;
}
// return status
return(g);
}