/**
* Must Call readDerivativeInfo before calling this function
*/
public void readTickDataDerivative(string rootFolder, int date,
HashSet<string> seriesFilter = null, HashSet<string> productFilter = null,
int startTime = 091500, int closeTime = 161500)
{
bool readAll = (seriesFilter == null || seriesFilter.Count == 0)
&& (productFilter == null || productFilter.Count == 0);
// date 20110627
string subFolder = String.Format("{0:####-##}", date / 100);
// construct access list for series and products as (seriesAccessList, productAccessList)
HashSet<string> seriesAccessList, productAccessList = null;
if (readAll)
{
seriesAccessList = new HashSet<string>(SeriesTable.Keys);
}
else
{
seriesAccessList = new HashSet<string>();
productAccessList = new HashSet<string>();
if (seriesFilter != null)
{
foreach (string seriesKey in seriesFilter)
if (SeriesTable.ContainsKey(seriesKey))
{
seriesAccessList.Add(seriesKey);
foreach (string productKey in SeriesTable[seriesKey].productList)
productAccessList.Add(productKey);
}
}
if (productFilter != null)
{
int count = 0;
foreach (string productKey in productFilter)
if (ProductTable.ContainsKey(productKey) && ProductTable[productKey] is DevProduct)
{
string seriesKey = (ProductTable[productKey] as DevProduct).series.getKey();
seriesAccessList.Add(seriesKey);
productAccessList.Add(productKey);
count++;
}
if (count == 0)
return; // not match any product in zip
}
}
// unzip and read on the fly
string parsedPath = rootFolder + this.ParsedDataFolder_DerivativeTick + "\\" + subFolder + "\\";
string zipFilename = parsedPath + date + "_DevTick.zip";
// read from parse zip file
ZipFile zip = new ZipFile(zipFilename);
foreach (string seriesCode in seriesAccessList)
{
string csvName = date + "_TR_" + seriesCode + ".csv";
if (zip.ContainsEntry(csvName) == false)
continue;
string seriesMain = seriesCode.Substring(0, seriesCode.Length - 2);
MemoryStream ms = new MemoryStream();
StreamReader sr = new StreamReader(ms);
string line = "";
// read
zip[csvName].Extract(ms);
ms.Position = 0;
while ((line = sr.ReadLine()) != null)
{
string[] val = line.Split(COMMA);
string prodKey = seriesMain + val[2];
if (productAccessList.Contains(prodKey) == false)
continue;
int TIME = int.Parse(val[0]);
int lineCount = int.Parse(val[1]);
float PRICE = float.Parse(val[3]);
int QUANTITY = int.Parse(val[4]);
short TRADETYPE = short.Parse(val[5]);
DevProduct product = ProductTable[prodKey] as DevProduct;
TradeEvent tradesEvent = new TradeEvent(product, date, TIME, QUANTITY, PRICE, TRADETYPE);
tradesEvent.lineNumber = lineCount;
if (!eventSecondList.ContainsKey(TIME))
eventSecondList.Add(TIME, new EventInSecond());
EventInSecond eis = eventSecondList[TIME];
eis.events.Add(tradesEvent);
}
ms.SetLength(0);
}
}
// if productFilter = null => read all
public void readDataSecurities(string rootFolder, int date, HashSet<string> productFilter = null, bool withTickData=false)
{
string subFolder = String.Format("{0:####-##}", date / 100);
string path = rootFolder + ParsedDataFolder_Securities + "\\" + subFolder + "\\";
string BAFileName = path + date + "_SecBA.zip";
// check if exist in product info.
bool readAll = (productFilter == null);
if (productFilter != null)
{
bool hasSec = false;
foreach (string pCode in productFilter)
if (ProductTable.ContainsKey(pCode) && ProductTable[pCode] is SecProduct)
hasSec = true;
if (!hasSec)
return;
}
// read BA
ZipFile BAzip = ZipFile.Read(BAFileName);
string[] strArgList = { };
int[] intArgList = { date };
HashSet<string>[] filterList = { };
foreach (ZipEntry entry in BAzip)
{
string entryName = entry.FileName;
string STKCODE = entryName.Substring(entryName.Length - 9, 5);
if (!readAll && !productFilter.Contains(STKCODE))
continue;
// read
MemoryStream ms = new MemoryStream();
StreamReader sr = new StreamReader(ms);
string line = "";
entry.Extract(ms);
ms.Position = 0;
while ((line = sr.ReadLine()) != null)
{
processSecBidAskLine(line, date, ProductTable[STKCODE] as SecProduct);
}
ms.SetLength(0);
}
// read TR
if (withTickData)
{
string TRFileName = path + date + "_SecTR.zip";
ZipFile TRzip = ZipFile.Read(TRFileName);
foreach (ZipEntry entry in TRzip)
{
string entryName = entry.FileName;
string STKCODE = entryName.Substring(entryName.Length - 9, 5);
SecProduct product = ProductTable[STKCODE] as SecProduct;
if (!readAll && !productFilter.Contains(STKCODE))
continue;
// read
MemoryStream ms = new MemoryStream();
StreamReader sr = new StreamReader(ms);
string line = "";
entry.Extract(ms);
ms.Position = 0;
while ((line = sr.ReadLine()) != null)
{
string[] val = line.Split(COMMA);
int lineCount = int.Parse(val[0]);
char TRAD_TYPE = val[1][0];
int TIME = int.Parse(val[2]);
float PRICE = float.Parse(val[3]);
int QUANTITY = int.Parse(val[4]);
TradeEvent tradesEvent = new TradeEvent(product, date, TIME, QUANTITY, PRICE, TradeType.getSecTradeTypeWithCharacter(TRAD_TYPE));
tradesEvent.lineNumber = lineCount;
if (!eventSecondList.ContainsKey(TIME))
eventSecondList.Add(TIME, new EventInSecond());
EventInSecond eis = eventSecondList[TIME];
eis.events.Add(tradesEvent);
}
ms.SetLength(0);
}
}
}