bool parseProductSecurities(string rootFolder, int date, string tempFolder=null)
{
// date 20110627
string subFolder = String.Format("{0:####-##}", date / 100);
string rawBidAskPath = rootFolder + RawDataFolder_SecuritiesBidAsk + "\\" + subFolder + "\\";
string rawBidAskFilename = rawBidAskPath + "ba_cm_"+date+".zip";
string rawTickPath = rootFolder + RawDataFolder_SecuritiesTick + "\\" + subFolder + "\\";
// if already have parsed Zip file, use it directly
string parsedPath = rootFolder + ParsedDataFolder_Securities + "\\" + subFolder + "\\";
string parsedBAZipName = parsedPath + date + "_SecBA.zip";
string parsedTRZipName = parsedPath + date + "_SecTR.zip";
// the folder to store intermediate data
tempFolder = (tempFolder == null ? parsedPath : tempFolder);
if (File.Exists(rawBidAskFilename) == false)
return false;
if (Directory.Exists(parsedPath) == false)
Directory.CreateDirectory(parsedPath);
if (File.Exists(parsedBAZipName) == false)
{
ZipFile rawZip = new ZipFile(rawBidAskFilename);
// Extract and combine Info
string[] masterList = rawZip.Entries.Where(x => x.FileName.StartsWith("mast_")).Select(y => y.FileName).ToArray();
List<string> output = new List<string>();
foreach (string masterFilename in masterList)
{
ZipEntry entry = rawZip[masterFilename];
string MB = masterFilename.StartsWith("mast_mb_") ? " M" : " G";
string line = "";
// read mast
MemoryStream ms = new MemoryStream();
StreamReader sr = new StreamReader(ms);
entry.Extract(ms);
ms.Position = 0;
while ((line = sr.ReadLine()) != null)
output.Add(line + MB);
ms.SetLength(0);
}
File.WriteAllLines(parsedPath + date + "_mast.txt", output);
output.Clear();
ZipFile parsedBAZip = new ZipFile(parsedBAZipName);
// Extract and split bid/ask Data
string[] baList = rawZip.Entries.Where(x => x.FileName.StartsWith("ba_")).Select(y => y.FileName).ToArray();
string[] strArgList = { tempFolder + date };
int[] intArgList = { };
HashSet<string>[] filterList = { };
this.rawLineNumber = 1;
MemoryParsingStream mps = new MemoryParsingStream(simplifySecBidAsk, simplifySecBidAskSub,
strArgList, intArgList, filterList, 54);
swList = new Dictionary<string, StreamWriter>();
bufList = new Dictionary<string, List<string>>();
mps.startConsumer();
rawLineNumber = 1;
foreach (string baFilename in baList)
rawZip[baFilename].Extract(mps); // extract uncompressed content and handling by processLine function
mps.JoinConsumer();
foreach (KeyValuePair<string, StreamWriter> pair in swList)
{
pair.Value.Close();
string filename = tempFolder + date + "_BA_" + pair.Key + ".txt";
parsedBAZip.AddFile(filename, "");
}
swList.Clear();
// convert bufList to stream and addEntry to zip
byte[] NEWLINE = System.Text.Encoding.ASCII.GetBytes(Environment.NewLine);
foreach (KeyValuePair<string, List<string>> pair in bufList)
{
string filename = date + "_BA_" + pair.Key + ".txt";
MemoryStream ms = new MemoryStream();
foreach (string str in pair.Value)
{
byte[] buf = System.Text.Encoding.ASCII.GetBytes(str);
ms.Write(buf, 0, buf.Length);
ms.Write(NEWLINE, 0, NEWLINE.Length);
}
ms.Position = 0;
parsedBAZip.AddEntry(filename, ms);
}
bufList.Clear();
// compress all files
parsedBAZip.CompressionLevel = Ionic.Zlib.CompressionLevel.Level8;
parsedBAZip.Save();
string[] BAfileList = parsedBAZip.Entries.Select(x => x.FileName).ToArray();
System.Threading.Thread t1 = new System.Threading.Thread
(delegate()
{
foreach (ZipEntry entry in parsedBAZip)
File.Delete(tempFolder + entry.FileName);
});
t1.Start();
}
// read and split tick data
if (File.Exists(parsedTRZipName) == false)
{
ZipFile parsedTRZip = new ZipFile(parsedTRZipName);
if (Directory.Exists(rawTickPath))
{
//abcd
string[] files = Directory.GetFiles(rawTickPath, "tt_*_" + date);
rawLineNumber = 1;
foreach (string filename in files)
{
string line = "";
StreamReader sr = new StreamReader(filename);
Dictionary<string, List<byte[]>> tickOutput = new Dictionary<string, List<byte[]>>();
while ((line = sr.ReadLine()) != null)
{
string STKCODE = line.Substring(0, 5).Trim();
float PRICE = float.Parse(line.Substring(14, 8));
int SHARES = int.Parse(line.Substring(22, 11).Trim());
char TRAD_TYPE = line[33];
int TIME = int.Parse(line.Substring(34, 6));
if (tickOutput.ContainsKey(STKCODE) == false)
tickOutput.Add(STKCODE, new List<byte[]>());
string str = (rawLineNumber++) + "," + TRAD_TYPE + "," + TIME + "," + PRICE + "," + SHARES + Environment.NewLine;
tickOutput[STKCODE].Add(System.Text.Encoding.ASCII.GetBytes(str));
}
sr.Close();
foreach (KeyValuePair<string, List<byte[]>> pair in tickOutput)
{
string tickFilename = date + "_TR_" + pair.Key + ".txt";
MemoryStream ms = new MemoryStream();
foreach (byte[] buf in pair.Value)
ms.Write(buf, 0, buf.Length);
ms.Position = 0;
parsedTRZip.AddEntry(tickFilename, ms);
}
tickOutput.Clear();
parsedTRZip.CompressionLevel = Ionic.Zlib.CompressionLevel.Level8;
parsedTRZip.Save();
}
}
}
return true;
}
/*
* Series List - null for any, of format "HSI_O", "HHI_F"
* Product List - null for any, of format "HSI201203C19000", "HHI201205F"
* if both filter is null, read all products
*/
public void readBidAskDataDerivative(string rootFolder, int date,
HashSet<string> seriesFilter = null, HashSet<string> productFilter = null,
int startTime = 091500, int closeTime = 161500)
{
bool readAll = (seriesFilter == null || seriesFilter.Count == 0)
&& (productFilter == null || productFilter.Count== 0);
// date 20110627
string subFolder = String.Format("{0:####-##}", date / 100);
// if already have parsed Zip file, use it directly
string parsedPath = rootFolder + this.ParsedDataFolder_DerivativeBidAsk + "\\" + subFolder + "\\";
string parsedZipName = parsedPath + date + "_DevBA.zip";
// construct access list for series and products as (seriesAccessList, productAccessList)
HashSet<string> seriesAccessList, productAccessList=null;
if (readAll){
seriesAccessList = new HashSet<string>(SeriesTable.Keys);
//productAccessList= new HashSet<string>(ProductTable.Keys);
}
else{
seriesAccessList = new HashSet<string>();
productAccessList = new HashSet<string>();
if (seriesFilter != null)
{
foreach (string seriesKey in seriesFilter)
if (SeriesTable.ContainsKey(seriesKey))
{
seriesAccessList.Add(seriesKey);
foreach (string productKey in SeriesTable[seriesKey].productList)
productAccessList.Add(productKey);
}
}
if (productFilter != null)
{
int count = 0;
foreach (string productKey in productFilter)
if (ProductTable.ContainsKey(productKey) && ProductTable[productKey] is DevProduct)
{
string seriesKey = (ProductTable[productKey] as DevProduct).series.getKey();
seriesAccessList.Add(seriesKey);
productAccessList.Add(productKey);
count++;
}
if (count == 0) // not match any product in zip
return;
}
}
// unzip and read on the fly
string zipFilename = parsedPath + date + "_DevBA.zip";
string[] strArgList = {""};
int[] intArgList = { date , (readAll?1:0) };
HashSet<string>[] filters = { productAccessList };
MemoryParsingStream ms = new MemoryParsingStream(processDevBidAskLineList, null, strArgList, intArgList, filters);
ZipFile zip = new ZipFile(zipFilename);
ms.startConsumer();
foreach (string seriesCode in seriesAccessList)
{
string csvName = date + "_BA_" + seriesCode + ".csv";
if (zip.ContainsEntry(csvName) == false)
continue;
string seriesMain = seriesCode.Substring(0, seriesCode.Length - 2);
ms.setStrArgument(0, seriesMain); // the lines in txt file don't have info for the series
zip[csvName].Extract(ms);// extract uncompressed content and handling by processLine function
}
ms.JoinConsumer();
/*
// only after 2011-07-01
// add dev. equilbrium close price (Symbol, DayOpen, DayHigh, DayLow, DayClose, Volume)
string filename = path + date + "Stat.csv";
if (File.Exists(filename))
{
foreach (string line in File.ReadLines(filename))
{
string[] strs = line.Split(COMMAS);
string prodCode = strs[0];
string prodKey = OptionNamingFunctions.convertDevCode(prodCode);
if (!readAll)
{
if (productFilter != null && !productFilter.Contains(prodKey))
continue;
if (!ProductTable.ContainsKey(prodKey))
continue;
}
if (!ProductTable.ContainsKey(prodKey)) continue;
DevProduct product = ProductTable[prodKey] as DevProduct;
// add close Price
{
DateTime closeDT = DateTimeFunctions.convertDateTime(date, closeTime);
closeDT.AddSeconds(2.0); // let the close price appear 2sec after close
closeDT.AddSeconds(1);
int time = DateTimeFunctions.getTimeInt(closeDT);
ProductEvent pEvent = new ProductEvent(product, date, time);
pEvent.EnablesStatistics();
pEvent.Day2SecHighest = float.Parse(strs[2]);
pEvent.Day2SecLowest = float.Parse(strs[3]);
pEvent.DayClose = float.Parse(strs[4]);
pEvent.Day2SecTradedVolume = float.Parse(strs[5]);
if (!eventSecondList.ContainsKey(time))
eventSecondList.Add(time, new EventInSecond());
eventSecondList[time].events.Add(pEvent);
}
}
}*/
}
