bool parseProductSecurities(string rootFolder, int date, string tempFolder=null) { // date 20110627 string subFolder = String.Format("{0:####-##}", date / 100); string rawBidAskPath = rootFolder + RawDataFolder_SecuritiesBidAsk + "\\" + subFolder + "\\"; string rawBidAskFilename = rawBidAskPath + "ba_cm_"+date+".zip"; string rawTickPath = rootFolder + RawDataFolder_SecuritiesTick + "\\" + subFolder + "\\"; // if already have parsed Zip file, use it directly string parsedPath = rootFolder + ParsedDataFolder_Securities + "\\" + subFolder + "\\"; string parsedBAZipName = parsedPath + date + "_SecBA.zip"; string parsedTRZipName = parsedPath + date + "_SecTR.zip"; // the folder to store intermediate data tempFolder = (tempFolder == null ? parsedPath : tempFolder); if (File.Exists(rawBidAskFilename) == false) return false; if (Directory.Exists(parsedPath) == false) Directory.CreateDirectory(parsedPath); if (File.Exists(parsedBAZipName) == false) { ZipFile rawZip = new ZipFile(rawBidAskFilename); // Extract and combine Info string[] masterList = rawZip.Entries.Where(x => x.FileName.StartsWith("mast_")).Select(y => y.FileName).ToArray(); List<string> output = new List<string>(); foreach (string masterFilename in masterList) { ZipEntry entry = rawZip[masterFilename]; string MB = masterFilename.StartsWith("mast_mb_") ? " M" : " G"; string line = ""; // read mast MemoryStream ms = new MemoryStream(); StreamReader sr = new StreamReader(ms); entry.Extract(ms); ms.Position = 0; while ((line = sr.ReadLine()) != null) output.Add(line + MB); ms.SetLength(0); } File.WriteAllLines(parsedPath + date + "_mast.txt", output); output.Clear(); ZipFile parsedBAZip = new ZipFile(parsedBAZipName); // Extract and split bid/ask Data string[] baList = rawZip.Entries.Where(x => x.FileName.StartsWith("ba_")).Select(y => y.FileName).ToArray(); string[] strArgList = { tempFolder + date }; int[] intArgList = { }; HashSet<string>[] filterList = { }; this.rawLineNumber = 1; MemoryParsingStream mps = new MemoryParsingStream(simplifySecBidAsk, simplifySecBidAskSub, strArgList, intArgList, filterList, 54); swList = new Dictionary<string, StreamWriter>(); bufList = new Dictionary<string, List<string>>(); mps.startConsumer(); rawLineNumber = 1; foreach (string baFilename in baList) rawZip[baFilename].Extract(mps); // extract uncompressed content and handling by processLine function mps.JoinConsumer(); foreach (KeyValuePair<string, StreamWriter> pair in swList) { pair.Value.Close(); string filename = tempFolder + date + "_BA_" + pair.Key + ".txt"; parsedBAZip.AddFile(filename, ""); } swList.Clear(); // convert bufList to stream and addEntry to zip byte[] NEWLINE = System.Text.Encoding.ASCII.GetBytes(Environment.NewLine); foreach (KeyValuePair<string, List<string>> pair in bufList) { string filename = date + "_BA_" + pair.Key + ".txt"; MemoryStream ms = new MemoryStream(); foreach (string str in pair.Value) { byte[] buf = System.Text.Encoding.ASCII.GetBytes(str); ms.Write(buf, 0, buf.Length); ms.Write(NEWLINE, 0, NEWLINE.Length); } ms.Position = 0; parsedBAZip.AddEntry(filename, ms); } bufList.Clear(); // compress all files parsedBAZip.CompressionLevel = Ionic.Zlib.CompressionLevel.Level8; parsedBAZip.Save(); string[] BAfileList = parsedBAZip.Entries.Select(x => x.FileName).ToArray(); System.Threading.Thread t1 = new System.Threading.Thread (delegate() { foreach (ZipEntry entry in parsedBAZip) File.Delete(tempFolder + entry.FileName); }); t1.Start(); } // read and split tick data if (File.Exists(parsedTRZipName) == false) { ZipFile parsedTRZip = new ZipFile(parsedTRZipName); if (Directory.Exists(rawTickPath)) { //abcd string[] files = Directory.GetFiles(rawTickPath, "tt_*_" + date); rawLineNumber = 1; foreach (string filename in files) { string line = ""; StreamReader sr = new StreamReader(filename); Dictionary<string, List<byte[]>> tickOutput = new Dictionary<string, List<byte[]>>(); while ((line = sr.ReadLine()) != null) { string STKCODE = line.Substring(0, 5).Trim(); float PRICE = float.Parse(line.Substring(14, 8)); int SHARES = int.Parse(line.Substring(22, 11).Trim()); char TRAD_TYPE = line[33]; int TIME = int.Parse(line.Substring(34, 6)); if (tickOutput.ContainsKey(STKCODE) == false) tickOutput.Add(STKCODE, new List<byte[]>()); string str = (rawLineNumber++) + "," + TRAD_TYPE + "," + TIME + "," + PRICE + "," + SHARES + Environment.NewLine; tickOutput[STKCODE].Add(System.Text.Encoding.ASCII.GetBytes(str)); } sr.Close(); foreach (KeyValuePair<string, List<byte[]>> pair in tickOutput) { string tickFilename = date + "_TR_" + pair.Key + ".txt"; MemoryStream ms = new MemoryStream(); foreach (byte[] buf in pair.Value) ms.Write(buf, 0, buf.Length); ms.Position = 0; parsedTRZip.AddEntry(tickFilename, ms); } tickOutput.Clear(); parsedTRZip.CompressionLevel = Ionic.Zlib.CompressionLevel.Level8; parsedTRZip.Save(); } } } return true; }
/* * Series List - null for any, of format "HSI_O", "HHI_F" * Product List - null for any, of format "HSI201203C19000", "HHI201205F" * if both filter is null, read all products */ public void readBidAskDataDerivative(string rootFolder, int date, HashSet<string> seriesFilter = null, HashSet<string> productFilter = null, int startTime = 091500, int closeTime = 161500) { bool readAll = (seriesFilter == null || seriesFilter.Count == 0) && (productFilter == null || productFilter.Count== 0); // date 20110627 string subFolder = String.Format("{0:####-##}", date / 100); // if already have parsed Zip file, use it directly string parsedPath = rootFolder + this.ParsedDataFolder_DerivativeBidAsk + "\\" + subFolder + "\\"; string parsedZipName = parsedPath + date + "_DevBA.zip"; // construct access list for series and products as (seriesAccessList, productAccessList) HashSet<string> seriesAccessList, productAccessList=null; if (readAll){ seriesAccessList = new HashSet<string>(SeriesTable.Keys); //productAccessList= new HashSet<string>(ProductTable.Keys); } else{ seriesAccessList = new HashSet<string>(); productAccessList = new HashSet<string>(); if (seriesFilter != null) { foreach (string seriesKey in seriesFilter) if (SeriesTable.ContainsKey(seriesKey)) { seriesAccessList.Add(seriesKey); foreach (string productKey in SeriesTable[seriesKey].productList) productAccessList.Add(productKey); } } if (productFilter != null) { int count = 0; foreach (string productKey in productFilter) if (ProductTable.ContainsKey(productKey) && ProductTable[productKey] is DevProduct) { string seriesKey = (ProductTable[productKey] as DevProduct).series.getKey(); seriesAccessList.Add(seriesKey); productAccessList.Add(productKey); count++; } if (count == 0) // not match any product in zip return; } } // unzip and read on the fly string zipFilename = parsedPath + date + "_DevBA.zip"; string[] strArgList = {""}; int[] intArgList = { date , (readAll?1:0) }; HashSet<string>[] filters = { productAccessList }; MemoryParsingStream ms = new MemoryParsingStream(processDevBidAskLineList, null, strArgList, intArgList, filters); ZipFile zip = new ZipFile(zipFilename); ms.startConsumer(); foreach (string seriesCode in seriesAccessList) { string csvName = date + "_BA_" + seriesCode + ".csv"; if (zip.ContainsEntry(csvName) == false) continue; string seriesMain = seriesCode.Substring(0, seriesCode.Length - 2); ms.setStrArgument(0, seriesMain); // the lines in txt file don't have info for the series zip[csvName].Extract(ms);// extract uncompressed content and handling by processLine function } ms.JoinConsumer(); /* // only after 2011-07-01 // add dev. equilbrium close price (Symbol, DayOpen, DayHigh, DayLow, DayClose, Volume) string filename = path + date + "Stat.csv"; if (File.Exists(filename)) { foreach (string line in File.ReadLines(filename)) { string[] strs = line.Split(COMMAS); string prodCode = strs[0]; string prodKey = OptionNamingFunctions.convertDevCode(prodCode); if (!readAll) { if (productFilter != null && !productFilter.Contains(prodKey)) continue; if (!ProductTable.ContainsKey(prodKey)) continue; } if (!ProductTable.ContainsKey(prodKey)) continue; DevProduct product = ProductTable[prodKey] as DevProduct; // add close Price { DateTime closeDT = DateTimeFunctions.convertDateTime(date, closeTime); closeDT.AddSeconds(2.0); // let the close price appear 2sec after close closeDT.AddSeconds(1); int time = DateTimeFunctions.getTimeInt(closeDT); ProductEvent pEvent = new ProductEvent(product, date, time); pEvent.EnablesStatistics(); pEvent.Day2SecHighest = float.Parse(strs[2]); pEvent.Day2SecLowest = float.Parse(strs[3]); pEvent.DayClose = float.Parse(strs[4]); pEvent.Day2SecTradedVolume = float.Parse(strs[5]); if (!eventSecondList.ContainsKey(time)) eventSecondList.Add(time, new EventInSecond()); eventSecondList[time].events.Add(pEvent); } } }*/ }