bool parseProductDerivativeBidAsk(string rootFolder, int date, string tempFolder) { string subFolder = String.Format("{0:####-##}", date / 100); // need to parse string rawPath = rootFolder + RawDataFolder_DerivativeBidAsk + "\\" + subFolder + "\\"; string parsedPath = rootFolder + ParsedDataFolder_DerivativeBidAsk + "\\" + subFolder + "\\"; if (Directory.Exists(rawPath) == false) return false; if (Directory.Exists(parsedPath) == false) Directory.CreateDirectory(parsedPath); string[] files = Directory.GetFiles(rawPath, date + "*_*.txt"); if (files.Length == 0) return false; var files_MP = files.Where(item => item.EndsWith("_MP.txt")); var files_MC = files.Where(item => item.EndsWith("_MC.txt")); var files_BA = files.Where(item => item.EndsWith("_BA.txt")); StreamWriter swMP = new StreamWriter(parsedPath + date + "_MP.txt"); foreach (string filePath in files_MP) { string FileSuffix = filePath.Substring(filePath.Length - 9, 2); string line = ""; System.IO.StreamReader file = new System.IO.StreamReader(filePath); while ((line = file.ReadLine()) != null) { swMP.WriteLine(line + " "+FileSuffix); string CLASS_CODE = line.Substring(0, 6).Trim(); // HSI, HHI ...etc char FUT_OPT = line[6]; // F or O int DATE = int.Parse(line.Substring(7, 8).Trim()); // 20121231 string PROD_NAME = line.Substring(15, 50).Trim(); // xyz limited char EX_STYLE = line[81]; // Exercise Style, E or A int CURRENCY = Currency.getType(line.Substring(82, 3)); // HK or US..etc float MULTIPLIER = float.Parse(line.Substring(85,16).Trim()); // 10 or 50 Series series = new Series(CLASS_CODE, PROD_NAME, FUT_OPT, EX_STYLE, CURRENCY, MULTIPLIER, DATE); series.FileSuffix = FileSuffix; SeriesTable[series.getKey()] = series; // HSI:F or HHI:O } file.Close(); } swMP.Close(); StreamWriter swMC = new StreamWriter(parsedPath + date + "_MC.txt"); foreach (string filePath in files_MC) { string line = ""; System.IO.StreamReader file = new System.IO.StreamReader(filePath); while ((line = file.ReadLine()) != null) { swMC.WriteLine(line); string SYMBOL = line.Substring(78).Trim(); // HSI21800Z1 string CLASS_CODE = line.Substring(0, 6).Trim(); // HSI, HHI ...etc char FUT_OPT = line[6]; // F or O Series series = SeriesTable[CLASS_CODE + '_' + FUT_OPT]; int EXPIRY_MTH = int.Parse(line.Substring(7, 4)); // YYMM as 1112 for 2011-Dec float STRIKE_PRC = float.Parse(line.Substring(11, 17).Trim()); // char productType = line[28] == ' ' ? 'F' : line[28]; // F - Future, C - Call, P - Put string EDate = line.Substring(37, 8).Trim(); int EXPIRY_DATE = EDate.Length == 0 ? 0 : int.Parse(EDate); // 20121229 float CON_SIZE = float.Parse(line.Substring(45, 17).Trim()); // 10 or 50 if (SYMBOL.Length == 0) continue; DevProduct product = new DevProduct(series, EXPIRY_DATE, EXPIRY_MTH, productType, CON_SIZE, STRIKE_PRC, SYMBOL); string prodKey = DevProduct.getKey(CLASS_CODE, EXPIRY_MTH, productType, STRIKE_PRC); // HSI1206F or HSI1206C12000 ProductTable[prodKey] = product; } file.Close(); } swMC.Close(); Dictionary<string, List<string>> output = new Dictionary<string,List<string>>(); List<string> csvList = new List<string>(); foreach (string filePath in files_BA) { string fileSuffix = filePath.Substring(filePath.Length-9, 2); string line = ""; System.IO.StreamReader file = new System.IO.StreamReader(filePath); int lineCount = 0; while ((line = file.ReadLine()) != null) { int TIME = int.Parse(line.Substring(37, 6).Trim()); string CLASS_CODE = line.Substring(0, 6).Trim(); // HSI, HHI ...etc int EXPIRY_MTH = int.Parse(line.Substring(7, 4).Trim()); // YYMM as 1112 for 2011-Dec char productType = line[28] == ' ' ? 'F' : line[28]; // F - Future, C - Call, P - Put float STRIKE_PRC = float.Parse(line.Substring(11, 17).Trim()); string prodKey = DevProduct.getKey(CLASS_CODE, EXPIRY_MTH, productType, STRIKE_PRC); if (!ProductTable.ContainsKey(prodKey)) continue; DevProduct product = ProductTable[prodKey] as DevProduct; //char FUT_OPT = line[6]; char BID_ASK = line[43]; float PRICE = float.Parse(line.Substring(44, 17).Trim()); int QUANTITY = int.Parse(line.Substring(61, 10).Trim()); if (PRICE == 9999999) { PRICE = 0; QUANTITY = 0; } string key = product.series.getKey(); if (output.ContainsKey(key) == false) output.Add(key, new List<string>()); List<string> list = output[key]; string str = TIME + "," + (++lineCount) + "," + prodKey.Substring(product.series.CLASS_CODE.Length) + "," + BID_ASK + "," + PRICE + "," + QUANTITY; list.Add(str); } file.Close(); // save at once to reduce memory usage foreach (KeyValuePair<string, List<string>> pair in output) { string filename = parsedPath + date + "_BA_" + pair.Key + ".csv"; File.WriteAllLines(filename, pair.Value); csvList.Add(filename); } output.Clear(); } // zip string zipFilename = parsedPath + date + "_DevBA.zip"; File.Delete(zipFilename); ZipFile zip = new ZipFile(zipFilename); zip.CompressionLevel = Ionic.Zlib.CompressionLevel.Level8; zip.AddFile(parsedPath + date + "_MP.txt", ""); zip.AddFile(parsedPath + date + "_MC.txt", ""); zip.AddFiles(csvList, ""); zip.Save(); // remove temp. file File.Delete(parsedPath + date + "_MP.txt"); File.Delete(parsedPath + date + "_MC.txt"); foreach (string csvFile in csvList) File.Delete(csvFile); return true; }
// if valid rawfile file exist, return true (would load info also, for easy process for tick raw data) public bool readProductInfoDerivative(string rootFolder, int date, string tempFolder=null) { // date 20110627 string subFolder = String.Format("{0:####-##}", date / 100); // if already have parsed Zip file, use it directly string parsedBidAskPath = rootFolder + this.ParsedDataFolder_DerivativeBidAsk + "\\" + subFolder + "\\"; string parsedBidAskZipName = parsedBidAskPath + date + "_DevBA.zip"; if (tempFolder != null && Directory.Exists(tempFolder) == false) Directory.CreateDirectory(tempFolder); if (File.Exists(parsedBidAskZipName) == false) if (parseProductDerivativeBidAsk(rootFolder, date, tempFolder) == false) // raw file don't exist also? return false; if (Directory.Exists(parsedBidAskPath) == false) Directory.CreateDirectory(parsedBidAskPath); // read from parse zip file ZipFile zip = new ZipFile(parsedBidAskZipName); MemoryStream ms = new MemoryStream(); StreamReader sr = new StreamReader(ms); ZipEntry files_MP = zip[date + "_MP.txt"]; ZipEntry files_MC = zip[date + "_MC.txt"]; string line = ""; // read MP files_MP.Extract(ms); ms.Position = 0; while ((line = sr.ReadLine()) != null) { string CLASS_CODE = line.Substring(0, 6).Trim(); // HSI, HHI ...etc char FUT_OPT = line[6]; // F or O int DATE = int.Parse(line.Substring(7, 8).Trim()); // 20121231 string PROD_NAME = line.Substring(15, 50).Trim(); // xyz limited char EX_STYLE = line[81]; // Exercise Style, E or A int CURRENCY = Currency.getType(line.Substring(82, 3)); // HK or US..etc float MULTIPLIER = float.Parse(line.Substring(85,16).Trim()); // 10 or 50 string FileSuffix = line.Substring(103).Trim(); Series series = new Series(CLASS_CODE, PROD_NAME, FUT_OPT, EX_STYLE, CURRENCY, MULTIPLIER, DATE); series.FileSuffix = FileSuffix; SeriesTable[series.getKey()] = series; // HSI:F or HHI:O } ms.SetLength(0); // read MC files_MC.Extract(ms); ms.Position = 0; while ((line = sr.ReadLine()) != null) { string SYMBOL = line.Substring(78).Trim(); // HSI21800Z1 string CLASS_CODE = line.Substring(0, 6).Trim(); // HSI, HHI ...etc char FUT_OPT = line[6]; // F or O int EXPIRY_MTH = int.Parse(line.Substring(7, 4)); // YYMM as 1112 for 2011-Dec float STRIKE_PRC = float.Parse(line.Substring(11, 17).Trim()); // char productType = line[28] == ' ' ? 'F' : line[28]; // F - Future, C - Call, P - Put string EDate = line.Substring(37, 8).Trim(); int EXPIRY_DATE = EDate.Length == 0 ? 0 : int.Parse(EDate); // 20121229 float CON_SIZE = float.Parse(line.Substring(45, 17).Trim()); // 10 or 50 if (SYMBOL.Length == 0) continue; Series series = SeriesTable[CLASS_CODE + '_' + FUT_OPT]; DevProduct product = new DevProduct(series, EXPIRY_DATE, EXPIRY_MTH, productType, CON_SIZE, STRIKE_PRC, SYMBOL); string prodKey = DevProduct.getKey(CLASS_CODE, EXPIRY_MTH, productType, STRIKE_PRC); // HSI1206F or HSI1206C12000 series.productList.Add(prodKey); ProductTable[prodKey] = product; } ms.SetLength(0); sr.Close(); // tick data // if already have parsed Zip file, use it directly string parsedTickPath = rootFolder + this.ParsedDataFolder_DerivativeTick + "\\" + subFolder + "\\"; string parsedTickZipName = parsedTickPath + date + "_DevTick.zip"; if (Directory.Exists(parsedTickPath) == false) Directory.CreateDirectory(parsedTickPath); if (File.Exists(parsedTickZipName) == false) parseProductDerivativeTick(rootFolder, date); // raw file don't exist also? return true; }