bool parseProductDerivativeBidAsk(string rootFolder, int date, string tempFolder)
{
string subFolder = String.Format("{0:####-##}", date / 100);
// need to parse
string rawPath = rootFolder + RawDataFolder_DerivativeBidAsk + "\\" + subFolder + "\\";
string parsedPath = rootFolder + ParsedDataFolder_DerivativeBidAsk + "\\" + subFolder + "\\";
if (Directory.Exists(rawPath) == false)
return false;
if (Directory.Exists(parsedPath) == false)
Directory.CreateDirectory(parsedPath);
string[] files = Directory.GetFiles(rawPath, date + "*_*.txt");
if (files.Length == 0)
return false;
var files_MP = files.Where(item => item.EndsWith("_MP.txt"));
var files_MC = files.Where(item => item.EndsWith("_MC.txt"));
var files_BA = files.Where(item => item.EndsWith("_BA.txt"));
StreamWriter swMP = new StreamWriter(parsedPath + date + "_MP.txt");
foreach (string filePath in files_MP)
{
string FileSuffix = filePath.Substring(filePath.Length - 9, 2);
string line = "";
System.IO.StreamReader file = new System.IO.StreamReader(filePath);
while ((line = file.ReadLine()) != null)
{
swMP.WriteLine(line + " "+FileSuffix);
string CLASS_CODE = line.Substring(0, 6).Trim(); // HSI, HHI ...etc
char FUT_OPT = line[6]; // F or O
int DATE = int.Parse(line.Substring(7, 8).Trim()); // 20121231
string PROD_NAME = line.Substring(15, 50).Trim(); // xyz limited
char EX_STYLE = line[81]; // Exercise Style, E or A
int CURRENCY = Currency.getType(line.Substring(82, 3)); // HK or US..etc
float MULTIPLIER = float.Parse(line.Substring(85,16).Trim()); // 10 or 50
Series series = new Series(CLASS_CODE, PROD_NAME, FUT_OPT, EX_STYLE, CURRENCY, MULTIPLIER, DATE);
series.FileSuffix = FileSuffix;
SeriesTable[series.getKey()] = series; // HSI:F or HHI:O
}
file.Close();
}
swMP.Close();
StreamWriter swMC = new StreamWriter(parsedPath + date + "_MC.txt");
foreach (string filePath in files_MC)
{
string line = "";
System.IO.StreamReader file = new System.IO.StreamReader(filePath);
while ((line = file.ReadLine()) != null)
{
swMC.WriteLine(line);
string SYMBOL = line.Substring(78).Trim(); // HSI21800Z1
string CLASS_CODE = line.Substring(0, 6).Trim(); // HSI, HHI ...etc
char FUT_OPT = line[6]; // F or O
Series series = SeriesTable[CLASS_CODE + '_' + FUT_OPT];
int EXPIRY_MTH = int.Parse(line.Substring(7, 4)); // YYMM as 1112 for 2011-Dec
float STRIKE_PRC = float.Parse(line.Substring(11, 17).Trim()); //
char productType = line[28] == ' ' ? 'F' : line[28]; // F - Future, C - Call, P - Put
string EDate = line.Substring(37, 8).Trim();
int EXPIRY_DATE = EDate.Length == 0 ? 0 : int.Parse(EDate); // 20121229
float CON_SIZE = float.Parse(line.Substring(45, 17).Trim()); // 10 or 50
if (SYMBOL.Length == 0)
continue;
DevProduct product = new DevProduct(series, EXPIRY_DATE, EXPIRY_MTH, productType, CON_SIZE, STRIKE_PRC, SYMBOL);
string prodKey = DevProduct.getKey(CLASS_CODE, EXPIRY_MTH, productType, STRIKE_PRC); // HSI1206F or HSI1206C12000
ProductTable[prodKey] = product;
}
file.Close();
}
swMC.Close();
Dictionary<string, List<string>> output = new Dictionary<string,List<string>>();
List<string> csvList = new List<string>();
foreach (string filePath in files_BA)
{
string fileSuffix = filePath.Substring(filePath.Length-9, 2);
string line = "";
System.IO.StreamReader file = new System.IO.StreamReader(filePath);
int lineCount = 0;
while ((line = file.ReadLine()) != null)
{
int TIME = int.Parse(line.Substring(37, 6).Trim());
string CLASS_CODE = line.Substring(0, 6).Trim(); // HSI, HHI ...etc
int EXPIRY_MTH = int.Parse(line.Substring(7, 4).Trim()); // YYMM as 1112 for 2011-Dec
char productType = line[28] == ' ' ? 'F' : line[28]; // F - Future, C - Call, P - Put
float STRIKE_PRC = float.Parse(line.Substring(11, 17).Trim());
string prodKey = DevProduct.getKey(CLASS_CODE, EXPIRY_MTH, productType, STRIKE_PRC);
if (!ProductTable.ContainsKey(prodKey))
continue;
DevProduct product = ProductTable[prodKey] as DevProduct;
//char FUT_OPT = line[6];
char BID_ASK = line[43];
float PRICE = float.Parse(line.Substring(44, 17).Trim());
int QUANTITY = int.Parse(line.Substring(61, 10).Trim());
if (PRICE == 9999999)
{
PRICE = 0;
QUANTITY = 0;
}
string key = product.series.getKey();
if (output.ContainsKey(key) == false)
output.Add(key, new List<string>());
List<string> list = output[key];
string str = TIME + "," + (++lineCount) + "," + prodKey.Substring(product.series.CLASS_CODE.Length) + "," + BID_ASK + "," + PRICE + "," + QUANTITY;
list.Add(str);
}
file.Close();
// save at once to reduce memory usage
foreach (KeyValuePair<string, List<string>> pair in output)
{
string filename = parsedPath + date + "_BA_" + pair.Key + ".csv";
File.WriteAllLines(filename, pair.Value);
csvList.Add(filename);
}
output.Clear();
}
// zip
string zipFilename = parsedPath + date + "_DevBA.zip";
File.Delete(zipFilename);
ZipFile zip = new ZipFile(zipFilename);
zip.CompressionLevel = Ionic.Zlib.CompressionLevel.Level8;
zip.AddFile(parsedPath + date + "_MP.txt", "");
zip.AddFile(parsedPath + date + "_MC.txt", "");
zip.AddFiles(csvList, "");
zip.Save();
// remove temp. file
File.Delete(parsedPath + date + "_MP.txt");
File.Delete(parsedPath + date + "_MC.txt");
foreach (string csvFile in csvList)
File.Delete(csvFile);
return true;
}
// if valid rawfile file exist, return true (would load info also, for easy process for tick raw data)
public bool readProductInfoDerivative(string rootFolder, int date, string tempFolder=null)
{
// date 20110627
string subFolder = String.Format("{0:####-##}", date / 100);
// if already have parsed Zip file, use it directly
string parsedBidAskPath = rootFolder + this.ParsedDataFolder_DerivativeBidAsk + "\\" + subFolder + "\\";
string parsedBidAskZipName = parsedBidAskPath + date + "_DevBA.zip";
if (tempFolder != null && Directory.Exists(tempFolder) == false)
Directory.CreateDirectory(tempFolder);
if (File.Exists(parsedBidAskZipName) == false)
if (parseProductDerivativeBidAsk(rootFolder, date, tempFolder) == false) // raw file don't exist also?
return false;
if (Directory.Exists(parsedBidAskPath) == false)
Directory.CreateDirectory(parsedBidAskPath);
// read from parse zip file
ZipFile zip = new ZipFile(parsedBidAskZipName);
MemoryStream ms = new MemoryStream();
StreamReader sr = new StreamReader(ms);
ZipEntry files_MP = zip[date + "_MP.txt"];
ZipEntry files_MC = zip[date + "_MC.txt"];
string line = "";
// read MP
files_MP.Extract(ms);
ms.Position = 0;
while ((line = sr.ReadLine()) != null)
{
string CLASS_CODE = line.Substring(0, 6).Trim(); // HSI, HHI ...etc
char FUT_OPT = line[6]; // F or O
int DATE = int.Parse(line.Substring(7, 8).Trim()); // 20121231
string PROD_NAME = line.Substring(15, 50).Trim(); // xyz limited
char EX_STYLE = line[81]; // Exercise Style, E or A
int CURRENCY = Currency.getType(line.Substring(82, 3)); // HK or US..etc
float MULTIPLIER = float.Parse(line.Substring(85,16).Trim()); // 10 or 50
string FileSuffix = line.Substring(103).Trim();
Series series = new Series(CLASS_CODE, PROD_NAME, FUT_OPT, EX_STYLE, CURRENCY, MULTIPLIER, DATE);
series.FileSuffix = FileSuffix;
SeriesTable[series.getKey()] = series; // HSI:F or HHI:O
}
ms.SetLength(0);
// read MC
files_MC.Extract(ms);
ms.Position = 0;
while ((line = sr.ReadLine()) != null)
{
string SYMBOL = line.Substring(78).Trim(); // HSI21800Z1
string CLASS_CODE = line.Substring(0, 6).Trim(); // HSI, HHI ...etc
char FUT_OPT = line[6]; // F or O
int EXPIRY_MTH = int.Parse(line.Substring(7, 4)); // YYMM as 1112 for 2011-Dec
float STRIKE_PRC = float.Parse(line.Substring(11, 17).Trim()); //
char productType = line[28] == ' ' ? 'F' : line[28]; // F - Future, C - Call, P - Put
string EDate = line.Substring(37, 8).Trim();
int EXPIRY_DATE = EDate.Length == 0 ? 0 : int.Parse(EDate); // 20121229
float CON_SIZE = float.Parse(line.Substring(45, 17).Trim()); // 10 or 50
if (SYMBOL.Length == 0)
continue;
Series series = SeriesTable[CLASS_CODE + '_' + FUT_OPT];
DevProduct product = new DevProduct(series, EXPIRY_DATE, EXPIRY_MTH, productType, CON_SIZE, STRIKE_PRC, SYMBOL);
string prodKey = DevProduct.getKey(CLASS_CODE, EXPIRY_MTH, productType, STRIKE_PRC); // HSI1206F or HSI1206C12000
series.productList.Add(prodKey);
ProductTable[prodKey] = product;
}
ms.SetLength(0);
sr.Close();
// tick data
// if already have parsed Zip file, use it directly
string parsedTickPath = rootFolder + this.ParsedDataFolder_DerivativeTick + "\\" + subFolder + "\\";
string parsedTickZipName = parsedTickPath + date + "_DevTick.zip";
if (Directory.Exists(parsedTickPath) == false)
Directory.CreateDirectory(parsedTickPath);
if (File.Exists(parsedTickZipName) == false)
parseProductDerivativeTick(rootFolder, date); // raw file don't exist also?
return true;
}