Beispiel #1
0
        bool parseProductDerivativeBidAsk(string rootFolder, int date, string tempFolder)
        {
            string subFolder = String.Format("{0:####-##}", date / 100);
            // need to parse
            string rawPath = rootFolder + RawDataFolder_DerivativeBidAsk + "\\" + subFolder + "\\";
            string parsedPath = rootFolder + ParsedDataFolder_DerivativeBidAsk + "\\" + subFolder + "\\";
            if (Directory.Exists(rawPath) == false)
                return false;

            if (Directory.Exists(parsedPath) == false)
                Directory.CreateDirectory(parsedPath);

            string[] files = Directory.GetFiles(rawPath, date  + "*_*.txt");

            if (files.Length == 0)
                return false;
            var files_MP = files.Where(item => item.EndsWith("_MP.txt"));
            var files_MC = files.Where(item => item.EndsWith("_MC.txt"));
            var files_BA = files.Where(item => item.EndsWith("_BA.txt"));

            StreamWriter swMP = new StreamWriter(parsedPath + date + "_MP.txt");
            foreach (string filePath in files_MP)
            {
                string FileSuffix = filePath.Substring(filePath.Length - 9, 2);
                string line = "";
                System.IO.StreamReader file = new System.IO.StreamReader(filePath);

                while ((line = file.ReadLine()) != null)
                {
                    swMP.WriteLine(line + " "+FileSuffix);

                    string CLASS_CODE = line.Substring(0, 6).Trim();        // HSI, HHI ...etc
                    char FUT_OPT = line[6];                                 // F or O
                    int DATE = int.Parse(line.Substring(7, 8).Trim());      // 20121231
                    string PROD_NAME = line.Substring(15, 50).Trim();       // xyz limited
                    char EX_STYLE = line[81];                               // Exercise Style, E or A
                    int CURRENCY = Currency.getType(line.Substring(82, 3)); // HK or US..etc
                    float MULTIPLIER = float.Parse(line.Substring(85,16).Trim()); // 10 or 50

                    Series series = new Series(CLASS_CODE, PROD_NAME, FUT_OPT, EX_STYLE, CURRENCY, MULTIPLIER, DATE);
                    series.FileSuffix = FileSuffix;
                    SeriesTable[series.getKey()] = series; // HSI:F or HHI:O

                }
                file.Close();
            }
            swMP.Close();

            StreamWriter swMC = new StreamWriter(parsedPath + date + "_MC.txt");
            foreach (string filePath in files_MC)
            {
                string line = "";
                System.IO.StreamReader file = new System.IO.StreamReader(filePath);
                while ((line = file.ReadLine()) != null)
                {
                    swMC.WriteLine(line);

                    string SYMBOL = line.Substring(78).Trim();                      // HSI21800Z1
                    string CLASS_CODE = line.Substring(0, 6).Trim();                // HSI, HHI ...etc
                    char FUT_OPT = line[6];                                         // F or O
                    Series series = SeriesTable[CLASS_CODE + '_' + FUT_OPT];
                    int EXPIRY_MTH = int.Parse(line.Substring(7, 4));               // YYMM as 1112 for 2011-Dec
                    float STRIKE_PRC = float.Parse(line.Substring(11, 17).Trim());  //
                    char productType = line[28] == ' ' ? 'F' : line[28];            // F - Future, C - Call, P - Put
                    string EDate = line.Substring(37, 8).Trim();
                    int EXPIRY_DATE = EDate.Length == 0 ? 0 : int.Parse(EDate);     // 20121229
                    float CON_SIZE = float.Parse(line.Substring(45, 17).Trim());    // 10 or 50

                    if (SYMBOL.Length == 0)
                        continue;

                    DevProduct product = new DevProduct(series, EXPIRY_DATE, EXPIRY_MTH, productType, CON_SIZE, STRIKE_PRC, SYMBOL);
                    string prodKey = DevProduct.getKey(CLASS_CODE, EXPIRY_MTH, productType, STRIKE_PRC); // HSI1206F or HSI1206C12000

                    ProductTable[prodKey] = product;
                }

                file.Close();
            }
            swMC.Close();

            Dictionary<string, List<string>> output = new Dictionary<string,List<string>>();
            List<string> csvList = new List<string>();
            foreach (string filePath in files_BA)
            {
                string fileSuffix = filePath.Substring(filePath.Length-9, 2);
                string line = "";
                System.IO.StreamReader file = new System.IO.StreamReader(filePath);
                int lineCount = 0;
                while ((line = file.ReadLine()) != null)
                {
                    int TIME = int.Parse(line.Substring(37, 6).Trim());

                    string CLASS_CODE = line.Substring(0, 6).Trim(); // HSI, HHI ...etc

                    int EXPIRY_MTH = int.Parse(line.Substring(7, 4).Trim()); // YYMM as 1112 for 2011-Dec
                    char productType = line[28] == ' ' ? 'F' : line[28];     // F - Future, C - Call, P - Put
                    float STRIKE_PRC = float.Parse(line.Substring(11, 17).Trim());
                    string prodKey = DevProduct.getKey(CLASS_CODE, EXPIRY_MTH, productType, STRIKE_PRC);

                    if (!ProductTable.ContainsKey(prodKey))
                        continue;
                    DevProduct product = ProductTable[prodKey] as DevProduct;

                    //char FUT_OPT = line[6];
                    char BID_ASK = line[43];
                    float PRICE = float.Parse(line.Substring(44, 17).Trim());
                    int QUANTITY = int.Parse(line.Substring(61, 10).Trim());

                    if (PRICE == 9999999)
                    {
                        PRICE = 0;
                        QUANTITY = 0;
                    }

                    string key = product.series.getKey();
                    if (output.ContainsKey(key) == false)
                        output.Add(key, new List<string>());
                    List<string> list = output[key];

                    string str = TIME + "," + (++lineCount) + "," + prodKey.Substring(product.series.CLASS_CODE.Length) + "," + BID_ASK + "," + PRICE + "," + QUANTITY;
                    list.Add(str);
                }
                file.Close();

                // save at once to reduce memory usage
                foreach (KeyValuePair<string, List<string>> pair in output)
                {
                    string filename = parsedPath + date + "_BA_" + pair.Key + ".csv";
                    File.WriteAllLines(filename, pair.Value);
                    csvList.Add(filename);
                }
                output.Clear();
            }

            // zip
            string zipFilename = parsedPath + date + "_DevBA.zip";
            File.Delete(zipFilename);
            ZipFile zip = new ZipFile(zipFilename);
            zip.CompressionLevel = Ionic.Zlib.CompressionLevel.Level8;
            zip.AddFile(parsedPath + date + "_MP.txt", "");
            zip.AddFile(parsedPath + date + "_MC.txt", "");
            zip.AddFiles(csvList, "");
            zip.Save();

            // remove temp. file
            File.Delete(parsedPath + date + "_MP.txt");
            File.Delete(parsedPath + date + "_MC.txt");
            foreach (string csvFile in csvList)
                File.Delete(csvFile);

            return true;
        }
Beispiel #2
0
        // if valid rawfile file exist, return true (would load info also, for easy process for tick raw data)
        public bool readProductInfoDerivative(string rootFolder, int date, string tempFolder=null)
        {
            // date 20110627
            string subFolder = String.Format("{0:####-##}", date / 100);

            // if already have parsed Zip file, use it directly
            string parsedBidAskPath = rootFolder + this.ParsedDataFolder_DerivativeBidAsk + "\\" + subFolder + "\\";
            string parsedBidAskZipName = parsedBidAskPath + date + "_DevBA.zip";

            if (tempFolder != null && Directory.Exists(tempFolder) == false)
                Directory.CreateDirectory(tempFolder);

            if (File.Exists(parsedBidAskZipName) == false)
                if (parseProductDerivativeBidAsk(rootFolder, date, tempFolder) == false) // raw file don't exist also?
                    return false;

            if (Directory.Exists(parsedBidAskPath) == false)
                Directory.CreateDirectory(parsedBidAskPath);

            // read from parse zip file
            ZipFile zip = new ZipFile(parsedBidAskZipName);
            MemoryStream ms = new MemoryStream();
            StreamReader sr = new StreamReader(ms);
            ZipEntry files_MP = zip[date + "_MP.txt"];
            ZipEntry files_MC = zip[date + "_MC.txt"];
            string line = "";

            // read MP
            files_MP.Extract(ms);
            ms.Position = 0;
            while ((line = sr.ReadLine()) != null)
            {
                string CLASS_CODE = line.Substring(0, 6).Trim();        // HSI, HHI ...etc
                char FUT_OPT = line[6];                                 // F or O
                int DATE = int.Parse(line.Substring(7, 8).Trim());      // 20121231
                string PROD_NAME = line.Substring(15, 50).Trim();       // xyz limited
                char EX_STYLE = line[81];                               // Exercise Style, E or A
                int CURRENCY = Currency.getType(line.Substring(82, 3)); // HK or US..etc
                float MULTIPLIER = float.Parse(line.Substring(85,16).Trim()); // 10 or 50
                string FileSuffix = line.Substring(103).Trim();
                Series series = new Series(CLASS_CODE, PROD_NAME, FUT_OPT, EX_STYLE, CURRENCY, MULTIPLIER, DATE);
                series.FileSuffix = FileSuffix;
                SeriesTable[series.getKey()] = series; // HSI:F or HHI:O
            }
            ms.SetLength(0);

            // read MC
            files_MC.Extract(ms);
            ms.Position = 0;
            while ((line = sr.ReadLine()) != null)
            {
                string SYMBOL = line.Substring(78).Trim();                      // HSI21800Z1
                string CLASS_CODE = line.Substring(0, 6).Trim();                // HSI, HHI ...etc
                char FUT_OPT = line[6];                                         // F or O
                int EXPIRY_MTH = int.Parse(line.Substring(7, 4));               // YYMM as 1112 for 2011-Dec
                float STRIKE_PRC = float.Parse(line.Substring(11, 17).Trim());  //
                char productType = line[28] == ' ' ? 'F' : line[28];            // F - Future, C - Call, P - Put
                string EDate = line.Substring(37, 8).Trim();
                int EXPIRY_DATE = EDate.Length == 0 ? 0 : int.Parse(EDate);     // 20121229
                float CON_SIZE = float.Parse(line.Substring(45, 17).Trim());    // 10 or 50

                if (SYMBOL.Length == 0)
                    continue;

                Series series = SeriesTable[CLASS_CODE + '_' + FUT_OPT];
                DevProduct product = new DevProduct(series, EXPIRY_DATE, EXPIRY_MTH, productType, CON_SIZE, STRIKE_PRC, SYMBOL);
                string prodKey = DevProduct.getKey(CLASS_CODE, EXPIRY_MTH, productType, STRIKE_PRC); // HSI1206F or HSI1206C12000
                series.productList.Add(prodKey);

                ProductTable[prodKey] = product;
            }
            ms.SetLength(0);

            sr.Close();

            // tick data
            // if already have parsed Zip file, use it directly
            string parsedTickPath = rootFolder + this.ParsedDataFolder_DerivativeTick + "\\" + subFolder + "\\";
            string parsedTickZipName = parsedTickPath + date + "_DevTick.zip";

            if (Directory.Exists(parsedTickPath) == false)
                Directory.CreateDirectory(parsedTickPath);

            if (File.Exists(parsedTickZipName) == false)
                parseProductDerivativeTick(rootFolder, date); // raw file don't exist also?

            return true;
        }