/// <summary> /// Enumerate through the lines of the algoseek files. /// </summary> /// <param name="file">BZ File for algoseek</param> /// <param name="date">Reference date of the folder</param> public AlgoSeekOptionsReader(string file, DateTime date, HashSet <string> symbolFilter = null) { _date = date; _underlyingCache = new Dictionary <string, Symbol>(); var streamProvider = StreamProvider.ForExtension(Path.GetExtension(file)); _stream = streamProvider.Open(file).First(); _streamReader = new StreamReader(_stream); _symbolFilter = symbolFilter; // detecting column order in the file var headerLine = _streamReader.ReadLine(); if (!string.IsNullOrEmpty(headerLine)) { var header = headerLine.ToCsv(); _columnTimestamp = header.FindIndex(x => x == "Timestamp"); _columnTicker = header.FindIndex(x => x == "Ticker"); _columnType = header.FindIndex(x => x == "Type"); _columnSide = header.FindIndex(x => x == "Side"); _columnPutCall = header.FindIndex(x => x == "PutCall"); _columnExpiration = header.FindIndex(x => x == "Expiration"); _columnStrike = header.FindIndex(x => x == "Strike"); _columnQuantity = header.FindIndex(x => x == "Quantity"); _columnPremium = header.FindIndex(x => x == "Premium"); _columnExchange = header.FindIndex(x => x == "Exchange"); _columnsCount = Enumerable.Max(new[] { _columnTimestamp, _columnTicker, _columnType, _columnSide, _columnPutCall, _columnExpiration, _columnStrike, _columnQuantity, _columnPremium, _columnExchange }); } //Prime the data pump, set the current. Current = null; MoveNext(); }
/// <summary> /// Enumerate through the lines of the algoseek files. /// </summary> /// <param name="file">BZ File for AlgoSeek</param> /// <param name="symbolMultipliers">Symbol price multiplier</param> /// <param name="symbolFilter">Symbol filter to apply, if any</param> public AlgoSeekFuturesReader(string file, Dictionary <string, decimal> symbolMultipliers, HashSet <string> symbolFilter = null) { var streamProvider = StreamProvider.ForExtension(Path.GetExtension(file)); _stream = streamProvider.Open(file).First(); _streamReader = new StreamReader(_stream); _symbolFilter = symbolFilter; _symbolMultipliers = symbolMultipliers.ToDictionary(); _symbolProperties = SymbolPropertiesDatabase.FromDataFolder(); // detecting column order in the file var headerLine = _streamReader.ReadLine(); if (!string.IsNullOrEmpty(headerLine)) { var header = headerLine.ToCsv(); _columnTimestamp = header.FindIndex(x => x == "Timestamp"); _columnTicker = header.FindIndex(x => x == "Ticker"); _columnType = header.FindIndex(x => x == "Type"); _columnSide = header.FindIndex(x => x == "Side"); _columnSecID = header.FindIndex(x => x == "SecurityID"); _columnQuantity = header.FindIndex(x => x == "Quantity"); _columnPrice = header.FindIndex(x => x == "Price"); _columnsCount = new[] { _columnTimestamp, _columnTicker, _columnType, _columnSide, _columnSecID, _columnQuantity, _columnPrice }.Max(); } //Prime the data pump, set the current. Current = null; MoveNext(); }
/// <summary> /// Converts gzipped backfill data. Reads all *.gz files in the raw directory and attempts to convert them /// </summary> public void ConvertHistoricalData() { foreach (var archive in _rawSourceDirectory.GetFiles("*.gz", SearchOption.TopDirectoryOnly)) { Log.Trace($"PsychSignalDataConverter.ConvertBackfill(): Begin converting historical data for file: {archive.FullName}"); using (var archiveStream = StreamProvider.ForExtension(".gz").Open(archive.FullName).First()) { Convert(archiveStream); } Log.Trace($"PsychSignalDataConverter.ConvertBackfill(): Finished converting historical data for file: {archive.FullName}"); } Dispose(); }
/// <summary> /// Enumerate through the lines of the algoseek files. /// </summary> /// <param name="file">BZ File for algoseek</param> /// <param name="date">Reference date of the folder</param> public AlgoSeekStocksReader(string file, DateTime date, HashSet <string> symbolFilter = null) { _date = date; _underlyingCache = new Dictionary <string, Symbol>(); var streamProvider = StreamProvider.ForExtension(Path.GetExtension(file)); _stream = streamProvider.Open(file).First(); _streamReader = new StreamReader(_stream); _symbolFilter = symbolFilter; _symbol = Symbol.Create("510050", SecurityType.Equity, Market.SSE); // detecting column order in the file var headerLine = _streamReader.ReadLine(); if (!string.IsNullOrEmpty(headerLine)) { var header = headerLine.ToCsv(); _columnTime = header.FindIndex(x => x == "dt"); _columnPrice = header.FindIndex(x => x == "close"); _columnVolume = header.FindIndex(x => x == "volume"); _columnAmount = header.FindIndex(x => x == "Amount"); _columnOpenInt = header.FindIndex(x => x == "OpenInt"); _columnTotalVol = header.FindIndex(x => x == "TotalVol"); _columnTotalAmount = header.FindIndex(x => x == "TotalAmount"); _columnLastClose = header.FindIndex(x => x == "close"); _columnOpen = header.FindIndex(x => x == "Open"); _columnHigh = header.FindIndex(x => x == "High"); _columnLow = header.FindIndex(x => x == "Low"); _columnSP1 = header.FindIndex(x => x == "SP1"); _columnSV1 = header.FindIndex(x => x == "SV1"); _columnBP1 = header.FindIndex(x => x == "BP1"); _columnBV1 = header.FindIndex(x => x == "BV1"); _columnsCount = Enumerable.Max(new[] { _columnTime, _columnPrice, _columnVolume, _columnAmount, _columnOpenInt, _columnTotalVol, _columnTotalAmount, _columnLastClose, _columnOpen, _columnHigh, _columnLow, _columnSP1, _columnSV1, _columnBP1, _columnBV1 }); } //Prime the data pump, set the current. Current = null; MoveNext(); }
/// <summary> /// Enumerate through the lines of the algoseek files. /// </summary> /// <param name="file">BZ File for algoseek</param> /// <param name="date">Reference date of the folder</param> public AlgoSeekOptionsReader(string file, DateTime date, HashSet <string> symbolFilter = null) { _date = date; _underlyingCache = new Dictionary <string, Symbol>(); var streamProvider = StreamProvider.ForExtension(Path.GetExtension(file)); _stream = streamProvider.Open(file).First(); _streamReader = new StreamReader(_stream); _symbolFilter = symbolFilter; var baseinfostreamProvider = StreamProvider.ForExtension(Path.GetExtension("E:\\data\\optiondata\\optionbaseinfo.csv")); var baseinfostream = baseinfostreamProvider.Open("E:\\data\\optiondata\\optionbaseinfo.csv").First(); var basestreamReader = new StreamReader(baseinfostream); var baseline = basestreamReader.ReadLine(); decimal strike = 2.2m; var underlying = "510050"; var optionRight = OptionRight.Call; var expiry = new DateTime(2015, 3, 25); while ((baseline = basestreamReader.ReadLine()) != null) { baseline = baseline.Replace("\"", ""); var csv = baseline.ToCsv(); if (csv[1] == Path.GetFileNameWithoutExtension(file).Substring(2, 8)) { strike = csv[12].ToDecimal(); underlying = csv[8]; optionRight = csv[11] == "CO" ? OptionRight.Call : OptionRight.Put; expiry = DateTime.ParseExact(csv[18], "yyyy-MM-dd", null); break; } } var optionStyle = OptionStyle.European; // couldn't see this specified in the file, maybe need a reference file if (!_underlyingCache.ContainsKey(underlying)) { _symbol = Symbol.CreateOption(underlying, Market.SSE, optionStyle, optionRight, strike, expiry, null, false); _underlyingCache[underlying] = _symbol.Underlying; } else { _symbol = Symbol.CreateOption(_underlyingCache[underlying], Market.SSE, optionStyle, optionRight, strike, expiry); } // detecting column order in the file var headerLine = _streamReader.ReadLine(); if (!string.IsNullOrEmpty(headerLine)) { var header = headerLine.ToCsv(); _columnTime = header.FindIndex(x => x == "Time"); _columnPrice = header.FindIndex(x => x == "Price"); _columnVolume = header.FindIndex(x => x == "Volume"); _columnAmount = header.FindIndex(x => x == "Amount"); _columnOpenInt = header.FindIndex(x => x == "OpenInt"); _columnTotalVol = header.FindIndex(x => x == "TotalVol"); _columnTotalAmount = header.FindIndex(x => x == "TotalAmount"); _columnLastClose = header.FindIndex(x => x == "LastClose"); _columnOpen = header.FindIndex(x => x == "Open"); _columnHigh = header.FindIndex(x => x == "High"); _columnLow = header.FindIndex(x => x == "Low"); _columnSP1 = header.FindIndex(x => x == "SP1"); _columnSV1 = header.FindIndex(x => x == "SV1"); _columnBP1 = header.FindIndex(x => x == "BP1"); _columnBV1 = header.FindIndex(x => x == "BV1"); _columnsCount = Enumerable.Max(new[] { _columnTime, _columnPrice, _columnVolume, _columnAmount, _columnOpenInt, _columnTotalVol, _columnTotalAmount, _columnLastClose, _columnOpen, _columnHigh, _columnLow, _columnSP1, _columnSV1, _columnBP1, _columnBV1 }); } //Prime the data pump, set the current. Current = null; MoveNext(); }
public static void IVolatilityEquityConverter(string sourceDirectory, string sourceMetaDirectory, string destinationDirectory, string resolution) { Console.WriteLine("QuantConnect.ToolBox: IVolatility Converter: "); Console.WriteLine("=============================================="); Console.WriteLine( "The IVolatility converter transforms IVolatility orders into the LEAN Algorithmic Trading Engine Data Format."); Console.WriteLine("Parameters required: --source-dir= --source-meta-dir= --destination-dir= --resolution="); Console.WriteLine(" 1> Source archived IVolatility data."); Console.WriteLine(" 2> Source archived IVolatility meta data."); Console.WriteLine(" 3> Destination Directory of LEAN Data Folder. (Typically located under Lean/Data)"); Console.WriteLine(" 4> Resolution of your IVolatility data. (min,hour,day)"); Console.WriteLine(" "); Console.WriteLine("NOTE: THIS WILL OVERWRITE ANY EXISTING FILES."); if (!(sourceDirectory.IsNullOrEmpty() || sourceMetaDirectory.IsNullOrEmpty() || destinationDirectory.IsNullOrEmpty() || resolution.IsNullOrEmpty())) { _sourceDirectory = sourceDirectory; _sourceMetaDirectory = sourceMetaDirectory; _destinationDirectory = destinationDirectory; _resolution = ParseResolution(resolution); } else { Console.WriteLine("1. Ivolatility equity data source directory: "); _sourceDirectory = (Console.ReadLine() ?? ""); Console.WriteLine("2. Ivolatility equity data source directory: "); _sourceMetaDirectory = (Console.ReadLine() ?? ""); Console.WriteLine("3. Destination LEAN Data directory: "); _destinationDirectory = (Console.ReadLine() ?? ""); Console.WriteLine("4. Enter resolution of source data (Minute, Hour, Daily): "); _resolution = ParseResolution(Console.ReadLine() ?? ""); } //Count the total files to process: Console.WriteLine("Counting Files..." + _sourceDirectory); var count = 1; var files = GetFiles(_sourceDirectory); Console.WriteLine("Processing {0} Files ...", files.Length); foreach (var file in files) { Console.WriteLine("Processing {0} of {1} files ...", count, files.Length); var symbol = GetSymbol(file); var streamProvider = StreamProvider.ForExtension(Path.GetExtension(file)); var inputStream = streamProvider.Open(file).First(); var streamReader = new StreamReader(inputStream); IList <TradeBar> fileTradeBars = new List <TradeBar>(); // IList<QuoteBar> fileQuoteBars = new List<QuoteBar>(); var tradeDataWriter = new LeanDataWriter(_resolution, symbol, _destinationDirectory); // var quoteDataWriter = new LeanDataWriter(_resolution, symbol, _destinationDirectory, TickType.Quote); string line; while ((line = streamReader.ReadLine()) != null) { var linearray = ParseCsv(line); if (linearray.Length <= 2) { continue; } var time = DateTime.ParseExact(linearray[TimeField], DateFormat.UI, CultureInfo.InvariantCulture); var priceBid = Decimal.Parse(linearray[PriceBidField]); var sizeBid = Decimal.Parse(linearray[SizeBidField]); var priceAsk = Decimal.Parse(linearray[PriceAskField]); var sizeAsk = Decimal.Parse(linearray[SizeAskField]); var priceLast = Decimal.Parse(linearray[PriceLastField]); //var sizeLast = Decimal.Parse(linearray[sizeLastField]); var volume = Decimal.Parse(linearray[VolumeField]); var tradeBar = new TradeBar(time, symbol, priceLast, priceLast, priceLast, priceLast, 0); tradeBar.Update(priceLast, priceBid, priceAsk, volume, sizeBid, sizeAsk); var bidBar = new Bar(priceBid, priceBid, priceBid, priceBid); var askBar = new Bar(priceAsk, priceAsk, priceAsk, priceAsk); var quoteBar = new QuoteBar(time, symbol, bidBar, sizeBid, askBar, sizeAsk, TimeSpan.FromSeconds(60)); fileTradeBars.Add(tradeBar); // fileQuoteBars.Add(quoteBar); } tradeDataWriter.Write(fileTradeBars); //TODO: implement quote bars in LeanDataWriter for minute equity resolution // quoteDataWriter.Write(fileQuoteBars); count++; } }
static void Main() { var directory = "F:/Downloads/AlgoSeek/smaller"; directory = "F:/AlgoSeek/20151224"; var dataDirectory = "./Data"; Log.LogHandler = new CompositeLogHandler(new ILogHandler[] { new ConsoleLogHandler(), new FileLogHandler(LogFilePath) }); // first process tick/second/minute -- we'll do hour/daily at the end on a per symbol basis var parallelism = 4; var options = new ParallelOptions { MaxDegreeOfParallelism = parallelism }; var resolutions = new[] { Resolution.Tick, Resolution.Minute, Resolution.Second, Resolution.Hour, Resolution.Daily }; // for testing only process the smallest 2 files var files = Directory.EnumerateFiles(directory).OrderByDescending(x => new FileInfo(x).Length); Parallel.ForEach(files, options, file => { Log.Trace("Begin tick/second/minute/hour/daily: " + file); var quotes = DataProcessor.Zip(dataDirectory, resolutions, TickType.Quote, true); var trades = DataProcessor.Zip(dataDirectory, resolutions, TickType.Trade, true); var streamProvider = StreamProvider.ForExtension(Path.GetExtension(file)); if (!RawFileProcessor.Run(file, new[] { file }, streamProvider, new AlgoSeekOptionsParser(), quotes, trades)) { return; } Log.Trace("Completed tick/second/minute/hour/daily: " + file); }); Log.Trace("Begin compressing csv files"); var root = Path.Combine(dataDirectory, "option", "usa"); var fine = from res in new[] { Resolution.Tick, Resolution.Second, Resolution.Minute } let path = Path.Combine(root, res.ToLower()) from sym in Directory.EnumerateDirectories(path) from dir in Directory.EnumerateDirectories(sym) select new DirectoryInfo(dir).FullName; var coarse = from res in new[] { Resolution.Hour, Resolution.Daily } let path = Path.Combine(root, res.ToLower()) from dir in Directory.EnumerateDirectories(path) select new DirectoryInfo(dir).FullName; var all = fine.Union(coarse); Parallel.ForEach(all, dir => { try { // zip the contents of the directory and then delete the directory Compression.ZipDirectory(dir, dir + ".zip", false); Directory.Delete(dir, true); } catch (Exception err) { Log.Error(err, "Zipping " + dir); } }); Log.Trace("Finished processing directory: " + directory); }