/// <summary>
/// Enumerate through the lines of the algoseek files.
/// </summary>
/// <param name="file">BZ File for algoseek</param>
/// <param name="date">Reference date of the folder</param>
public AlgoSeekOptionsReader(string file, DateTime date, HashSet <string> symbolFilter = null)
{
_date = date;
_underlyingCache = new Dictionary <string, Symbol>();
var streamProvider = StreamProvider.ForExtension(Path.GetExtension(file));
_stream = streamProvider.Open(file).First();
_streamReader = new StreamReader(_stream);
_symbolFilter = symbolFilter;
// detecting column order in the file
var headerLine = _streamReader.ReadLine();
if (!string.IsNullOrEmpty(headerLine))
{
var header = headerLine.ToCsv();
_columnTimestamp = header.FindIndex(x => x == "Timestamp");
_columnTicker = header.FindIndex(x => x == "Ticker");
_columnType = header.FindIndex(x => x == "Type");
_columnSide = header.FindIndex(x => x == "Side");
_columnPutCall = header.FindIndex(x => x == "PutCall");
_columnExpiration = header.FindIndex(x => x == "Expiration");
_columnStrike = header.FindIndex(x => x == "Strike");
_columnQuantity = header.FindIndex(x => x == "Quantity");
_columnPremium = header.FindIndex(x => x == "Premium");
_columnExchange = header.FindIndex(x => x == "Exchange");
_columnsCount = Enumerable.Max(new[] { _columnTimestamp, _columnTicker, _columnType, _columnSide,
_columnPutCall, _columnExpiration, _columnStrike, _columnQuantity, _columnPremium, _columnExchange });
}
//Prime the data pump, set the current.
Current = null;
MoveNext();
}
/// <summary>
/// Enumerate through the lines of the algoseek files.
/// </summary>
/// <param name="file">BZ File for AlgoSeek</param>
/// <param name="symbolMultipliers">Symbol price multiplier</param>
/// <param name="symbolFilter">Symbol filter to apply, if any</param>
public AlgoSeekFuturesReader(string file, Dictionary <string, decimal> symbolMultipliers, HashSet <string> symbolFilter = null)
{
var streamProvider = StreamProvider.ForExtension(Path.GetExtension(file));
_stream = streamProvider.Open(file).First();
_streamReader = new StreamReader(_stream);
_symbolFilter = symbolFilter;
_symbolMultipliers = symbolMultipliers.ToDictionary();
_symbolProperties = SymbolPropertiesDatabase.FromDataFolder();
// detecting column order in the file
var headerLine = _streamReader.ReadLine();
if (!string.IsNullOrEmpty(headerLine))
{
var header = headerLine.ToCsv();
_columnTimestamp = header.FindIndex(x => x == "Timestamp");
_columnTicker = header.FindIndex(x => x == "Ticker");
_columnType = header.FindIndex(x => x == "Type");
_columnSide = header.FindIndex(x => x == "Side");
_columnSecID = header.FindIndex(x => x == "SecurityID");
_columnQuantity = header.FindIndex(x => x == "Quantity");
_columnPrice = header.FindIndex(x => x == "Price");
_columnsCount = new[] { _columnTimestamp, _columnTicker, _columnType, _columnSide, _columnSecID, _columnQuantity, _columnPrice }.Max();
}
//Prime the data pump, set the current.
Current = null;
MoveNext();
}
/// <summary>
/// Converts gzipped backfill data. Reads all *.gz files in the raw directory and attempts to convert them
/// </summary>
public void ConvertHistoricalData()
{
foreach (var archive in _rawSourceDirectory.GetFiles("*.gz", SearchOption.TopDirectoryOnly))
{
Log.Trace($"PsychSignalDataConverter.ConvertBackfill(): Begin converting historical data for file: {archive.FullName}");
using (var archiveStream = StreamProvider.ForExtension(".gz").Open(archive.FullName).First())
{
Convert(archiveStream);
}
Log.Trace($"PsychSignalDataConverter.ConvertBackfill(): Finished converting historical data for file: {archive.FullName}");
}
Dispose();
}
/// <summary>
/// Enumerate through the lines of the algoseek files.
/// </summary>
/// <param name="file">BZ File for algoseek</param>
/// <param name="date">Reference date of the folder</param>
public AlgoSeekStocksReader(string file, DateTime date, HashSet <string> symbolFilter = null)
{
_date = date;
_underlyingCache = new Dictionary <string, Symbol>();
var streamProvider = StreamProvider.ForExtension(Path.GetExtension(file));
_stream = streamProvider.Open(file).First();
_streamReader = new StreamReader(_stream);
_symbolFilter = symbolFilter;
_symbol = Symbol.Create("510050", SecurityType.Equity, Market.SSE);
// detecting column order in the file
var headerLine = _streamReader.ReadLine();
if (!string.IsNullOrEmpty(headerLine))
{
var header = headerLine.ToCsv();
_columnTime = header.FindIndex(x => x == "dt");
_columnPrice = header.FindIndex(x => x == "close");
_columnVolume = header.FindIndex(x => x == "volume");
_columnAmount = header.FindIndex(x => x == "Amount");
_columnOpenInt = header.FindIndex(x => x == "OpenInt");
_columnTotalVol = header.FindIndex(x => x == "TotalVol");
_columnTotalAmount = header.FindIndex(x => x == "TotalAmount");
_columnLastClose = header.FindIndex(x => x == "close");
_columnOpen = header.FindIndex(x => x == "Open");
_columnHigh = header.FindIndex(x => x == "High");
_columnLow = header.FindIndex(x => x == "Low");
_columnSP1 = header.FindIndex(x => x == "SP1");
_columnSV1 = header.FindIndex(x => x == "SV1");
_columnBP1 = header.FindIndex(x => x == "BP1");
_columnBV1 = header.FindIndex(x => x == "BV1");
_columnsCount = Enumerable.Max(new[] { _columnTime, _columnPrice, _columnVolume, _columnAmount,
_columnOpenInt, _columnTotalVol, _columnTotalAmount, _columnLastClose, _columnOpen, _columnHigh,
_columnLow, _columnSP1, _columnSV1, _columnBP1, _columnBV1 });
}
//Prime the data pump, set the current.
Current = null;
MoveNext();
}
/// <summary>
/// Enumerate through the lines of the algoseek files.
/// </summary>
/// <param name="file">BZ File for algoseek</param>
/// <param name="date">Reference date of the folder</param>
public AlgoSeekOptionsReader(string file, DateTime date, HashSet <string> symbolFilter = null)
{
_date = date;
_underlyingCache = new Dictionary <string, Symbol>();
var streamProvider = StreamProvider.ForExtension(Path.GetExtension(file));
_stream = streamProvider.Open(file).First();
_streamReader = new StreamReader(_stream);
_symbolFilter = symbolFilter;
var baseinfostreamProvider = StreamProvider.ForExtension(Path.GetExtension("E:\\data\\optiondata\\optionbaseinfo.csv"));
var baseinfostream = baseinfostreamProvider.Open("E:\\data\\optiondata\\optionbaseinfo.csv").First();
var basestreamReader = new StreamReader(baseinfostream);
var baseline = basestreamReader.ReadLine();
decimal strike = 2.2m;
var underlying = "510050";
var optionRight = OptionRight.Call;
var expiry = new DateTime(2015, 3, 25);
while ((baseline = basestreamReader.ReadLine()) != null)
{
baseline = baseline.Replace("\"", "");
var csv = baseline.ToCsv();
if (csv[1] == Path.GetFileNameWithoutExtension(file).Substring(2, 8))
{
strike = csv[12].ToDecimal();
underlying = csv[8];
optionRight = csv[11] == "CO" ? OptionRight.Call : OptionRight.Put;
expiry = DateTime.ParseExact(csv[18], "yyyy-MM-dd", null);
break;
}
}
var optionStyle = OptionStyle.European; // couldn't see this specified in the file, maybe need a reference file
if (!_underlyingCache.ContainsKey(underlying))
{
_symbol = Symbol.CreateOption(underlying, Market.SSE, optionStyle, optionRight, strike, expiry, null, false);
_underlyingCache[underlying] = _symbol.Underlying;
}
else
{
_symbol = Symbol.CreateOption(_underlyingCache[underlying], Market.SSE, optionStyle, optionRight, strike, expiry);
}
// detecting column order in the file
var headerLine = _streamReader.ReadLine();
if (!string.IsNullOrEmpty(headerLine))
{
var header = headerLine.ToCsv();
_columnTime = header.FindIndex(x => x == "Time");
_columnPrice = header.FindIndex(x => x == "Price");
_columnVolume = header.FindIndex(x => x == "Volume");
_columnAmount = header.FindIndex(x => x == "Amount");
_columnOpenInt = header.FindIndex(x => x == "OpenInt");
_columnTotalVol = header.FindIndex(x => x == "TotalVol");
_columnTotalAmount = header.FindIndex(x => x == "TotalAmount");
_columnLastClose = header.FindIndex(x => x == "LastClose");
_columnOpen = header.FindIndex(x => x == "Open");
_columnHigh = header.FindIndex(x => x == "High");
_columnLow = header.FindIndex(x => x == "Low");
_columnSP1 = header.FindIndex(x => x == "SP1");
_columnSV1 = header.FindIndex(x => x == "SV1");
_columnBP1 = header.FindIndex(x => x == "BP1");
_columnBV1 = header.FindIndex(x => x == "BV1");
_columnsCount = Enumerable.Max(new[] { _columnTime, _columnPrice, _columnVolume, _columnAmount,
_columnOpenInt, _columnTotalVol, _columnTotalAmount, _columnLastClose, _columnOpen, _columnHigh,
_columnLow, _columnSP1, _columnSV1, _columnBP1, _columnBV1 });
}
//Prime the data pump, set the current.
Current = null;
MoveNext();
}
public static void IVolatilityEquityConverter(string sourceDirectory, string sourceMetaDirectory, string destinationDirectory, string resolution)
{
Console.WriteLine("QuantConnect.ToolBox: IVolatility Converter: ");
Console.WriteLine("==============================================");
Console.WriteLine(
"The IVolatility converter transforms IVolatility orders into the LEAN Algorithmic Trading Engine Data Format.");
Console.WriteLine("Parameters required: --source-dir= --source-meta-dir= --destination-dir= --resolution=");
Console.WriteLine(" 1> Source archived IVolatility data.");
Console.WriteLine(" 2> Source archived IVolatility meta data.");
Console.WriteLine(" 3> Destination Directory of LEAN Data Folder. (Typically located under Lean/Data)");
Console.WriteLine(" 4> Resolution of your IVolatility data. (min,hour,day)");
Console.WriteLine(" ");
Console.WriteLine("NOTE: THIS WILL OVERWRITE ANY EXISTING FILES.");
if (!(sourceDirectory.IsNullOrEmpty() || sourceMetaDirectory.IsNullOrEmpty() ||
destinationDirectory.IsNullOrEmpty() || resolution.IsNullOrEmpty()))
{
_sourceDirectory = sourceDirectory;
_sourceMetaDirectory = sourceMetaDirectory;
_destinationDirectory = destinationDirectory;
_resolution = ParseResolution(resolution);
}
else
{
Console.WriteLine("1. Ivolatility equity data source directory: ");
_sourceDirectory = (Console.ReadLine() ?? "");
Console.WriteLine("2. Ivolatility equity data source directory: ");
_sourceMetaDirectory = (Console.ReadLine() ?? "");
Console.WriteLine("3. Destination LEAN Data directory: ");
_destinationDirectory = (Console.ReadLine() ?? "");
Console.WriteLine("4. Enter resolution of source data (Minute, Hour, Daily): ");
_resolution = ParseResolution(Console.ReadLine() ?? "");
}
//Count the total files to process:
Console.WriteLine("Counting Files..." + _sourceDirectory);
var count = 1;
var files = GetFiles(_sourceDirectory);
Console.WriteLine("Processing {0} Files ...", files.Length);
foreach (var file in files)
{
Console.WriteLine("Processing {0} of {1} files ...", count, files.Length);
var symbol = GetSymbol(file);
var streamProvider = StreamProvider.ForExtension(Path.GetExtension(file));
var inputStream = streamProvider.Open(file).First();
var streamReader = new StreamReader(inputStream);
IList <TradeBar> fileTradeBars = new List <TradeBar>();
// IList<QuoteBar> fileQuoteBars = new List<QuoteBar>();
var tradeDataWriter = new LeanDataWriter(_resolution, symbol, _destinationDirectory);
// var quoteDataWriter = new LeanDataWriter(_resolution, symbol, _destinationDirectory, TickType.Quote);
string line;
while ((line = streamReader.ReadLine()) != null)
{
var linearray = ParseCsv(line);
if (linearray.Length <= 2)
{
continue;
}
var time = DateTime.ParseExact(linearray[TimeField], DateFormat.UI,
CultureInfo.InvariantCulture);
var priceBid = Decimal.Parse(linearray[PriceBidField]);
var sizeBid = Decimal.Parse(linearray[SizeBidField]);
var priceAsk = Decimal.Parse(linearray[PriceAskField]);
var sizeAsk = Decimal.Parse(linearray[SizeAskField]);
var priceLast = Decimal.Parse(linearray[PriceLastField]);
//var sizeLast = Decimal.Parse(linearray[sizeLastField]);
var volume = Decimal.Parse(linearray[VolumeField]);
var tradeBar = new TradeBar(time, symbol, priceLast, priceLast, priceLast, priceLast, 0);
tradeBar.Update(priceLast, priceBid, priceAsk, volume, sizeBid, sizeAsk);
var bidBar = new Bar(priceBid, priceBid, priceBid, priceBid);
var askBar = new Bar(priceAsk, priceAsk, priceAsk, priceAsk);
var quoteBar = new QuoteBar(time, symbol, bidBar, sizeBid, askBar, sizeAsk,
TimeSpan.FromSeconds(60));
fileTradeBars.Add(tradeBar);
// fileQuoteBars.Add(quoteBar);
}
tradeDataWriter.Write(fileTradeBars);
//TODO: implement quote bars in LeanDataWriter for minute equity resolution
// quoteDataWriter.Write(fileQuoteBars);
count++;
}
}
static void Main()
{
var directory = "F:/Downloads/AlgoSeek/smaller";
directory = "F:/AlgoSeek/20151224";
var dataDirectory = "./Data";
Log.LogHandler = new CompositeLogHandler(new ILogHandler[]
{
new ConsoleLogHandler(),
new FileLogHandler(LogFilePath)
});
// first process tick/second/minute -- we'll do hour/daily at the end on a per symbol basis
var parallelism = 4;
var options = new ParallelOptions {
MaxDegreeOfParallelism = parallelism
};
var resolutions = new[] { Resolution.Tick, Resolution.Minute, Resolution.Second, Resolution.Hour, Resolution.Daily };
// for testing only process the smallest 2 files
var files = Directory.EnumerateFiles(directory).OrderByDescending(x => new FileInfo(x).Length);
Parallel.ForEach(files, options, file =>
{
Log.Trace("Begin tick/second/minute/hour/daily: " + file);
var quotes = DataProcessor.Zip(dataDirectory, resolutions, TickType.Quote, true);
var trades = DataProcessor.Zip(dataDirectory, resolutions, TickType.Trade, true);
var streamProvider = StreamProvider.ForExtension(Path.GetExtension(file));
if (!RawFileProcessor.Run(file, new[] { file }, streamProvider, new AlgoSeekOptionsParser(), quotes, trades))
{
return;
}
Log.Trace("Completed tick/second/minute/hour/daily: " + file);
});
Log.Trace("Begin compressing csv files");
var root = Path.Combine(dataDirectory, "option", "usa");
var fine =
from res in new[] { Resolution.Tick, Resolution.Second, Resolution.Minute }
let path = Path.Combine(root, res.ToLower())
from sym in Directory.EnumerateDirectories(path)
from dir in Directory.EnumerateDirectories(sym)
select new DirectoryInfo(dir).FullName;
var coarse =
from res in new[] { Resolution.Hour, Resolution.Daily }
let path = Path.Combine(root, res.ToLower())
from dir in Directory.EnumerateDirectories(path)
select new DirectoryInfo(dir).FullName;
var all = fine.Union(coarse);
Parallel.ForEach(all, dir =>
{
try
{
// zip the contents of the directory and then delete the directory
Compression.ZipDirectory(dir, dir + ".zip", false);
Directory.Delete(dir, true);
}
catch (Exception err)
{
Log.Error(err, "Zipping " + dir);
}
});
Log.Trace("Finished processing directory: " + directory);
}
