/// <summary> /// Runs this instance. /// </summary> /// <returns></returns> public bool Run() { var stopwatch = Stopwatch.StartNew(); var symbolMapper = new CoinApiSymbolMapper(); var success = true; var fileToProcess = _rawDataFolder.EnumerateFiles("*.gz") .DistinctBy( x => { var parts = x.Name.Split('-').Take(2); return(string.Join("-", parts)); } ); Parallel.ForEach(fileToProcess, (file, loopState) => { Log.Trace($"CoinApiDataConverter(): Starting data conversion from source file: {file.Name}..."); try { ProcessEntry(new CoinApiDataReader(symbolMapper), file); } catch (Exception e) { Log.Error(e, $"CoinApiDataConverter(): Error processing entry: {file.Name}"); success = false; loopState.Break(); } } ); Log.Trace($"CoinApiDataConverter(): Finished in {stopwatch.Elapsed}"); return(success); }
public void ReturnsCorrectBrokerageSymbol(string leanTicker, string market, string coinApiSymbolId) { var mapper = new CoinApiSymbolMapper(); var symbol = Symbol.Create(leanTicker, SecurityType.Crypto, market); var symbolId = mapper.GetBrokerageSymbol(symbol); Assert.AreEqual(coinApiSymbolId, symbolId); }
public void ReturnsCorrectLeanSymbol(string coinApiSymbolId, string leanTicker, string market) { var mapper = new CoinApiSymbolMapper(); var symbol = mapper.GetLeanSymbol(coinApiSymbolId, SecurityType.Crypto, string.Empty); Assert.AreEqual(leanTicker, symbol.Value); Assert.AreEqual(SecurityType.Crypto, symbol.ID.SecurityType); Assert.AreEqual(market, symbol.ID.Market); }
public void ReturnsCorrectBrokerageSymbol() { var symbol = Symbol.Create("QTMUSD", SecurityType.Crypto, Market.Bitfinex); var mapper = new CoinApiSymbolMapper(); var symbolId = mapper.GetBrokerageSymbol(symbol); Assert.AreEqual("BITFINEX_SPOT_QTUM_USD", symbolId); }
public void ReturnsCorrectLeanSymbol() { const string symbolId = "BITFINEX_SPOT_ANIO_USD"; var mapper = new CoinApiSymbolMapper(); var symbol = mapper.GetLeanSymbol(symbolId, SecurityType.Crypto, string.Empty); Assert.AreEqual("NIOUSD", symbol.Value); Assert.AreEqual(SecurityType.Crypto, symbol.ID.SecurityType); Assert.AreEqual(Market.Bitfinex, symbol.ID.Market); }
/// <summary> /// Runs this instance. /// </summary> /// <returns></returns> public bool Run() { var stopwatch = Stopwatch.StartNew(); var symbolMapper = new CoinApiSymbolMapper(); var success = true; // There were cases of files with with an extra suffix, following pattern: // <TickType>-<ID>-<Exchange>_SPOT_<BaseCurrency>_<QuoteCurrency>_<ExtraSuffix>.csv.gz // Those cases should be ignored for SPOT prices. var tradesFolder = new DirectoryInfo( Path.Combine( _rawDataFolder.FullName, "trades", _processingDate.ToStringInvariant(DateFormat.EightCharacter))); var quotesFolder = new DirectoryInfo( Path.Combine( _rawDataFolder.FullName, "quotes", _processingDate.ToStringInvariant(DateFormat.EightCharacter))); // Distinct by tick type and first two parts of the raw file name, separated by '-'. // This prevents us from double processing the same ticker twice, in case we're given // two raw data files for the same symbol. Related: https://github.com/QuantConnect/Lean/pull/3262 var apiDataReader = new CoinApiDataReader(symbolMapper); var fileToProcess = tradesFolder.EnumerateFiles("*.gz") .Concat(quotesFolder.EnumerateFiles("*.gz")) .Where(f => f.Name.Contains("SPOT")) .Where(f => f.Name.Split('_').Length == 4) .DistinctBy(x => x.Directory.Parent.Name + apiDataReader.GetCoinApiEntryData(x, _processingDate).Symbol.ID); Parallel.ForEach(fileToProcess, (file, loopState) => { Log.Trace($"CoinApiDataConverter(): Starting data conversion from source file: {file.Name}..."); try { ProcessEntry(apiDataReader, file); } catch (Exception e) { Log.Error(e, $"CoinApiDataConverter(): Error processing entry: {file.Name}"); success = false; loopState.Break(); } } ); Log.Trace($"CoinApiDataConverter(): Finished in {stopwatch.Elapsed}"); return(success); }
/// <summary> /// Runs this instance. /// </summary> /// <returns></returns> public bool Run() { var stopwatch = Stopwatch.StartNew(); var symbolMapper = new CoinApiSymbolMapper(); var success = true; // There were cases of files with with an extra suffix, following pattern: // <TickType>-<ID>-<Exchange>_SPOT_<BaseCurrency>_<QuoteCurrency>_<ExtraSuffix>.csv.gz // Those cases should be ignored for SPOT prices. var tradesFolder = new DirectoryInfo( Path.Combine( _rawDataFolder.FullName, "trades", _processingDate.ToStringInvariant(DateFormat.EightCharacter))); var quotesFolder = new DirectoryInfo( Path.Combine( _rawDataFolder.FullName, "quotes", _processingDate.ToStringInvariant(DateFormat.EightCharacter))); var fileToProcess = tradesFolder.EnumerateFiles("*.gz") .Concat(quotesFolder.EnumerateFiles("*.gz")) .Where(f => f.Name.Contains("SPOT")) .Where(f => f.Name.Split('_').Length == 4) .DistinctBy(x => x.Directory.Parent.Name + x.Name.Split('-')[0]); Parallel.ForEach(fileToProcess, (file, loopState) => { Log.Trace($"CoinApiDataConverter(): Starting data conversion from source file: {file.Name}..."); try { ProcessEntry(new CoinApiDataReader(symbolMapper), file); } catch (Exception e) { Log.Error(e, $"CoinApiDataConverter(): Error processing entry: {file.Name}"); success = false; loopState.Break(); } } ); Log.Trace($"CoinApiDataConverter(): Finished in {stopwatch.Elapsed}"); return(success); }
/// <summary> /// Runs this instance. /// </summary> /// <returns></returns> public bool Run() { var stopwatch = Stopwatch.StartNew(); var symbolMapper = new CoinApiSymbolMapper(); var success = true; // There were cases of files with with an extra suffix, following pattern: // <TickType>-<ID>-<Exchange>_SPOT_<BaseCurrency>_<QuoteCurrency>_<ExtraSuffix>.csv.gz // Those cases should be ignored for SPOT prices. var fileToProcess = _rawDataFolder.EnumerateFiles("*.gz") .Where(f => f.Name.Contains("SPOT")) .Where(f => f.Name.Split('_').Length == 4) .DistinctBy( x => { var parts = x.Name.Split('-').Take(2); return(string.Join("-", parts)); } ); Parallel.ForEach(fileToProcess, (file, loopState) => { Log.Trace($"CoinApiDataConverter(): Starting data conversion from source file: {file.Name}..."); try { ProcessEntry(new CoinApiDataReader(symbolMapper), file); } catch (Exception e) { Log.Error(e, $"CoinApiDataConverter(): Error processing entry: {file.Name}"); success = false; loopState.Break(); } } ); Log.Trace($"CoinApiDataConverter(): Finished in {stopwatch.Elapsed}"); return(success); }
/// <summary> /// Runs this instance. /// </summary> /// <returns></returns> public bool Run() { var stopwatch = Stopwatch.StartNew(); var symbolMapper = new CoinApiSymbolMapper(); var success = true; // There were cases of files with with an extra suffix, following pattern: // <TickType>-<ID>-<Exchange>_SPOT_<BaseCurrency>_<QuoteCurrency>_<ExtraSuffix>.csv.gz // Those cases should be ignored for SPOT prices. var tradesFolder = new DirectoryInfo( Path.Combine( _rawDataFolder.FullName, "trades", _processingDate.ToStringInvariant(DateFormat.EightCharacter))); var quotesFolder = new DirectoryInfo( Path.Combine( _rawDataFolder.FullName, "quotes", _processingDate.ToStringInvariant(DateFormat.EightCharacter))); // Distinct by tick type and first two parts of the raw file name, separated by '-'. // This prevents us from double processing the same ticker twice, in case we're given // two raw data files for the same symbol. Related: https://github.com/QuantConnect/Lean/pull/3262 var apiDataReader = new CoinApiDataReader(symbolMapper); var filesToProcessCandidates = tradesFolder.EnumerateFiles("*.gz") .Concat(quotesFolder.EnumerateFiles("*.gz")) .Where(f => f.Name.Contains("SPOT")) .Where(f => f.Name.Split('_').Length == 4) .ToList(); var filesToProcessKeys = new HashSet <string>(); var filesToProcess = new List <FileInfo>(); foreach (var candidate in filesToProcessCandidates) { try { var key = candidate.Directory.Parent.Name + apiDataReader.GetCoinApiEntryData(candidate, _processingDate).Symbol.ID; if (filesToProcessKeys.Add(key)) { // Separate list from HashSet to preserve ordering of viable candidates filesToProcess.Add(candidate); } } catch (Exception err) { // Most likely the exchange isn't supported. Log exception message to avoid excessive stack trace spamming in console output Log.Error(err.Message); } } Parallel.ForEach(filesToProcess, (file, loopState) => { Log.Trace($"CoinApiDataConverter(): Starting data conversion from source file: {file.Name}..."); try { ProcessEntry(apiDataReader, file); } catch (Exception e) { Log.Error(e, $"CoinApiDataConverter(): Error processing entry: {file.Name}"); success = false; loopState.Break(); } } ); Log.Trace($"CoinApiDataConverter(): Finished in {stopwatch.Elapsed}"); return(success); }