Beispiel #1
0
        public static void CheckFirstSeenDates(Dictionary <string, Market> markets, ref Dictionary <string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
        {
            log.DoLogInfo("BinanceFutures - Checking first seen dates for " + markets.Count + " markets. This may take a while...");

            int marketsChecked = 0;

            foreach (string key in markets.Keys)
            {
                // Save market info
                MarketInfo marketInfo = null;
                if (marketInfos.ContainsKey(key))
                {
                    marketInfo = marketInfos[key];
                }

                if (marketInfo == null)
                {
                    marketInfo      = new MarketInfo();
                    marketInfo.Name = key;
                    marketInfos.Add(key, marketInfo);
                    marketInfo.FirstSeen = BinanceFutures.GetFirstSeenDate(key, systemConfiguration, log);
                }
                else
                {
                    if (marketInfo.FirstSeen == Constants.confMinDate)
                    {
                        marketInfo.FirstSeen = BinanceFutures.GetFirstSeenDate(key, systemConfiguration, log);
                    }
                }
                marketInfo.LastSeen = DateTime.UtcNow;

                marketsChecked++;

                if ((marketsChecked % 20) == 0)
                {
                    log.DoLogInfo("BinanceFutures - Yes, I am still checking first seen dates... " + marketsChecked + "/" + markets.Count + " markets done...");
                }
            }
        }
Beispiel #2
0
        public static List <string> GetMarketData(string mainMarket, Dictionary <string, MarketInfo> marketInfos, PTMagicConfiguration systemConfiguration, LogHelper log)
        {
            List <string> result = new List <string>();

            string lastMarket = "";

            Newtonsoft.Json.Linq.JObject lastTicker = null;
            try
            {
                string baseUrl = "https://fapi.binance.com/fapi/v1/ticker/24hr";

                log.DoLogInfo("BinanceFutures - Getting market data...");
                Newtonsoft.Json.Linq.JArray jsonArray = GetSimpleJsonArrayFromURL(baseUrl, log);
                if (jsonArray.Count > 0)
                {
                    double mainCurrencyPrice = 1;
                    if (!mainMarket.Equals("USDT", StringComparison.InvariantCultureIgnoreCase))
                    {
                        mainCurrencyPrice = BinanceFutures.GetMainCurrencyPrice(mainMarket, systemConfiguration, log);
                    }

                    log.DoLogInfo("BinanceFutures - Market data received for " + jsonArray.Count.ToString() + " currencies");

                    if (mainCurrencyPrice > 0)
                    {
                        Dictionary <string, Market> markets = new Dictionary <string, Market>();
                        foreach (Newtonsoft.Json.Linq.JObject currencyTicker in jsonArray)
                        {
                            string marketName = currencyTicker["symbol"].ToString();
                            //New variables for filtering out bad markets
                            float marketLastPrice = currencyTicker["lastPrice"].ToObject <float>();
                            float marketVolume    = currencyTicker["volume"].ToObject <float>();
                            if (marketName.EndsWith(mainMarket, StringComparison.InvariantCultureIgnoreCase))
                            {
                                if (marketLastPrice > 0 && marketVolume > 0)
                                {
                                    // Set last values in case any error occurs
                                    lastMarket = marketName;
                                    lastTicker = currencyTicker;

                                    Market market = new Market();
                                    market.Position             = markets.Count + 1;
                                    market.Name                 = marketName;
                                    market.Symbol               = currencyTicker["symbol"].ToString();
                                    market.Price                = SystemHelper.TextToDouble(currencyTicker["lastPrice"].ToString(), 0, "en-US");
                                    market.Volume24h            = SystemHelper.TextToDouble(currencyTicker["quoteVolume"].ToString(), 0, "en-US");
                                    market.MainCurrencyPriceUSD = mainCurrencyPrice;

                                    markets.Add(market.Name, market);

                                    result.Add(market.Name);
                                }
                                else
                                {
                                    //Let the user know that the problem market was ignored.
                                    log.DoLogInfo("BinanceFutures - Ignoring bad market data for " + marketName);
                                }
                            }
                        }

                        BinanceFutures.CheckFirstSeenDates(markets, ref marketInfos, systemConfiguration, log);

                        BaseAnalyzer.SaveMarketInfosToFile(marketInfos, systemConfiguration, log);

                        BinanceFutures.CheckForMarketDataRecreation(mainMarket, markets, systemConfiguration, log);

                        DateTime fileDateTime = DateTime.UtcNow;

                        FileHelper.WriteTextToFile(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathExchange + Path.DirectorySeparatorChar, "MarketData_" + fileDateTime.ToString("yyyy-MM-dd_HH.mm") + ".json", JsonConvert.SerializeObject(markets), fileDateTime, fileDateTime);

                        log.DoLogInfo("BinanceFutures - Market data saved for " + markets.Count.ToString() + " markets with " + mainMarket + ".");

                        FileHelper.CleanupFiles(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathExchange + Path.DirectorySeparatorChar, systemConfiguration.AnalyzerSettings.MarketAnalyzer.StoreDataMaxHours);
                        log.DoLogInfo("BinanceFutures - Market data cleaned.");
                    }
                    else
                    {
                        log.DoLogError("BinanceFutures - Failed to get main market price for " + mainMarket + ".");
                        result = null;
                    }
                }
            }
            catch (WebException ex)
            {
                if (ex.Response != null)
                {
                    using (HttpWebResponse errorResponse = (HttpWebResponse)ex.Response)
                    {
                        using (StreamReader reader = new StreamReader(errorResponse.GetResponseStream()))
                        {
                            Dictionary <string, string> errorData = JsonConvert.DeserializeObject <Dictionary <string, string> >(reader.ReadToEnd());
                            if (errorData != null)
                            {
                                string errorMessage = "Unable to get data from BinanceFutures with URL '" + errorResponse.ResponseUri + "'!";
                                if (errorData.ContainsKey("code"))
                                {
                                    errorMessage += " - Code: " + errorData["code"];
                                }

                                if (errorData.ContainsKey("msg"))
                                {
                                    errorMessage += " - Message: " + errorData["msg"];
                                }

                                log.DoLogError(errorMessage);
                            }
                        }
                    }
                }
                result = null;
            }
            catch (Exception ex)
            {
                log.DoLogCritical("Exception while getting data for '" + lastMarket + "': " + ex.Message, ex);
                result = null;
            }

            return(result);
        }
Beispiel #3
0
        public static void CheckForMarketDataRecreation(string mainMarket, Dictionary <string, Market> markets, PTMagicConfiguration systemConfiguration, LogHelper log)
        {
            string binanceFuturesDataDirectoryPath = Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathExchange + Path.DirectorySeparatorChar;

            if (!Directory.Exists(binanceFuturesDataDirectoryPath))
            {
                Directory.CreateDirectory(binanceFuturesDataDirectoryPath);
            }

            DirectoryInfo dataDirectory = new DirectoryInfo(binanceFuturesDataDirectoryPath);

            // Check for existing market files
            DateTime        latestMarketDataFileDateTime = Constants.confMinDate;
            List <FileInfo> marketFiles          = dataDirectory.EnumerateFiles("MarketData*").ToList();
            FileInfo        latestMarketDataFile = null;

            if (marketFiles.Count > 0)
            {
                latestMarketDataFile         = marketFiles.OrderByDescending(mdf => mdf.LastWriteTimeUtc).First();
                latestMarketDataFileDateTime = latestMarketDataFile.LastWriteTimeUtc;
            }

            if (latestMarketDataFileDateTime < DateTime.UtcNow.AddMinutes(-20))
            {
                int lastMarketDataAgeInSeconds = (int)Math.Ceiling(DateTime.UtcNow.Subtract(latestMarketDataFileDateTime).TotalSeconds);

                // Go back in time and create market data
                DateTime startDateTime = DateTime.UtcNow;
                DateTime endDateTime   = DateTime.UtcNow.AddHours(-systemConfiguration.AnalyzerSettings.MarketAnalyzer.StoreDataMaxHours);
                if (latestMarketDataFileDateTime != Constants.confMinDate && latestMarketDataFileDateTime > endDateTime)
                {
                    // Existing market files too old => Recreate market data for configured timeframe
                    log.DoLogInfo("BinanceFutures - Recreating market data for " + markets.Count + " markets over " + SystemHelper.GetProperDurationTime(lastMarketDataAgeInSeconds) + ". This may take a while...");
                    endDateTime = latestMarketDataFileDateTime;
                }
                else
                {
                    // No existing market files found => Recreate market data for configured timeframe
                    log.DoLogInfo("BinanceFutures - Recreating market data for " + markets.Count + " markets over " + systemConfiguration.AnalyzerSettings.MarketAnalyzer.StoreDataMaxHours + " hours. This may take a while...");
                }

                int totalTicks = (int)Math.Ceiling(startDateTime.Subtract(endDateTime).TotalMinutes);

                // Get Ticks for main market
                List <MarketTick> mainMarketTicks = new List <MarketTick>();
                if (!mainMarket.Equals("USDT", StringComparison.InvariantCultureIgnoreCase))
                {
                    mainMarketTicks = BinanceFutures.GetMarketTicks(mainMarket + "USDT", totalTicks, systemConfiguration, log);
                }

                // Get Ticks for all markets
                log.DoLogDebug("BinanceFutures - Getting ticks for '" + markets.Count + "' markets");
                ConcurrentDictionary <string, List <MarketTick> > marketTicks = new ConcurrentDictionary <string, List <MarketTick> >();

                int ParallelThrottle = 4;
                if (systemConfiguration.AnalyzerSettings.MarketAnalyzer.StoreDataMaxHours > 200)
                {
                    ParallelThrottle = 2;
                    log.DoLogInfo("----------------------------------------------------------------------------");
                    log.DoLogInfo("StoreDataMaxHours is greater than 200.  Historical data requests will be");
                    log.DoLogInfo("throttled to avoid exceeding exchange data request limits.  This initial ");
                    log.DoLogInfo("run could take more than 30 minutes.  Please go outside for a walk...");
                    log.DoLogInfo("----------------------------------------------------------------------------");
                }

                Parallel.ForEach(markets.Keys,
                                 new ParallelOptions {
                    MaxDegreeOfParallelism = ParallelThrottle
                },
                                 (key) =>
                {
                    if (!marketTicks.TryAdd(key, GetMarketTicks(key, totalTicks, systemConfiguration, log)))
                    {
                        // Failed to add ticks to dictionary
                        throw new Exception("Failed to add ticks for " + key + " to the memory dictionary, results may be incorrectly calculated!");
                    }

                    if ((marketTicks.Count % 10) == 0)
                    {
                        log.DoLogInfo("BinanceFutures - No worries, I am still alive... " + marketTicks.Count + "/" + markets.Count + " markets done...");
                    }
                });

                log.DoLogInfo("BinanceFutures - Ticks completed.");

                log.DoLogInfo("BinanceFutures - Creating initial market data ticks. This may take another while...");

                // Go back in time and create market data
                int completedTicks = 0;
                if (marketTicks.Count > 0)
                {
                    for (DateTime tickTime = startDateTime; tickTime >= endDateTime; tickTime = tickTime.AddMinutes(-1))
                    {
                        completedTicks++;

                        double mainCurrencyPrice = 1;
                        if (mainMarketTicks.Count > 0)
                        {
                            List <MarketTick> mainCurrencyTickRange = mainMarketTicks.FindAll(t => t.Time <= tickTime);
                            if (mainCurrencyTickRange.Count > 0)
                            {
                                MarketTick mainCurrencyTick = mainCurrencyTickRange.OrderByDescending(t => t.Time).First();
                                mainCurrencyPrice = mainCurrencyTick.Price;
                            }
                        }

                        Dictionary <string, Market> tickMarkets = new Dictionary <string, Market>();
                        foreach (string key in markets.Keys)
                        {
                            List <MarketTick> tickRange = marketTicks[key] != null ? marketTicks[key].FindAll(t => t.Time <= tickTime) : new List <MarketTick>();

                            if (tickRange.Count > 0)
                            {
                                MarketTick marketTick = tickRange.OrderByDescending(t => t.Time).First();

                                Market market = new Market();
                                market.Position = markets.Count + 1;
                                market.Name     = key;
                                market.Symbol   = key;
                                market.Price    = marketTick.Price;
                                //market.Volume24h = marketTick.Volume24h;
                                market.MainCurrencyPriceUSD = mainCurrencyPrice;

                                tickMarkets.Add(market.Name, market);
                            }
                        }

                        DateTime fileDateTime = new DateTime(tickTime.ToLocalTime().Year, tickTime.ToLocalTime().Month, tickTime.ToLocalTime().Day, tickTime.ToLocalTime().Hour, tickTime.ToLocalTime().Minute, 0).ToUniversalTime();

                        FileHelper.WriteTextToFile(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + Constants.PTMagicPathData + Path.DirectorySeparatorChar + Constants.PTMagicPathExchange + Path.DirectorySeparatorChar, "MarketData_" + fileDateTime.ToString("yyyy-MM-dd_HH.mm") + ".json", JsonConvert.SerializeObject(tickMarkets), fileDateTime, fileDateTime);

                        log.DoLogDebug("BinanceFutures - Market data saved for tick " + fileDateTime.ToString() + " - MainCurrencyPrice=" + mainCurrencyPrice.ToString("#,#0.00") + " USD.");

                        if ((completedTicks % 100) == 0)
                        {
                            log.DoLogInfo("BinanceFutures - Our magicbots are still at work, hang on... " + completedTicks + "/" + totalTicks + " ticks done...");
                        }
                    }
                }

                log.DoLogInfo("BinanceFutures - Initial market data created. Ready to go!");
            }
        }