public Calendrier() { this.InitializeComponent(); Task download = DataDownloader.download <Models.Calendrier>(ServerConstants.CALENDRIER_URL["equipe1"], null); download.ContinueWith(result => downloadDone((Task <List <Models.Calendrier> >)result)); }
protected void Page_Load(object sender, EventArgs e) { if (Session["field1"] == null) { Response.Redirect("Login.aspx"); } else { UserName = Session["field1"].ToString(); Session["field2"] = "MyLocationtrue"; vdm = new VehicleDBMgr(); vdm.InitializeDB(); if (!Page.IsPostBack) { if (!Page.IsCallback) { ddwnldr = new DataDownloader(); vdm.InitializeDB(); fillPlant(); fillperson_name(); UpdateLogins(); } } } }
protected void Page_Load(object sender, EventArgs e) { /////////////////.................MyLocation....................//////////////// if (Session["field1"] == null) { Response.Redirect("Login.aspx"); } else { UserName = Session["field1"].ToString(); User_sno = Session["field3"].ToString(); Session["field2"] = "MyLocationtrue"; vdm = new VehicleDBMgr(); vdm.InitializeDB(); if (!Page.IsPostBack) { if (!Page.IsCallback) { // FillplantName(); ddwnldr = new DataDownloader(); vdm.InitializeDB(); // ImageButton1.ImageUrl = "UserImgs/build10.png"; UpdateMyLocationUI(); FillCategoryName(); } } } }
protected void Page_Load(object sender, EventArgs e) { /////////////////.................MyLocation....................//////////////// if (Session["field1"] == null) { Response.Redirect("Login.aspx"); } else { UserName = Session["field1"].ToString(); Session["field2"] = "MyLocationtrue"; vdm = new VehicleDBMgr(); vdm.InitializeDB(); if (!Page.IsPostBack) { if (!Page.IsCallback) { ddwnldr = new DataDownloader(); vdm.InitializeDB(); FillSelectVehicle(); UpdateVehicleGroupData(); } } } }
public static List <PriceCurvesModel> GetNordpoolMarketCurves(DateTime dateTime) { var file = string.Format(@"mcp_data_report_{0}-00_00_00", dateTime.ToString("dd-MM-yyyy")); string filePath = GetAppDataServerPathWith(@"\NordpoolSpot\" + file); var dd = new DataDownloader(); var er = new Utils.ExcelReader(); Stream fs; List <PriceCurvesModel> priceCurvesModels; /* If it doesn't exist, download it */ if (!File.Exists(filePath + ".json")) { var data = dd.DownloadFile(file + ".xls"); fs = new MemoryStream(data); priceCurvesModels = er.ReadNordPoolSpotPriceCurves(fs); fs.Dispose(); Commons.SaveFileAsJson(filePath + ".json", priceCurvesModels); } else { var json = File.ReadAllText(filePath + ".json"); priceCurvesModels = JsonConvert.DeserializeObject <List <PriceCurvesModel> >(json); } return(priceCurvesModels); }
public static void ExcelToJsonSeries(DataItem di) { Func <int, string, string> downloadMethod = (int p1, string p2) => p2.Substring(p1); var er = new Utils.ExcelReader(); var nd = new DataDownloader(); var p = nd.BuildFileName(DataItem.Elspot_Prices, Resolution.Hourly, FromYear(2013), Currency.EUR); var hd = er.ReadNordPoolSpotHistoricalPricesInterop(DataDownloader.SavePath + p, 3); p = nd.BuildFileName(DataItem.Elspot_Prices, Resolution.Hourly, FromYear(2014), Currency.EUR); var hd1 = er.ReadNordPoolSpotHistoricalPricesInterop(DataDownloader.SavePath + p, 3); hd.AddRange(hd1); p = nd.BuildFileName(DataItem.Elspot_Prices, Resolution.Hourly, FromYear(2015), Currency.EUR); var hd2 = er.ReadNordPoolSpotHistoricalPricesInterop(DataDownloader.SavePath + p, 3); hd.AddRange(hd2); // do here some preprocessing?... interpolate missing and ourliers... but not spikes var negvs = hd.Where(x => x.Value < 0).ToList(); var nulls = hd.Where(x => !x.Value.HasValue).ToList(); var name = nd.BuildFileName(DataItem.Elspot_Prices, Resolution.Hourly, Currency.EUR); nd.SaveFileAsJson(GetAppDataServerPath() + name + ".json", hd); }
public MergeController() { _webClient = new WebClient(); _imageRepo = new ImageRepo(); _dataDownloader = new DataDownloader(); _imageMerger = new ImageMerger(); }
protected override void OnContinue() { DataDownloader.DownloadFile(Session.settingsPath); CovidData data = CovidData.LoadJson(); GraphGenerator.GenerateGraph(1200, 1200, data.getNewCasesForDays(14)); EmailSender.sendMail(); timer.Change(3600000, Timeout.Infinite); }
public static void ExcelToJsonSeriesPredictors2(DataItem di) { var er = new Utils.ExcelReader(); Func <string, int, int, List <HistoricalPrice> > excelRead = (string path, int fromColumn, int toColumn) => { var cols = new List <List <HistoricalPrice> >(); for (int i = fromColumn; i <= toColumn; i++) { cols.Add(er.ReadNordPoolSpotHistoricalPricesInterop(DataDownloader.SavePath + path, i)); } //needs to be done, since the sum is not already consistently calculated //assumed same data lengths, ofc for (int i = 0; i < cols[0].Count; i++) { decimal?val = 0m; for (int j = 1; j < cols.Count; j++) { val += cols[j][i].Value; } cols[0][i].Value += val; } return(cols[0]); }; var nd = new DataDownloader(); Func <int, List <HistoricalPrice> > readYear = (int year) => { var p = nd.BuildFileName(di, Resolution.Hourly, FromYear(year)); if (di == DataItem.Consumption_Prognosis) { return(excelRead(p, 3, 7)); } else { return(excelRead(p, 3, 14)); } }; var hd = readYear(2013); hd.AddRange(readYear(2014)); hd.AddRange(readYear(2015)); // do here some preprocessing?... interpolate missing and ourliers... but not spikes var negvs = hd.Where(x => x.Value < 0).ToList(); var nulls = hd.Where(x => !x.Value.HasValue).ToList(); var name = nd.BuildFileName(di, Country.All, Resolution.Hourly); nd.SaveFileAsJson(GetAppDataServerPath() + name + ".json", hd); }
private static void DelayedDataDownload(Object o) { DataDownloader dd = (DataDownloader)o; //update the end date to make sure we inclued the most recent data dd.EndDate = DateTime.UtcNow; //set the from date to 3 times the lookback to take into account weeknds and public holidays dd.GetPriceDataAsync(); }
public void SetUp() { output = Path.Combine(TestContext.CurrentContext.TestDirectory, "out"); if (Directory.Exists(output)) { Directory.Delete(output, true); } instance = new DataDownloader(new NullLoggerFactory()); }
/// <summary> /// Download a text file, and call action() when done.</summary> /// <param name="url">Url of the file to download</param> /// <param name="action">Callback with content in string or error in case of problem, and bool containing download status (true:success).</param> public static void Get(string url, Action <string, bool> action) { if (network_go == null) { network_go = new GameObject("NetworkManager"); network_go.tag = Game.BG_TASKS; } //Use WebClient Class DataDownloader dd = network_go.AddComponent <DataDownloader>(); dd.DownloadAsync(url, action); }
private void button1_Click(object sender, EventArgs e) { var loot = DataDownloader.GetItemsFromLootFarm(lootUrl); var swap = DataDownloader.GetItemsFromSwapGG(swapUrl); var flag = sourcesCB1.SelectedIndex == 0 ? false : true; var dataSource = new TableConverter().ConvertModels(loot, swap, flag); mainData.Init(); mainData.DataSource = dataSource; btnGetData.Enabled = false; }
public void TestDataDownloader() { List <SourceData> data = DataDownloader.DownloadData(); Assert.AreEqual <int>(data.Count, 10); //下面 3 个主要是用来测试 SourceData 各个属性的 get 访问器,没有实际作用。 Assert.IsInstanceOfType(data[0].OnlineChange, typeof(string)); Assert.IsInstanceOfType(data[0].OnlineTime, typeof(DateTime)); Assert.IsInstanceOfType(data[0].OnlineNumber, typeof(string)); Assert.IsTrue(data[0].Equals(data[0])); Assert.IsFalse(data[0].Equals(data[1])); }
public void JSONConvertTest() { ConfigLoader _testloader = new ConfigLoader(); _testloader.Load(); DataDownloader _testDownloader = new DataDownloader(_testloader.classConfigResult); _testDownloader.DataDownload(); List<Dictionary<string, object>> _output = JsonConvert.DeserializeObject<List<Dictionary<string, object>>>(_testDownloader.strDownloadData); Console.WriteLine(_output[253]["經度Lng"] == null); Assert.IsNull(_output[253]["經度Lng"]); }
public void DataDownloadTest() { ConfigLoader _testloader = new ConfigLoader(); _testloader.Load(); DataDownloader _testDownloader = new DataDownloader(_testloader.classConfigResult); _testDownloader.DataDownload(); Assert.IsNotNull(_testDownloader.strDownloadData); Console.WriteLine(_testDownloader.strDownloadData); }
public void LoadCSVTest() { ConfigLoader _testloader = new ConfigLoader(); _testloader.Load(); DataDownloader _testDownloader = new DataDownloader(_testloader.classConfigResult); _testDownloader.DataDownload(); List<Dictionary<string, object>> output = CSVReader.Read(_testDownloader.strDownloadData); Assert.IsTrue(output[5]["經度Lng"].ToString() == ""); Console.WriteLine(output[5]["經度Lng"]); }
public void UpdateElementFromFeatureTest() { DataDownloader target = new DataDownloader(config); // TODO: Initialize to an appropriate value long pageid = pagetotest; // TODO: Initialize to an appropriate value string tablename = "unit_testing"; // TODO: Initialize to an appropriate value string workspacepath = @"C:\Users\trav5516\Sandbox\Output"; IWorkspace workspace; workspace = target.SchemaBuilder(pageid, workspacepath,false); workspace = target.DownloadData(pageid, tablename, workspace,1); DataUpdater updater = new DataUpdater(config); IFeature feature = null; Page schemapage = null; updater.UpdateElementFromFeature(feature, schemapage); Assert.Inconclusive("A method that does not return a value cannot be verified."); }
public override void ViewDidLoad() { base.ViewDidLoad(); // Perform any additional setup after loading the view, typically from a nib. _table = new UITableView(View.Bounds); var tableItems = new string[] { "Downloading.." }; _table.Source = new TableSource(tableItems); View.AddSubview(_table); var url = @"http://partner.market.yandex.ru/pages/help/YML.xml"; var downloader = new DataDownloader(url); downloader.Load(); downloader.TextChanged += OnTextChanged; }
public static async Task Main(string[] args) { NLog.LogManager.LoadConfiguration("nlog.config"); starter = new AutoStarter(ApplicationLogging.LoggerFactory, "Sentiment analysis", args); starter.LoggerFactory.AddNLog(); starter.RegisterCommand <TestingCommand, TestingConfig>("test"); starter.RegisterCommand <TrainCommand, TrainingConfig>("train"); starter.RegisterCommand <BoostrapCommand, BootsrapConfig>("boot"); var configuration = new ConfigurationHandler(); var resourcesPath = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), configuration.GetConfiguration("Resources")); if (Directory.Exists(resourcesPath)) { log.LogInformation("Resources folder {0} found.", resourcesPath); } else { var dataDownloader = new DataDownloader(ApplicationLogging.LoggerFactory); Task download = dataDownloader.DownloadFile(new Uri(configuration.GetConfiguration("dataset")), resourcesPath); await download.ConfigureAwait(false); } #if NET472 var fPreviousExecutionState = NativeMethods.SetThreadExecutionState(NativeMethods.ES_CONTINUOUS | NativeMethods.ES_SYSTEM_REQUIRED); if (fPreviousExecutionState == 0) { log.LogError("SetThreadExecutionState failed."); return; } #endif try { source = new CancellationTokenSource(); task = starter.StartAsync(source.Token); System.Console.WriteLine("Please press CTRL+C to break..."); System.Console.CancelKeyPress += ConsoleOnCancelKeyPress; await starter.Status.LastOrDefaultAsync(); System.Console.WriteLine("Exiting..."); } catch (Exception e) { System.Console.WriteLine(e); } }
public MainWindow() { InitializeComponent(); ResultContainer.Instance.addOutputHandler((s) => text.Text = s + Environment.NewLine + text.Text); startButton.Click += (s, o) => { click(); }; analysisButton.Click += (s, o) => { navigateToAnalysis(); }; downloadButton.Click += (s, o) => { DataDownloader.run(); }; }
public void CSVConvertTest() { ConfigLoader _testloader = new ConfigLoader(); _testloader.Load(); DataDownloader _testDownloader = new DataDownloader(_testloader.classConfigResult); _testDownloader.DataDownload(); ProcessorFactory _testFactory = new ProcessorFactory(_testloader.classConfigResult); CSVProcessor _processor = _testFactory.GetYourProcessor() as CSVProcessor; Way _way = _processor.Process(_testDownloader.strDownloadData , _testloader.classConfigResult); Console.WriteLine(_way.Nodes.Count); Assert.IsNotNull(_way.Nodes); }
//only for consumption and production... public static void ExcelToJsonSeriesPredictors1(DataItem di) { var er = new Utils.ExcelReader(); Func <string, List <HistoricalPrice> > excelRead = (string path) => { var hdx1 = er.ReadNordPoolSpotHistoricalPricesInterop(DataDownloader.SavePath + path, 3); var hdx2 = er.ReadNordPoolSpotHistoricalPricesInterop(DataDownloader.SavePath + path, 4); var hdx3 = er.ReadNordPoolSpotHistoricalPricesInterop(DataDownloader.SavePath + path, 5); var hdx4 = er.ReadNordPoolSpotHistoricalPricesInterop(DataDownloader.SavePath + path, 6); //needs to be done, since the sum is not already consistently calculated for (int i = 0; i < hdx1.Count; i++) { hdx1[i].Value = hdx2[i].Value + hdx3[i].Value + hdx4[i].Value; } return(hdx1); }; var nd = new DataDownloader(); Func <int, List <HistoricalPrice> > readYear = (int year) => { var p = nd.BuildFileName(di, Country.All, Resolution.Hourly, FromYear(year)); return(excelRead(p)); }; var hd = readYear(2013); hd.AddRange(readYear(2014)); hd.AddRange(readYear(2015)); // do here some preprocessing?... interpolate missing and ourliers... but not spikes var negvs = hd.Where(x => x.Value < 0).ToList(); var nulls = hd.Where(x => !x.Value.HasValue).ToList(); var name = nd.BuildFileName(di, Country.All, Resolution.Hourly); nd.SaveFileAsJson(GetAppDataServerPath() + name + ".json", hd); }
/// <summary> /// Инициализирует главную форму приложения и её зависимости. /// </summary> public void Initialize() { var dataDownloaded = new DataDownloader(); var htmlParser = new HtmlParser(); IPageSaver pageSaver; IPageModelReader reader; IResultWatcher watcher; if (bool.Parse(ConfigurationManager.AppSettings["UseSqlite"])) { var dbAccessor = new DbAccessor(); dbAccessor.Initialize(new DbConfiguration { DataSource = ConfigurationManager.AppSettings["SqliteDbPath"] }); pageSaver = new DbPageSaver(dbAccessor); reader = new PageModelFromDbReader(dbAccessor); watcher = new ResultDbWatcher(dbAccessor); } else { var path = ConfigurationManager.AppSettings["JsonDirPath"]; var jsonPageSaver = new JsonPageSaver(); jsonPageSaver.Initialize(path); pageSaver = jsonPageSaver; var jsonConverter = new PageModelFromJsonReader(); jsonConverter.Initialize(path); reader = jsonConverter; var jsonWatcher = new ResultDirectoryWatcher(); jsonWatcher.Initialize(path); watcher = jsonWatcher; } var processor = new RiaPageProcessor(dataDownloaded, htmlParser, pageSaver); MainForm = new MainForm(); MainForm.InjectDependencies(processor, reader, watcher); }
static void Main(string[] args) { // Set the Certificate Policy System.Net.ServicePointManager.CertificatePolicy = new CustomCertificate(); Arguments arguments; // Parse the command line arguments if (!parseCommandLineArguments(args, out arguments)) { Console.Error.WriteLine(("Usage DownloadRepoData <userName> <password> <companyId> <fromDate> <toDate> <outputFileName>")); return; } // Create an instance of the DataDownloader and connect var downloader = new DataDownloader(); downloader.Connect(arguments); //// Download the data downloader.DownloadData(); } // method Main
protected void Page_Load(object sender, EventArgs e) { if (Session["field1"] == null) { Response.Redirect("Login.aspx"); } else { UserName = Session["field1"].ToString(); vdm = new VehicleDBMgr(); vdm.InitializeDB(); if (!this.IsPostBack) { if (!Page.IsCallback) { ddwnldr = new DataDownloader(); ddwnldr.UpdateBranchDetails(UserName); FillRoute(); } } } }
public static void Main() { Console.WriteLine("Enter the path to the directory:"); var path = Console.ReadLine(); var dataDownloaded = new DataDownloader(); var htmlParser = new HtmlParser(); var jsonPageSaver = new JsonPageSaver(); jsonPageSaver.Initialize(path); var processor = new RiaPageProcessor(dataDownloaded, htmlParser, jsonPageSaver); while (true) { try { Console.WriteLine("Enter your link to ria.ru:"); var url = Console.ReadLine(); Console.WriteLine(); Console.WriteLine(); processor.ProcessPage(url); Console.WriteLine("Press Y to continue or N to close: "); Console.WriteLine(); } catch (Exception e) { Console.WriteLine(e.Message); } if (Console.ReadKey(true).Key != ConsoleKey.Y) { break; } } }
protected void Page_Load(object sender, EventArgs e) { vdm = new VehicleDBMgr(); ddwnldr = new DataDownloader(); vdm.InitializeDB(); }
/// <summary> /// Процессор по получению спарсенных данных из ria и их сохранению в файл /// </summary> /// <param name="downloader">Загрузчик данных из интернета.</param> /// <param name="parser">Парсер HTML кода.</param> /// <param name="saver">Интерфейс, предоставляющий метод сохранения модели страницы.</param> public RiaPageProcessor(DataDownloader downloader, HtmlParser parser, IPageSaver saver) { _downloader = downloader; _parser = parser; _saver = saver; }
public void XMLConvertTest() { ConfigLoader _testloader = new ConfigLoader(); _testloader.Load(); DataDownloader _testDownloader = new DataDownloader(_testloader.classConfigResult); _testDownloader.DataDownload(); XmlDocument _document = new XmlDocument(); _document.LoadXml(_testDownloader.strDownloadData); if(_document.FirstChild.NodeType == XmlNodeType.XmlDeclaration) _document.RemoveChild(_document.FirstChild); string json = JsonConvert.SerializeXmlNode(_document.FirstChild , Newtonsoft.Json.Formatting.None , true); Console.WriteLine(json); }
// Use this for initialization void Start() { Downloader = new DataDownloader <MotorsProtocol>(DataUri); }
public void GetData() { var dataDownloader = new DataDownloader(); //dataDownloader.DownloadExcelData(); }
static void TickReceived(UserConfig user, long symbolId, bool isBid, ulong value, DateTime tickTime) { string symbolName = user.Symbols.SymbolName((int)symbolId); //do nothing if this tick is not required if (!priceData.ContainsKey(symbolName)) { return; } string bidText = "Ask"; if (isBid) { bidText = "Bid"; } float val = (float)value / 100000; //display the tick string text = symbolName + " " + tickTime.ToString("dd-MM-yy HH:mm:ss.fff") + " " + val + " " + bidText; WriteConsole(text, 1, false); //get the strategies affected by this tick Strategy[] runStrategies = liveStrategies.Where(x => x.Asset.Name == symbolName).ToArray(); //update the price data Dictionary <int, Bar[]> datasets = priceData[symbolName]; Dictionary <int, int> indicies = barIndices[symbolName]; foreach (KeyValuePair <int, Bar[]> data in datasets) { //on first run we need to initialse a bar but don't set the BidOpen or AskOpen so we know the tick recorded didn't start from the start of the bar if (data.Value[0] == null) { Bar bar = new Bar(); TimeSpan d = TimeSpan.FromMinutes(data.Key); DateTime barDate = (new DateTime((tickTime.Ticks + d.Ticks) / d.Ticks * d.Ticks, tickTime.Kind)).AddMinutes(-data.Key); bar.OpenTime = barDate; data.Value[0] = bar; } //if this is not a new bar just update bar at zero if (tickTime < data.Value[0].OpenTime.AddMinutes(data.Key)) { data.Value[0].Update(val, isBid); } ////////////////////////////////////// //// NEW BAR //////////////////////////////////////// else // this is a new bar { //shift the bar array Array.Copy(data.Value, 0, data.Value, 1, data.Value.Length - 1); //peg the date to the nearest timeframe TimeSpan d = TimeSpan.FromMinutes(data.Key); DateTime barDate = (new DateTime((tickTime.Ticks + d.Ticks) / d.Ticks * d.Ticks, tickTime.Kind)).AddMinutes(-data.Key); //Create the new incomplete bar data.Value[0] = new Bar(barDate, val, isBid); //update the bar index - this would have started with a value of lookback and continues to count up indefinately - mainly used in backtesting indicies[data.Key]++; //get the strategies for this timeframe only Strategy[] tfStrategies = runStrategies.Where(x => x.RequiredData.ContainsKey(data.Key)).ToArray(); //we need to download the historic data to fill the lookback array //Note it is best to do this on the start of a new bar because we can only download full bars from history //Therefore by waiting until a new bar is formed we ensure that the new bar collects all ticks and historic bars are complete if (!assetDetails[symbolName].LookbackDownloaded.ContainsKey(data.Key) || !assetDetails[symbolName].LookbackDownloaded[data.Key]) { //delay by 5 seconds so that enough time has elapsed for the bar to close int lookback = data.Value.Length; DataDownloader dd = new DataDownloader(user, symbolName, data.Key, DateTime.UtcNow.AddMinutes(-lookback * data.Key * 3), DateTime.UtcNow, new PriceDataDownloadedEventHandler(DownloadPriceDataCompleted)); dd.OnError = new ErrorHandler(DisplayError); dd.PostRunStrategies = tfStrategies; Timer t = new Timer(DelayedDataDownload, dd, 5000, -1); } else { //load in the python calculated data if required - only need the most recent bar if (PythonBridge != null) { CalculatePythonData(PythonBridge, symbolName, data.Key, 1); } foreach (Strategy strategy in tfStrategies) { strategy.Run(data.Key, symbolName); } } } } //update the trades with the most recent incomplete bar ie. the current minute bar will have the close prices at the current spot price //This needs to run after the OnBar event because OnBar will process the strategy for the last closed bar then UpdateTrades will execute any //trades that were generated from OnBar foreach (Strategy strategy in runStrategies) { strategy.UpdateTrades(datasets[1][0]); } }