public Calendrier()
{
this.InitializeComponent();
Task download = DataDownloader.download <Models.Calendrier>(ServerConstants.CALENDRIER_URL["equipe1"], null);
download.ContinueWith(result => downloadDone((Task <List <Models.Calendrier> >)result));
}
protected void Page_Load(object sender, EventArgs e)
{
if (Session["field1"] == null)
{
Response.Redirect("Login.aspx");
}
else
{
UserName = Session["field1"].ToString();
Session["field2"] = "MyLocationtrue";
vdm = new VehicleDBMgr();
vdm.InitializeDB();
if (!Page.IsPostBack)
{
if (!Page.IsCallback)
{
ddwnldr = new DataDownloader();
vdm.InitializeDB();
fillPlant();
fillperson_name();
UpdateLogins();
}
}
}
}
protected void Page_Load(object sender, EventArgs e)
{
/////////////////.................MyLocation....................////////////////
if (Session["field1"] == null)
{
Response.Redirect("Login.aspx");
}
else
{
UserName = Session["field1"].ToString();
User_sno = Session["field3"].ToString();
Session["field2"] = "MyLocationtrue";
vdm = new VehicleDBMgr();
vdm.InitializeDB();
if (!Page.IsPostBack)
{
if (!Page.IsCallback)
{
// FillplantName();
ddwnldr = new DataDownloader();
vdm.InitializeDB();
// ImageButton1.ImageUrl = "UserImgs/build10.png";
UpdateMyLocationUI();
FillCategoryName();
}
}
}
}
protected void Page_Load(object sender, EventArgs e)
{
/////////////////.................MyLocation....................////////////////
if (Session["field1"] == null)
{
Response.Redirect("Login.aspx");
}
else
{
UserName = Session["field1"].ToString();
Session["field2"] = "MyLocationtrue";
vdm = new VehicleDBMgr();
vdm.InitializeDB();
if (!Page.IsPostBack)
{
if (!Page.IsCallback)
{
ddwnldr = new DataDownloader();
vdm.InitializeDB();
FillSelectVehicle();
UpdateVehicleGroupData();
}
}
}
}
public static List <PriceCurvesModel> GetNordpoolMarketCurves(DateTime dateTime)
{
var file = string.Format(@"mcp_data_report_{0}-00_00_00",
dateTime.ToString("dd-MM-yyyy"));
string filePath = GetAppDataServerPathWith(@"\NordpoolSpot\" + file);
var dd = new DataDownloader();
var er = new Utils.ExcelReader();
Stream fs;
List <PriceCurvesModel> priceCurvesModels;
/* If it doesn't exist, download it */
if (!File.Exists(filePath + ".json"))
{
var data = dd.DownloadFile(file + ".xls");
fs = new MemoryStream(data);
priceCurvesModels = er.ReadNordPoolSpotPriceCurves(fs);
fs.Dispose();
Commons.SaveFileAsJson(filePath + ".json", priceCurvesModels);
}
else
{
var json = File.ReadAllText(filePath + ".json");
priceCurvesModels = JsonConvert.DeserializeObject <List <PriceCurvesModel> >(json);
}
return(priceCurvesModels);
}
public static void ExcelToJsonSeries(DataItem di)
{
Func <int, string, string> downloadMethod = (int p1, string p2) => p2.Substring(p1);
var er = new Utils.ExcelReader();
var nd = new DataDownloader();
var p = nd.BuildFileName(DataItem.Elspot_Prices, Resolution.Hourly, FromYear(2013), Currency.EUR);
var hd = er.ReadNordPoolSpotHistoricalPricesInterop(DataDownloader.SavePath + p, 3);
p = nd.BuildFileName(DataItem.Elspot_Prices, Resolution.Hourly, FromYear(2014), Currency.EUR);
var hd1 = er.ReadNordPoolSpotHistoricalPricesInterop(DataDownloader.SavePath + p, 3);
hd.AddRange(hd1);
p = nd.BuildFileName(DataItem.Elspot_Prices, Resolution.Hourly, FromYear(2015), Currency.EUR);
var hd2 = er.ReadNordPoolSpotHistoricalPricesInterop(DataDownloader.SavePath + p, 3);
hd.AddRange(hd2);
// do here some preprocessing?... interpolate missing and ourliers... but not spikes
var negvs = hd.Where(x => x.Value < 0).ToList();
var nulls = hd.Where(x => !x.Value.HasValue).ToList();
var name = nd.BuildFileName(DataItem.Elspot_Prices, Resolution.Hourly, Currency.EUR);
nd.SaveFileAsJson(GetAppDataServerPath() + name + ".json", hd);
}
public MergeController()
{
_webClient = new WebClient();
_imageRepo = new ImageRepo();
_dataDownloader = new DataDownloader();
_imageMerger = new ImageMerger();
}
protected override void OnContinue()
{
DataDownloader.DownloadFile(Session.settingsPath);
CovidData data = CovidData.LoadJson();
GraphGenerator.GenerateGraph(1200, 1200, data.getNewCasesForDays(14));
EmailSender.sendMail();
timer.Change(3600000, Timeout.Infinite);
}
public static void ExcelToJsonSeriesPredictors2(DataItem di)
{
var er = new Utils.ExcelReader();
Func <string, int, int, List <HistoricalPrice> > excelRead = (string path, int fromColumn, int toColumn) =>
{
var cols = new List <List <HistoricalPrice> >();
for (int i = fromColumn; i <= toColumn; i++)
{
cols.Add(er.ReadNordPoolSpotHistoricalPricesInterop(DataDownloader.SavePath + path, i));
}
//needs to be done, since the sum is not already consistently calculated
//assumed same data lengths, ofc
for (int i = 0; i < cols[0].Count; i++)
{
decimal?val = 0m;
for (int j = 1; j < cols.Count; j++)
{
val += cols[j][i].Value;
}
cols[0][i].Value += val;
}
return(cols[0]);
};
var nd = new DataDownloader();
Func <int, List <HistoricalPrice> > readYear = (int year) =>
{
var p = nd.BuildFileName(di, Resolution.Hourly, FromYear(year));
if (di == DataItem.Consumption_Prognosis)
{
return(excelRead(p, 3, 7));
}
else
{
return(excelRead(p, 3, 14));
}
};
var hd = readYear(2013);
hd.AddRange(readYear(2014));
hd.AddRange(readYear(2015));
// do here some preprocessing?... interpolate missing and ourliers... but not spikes
var negvs = hd.Where(x => x.Value < 0).ToList();
var nulls = hd.Where(x => !x.Value.HasValue).ToList();
var name = nd.BuildFileName(di, Country.All, Resolution.Hourly);
nd.SaveFileAsJson(GetAppDataServerPath() + name + ".json", hd);
}
private static void DelayedDataDownload(Object o)
{
DataDownloader dd = (DataDownloader)o;
//update the end date to make sure we inclued the most recent data
dd.EndDate = DateTime.UtcNow;
//set the from date to 3 times the lookback to take into account weeknds and public holidays
dd.GetPriceDataAsync();
}
public void SetUp()
{
output = Path.Combine(TestContext.CurrentContext.TestDirectory, "out");
if (Directory.Exists(output))
{
Directory.Delete(output, true);
}
instance = new DataDownloader(new NullLoggerFactory());
}
/// <summary>
/// Download a text file, and call action() when done.</summary>
/// <param name="url">Url of the file to download</param>
/// <param name="action">Callback with content in string or error in case of problem, and bool containing download status (true:success).</param>
public static void Get(string url, Action <string, bool> action)
{
if (network_go == null)
{
network_go = new GameObject("NetworkManager");
network_go.tag = Game.BG_TASKS;
}
//Use WebClient Class
DataDownloader dd = network_go.AddComponent <DataDownloader>();
dd.DownloadAsync(url, action);
}
private void button1_Click(object sender, EventArgs e)
{
var loot = DataDownloader.GetItemsFromLootFarm(lootUrl);
var swap = DataDownloader.GetItemsFromSwapGG(swapUrl);
var flag = sourcesCB1.SelectedIndex == 0 ? false : true;
var dataSource = new TableConverter().ConvertModels(loot, swap, flag);
mainData.Init();
mainData.DataSource = dataSource;
btnGetData.Enabled = false;
}
public void TestDataDownloader()
{
List <SourceData> data = DataDownloader.DownloadData();
Assert.AreEqual <int>(data.Count, 10);
//下面 3 个主要是用来测试 SourceData 各个属性的 get 访问器，没有实际作用。
Assert.IsInstanceOfType(data[0].OnlineChange, typeof(string));
Assert.IsInstanceOfType(data[0].OnlineTime, typeof(DateTime));
Assert.IsInstanceOfType(data[0].OnlineNumber, typeof(string));
Assert.IsTrue(data[0].Equals(data[0]));
Assert.IsFalse(data[0].Equals(data[1]));
}
public void JSONConvertTest()
{
ConfigLoader _testloader = new ConfigLoader();
_testloader.Load();
DataDownloader _testDownloader = new DataDownloader(_testloader.classConfigResult);
_testDownloader.DataDownload();
List<Dictionary<string, object>> _output = JsonConvert.DeserializeObject<List<Dictionary<string, object>>>(_testDownloader.strDownloadData);
Console.WriteLine(_output[253]["經度Lng"] == null);
Assert.IsNull(_output[253]["經度Lng"]);
}
public void DataDownloadTest()
{
ConfigLoader _testloader = new ConfigLoader();
_testloader.Load();
DataDownloader _testDownloader = new DataDownloader(_testloader.classConfigResult);
_testDownloader.DataDownload();
Assert.IsNotNull(_testDownloader.strDownloadData);
Console.WriteLine(_testDownloader.strDownloadData);
}
public void LoadCSVTest()
{
ConfigLoader _testloader = new ConfigLoader();
_testloader.Load();
DataDownloader _testDownloader = new DataDownloader(_testloader.classConfigResult);
_testDownloader.DataDownload();
List<Dictionary<string, object>> output = CSVReader.Read(_testDownloader.strDownloadData);
Assert.IsTrue(output[5]["經度Lng"].ToString() == "");
Console.WriteLine(output[5]["經度Lng"]);
}
public void UpdateElementFromFeatureTest()
{
DataDownloader target = new DataDownloader(config); // TODO: Initialize to an appropriate value
long pageid = pagetotest; // TODO: Initialize to an appropriate value
string tablename = "unit_testing"; // TODO: Initialize to an appropriate value
string workspacepath = @"C:\Users\trav5516\Sandbox\Output";
IWorkspace workspace;
workspace = target.SchemaBuilder(pageid, workspacepath,false);
workspace = target.DownloadData(pageid, tablename, workspace,1);
DataUpdater updater = new DataUpdater(config);
IFeature feature = null;
Page schemapage = null;
updater.UpdateElementFromFeature(feature, schemapage);
Assert.Inconclusive("A method that does not return a value cannot be verified.");
}
public override void ViewDidLoad()
{
base.ViewDidLoad();
// Perform any additional setup after loading the view, typically from a nib.
_table = new UITableView(View.Bounds);
var tableItems = new string[] { "Downloading.." };
_table.Source = new TableSource(tableItems);
View.AddSubview(_table);
var url = @"http://partner.market.yandex.ru/pages/help/YML.xml";
var downloader = new DataDownloader(url);
downloader.Load();
downloader.TextChanged += OnTextChanged;
}
public static async Task Main(string[] args)
{
NLog.LogManager.LoadConfiguration("nlog.config");
starter = new AutoStarter(ApplicationLogging.LoggerFactory, "Sentiment analysis", args);
starter.LoggerFactory.AddNLog();
starter.RegisterCommand <TestingCommand, TestingConfig>("test");
starter.RegisterCommand <TrainCommand, TrainingConfig>("train");
starter.RegisterCommand <BoostrapCommand, BootsrapConfig>("boot");
var configuration = new ConfigurationHandler();
var resourcesPath = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), configuration.GetConfiguration("Resources"));
if (Directory.Exists(resourcesPath))
{
log.LogInformation("Resources folder {0} found.", resourcesPath);
}
else
{
var dataDownloader = new DataDownloader(ApplicationLogging.LoggerFactory);
Task download = dataDownloader.DownloadFile(new Uri(configuration.GetConfiguration("dataset")), resourcesPath);
await download.ConfigureAwait(false);
}
#if NET472
var fPreviousExecutionState = NativeMethods.SetThreadExecutionState(NativeMethods.ES_CONTINUOUS | NativeMethods.ES_SYSTEM_REQUIRED);
if (fPreviousExecutionState == 0)
{
log.LogError("SetThreadExecutionState failed.");
return;
}
#endif
try
{
source = new CancellationTokenSource();
task = starter.StartAsync(source.Token);
System.Console.WriteLine("Please press CTRL+C to break...");
System.Console.CancelKeyPress += ConsoleOnCancelKeyPress;
await starter.Status.LastOrDefaultAsync();
System.Console.WriteLine("Exiting...");
}
catch (Exception e)
{
System.Console.WriteLine(e);
}
}
public MainWindow()
{
InitializeComponent();
ResultContainer.Instance.addOutputHandler((s) => text.Text = s + Environment.NewLine + text.Text);
startButton.Click += (s, o) =>
{
click();
};
analysisButton.Click += (s, o) =>
{
navigateToAnalysis();
};
downloadButton.Click += (s, o) =>
{
DataDownloader.run();
};
}
public void CSVConvertTest()
{
ConfigLoader _testloader = new ConfigLoader();
_testloader.Load();
DataDownloader _testDownloader = new DataDownloader(_testloader.classConfigResult);
_testDownloader.DataDownload();
ProcessorFactory _testFactory = new ProcessorFactory(_testloader.classConfigResult);
CSVProcessor _processor = _testFactory.GetYourProcessor() as CSVProcessor;
Way _way = _processor.Process(_testDownloader.strDownloadData , _testloader.classConfigResult);
Console.WriteLine(_way.Nodes.Count);
Assert.IsNotNull(_way.Nodes);
}
//only for consumption and production...
public static void ExcelToJsonSeriesPredictors1(DataItem di)
{
var er = new Utils.ExcelReader();
Func <string, List <HistoricalPrice> > excelRead = (string path) =>
{
var hdx1 = er.ReadNordPoolSpotHistoricalPricesInterop(DataDownloader.SavePath + path, 3);
var hdx2 = er.ReadNordPoolSpotHistoricalPricesInterop(DataDownloader.SavePath + path, 4);
var hdx3 = er.ReadNordPoolSpotHistoricalPricesInterop(DataDownloader.SavePath + path, 5);
var hdx4 = er.ReadNordPoolSpotHistoricalPricesInterop(DataDownloader.SavePath + path, 6);
//needs to be done, since the sum is not already consistently calculated
for (int i = 0; i < hdx1.Count; i++)
{
hdx1[i].Value = hdx2[i].Value + hdx3[i].Value + hdx4[i].Value;
}
return(hdx1);
};
var nd = new DataDownloader();
Func <int, List <HistoricalPrice> > readYear = (int year) =>
{
var p = nd.BuildFileName(di, Country.All, Resolution.Hourly, FromYear(year));
return(excelRead(p));
};
var hd = readYear(2013);
hd.AddRange(readYear(2014));
hd.AddRange(readYear(2015));
// do here some preprocessing?... interpolate missing and ourliers... but not spikes
var negvs = hd.Where(x => x.Value < 0).ToList();
var nulls = hd.Where(x => !x.Value.HasValue).ToList();
var name = nd.BuildFileName(di, Country.All, Resolution.Hourly);
nd.SaveFileAsJson(GetAppDataServerPath() + name + ".json", hd);
}
/// <summary>
/// Инициализирует главную форму приложения и её зависимости.
/// </summary>
public void Initialize()
{
var dataDownloaded = new DataDownloader();
var htmlParser = new HtmlParser();
IPageSaver pageSaver;
IPageModelReader reader;
IResultWatcher watcher;
if (bool.Parse(ConfigurationManager.AppSettings["UseSqlite"]))
{
var dbAccessor = new DbAccessor();
dbAccessor.Initialize(new DbConfiguration {
DataSource = ConfigurationManager.AppSettings["SqliteDbPath"]
});
pageSaver = new DbPageSaver(dbAccessor);
reader = new PageModelFromDbReader(dbAccessor);
watcher = new ResultDbWatcher(dbAccessor);
}
else
{
var path = ConfigurationManager.AppSettings["JsonDirPath"];
var jsonPageSaver = new JsonPageSaver();
jsonPageSaver.Initialize(path);
pageSaver = jsonPageSaver;
var jsonConverter = new PageModelFromJsonReader();
jsonConverter.Initialize(path);
reader = jsonConverter;
var jsonWatcher = new ResultDirectoryWatcher();
jsonWatcher.Initialize(path);
watcher = jsonWatcher;
}
var processor = new RiaPageProcessor(dataDownloaded, htmlParser, pageSaver);
MainForm = new MainForm();
MainForm.InjectDependencies(processor, reader, watcher);
}
static void Main(string[] args)
{
// Set the Certificate Policy
System.Net.ServicePointManager.CertificatePolicy = new CustomCertificate();
Arguments arguments;
// Parse the command line arguments
if (!parseCommandLineArguments(args, out arguments))
{
Console.Error.WriteLine(("Usage DownloadRepoData <userName> <password> <companyId> <fromDate> <toDate> <outputFileName>"));
return;
}
// Create an instance of the DataDownloader and connect
var downloader = new DataDownloader();
downloader.Connect(arguments);
//// Download the data
downloader.DownloadData();
} // method Main
protected void Page_Load(object sender, EventArgs e)
{
if (Session["field1"] == null)
{
Response.Redirect("Login.aspx");
}
else
{
UserName = Session["field1"].ToString();
vdm = new VehicleDBMgr();
vdm.InitializeDB();
if (!this.IsPostBack)
{
if (!Page.IsCallback)
{
ddwnldr = new DataDownloader();
ddwnldr.UpdateBranchDetails(UserName);
FillRoute();
}
}
}
}
public static void Main()
{
Console.WriteLine("Enter the path to the directory:");
var path = Console.ReadLine();
var dataDownloaded = new DataDownloader();
var htmlParser = new HtmlParser();
var jsonPageSaver = new JsonPageSaver();
jsonPageSaver.Initialize(path);
var processor = new RiaPageProcessor(dataDownloaded, htmlParser, jsonPageSaver);
while (true)
{
try
{
Console.WriteLine("Enter your link to ria.ru:");
var url = Console.ReadLine();
Console.WriteLine();
Console.WriteLine();
processor.ProcessPage(url);
Console.WriteLine("Press Y to continue or N to close: ");
Console.WriteLine();
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
if (Console.ReadKey(true).Key != ConsoleKey.Y)
{
break;
}
}
}
protected void Page_Load(object sender, EventArgs e)
{
vdm = new VehicleDBMgr();
ddwnldr = new DataDownloader();
vdm.InitializeDB();
}
/// <summary>
/// Процессор по получению спарсенных данных из ria и их сохранению в файл
/// </summary>
/// <param name="downloader">Загрузчик данных из интернета.</param>
/// <param name="parser">Парсер HTML кода.</param>
/// <param name="saver">Интерфейс, предоставляющий метод сохранения модели страницы.</param>
public RiaPageProcessor(DataDownloader downloader, HtmlParser parser, IPageSaver saver)
{
_downloader = downloader;
_parser = parser;
_saver = saver;
}
public void XMLConvertTest()
{
ConfigLoader _testloader = new ConfigLoader();
_testloader.Load();
DataDownloader _testDownloader = new DataDownloader(_testloader.classConfigResult);
_testDownloader.DataDownload();
XmlDocument _document = new XmlDocument();
_document.LoadXml(_testDownloader.strDownloadData);
if(_document.FirstChild.NodeType == XmlNodeType.XmlDeclaration)
_document.RemoveChild(_document.FirstChild);
string json = JsonConvert.SerializeXmlNode(_document.FirstChild , Newtonsoft.Json.Formatting.None , true);
Console.WriteLine(json);
}
// Use this for initialization
void Start()
{
Downloader = new DataDownloader <MotorsProtocol>(DataUri);
}
public void GetData()
{
var dataDownloader = new DataDownloader();
//dataDownloader.DownloadExcelData();
}
static void TickReceived(UserConfig user, long symbolId, bool isBid, ulong value, DateTime tickTime)
{
string symbolName = user.Symbols.SymbolName((int)symbolId);
//do nothing if this tick is not required
if (!priceData.ContainsKey(symbolName))
{
return;
}
string bidText = "Ask";
if (isBid)
{
bidText = "Bid";
}
float val = (float)value / 100000;
//display the tick
string text = symbolName + " " + tickTime.ToString("dd-MM-yy HH:mm:ss.fff") + " " + val + " " + bidText;
WriteConsole(text, 1, false);
//get the strategies affected by this tick
Strategy[] runStrategies = liveStrategies.Where(x => x.Asset.Name == symbolName).ToArray();
//update the price data
Dictionary <int, Bar[]> datasets = priceData[symbolName];
Dictionary <int, int> indicies = barIndices[symbolName];
foreach (KeyValuePair <int, Bar[]> data in datasets)
{
//on first run we need to initialse a bar but don't set the BidOpen or AskOpen so we know the tick recorded didn't start from the start of the bar
if (data.Value[0] == null)
{
Bar bar = new Bar();
TimeSpan d = TimeSpan.FromMinutes(data.Key);
DateTime barDate = (new DateTime((tickTime.Ticks + d.Ticks) / d.Ticks * d.Ticks, tickTime.Kind)).AddMinutes(-data.Key);
bar.OpenTime = barDate;
data.Value[0] = bar;
}
//if this is not a new bar just update bar at zero
if (tickTime < data.Value[0].OpenTime.AddMinutes(data.Key))
{
data.Value[0].Update(val, isBid);
}
//////////////////////////////////////
//// NEW BAR
////////////////////////////////////////
else // this is a new bar
{
//shift the bar array
Array.Copy(data.Value, 0, data.Value, 1, data.Value.Length - 1);
//peg the date to the nearest timeframe
TimeSpan d = TimeSpan.FromMinutes(data.Key);
DateTime barDate = (new DateTime((tickTime.Ticks + d.Ticks) / d.Ticks * d.Ticks, tickTime.Kind)).AddMinutes(-data.Key);
//Create the new incomplete bar
data.Value[0] = new Bar(barDate, val, isBid);
//update the bar index - this would have started with a value of lookback and continues to count up indefinately - mainly used in backtesting
indicies[data.Key]++;
//get the strategies for this timeframe only
Strategy[] tfStrategies = runStrategies.Where(x => x.RequiredData.ContainsKey(data.Key)).ToArray();
//we need to download the historic data to fill the lookback array
//Note it is best to do this on the start of a new bar because we can only download full bars from history
//Therefore by waiting until a new bar is formed we ensure that the new bar collects all ticks and historic bars are complete
if (!assetDetails[symbolName].LookbackDownloaded.ContainsKey(data.Key) || !assetDetails[symbolName].LookbackDownloaded[data.Key])
{
//delay by 5 seconds so that enough time has elapsed for the bar to close
int lookback = data.Value.Length;
DataDownloader dd = new DataDownloader(user, symbolName, data.Key, DateTime.UtcNow.AddMinutes(-lookback * data.Key * 3), DateTime.UtcNow,
new PriceDataDownloadedEventHandler(DownloadPriceDataCompleted));
dd.OnError = new ErrorHandler(DisplayError);
dd.PostRunStrategies = tfStrategies;
Timer t = new Timer(DelayedDataDownload, dd, 5000, -1);
}
else
{
//load in the python calculated data if required - only need the most recent bar
if (PythonBridge != null)
{
CalculatePythonData(PythonBridge, symbolName, data.Key, 1);
}
foreach (Strategy strategy in tfStrategies)
{
strategy.Run(data.Key, symbolName);
}
}
}
}
//update the trades with the most recent incomplete bar ie. the current minute bar will have the close prices at the current spot price
//This needs to run after the OnBar event because OnBar will process the strategy for the last closed bar then UpdateTrades will execute any
//trades that were generated from OnBar
foreach (Strategy strategy in runStrategies)
{
strategy.UpdateTrades(datasets[1][0]);
}
}
