/// <summary> /// Reads the specified <paramref name="source"/> /// </summary> /// <param name="source">The source to be read</param> /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns> public IEnumerable <BaseData> Read(SubscriptionDataSource source) { List <string> entryNames; try { var stream = _dataProvider.Fetch(source.Source); if (stream == null) { OnInvalidSource(source, new ArgumentException($"Failed to create source stream {source.Source}")); yield break; } entryNames = Compression.GetZipEntryFileNames(stream).ToList(); stream.DisposeSafely(); } catch (ZipException err) { OnInvalidSource(source, err); yield break; } foreach (var entryFileName in entryNames) { var instance = _factory.Reader(_config, entryFileName, _date, _isLiveMode); if (instance != null && instance.EndTime != default(DateTime)) { yield return(instance); } } }
/// <summary> /// Processes an order cancelled event /// </summary> /// <param name="order">Order that has been cancelled</param> public static void OrderCancelled(Order order) { order.Notes.Add(new OrderNote(order.OrderId, order.UserId, LocaleHelper.LocalNow, Properties.Resources.OrderCancelled, NoteType.SystemPrivate)); order.Notes.Save(); UpdateOrderStatus(StoreEvent.OrderCancelled, order); //if this is a google checkout order update google checkout if (!string.IsNullOrEmpty(order.GoogleOrderNumber)) { GoogleCheckout instance = GoogleCheckout.GetInstance(); string comment = order.GetLastPublicNote(); if (string.IsNullOrEmpty(comment)) { comment = "N/A"; } instance.CancelOrder(order.GoogleOrderNumber, comment); } // DELETE ALL ASSOCIATED SUBSCRIPTIONS SubscriptionDataSource.DeleteForOrder(order.OrderId); Hashtable parameters = new Hashtable(); parameters["order"] = order; parameters["customer"] = order.User; parameters["payments"] = order.Payments; ProcessEmails(StoreEvent.OrderCancelled, parameters); }
/// <summary> /// Reads the specified <paramref name="source"/> /// </summary> /// <param name="source">The source to be read</param> /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns> public IEnumerable <BaseData> Read(SubscriptionDataSource source) { using (var reader = CreateStreamReader(source)) { // if the reader doesn't have data then we're done with this subscription if (reader == null || reader.EndOfStream) { OnCreateStreamReaderError(_date, source); yield break; } // while the reader has data while (!reader.EndOfStream) { // read a line and pass it to the base data factory var line = reader.ReadLine(); BaseData instance = null; try { instance = _factory.Reader(_config, line, _date, _isLiveMode); } catch (Exception err) { OnReaderError(line, err); } if (instance != null) { yield return(instance); } } } }
/// <summary> /// Load subscriptions in the id list /// </summary> /// <param name="idList">List of subscription ids</param> private void ParseSubscriptionIdList(string idList) { // VALIDATE THE INPUT if (string.IsNullOrEmpty(idList)) { throw new ArgumentNullException("idList"); } if (!Regex.IsMatch(idList, "^\\d+(,\\d+)*$")) { throw new ArgumentException("Id list can only be a comma delimited list of integer.", "idList"); } // PARSE THE LIST OF INTEGERS if (idList.Contains(",")) { ICriteria criteria = NHibernateHelper.CreateCriteria <Subscription>(); criteria.Add(Restrictions.In("Id", AlwaysConvert.ToIntArray(idList))); _SubscriptionList = SubscriptionDataSource.LoadForCriteria(criteria); } else { ICriteria criteria = NHibernateHelper.CreateCriteria <Subscription>(); criteria.Add(Restrictions.Eq("Id", AlwaysConvert.ToInt(idList))); _SubscriptionList = SubscriptionDataSource.LoadForCriteria(criteria); } }
public void ComparesNotEqualWithDifferentTransportMedium() { var one = new SubscriptionDataSource("source", SubscriptionTransportMedium.LocalFile); var two = new SubscriptionDataSource("source", SubscriptionTransportMedium.RemoteFile); Assert.IsTrue(one != two); Assert.IsTrue(!one.Equals(two)); }
/// <summary> /// Reads the specified <paramref name="source"/> /// </summary> /// <param name="source">The source to be read</param> /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns> public IEnumerable<BaseData> Read(SubscriptionDataSource source) { using (var reader = CreateStreamReader(source)) { // if the reader doesn't have data then we're done with this subscription if (reader == null || reader.EndOfStream) { OnCreateStreamReaderError(_date, source); yield break; } // while the reader has data while (!reader.EndOfStream) { // read a line and pass it to the base data factory var line = reader.ReadLine(); BaseData instance = null; try { instance = _factory.Reader(_config, line, _date, _isLiveMode); } catch (Exception err) { OnReaderError(line, err); } if (instance != null) { yield return instance; } } } }
/// <summary> /// Reads the specified <paramref name="source"/> /// </summary> /// <param name="source">The source to be read</param> /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns> public IEnumerable <BaseData> Read(SubscriptionDataSource source) { ICollection <string> entryNames; try { using (var zip = new ZipFile(source.Source)) { entryNames = zip.EntryFileNames; } } catch (ZipException err) { OnInvalidSource(source, err); yield break; } foreach (var entryFileName in entryNames) { var instance = _factory.Reader(_config, entryFileName, _date, _isLiveMode); if (instance != null && instance.EndTime != default(DateTime)) { yield return(instance); } } }
public void ComparesEqualWithIdenticalSourceAndTransportMedium() { var one = new SubscriptionDataSource("source", SubscriptionTransportMedium.LocalFile); var two = new SubscriptionDataSource("source", SubscriptionTransportMedium.LocalFile); Assert.IsTrue(one == two); Assert.IsTrue(one.Equals(two)); }
/// <summary> /// Opens up an IStreamReader for a remote file source /// </summary> private IStreamReader HandleRemoteSourceFile(SubscriptionDataSource source) { // clean old files out of the cache if (!Directory.Exists(Globals.Cache)) { Directory.CreateDirectory(Globals.Cache); } foreach (var file in Directory.EnumerateFiles(Globals.Cache)) { if (File.GetCreationTime(file) < DateTime.Now.AddHours(-24)) { File.Delete(file); } } try { // this will fire up a web client in order to download the 'source' file to the cache return(new RemoteFileSubscriptionStreamReader(source.Source, Globals.Cache)); } catch (Exception err) { OnInvalidSource(source, err); return(null); } }
private void ExportSubscriptions(bool all, bool perUser) { List <int> selectedSubscriptions = new List <int>(); if (all) { int subscriptionPlanId = AlwaysConvert.ToInt(SubscriptionPlan.SelectedValue); IList <Subscription> subscriptions = SubscriptionDataSource.Search(subscriptionPlanId, OrderNumber.Text, string.Empty, FirstName.Text, LastName.Text, Email.Text, ExpirationStart.SelectedDate, ExpirationEnd.SelectedDate, BitFieldState.Any); foreach (Subscription subscription in subscriptions) { selectedSubscriptions.Add(subscription.Id); } } else { List <DataKey> selectedItems = GetSelectedItems(); foreach (DataKey item in selectedItems) { selectedSubscriptions.Add((int)item.Value); } } if (selectedSubscriptions.Count <= 0) { return; } Session["Subscriptions_To_Export"] = selectedSubscriptions; StringBuilder exportJS = new StringBuilder(); exportJS.AppendLine("var iframe = document.createElement(\"iframe\");"); exportJS.AppendLine("iframe.src = \"ExportSubscriptions.ashx?PerUser="******"\";"); exportJS.AppendLine("iframe.style.display = \"none\";"); exportJS.AppendLine("document.body.appendChild(iframe);"); ScriptManager.RegisterClientScriptBlock(SubscriptionsAjax, typeof(UpdatePanel), "downloadexportfile", exportJS.ToString(), true); }
private ISubscriptionDataSourceReader CreateSubscriptionFactory(SubscriptionDataSource source, BaseData baseDataInstance) { var factory = SubscriptionDataSourceReader.ForSource(source, _dataCacheProvider, _config, _tradeableDates.Current, _isLiveMode, baseDataInstance); AttachEventHandlers(factory, source); return(factory); }
protected void Page_Init(object sender, EventArgs e) { _subscriptionId = AlwaysConvert.ToInt(Request.QueryString["SubscriptionId"]); _subscription = SubscriptionDataSource.Load(_subscriptionId); CountryCode.DataSource = CountryDataSource.LoadAll("Name"); CountryCode.DataBind(); //INIT ADDRESS Address address = _subscription.GetShippingAddress(); FirstName.Text = address.FirstName; LastName.Text = address.LastName; Company.Text = address.Company; Address1.Text = address.Address1; Address2.Text = address.Address2; City.Text = address.City; Province.Text = address.Province; PostalCode.Text = address.PostalCode; ListItem selectedCountry = CountryCode.Items.FindByValue(AbleContext.Current.Store.DefaultWarehouse.CountryCode); if (!String.IsNullOrEmpty(address.CountryCode)) { selectedCountry = CountryCode.Items.FindByValue(address.CountryCode.ToString()); } if (selectedCountry != null) { CountryCode.SelectedIndex = CountryCode.Items.IndexOf(selectedCountry); } Phone.Text = address.Phone; Fax.Text = address.Fax; Residence.SelectedIndex = (address.Residence ? 0 : 1); }
private ISubscriptionFactory CreateSubscriptionFactory(SubscriptionDataSource source) { var factory = SubscriptionFactory.ForSource(source, _config, _tradeableDates.Current, _isLiveMode); AttachEventHandlers(factory, source); return(factory); }
protected int CountExpiringSubscriptions(Object dataItem, int expDays) { SubscriptionPlan subscriptionPlan = (SubscriptionPlan)dataItem; DateTime expDateEnd = GetEndOfDay(LocaleHelper.LocalNow.AddDays(expDays)); return(SubscriptionDataSource.SearchCount(subscriptionPlan.Product.Id, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, DateTime.MinValue, expDateEnd, BitFieldState.Any)); }
protected void SubscriptionGrid_RowUpdating(object sender, GridViewUpdateEventArgs e) { GridViewRow row = SubscriptionGrid.Rows[e.RowIndex]; int subscriptionId = (int)SubscriptionGrid.DataKeys[e.RowIndex].Value; Subscription subscription = SubscriptionDataSource.Load(subscriptionId); if (subscription != null) { DropDownList AutoDeliveryInterval = row.FindControl("AutoDeliveryInterval") as DropDownList; if (AutoDeliveryInterval != null && !string.IsNullOrEmpty(AutoDeliveryInterval.SelectedValue)) { subscription.PaymentFrequency = AlwaysConvert.ToInt16(AutoDeliveryInterval.SelectedValue); subscription.RecalculateNextOrderDueDate(); subscription.RecalculateExpiration(); try { EmailProcessor.NotifySubscriptionUpdated(subscription); } catch (Exception ex) { Logger.Error("Error sending subscription updated email.", ex); } subscription.Save(); } } SubscriptionGrid.EditIndex = -1; e.Cancel = true; }
protected void SubscriptionGrid_RowCommand(object sender, GridViewCommandEventArgs e) { int subscriptionId = AlwaysConvert.ToInt(e.CommandArgument); Subscription subscription = SubscriptionDataSource.Load(subscriptionId); switch (e.CommandName) { case "Activate": subscription.Activate(); SubscriptionGrid.DataBind(); break; case "Deactivate": subscription.Deactivate(); SubscriptionGrid.DataBind(); break; case "CancelSubscription": subscription.Delete(); SubscriptionGrid.DataBind(); break; } BindSubscriptions(); }
private void BindSubscriptions() { IList <Subscription> subscriptions = new List <Subscription>(); if (_Order.IsSubscriptionGenerated) { IList <SubscriptionOrder> subOrders = SubscriptionOrderDataSource.LoadForOrder(_Order.Id); foreach (SubscriptionOrder subOrder in subOrders) { if (subOrder.Subscription != null) { subscriptions.Add(subOrder.Subscription); } } SubscriptionGrid.DataSource = subscriptions; SubscriptionGrid.DataBind(); } else { subscriptions = SubscriptionDataSource.LoadForOrder(_Order.Id); SubscriptionGrid.DataSource = subscriptions; SubscriptionGrid.DataBind(); } }
protected void Page_Load(object sender, EventArgs e) { if (!Page.IsPostBack) { BackButton.NavigateUrl = "ViewOrder.aspx?OrderNumber=" + Order.OrderNumber.ToString(); } int count = GetPendingTransactionCount(); PendingPaymentsPH.Visible = (count > 0); if (PendingPaymentsPH.Visible) { PendingPaymentsLabel.Text = string.Format(PendingPaymentsLabel.Text, count); } // ACTIVE GIFTCERTIFICATES int gcCount = 0; foreach (GiftCertificate gc in Order.GiftCertificates) { if (gc.SerialNumber != null && gc.SerialNumber.Length > 0) { gcCount++; } } PHActiveGCs.Visible = (gcCount > 0); if (PHActiveGCs.Visible) { ActiveGCsLabel.Text = string.Format(ActiveGCsLabel.Text, gcCount); } // ACTIVE DIGITAL GOODS int dgCount = 0; foreach (OrderItem oi in Order.Items) { foreach (OrderItemDigitalGood oidg in oi.DigitalGoods) { if (oidg.IsActivated()) { dgCount++; } } } PHActiveDGs.Visible = (dgCount > 0); if (PHActiveDGs.Visible) { ActiveDGsLabel.Text = string.Format(ActiveDGsLabel.Text, dgCount); } // DEACTIVATE ALL ASSOCIATED SUBSCRIPTIONS int subCount = SubscriptionDataSource.CountActiveForOrder(_Order.Id); PHActiveSubscriptions.Visible = (subCount > 0); if (PHActiveSubscriptions.Visible) { ActiveSubscriptionsLabel.Text = string.Format(ActiveSubscriptionsLabel.Text, subCount); } }
/// <summary> /// Return the URL string source of the file. This will be converted to a stream /// </summary> /// <param name="config">Configuration object</param> /// <param name="date">Date of this source file</param> /// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param> /// <returns> /// String URL of source file. /// </returns> /// <remarks> /// Given Intrinio's API limits, we cannot make more than one CSV request per second. That's why in backtesting mode /// we make sure we make just one call to retrieve all the data needed. Also, to avoid the problem of many sources /// asking the data at the beginning of the algorithm, a pause of a second is added. /// </remarks> public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode) { // If the user and the password is not set then... if (!IntrinioConfig.IsInitialized) { // ... try to get the user and password from the config file. var user = Config.Get("intrinio-username"); var password = Config.Get("intrinio-password"); IntrinioConfig.SetUserAndPassword(user, password); // If the user and password aren't available in the config file, then throw error. if (!IntrinioConfig.IsInitialized) { throw new InvalidOperationException("Please set a valid Intrinio user and password using the 'IntrinioEconomicData.SetUserAndPassword' static method. " + "For local backtesting, the user and password can be set in the 'config.json' file."); } } SubscriptionDataSource subscription; var intrinioApiCallLimit = 1000 - (int)(DateTime.Now - LastIntrinoAPICall).TotalMilliseconds; // We want to make just one call with all the data in backtesting mode. // Also we want to make one call per second becasue of the API limit. if (_firstTime || intrinioApiCallLimit > 0) { var order = string.Empty; if (isLiveMode) { // In Live mode, we only want the last observation. order = "desc"; } else { // In backtesting mode, we need the data in ascending order... order = "asc"; // and avoid making furthers calls for the same data. _firstTime = false; } var item = GetStringForDataTransformation(_dataTransformation); var url = $"{_baseUrl}identifier={config.Symbol.Value}&item={item}&sort_order={order}"; var byteKey = Encoding.ASCII.GetBytes($"{IntrinioConfig.User}:{IntrinioConfig.Password}"); var authorizationHeaders = new List <KeyValuePair <string, string> > { new KeyValuePair <string, string>("Authorization", $"Basic ({Convert.ToBase64String(byteKey)})") }; subscription = new SubscriptionDataSource(url, SubscriptionTransportMedium.RemoteFile, FileFormat.Csv, authorizationHeaders); LastIntrinoAPICall = DateTime.Now; } else { subscription = new SubscriptionDataSource("", SubscriptionTransportMedium.LocalFile); } return(subscription); }
/// <summary> /// Event invocator for the <see cref="CreateStreamReaderError"/> event /// </summary> /// <param name="date">The date of the source</param> /// <param name="source">The source that caused the error</param> private void OnCreateStreamReaderError(DateTime date, SubscriptionDataSource source) { var handler = CreateStreamReaderError; if (handler != null) { handler(this, new CreateStreamReaderErrorEventArgs(date, source)); } }
/// <summary> /// Event invocator for the <see cref="InvalidSource"/> event /// </summary> /// <param name="source">The <see cref="SubscriptionDataSource"/> that was invalid</param> /// <param name="exception">The exception if one was raised, otherwise null</param> private void OnInvalidSource(SubscriptionDataSource source, Exception exception) { var handler = InvalidSource; if (handler != null) { handler(this, new InvalidSourceEventArgs(source, exception)); } }
/// <summary> /// Get fundamental data from given symbols /// </summary> /// <param name="pyObject">The symbols to retrieve fundamental data for</param> /// <param name="selector">Selects a value from the Fundamental data to filter the request output</param> /// <param name="start">The start date of selected data</param> /// <param name="end">The end date of selected data</param> /// <returns></returns> public PyObject GetFundamental(PyObject tickers, string selector, DateTime?start = null, DateTime?end = null) { if (string.IsNullOrWhiteSpace(selector)) { return("Invalid selector. Cannot be None, empty or consist only of white-space characters".ToPython()); } using (Py.GIL()) { // If tickers are not a PyList, we create one if (!PyList.IsListType(tickers)) { var tmp = new PyList(); tmp.Append(tickers); tickers = tmp; } var list = new List <Tuple <Symbol, DateTime, object> >(); foreach (var ticker in tickers) { var symbol = QuantConnect.Symbol.Create(ticker.ToString(), SecurityType.Equity, Market.USA); var dir = new DirectoryInfo(Path.Combine(Globals.DataFolder, "equity", symbol.ID.Market, "fundamental", "fine", symbol.Value.ToLowerInvariant())); if (!dir.Exists) { continue; } var config = new SubscriptionDataConfig(typeof(FineFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false); foreach (var fileName in dir.EnumerateFiles()) { var date = DateTime.ParseExact(fileName.Name.Substring(0, 8), DateFormat.EightCharacter, CultureInfo.InvariantCulture); if (date < start || date > end) { continue; } var factory = new TextSubscriptionDataSourceReader(_dataCacheProvider, config, date, false); var source = new SubscriptionDataSource(fileName.FullName, SubscriptionTransportMedium.LocalFile); var value = factory.Read(source).Select(x => GetPropertyValue(x, selector)).First(); list.Add(Tuple.Create(symbol, date, value)); } } var data = new PyDict(); foreach (var item in list.GroupBy(x => x.Item1)) { var index = item.Select(x => x.Item2); data.SetItem(item.Key, _pandas.Series(item.Select(x => x.Item3).ToList(), index)); } return(_pandas.DataFrame(data)); } }
/// <summary> /// Reads the specified <paramref name="source"/> /// </summary> /// <param name="source">The source to be read</param> /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns> public IEnumerable <BaseData> Read(SubscriptionDataSource source) { SubscriptionDataSourceReader.CheckRemoteFileCache(); IStreamReader reader = null; try { switch (source.TransportMedium) { default: case SubscriptionTransportMedium.Rest: reader = new RestSubscriptionStreamReader(source.Source); break; case SubscriptionTransportMedium.LocalFile: reader = new LocalFileSubscriptionStreamReader(_dataCacheProvider, source.Source); break; case SubscriptionTransportMedium.RemoteFile: reader = new RemoteFileSubscriptionStreamReader(_dataCacheProvider, source.Source, Globals.Cache); break; } var raw = ""; while (!reader.EndOfStream) { BaseDataCollection instances; try { raw = reader.ReadLine(); var result = _factory.Reader(_config, raw, _date, _isLiveMode); instances = result as BaseDataCollection; if (instances == null) { OnInvalidSource(source, new Exception("Reader must generate a BaseDataCollection with the FileFormat.Collection")); continue; } } catch (Exception err) { OnReaderError(raw, err); continue; } yield return(instances); } } finally { if (reader != null) { reader.Dispose(); } } }
/// <summary> /// Reads the specified <paramref name="source"/> /// </summary> /// <param name="source">The source to be read</param> /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns> public override IEnumerable <BaseData> Read(SubscriptionDataSource source) { // handles zip or text files using (var reader = CreateStreamReader(source)) { // if the reader doesn't have data then we're done with this subscription if (reader == null || reader.EndOfStream) { OnInvalidSource(source, new Exception($"The reader was empty for source: ${source.Source}")); yield break; } // while the reader has data while (!reader.EndOfStream) { // read a line and pass it to the base data factory var line = reader.ReadLine(); if (line.IsNullOrEmpty()) { continue; } SubscriptionDataSource dataSource; try { dataSource = _factory.GetSourceForAnIndex(_config, _date, line, IsLiveMode); } catch { OnInvalidSource(source, new Exception("Factory.GetSourceForAnIndex() failed to return a valid source")); yield break; } if (dataSource != null) { var dataReader = SubscriptionDataSourceReader.ForSource( dataSource, DataCacheProvider, _config, _date, IsLiveMode, _factory, _dataProvider); var enumerator = dataReader.Read(dataSource).GetEnumerator(); while (enumerator.MoveNext()) { yield return(enumerator.Current); } enumerator.DisposeSafely(); } } } }
/// <summary> /// Createsa new <see cref="ISubscriptionFactory"/> capable of handling the specified <paramref name="source"/> /// </summary> /// <param name="source">The subscription data source to create a factory for</param> /// <param name="config">The configuration of the subscription</param> /// <param name="date">The date to be processed</param> /// <param name="isLiveMode">True for live mode, false otherwise</param> /// <returns>A new <see cref="ISubscriptionFactory"/> that can read the specified <paramref name="source"/></returns> public static ISubscriptionFactory ForSource(SubscriptionDataSource source, SubscriptionDataConfig config, DateTime date, bool isLiveMode) { switch (source.Format) { case FileFormat.Csv: return new TextSubscriptionFactory(config, date, isLiveMode); default: throw new NotImplementedException("SubscriptionFactory.ForSource(" + source + ") has not been implemented yet."); } }
/// <summary> /// Initializes a new instance of the C2dmConnection class. /// </summary> /// <param name="retries"></param> public C2dmConnection(int retries) { this.numOfRetries = retries; this.sds = new SubscriptionDataSource(); this.dds = new DeviceDataSource(); if (this.authKey == null) { //Uncomment this after registering an account and configuring Google C2DM //this.Login(Username, Password); } }
/// <summary> /// Opens up an IStreamReader for a local file source /// </summary> private IStreamReader HandleLocalFileSource(SubscriptionDataSource source) { if (!File.Exists(source.Source)) { OnInvalidSource(source, new FileNotFoundException("The specified file was not found", source.Source)); return(null); } // handles zip or text files return(new LocalFileSubscriptionStreamReader(source.Source)); }
/// <summary> /// Createsa new <see cref="ISubscriptionFactory"/> capable of handling the specified <paramref name="source"/> /// </summary> /// <param name="source">The subscription data source to create a factory for</param> /// <param name="config">The configuration of the subscription</param> /// <param name="date">The date to be processed</param> /// <param name="isLiveMode">True for live mode, false otherwise</param> /// <returns>A new <see cref="ISubscriptionFactory"/> that can read the specified <paramref name="source"/></returns> public static ISubscriptionFactory ForSource(SubscriptionDataSource source, SubscriptionDataConfig config, DateTime date, bool isLiveMode) { switch (source.Format) { case FileFormat.Csv: return(new TextSubscriptionFactory(config, date, isLiveMode)); default: throw new NotImplementedException("SubscriptionFactory.ForSource(" + source + ") has not been implemented yet."); } }
protected void SubscriptionGrid_RowUpdating(object sender, GridViewUpdateEventArgs e) { int subscriptionId = (int)SubscriptionGrid.DataKeys[e.RowIndex].Value; Subscription subscription = SubscriptionDataSource.Load(subscriptionId); if (subscription != null) { subscription.ExpirationDate = AlwaysConvert.ToDateTime(e.NewValues["ExpirationDate"], DateTime.MinValue); subscription.Save(); } SubscriptionGrid.EditIndex = -1; e.Cancel = true; }
/// <summary> /// Resolves the next enumerator to be used in <see cref="MoveNext"/> /// </summary> private IEnumerator <BaseData> ResolveDataEnumerator(bool endOfEnumerator) { do { // always advance the date enumerator, this function is intended to be // called on date changes, never return null for live mode, we'll always // just keep trying to refresh the subscription DateTime date; if (!TryGetNextDate(out date) && !_isLiveMode) { // if we run out of dates then we're finished with this subscription return(null); } // fetch the new source var newSource = _dataFactory.GetSource(_config, date, _isLiveMode); // check if we should create a new subscription factory var sourceChanged = _source != newSource && newSource.Source != ""; var liveRemoteFile = _isLiveMode && (_source == null || _source.TransportMedium == SubscriptionTransportMedium.RemoteFile); if (sourceChanged || liveRemoteFile) { // dispose of the current enumerator before creating a new one if (_subscriptionFactoryEnumerator != null) { _subscriptionFactoryEnumerator.Dispose(); } // save off for comparison next time _source = newSource; var subscriptionFactory = CreateSubscriptionFactory(newSource); var previousTime = _previous == null ? DateTime.MinValue : _previous.EndTime; return(subscriptionFactory.Read(newSource) // prevent the enumerator from emitting data before the last emitted time .Where(instance => instance != null && previousTime < instance.EndTime) .GetEnumerator()); } // if there's still more in the enumerator and we received the same source from the GetSource call // above, then just keep using the same enumerator as we were before if (!endOfEnumerator) // && !sourceChanged is always true here { return(_subscriptionFactoryEnumerator); } // keep churning until we find a new source or run out of tradeable dates // in live mode tradeable dates won't advance beyond today's date, but // TryGetNextDate will return false if it's already at today }while (true); }
private ISubscriptionFactory HandleCsvFileFormat(SubscriptionDataSource source) { // convert the date to the data time zone var dateInDataTimeZone = _tradeableDates.Current.ConvertTo(_config.ExchangeTimeZone, _config.DataTimeZone).Date; var factory = SubscriptionFactory.ForSource(source, _config, dateInDataTimeZone, _isLiveMode); // handle missing files factory.InvalidSource += (sender, args) => { switch (args.Source.TransportMedium) { case SubscriptionTransportMedium.LocalFile: // the local uri doesn't exist, write an error and return null so we we don't try to get data for today Log.Trace(string.Format("SubscriptionDataReader.GetReader(): Could not find QC Data, skipped: {0}", source)); _resultHandler.SamplePerformance(_tradeableDates.Current, 0); break; case SubscriptionTransportMedium.RemoteFile: _resultHandler.ErrorMessage(string.Format("Error downloading custom data source file, skipped: {0} Error: {1}", source, args.Exception.Message), args.Exception.StackTrace); _resultHandler.SamplePerformance(_tradeableDates.Current.Date, 0); break; case SubscriptionTransportMedium.Rest: break; default: throw new ArgumentOutOfRangeException(); } }; if (factory is TextSubscriptionFactory) { // handle empty files/instantiation errors var textSubscriptionFactory = (TextSubscriptionFactory)factory; textSubscriptionFactory.CreateStreamReaderError += (sender, args) => { Log.Error(string.Format("Failed to get StreamReader for data source({0}), symbol({1}). Skipping date({2}). Reader is null.", args.Source.Source, _mappedSymbol, args.Date.ToShortDateString())); if (_config.IsCustomData) { _resultHandler.ErrorMessage(string.Format("We could not fetch the requested data. This may not be valid data, or a failed download of custom data. Skipping source ({0}).", args.Source.Source)); } }; // handle parser errors textSubscriptionFactory.ReaderError += (sender, args) => { _resultHandler.RuntimeError(string.Format("Error invoking {0} data reader. Line: {1} Error: {2}", _config.Symbol, args.Line, args.Exception.Message), args.Exception.StackTrace); }; } return(factory); }
private ISubscriptionFactory CreateSubscriptionFactory(SubscriptionDataSource source) { switch (source.Format) { case FileFormat.Csv: return(HandleCsvFileFormat(source)); case FileFormat.Binary: throw new NotSupportedException("Binary file format is not supported"); default: throw new ArgumentOutOfRangeException(); } }
/// <summary> /// Reads the specified <paramref name="source"/> /// </summary> /// <param name="source">The source to be read</param> /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns> public IEnumerable<BaseData> Read(SubscriptionDataSource source) { IStreamReader reader = null; var instances = new BaseDataCollection(); try { switch (source.TransportMedium) { default: case SubscriptionTransportMedium.Rest: reader = new RestSubscriptionStreamReader(source.Source); break; case SubscriptionTransportMedium.LocalFile: reader = new LocalFileSubscriptionStreamReader(source.Source); break; case SubscriptionTransportMedium.RemoteFile: reader = new RemoteFileSubscriptionStreamReader(source.Source, Globals.Cache); break; } var raw = ""; try { raw = reader.ReadLine(); var result = _factory.Reader(_config, raw, _date, _isLiveMode); instances = result as BaseDataCollection; if (instances == null) { OnInvalidSource(source, new Exception("Reader must generate a BaseDataCollection with the FileFormat.Collection")); } } catch (Exception err) { OnReaderError(raw, err); } foreach (var instance in instances.Data) { yield return instance; } } finally { if (reader != null) reader.Dispose(); } }
/// <summary> /// Creates a new <see cref="ISubscriptionDataSourceReader"/> capable of handling the specified <paramref name="source"/> /// </summary> /// <param name="source">The subscription data source to create a factory for</param> /// <param name="dataFileProvider">Retrieves files if not found on disk</param> /// <param name="config">The configuration of the subscription</param> /// <param name="date">The date to be processed</param> /// <param name="isLiveMode">True for live mode, false otherwise</param> /// <returns>A new <see cref="ISubscriptionDataSourceReader"/> that can read the specified <paramref name="source"/></returns> public static ISubscriptionDataSourceReader ForSource(SubscriptionDataSource source, IDataFileProvider dataFileProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode) { switch (source.Format) { case FileFormat.Csv: return new TextSubscriptionDataSourceReader(dataFileProvider, config, date, isLiveMode); case FileFormat.Collection: return new CollectionSubscriptionDataSourceReader(config, date, isLiveMode); case FileFormat.ZipEntryName: return new ZipEntryNameSubscriptionDataSourceReader(dataFileProvider, config, date, isLiveMode); default: throw new NotImplementedException("SubscriptionFactory.ForSource(" + source + ") has not been implemented yet."); } }
/// <summary> /// Reads the specified <paramref name="source"/> /// </summary> /// <param name="source">The source to be read</param> /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns> public IEnumerable<BaseData> Read(SubscriptionDataSource source) { if (!File.Exists(source.Source)) { OnInvalidSource(source, new FileNotFoundException("The specified file was not found", source.Source)); } ZipFile zip; try { zip = new ZipFile(source.Source); } catch (ZipException err) { OnInvalidSource(source, err); yield break; } foreach (var entryFileName in zip.EntryFileNames) { yield return _factory.Reader(_config, entryFileName, _dateTime, _isLiveMode); } }
/// <summary> /// Initializes a new instance of the <see cref="CreateStreamReaderErrorEventArgs"/> class /// </summary> /// <param name="date">The date of the source</param> /// <param name="source">The source that cause the error</param> public CreateStreamReaderErrorEventArgs(DateTime date, SubscriptionDataSource source) { Date = date; Source = source; }
/// <summary> /// 2. RETURN THE STRING URL SOURCE LOCATION FOR YOUR DATA: /// This is a powerful and dynamic select source file method. If you have a large dataset, 10+mb we recommend you break it into smaller files. E.g. One zip per year. /// We can accept raw text or ZIP files. We read the file extension to determine if it is a zip file. /// </summary> /// <param name="config">Configuration object</param> /// <param name="date">Date of this source file</param> /// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param> /// <returns>String URL of source file.</returns> public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime currentDate, bool isLiveMode) { if (isLiveMode) { return new SubscriptionDataSource(string.Format(DATA_SOURCE_URI, currentDate.ToShortDateString(), currentDate.ToShortDateString()), SubscriptionTransportMedium.RemoteFile); //return new SubscriptionDataSource("https://www.bitstamp.net/api/ticker/", SubscriptionTransportMedium.Rest); } // OR simply return a fixed small data file. Large files will slow down your backtest FileDataImport fi = new FileDataImport(); string uri = string.Format(DATA_SOURCE_URI, currentDate.ToShortDateString(), currentDate.ToShortDateString()); string logentry = string.Format("{0}:{1}:{2}, {3}", DateTime.Now.Minute, DateTime.Now.Second, DateTime.Now.Millisecond, uri); //urls.Add(logentry); //WriteLog(logentry); var source = new SubscriptionDataSource(uri, SubscriptionTransportMedium.RemoteFile); return source; //return new SubscriptionDataSource(fi._dataSourceUri, SubscriptionTransportMedium.LocalFile); }
/// <summary> /// Opens up an IStreamReader for a local file source /// </summary> private IStreamReader HandleLocalFileSource(SubscriptionDataSource source) { string entryName = null; // default to all entries var file = source.Source; var hashIndex = source.Source.LastIndexOf("#", StringComparison.Ordinal); if (hashIndex != -1) { entryName = source.Source.Substring(hashIndex + 1); file = source.Source.Substring(0, hashIndex); } if (!File.Exists(file)) { OnInvalidSource(source, new FileNotFoundException("The specified file was not found", file)); return null; } // handles zip or text files return new LocalFileSubscriptionStreamReader(file, entryName); }
/// <summary> /// Initializes a new instance of the <see cref="InvalidSourceEventArgs"/> class /// </summary> /// <param name="source">The source that was considered invalid</param> /// <param name="exception">The exception that was encountered</param> public InvalidSourceEventArgs(SubscriptionDataSource source, Exception exception) { Source = source; Exception = exception; }
/// <summary> /// Creates a new <see cref="IStreamReader"/> for the specified <paramref name="subscriptionDataSource"/> /// </summary> /// <param name="subscriptionDataSource">The source to produce an <see cref="IStreamReader"/> for</param> /// <returns>A new instance of <see cref="IStreamReader"/> to read the source, or null if there was an error</returns> private IStreamReader CreateStreamReader(SubscriptionDataSource subscriptionDataSource) { IStreamReader reader; switch (subscriptionDataSource.TransportMedium) { case SubscriptionTransportMedium.LocalFile: reader = HandleLocalFileSource(subscriptionDataSource); break; case SubscriptionTransportMedium.RemoteFile: reader = HandleRemoteSourceFile(subscriptionDataSource); break; case SubscriptionTransportMedium.Rest: reader = new RestSubscriptionStreamReader(subscriptionDataSource.Source); break; default: throw new InvalidEnumArgumentException("Unexpected SubscriptionTransportMedium specified: " + subscriptionDataSource.TransportMedium); } return reader; }
/// <summary> /// Opens up an IStreamReader for a remote file source /// </summary> private IStreamReader HandleRemoteSourceFile(SubscriptionDataSource source) { // clean old files out of the cache if (!Directory.Exists(Constants.Cache)) Directory.CreateDirectory(Constants.Cache); foreach (var file in Directory.EnumerateFiles(Constants.Cache)) { if (File.GetCreationTime(file) < DateTime.Now.AddHours(-24)) File.Delete(file); } try { // this will fire up a web client in order to download the 'source' file to the cache return new RemoteFileSubscriptionStreamReader(source.Source, Constants.Cache); } catch (Exception err) { OnInvalidSource(source, err); return null; } }
/// <summary> /// Opens up an IStreamReader for a local file source /// </summary> private IStreamReader HandleLocalFileSource(SubscriptionDataSource source) { if (!File.Exists(source.Source)) { OnInvalidSource(source, new FileNotFoundException("The specified file was not found", source.Source)); return null; } // handles zip or text files return new LocalFileSubscriptionStreamReader(source.Source); }
/// <summary> /// Event invocator for the <see cref="CreateStreamReaderError"/> event /// </summary> /// <param name="date">The date of the source</param> /// <param name="source">The source that caused the error</param> private void OnCreateStreamReaderError(DateTime date, SubscriptionDataSource source) { var handler = CreateStreamReaderError; if (handler != null) handler(this, new CreateStreamReaderErrorEventArgs(date, source)); }
/// <summary> /// Event invocator for the <see cref="InvalidSource"/> event /// </summary> /// <param name="source">The <see cref="SubscriptionDataSource"/> that was invalid</param> /// <param name="exception">The exception if one was raised, otherwise null</param> private void OnInvalidSource(SubscriptionDataSource source, Exception exception) { var handler = InvalidSource; if (handler != null) handler(this, new InvalidSourceEventArgs(source, exception)); }