/// <summary>
/// Reads the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The source to be read</param>
/// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
public IEnumerable <BaseData> Read(SubscriptionDataSource source)
{
List <string> entryNames;
try
{
var stream = _dataProvider.Fetch(source.Source);
if (stream == null)
{
OnInvalidSource(source, new ArgumentException($"Failed to create source stream {source.Source}"));
yield break;
}
entryNames = Compression.GetZipEntryFileNames(stream).ToList();
stream.DisposeSafely();
}
catch (ZipException err)
{
OnInvalidSource(source, err);
yield break;
}
foreach (var entryFileName in entryNames)
{
var instance = _factory.Reader(_config, entryFileName, _date, _isLiveMode);
if (instance != null && instance.EndTime != default(DateTime))
{
yield return(instance);
}
}
}
/// <summary>
/// Processes an order cancelled event
/// </summary>
/// <param name="order">Order that has been cancelled</param>
public static void OrderCancelled(Order order)
{
order.Notes.Add(new OrderNote(order.OrderId, order.UserId, LocaleHelper.LocalNow, Properties.Resources.OrderCancelled, NoteType.SystemPrivate));
order.Notes.Save();
UpdateOrderStatus(StoreEvent.OrderCancelled, order);
//if this is a google checkout order update google checkout
if (!string.IsNullOrEmpty(order.GoogleOrderNumber))
{
GoogleCheckout instance = GoogleCheckout.GetInstance();
string comment = order.GetLastPublicNote();
if (string.IsNullOrEmpty(comment))
{
comment = "N/A";
}
instance.CancelOrder(order.GoogleOrderNumber, comment);
}
// DELETE ALL ASSOCIATED SUBSCRIPTIONS
SubscriptionDataSource.DeleteForOrder(order.OrderId);
Hashtable parameters = new Hashtable();
parameters["order"] = order;
parameters["customer"] = order.User;
parameters["payments"] = order.Payments;
ProcessEmails(StoreEvent.OrderCancelled, parameters);
}
/// <summary>
/// Reads the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The source to be read</param>
/// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
public IEnumerable <BaseData> Read(SubscriptionDataSource source)
{
using (var reader = CreateStreamReader(source))
{
// if the reader doesn't have data then we're done with this subscription
if (reader == null || reader.EndOfStream)
{
OnCreateStreamReaderError(_date, source);
yield break;
}
// while the reader has data
while (!reader.EndOfStream)
{
// read a line and pass it to the base data factory
var line = reader.ReadLine();
BaseData instance = null;
try
{
instance = _factory.Reader(_config, line, _date, _isLiveMode);
}
catch (Exception err)
{
OnReaderError(line, err);
}
if (instance != null)
{
yield return(instance);
}
}
}
}
/// <summary>
/// Load subscriptions in the id list
/// </summary>
/// <param name="idList">List of subscription ids</param>
private void ParseSubscriptionIdList(string idList)
{
// VALIDATE THE INPUT
if (string.IsNullOrEmpty(idList))
{
throw new ArgumentNullException("idList");
}
if (!Regex.IsMatch(idList, "^\\d+(,\\d+)*$"))
{
throw new ArgumentException("Id list can only be a comma delimited list of integer.", "idList");
}
// PARSE THE LIST OF INTEGERS
if (idList.Contains(","))
{
ICriteria criteria = NHibernateHelper.CreateCriteria <Subscription>();
criteria.Add(Restrictions.In("Id", AlwaysConvert.ToIntArray(idList)));
_SubscriptionList = SubscriptionDataSource.LoadForCriteria(criteria);
}
else
{
ICriteria criteria = NHibernateHelper.CreateCriteria <Subscription>();
criteria.Add(Restrictions.Eq("Id", AlwaysConvert.ToInt(idList)));
_SubscriptionList = SubscriptionDataSource.LoadForCriteria(criteria);
}
}
public void ComparesNotEqualWithDifferentTransportMedium()
{
var one = new SubscriptionDataSource("source", SubscriptionTransportMedium.LocalFile);
var two = new SubscriptionDataSource("source", SubscriptionTransportMedium.RemoteFile);
Assert.IsTrue(one != two);
Assert.IsTrue(!one.Equals(two));
}
/// <summary>
/// Reads the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The source to be read</param>
/// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
public IEnumerable<BaseData> Read(SubscriptionDataSource source)
{
using (var reader = CreateStreamReader(source))
{
// if the reader doesn't have data then we're done with this subscription
if (reader == null || reader.EndOfStream)
{
OnCreateStreamReaderError(_date, source);
yield break;
}
// while the reader has data
while (!reader.EndOfStream)
{
// read a line and pass it to the base data factory
var line = reader.ReadLine();
BaseData instance = null;
try
{
instance = _factory.Reader(_config, line, _date, _isLiveMode);
}
catch (Exception err)
{
OnReaderError(line, err);
}
if (instance != null)
{
yield return instance;
}
}
}
}
/// <summary>
/// Reads the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The source to be read</param>
/// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
public IEnumerable <BaseData> Read(SubscriptionDataSource source)
{
ICollection <string> entryNames;
try
{
using (var zip = new ZipFile(source.Source))
{
entryNames = zip.EntryFileNames;
}
}
catch (ZipException err)
{
OnInvalidSource(source, err);
yield break;
}
foreach (var entryFileName in entryNames)
{
var instance = _factory.Reader(_config, entryFileName, _date, _isLiveMode);
if (instance != null && instance.EndTime != default(DateTime))
{
yield return(instance);
}
}
}
public void ComparesEqualWithIdenticalSourceAndTransportMedium()
{
var one = new SubscriptionDataSource("source", SubscriptionTransportMedium.LocalFile);
var two = new SubscriptionDataSource("source", SubscriptionTransportMedium.LocalFile);
Assert.IsTrue(one == two);
Assert.IsTrue(one.Equals(two));
}
/// <summary>
/// Opens up an IStreamReader for a remote file source
/// </summary>
private IStreamReader HandleRemoteSourceFile(SubscriptionDataSource source)
{
// clean old files out of the cache
if (!Directory.Exists(Globals.Cache))
{
Directory.CreateDirectory(Globals.Cache);
}
foreach (var file in Directory.EnumerateFiles(Globals.Cache))
{
if (File.GetCreationTime(file) < DateTime.Now.AddHours(-24))
{
File.Delete(file);
}
}
try
{
// this will fire up a web client in order to download the 'source' file to the cache
return(new RemoteFileSubscriptionStreamReader(source.Source, Globals.Cache));
}
catch (Exception err)
{
OnInvalidSource(source, err);
return(null);
}
}
private void ExportSubscriptions(bool all, bool perUser)
{
List <int> selectedSubscriptions = new List <int>();
if (all)
{
int subscriptionPlanId = AlwaysConvert.ToInt(SubscriptionPlan.SelectedValue);
IList <Subscription> subscriptions = SubscriptionDataSource.Search(subscriptionPlanId, OrderNumber.Text, string.Empty, FirstName.Text, LastName.Text, Email.Text, ExpirationStart.SelectedDate, ExpirationEnd.SelectedDate, BitFieldState.Any);
foreach (Subscription subscription in subscriptions)
{
selectedSubscriptions.Add(subscription.Id);
}
}
else
{
List <DataKey> selectedItems = GetSelectedItems();
foreach (DataKey item in selectedItems)
{
selectedSubscriptions.Add((int)item.Value);
}
}
if (selectedSubscriptions.Count <= 0)
{
return;
}
Session["Subscriptions_To_Export"] = selectedSubscriptions;
StringBuilder exportJS = new StringBuilder();
exportJS.AppendLine("var iframe = document.createElement(\"iframe\");");
exportJS.AppendLine("iframe.src = \"ExportSubscriptions.ashx?PerUser="******"\";");
exportJS.AppendLine("iframe.style.display = \"none\";");
exportJS.AppendLine("document.body.appendChild(iframe);");
ScriptManager.RegisterClientScriptBlock(SubscriptionsAjax, typeof(UpdatePanel), "downloadexportfile", exportJS.ToString(), true);
}
private ISubscriptionDataSourceReader CreateSubscriptionFactory(SubscriptionDataSource source, BaseData baseDataInstance)
{
var factory = SubscriptionDataSourceReader.ForSource(source, _dataCacheProvider, _config, _tradeableDates.Current, _isLiveMode, baseDataInstance);
AttachEventHandlers(factory, source);
return(factory);
}
protected void Page_Init(object sender, EventArgs e)
{
_subscriptionId = AlwaysConvert.ToInt(Request.QueryString["SubscriptionId"]);
_subscription = SubscriptionDataSource.Load(_subscriptionId);
CountryCode.DataSource = CountryDataSource.LoadAll("Name");
CountryCode.DataBind();
//INIT ADDRESS
Address address = _subscription.GetShippingAddress();
FirstName.Text = address.FirstName;
LastName.Text = address.LastName;
Company.Text = address.Company;
Address1.Text = address.Address1;
Address2.Text = address.Address2;
City.Text = address.City;
Province.Text = address.Province;
PostalCode.Text = address.PostalCode;
ListItem selectedCountry = CountryCode.Items.FindByValue(AbleContext.Current.Store.DefaultWarehouse.CountryCode);
if (!String.IsNullOrEmpty(address.CountryCode))
{
selectedCountry = CountryCode.Items.FindByValue(address.CountryCode.ToString());
}
if (selectedCountry != null)
{
CountryCode.SelectedIndex = CountryCode.Items.IndexOf(selectedCountry);
}
Phone.Text = address.Phone;
Fax.Text = address.Fax;
Residence.SelectedIndex = (address.Residence ? 0 : 1);
}
private ISubscriptionFactory CreateSubscriptionFactory(SubscriptionDataSource source)
{
var factory = SubscriptionFactory.ForSource(source, _config, _tradeableDates.Current, _isLiveMode);
AttachEventHandlers(factory, source);
return(factory);
}
protected int CountExpiringSubscriptions(Object dataItem, int expDays)
{
SubscriptionPlan subscriptionPlan = (SubscriptionPlan)dataItem;
DateTime expDateEnd = GetEndOfDay(LocaleHelper.LocalNow.AddDays(expDays));
return(SubscriptionDataSource.SearchCount(subscriptionPlan.Product.Id, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, DateTime.MinValue, expDateEnd, BitFieldState.Any));
}
protected void SubscriptionGrid_RowUpdating(object sender, GridViewUpdateEventArgs e)
{
GridViewRow row = SubscriptionGrid.Rows[e.RowIndex];
int subscriptionId = (int)SubscriptionGrid.DataKeys[e.RowIndex].Value;
Subscription subscription = SubscriptionDataSource.Load(subscriptionId);
if (subscription != null)
{
DropDownList AutoDeliveryInterval = row.FindControl("AutoDeliveryInterval") as DropDownList;
if (AutoDeliveryInterval != null && !string.IsNullOrEmpty(AutoDeliveryInterval.SelectedValue))
{
subscription.PaymentFrequency = AlwaysConvert.ToInt16(AutoDeliveryInterval.SelectedValue);
subscription.RecalculateNextOrderDueDate();
subscription.RecalculateExpiration();
try
{
EmailProcessor.NotifySubscriptionUpdated(subscription);
}
catch (Exception ex)
{
Logger.Error("Error sending subscription updated email.", ex);
}
subscription.Save();
}
}
SubscriptionGrid.EditIndex = -1;
e.Cancel = true;
}
protected void SubscriptionGrid_RowCommand(object sender, GridViewCommandEventArgs e)
{
int subscriptionId = AlwaysConvert.ToInt(e.CommandArgument);
Subscription subscription = SubscriptionDataSource.Load(subscriptionId);
switch (e.CommandName)
{
case "Activate":
subscription.Activate();
SubscriptionGrid.DataBind();
break;
case "Deactivate":
subscription.Deactivate();
SubscriptionGrid.DataBind();
break;
case "CancelSubscription":
subscription.Delete();
SubscriptionGrid.DataBind();
break;
}
BindSubscriptions();
}
private void BindSubscriptions()
{
IList <Subscription> subscriptions = new List <Subscription>();
if (_Order.IsSubscriptionGenerated)
{
IList <SubscriptionOrder> subOrders = SubscriptionOrderDataSource.LoadForOrder(_Order.Id);
foreach (SubscriptionOrder subOrder in subOrders)
{
if (subOrder.Subscription != null)
{
subscriptions.Add(subOrder.Subscription);
}
}
SubscriptionGrid.DataSource = subscriptions;
SubscriptionGrid.DataBind();
}
else
{
subscriptions = SubscriptionDataSource.LoadForOrder(_Order.Id);
SubscriptionGrid.DataSource = subscriptions;
SubscriptionGrid.DataBind();
}
}
protected void Page_Load(object sender, EventArgs e)
{
if (!Page.IsPostBack)
{
BackButton.NavigateUrl = "ViewOrder.aspx?OrderNumber=" + Order.OrderNumber.ToString();
}
int count = GetPendingTransactionCount();
PendingPaymentsPH.Visible = (count > 0);
if (PendingPaymentsPH.Visible)
{
PendingPaymentsLabel.Text = string.Format(PendingPaymentsLabel.Text, count);
}
// ACTIVE GIFTCERTIFICATES
int gcCount = 0;
foreach (GiftCertificate gc in Order.GiftCertificates)
{
if (gc.SerialNumber != null && gc.SerialNumber.Length > 0)
{
gcCount++;
}
}
PHActiveGCs.Visible = (gcCount > 0);
if (PHActiveGCs.Visible)
{
ActiveGCsLabel.Text = string.Format(ActiveGCsLabel.Text, gcCount);
}
// ACTIVE DIGITAL GOODS
int dgCount = 0;
foreach (OrderItem oi in Order.Items)
{
foreach (OrderItemDigitalGood oidg in oi.DigitalGoods)
{
if (oidg.IsActivated())
{
dgCount++;
}
}
}
PHActiveDGs.Visible = (dgCount > 0);
if (PHActiveDGs.Visible)
{
ActiveDGsLabel.Text = string.Format(ActiveDGsLabel.Text, dgCount);
}
// DEACTIVATE ALL ASSOCIATED SUBSCRIPTIONS
int subCount = SubscriptionDataSource.CountActiveForOrder(_Order.Id);
PHActiveSubscriptions.Visible = (subCount > 0);
if (PHActiveSubscriptions.Visible)
{
ActiveSubscriptionsLabel.Text = string.Format(ActiveSubscriptionsLabel.Text, subCount);
}
}
public void ComparesNotEqualWithDifferentTransportMedium()
{
var one = new SubscriptionDataSource("source", SubscriptionTransportMedium.LocalFile);
var two = new SubscriptionDataSource("source", SubscriptionTransportMedium.RemoteFile);
Assert.IsTrue(one != two);
Assert.IsTrue(!one.Equals(two));
}
public void ComparesEqualWithIdenticalSourceAndTransportMedium()
{
var one = new SubscriptionDataSource("source", SubscriptionTransportMedium.LocalFile);
var two = new SubscriptionDataSource("source", SubscriptionTransportMedium.LocalFile);
Assert.IsTrue(one == two);
Assert.IsTrue(one.Equals(two));
}
/// <summary>
/// Return the URL string source of the file. This will be converted to a stream
/// </summary>
/// <param name="config">Configuration object</param>
/// <param name="date">Date of this source file</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns>
/// String URL of source file.
/// </returns>
/// <remarks>
/// Given Intrinio's API limits, we cannot make more than one CSV request per second. That's why in backtesting mode
/// we make sure we make just one call to retrieve all the data needed. Also, to avoid the problem of many sources
/// asking the data at the beginning of the algorithm, a pause of a second is added.
/// </remarks>
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
// If the user and the password is not set then...
if (!IntrinioConfig.IsInitialized)
{
// ... try to get the user and password from the config file.
var user = Config.Get("intrinio-username");
var password = Config.Get("intrinio-password");
IntrinioConfig.SetUserAndPassword(user, password);
// If the user and password aren't available in the config file, then throw error.
if (!IntrinioConfig.IsInitialized)
{
throw new InvalidOperationException("Please set a valid Intrinio user and password using the 'IntrinioEconomicData.SetUserAndPassword' static method. " +
"For local backtesting, the user and password can be set in the 'config.json' file.");
}
}
SubscriptionDataSource subscription;
var intrinioApiCallLimit = 1000 - (int)(DateTime.Now - LastIntrinoAPICall).TotalMilliseconds;
// We want to make just one call with all the data in backtesting mode.
// Also we want to make one call per second becasue of the API limit.
if (_firstTime || intrinioApiCallLimit > 0)
{
var order = string.Empty;
if (isLiveMode)
{
// In Live mode, we only want the last observation.
order = "desc";
}
else
{
// In backtesting mode, we need the data in ascending order...
order = "asc";
// and avoid making furthers calls for the same data.
_firstTime = false;
}
var item = GetStringForDataTransformation(_dataTransformation);
var url = $"{_baseUrl}identifier={config.Symbol.Value}&item={item}&sort_order={order}";
var byteKey = Encoding.ASCII.GetBytes($"{IntrinioConfig.User}:{IntrinioConfig.Password}");
var authorizationHeaders = new List <KeyValuePair <string, string> >
{
new KeyValuePair <string, string>("Authorization",
$"Basic ({Convert.ToBase64String(byteKey)})")
};
subscription = new SubscriptionDataSource(url, SubscriptionTransportMedium.RemoteFile, FileFormat.Csv,
authorizationHeaders);
LastIntrinoAPICall = DateTime.Now;
}
else
{
subscription = new SubscriptionDataSource("", SubscriptionTransportMedium.LocalFile);
}
return(subscription);
}
/// <summary>
/// Event invocator for the <see cref="CreateStreamReaderError"/> event
/// </summary>
/// <param name="date">The date of the source</param>
/// <param name="source">The source that caused the error</param>
private void OnCreateStreamReaderError(DateTime date, SubscriptionDataSource source)
{
var handler = CreateStreamReaderError;
if (handler != null)
{
handler(this, new CreateStreamReaderErrorEventArgs(date, source));
}
}
/// <summary>
/// Event invocator for the <see cref="InvalidSource"/> event
/// </summary>
/// <param name="source">The <see cref="SubscriptionDataSource"/> that was invalid</param>
/// <param name="exception">The exception if one was raised, otherwise null</param>
private void OnInvalidSource(SubscriptionDataSource source, Exception exception)
{
var handler = InvalidSource;
if (handler != null)
{
handler(this, new InvalidSourceEventArgs(source, exception));
}
}
/// <summary>
/// Get fundamental data from given symbols
/// </summary>
/// <param name="pyObject">The symbols to retrieve fundamental data for</param>
/// <param name="selector">Selects a value from the Fundamental data to filter the request output</param>
/// <param name="start">The start date of selected data</param>
/// <param name="end">The end date of selected data</param>
/// <returns></returns>
public PyObject GetFundamental(PyObject tickers, string selector, DateTime?start = null, DateTime?end = null)
{
if (string.IsNullOrWhiteSpace(selector))
{
return("Invalid selector. Cannot be None, empty or consist only of white-space characters".ToPython());
}
using (Py.GIL())
{
// If tickers are not a PyList, we create one
if (!PyList.IsListType(tickers))
{
var tmp = new PyList();
tmp.Append(tickers);
tickers = tmp;
}
var list = new List <Tuple <Symbol, DateTime, object> >();
foreach (var ticker in tickers)
{
var symbol = QuantConnect.Symbol.Create(ticker.ToString(), SecurityType.Equity, Market.USA);
var dir = new DirectoryInfo(Path.Combine(Globals.DataFolder, "equity", symbol.ID.Market, "fundamental", "fine", symbol.Value.ToLowerInvariant()));
if (!dir.Exists)
{
continue;
}
var config = new SubscriptionDataConfig(typeof(FineFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false);
foreach (var fileName in dir.EnumerateFiles())
{
var date = DateTime.ParseExact(fileName.Name.Substring(0, 8), DateFormat.EightCharacter, CultureInfo.InvariantCulture);
if (date < start || date > end)
{
continue;
}
var factory = new TextSubscriptionDataSourceReader(_dataCacheProvider, config, date, false);
var source = new SubscriptionDataSource(fileName.FullName, SubscriptionTransportMedium.LocalFile);
var value = factory.Read(source).Select(x => GetPropertyValue(x, selector)).First();
list.Add(Tuple.Create(symbol, date, value));
}
}
var data = new PyDict();
foreach (var item in list.GroupBy(x => x.Item1))
{
var index = item.Select(x => x.Item2);
data.SetItem(item.Key, _pandas.Series(item.Select(x => x.Item3).ToList(), index));
}
return(_pandas.DataFrame(data));
}
}
/// <summary>
/// Reads the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The source to be read</param>
/// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
public IEnumerable <BaseData> Read(SubscriptionDataSource source)
{
SubscriptionDataSourceReader.CheckRemoteFileCache();
IStreamReader reader = null;
try
{
switch (source.TransportMedium)
{
default:
case SubscriptionTransportMedium.Rest:
reader = new RestSubscriptionStreamReader(source.Source);
break;
case SubscriptionTransportMedium.LocalFile:
reader = new LocalFileSubscriptionStreamReader(_dataCacheProvider, source.Source);
break;
case SubscriptionTransportMedium.RemoteFile:
reader = new RemoteFileSubscriptionStreamReader(_dataCacheProvider, source.Source, Globals.Cache);
break;
}
var raw = "";
while (!reader.EndOfStream)
{
BaseDataCollection instances;
try
{
raw = reader.ReadLine();
var result = _factory.Reader(_config, raw, _date, _isLiveMode);
instances = result as BaseDataCollection;
if (instances == null)
{
OnInvalidSource(source, new Exception("Reader must generate a BaseDataCollection with the FileFormat.Collection"));
continue;
}
}
catch (Exception err)
{
OnReaderError(raw, err);
continue;
}
yield return(instances);
}
}
finally
{
if (reader != null)
{
reader.Dispose();
}
}
}
/// <summary>
/// Reads the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The source to be read</param>
/// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
public override IEnumerable <BaseData> Read(SubscriptionDataSource source)
{
// handles zip or text files
using (var reader = CreateStreamReader(source))
{
// if the reader doesn't have data then we're done with this subscription
if (reader == null || reader.EndOfStream)
{
OnInvalidSource(source, new Exception($"The reader was empty for source: ${source.Source}"));
yield break;
}
// while the reader has data
while (!reader.EndOfStream)
{
// read a line and pass it to the base data factory
var line = reader.ReadLine();
if (line.IsNullOrEmpty())
{
continue;
}
SubscriptionDataSource dataSource;
try
{
dataSource = _factory.GetSourceForAnIndex(_config, _date, line, IsLiveMode);
}
catch
{
OnInvalidSource(source, new Exception("Factory.GetSourceForAnIndex() failed to return a valid source"));
yield break;
}
if (dataSource != null)
{
var dataReader = SubscriptionDataSourceReader.ForSource(
dataSource,
DataCacheProvider,
_config,
_date,
IsLiveMode,
_factory,
_dataProvider);
var enumerator = dataReader.Read(dataSource).GetEnumerator();
while (enumerator.MoveNext())
{
yield return(enumerator.Current);
}
enumerator.DisposeSafely();
}
}
}
}
/// <summary>
/// Createsa new <see cref="ISubscriptionFactory"/> capable of handling the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The subscription data source to create a factory for</param>
/// <param name="config">The configuration of the subscription</param>
/// <param name="date">The date to be processed</param>
/// <param name="isLiveMode">True for live mode, false otherwise</param>
/// <returns>A new <see cref="ISubscriptionFactory"/> that can read the specified <paramref name="source"/></returns>
public static ISubscriptionFactory ForSource(SubscriptionDataSource source, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
switch (source.Format)
{
case FileFormat.Csv:
return new TextSubscriptionFactory(config, date, isLiveMode);
default:
throw new NotImplementedException("SubscriptionFactory.ForSource(" + source + ") has not been implemented yet.");
}
}
/// <summary>
/// Initializes a new instance of the C2dmConnection class.
/// </summary>
/// <param name="retries"></param>
public C2dmConnection(int retries)
{
this.numOfRetries = retries;
this.sds = new SubscriptionDataSource();
this.dds = new DeviceDataSource();
if (this.authKey == null)
{
//Uncomment this after registering an account and configuring Google C2DM
//this.Login(Username, Password);
}
}
/// <summary>
/// Opens up an IStreamReader for a local file source
/// </summary>
private IStreamReader HandleLocalFileSource(SubscriptionDataSource source)
{
if (!File.Exists(source.Source))
{
OnInvalidSource(source, new FileNotFoundException("The specified file was not found", source.Source));
return(null);
}
// handles zip or text files
return(new LocalFileSubscriptionStreamReader(source.Source));
}
/// <summary>
/// Createsa new <see cref="ISubscriptionFactory"/> capable of handling the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The subscription data source to create a factory for</param>
/// <param name="config">The configuration of the subscription</param>
/// <param name="date">The date to be processed</param>
/// <param name="isLiveMode">True for live mode, false otherwise</param>
/// <returns>A new <see cref="ISubscriptionFactory"/> that can read the specified <paramref name="source"/></returns>
public static ISubscriptionFactory ForSource(SubscriptionDataSource source, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
switch (source.Format)
{
case FileFormat.Csv:
return(new TextSubscriptionFactory(config, date, isLiveMode));
default:
throw new NotImplementedException("SubscriptionFactory.ForSource(" + source + ") has not been implemented yet.");
}
}
protected void SubscriptionGrid_RowUpdating(object sender, GridViewUpdateEventArgs e)
{
int subscriptionId = (int)SubscriptionGrid.DataKeys[e.RowIndex].Value;
Subscription subscription = SubscriptionDataSource.Load(subscriptionId);
if (subscription != null)
{
subscription.ExpirationDate = AlwaysConvert.ToDateTime(e.NewValues["ExpirationDate"], DateTime.MinValue);
subscription.Save();
}
SubscriptionGrid.EditIndex = -1;
e.Cancel = true;
}
/// <summary>
/// Resolves the next enumerator to be used in <see cref="MoveNext"/>
/// </summary>
private IEnumerator <BaseData> ResolveDataEnumerator(bool endOfEnumerator)
{
do
{
// always advance the date enumerator, this function is intended to be
// called on date changes, never return null for live mode, we'll always
// just keep trying to refresh the subscription
DateTime date;
if (!TryGetNextDate(out date) && !_isLiveMode)
{
// if we run out of dates then we're finished with this subscription
return(null);
}
// fetch the new source
var newSource = _dataFactory.GetSource(_config, date, _isLiveMode);
// check if we should create a new subscription factory
var sourceChanged = _source != newSource && newSource.Source != "";
var liveRemoteFile = _isLiveMode && (_source == null || _source.TransportMedium == SubscriptionTransportMedium.RemoteFile);
if (sourceChanged || liveRemoteFile)
{
// dispose of the current enumerator before creating a new one
if (_subscriptionFactoryEnumerator != null)
{
_subscriptionFactoryEnumerator.Dispose();
}
// save off for comparison next time
_source = newSource;
var subscriptionFactory = CreateSubscriptionFactory(newSource);
var previousTime = _previous == null ? DateTime.MinValue : _previous.EndTime;
return(subscriptionFactory.Read(newSource)
// prevent the enumerator from emitting data before the last emitted time
.Where(instance => instance != null && previousTime < instance.EndTime)
.GetEnumerator());
}
// if there's still more in the enumerator and we received the same source from the GetSource call
// above, then just keep using the same enumerator as we were before
if (!endOfEnumerator) // && !sourceChanged is always true here
{
return(_subscriptionFactoryEnumerator);
}
// keep churning until we find a new source or run out of tradeable dates
// in live mode tradeable dates won't advance beyond today's date, but
// TryGetNextDate will return false if it's already at today
}while (true);
}
private ISubscriptionFactory HandleCsvFileFormat(SubscriptionDataSource source)
{
// convert the date to the data time zone
var dateInDataTimeZone = _tradeableDates.Current.ConvertTo(_config.ExchangeTimeZone, _config.DataTimeZone).Date;
var factory = SubscriptionFactory.ForSource(source, _config, dateInDataTimeZone, _isLiveMode);
// handle missing files
factory.InvalidSource += (sender, args) =>
{
switch (args.Source.TransportMedium)
{
case SubscriptionTransportMedium.LocalFile:
// the local uri doesn't exist, write an error and return null so we we don't try to get data for today
Log.Trace(string.Format("SubscriptionDataReader.GetReader(): Could not find QC Data, skipped: {0}", source));
_resultHandler.SamplePerformance(_tradeableDates.Current, 0);
break;
case SubscriptionTransportMedium.RemoteFile:
_resultHandler.ErrorMessage(string.Format("Error downloading custom data source file, skipped: {0} Error: {1}", source, args.Exception.Message), args.Exception.StackTrace);
_resultHandler.SamplePerformance(_tradeableDates.Current.Date, 0);
break;
case SubscriptionTransportMedium.Rest:
break;
default:
throw new ArgumentOutOfRangeException();
}
};
if (factory is TextSubscriptionFactory)
{
// handle empty files/instantiation errors
var textSubscriptionFactory = (TextSubscriptionFactory)factory;
textSubscriptionFactory.CreateStreamReaderError += (sender, args) =>
{
Log.Error(string.Format("Failed to get StreamReader for data source({0}), symbol({1}). Skipping date({2}). Reader is null.", args.Source.Source, _mappedSymbol, args.Date.ToShortDateString()));
if (_config.IsCustomData)
{
_resultHandler.ErrorMessage(string.Format("We could not fetch the requested data. This may not be valid data, or a failed download of custom data. Skipping source ({0}).", args.Source.Source));
}
};
// handle parser errors
textSubscriptionFactory.ReaderError += (sender, args) =>
{
_resultHandler.RuntimeError(string.Format("Error invoking {0} data reader. Line: {1} Error: {2}", _config.Symbol, args.Line, args.Exception.Message), args.Exception.StackTrace);
};
}
return(factory);
}
private ISubscriptionFactory CreateSubscriptionFactory(SubscriptionDataSource source)
{
switch (source.Format)
{
case FileFormat.Csv:
return(HandleCsvFileFormat(source));
case FileFormat.Binary:
throw new NotSupportedException("Binary file format is not supported");
default:
throw new ArgumentOutOfRangeException();
}
}
/// <summary>
/// Reads the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The source to be read</param>
/// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
public IEnumerable<BaseData> Read(SubscriptionDataSource source)
{
IStreamReader reader = null;
var instances = new BaseDataCollection();
try
{
switch (source.TransportMedium)
{
default:
case SubscriptionTransportMedium.Rest:
reader = new RestSubscriptionStreamReader(source.Source);
break;
case SubscriptionTransportMedium.LocalFile:
reader = new LocalFileSubscriptionStreamReader(source.Source);
break;
case SubscriptionTransportMedium.RemoteFile:
reader = new RemoteFileSubscriptionStreamReader(source.Source, Globals.Cache);
break;
}
var raw = "";
try
{
raw = reader.ReadLine();
var result = _factory.Reader(_config, raw, _date, _isLiveMode);
instances = result as BaseDataCollection;
if (instances == null)
{
OnInvalidSource(source, new Exception("Reader must generate a BaseDataCollection with the FileFormat.Collection"));
}
}
catch (Exception err)
{
OnReaderError(raw, err);
}
foreach (var instance in instances.Data)
{
yield return instance;
}
}
finally
{
if (reader != null)
reader.Dispose();
}
}
/// <summary>
/// Creates a new <see cref="ISubscriptionDataSourceReader"/> capable of handling the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The subscription data source to create a factory for</param>
/// <param name="dataFileProvider">Retrieves files if not found on disk</param>
/// <param name="config">The configuration of the subscription</param>
/// <param name="date">The date to be processed</param>
/// <param name="isLiveMode">True for live mode, false otherwise</param>
/// <returns>A new <see cref="ISubscriptionDataSourceReader"/> that can read the specified <paramref name="source"/></returns>
public static ISubscriptionDataSourceReader ForSource(SubscriptionDataSource source, IDataFileProvider dataFileProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
switch (source.Format)
{
case FileFormat.Csv:
return new TextSubscriptionDataSourceReader(dataFileProvider, config, date, isLiveMode);
case FileFormat.Collection:
return new CollectionSubscriptionDataSourceReader(config, date, isLiveMode);
case FileFormat.ZipEntryName:
return new ZipEntryNameSubscriptionDataSourceReader(dataFileProvider, config, date, isLiveMode);
default:
throw new NotImplementedException("SubscriptionFactory.ForSource(" + source + ") has not been implemented yet.");
}
}
/// <summary>
/// Reads the specified <paramref name="source"/>
/// </summary>
/// <param name="source">The source to be read</param>
/// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
public IEnumerable<BaseData> Read(SubscriptionDataSource source)
{
if (!File.Exists(source.Source))
{
OnInvalidSource(source, new FileNotFoundException("The specified file was not found", source.Source));
}
ZipFile zip;
try
{
zip = new ZipFile(source.Source);
}
catch (ZipException err)
{
OnInvalidSource(source, err);
yield break;
}
foreach (var entryFileName in zip.EntryFileNames)
{
yield return _factory.Reader(_config, entryFileName, _dateTime, _isLiveMode);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="CreateStreamReaderErrorEventArgs"/> class
/// </summary>
/// <param name="date">The date of the source</param>
/// <param name="source">The source that cause the error</param>
public CreateStreamReaderErrorEventArgs(DateTime date, SubscriptionDataSource source)
{
Date = date;
Source = source;
}
/// <summary>
/// 2. RETURN THE STRING URL SOURCE LOCATION FOR YOUR DATA:
/// This is a powerful and dynamic select source file method. If you have a large dataset, 10+mb we recommend you break it into smaller files. E.g. One zip per year.
/// We can accept raw text or ZIP files. We read the file extension to determine if it is a zip file.
/// </summary>
/// <param name="config">Configuration object</param>
/// <param name="date">Date of this source file</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns>String URL of source file.</returns>
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime currentDate, bool isLiveMode)
{
if (isLiveMode)
{
return new SubscriptionDataSource(string.Format(DATA_SOURCE_URI, currentDate.ToShortDateString(), currentDate.ToShortDateString()), SubscriptionTransportMedium.RemoteFile);
//return new SubscriptionDataSource("https://www.bitstamp.net/api/ticker/", SubscriptionTransportMedium.Rest);
}
// OR simply return a fixed small data file. Large files will slow down your backtest
FileDataImport fi = new FileDataImport();
string uri = string.Format(DATA_SOURCE_URI, currentDate.ToShortDateString(), currentDate.ToShortDateString());
string logentry = string.Format("{0}:{1}:{2}, {3}", DateTime.Now.Minute, DateTime.Now.Second,
DateTime.Now.Millisecond, uri);
//urls.Add(logentry);
//WriteLog(logentry);
var source = new SubscriptionDataSource(uri, SubscriptionTransportMedium.RemoteFile);
return source;
//return new SubscriptionDataSource(fi._dataSourceUri, SubscriptionTransportMedium.LocalFile);
}
/// <summary>
/// Opens up an IStreamReader for a local file source
/// </summary>
private IStreamReader HandleLocalFileSource(SubscriptionDataSource source)
{
string entryName = null; // default to all entries
var file = source.Source;
var hashIndex = source.Source.LastIndexOf("#", StringComparison.Ordinal);
if (hashIndex != -1)
{
entryName = source.Source.Substring(hashIndex + 1);
file = source.Source.Substring(0, hashIndex);
}
if (!File.Exists(file))
{
OnInvalidSource(source, new FileNotFoundException("The specified file was not found", file));
return null;
}
// handles zip or text files
return new LocalFileSubscriptionStreamReader(file, entryName);
}
/// <summary>
/// Initializes a new instance of the <see cref="InvalidSourceEventArgs"/> class
/// </summary>
/// <param name="source">The source that was considered invalid</param>
/// <param name="exception">The exception that was encountered</param>
public InvalidSourceEventArgs(SubscriptionDataSource source, Exception exception)
{
Source = source;
Exception = exception;
}
/// <summary>
/// Creates a new <see cref="IStreamReader"/> for the specified <paramref name="subscriptionDataSource"/>
/// </summary>
/// <param name="subscriptionDataSource">The source to produce an <see cref="IStreamReader"/> for</param>
/// <returns>A new instance of <see cref="IStreamReader"/> to read the source, or null if there was an error</returns>
private IStreamReader CreateStreamReader(SubscriptionDataSource subscriptionDataSource)
{
IStreamReader reader;
switch (subscriptionDataSource.TransportMedium)
{
case SubscriptionTransportMedium.LocalFile:
reader = HandleLocalFileSource(subscriptionDataSource);
break;
case SubscriptionTransportMedium.RemoteFile:
reader = HandleRemoteSourceFile(subscriptionDataSource);
break;
case SubscriptionTransportMedium.Rest:
reader = new RestSubscriptionStreamReader(subscriptionDataSource.Source);
break;
default:
throw new InvalidEnumArgumentException("Unexpected SubscriptionTransportMedium specified: " + subscriptionDataSource.TransportMedium);
}
return reader;
}
/// <summary>
/// Opens up an IStreamReader for a remote file source
/// </summary>
private IStreamReader HandleRemoteSourceFile(SubscriptionDataSource source)
{
// clean old files out of the cache
if (!Directory.Exists(Constants.Cache)) Directory.CreateDirectory(Constants.Cache);
foreach (var file in Directory.EnumerateFiles(Constants.Cache))
{
if (File.GetCreationTime(file) < DateTime.Now.AddHours(-24)) File.Delete(file);
}
try
{
// this will fire up a web client in order to download the 'source' file to the cache
return new RemoteFileSubscriptionStreamReader(source.Source, Constants.Cache);
}
catch (Exception err)
{
OnInvalidSource(source, err);
return null;
}
}
/// <summary>
/// Opens up an IStreamReader for a local file source
/// </summary>
private IStreamReader HandleLocalFileSource(SubscriptionDataSource source)
{
if (!File.Exists(source.Source))
{
OnInvalidSource(source, new FileNotFoundException("The specified file was not found", source.Source));
return null;
}
// handles zip or text files
return new LocalFileSubscriptionStreamReader(source.Source);
}
/// <summary>
/// Event invocator for the <see cref="CreateStreamReaderError"/> event
/// </summary>
/// <param name="date">The date of the source</param>
/// <param name="source">The source that caused the error</param>
private void OnCreateStreamReaderError(DateTime date, SubscriptionDataSource source)
{
var handler = CreateStreamReaderError;
if (handler != null) handler(this, new CreateStreamReaderErrorEventArgs(date, source));
}
/// <summary>
/// Event invocator for the <see cref="InvalidSource"/> event
/// </summary>
/// <param name="source">The <see cref="SubscriptionDataSource"/> that was invalid</param>
/// <param name="exception">The exception if one was raised, otherwise null</param>
private void OnInvalidSource(SubscriptionDataSource source, Exception exception)
{
var handler = InvalidSource;
if (handler != null) handler(this, new InvalidSourceEventArgs(source, exception));
}
){kind=link}