/// <summary>
/// Reads a reader as comma separated values.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rowBuilder">
/// The data Set.
/// </param>
/// <param name="reader">
/// The stream.
/// </param>
/// <param name="fields">
/// The fields, or <c>null</c> to retrieve all fields.
/// </param>
/// <returns>
/// The rows.
/// </returns>
public IEnumerable <Row> Read(IFileFormatExecutionContext context, IRowBuilder rowBuilder, StreamReader reader, HashSet <string> fields)
{
var separator = context.GetDefault("SEPARATOR", false) as string ?? ",";
var splitter = GetSplitter(separator);
var headers = GetHeaders(splitter, reader, separator);
if (headers.Length == 1 && string.IsNullOrEmpty(headers[0]))
{
yield break;
}
var idx = 0L;
do
{
var line = splitter.Matches($"{reader.ReadLine()}{separator}")
.Cast <Match>()
.Select(match => match.Groups[1].Value)
.Select(value => value.Trim())
.Select(value => EscapedString.IsMatch(value) ? value.Substring(1, value.Length - 2).Replace("\"\"", "\"") : value)
.ToArray();
if (line.Length == headers.Length)
{
// ReSharper disable once AccessToDisposedClosure
yield return(rowBuilder.CreateRow(idx++, headers.Select((header, i) => new KeyValuePair <string, object>(header, line[i]))));
}
}while (!reader.EndOfStream);
}
/// <summary>
/// Reads JSON objects from the stream.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <param name="reader">
/// The stream.
/// </param>
/// <param name="fields">
/// The fields, or <c>null</c> to retrieve all fields.
/// </param>
/// <returns>
/// The rows.
/// </returns>
public IEnumerable <Row> Read(IFileFormatExecutionContext context, IRowBuilder rowBuilder, StreamReader reader, HashSet <string> fields)
{
var idx = 0L;
var serializer = JsonSerializer.Create();
var filterFields = fields == null
? (Func <KeyValuePair <string, object>, bool>)(kv => true)
: kv => fields.Contains(kv.Key);
using (var jsonTextReader = new JsonTextReader(reader))
{
jsonTextReader.SupportMultipleContent = true;
jsonTextReader.Read();
if (jsonTextReader.TokenType == JsonToken.StartArray)
{
while (jsonTextReader.Read() && jsonTextReader.TokenType == JsonToken.StartObject)
{
yield return(rowBuilder.CreateRow(idx++, serializer.Deserialize <Dictionary <string, object> >(jsonTextReader).Where(filterFields)));
}
}
else
{
do
{
yield return(rowBuilder.CreateRow(idx++, serializer.Deserialize <Dictionary <string, object> >(jsonTextReader).Where(filterFields)));
}while (jsonTextReader.Read() && jsonTextReader.TokenType == JsonToken.StartObject);
}
}
}
/// <summary>
/// Gets the data from the file source.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <param name="query">
/// The query.
/// </param>
/// <returns>
/// The <see cref="IAsyncEnumerable{T}"/>.
/// </returns>
IAsyncEnumerable <Row> IDataSource.GetRows(IExecutionContext context, IRowBuilder rowBuilder, [NotNull] IQuery query)
{
var whereFilter = query.GetFilter(context);
var sortOrders = query.GetSortOrders(context).ToArray();
var fields = query.RetrieveAllFields ? null : new HashSet <string>(query.Fields);
var fileFormatContext = new FileFormatExecutionContext(context, this);
IFileReader fileReader = null;
return(context.CreateAsyncEnumerableAndRunOnce(
async() =>
{
var streamBuffer = await StreamBuffer.CreateAsync(await this.OpenStreamAsync(context, UriResolveMode.Read, this.Uri).ConfigureAwait(false), FileDataSource.PreviewByteCount).ConfigureAwait(false);
fileReader = this.GetFileReader(context, streamBuffer);
if (fileReader == null)
{
throw new InvalidOperationException($"Unable to load file with extension '{Path.GetExtension(this.Uri)}', did you load the file format?");
}
return Tuple.Create(this.GetStreamReader(streamBuffer, fileReader), streamBuffer);
},
stream => fileReader.Read(fileFormatContext, rowBuilder, stream.Item1, fields),
stream =>
{
stream?.Item1?.Dispose();
stream?.Item2?.Dispose();
})
.Where(whereFilter?.GetRowFilter())
.OrderBy(sortOrders));
}
/// <summary>
/// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}"/>.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <param name="query">
/// The query expression. Can be <c>null</c>.
/// </param>
/// <returns>
/// A task returning the data set.
/// </returns>
public IAsyncEnumerable <Row> GetRows(IExecutionContext context, IRowBuilder rowBuilder, [NotNull] IQuery query)
{
var filter = query.GetFilter(context).GetRowFilter();
DateTime start;
// ReSharper disable once SwitchStatementMissingSomeCases
switch (this.offset)
{
case TimeOffset.Midnight:
start = DateTime.Today - this.past;
break;
case TimeOffset.UtcNow:
start = DateTime.UtcNow - this.past;
break;
case TimeOffset.UtcMidnight:
start = DateTime.UtcNow.Date - this.past;
break;
default:
start = DateTime.Now - this.past;
break;
}
var num = (int)(((long)this.future.TotalMilliseconds + (long)this.past.TotalMilliseconds) / (long)this.interval.TotalMilliseconds);
return(context.ToAsyncEnumerable(Enumerable
.Range(0, num)
.Select(i => start + TimeSpan.FromMilliseconds(i * this.interval.TotalMilliseconds))
.Select(i => rowBuilder.CreateRow(i.Ticks, new[] { new KeyValuePair <string, object>("Time", i), }))
.Where(filter)).OrderBy(query.OrderByExpressions));
}
/// <summary>
/// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}"/>.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <param name="query">
/// The query expression. Can be <c>null</c>.
/// </param>
/// <returns>
/// A task returning the data set.
/// </returns>
public IAsyncEnumerable <Row> GetRows(IExecutionContext context, IRowBuilder rowBuilder, IQuery query)
{
return
(this.rows?.Select(rowBuilder.Attach)
.Where(query.GetFilter(context).GetRowFilter())
.OrderBy(query.GetSortOrders(context))
?? context.CreateAsyncEnumerable(Enumerable.Empty <Row>()));
}
/// <summary>
/// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}"/>.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <param name="query">
/// The query expression. Can be <c>null</c>.
/// </param>
/// <returns>
/// A task returning the data set.
/// </returns>
public IAsyncEnumerable <Row> GetRows(IExecutionContext context, IRowBuilder rowBuilder, [NotNull] IQuery query)
{
var idx = 0L;
return(context
.CreateAsyncEnumerable(this.enumerable)
.Select(value => rowBuilder.CreateRow(idx++, this.rowGenerator(value)))
.Where(query.GetFilter(context).GetRowFilter())
.OrderBy(query.GetSortOrders(context))
.Take(query.Count));
}
public override DeliverOrderBuilder AddFee(IRowBuilder itemFee)
{
if (itemFee is ShippingFeeBuilder)
{
ShippingFeeRows.Add(itemFee as ShippingFeeBuilder);
}
else
{
InvoiceFeeRows.Add(itemFee as InvoiceFeeBuilder);
}
return(this);
}
public override CreateOrderBuilder AddDiscount(IRowBuilder itemDiscount)
{
if (itemDiscount is FixedDiscountBuilder)
{
FixedDiscountRows.Add(itemDiscount as FixedDiscountBuilder);
}
else
{
RelativeDiscountRows.Add(itemDiscount as RelativeDiscountBuilder);
}
return(this);
}
public override CreateOrderBuilder AddFee(IRowBuilder itemFee)
{
if (itemFee.GetType() == new ShippingFeeBuilder().GetType())
{
ShippingFeeRows.Add((ShippingFeeBuilder)itemFee);
}
else
{
InvoiceFeeRows.Add((InvoiceFeeBuilder)itemFee);
}
return(this);
}
private OrderRow NewRowBasedOnExisting(IRowBuilder existingRow)
{
var newOrderRow = new OrderRow();
newOrderRow = SerializeOrder(existingRow.GetArticleNumber(), existingRow.GetDescription(),
existingRow.GetName(), existingRow.GetUnit(), newOrderRow);
newOrderRow.DiscountPercent = existingRow.GetDiscountPercent();
newOrderRow.NumberOfUnits = existingRow.GetQuantity();
return(newOrderRow);
}
/// <summary>
/// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}"/>.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <param name="query">
/// The query expression. Can be <c>null</c>.
/// </param>
/// <returns>
/// A task returning the data set.
/// </returns>
public IAsyncEnumerable <Row> GetRows(IExecutionContext context, IRowBuilder rowBuilder, IQuery query)
{
var table = this.GetTable(context);
var tableQuery =
new TableQuery().Select(query.RetrieveAllFields ? null : query.Fields.Select(f => f).ToList())
.Where(ToTableQuery(query.GetFilter(context)))
.Take(query.Count);
var result = context.CreateAsyncEnumerable(
async(GetDataState state) =>
{
if (state.Done)
{
return(null);
}
var segment = await table.ExecuteQuerySegmentedAsync(tableQuery, CreateRowEntity, state.Token);
state.Done = segment.ContinuationToken == null;
state.Token = segment.ContinuationToken;
return(segment.Results.Count != 0
? segment.Select(
row =>
rowBuilder.CreateRow(
Tuple.Create(row.PartitionKey, row.RowKey),
row.GetValues(tableQuery.SelectColumns)))
: null);
});
result.BeforeFirstElement(
() =>
context.Logger.Verbose(
tableQuery.FilterString == null
? $"Retrieving all records from table '{table.Name}'."
: $"Retrieving records from table '{table.Name}', query : {tableQuery.FilterString}."));
result.AfterLastElement(
count => context.Logger.Verbose($"Retrieved {count} records from table '{table.Name}'."));
return(result);
}
/// <summary>
/// Reads objects from the stream.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <param name="reader">
/// The stream.
/// </param>
/// <param name="fields">
/// The fields, or <c>null</c> to retrieve all fields.
/// </param>
/// <returns>
/// The rows.
/// </returns>
public IEnumerable <Row> Read(IFileFormatExecutionContext context, IRowBuilder rowBuilder, StreamReader reader, HashSet <string> fields)
{
using (var package = new ExcelPackage(reader.BaseStream))
{
var sheet = package.Workbook.Worksheets.FirstOrDefault();
if (sheet == null)
{
yield break;
}
long idx = 0;
var headers = Enumerable.Range(1, sheet.Dimension.End.Column).Select(col => sheet.Cells[1, col].Value?.ToString()).TakeWhile(header => header != null).ToArray();
foreach (var range in Enumerable.Range(2, sheet.Dimension.End.Row - 1).Select(row => sheet.Cells[row, 1, row, headers.Length].Select(cell => cell.Value)))
{
yield return(rowBuilder.CreateRow(idx++, headers.Zip(range, (header, value) => new KeyValuePair <string, object>(header, value))));
}
}
}
private HostedOrderRowBuilder GetNewTempRow(IRowBuilder row, string sku)
{
var tempRow = new HostedOrderRowBuilder();
if (row.GetName() != null)
{
tempRow.SetName(row.GetName());
}
if (row.GetDescription() != null)
{
tempRow.SetDescription(row.GetDescription());
}
tempRow.SetQuantity(1);
tempRow.SetUnit(row.GetUnit());
tempRow.SetSku(sku);
return(tempRow);
}
private static string FormatDescription(IRowBuilder row)
{
return(string.Format(row.GetName() != null && row.GetName().Length > 0 ? "{0}: {1}" : "{1}", row.GetName(), row.GetDescription()));
}
/// <summary>
/// Joins the other row to the current row.
/// </summary>
/// <typeparam name="TOther">
/// The type of the other row's unique id.
/// </typeparam>
/// <param name="rowBuilder">
/// The row Builder.
/// </param>
/// <param name="other">
/// The other row.
/// </param>
/// <returns>
/// The joined row, or null when the rows cannot be joined.
/// </returns>
Row IRowImplementation.CombineFrom <TOther>(IRowBuilder rowBuilder, RowImplementation <TOther> other)
{
return(this.CombineFrom(rowBuilder, other));
}
public abstract T AddFee(IRowBuilder itemFee);
/// <summary>
/// Joins the other row to the current row.
/// </summary>
/// <typeparam name="TOther">
/// The type of the other row's unique id.
/// </typeparam>
/// <param name="builder">
/// The builder.
/// </param>
/// <param name="other">
/// The other row.
/// </param>
/// <returns>
/// The joined row, or null when the rows cannot be joined.
/// </returns>
protected override Row CombineFrom <TOther>(IRowBuilder builder, RowImplementation <TOther> other)
{
return(IdCombiner <T, TOther> .Combine(builder, this, other));
}
/// <summary> /// Clones the row in the specified row builder. /// </summary> /// <param name="rowBuilder"> /// The builder to clone the row into. /// </param> /// <returns> /// The cloned row. /// </returns> internal abstract Row Clone(IRowBuilder rowBuilder);
/// <summary>
/// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}" />.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <param name="query">
/// The query expression. Can be <c>null</c>.
/// </param>
/// <returns>
/// A task returning the data set.
/// </returns>
public IAsyncEnumerable <Row> GetRows(IExecutionContext context, IRowBuilder rowBuilder, IQuery query)
{
return(context.CreateAsyncEnumerable(() => this.GetRowsAsync(context, rowBuilder)));
}
/// <summary>
/// Retrieves the data from multiple sources as an <see cref="IAsyncEnumerable{T}"/>.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <param name="query">
/// The join query expression. Can be <c>null</c>.
/// </param>
/// <returns>
/// A task returning the data set.
/// </returns>
IAsyncEnumerable <Row> ISupportsJoin.GetRows(IExecutionContext context, IRowBuilder rowBuilder, IJoinQuery query)
{
throw new NotSupportedException("Join support is not yet implemented.");
}
private bool WriteToOutput(IRowBuilder builder)
{
return output.Write(builder.BuildRow());
}
/// <summary> /// When implemented in a derived class, joins the other row to the current row. /// </summary> /// <typeparam name="TOther"> /// The type of the other row's unique id. /// </typeparam> /// <param name="rowBuilder"> /// The row Builder. /// </param> /// <param name="other"> /// The other row. /// </param> /// <returns> /// The joined row, or null when the rows cannot be joined. /// </returns> protected abstract Row CombineFrom <TOther>(IRowBuilder rowBuilder, RowImplementation <TOther> other);
public void SetBuilder(IRowBuilder builder)
{
this.builder = builder;
}
/// <summary>
/// Clones the row in the specified row builder.
/// </summary>
/// <param name="rowBuilder">
/// The builder to clone the row into.
/// </param>
/// <returns>
/// The cloned row.
/// </returns>
internal override Row Clone([NotNull] IRowBuilder rowBuilder)
{
return(rowBuilder.CreateRow(this.Id, this.ToDictionary()));
}
/// <summary>
/// Joins the current row with the other row.
/// </summary>
/// <param name="rowBuilder">
/// The row Builder.
/// </param>
/// <param name="row">
/// The row to join.
/// </param>
/// <returns>
/// The joined row, or null if <paramref name="row"/> is null and this is an inner join.
/// </returns>
internal override Row CombineWith(IRowBuilder rowBuilder, Row row)
{
return(((IRowImplementation)row).CombineFrom(rowBuilder, this));
}
private static string FormatDiscountDescription(IRowBuilder row)
{
return(string.Format(row.GetName() != null && row.GetName().Length > 0 ? "{0}: {1} ({2}%)" : "{1} ({2}%)", row.GetName(), row.GetDescription(), (int)row.GetVatPercent().GetValueOrDefault()));
}
/// <summary>
/// Gets the rows asynchronously.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <returns>
/// The <see cref="Task" />.
/// </returns>
private async Task <IAsyncEnumerable <Row> > GetRowsAsync(IExecutionContext context, IRowBuilder rowBuilder)
{
var rows = await this.source.MaterializeAsync().ConfigureAwait(false);
var first = await rows.FirstOrDefaultAsync().ConfigureAwait(false);
if (first == null)
{
return(context.CreateEmptyAsyncEnumerable <Row>());
}
var sortColumns = first.ColumnNames.Where(c => c != this.columnName && c != this.columnValue).ToArray();
var groups = rows.GroupBy(row => sortColumns.Select(c => row[c]).ToArray(), ArrayOfObjectComparer.Default);
return(groups.Select(
async group =>
{
var values = sortColumns
.Zip(group.Key, (name, value) => new KeyValuePair <string, object>(name, value))
.ToDictionary(kv => kv.Key, kv => kv.Value);
var counts = values.ToDictionary(kv => kv.Key, kv => 0);
object id = null;
await group.ForEachAsync(
row =>
{
id = row.UniqueId;
var name = row[this.columnName] as string;
var value = row[this.columnValue];
if (name != null)
{
switch (this.function)
{
case DCastFunction.Concat:
if (values.TryGetValue(name, out object existingValue))
{
values[name] = $"{(string)existingValue}, {value}";
}
else
{
values[name] = value;
}
break;
case DCastFunction.First:
if (!values.ContainsKey(name))
{
values[name] = value;
}
break;
case DCastFunction.Last:
values[name] = value;
break;
case DCastFunction.Min:
if (values.TryGetValue(name, out existingValue))
{
if (Comparer <object> .Default.Compare(existingValue, value) > 0)
{
values[name] = value;
}
}
else
{
values[name] = value;
}
break;
case DCastFunction.Max:
if (values.TryGetValue(name, out existingValue))
{
if (Comparer <object> .Default.Compare(existingValue, value) < 0)
{
values[name] = value;
}
}
else
{
values[name] = value;
}
break;
case DCastFunction.Avg:
if (values.TryGetValue(name, out existingValue))
{
var valueDouble = DCast.ToDouble(value);
var existingDouble = DCast.ToDouble(existingValue);
if (valueDouble != null && existingDouble != null)
{
values[name] = valueDouble + existingDouble;
}
}
else
{
values[name] = DCast.ToDouble(value) ?? value;
}
break;
}
values[name] = row[this.columnValue];
counts[name] = counts.TryGetValue(name, out int count) ? count + 1 : 1;
}
});
if (this.function == DCastFunction.Avg)
{
foreach (var key in values.Keys.Except(sortColumns))
{
if (values[key] is double)
{
values[key] = ((double)values[key]) / counts[key];
}
}
}
return rowBuilder.CreateRow(id, values);
}));
}
/// <summary>
/// Retrieves the data from the source as an <see cref="T:ConnectQl.AsyncEnumerables.IAsyncEnumerable`1" />.
/// </summary>
/// <param name="context">The context.</param>
/// <param name="rowBuilder">The row builder.</param>
/// <param name="query">The query expression. Can be <c>null</c>.</param>
/// <returns>
/// A task returning the data set.
/// </returns>
public IAsyncEnumerable <Row> GetRows(IExecutionContext context, IRowBuilder rowBuilder, IQuery query)
{
this.GetQueryExpression(query.GetFilter(context));
return(context.CreateEmptyAsyncEnumerable <Row>());
}
/// <summary> /// Joins the row with this row and returns the result as new row. /// </summary> /// <param name="set"> /// The set. /// </param> /// <param name="row"> /// The row to join this row with. /// </param> /// <returns> /// A new <see cref="Row"/> containing fields for both the left row and the right row, or null if he join was an inner /// join /// and <paramref name="row"/> was null. /// </returns> internal abstract Row CombineWith(IRowBuilder set, Row row);
public SpinSlotsBuilder(IRowBuilder rowBuilder, IOutcomeBuilder outcomeBuilder, ISpinSlotsResponseBuilder spinSlotsResponseBuilder)
{
RowBuilder = rowBuilder ?? throw new ArgumentNullException(nameof(rowBuilder));
OutcomeBuilder = outcomeBuilder ?? throw new ArgumentNullException(nameof(outcomeBuilder));
ResponseBuilder = spinSlotsResponseBuilder ?? throw new ArgumentNullException(nameof(spinSlotsResponseBuilder));
}
/// <summary>
/// Creates a row.
/// </summary>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <param name="uniqueId">
/// The unique id of the row.
/// </param>
/// <param name="fields">
/// The fields in the row.
/// </param>
/// <typeparam name="T">
/// The type of the unique id of the row.
/// </typeparam>
/// <returns>
/// The <see cref="Row"/>.
/// </returns>
public static Row CreateRow <T>([NotNull] this IRowBuilder rowBuilder, T uniqueId, params KeyValuePair <string, object>[] fields)
{
return(rowBuilder.CreateRow(uniqueId, fields));
}