/// <summary>
/// Reads JSON objects from the stream.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <param name="reader">
/// The stream.
/// </param>
/// <param name="fields">
/// The fields, or <c>null</c> to retrieve all fields.
/// </param>
/// <returns>
/// The rows.
/// </returns>
public IEnumerable <Row> Read(IFileFormatExecutionContext context, IRowBuilder rowBuilder, StreamReader reader, HashSet <string> fields)
{
var idx = 0L;
var serializer = JsonSerializer.Create();
var filterFields = fields == null
? (Func <KeyValuePair <string, object>, bool>)(kv => true)
: kv => fields.Contains(kv.Key);
using (var jsonTextReader = new JsonTextReader(reader))
{
jsonTextReader.SupportMultipleContent = true;
jsonTextReader.Read();
if (jsonTextReader.TokenType == JsonToken.StartArray)
{
while (jsonTextReader.Read() && jsonTextReader.TokenType == JsonToken.StartObject)
{
yield return(rowBuilder.CreateRow(idx++, serializer.Deserialize <Dictionary <string, object> >(jsonTextReader).Where(filterFields)));
}
}
else
{
do
{
yield return(rowBuilder.CreateRow(idx++, serializer.Deserialize <Dictionary <string, object> >(jsonTextReader).Where(filterFields)));
}while (jsonTextReader.Read() && jsonTextReader.TokenType == JsonToken.StartObject);
}
}
}
/// <summary>
/// Reads a reader as comma separated values.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rowBuilder">
/// The data Set.
/// </param>
/// <param name="reader">
/// The stream.
/// </param>
/// <param name="fields">
/// The fields, or <c>null</c> to retrieve all fields.
/// </param>
/// <returns>
/// The rows.
/// </returns>
public IEnumerable <Row> Read(IFileFormatExecutionContext context, IRowBuilder rowBuilder, StreamReader reader, HashSet <string> fields)
{
var separator = context.GetDefault("SEPARATOR", false) as string ?? ",";
var splitter = GetSplitter(separator);
var headers = GetHeaders(splitter, reader, separator);
if (headers.Length == 1 && string.IsNullOrEmpty(headers[0]))
{
yield break;
}
var idx = 0L;
do
{
var line = splitter.Matches($"{reader.ReadLine()}{separator}")
.Cast <Match>()
.Select(match => match.Groups[1].Value)
.Select(value => value.Trim())
.Select(value => EscapedString.IsMatch(value) ? value.Substring(1, value.Length - 2).Replace("\"\"", "\"") : value)
.ToArray();
if (line.Length == headers.Length)
{
// ReSharper disable once AccessToDisposedClosure
yield return(rowBuilder.CreateRow(idx++, headers.Select((header, i) => new KeyValuePair <string, object>(header, line[i]))));
}
}while (!reader.EndOfStream);
}
/// <summary>
/// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}"/>.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <param name="query">
/// The query expression. Can be <c>null</c>.
/// </param>
/// <returns>
/// A task returning the data set.
/// </returns>
public IAsyncEnumerable <Row> GetRows(IExecutionContext context, IRowBuilder rowBuilder, [NotNull] IQuery query)
{
var filter = query.GetFilter(context).GetRowFilter();
DateTime start;
// ReSharper disable once SwitchStatementMissingSomeCases
switch (this.offset)
{
case TimeOffset.Midnight:
start = DateTime.Today - this.past;
break;
case TimeOffset.UtcNow:
start = DateTime.UtcNow - this.past;
break;
case TimeOffset.UtcMidnight:
start = DateTime.UtcNow.Date - this.past;
break;
default:
start = DateTime.Now - this.past;
break;
}
var num = (int)(((long)this.future.TotalMilliseconds + (long)this.past.TotalMilliseconds) / (long)this.interval.TotalMilliseconds);
return(context.ToAsyncEnumerable(Enumerable
.Range(0, num)
.Select(i => start + TimeSpan.FromMilliseconds(i * this.interval.TotalMilliseconds))
.Select(i => rowBuilder.CreateRow(i.Ticks, new[] { new KeyValuePair <string, object>("Time", i), }))
.Where(filter)).OrderBy(query.OrderByExpressions));
}
/// <summary>
/// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}"/>.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <param name="query">
/// The query expression. Can be <c>null</c>.
/// </param>
/// <returns>
/// A task returning the data set.
/// </returns>
public IAsyncEnumerable <Row> GetRows(IExecutionContext context, IRowBuilder rowBuilder, [NotNull] IQuery query)
{
var idx = 0L;
return(context
.CreateAsyncEnumerable(this.enumerable)
.Select(value => rowBuilder.CreateRow(idx++, this.rowGenerator(value)))
.Where(query.GetFilter(context).GetRowFilter())
.OrderBy(query.GetSortOrders(context))
.Take(query.Count));
}
/// <summary>
/// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}"/>.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <param name="query">
/// The query expression. Can be <c>null</c>.
/// </param>
/// <returns>
/// A task returning the data set.
/// </returns>
public IAsyncEnumerable <Row> GetRows(IExecutionContext context, IRowBuilder rowBuilder, IQuery query)
{
var table = this.GetTable(context);
var tableQuery =
new TableQuery().Select(query.RetrieveAllFields ? null : query.Fields.Select(f => f).ToList())
.Where(ToTableQuery(query.GetFilter(context)))
.Take(query.Count);
var result = context.CreateAsyncEnumerable(
async(GetDataState state) =>
{
if (state.Done)
{
return(null);
}
var segment = await table.ExecuteQuerySegmentedAsync(tableQuery, CreateRowEntity, state.Token);
state.Done = segment.ContinuationToken == null;
state.Token = segment.ContinuationToken;
return(segment.Results.Count != 0
? segment.Select(
row =>
rowBuilder.CreateRow(
Tuple.Create(row.PartitionKey, row.RowKey),
row.GetValues(tableQuery.SelectColumns)))
: null);
});
result.BeforeFirstElement(
() =>
context.Logger.Verbose(
tableQuery.FilterString == null
? $"Retrieving all records from table '{table.Name}'."
: $"Retrieving records from table '{table.Name}', query : {tableQuery.FilterString}."));
result.AfterLastElement(
count => context.Logger.Verbose($"Retrieved {count} records from table '{table.Name}'."));
return(result);
}
/// <summary>
/// Reads objects from the stream.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <param name="reader">
/// The stream.
/// </param>
/// <param name="fields">
/// The fields, or <c>null</c> to retrieve all fields.
/// </param>
/// <returns>
/// The rows.
/// </returns>
public IEnumerable <Row> Read(IFileFormatExecutionContext context, IRowBuilder rowBuilder, StreamReader reader, HashSet <string> fields)
{
using (var package = new ExcelPackage(reader.BaseStream))
{
var sheet = package.Workbook.Worksheets.FirstOrDefault();
if (sheet == null)
{
yield break;
}
long idx = 0;
var headers = Enumerable.Range(1, sheet.Dimension.End.Column).Select(col => sheet.Cells[1, col].Value?.ToString()).TakeWhile(header => header != null).ToArray();
foreach (var range in Enumerable.Range(2, sheet.Dimension.End.Row - 1).Select(row => sheet.Cells[row, 1, row, headers.Length].Select(cell => cell.Value)))
{
yield return(rowBuilder.CreateRow(idx++, headers.Zip(range, (header, value) => new KeyValuePair <string, object>(header, value))));
}
}
}
/// <summary>
/// Gets the rows asynchronously.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <returns>
/// The <see cref="Task" />.
/// </returns>
private async Task <IAsyncEnumerable <Row> > GetRowsAsync(IExecutionContext context, IRowBuilder rowBuilder)
{
var rows = await this.source.MaterializeAsync().ConfigureAwait(false);
var first = await rows.FirstOrDefaultAsync().ConfigureAwait(false);
if (first == null)
{
return(context.CreateEmptyAsyncEnumerable <Row>());
}
var sortColumns = first.ColumnNames.Where(c => c != this.columnName && c != this.columnValue).ToArray();
var groups = rows.GroupBy(row => sortColumns.Select(c => row[c]).ToArray(), ArrayOfObjectComparer.Default);
return(groups.Select(
async group =>
{
var values = sortColumns
.Zip(group.Key, (name, value) => new KeyValuePair <string, object>(name, value))
.ToDictionary(kv => kv.Key, kv => kv.Value);
var counts = values.ToDictionary(kv => kv.Key, kv => 0);
object id = null;
await group.ForEachAsync(
row =>
{
id = row.UniqueId;
var name = row[this.columnName] as string;
var value = row[this.columnValue];
if (name != null)
{
switch (this.function)
{
case DCastFunction.Concat:
if (values.TryGetValue(name, out object existingValue))
{
values[name] = $"{(string)existingValue}, {value}";
}
else
{
values[name] = value;
}
break;
case DCastFunction.First:
if (!values.ContainsKey(name))
{
values[name] = value;
}
break;
case DCastFunction.Last:
values[name] = value;
break;
case DCastFunction.Min:
if (values.TryGetValue(name, out existingValue))
{
if (Comparer <object> .Default.Compare(existingValue, value) > 0)
{
values[name] = value;
}
}
else
{
values[name] = value;
}
break;
case DCastFunction.Max:
if (values.TryGetValue(name, out existingValue))
{
if (Comparer <object> .Default.Compare(existingValue, value) < 0)
{
values[name] = value;
}
}
else
{
values[name] = value;
}
break;
case DCastFunction.Avg:
if (values.TryGetValue(name, out existingValue))
{
var valueDouble = DCast.ToDouble(value);
var existingDouble = DCast.ToDouble(existingValue);
if (valueDouble != null && existingDouble != null)
{
values[name] = valueDouble + existingDouble;
}
}
else
{
values[name] = DCast.ToDouble(value) ?? value;
}
break;
}
values[name] = row[this.columnValue];
counts[name] = counts.TryGetValue(name, out int count) ? count + 1 : 1;
}
});
if (this.function == DCastFunction.Avg)
{
foreach (var key in values.Keys.Except(sortColumns))
{
if (values[key] is double)
{
values[key] = ((double)values[key]) / counts[key];
}
}
}
return rowBuilder.CreateRow(id, values);
}));
}
/// <summary>
/// Creates a row.
/// </summary>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <param name="uniqueId">
/// The unique id of the row.
/// </param>
/// <param name="fields">
/// The fields in the row.
/// </param>
/// <typeparam name="T">
/// The type of the unique id of the row.
/// </typeparam>
/// <returns>
/// The <see cref="Row"/>.
/// </returns>
public static Row CreateRow <T>([NotNull] this IRowBuilder rowBuilder, T uniqueId, params KeyValuePair <string, object>[] fields)
{
return(rowBuilder.CreateRow(uniqueId, fields));
}
/// <summary>
/// Clones the row in the specified row builder.
/// </summary>
/// <param name="rowBuilder">
/// The builder to clone the row into.
/// </param>
/// <returns>
/// The cloned row.
/// </returns>
internal override Row Clone([NotNull] IRowBuilder rowBuilder)
{
return(rowBuilder.CreateRow(this.Id, this.ToDictionary()));
}
/// <summary>
/// Retrieves the data from the source as an <see cref="IAsyncEnumerable{T}"/>.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <param name="query">
/// The query expression. Can be <c>null</c>.
/// </param>
/// <returns>
/// A task returning the data set.
/// </returns>
public IAsyncEnumerable <Row> GetRows(IExecutionContext context, IRowBuilder rowBuilder, IQuery query)
{
var row = rowBuilder.CreateRow(0, new KeyValuePair <string, object>(this.columnName, this.columnValueSet ? this.columnValue : true));
return(context.CreateAsyncEnumerable(Enumerable.Repeat(row, 1)));
}
