/// <summary>
/// Gets the data from the file source.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rowBuilder">
/// The row builder.
/// </param>
/// <param name="query">
/// The query.
/// </param>
/// <returns>
/// The <see cref="IAsyncEnumerable{T}"/>.
/// </returns>
IAsyncEnumerable <Row> IDataSource.GetRows(IExecutionContext context, IRowBuilder rowBuilder, [NotNull] IQuery query)
{
var whereFilter = query.GetFilter(context);
var sortOrders = query.GetSortOrders(context).ToArray();
var fields = query.RetrieveAllFields ? null : new HashSet <string>(query.Fields);
var fileFormatContext = new FileFormatExecutionContext(context, this);
IFileReader fileReader = null;
return(context.CreateAsyncEnumerableAndRunOnce(
async() =>
{
var streamBuffer = await StreamBuffer.CreateAsync(await this.OpenStreamAsync(context, UriResolveMode.Read, this.Uri).ConfigureAwait(false), FileDataSource.PreviewByteCount).ConfigureAwait(false);
fileReader = this.GetFileReader(context, streamBuffer);
if (fileReader == null)
{
throw new InvalidOperationException($"Unable to load file with extension '{Path.GetExtension(this.Uri)}', did you load the file format?");
}
return Tuple.Create(this.GetStreamReader(streamBuffer, fileReader), streamBuffer);
},
stream => fileReader.Read(fileFormatContext, rowBuilder, stream.Item1, fields),
stream =>
{
stream?.Item1?.Dispose();
stream?.Item2?.Dispose();
})
.Where(whereFilter?.GetRowFilter())
.OrderBy(sortOrders));
}
/// <summary>
/// Writes the rowsToWrite to the specified target.
/// </summary>
/// <param name="context">
/// The context.
/// </param>
/// <param name="rowsToWrite">
/// The rowsToWrite.
/// </param>
/// <param name="upsert">
/// True to also update records, false to insert.
/// </param>
/// <returns>
/// The <see cref="Task"/>.
/// </returns>
async Task <long> IDataTarget.WriteRowsAsync([NotNull] IExecutionContext context, IAsyncEnumerable <Row> rowsToWrite, bool upsert)
{
var fileWriter = context.FileFormats.OfType <IFileWriter>().FirstOrDefault(writer => writer.CanWriteThisFile(this.Uri));
if (fileWriter == null)
{
throw new InvalidOperationException($"Unable to load file with extension '{Path.GetExtension(this.Uri)}', did you load the file format?");
}
if (fileWriter.ShouldMaterialize)
{
rowsToWrite = await rowsToWrite.MaterializeAsync().ConfigureAwait(false);
}
var result = 0L;
var fileFormatContext = new FileFormatExecutionContext(context, this);
using (var stream = await this.OpenStreamAsync(context, UriResolveMode.Write, this.Uri))
{
using (var writer = new StreamWriter(stream, (fileWriter as IOverrideEncoding)?.Encoding ?? this.encoding))
{
using (var enumerator = rowsToWrite.GetAsyncEnumerator())
{
var emptyEnumerable = false;
while (!enumerator.MoveNext())
{
if (!enumerator.IsSynchronous && await enumerator.NextBatchAsync())
{
continue;
}
emptyEnumerable = true;
break;
}
if (!emptyEnumerable)
{
var done = false;
fileWriter.WriteHeader(fileFormatContext, writer, enumerator.Current.ColumnNames);
while (!done)
{
result += fileWriter.WriteRows(fileFormatContext, writer, FileDataSource.CurrentAndRest(enumerator), upsert);
done = true;
if (enumerator.IsSynchronous)
{
continue;
}
while (await enumerator.NextBatchAsync())
{
if (!enumerator.MoveNext())
{
continue;
}
done = false;
break;
}
}
}
else
{
fileWriter.WriteHeader(fileFormatContext, writer, Enumerable.Empty <string>());
}
}
fileWriter.WriteFooter(fileFormatContext, writer);
}
}
return(result);
}
