private async Task <string> ReadTextFileInStore(string fileToken)
{
using (var fileStore = _fileStoreFactory.GetFileStoreFromFileToken(fileToken))
await using (var stream = await fileStore.OpenRead(fileToken))
using (var reader = new StreamReader(stream, Encoding.UTF8))
{
return(reader.ReadToEnd());
}
}
private string GetFileTokenContent(string fileToken)
{
using (var fileStore = _fileStoreFactory.GetFileStoreFromFileToken(fileToken))
using (var input = fileStore.OpenRead(fileToken).Result)
using (var reader = new StreamReader(input, Encoding.UTF8))
{
return(reader.ReadToEnd());
}
}
private async Task <SqlQuery> RetrieveQuery(SqlExportToCsvInput input)
{
if (input.QueryFileToken != null)
{
using var fileStore = _fileStoreFactory.GetFileStoreFromFileToken(input.QueryFileToken);
using Stream stream = await fileStore.OpenRead(input.QueryFileToken);
using StreamReader reader = new StreamReader(stream);
input.Query = reader.ReadToEnd();
}
return(input);
}
public override async Task <string> DoWorkAsync(GZipInput input, CancellationToken token)
{
string outputFileName = input.OutputFileName;
if (input.InputFileName != null)
{
using var inputfs = _fileStoreFactory.GetFileStoreFromName(input.InputFileStoreName);
input.FileToken = await inputfs.CreateFileToken(input.InputFileName);
}
using var inputFileStore = _fileStoreFactory.GetFileStoreFromFileToken(input.FileToken);
using var outputFileStore = string.IsNullOrEmpty(input.OutputFileStoreName) ? inputFileStore : _fileStoreFactory.GetFileStoreFromName(input.OutputFileStoreName);
if (string.IsNullOrEmpty(outputFileName))
{
string fileName = await outputFileStore.GetFileName(input.FileToken);
outputFileName = input.Compress ? GetCompressedFileName(fileName) : GetDecompressedFileName(fileName);
}
_logger.LogDebug($"GZip output file name '{outputFileName}'");
string outputFileToken = await outputFileStore.CreateFileToken(outputFileName);
using (var inputStream = await inputFileStore.OpenRead(input.FileToken))
using (var outputStream = await outputFileStore.OpenWrite(outputFileToken))
using (var gzipStream = input.Compress ?
new GZipStream(outputStream, CompressionMode.Compress) :
new GZipStream(inputStream, CompressionMode.Decompress))
{
if (input.Compress)
{
await inputStream.CopyToAsync(gzipStream, input.BufferSize, token);
}
else
{
await gzipStream.CopyToAsync(outputStream, input.BufferSize, token);
}
}
if (input.DeleteSource)
{
_logger.LogDebug("Deleting source filetoken");
await inputFileStore.Delete(input.FileToken);
}
return(outputFileToken);
}
private async Task <HttpContent> GetRequestContent(HttpWorkerInput workerInput)
{
HttpContent requestContent = null;
if (workerInput.JsonContent.HasValue && workerInput.JsonContent.Value.ValueKind != JsonValueKind.Null &&
workerInput.JsonContent.Value.ValueKind != JsonValueKind.Undefined)
{
_logger.LogDebug("Using RequestBody Json as Request Content");
requestContent = new StringContent(workerInput.JsonContent.Value.GetRawText(), Encoding.UTF8, JsonMediaType);
}
else if (workerInput.ContentFileToken != null)
{
_logger.LogDebug($"Using RequestBody from Token '{workerInput.ContentFileToken}' as Request Content");
var inputStore = _storeFactory.GetFileStoreFromFileToken(workerInput.ContentFileToken);
requestContent = new StreamContent(await inputStore.OpenRead(workerInput.ContentFileToken));
requestContent.Headers.ContentType = new MediaTypeHeaderValue("text/plain");
requestContent.Headers.ContentType.CharSet = "utf-8";
}
return(requestContent);
}
public override async Task <SqlQueryOutput> DoWorkAsync(SqlQueryInput input, CancellationToken token)
{
var database = _databases.FirstOrDefault(x => x.Name == input.ConnectionName);
if (database == null)
{
throw new ArgumentException("Invalid ConnectionName");
}
using DbConnection connection = database.CreateConnection();
try
{
_logger.LogDebug("Opening connection...");
await connection.OpenAsync(token);
}
catch (Exception e)
{
throw new SqlConnectionException(e);
}
if (input.QueryFileToken != null || input.QueryFileName != null)
{
using (var fileStore = _fileStoreFactory.GetFileStoreFromFileToken(input.QueryFileToken))
{
if (input.QueryFileName != null)
{
input.QueryFileToken = await fileStore.CreateFileToken(input.QueryFileName);
}
using (Stream stream = await fileStore.OpenRead(input.QueryFileToken))
using (StreamReader reader = new StreamReader(stream))
{
input.Query = reader.ReadToEnd();
}
}
}
_logger.LogDebug("Creating command...");
using var cmd = connection.CreateCommand(input);
var output = new SqlQueryOutput();
try
{
switch (input.Type)
{
case SqlQueryType.Scalar:
_logger.LogDebug("Execute scalar...");
output.Scalar = await cmd.ExecuteScalarAsync(token);
break;
case SqlQueryType.NonQuery:
_logger.LogDebug("Execute non query...");
await cmd.ExecuteNonQueryAsync(token);
break;
case SqlQueryType.Reader:
_logger.LogDebug("Execute Reader...");
using (var reader = await cmd.ExecuteReaderAsync(token))
{
output.Result = new List <Dictionary <string, object> >();
while (reader.Read())
{
Dictionary <string, object> line = new Dictionary <string, object>();
for (int i = 0; i < reader.FieldCount; i++)
{
line.Add(reader.GetName(i), reader.GetValue(i));
}
output.Result.Add(line);
if (output.Result.Count >= input.MaxResults)
{
break;
}
}
}
break;
default:
throw new ArgumentOutOfRangeException();
}
}
catch (Exception e)
{
_logger.LogError(e, "Error while executing query");
throw new SqlQueryException(e);
}
return(output);
}
public override async Task<bool> DoWorkAsync(BulkInsertInput input, CancellationToken token)
{
var destinationDatabase = _databases.FirstOrDefault(x => x.Name == input.Destination.ConnectionName);
if (destinationDatabase == null)
{
throw new ArgumentException("Invalid Destination ConnectionName");
}
if (input.Destination.TruncateFirst)
{
await destinationDatabase.TruncateTable(input.Destination.SchemaName, input.Destination.TableName);
}
DbConnection sourceConnection = null;
DbCommand sourceCommand = null;
IDataReader dataReader = null;
if (!string.IsNullOrEmpty(input.Source.ConnectionName))
{
_logger.LogDebug($"Quering '{input.Source.ConnectionName}' as source");
var sourceDatabase = _databases.FirstOrDefault(x => x.Name == input.Source.ConnectionName);
if (sourceDatabase == null)
{
throw new ArgumentException("Invalid Source ConnectionName");
}
if (input.Source.Query.QueryFileToken != null)
{
using(var fileStore = _fileStoreFactory.GetFileStoreFromFileToken(input.Source.Query.QueryFileToken))
using(Stream stream = await fileStore.OpenRead(input.Source.Query.QueryFileToken))
using(StreamReader reader = new StreamReader(stream))
{
input.Source.Query.Query = reader.ReadToEnd();
}
}
sourceConnection = sourceDatabase.CreateConnection();
await sourceConnection.OpenAsync(token);
sourceCommand = sourceConnection.CreateCommand(input.Source.Query);
dataReader = await sourceCommand.ExecuteReaderAsync(token);
}
else if(input.Source.FileToken != null)
{
using var fileStore = _fileStoreFactory.GetFileStoreFromFileToken(input.Source.FileToken);
string tmpFileName = Path.GetTempFileName();
_logger.LogDebug($"Copying filetoken '{input.Source.FileToken}' into '{tmpFileName}'");
using (var reader = await fileStore.OpenRead(input.Source.FileToken))
using (var writer = File.OpenWrite(tmpFileName))
{
await reader.CopyToAsync(writer,2048, token);
}
input.Source.FileName = tmpFileName;
}
if(dataReader == null)
{
if (string.IsNullOrEmpty(input.Source.FileName))
{
throw new ArgumentException("No Source specified");
}
if (!File.Exists(input.Source.FileName))
{
throw new ArgumentException($"File '{input.Source.FileName}' not found");
}
_logger.LogDebug($"Loading file '{input.Source.FileName}' as source");
dataReader = new CsvDataReader(input.Source.FileName,input.Source.Separator,input.Source.Headers, CultureInfo.GetCultureInfo(input.Source.CultureName));
}
try
{
_logger.LogDebug("Starting BulkInsert");
await destinationDatabase.BulkInsert(
dataReader,
input.Destination.SchemaName,
input.Destination.TableName,
input.Destination.Timeout,
token);
_logger.LogDebug("BulkInsert completed");
}
finally
{
dataReader?.Dispose();
sourceCommand?.Dispose();
sourceConnection?.Dispose();
}
return true;
}