public ImportImpl(StreamReader reader, string file, TaskModel taskModel, Action <string> output, IAsyncDatabaseCommands databaseCommands)
{
this.taskModel = taskModel;
this.output = output;
this.databaseCommands = databaseCommands;
csvReader = new CsvReader(reader);
header = csvReader.ReadHeaderRecord();
entity = Inflector.Pluralize(CSharpClassName.ConvertToValidClassName(Path.GetFileNameWithoutExtension(file)));
if (entity.Length > 0 && char.IsLower(entity[0]))
{
entity = char.ToUpper(entity[0]) + entity.Substring(1);
}
sw = Stopwatch.StartNew();
enumerator = csvReader.DataRecords.GetEnumerator();
}
public async Task <HttpResponseMessage> LoadCsvFile()
{
if (!Request.Content.IsMimeMultipartContent())
{
throw new Exception(); // divided by zero
}
var provider = new MultipartMemoryStreamProvider();
await Request.Content.ReadAsMultipartAsync(provider);
foreach (var file in provider.Contents)
{
var filename = file.Headers.ContentDisposition.FileName.Trim('\"');
var stream = await file.ReadAsStreamAsync();
using (var csvReader = new TextFieldParser(stream))
{
csvReader.SetDelimiters(",");
var headers = csvReader.ReadFields();
var entity =
Inflector.Pluralize(CSharpClassName.ConvertToValidClassName(Path.GetFileNameWithoutExtension(filename)));
if (entity.Length > 0 && char.IsLower(entity[0]))
{
entity = char.ToUpper(entity[0]) + entity.Substring(1);
}
var totalCount = 0;
var batch = new List <RavenJObject>();
var columns = headers.Where(x => x.StartsWith("@") == false).ToArray();
batch.Clear();
while (csvReader.EndOfData == false)
{
var record = csvReader.ReadFields();
var document = new RavenJObject();
string id = null;
RavenJObject metadata = null;
for (int index = 0; index < columns.Length; index++)
{
var column = columns[index];
if (string.IsNullOrEmpty(column))
{
continue;
}
if (string.Equals("id", column, StringComparison.OrdinalIgnoreCase))
{
id = record[index];
}
else if (string.Equals(Constants.RavenEntityName, column, StringComparison.OrdinalIgnoreCase))
{
metadata = metadata ?? new RavenJObject();
metadata[Constants.RavenEntityName] = record[index];
id = id ?? record[index] + "/";
}
else if (string.Equals(Constants.RavenClrType, column, StringComparison.OrdinalIgnoreCase))
{
metadata = metadata ?? new RavenJObject();
metadata[Constants.RavenClrType] = record[index];
id = id ?? record[index] + "/";
}
else
{
document[column] = SetValueInDocument(record[index]);
}
}
metadata = metadata ?? new RavenJObject {
{ "Raven-Entity-Name", entity }
};
document.Add("@metadata", metadata);
metadata.Add("@id", id ?? Guid.NewGuid().ToString());
batch.Add(document);
totalCount++;
if (batch.Count >= CsvImportBatchSize)
{
await FlushBatch(batch);
batch.Clear();
}
}
if (batch.Count > 0)
{
await FlushBatch(batch);
}
}
}
return(GetEmptyMessage());
}
public async Task <HttpResponseMessage> LoadCsvFile()
{
const int csvImportBatchSize = 512;
if (Request.Content.IsMimeMultipartContent() == false)
{
throw new Exception(); // divided by zero
}
var provider = new MultipartMemoryStreamProvider();
await Request.Content.ReadAsMultipartAsync(provider).ConfigureAwait(false);
foreach (var file in provider.Contents)
{
var filename = file.Headers.ContentDisposition.FileName.Trim('\"');
using (var stream = await file.ReadAsStreamAsync().ConfigureAwait(false))
using (var textReader = new StreamReader(stream))
using (var csvReader = new CsvReader(textReader))
{
csvReader.Configuration.Delimiter = ",";
if (csvReader.Read() == false)
{
return(GetEmptyMessage());
}
var headers = csvReader.FieldHeaders;
var entity =
Inflector.Pluralize(CSharpClassName.ConvertToValidClassName(Path.GetFileNameWithoutExtension(filename)));
if (entity.Length > 0 && char.IsLower(entity[0]))
{
entity = char.ToUpper(entity[0]) + entity.Substring(1);
}
var totalCount = 0;
var batch = new List <RavenJObject>();
var validColumnIndexes = headers.Select((h, i) => new { Header = h, Index = i })
.Where(x => x.Header.StartsWith("@") == false)
.Select(s => s.Index)
.ToArray();
batch.Clear();
do
{
var record = csvReader.CurrentRecord;
var document = new RavenJObject();
string id = null;
RavenJObject metadata = null;
foreach (var index in validColumnIndexes)
{
var column = headers[index];
if (string.IsNullOrEmpty(column))
{
continue;
}
var value = record[index];
if (string.Equals("id", column, StringComparison.OrdinalIgnoreCase))
{
id = value;
}
else if (string.Equals(Constants.RavenEntityName, column, StringComparison.OrdinalIgnoreCase))
{
metadata = metadata ?? new RavenJObject();
metadata[Constants.RavenEntityName] = value;
id = id ?? value + "/";
}
else if (string.Equals(Constants.RavenClrType, column, StringComparison.OrdinalIgnoreCase))
{
metadata = metadata ?? new RavenJObject();
metadata[Constants.RavenClrType] = value;
id = id ?? value + "/";
}
else
{
SetValueInDocumentForColumn(document, column, value);
}
}
metadata = metadata ?? new RavenJObject {
{ "Raven-Entity-Name", entity }
};
document.Add("@metadata", metadata);
metadata.Add("@id", id ?? Guid.NewGuid().ToString());
batch.Add(document);
totalCount++;
if (batch.Count >= csvImportBatchSize)
{
await FlushBatch(batch).ConfigureAwait(false);
batch.Clear();
}
} while (csvReader.Read());
if (batch.Count > 0)
{
await FlushBatch(batch).ConfigureAwait(false);
}
}
}
return(GetEmptyMessage());
}
private async Task ImportAsync(StreamReader streamReader, string name)
{
using (var csvReader = new CsvReader(streamReader))
{
var header = csvReader.ReadHeaderRecord();
var entity =
Inflector.Pluralize(CSharpClassName.ConvertToValidClassName(Path.GetFileNameWithoutExtension(name)));
if (entity.Length > 0 && char.IsLower(entity[0]))
{
entity = char.ToUpper(entity[0]) + entity.Substring(1);
}
var totalCount = 0;
var batch = new List <RavenJObject>();
var columns = header.Values.Where(x => x.StartsWith("@") == false).ToArray();
batch.Clear();
foreach (var record in csvReader.DataRecords)
{
var document = new RavenJObject();
string id = null;
RavenJObject metadata = null;
foreach (var column in columns)
{
if (string.IsNullOrEmpty(column))
{
continue;
}
if (string.Equals("id", column, StringComparison.OrdinalIgnoreCase))
{
id = record[column];
}
else if (string.Equals(Constants.RavenEntityName, column, StringComparison.OrdinalIgnoreCase))
{
metadata = metadata ?? new RavenJObject();
metadata[Constants.RavenEntityName] = record[column];
id = id ?? record[column] + "/";
}
else if (string.Equals(Constants.RavenClrType, column, StringComparison.OrdinalIgnoreCase))
{
metadata = metadata ?? new RavenJObject();
metadata[Constants.RavenClrType] = record[column];
id = id ?? record[column] + "/";
}
else
{
document[column] = SetValueInDocument(record[column]);
}
}
metadata = metadata ?? new RavenJObject {
{ "Raven-Entity-Name", entity }
};
document.Add("@metadata", metadata);
metadata.Add("@id", id ?? Guid.NewGuid().ToString());
batch.Add(document);
totalCount++;
if (batch.Count >= BatchSize)
{
await FlushBatch(batch);
batch.Clear();
}
}
if (batch.Count > 0)
{
await FlushBatch(batch);
}
Report(String.Format("Imported {0:#,#;;0} documents", totalCount));
}
}
public async Task ImportFromCsv()
{
using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
if (HttpContext.Request.HasFormContentType == false)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.BadRequest;
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, new DynamicJsonValue
{
["Type"] = "Error",
["Error"] = "Import from csv requires form content type"
});
return;
}
}
var token = new OperationCancelToken(Database.DatabaseShutdown);
var result = new SmugglerResult();
var operationId = GetLongQueryString("operationId", false) ?? Database.Operations.GetNextOperationId();
var collection = GetStringQueryString("collection", false);
var operationDescription = collection != null ? "Import collection: " + collection : "Import collection from CSV";
await Database.Operations.AddOperation(Database, operationDescription, Raven.Server.Documents.Operations.Operations.OperationType.CollectionImportFromCsv,
onProgress =>
{
return Task.Run(async () =>
{
try
{
var reader = new MultipartReader(MultipartRequestHelper.GetBoundary(MediaTypeHeaderValue.Parse(HttpContext.Request.ContentType),
MultipartRequestHelper.MultipartBoundaryLengthLimit), HttpContext.Request.Body);
while (true)
{
var section = await reader.ReadNextSectionAsync().ConfigureAwait(false);
if (section == null)
break;
if (ContentDispositionHeaderValue.TryParse(section.ContentDisposition, out ContentDispositionHeaderValue contentDisposition) == false)
continue;
if (MultipartRequestHelper.HasFileContentDisposition(contentDisposition))
{
if (ContentDispositionHeaderValue.TryParse(section.ContentDisposition, out contentDisposition) == false)
continue;
if (string.IsNullOrEmpty(collection))
{
var fileName = contentDisposition.FileName.ToString().Trim('\"');
collection = Inflector.Pluralize(CSharpClassName.ConvertToValidClassName(Path.GetFileNameWithoutExtension(fileName)));
}
var options = new DatabaseSmugglerOptionsServerSide();
if (section.Headers.ContainsKey("Content-Encoding") && section.Headers["Content-Encoding"] == "gzip")
{
using (var gzipStream = new GZipStream(section.Body, CompressionMode.Decompress))
{
ImportDocumentsFromCsvStream(gzipStream, context, collection, options, result, onProgress, token);
}
}
else
{
ImportDocumentsFromCsvStream(section.Body, context, collection, options, result, onProgress, token);
}
}
}
}
catch (Exception e)
{
result.AddError($"Error occurred during csv import. Exception: {e.Message}");
throw;
}
return (IOperationResult)result;
});
}, operationId, token);
WriteImportResult(context, result, ResponseBodyStream());
}
}
