public ImportFormParserResult(ImportJobRequestInfo importJob, SchemaImportJobRequestInfo schemaImportJob, string metadata)
{
IsValid = true;
ValidationErrors = new List <string>(0);
ImportJobRequest = importJob;
SchemaImportJobRequest = schemaImportJob;
Metadata = metadata;
}
public async Task <ImportFormParserResult> ParseImportFormAsync(HttpRequest request, string userId,
Func <ImportFormData, IFormCollection, Task <bool> > bindAsyncFunc)
{
if (request == null)
{
throw new ArgumentNullException(nameof(request));
}
if (userId == null)
{
throw new ArgumentNullException(nameof(userId));
}
if (bindAsyncFunc == null)
{
throw new ArgumentNullException(nameof(bindAsyncFunc));
}
if (!MultipartRequestHelper.IsMultipartContentType(request.ContentType))
{
return(new ImportFormParserResult($"Expected a multipart request, but got {request.ContentType}"));
}
var defaultFormOptions = new FormOptions();
var jobInfo = new ImportJobRequestInfo
{
UserId = userId
};
var formAccumulator = new KeyValueAccumulator();
var boundary = MultipartRequestHelper.GetBoundary(MediaTypeHeaderValue.Parse(request.ContentType),
defaultFormOptions.MultipartBoundaryLengthLimit);
var reader = new MultipartReader(boundary, request.Body);
var section = await reader.ReadNextSectionAsync();
while (section != null)
{
var hasContentDispositionHeader =
ContentDispositionHeaderValue.TryParse(section.ContentDisposition, out var contentDisposition);
if (hasContentDispositionHeader)
{
if (MultipartRequestHelper.HasFileContentDisposition(contentDisposition))
{
jobInfo.CsvFileName = string.Empty;
jobInfo.DatasetId = string.Empty;
Log.Information("api/data(POST): Starting conversion job. UserId='{0}', File='{1}'", userId,
"");
var csvFileId = await _fileStore.AddFileAsync(section.Body);
Log.Information($"Saved the uploaded CSV file '{csvFileId}'");
jobInfo.CsvFileId = csvFileId;
}
else if (MultipartRequestHelper.HasFormDataContentDisposition(contentDisposition))
{
var key = HeaderUtilities.RemoveQuotes(contentDisposition.Name);
var encoding = GetEncoding(section);
using (var streamReader = new StreamReader(
section.Body,
encoding,
detectEncodingFromByteOrderMarks: true,
bufferSize: 1024,
leaveOpen: true))
{
// The value length limit is enforced by MultipartBodyLengthLimit
var value = await streamReader.ReadToEndAsync();
if (String.Equals(value, "undefined", StringComparison.OrdinalIgnoreCase))
{
value = String.Empty;
}
formAccumulator.Append(key.ToString(), value);
if (formAccumulator.ValueCount > defaultFormOptions.ValueCountLimit)
{
throw new InvalidDataException(
$"Form key count limit {defaultFormOptions.ValueCountLimit} exceeded.");
}
}
}
}
// Drains any remaining section body that has not been consumed and
// reads the headers for the next section.
section = await reader.ReadNextSectionAsync();
}
// Bind form data to a model
var formData = new ImportFormData();
var bindingSuccessful = await bindAsyncFunc(formData, new FormCollection(formAccumulator.GetResults()));
if (!bindingSuccessful)
{
return(new ImportFormParserResult("Import form data validation failed"));
}
jobInfo.OwnerId = formData.OwnerId;
jobInfo.RepositoryId = formData.RepoId;
if (string.IsNullOrEmpty(formData.OwnerId) || string.IsNullOrEmpty(formData.RepoId))
{
Log.Error("DataController: POST called with no owner or repo set in FormData");
return(new ImportFormParserResult("No target repository supplied"));
}
if (formData.Metadata == null)
{
Log.Error("DataController: POST called with no metadata present in FormData");
return(new ImportFormParserResult("No metadata supplied"));
}
var parser = new JsonSerializer();
Log.Debug("DataController: Metadata: {0}", formData.Metadata);
var metadataObject = parser.Deserialize(new JsonTextReader(new StringReader(formData.Metadata))) as JObject;
if (metadataObject == null)
{
Log.Error(
"DataController: Error deserializing metadata as object, unable to create conversion job. Metadata = '{0}'",
formData.Metadata);
return(new ImportFormParserResult("Metadata badly formatted"));
}
var datasetIri = metadataObject["url"]?.ToString();
var datasetTitle = metadataObject["dc:title"]?.ToString();
if (string.IsNullOrEmpty(datasetIri))
{
Log.Error("DataController: No dataset IRI supplied in metadata.");
return(new ImportFormParserResult("No dataset IRI supplied in metadata"));
}
var datasetId = formData.Filename;
var datasetIriSplit = datasetIri.Split("/");
if (datasetIriSplit != null && datasetIriSplit.Length > 1)
{
datasetId = datasetIriSplit[datasetIriSplit.Length - 1];
}
Log.Debug("DataController: datasetIri = '{0}'", datasetIri);
jobInfo.DatasetIri = datasetIri;
// save CSVW to file storage
if (string.IsNullOrEmpty(jobInfo.CsvFileName))
{
jobInfo.CsvFileName = formData.Filename;
jobInfo.DatasetId = datasetId;
}
jobInfo.IsPublic = formData.ShowOnHomePage;
byte[] byteArray = Encoding.UTF8.GetBytes(formData.Metadata);
var metadataStream = new MemoryStream(byteArray);
var csvwFileId = await _fileStore.AddFileAsync(metadataStream);
jobInfo.CsvmFileId = csvwFileId;
jobInfo.OverwriteExistingData = formData.OverwriteExisting;
if (formData.SaveAsSchema)
{
Log.Information("api/data(POST): Saving metadata as template.");
var schema = new JObject(new JProperty("dc:title", "Template from " + datasetTitle),
new JProperty("metadata", metadataObject));
Log.Information("api/data(POST): Starting schema creation job. UserId='{0}', Repository='{1}'", userId,
formData.RepoId);
byte[] schemaByteArray = Encoding.UTF8.GetBytes(schema.ToString());
var schemaStream = new MemoryStream(schemaByteArray);
var schemaFileId = await _fileStore.AddFileAsync(schemaStream);
if (!string.IsNullOrEmpty(schemaFileId))
{
Log.Information("api/data(POST): Schema temp file saved: {0}.", schemaFileId);
var schemaJobRequest = new SchemaImportJobRequestInfo()
{
UserId = userId,
SchemaFileId = schemaFileId,
OwnerId = formData.OwnerId,
RepositoryId = formData.RepoId
};
return(new ImportFormParserResult(jobInfo, schemaJobRequest, formData.Metadata));
}
Log.Error(
"api/data(POST): Error saving schema content to temporary file storage, unable to start schema creation job");
}
return(new ImportFormParserResult(jobInfo, formData.Metadata));
}
public async Task <JobInfo> SubmitSchemaImportJobAsync(SchemaImportJobRequestInfo jobRequest)
{
var jobInfo = new JobInfo(jobRequest);
return(await SubmitJobAsync(jobInfo));
}
