public ImportResult Import(Connection connection)
{
if (_reader is NullSchemaReader)
{
_context.Error($"Unable to read schema of {connection}.");
return(new ImportResult {
Connection = connection,
Fields = new List <Field>(),
Rows = Enumerable.Empty <IRow>(),
Schema = null
});
}
var schema = _reader.Read();
var entity = schema.Entities.First();
foreach (var field in entity.Fields.Where(f => Constants.InvalidFieldNames.Contains(f.Name)).Where(field => field.Alias == field.Name))
{
field.Alias = field.Name + "Source";
_context.Warn($"Reserved column name {field.Name} aliased as {field.Alias}.");
}
var cfg = new Process {
Name = "Import",
Connections = new List <Connection> {
schema.Connection
},
Entities = new List <Entity> {
entity
}
}.Serialize();
var process = new Process();
process.Load(cfg);
if (process.Errors().Any())
{
foreach (var error in process.Errors())
{
_context.Error(error);
}
return(new ImportResult {
Connection = schema.Connection,
Fields = new List <Field>(),
Rows = Enumerable.Empty <IRow>(),
Schema = schema
});
}
return(new ImportResult {
Connection = schema.Connection,
Fields = process.Entities.First().Fields.Where(f => !f.System).ToList(),
Schema = schema,
Rows = _runner.Run(process),
});
}
public PageResult GetPage(int page, int pageSize)
{
var result = new PageResult();
_entity.Page = page;
_entity.PageSize = pageSize;
result.Rows = _reader.Run(_process).ToArray(); // enumerate so i can get hits count back
result.Fields = _fields;
result.Hits = _entity.Hits;
return(result);
}
private void ModifyFields(IEnumerable <Field> fields)
{
var expanded = fields.ToArray();
if (_context.Connection.MaxLength > 0)
{
foreach (var field in expanded)
{
field.Length = _context.Connection.MaxLength.ToString();
}
}
var checkLength = _context.Connection.MinLength > 0 || _context.Connection.MaxLength > 0;
var checkTypes = _context.Connection.Types.Any(t => t.Type != "string");
if (checkTypes || checkLength)
{
var rows = _runner.Run(_process).ToArray(); // can take a lot of memory
if (rows.Length == 0)
{
return;
}
if (checkLength)
{
Parallel.ForEach(expanded, f => {
var length = _context.Connection.MaxLength == 0 ? rows.Max(row => row[f].ToString().Length) + 1 : Math.Min(rows.Max(row => row[f].ToString().Length) + 1, _context.Connection.MaxLength);
if (_context.Connection.MinLength > 0 && length < _context.Connection.MinLength)
{
length = _context.Connection.MinLength;
}
f.Length = length.ToString();
});
}
if (checkTypes)
{
var canConvert = Constants.CanConvert();
Parallel.ForEach(expanded, f => {
foreach (var dataType in _context.Connection.Types.Where(t => t.Type != "string"))
{
if (rows.All(r => canConvert[dataType.Type](r[f].ToString())))
{
f.Type = dataType.Type;
break;
}
}
});
}
}
}
