private static void ImportFile(string file, CmdOptions options)
{
Log.WriteLine("Importing {0}", file);
string typename = Path.GetFileNameWithoutExtension(file);
IImporter importer = GetImporter(file, options);
if (importer == null)
{
Log.WriteLine("Unrecognized file type: {0}", typename);
return;
}
if (!importer.IgnoreType() && Global.StorageSchema.CellDescriptors.FirstOrDefault(cd => cd.TypeName == typename) == null)
{
Log.WriteLine("File {0} does not match any types defined in the storage extension.", file);
return;
}
var importer_type = importer.GetType();
var importer_interface_type = importer_type.FindInterfaces((t, _) =>
{
return(t.Name == "IImporter`1" && t.IsGenericType);
}, null).First();
var importer_entry_type = importer_interface_type.GetGenericArguments()[0];
var importer_method = typeof(Importer).GetMethod("_Import", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static).MakeGenericMethod(importer_entry_type);
importer_method.Invoke(null, new object[] { importer, file, typename });
}
private static IImporter GetImporter(string path, CmdOptions options)
{
const char comma = ',';
const char tab = '\t';
string filename, filetype;
bool delimiterSpecify = (options.Delimiter == '\0') ? false : true;
if (options.FileFormat != null)
{
filetype = options.FileFormat.StartsWith(".") ? options.FileFormat : "." + options.FileFormat;
}
else
{
filename = Path.GetFileName(path);
filetype = Path.GetExtension(filename).ToLower();
if (filetype == ".gz" || filetype == ".zip")
{
filetype = Path.GetExtension(Path.GetFileNameWithoutExtension(filename)).ToLower();
}
}
switch (filetype)
{
case ".json":
return(new JsonImporter());
case ".csv":
return(new CsvImporter(delimiterSpecify ? options.Delimiter : comma, !options.NoTrim));
case ".tsv":
return(new CsvImporter(delimiterSpecify ? options.Delimiter : tab, !options.NoTrim));
case ".ntriples":
return(g_opts.Sorted ? (IImporter) new SortedRDFImporter() : new UnsortedRDFImporter());
default:
{
return(null);
}
}
}
internal static void Generate(List <string> files, CmdOptions opts)
{
Log.WriteLine("Scanning data files to generate a TSL file.");
if (opts.Output != null)
{
g_output_tsl_name = opts.Output;
}
Global.LocalStorage.ResetStorage();
g_opts = opts;
g_field_type_aggregation_threshold = g_opts.DominatingTypeThreshold;
g_field_idmap.Clear();
foreach (var file in files)
{
ScanFile(file);
}
Log.WriteLine("All files scanned. Aggregating type information.");
AggregateTypeInformation();
Log.WriteLine("TSL file generated successfully.");
}
static void Main(string[] args)
{
CmdOptions opts = new CmdOptions();
if (CommandLine.Parser.Default.ParseArguments(args, opts))
{
List <string> files = new List <string>(opts.ExplicitFiles.Select(_ => _.Trim()));
if (opts.InputDirectory != null)
{
// Log.WriteLine("Including files from directory {0}", Path.GetFullPath(opts.InputDirectory));
files.AddRange(Directory.GetFiles(opts.InputDirectory).Select(_ => _.Trim()));
}
Stopwatch timer = Stopwatch.StartNew();
if (opts.TSL != null)
{
var tslCompiler = new TSLCompiler();
opts.TSLAssembly = tslCompiler.Compile(opts.TSL);
if (opts.TSLAssembly != null)
{
Importer.Import(opts.TSLAssembly, files, opts);
}
else
{
Log.WriteLine("TSL File Compile Error.");
}
}
else if (opts.TSLAssembly != null)
{
Importer.Import(opts.TSLAssembly, files, opts);
}
else if (opts.GenerateTSL)
{
TSLGenerator.Generate(files, opts);
}
timer.Stop();
Log.WriteLine("Time: {0} seconds.", timer.ElapsedMilliseconds / 1000);
}
}
internal static void Import(string tslAssembly, IEnumerable <string> files, CmdOptions options)
{
Log.WriteLine("Importing data files.");
if (options.Output != null)
{
TrinityConfig.StorageRoot = options.Output;
}
Global.LocalStorage.ResetStorage();
g_opts = options;
InitializeSchema(tslAssembly);
foreach (var file in files)
{
ImportFile(file, options);
}
ImportReverseEdges();
Global.LocalStorage.SaveStorage();
}