private static void ImportFile(string file, CmdOptions options) { Log.WriteLine("Importing {0}", file); string typename = Path.GetFileNameWithoutExtension(file); IImporter importer = GetImporter(file, options); if (importer == null) { Log.WriteLine("Unrecognized file type: {0}", typename); return; } if (!importer.IgnoreType() && Global.StorageSchema.CellDescriptors.FirstOrDefault(cd => cd.TypeName == typename) == null) { Log.WriteLine("File {0} does not match any types defined in the storage extension.", file); return; } var importer_type = importer.GetType(); var importer_interface_type = importer_type.FindInterfaces((t, _) => { return(t.Name == "IImporter`1" && t.IsGenericType); }, null).First(); var importer_entry_type = importer_interface_type.GetGenericArguments()[0]; var importer_method = typeof(Importer).GetMethod("_Import", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static).MakeGenericMethod(importer_entry_type); importer_method.Invoke(null, new object[] { importer, file, typename }); }
private static IImporter GetImporter(string path, CmdOptions options) { const char comma = ','; const char tab = '\t'; string filename, filetype; bool delimiterSpecify = (options.Delimiter == '\0') ? false : true; if (options.FileFormat != null) { filetype = options.FileFormat.StartsWith(".") ? options.FileFormat : "." + options.FileFormat; } else { filename = Path.GetFileName(path); filetype = Path.GetExtension(filename).ToLower(); if (filetype == ".gz" || filetype == ".zip") { filetype = Path.GetExtension(Path.GetFileNameWithoutExtension(filename)).ToLower(); } } switch (filetype) { case ".json": return(new JsonImporter()); case ".csv": return(new CsvImporter(delimiterSpecify ? options.Delimiter : comma, !options.NoTrim)); case ".tsv": return(new CsvImporter(delimiterSpecify ? options.Delimiter : tab, !options.NoTrim)); case ".ntriples": return(g_opts.Sorted ? (IImporter) new SortedRDFImporter() : new UnsortedRDFImporter()); default: { return(null); } } }
internal static void Generate(List <string> files, CmdOptions opts) { Log.WriteLine("Scanning data files to generate a TSL file."); if (opts.Output != null) { g_output_tsl_name = opts.Output; } Global.LocalStorage.ResetStorage(); g_opts = opts; g_field_type_aggregation_threshold = g_opts.DominatingTypeThreshold; g_field_idmap.Clear(); foreach (var file in files) { ScanFile(file); } Log.WriteLine("All files scanned. Aggregating type information."); AggregateTypeInformation(); Log.WriteLine("TSL file generated successfully."); }
static void Main(string[] args) { CmdOptions opts = new CmdOptions(); if (CommandLine.Parser.Default.ParseArguments(args, opts)) { List <string> files = new List <string>(opts.ExplicitFiles.Select(_ => _.Trim())); if (opts.InputDirectory != null) { // Log.WriteLine("Including files from directory {0}", Path.GetFullPath(opts.InputDirectory)); files.AddRange(Directory.GetFiles(opts.InputDirectory).Select(_ => _.Trim())); } Stopwatch timer = Stopwatch.StartNew(); if (opts.TSL != null) { var tslCompiler = new TSLCompiler(); opts.TSLAssembly = tslCompiler.Compile(opts.TSL); if (opts.TSLAssembly != null) { Importer.Import(opts.TSLAssembly, files, opts); } else { Log.WriteLine("TSL File Compile Error."); } } else if (opts.TSLAssembly != null) { Importer.Import(opts.TSLAssembly, files, opts); } else if (opts.GenerateTSL) { TSLGenerator.Generate(files, opts); } timer.Stop(); Log.WriteLine("Time: {0} seconds.", timer.ElapsedMilliseconds / 1000); } }
internal static void Import(string tslAssembly, IEnumerable <string> files, CmdOptions options) { Log.WriteLine("Importing data files."); if (options.Output != null) { TrinityConfig.StorageRoot = options.Output; } Global.LocalStorage.ResetStorage(); g_opts = options; InitializeSchema(tslAssembly); foreach (var file in files) { ImportFile(file, options); } ImportReverseEdges(); Global.LocalStorage.SaveStorage(); }