public override void ExtractFeature() { int count = 0; var reader = new InstanceReaderByLine(source); var writer = new EventWriterByLine(des); while (reader.HasNext()) { if (++count % 1000 == 0) { Console.Clear(); Console.WriteLine("{0} has processed {1}", Thread.CurrentThread.Name, count); } var instance = reader.GetNextInstance(); try { var feature = extractor.ExtractFeature(instance); var e = new Event(instance.Label, feature); writer.WriteEvent(e); } catch (Exception e) { Console.WriteLine(e.Message); //Console.WriteLine(e.StackTrace); Console.WriteLine(instance); } } reader.Close(); writer.Close(); }
public Pair <List <string>, List <string> > SplitData(string source, string des, int numPerThread) { var reader = new InstanceReaderByLine(source); var writer = new InstanceWriterByLine(des); var directory = Path.GetDirectoryName(source); var name = Path.GetFileNameWithoutExtension(source); var ext = Path.GetExtension(source); // seperate source file into parts int part = 0; var partFile = Path.Combine(directory, name + "-part" + part + ext); var sourceFiles = new List <string>(); sourceFiles.Add(partFile); // Create corresponding des files string desPartFile = null; var desFiles = new List <string>(); writer = new InstanceWriterByLine(partFile); Instance instance = null; int count = 0; while (reader.HasNext()) { try { instance = reader.GetNextInstance(); } catch (Exception) { continue; } if (++count < numPerThread) { writer.WriteInstance(instance); } else { writer.Close(); // add des path to desfiles desPartFile = Path.Combine(directory, name + "-feature-part" + part + ext); desFiles.Add(desPartFile); // create another part file part++; partFile = Path.Combine(directory, name + "-part" + part + ext); writer = new InstanceWriterByLine(partFile); count = 0; sourceFiles.Add(partFile); } } if (count > 0) { writer.Close(); desPartFile = Path.Combine(directory, name + "-feature-part" + part + ext); desFiles.Add(desPartFile); } else { writer.Close(); sourceFiles.Remove(partFile); } reader.Close(); return(new Pair <List <string>, List <string> >(sourceFiles, desFiles)); }