public override void ExtractFeature()
{
int count = 0;
var reader = new InstanceReaderByLine(source);
var writer = new EventWriterByLine(des);
while (reader.HasNext())
{
if (++count % 1000 == 0)
{
Console.Clear();
Console.WriteLine("{0} has processed {1}", Thread.CurrentThread.Name, count);
}
var instance = reader.GetNextInstance();
try
{
var feature = extractor.ExtractFeature(instance);
var e = new Event(instance.Label, feature);
writer.WriteEvent(e);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
//Console.WriteLine(e.StackTrace);
Console.WriteLine(instance);
}
}
reader.Close();
writer.Close();
}
public Pair <List <string>, List <string> > SplitData(string source, string des, int numPerThread)
{
var reader = new InstanceReaderByLine(source);
var writer = new InstanceWriterByLine(des);
var directory = Path.GetDirectoryName(source);
var name = Path.GetFileNameWithoutExtension(source);
var ext = Path.GetExtension(source);
// seperate source file into parts
int part = 0;
var partFile = Path.Combine(directory, name + "-part" + part + ext);
var sourceFiles = new List <string>();
sourceFiles.Add(partFile);
// Create corresponding des files
string desPartFile = null;
var desFiles = new List <string>();
writer = new InstanceWriterByLine(partFile);
Instance instance = null;
int count = 0;
while (reader.HasNext())
{
try
{
instance = reader.GetNextInstance();
}
catch (Exception)
{
continue;
}
if (++count < numPerThread)
{
writer.WriteInstance(instance);
}
else
{
writer.Close();
// add des path to desfiles
desPartFile = Path.Combine(directory, name + "-feature-part" + part + ext);
desFiles.Add(desPartFile);
// create another part file
part++;
partFile = Path.Combine(directory, name + "-part" + part + ext);
writer = new InstanceWriterByLine(partFile);
count = 0;
sourceFiles.Add(partFile);
}
}
if (count > 0)
{
writer.Close();
desPartFile = Path.Combine(directory, name + "-feature-part" + part + ext);
desFiles.Add(desPartFile);
}
else
{
writer.Close();
sourceFiles.Remove(partFile);
}
reader.Close();
return(new Pair <List <string>, List <string> >(sourceFiles, desFiles));
}