static void Write(string[] args)
{
var take = 1000;
if (Array.IndexOf(args, "--take") > 0)
{
take = int.Parse(args[Array.IndexOf(args, "--take") + 1]);
}
var fileName = args[Array.IndexOf(args, "--file") + 1];
string dir = null;
string indexName = null;
if (Array.IndexOf(args, "--dir") > 0)
{
dir = args[Array.IndexOf(args, "--dir") + 1];
}
if (Array.IndexOf(args, "--name") > 0)
{
indexName = args[Array.IndexOf(args, "--name") + 1];
}
var url = ConfigurationManager.AppSettings.Get("sir.endpoint");
var inproc = !string.IsNullOrWhiteSpace(dir);
Console.WriteLine("writing...");
var docs = new List <Dictionary <string, string> >();
var writeTimer = new Stopwatch();
writeTimer.Start();
if (inproc)
{
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
using (var writer = new StreamWriteOperation(dir, new Analyzer(), fileName, take))
{
writer.Execute();
}
}
else
{
Console.WriteLine("Executing HTTP POST");
using (var client = new WriterClient(indexName, url))
{
client.Write(docs);
}
}
Console.WriteLine("write operation took {0}", writeTimer.Elapsed);
}
static void Write(string[] args)
{
var take = 1000;
var skip = 0;
var skipped = 0;
if (Array.IndexOf(args, "--take") > 0)
{
take = int.Parse(args[Array.IndexOf(args, "--take") + 1]);
}
if (Array.IndexOf(args, "--skip") > 0)
{
skip = int.Parse(args[Array.IndexOf(args, "--skip") + 1]);
}
var fileName = args[Array.IndexOf(args, "--file") + 1];
string dir = null;
string indexName = null;
if (Array.IndexOf(args, "--dir") > 0)
{
dir = args[Array.IndexOf(args, "--dir") + 1];
}
if (Array.IndexOf(args, "--name") > 0)
{
indexName = args[Array.IndexOf(args, "--name") + 1];
}
var url = ConfigurationManager.AppSettings.Get("resin.endpoint");
var inproc = !string.IsNullOrWhiteSpace(dir);
IndexWriter w = inproc ? new IndexWriter(dir, new Analyzer(), new Tfidf()) : null;
Console.Write(inproc ? "Writing " : "Collecting docs ");
var cursorPos = Console.CursorLeft;
var count = 0;
var docs = new List <Dictionary <string, string> >();
var timer = new Stopwatch();
timer.Start();
using (var fs = File.Open(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (var bs = new BufferedStream(fs))
using (var sr = new StreamReader(bs))
{
string line;
sr.ReadLine();
while (skipped++ < skip)
{
sr.ReadLine();
}
while ((line = sr.ReadLine()) != null)
{
if (line[0] == ']')
{
break;
}
var doc = JsonConvert.DeserializeObject <Dictionary <string, string> >(line.Substring(0, line.Length - 1));
Console.SetCursorPosition(cursorPos, Console.CursorTop);
Console.Write(++count);
if (inproc)
{
w.Write(doc);
}
else
{
docs.Add(doc);
}
if (count == take)
{
break;
}
}
Console.WriteLine();
}
if (inproc)
{
w.Dispose();
}
else
{
Console.Write("Executing HTTP POST");
using (var client = new WriterClient(indexName, url))
{
client.Write(docs);
}
}
Console.Write("total time elapsed {0}", timer.Elapsed);
}