//static void Analyze(string[] args)
//{
// var dir = args[Array.IndexOf(args, "--dir") + 1];
// var field = args[Array.IndexOf(args, "--field") + 1];
// var timer = new Stopwatch();
// timer.Start();
// var fileName = Path.Combine(dir, field.ToHash() + ".tr");
// var trie = Trie.Load(fileName);
// var tokens = trie.All().Take(100).ToList();
// Console.WriteLine("Tokens fetched from disk in {0} ms. Writing...\r\n", timer.ElapsedMilliseconds);
// File.WriteAllLines(Path.Combine(dir, "_" + field + ".txt"), tokens);
//}
//static void Optimize(string[] args)
//{
// var dir = args[Array.IndexOf(args, "--dir") + 1];
// var truncate = Array.IndexOf(args, "--truncate") > 0;
// var timer = new Stopwatch();
// timer.Start();
// var ixFileName = Helper.GetFileNameOfLatestIndex(dir);
// var ix = IxFile.Load(ixFileName);
// var dix = DixFile.Load(Path.Combine(dir, ix.DixFileName));
// var fix = FixFile.Load(Path.Combine(dir, ix.FixFileName));
// var optimizer = new Optimizer(dir, ixFileName, dix, fix);
// optimizer.Rebase();
// if (truncate) optimizer.Truncate();
// optimizer.Save();
//}
static void Remove(string[] args)
{
string dir = null;
string indexName = null;
if (Array.IndexOf(args, "--dir") > 0)
{
dir = args[Array.IndexOf(args, "--dir") + 1];
}
if (Array.IndexOf(args, "--name") > 0)
{
indexName = args[Array.IndexOf(args, "--name") + 1];
}
var inproc = !string.IsNullOrWhiteSpace(dir);
var docId = args[Array.IndexOf(args, "--docid") + 1];
var timer = new Stopwatch();
timer.Start();
if (inproc)
{
using (var writer = new IndexWriter(dir, new Analyzer(), new Tfidf()))
{
writer.Remove(docId);
}
}
else
{
var url = ConfigurationManager.AppSettings.Get("resin.endpoint");
using (var client = new WriterClient(indexName, url))
{
client.Remove(docId);
}
}
Console.WriteLine("deleted {0} in {1}", docId, timer.Elapsed);
}
static void Write(string[] args)
{
var take = 1000;
if (Array.IndexOf(args, "--take") > 0)
{
take = int.Parse(args[Array.IndexOf(args, "--take") + 1]);
}
var fileName = args[Array.IndexOf(args, "--file") + 1];
string dir = null;
string indexName = null;
if (Array.IndexOf(args, "--dir") > 0)
{
dir = args[Array.IndexOf(args, "--dir") + 1];
}
if (Array.IndexOf(args, "--name") > 0)
{
indexName = args[Array.IndexOf(args, "--name") + 1];
}
var url = ConfigurationManager.AppSettings.Get("sir.endpoint");
var inproc = !string.IsNullOrWhiteSpace(dir);
Console.WriteLine("writing...");
var docs = new List <Dictionary <string, string> >();
var writeTimer = new Stopwatch();
writeTimer.Start();
if (inproc)
{
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
using (var writer = new StreamWriteOperation(dir, new Analyzer(), fileName, take))
{
writer.Execute();
}
}
else
{
Console.WriteLine("Executing HTTP POST");
using (var client = new WriterClient(indexName, url))
{
client.Write(docs);
}
}
Console.WriteLine("write operation took {0}", writeTimer.Elapsed);
}
static void Write(string[] args)
{
var take = 1000;
var skip = 0;
var skipped = 0;
if (Array.IndexOf(args, "--take") > 0)
{
take = int.Parse(args[Array.IndexOf(args, "--take") + 1]);
}
if (Array.IndexOf(args, "--skip") > 0)
{
skip = int.Parse(args[Array.IndexOf(args, "--skip") + 1]);
}
var fileName = args[Array.IndexOf(args, "--file") + 1];
string dir = null;
string indexName = null;
if (Array.IndexOf(args, "--dir") > 0)
{
dir = args[Array.IndexOf(args, "--dir") + 1];
}
if (Array.IndexOf(args, "--name") > 0)
{
indexName = args[Array.IndexOf(args, "--name") + 1];
}
var url = ConfigurationManager.AppSettings.Get("resin.endpoint");
var inproc = !string.IsNullOrWhiteSpace(dir);
IndexWriter w = inproc ? new IndexWriter(dir, new Analyzer(), new Tfidf()) : null;
Console.Write(inproc ? "Writing " : "Collecting docs ");
var cursorPos = Console.CursorLeft;
var count = 0;
var docs = new List <Dictionary <string, string> >();
var timer = new Stopwatch();
timer.Start();
using (var fs = File.Open(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (var bs = new BufferedStream(fs))
using (var sr = new StreamReader(bs))
{
string line;
sr.ReadLine();
while (skipped++ < skip)
{
sr.ReadLine();
}
while ((line = sr.ReadLine()) != null)
{
if (line[0] == ']')
{
break;
}
var doc = JsonConvert.DeserializeObject <Dictionary <string, string> >(line.Substring(0, line.Length - 1));
Console.SetCursorPosition(cursorPos, Console.CursorTop);
Console.Write(++count);
if (inproc)
{
w.Write(doc);
}
else
{
docs.Add(doc);
}
if (count == take)
{
break;
}
}
Console.WriteLine();
}
if (inproc)
{
w.Dispose();
}
else
{
Console.Write("Executing HTTP POST");
using (var client = new WriterClient(indexName, url))
{
client.Write(docs);
}
}
Console.Write("total time elapsed {0}", timer.Elapsed);
}
