private void Write(IndexJob job)
{
try
{
var docCount = 0;
var timer = new Stopwatch();
timer.Start();
foreach (var doc in job.Documents)
{
var docId = (ulong)doc["__docid"];
var keys = doc.Keys
.Cast <string>()
.Where(x => !x.StartsWith("__"));
foreach (var key in keys)
{
var keyHash = key.ToHash();
var keyId = SessionFactory.GetKeyId(keyHash);
VectorNode ix;
if (!_dirty.TryGetValue(keyId, out ix))
{
ix = GetIndex(keyHash) ?? new VectorNode();
_dirty.Add(keyId, ix);
}
var val = (IComparable)doc[key];
var str = val as string;
var tokens = new HashSet <string>();
if (str == null || key[0] == '_')
{
tokens.Add(val.ToString());
}
else
{
var tokenlist = _tokenizer.Tokenize(str);
foreach (var token in tokenlist)
{
tokens.Add(token);
}
}
_buildQueue.Enqueue(new BuildJob(CollectionId, docId, tokens, ix));
}
if (++docCount == 100)
{
_log.Log(string.Format("analyzed doc {0}", doc["__docid"]));
docCount = 0;
}
}
_log.Log(string.Format("executed {0} analyze job in {1}",
job.CollectionId, timer.Elapsed));
}
catch (Exception ex)
{
_log.Log(ex.ToString());
throw;
}
}
public void WriteToIndex(IndexJob job)
{
_indexQueue.Enqueue(job);
}
