private void Write(IndexJob job) { try { var docCount = 0; var timer = new Stopwatch(); timer.Start(); foreach (var doc in job.Documents) { var docId = (ulong)doc["__docid"]; var keys = doc.Keys .Cast <string>() .Where(x => !x.StartsWith("__")); foreach (var key in keys) { var keyHash = key.ToHash(); var keyId = SessionFactory.GetKeyId(keyHash); VectorNode ix; if (!_dirty.TryGetValue(keyId, out ix)) { ix = GetIndex(keyHash) ?? new VectorNode(); _dirty.Add(keyId, ix); } var val = (IComparable)doc[key]; var str = val as string; var tokens = new HashSet <string>(); if (str == null || key[0] == '_') { tokens.Add(val.ToString()); } else { var tokenlist = _tokenizer.Tokenize(str); foreach (var token in tokenlist) { tokens.Add(token); } } _buildQueue.Enqueue(new BuildJob(CollectionId, docId, tokens, ix)); } if (++docCount == 100) { _log.Log(string.Format("analyzed doc {0}", doc["__docid"])); docCount = 0; } } _log.Log(string.Format("executed {0} analyze job in {1}", job.CollectionId, timer.Elapsed)); } catch (Exception ex) { _log.Log(ex.ToString()); throw; } }
public void WriteToIndex(IndexJob job) { _indexQueue.Enqueue(job); }