public object Detect()
{
if (string.IsNullOrEmpty(_context.Entity.Version))
{
return(null);
}
var version = _context.Entity.GetVersionField();
_context.Debug(() => $"Detecting max output version: {_context.Connection.Folder}.{_context.Entity.Alias}.{version.Alias}.");
var tflDeleted = _context.Entity.TflDeleted();
var sort = new Sort(new SortField(version.Alias, LuceneConversion.TypeSort(version.Type), true));
using (var searcher = _searcherFactory.Create()) {
var hits = searcher.Search(LuceneConversion.TypeSearch(tflDeleted, tflDeleted.Alias, false), null, 1, sort);
if (hits.TotalHits > 0)
{
var doc = searcher.Doc(hits.ScoreDocs[0].Doc);
var value = doc.Get(version.Alias);
_context.Debug(() => $"Found value: {value}");
return(version.Convert(value));
}
}
_context.Debug(() => "Did not find max output version");
return(null);
}
public object Detect()
{
if (string.IsNullOrEmpty(_context.Entity.Version))
{
return(null);
}
using (var searcher = _searcherFactory.Create()) {
var version = _context.Entity.GetVersionField();
_context.Debug(() => $"Detecting max input version: {_context.Connection.Folder}:{version.Name}.");
var hits = searcher.Search(new MatchAllDocsQuery(), null, 1,
new Sort(new SortField(version.Name, LuceneConversion.TypeSort(version.Type), true))
);
if (hits.TotalHits > 0)
{
var doc = searcher.Doc(hits.ScoreDocs[0].Doc);
var value = doc.Get(version.Name);
_context.Debug(() => $"Found value: {value}");
return(version.Convert(value));
}
}
_context.Debug(() => "Did not find max input version");
return(null);
}
public void Write(IEnumerable <IRow> rows)
{
var tflKey = _output.Entity.TflKey();
using (var searcher = _searcherFactory.Create()) {
using (var writer = _writerFactory.Create()) {
foreach (var row in rows)
{
var tflId = string.Concat(_primaryKey.Select(pk => row[pk].ToString()));
var doc = new Document();
foreach (var field in _fieldSearchTypes.Where(field => field.SearchType.Store || field.SearchType.Index))
{
doc.Add(CreateField(field, row[field.Field]));
}
doc.Add(new LuceneField("TflId", tflId, LuceneField.Store.YES, LuceneField.Index.NOT_ANALYZED_NO_NORMS));
if (_output.Entity.IsFirstRun)
{
writer.AddDocument(doc);
}
else
{
var term = new Term("TflId", tflId);
var hits = searcher.Search(new TermQuery(term), null, 1);
if (hits.TotalHits > 0)
{
var old = searcher.Doc(hits.ScoreDocs[0].Doc);
doc.RemoveField(tflKey.Alias);
doc.Add(new NumericField(tflKey.Alias, LuceneField.Store.YES, true).SetIntValue(Convert.ToInt32(old.Get(tflKey.Alias))));
writer.UpdateDocument(term, doc);
}
else
{
writer.AddDocument(doc);
}
}
}
writer.Commit();
writer.Optimize();
}
}
}
// get max tflbatchid, max tflkey
public override void Start()
{
base.Start();
var tflBatchId = Context.Entity.TflBatchId();
var tflKey = Context.Entity.TflKey();
using (var searcher = _searcherFactory.Create()) {
var batchHits = searcher.Search(new MatchAllDocsQuery(), null, 1,
new Sort(new SortField(tflBatchId.Alias, LuceneConversion.TypeSort(tflBatchId.Type), true))
);
Context.Entity.BatchId = (batchHits.TotalHits > 0 ? System.Convert.ToInt32(searcher.Doc(batchHits.ScoreDocs[0].Doc).Get(tflBatchId.Alias)) : 0) + 1;
var keyHits = searcher.Search(new MatchAllDocsQuery(), null, 1,
new Sort(new SortField(tflKey.Alias, LuceneConversion.TypeSort(tflKey.Type), true))
);
Context.Entity.Identity = (keyHits.TotalHits > 0 ? System.Convert.ToInt32(searcher.Doc(keyHits.ScoreDocs[0].Doc).Get(tflKey.Alias)) : 0);
}
Context.Debug(() => $"Next {tflBatchId.Alias}: {Context.Entity.BatchId}.");
Context.Debug(() => $"Last {tflKey.Alias}: {Context.Entity.Identity}.");
using (var reader = _readerFactory.Create()) {
Context.Entity.IsFirstRun = Context.Entity.MinVersion == null && reader.NumDocs() == 0;
}
}
public IEnumerable <IRow> Read()
{
var reader = _readerFactory.Create();
var numDocs = reader.NumDocs();
var selector = new MapFieldSelector(_fields.Select(f => f.Name).ToArray());
using (var searcher = _searcherFactory.Create()) {
// read from input? consider filters, and field names
if (_readFrom == ReadFrom.Input)
{
if (_context.Entity.Filter.Any())
{
var queryFields = _context.Entity.Filter.Select(f => f.Field).ToArray();
var query = string.Join(" ", _context.Entity.Filter.Select(f => "(" + (string.IsNullOrEmpty(f.Expression) ? f.Field + ":" + f.Value : f.Expression) + ") " + f.Continuation.ToUpper()));
query = query.Remove(query.Length - 3);
var topFieldCollector = TopFieldCollector.Create(Sort.INDEXORDER, numDocs, false, false, false, false);
searcher.Search(new MultiFieldQueryParser(V, queryFields, _analyzer).Parse(query), topFieldCollector);
var topDocs = topFieldCollector.TopDocs();
if (topDocs == null)
{
yield break;
}
for (var i = 0; i < topDocs.TotalHits; i++)
{
var row = _rowFactory.Create();
var doc = searcher.Doc(i, selector);
foreach (var field in _fields)
{
row[field] = field.Convert(doc.Get(field.Name));
}
yield return(row);
}
}
else
{
for (var i = 0; i < numDocs; i++)
{
if (reader.IsDeleted(i))
{
continue;
}
var doc = reader.Document(i, selector);
var row = _rowFactory.Create();
foreach (var field in _fields)
{
row[field] = field.Convert(doc.Get(field.Name));
}
yield return(row);
}
}
}
else // read from output? consider tfldeleted and field aliases
{
var tflDeleted = _context.Entity.TflDeleted();
var collector = TopFieldCollector.Create(Sort.INDEXORDER, numDocs, false, false, false, false);
searcher.Search(LuceneConversion.TypeSearch(tflDeleted, tflDeleted.Alias, false), collector);
var topDocs = collector.TopDocs();
if (topDocs == null)
{
yield break;
}
for (var i = 0; i < topDocs.TotalHits; i++)
{
var row = _rowFactory.Create();
var doc = searcher.Doc(i, selector);
foreach (var field in _fields)
{
row[field] = field.Convert(doc.Get(field.Alias));
}
yield return(row);
}
}
}
}