public override void WriteEndVersion(Process process, AbstractConnection input, Entity entity, bool force = false) {
if (entity.Updates + entity.Inserts <= 0 && !force)
return;
var versionType = entity.Version == null ? "string" : entity.Version.SimpleType;
var end = entity.End ?? new DefaultFactory(Logger).Convert(entity.End, versionType);
using (var dir = LuceneDirectoryFactory.Create(this, TflBatchEntity(entity.ProcessName))) {
using (var writer = new IndexWriter(dir, new KeywordAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED)) {
var doc = new Document();
doc.fields.Add(new NumericField("id", Libs.Lucene.Net.Document.Field.Store.YES, true).SetIntValue(entity.TflBatchId));
doc.fields.Add(new Libs.Lucene.Net.Document.Field("process", entity.ProcessName, Libs.Lucene.Net.Document.Field.Store.YES, Libs.Lucene.Net.Document.Field.Index.NOT_ANALYZED_NO_NORMS));
doc.fields.Add(new Libs.Lucene.Net.Document.Field("connection", input.Name, Libs.Lucene.Net.Document.Field.Store.YES, Libs.Lucene.Net.Document.Field.Index.NOT_ANALYZED_NO_NORMS));
doc.fields.Add(new Libs.Lucene.Net.Document.Field("entity", entity.Alias, Libs.Lucene.Net.Document.Field.Store.YES, Libs.Lucene.Net.Document.Field.Index.NOT_ANALYZED_NO_NORMS));
doc.fields.Add(new NumericField("updates", Libs.Lucene.Net.Document.Field.Store.YES, true).SetLongValue(entity.Updates));
doc.fields.Add(new NumericField("inserts", Libs.Lucene.Net.Document.Field.Store.YES, true).SetLongValue(entity.Inserts));
doc.fields.Add(new NumericField("deletes", Libs.Lucene.Net.Document.Field.Store.YES, true).SetLongValue(entity.Deletes));
doc.fields.Add(LuceneWriter.CreateField("version", versionType, new SearchType { Analyzer = "keyword" }, end));
doc.fields.Add(new Libs.Lucene.Net.Document.Field("version_type", versionType, Libs.Lucene.Net.Document.Field.Store.YES, Libs.Lucene.Net.Document.Field.Index.NOT_ANALYZED_NO_NORMS));
doc.fields.Add(new NumericField("tflupdate", Libs.Lucene.Net.Document.Field.Store.YES, true).SetLongValue(DateTime.UtcNow.Ticks));
writer.AddDocument(doc);
writer.Commit();
writer.Optimize();
}
}
}
internal void AddDocument(Document.Document doc)
{
indexStream.WriteLong(fieldsStream.FilePointer);
System.Collections.Generic.IList<IFieldable> fields = doc.GetFields();
int storedCount = fields.Count(field => field.IsStored);
fieldsStream.WriteVInt(storedCount);
foreach(IFieldable field in fields)
{
if (field.IsStored)
WriteField(fieldInfos.FieldInfo(field.Name), field);
}
}
/// <summary>Adds field info for a Document. </summary>
public void Add(Document.Document doc)
{
lock (this)
{
System.Collections.Generic.IList<IFieldable> fields = doc.GetFields();
foreach(IFieldable field in fields)
{
Add(field.Name, field.IsIndexed, field.IsTermVectorStored,
field.IsStorePositionWithTermVector, field.IsStoreOffsetWithTermVector, field.OmitNorms,
false, field.OmitTermFreqAndPositions);
}
}
}
private void InitBlock(Document enclosingInstance)
{
this.enclosingInstance = enclosingInstance;
iter = Enclosing_Instance.fields.GetEnumerator();
}
public AnonymousClassEnumeration(Document enclosingInstance)
{
InitBlock(enclosingInstance);
}
// Add the size of field as a byte[] containing the 4 bytes of the integer byte size (high order byte first; char = 2 bytes)
// Read just the size -- caller must skip the field content to continue reading fields
// Return the size in bytes or chars, depending on field type
private int AddFieldSize(Document.Document doc, FieldInfo fi, bool binary, bool compressed)
{
int size = fieldsStream.ReadVInt(), bytesize = binary || compressed?size:2 * size;
var sizebytes = new byte[4];
sizebytes[0] = (byte) (Number.URShift(bytesize, 24));
sizebytes[1] = (byte) (Number.URShift(bytesize, 16));
sizebytes[2] = (byte) (Number.URShift(bytesize, 8));
sizebytes[3] = (byte) bytesize;
doc.Add(new Field(fi.name, sizebytes, Field.Store.YES));
return size;
}
private void AddField(Document.Document doc, FieldInfo fi, bool binary, bool compressed, bool tokenize)
{
//we have a binary stored field, and it may be compressed
if (binary)
{
int toRead = fieldsStream.ReadVInt();
var b = new byte[toRead];
fieldsStream.ReadBytes(b, 0, b.Length);
doc.Add(compressed ? new Field(fi.name, Uncompress(b), Field.Store.YES) : new Field(fi.name, b, Field.Store.YES));
}
else
{
const Field.Store store = Field.Store.YES;
Field.Index index = FieldExtensions.ToIndex(fi.isIndexed, tokenize);
Field.TermVector termVector = FieldExtensions.ToTermVector(fi.storeTermVector, fi.storeOffsetWithTermVector, fi.storePositionWithTermVector);
AbstractField f;
if (compressed)
{
int toRead = fieldsStream.ReadVInt();
var b = new byte[toRead];
fieldsStream.ReadBytes(b, 0, b.Length);
f = new Field(fi.name, false, System.Text.Encoding.GetEncoding("UTF-8").GetString(Uncompress(b)), store, index,
termVector) {OmitTermFreqAndPositions = fi.omitTermFreqAndPositions, OmitNorms = fi.omitNorms};
}
else
{
f = new Field(fi.name, false, fieldsStream.ReadString(), store, index, termVector)
{OmitTermFreqAndPositions = fi.omitTermFreqAndPositions, OmitNorms = fi.omitNorms};
}
doc.Add(f);
}
}
private void AddFieldLazy(Document.Document doc, FieldInfo fi, bool binary, bool compressed, bool tokenize)
{
if (binary)
{
int toRead = fieldsStream.ReadVInt();
long pointer = fieldsStream.FilePointer;
//was: doc.add(new Fieldable(fi.name, b, Fieldable.Store.YES));
doc.Add(new LazyField(this, fi.name, Field.Store.YES, toRead, pointer, binary, compressed));
//Need to move the pointer ahead by toRead positions
fieldsStream.Seek(pointer + toRead);
}
else
{
const Field.Store store = Field.Store.YES;
Field.Index index = FieldExtensions.ToIndex(fi.isIndexed, tokenize);
Field.TermVector termVector = FieldExtensions.ToTermVector(fi.storeTermVector, fi.storeOffsetWithTermVector, fi.storePositionWithTermVector);
AbstractField f;
if (compressed)
{
int toRead = fieldsStream.ReadVInt();
long pointer = fieldsStream.FilePointer;
f = new LazyField(this, fi.name, store, toRead, pointer, binary, compressed);
//skip over the part that we aren't loading
fieldsStream.Seek(pointer + toRead);
f.OmitNorms = fi.omitNorms;
f.OmitTermFreqAndPositions = fi.omitTermFreqAndPositions;
}
else
{
int length = fieldsStream.ReadVInt();
long pointer = fieldsStream.FilePointer;
//Skip ahead of where we are by the length of what is stored
if (format >= FieldsWriter.FORMAT_VERSION_UTF8_LENGTH_IN_BYTES)
{
fieldsStream.Seek(pointer + length);
}
else
{
fieldsStream.SkipChars(length);
}
f = new LazyField(this, fi.name, store, index, termVector, length, pointer, binary, compressed)
{OmitNorms = fi.omitNorms, OmitTermFreqAndPositions = fi.omitTermFreqAndPositions};
}
doc.Add(f);
}
}
public static Document CreateDocument(Dictionary<string, KeyValuePair<Field, SearchType>> fieldMap, Row row) {
var doc = new Document();
foreach (var pair in fieldMap) {
var name = pair.Key;
var field = pair.Value.Key;
var searchType = pair.Value.Value;
if (!searchType.Store && !searchType.Index)
continue;
doc.Add(CreateField(name, field.SimpleType, searchType, row[field.Alias]));
}
return doc;
}