public static LDocument Convert(Document doc, Schema schema)
{
var ldoc = new LDocument();
foreach (var sf in schema.Fields)
{
foreach (var lf in Convert(sf, doc))
{
ldoc.Add(lf);
}
}
ldoc.Add(
new LField(
SchemaNameField,
schema.Name,
ConvertToStore(true, false),
ConvertToIndexFlag(false, false)
)
);
ldoc.Add(
new LField(
SchemaVersionField,
schema.Version,
ConvertToStore(true, false),
ConvertToIndexFlag(false, false)
)
);
return ldoc;
}
public void Update(Document doc, IndexWorkItem workItem)
{
Update(
new IndexOperation { Document = doc, Type = IndexOperationType.Save },
workItem
);
}
public IEnumerable<Field> GetUniqueFields(Document doc)
{
var blah = from f in doc.Fields.All
join sf in this.Fields on f.Name equals sf.Name
where sf.Unique
orderby f.Name
select f;
return blah;
}
public static Document GetDocument()
{
var doc = new Document();
doc.Fields.Add(new Field { Name = "ID", Values = { "12345" } });
doc.Fields.Add(new Field { Name = "Title", Values = { "Kurt Rocks" } });
doc.Fields.Add(new Field { Name = "Text", Values = { "Lorem Ipsum Kurtum Spurtum Kurt rules" } });
doc.Fields.Add(new Field { Name = "Tag", Values = { "Kurt", "Owns", "Joo" } });
doc.Schema = GetSchema();
return doc;
}
public static IEnumerable<LField> Convert(SchemaField field, Document doc)
{
var f = doc.Fields[field.Name];
if (f == null) yield break;
var values = field.MultiValued ? f.Values.ToArray() : new string[]{ f.GetValue() };
foreach (var value in values)
{
yield return new LField(
field.Name,
FieldHelper.Format(field, value),
ConvertToStore(field.Stored, field.Compressed),
ConvertToIndexFlag(field.Indexed, field.Tokenized)
);
}
}
public void Update(Document document, IndexWorkItem workItem)
{
//TODO: Validate against the supplied schema
this.writer.Update(document, workItem);
}
public IndexWorkItem Update(Document document)
{
var wi = new IndexWorkItem();
Update(document, wi);
return wi;
}
public void Indexing()
{
var schema = new Schema("PerfTest", "1.0")
{
Fields =
{
new SchemaField{ Name = "ID", Indexed = true, Stored = true, Tokenized = false, Unique = true},
new SchemaField{ Name = "Title",Indexed = true, Stored = true, Tokenized = true, Unique = false},
new SchemaField{ Name = "Body", Indexed = true, Stored = true, Tokenized = true, Unique = false}
},
DefaultSearchField = "Body"
};
var docs = new List<Document>();
var fields = new string[] { "ID", "Title", "Body" };
foreach (var file in System.IO.Directory.GetFiles("ReutersData", "*.sgm"))
{
var text = System.IO.File.ReadAllText(file);
var matches = reutersSgm.Matches(text);
foreach (Match match in matches)
{
if (match.Success)
{
var doc = new Document();
doc.Schema = schema;
foreach(var key in fields)
{
doc.Fields.Add(new Field{ Name = key, Values = { match.Groups[key].Value }});
}
docs.Add(doc);
}
}
}
if (System.IO.Directory.Exists("PerfIndex"))
{
System.IO.Directory.Delete("PerfIndex", true);
}
var idx = Index.Open("PerfIndex");
idx.Update(schema);
var rand = new Random();
var max = docs.Count;
var results = new List<Mubble.Indexing.IndexWorkItem>();
var curr = 0;
Mubble.Treadmill.Func<Mubble.Treadmill.Result> action = () =>
{
var i = System.Threading.Interlocked.Increment(ref curr);
if (i > docs.Count)
{
i = i % docs.Count;
}
var wi = idx.Update(docs[i]);
results.Add(wi);
return new Mubble.Treadmill.Result(true, "Indexed!");
};
var cpu = new System.Diagnostics.PerformanceCounter("Processor", "% Processor Time", "_Total", true);
Mubble.Treadmill.Func<Mubble.Treadmill.Benchmark> bench = () =>
{
var start = DateTime.Now;
var r = idx.Search("Body:output");
var msg = string.Concat("Search for output - ", r.Hits.Count, " results");
var elapsed = DateTime.Now - start;
return new Mubble.Treadmill.Benchmark(msg, elapsed.Milliseconds, DateTime.Now);
};
Mubble.Treadmill.Func<Mubble.Treadmill.Benchmark> proc = () =>
{
var count = cpu.NextValue();
return new Mubble.Treadmill.Benchmark("Processor", count, DateTime.Now);
};
var actions = new List<Mubble.Treadmill.Command>(1);
var benchmarks = new List<Mubble.Treadmill.Collect>(1);
actions.Add(new Mubble.Treadmill.Command(action, "Index doc"));
benchmarks.Add(new Mubble.Treadmill.Collect(bench));
benchmarks.Add(new Mubble.Treadmill.Collect(proc));
var commands = new Mubble.Treadmill.Commands(actions, () =>
{
var total = results.Count;
Mubble.Tests.Indexing.WaitFor(results);
idx.Dispose();
return new Mubble.Treadmill.Result(true, string.Concat("Waited for ", total, " operations"));
});
Mubble.Treadmill.Runner.Start(50000, 8, commands, benchmarks);
}
public void Save(Document doc, object id)
{
Save(doc, id, RowStatus.Active);
}
public void Save(Document doc)
{
Save(doc, null);
}
void Save(Document doc, object id, RowStatus status)
{
var sql = this.settings.GetUpdateSql();
using (var cn = new SqlConnection(this.settings.ConnectionString))
using (var cmd = new SqlCommand(sql, cn))
{
cmd.Parameters.AddWithValue("@ID", id);
cmd.Parameters.AddWithValue("@Document", doc != null ? (object)Utility.Serialize(doc) : DBNull.Value);
cmd.Parameters.AddWithValue("@Status", status);
cn.Open();
cmd.ExecuteNonQuery();
cn.Close();
}
}
