public static LDocument Convert(Document doc, Schema schema)
{
var ldoc = new LDocument();
foreach (var sf in schema.Fields)
{
foreach (var lf in Convert(sf, doc))
{
ldoc.Add(lf);
}
}
ldoc.Add(
new LField(
SchemaNameField,
schema.Name,
ConvertToStore(true, false),
ConvertToIndexFlag(false, false)
)
);
ldoc.Add(
new LField(
SchemaVersionField,
schema.Version,
ConvertToStore(true, false),
ConvertToIndexFlag(false, false)
)
);
return ldoc;
}
public void Save(Schema schema)
{
using (var file = File.Create(IndexHelper.SchemaPath(this.index, schema)))
{
Utility.Serialize(schema, file);
}
}
public void AdHocSchema()
{
var doc = GetDocument();
var schema = new Mubble.Indexing.Schema("mubble", "1.1");
schema.Fields.Add(new Mubble.Indexing.SchemaField { Name = "ID", Unique = true, Tokenized = false, Stored = true });
schema.Fields.Add(new Mubble.Indexing.SchemaField { Name = "Title", Stored = true });
schema.Fields.Add(new Mubble.Indexing.SchemaField { Name = "Text", Stored = true });
schema.Fields.Add(new Mubble.Indexing.SchemaField { Name = "Tag", Stored = true, MultiValued = true, Tokenized = false });
doc.Schema = schema;
var index = Index.Open("LuceneIndex");
var wi = index.Update(doc);
WaitFor(wi);
Assert.IsFalse(IndexWorkItemStatus.Error == wi.Status, "Error indexing document");
}
public void Update(Schema schema)
{
this.schemaManager.Save(schema);
}
static Schema GetSchema()
{
var schema = new Mubble.Indexing.Schema("mubble", "1.0");
schema.Fields.Add(new Mubble.Indexing.SchemaField { Name = "ID", Unique = true, Tokenized = false, Stored = true });
schema.Fields.Add(new Mubble.Indexing.SchemaField { Name = "Title", Stored = true });
schema.Fields.Add(new Mubble.Indexing.SchemaField { Name = "Text", Stored = true });
schema.Fields.Add(new Mubble.Indexing.SchemaField { Name = "Tag", Stored = true, MultiValued = true, Tokenized = false });
return schema;
}
public static string SchemaPath(this Index index, Schema schema)
{
return SchemaPath(index, schema.Name, schema.Version);
}
public void Indexing()
{
var schema = new Schema("PerfTest", "1.0")
{
Fields =
{
new SchemaField{ Name = "ID", Indexed = true, Stored = true, Tokenized = false, Unique = true},
new SchemaField{ Name = "Title",Indexed = true, Stored = true, Tokenized = true, Unique = false},
new SchemaField{ Name = "Body", Indexed = true, Stored = true, Tokenized = true, Unique = false}
},
DefaultSearchField = "Body"
};
var docs = new List<Document>();
var fields = new string[] { "ID", "Title", "Body" };
foreach (var file in System.IO.Directory.GetFiles("ReutersData", "*.sgm"))
{
var text = System.IO.File.ReadAllText(file);
var matches = reutersSgm.Matches(text);
foreach (Match match in matches)
{
if (match.Success)
{
var doc = new Document();
doc.Schema = schema;
foreach(var key in fields)
{
doc.Fields.Add(new Field{ Name = key, Values = { match.Groups[key].Value }});
}
docs.Add(doc);
}
}
}
if (System.IO.Directory.Exists("PerfIndex"))
{
System.IO.Directory.Delete("PerfIndex", true);
}
var idx = Index.Open("PerfIndex");
idx.Update(schema);
var rand = new Random();
var max = docs.Count;
var results = new List<Mubble.Indexing.IndexWorkItem>();
var curr = 0;
Mubble.Treadmill.Func<Mubble.Treadmill.Result> action = () =>
{
var i = System.Threading.Interlocked.Increment(ref curr);
if (i > docs.Count)
{
i = i % docs.Count;
}
var wi = idx.Update(docs[i]);
results.Add(wi);
return new Mubble.Treadmill.Result(true, "Indexed!");
};
var cpu = new System.Diagnostics.PerformanceCounter("Processor", "% Processor Time", "_Total", true);
Mubble.Treadmill.Func<Mubble.Treadmill.Benchmark> bench = () =>
{
var start = DateTime.Now;
var r = idx.Search("Body:output");
var msg = string.Concat("Search for output - ", r.Hits.Count, " results");
var elapsed = DateTime.Now - start;
return new Mubble.Treadmill.Benchmark(msg, elapsed.Milliseconds, DateTime.Now);
};
Mubble.Treadmill.Func<Mubble.Treadmill.Benchmark> proc = () =>
{
var count = cpu.NextValue();
return new Mubble.Treadmill.Benchmark("Processor", count, DateTime.Now);
};
var actions = new List<Mubble.Treadmill.Command>(1);
var benchmarks = new List<Mubble.Treadmill.Collect>(1);
actions.Add(new Mubble.Treadmill.Command(action, "Index doc"));
benchmarks.Add(new Mubble.Treadmill.Collect(bench));
benchmarks.Add(new Mubble.Treadmill.Collect(proc));
var commands = new Mubble.Treadmill.Commands(actions, () =>
{
var total = results.Count;
Mubble.Tests.Indexing.WaitFor(results);
idx.Dispose();
return new Mubble.Treadmill.Result(true, string.Concat("Waited for ", total, " operations"));
});
Mubble.Treadmill.Runner.Start(50000, 8, commands, benchmarks);
}
public IEnumerable<Field> GetUniqueFields(Schema schema)
{
return schema.GetUniqueFields(this);
}
public string GetSignature(Schema schema)
{
var values = GetValue();
return string.Concat(schema.Name, ".[", this.Name, "]=", values);
}