public void TestRealValuedWeightsVsRepeatWeighting()
{
var rvfes1 = new RealValueFileEventStream(Tests.OpenFile("opennlp/data/maxent/real-valued-weights-training-data.txt"));
var rvfes2 = new FileEventStream(Tests.OpenFile("opennlp/data/maxent/repeat-weighting-training-data.txt"));
var realModel = GIS.TrainModel(100, new OnePassRealValueDataIndexer(rvfes1, 1));
var repeatModel = GIS.TrainModel(100, new OnePassRealValueDataIndexer(rvfes2, 1));
rvfes1.Dispose();
rvfes2.Dispose();
Assert.NotNull(realModel);
Assert.NotNull(repeatModel);
var features2Classify = new[] { "feature2", "feature5" };
var realResults = realModel.Eval(features2Classify);
var repeatResults = repeatModel.Eval(features2Classify);
Assert.AreEqual(realResults.Length, repeatResults.Length);
for (var i = 0; i < realResults.Length; i++)
{
Assert.AreEqual(realResults[i], repeatResults[i], 0.01f);
}
features2Classify = new[] { "feature1", "feature2", "feature3", "feature4", "feature5" };
realResults = realModel.Eval(features2Classify, new[] { 5.5f, 6.1f, 9.1f, 4.0f, 1.8f });
repeatResults = repeatModel.Eval(features2Classify, new[] { 5.5f, 6.1f, 9.1f, 4.0f, 1.8f });
Assert.AreEqual(realResults.Length, repeatResults.Length);
for (var i = 0; i < realResults.Length; i++)
{
Assert.AreEqual(realResults[i], repeatResults[i], 0.01f);
}
}
public void TestRealValuedWeightsVsRepeatWeighting() {
var rvfes1 = new RealValueFileEventStream(Tests.OpenFile("opennlp/data/maxent/real-valued-weights-training-data.txt"));
var rvfes2 = new FileEventStream(Tests.OpenFile("opennlp/data/maxent/repeat-weighting-training-data.txt"));
var realModel = GIS.TrainModel(100, new OnePassRealValueDataIndexer(rvfes1, 1));
var repeatModel = GIS.TrainModel(100, new OnePassRealValueDataIndexer(rvfes2, 1));
rvfes1.Dispose();
rvfes2.Dispose();
Assert.NotNull(realModel);
Assert.NotNull(repeatModel);
var features2Classify = new[] {"feature2", "feature5"};
var realResults = realModel.Eval(features2Classify);
var repeatResults = repeatModel.Eval(features2Classify);
Assert.AreEqual(realResults.Length, repeatResults.Length);
for (var i = 0; i < realResults.Length; i++) {
Assert.AreEqual(realResults[i], repeatResults[i], 0.01f);
}
features2Classify = new[] {"feature1", "feature2", "feature3", "feature4", "feature5"};
realResults = realModel.Eval(features2Classify, new[] {5.5f, 6.1f, 9.1f, 4.0f, 1.8f});
repeatResults = repeatModel.Eval(features2Classify, new[] {5.5f, 6.1f, 9.1f, 4.0f, 1.8f});
Assert.AreEqual(realResults.Length, repeatResults.Length);
for (var i = 0; i < realResults.Length; i++) {
Assert.AreEqual(realResults[i], repeatResults[i], 0.01f);
}
}
public void PublishEvents(IEnumerable <ModelEvent> events)
{
UpdateEventCache();
var eventsToPublish = events.Where(e => !_publishedEvents.Contains(e.EventID)).OrderBy(e => e.EventVector).ToList();
if (eventsToPublish.Count == 0)
{
return;
}
var streamId = Guid.NewGuid();
EventStreamHeader header = new EventStreamHeader(eventsToPublish[0].EventVector, eventsToPublish.Last().EventVector, _deviceId, streamId);
string filePath = Path.Combine(_deviceDirectory, streamId.ToString() + ".eventstream");
using (Stream file = new FileStream(filePath, FileMode.CreateNew, FileAccess.Write))
using (EventStreamWriter writer = new EventStreamWriter(file, header, true))
{
writer.WriteEvents(eventsToPublish);
}
FileEventStream stream = new FileEventStream(filePath);
_cachedEventStreams.Add(filePath, stream);
var streamEventIDs = eventsToPublish.Select(e => e.EventID).ToList();
foreach (var eventId in streamEventIDs)
{
if (!_publishedEvents.Add(eventId))
{
throw new InvalidOperationException("This eventID already exists!");
}
}
_eventsInStreams.Add(filePath, streamEventIDs);
}
private void UpdateEventCache()
{
foreach (var directory in Directory.EnumerateDirectories(_rootDirectory))
{
foreach (var eventFile in Directory.EnumerateFiles(directory, "*.eventstream"))
{
if (!_cachedEventStreams.ContainsKey(eventFile))
{
FileEventStream stream = new FileEventStream(eventFile);
_cachedEventStreams.Add(eventFile, stream);
var streamEventIDs = stream.CreateEventIterator().Select(e => e.EventID).ToList();
foreach (var eventId in streamEventIDs)
{
if (!_publishedEvents.Add(eventId))
{
throw new InvalidOperationException("This eventID already exists!");
}
}
_eventsInStreams.Add(eventFile, streamEventIDs);
}
}
}
}
protected override void PerformIndexing() {
Display("Indexing events using cutoff of " + Cutoff);
Display("\tComputing event counts...");
var fileName = Path.GetTempFileName();
var predicateIndex = new Dictionary<string, int>();
using (var file = new FileStream(fileName, FileMode.Open, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.DeleteOnClose)) {
var writer = new StreamWriter(new UnclosableStream(file), Encoding.UTF8);
int numEvents = ComputeEventCounts(writer, predicateIndex);
Display("done. " + numEvents + " events");
file.Seek(0, SeekOrigin.Begin);
Display("\tIndexing...");
var fes = new FileEventStream(file);
var eventsToCompare = Index(fes, predicateIndex);
file.Close();
Display("done.");
Display(Sort ? "Sorting and merging events..." : "Collecting events...");
SortAndMerge(eventsToCompare, Sort);
Display("Done indexing.");
}
}