/// <summary>
/// Reads events from <tt>eventReader</tt> into a List<TrainingEvent>. The
/// predicates associated with each event are counted and any which
/// occur at least <tt>cutoff</tt> times are added to the
/// <tt>predicatesInOut</tt> dictionary along with a unique integer index.
/// </summary>
/// <param name="eventReader">
/// an <code>ITrainingEventReader</code> value
/// </param>
/// <param name="predicatesInOut">
/// a <code>Dictionary</code> value
/// </param>
/// <param name="cutoff">
/// an <code>int</code> value
/// </param>
/// <returns>
/// an <code>List of TrainingEvents</code> value
/// </returns>
private List <TrainingEvent> ComputeEventCounts(ITrainingEventReader eventReader, Dictionary <string, int> predicatesInOut, int cutoff)
{
var counter = new Dictionary <string, int>();
var events = new List <TrainingEvent>();
int predicateIndex = 0;
while (eventReader.HasNext())
{
TrainingEvent trainingEvent = eventReader.ReadNextEvent();
events.Add(trainingEvent);
string[] eventContext = trainingEvent.Context;
for (int currentEventContext = 0; currentEventContext < eventContext.Length; currentEventContext++)
{
if (!predicatesInOut.ContainsKey(eventContext[currentEventContext]))
{
if (counter.ContainsKey(eventContext[currentEventContext]))
{
counter[eventContext[currentEventContext]]++;
}
else
{
counter.Add(eventContext[currentEventContext], 1);
}
if (counter[eventContext[currentEventContext]] >= cutoff)
{
predicatesInOut.Add(eventContext[currentEventContext], predicateIndex++);
counter.Remove(eventContext[currentEventContext]);
}
}
}
}
return(events);
}
/// <summary>
/// Test whether there are any Events remaining in this EventReader.
/// </summary>
/// <returns>
/// true if this EventReader has more Events
/// </returns>
public virtual bool HasNext()
{
while (mNextEvent == null && mDataReader.HasNext())
{
mNextEvent = CreateEvent(mDataReader.NextToken());
}
return(mNextEvent != null);
}
private List <ComparableEvent> Index(List <TrainingEvent> events, Dictionary <string, int> predicateIndex)
{
var map = new Dictionary <string, int>();
int eventCount = events.Count;
int outcomeCount = 0;
var eventsToCompare = new List <ComparableEvent>(eventCount);
var indexedContext = new List <int>();
for (int eventIndex = 0; eventIndex < eventCount; eventIndex++)
{
TrainingEvent currentTrainingEvent = events[eventIndex];
string[] eventContext = currentTrainingEvent.Context;
ComparableEvent comparableEvent;
int outcomeIndex;
string outcome = currentTrainingEvent.Outcome;
if (map.ContainsKey(outcome))
{
outcomeIndex = map[outcome];
}
else
{
outcomeIndex = outcomeCount++;
map.Add(outcome, outcomeIndex);
}
for (int currentEventContext = 0; currentEventContext < eventContext.Length; currentEventContext++)
{
string predicate = eventContext[currentEventContext];
if (predicateIndex.ContainsKey(predicate))
{
indexedContext.Add(predicateIndex[predicate]);
}
}
// drop events with no active features
if (indexedContext.Count > 0)
{
comparableEvent = new ComparableEvent(outcomeIndex, indexedContext.ToArray());
eventsToCompare.Add(comparableEvent);
}
else
{
//"Dropped event " + oEvent.Outcome + ":" + oEvent.Context);
}
// recycle the list
indexedContext.Clear();
}
SetOutcomeLabels(ToIndexedStringArray(map));
SetPredicateLabels(ToIndexedStringArray(predicateIndex));
return(eventsToCompare);
}
/// <summary>
/// Constructor sets up the training event reader based on a stream of training data.
/// </summary>
/// <param name="dataReader">
/// Stream of training data.
/// </param>
public BasicEventReader(ITrainingDataReader<string> dataReader)
{
mContext = new BasicContextGenerator();
mDataReader = dataReader;
if (mDataReader.HasNext())
{
mNextEvent = CreateEvent(mDataReader.NextToken());
}
}
/// <summary>
/// Constructor sets up the training event reader based on a stream of training data.
/// </summary>
/// <param name="dataReader">
/// Stream of training data.
/// </param>
public BasicEventReader(ITrainingDataReader <string> dataReader)
{
mContext = new BasicContextGenerator();
mDataReader = dataReader;
if (mDataReader.HasNext())
{
mNextEvent = CreateEvent(mDataReader.NextToken());
}
}
public static string ToLine(TrainingEvent eventToConvert)
{
System.Text.StringBuilder lineBuilder = new System.Text.StringBuilder();
lineBuilder.Append(eventToConvert.Outcome);
string[] context = eventToConvert.GetContext();
for (int contextIndex = 0, contextLength = context.Length; contextIndex < contextLength; contextIndex++)
{
lineBuilder.Append(" " + context[contextIndex]);
}
lineBuilder.Append(System.Environment.NewLine);
return(lineBuilder.ToString());
}
private List <ComparableEvent> Index(int eventCount, ITrainingEventReader eventReader, Dictionary <string, int> predicateIndex)
{
var outcomeMap = new Dictionary <string, int>();
int outcomeCount = 0;
var eventsToCompare = new List <ComparableEvent>(eventCount);
var indexedContext = new List <int>();
while (eventReader.HasNext())
{
TrainingEvent currentTrainingEvent = eventReader.ReadNextEvent();
string[] eventContext = currentTrainingEvent.Context;
ComparableEvent comparableEvent;
int outcomeId;
string outcome = currentTrainingEvent.Outcome;
if (outcomeMap.ContainsKey(outcome))
{
outcomeId = outcomeMap[outcome];
}
else
{
outcomeId = outcomeCount++;
outcomeMap.Add(outcome, outcomeId);
}
for (int currentPredicate = 0; currentPredicate < eventContext.Length; currentPredicate++)
{
string predicate = eventContext[currentPredicate];
if (predicateIndex.ContainsKey(predicate))
{
indexedContext.Add(predicateIndex[predicate]);
}
}
// drop events with no active features
if (indexedContext.Count > 0)
{
comparableEvent = new ComparableEvent(outcomeId, indexedContext.ToArray());
eventsToCompare.Add(comparableEvent);
}
else
{
//"Dropped event " + currentTrainingEvent.Outcome + ":" + currentTrainingEvent.Context);
}
// recycle the list
indexedContext.Clear();
}
SetOutcomeLabels(ToIndexedStringArray(outcomeMap));
SetPredicateLabels(ToIndexedStringArray(predicateIndex));
return(eventsToCompare);
}
/// <summary>
/// Returns the next Event object held in this EventReader. Each call to ReadNextEvent advances the EventReader.
/// </summary>
/// <returns>
/// the Event object which is next in this EventReader
/// </returns>
public virtual TrainingEvent ReadNextEvent()
{
while (mNextEvent == null && mDataReader.HasNext())
{
mNextEvent = CreateEvent(mDataReader.NextToken());
}
TrainingEvent currentEvent = mNextEvent;
if (mDataReader.HasNext())
{
mNextEvent = CreateEvent(mDataReader.NextToken());
}
else
{
mNextEvent = null;
}
return currentEvent;
}
/// <summary>
/// Returns the next Event object held in this EventReader. Each call to ReadNextEvent advances the EventReader.
/// </summary>
/// <returns>
/// the Event object which is next in this EventReader
/// </returns>
public virtual TrainingEvent ReadNextEvent()
{
while (mNextEvent == null && mDataReader.HasNext())
{
mNextEvent = CreateEvent(mDataReader.NextToken());
}
TrainingEvent currentEvent = mNextEvent;
if (mDataReader.HasNext())
{
mNextEvent = CreateEvent(mDataReader.NextToken());
}
else
{
mNextEvent = null;
}
return(currentEvent);
}
/// <summary>
/// Reads events from <tt>eventStream</tt> into a dictionary. The
/// predicates associated with each event are counted and any which
/// occur at least <tt>cutoff</tt> times are added to the
/// <tt>predicatesInOut</tt> map along with a unique integer index.
/// </summary>
/// <param name="eventReader">
/// an <code>ITrainingEventReader</code> value
/// </param>
/// <param name="eventStoreFile">
/// a file name to which the events are written to for later processing.
/// </param>
/// <param name="predicatesInOut">
/// a <code>Dictionary</code> value
/// </param>
/// <param name="cutoff">
/// an <code>int</code> value
/// </param>
private int ComputeEventCounts(ITrainingEventReader eventReader, string eventStoreFile, Dictionary <string, int> predicatesInOut, int cutoff)
{
var counter = new Dictionary <string, int>();
int predicateIndex = 0;
int eventCount = 0;
#if DNF
using (var eventStoreWriter = new StreamWriter(eventStoreFile))
#else
using (var stream = new FileStream(eventStoreFile, FileMode.Open))
using (var eventStoreWriter = new StreamWriter(stream))
#endif
{
while (eventReader.HasNext())
{
TrainingEvent currentTrainingEvent = eventReader.ReadNextEvent();
eventCount++;
eventStoreWriter.Write(FileEventReader.ToLine(currentTrainingEvent));
string[] eventContext = currentTrainingEvent.Context;
for (int currentPredicate = 0; currentPredicate < eventContext.Length; currentPredicate++)
{
if (!predicatesInOut.ContainsKey(eventContext[currentPredicate]))
{
if (counter.ContainsKey(eventContext[currentPredicate]))
{
counter[eventContext[currentPredicate]]++;
}
else
{
counter.Add(eventContext[currentPredicate], 1);
}
if (counter[eventContext[currentPredicate]] >= cutoff)
{
predicatesInOut.Add(eventContext[currentPredicate], predicateIndex++);
counter.Remove(eventContext[currentPredicate]);
}
}
}
}
}
return(eventCount);
}
/// <summary>
/// Test whether there are any Events remaining in this EventReader.
/// </summary>
/// <returns>
/// true if this EventReader has more Events
/// </returns>
public virtual bool HasNext()
{
while (mNextEvent == null && mDataReader.HasNext())
{
mNextEvent = CreateEvent(mDataReader.NextToken());
}
return mNextEvent != null;
}
public static string ToLine(TrainingEvent eventToConvert)
{
var lineBuilder = new StringBuilder();
lineBuilder.Append(eventToConvert.Outcome);
string[] context = eventToConvert.Context;
for (int contextIndex = 0, contextLength = context.Length; contextIndex < contextLength; contextIndex++)
{
lineBuilder.Append(" " + context[contextIndex]);
}
lineBuilder.Append(System.Environment.NewLine);
return lineBuilder.ToString();
}