public WordPruningBreadthFirstLookaheadSearchManager(Linguist.Linguist linguist, Linguist.Linguist fastmatchLinguist, ILoader loader,
IPruner pruner, IAcousticScorer scorer, ActiveListManager activeListManager,
ActiveListFactory fastmatchActiveListFactory, bool showTokenCount, double relativeWordBeamWidth,
int growSkipInterval, bool checkStateOrder, bool buildWordLattice, int lookaheadWindow, float lookaheadWeight,
int maxLatticeEdges, float acousticLookaheadFrames, bool keepAllTokens)
: base(linguist, pruner, scorer, activeListManager, showTokenCount, relativeWordBeamWidth, growSkipInterval,
checkStateOrder, buildWordLattice, maxLatticeEdges, acousticLookaheadFrames, keepAllTokens)
{
_loader = loader;
_fastmatchLinguist = fastmatchLinguist;
_fastmatchActiveListFactory = fastmatchActiveListFactory;
_lookaheadWindow = lookaheadWindow;
_lookaheadWeight = lookaheadWeight;
if (lookaheadWindow < 1 || lookaheadWindow > 10)
{
throw new ArgumentException("Unsupported lookahead window size: " + lookaheadWindow
+ ". Value in range [1..10] is expected");
}
_ciScores = new LinkedList <FrameCiScores>();
_penalties = new HashMap <Integer, Float>();
if (loader is Sphinx3Loader && ((Sphinx3Loader)loader).HasTiedMixtures())
{
((Sphinx3Loader)loader).SetGauScoresQueueLength(lookaheadWindow + 2);
}
}
/*
* /// (non-Javadoc)
*
* /// @see edu.cmu.sphinx.util.props.Configurable#newProperties(edu.cmu.sphinx.util.props.PropertySheet)
*/
public override void newProperties(PropertySheet ps)
{
logMath = LogMath.getLogMath();
linguist = (Linguist.Linguist)ps.getComponent(PROP_LINGUIST);
pruner = (IPruner)ps.getComponent(PROP_PRUNER);
scorer = (IAcousticScorer)ps.getComponent(PROP_SCORER);
activeListManager = (ActiveListManager)ps.getComponent(PROP_ACTIVE_LIST_MANAGER);
_showTokenCount = ps.getBoolean(PROP_SHOW_TOKEN_COUNT);
growSkipInterval = ps.getInt(PROP_GROW_SKIP_INTERVAL);
_checkStateOrder = ps.getBoolean(PROP_CHECK_STATE_ORDER);
maxLatticeEdges = ps.getInt(PROP_MAX_LATTICE_EDGES);
acousticLookaheadFrames = ps.getFloat(PROP_ACOUSTIC_LOOKAHEAD_FRAMES);
relativeBeamWidth = logMath.linearToLog(ps.getDouble(PROP_RELATIVE_BEAM_WIDTH));
}
/**
* ///
* /// @param linguist
* /// @param pruner
* /// @param scorer
* /// @param activeListFactory
* /// @param showTokenCount
* /// @param relativeWordBeamWidth
* /// @param growSkipInterval
* /// @param wantEntryPruning
*/
public SimpleBreadthFirstSearchManager(Linguist.Linguist linguist, IPruner pruner,
IAcousticScorer scorer, ActiveListFactory activeListFactory,
Boolean showTokenCount, double relativeWordBeamWidth,
int growSkipInterval, Boolean wantEntryPruning)
{
this.name = GetType().Name;
this.logMath = LogMath.getLogMath();
this.linguist = linguist;
this.pruner = pruner;
this.scorer = scorer;
this.activeListFactory = activeListFactory;
this._showTokenCount = showTokenCount;
this.growSkipInterval = growSkipInterval;
this.wantEntryPruning = wantEntryPruning;
this.logRelativeWordBeamWidth = logMath.linearToLog(relativeWordBeamWidth);
this.keepAllTokens = true;
}
/**
* ///
* /// @param linguist
* /// @param pruner
* /// @param scorer
* /// @param activeListFactory
* /// @param showTokenCount
* /// @param relativeWordBeamWidth
* /// @param growSkipInterval
* /// @param wantEntryPruning
*/
public SimpleBreadthFirstSearchManager(Linguist.Linguist linguist, IPruner pruner,
IAcousticScorer scorer, ActiveListFactory activeListFactory,
Boolean showTokenCount, double relativeWordBeamWidth,
int growSkipInterval, Boolean wantEntryPruning, int totalHmms)
{
_name = GetType().Name;
LogMath = LogMath.GetLogMath();
Linguist = linguist;
_pruner = pruner;
_scorer = scorer;
ActiveListFactory = activeListFactory;
_showTokenCount = showTokenCount;
_growSkipInterval = growSkipInterval;
_wantEntryPruning = wantEntryPruning;
_totalHmms = totalHmms;
_logRelativeWordBeamWidth = LogMath.LinearToLog(relativeWordBeamWidth);
KeepAllTokens = true;
}
public override void NewProperties(PropertySheet ps)
{
LogMath = LogMath.GetLogMath();
_name = ps.InstanceName;
Linguist = (Linguist.Linguist)ps.GetComponent(PropLinguist);
_pruner = (IPruner)ps.GetComponent(PropPruner);
_scorer = (IAcousticScorer)ps.GetComponent(PropScorer);
ActiveListFactory = (ActiveListFactory)ps.GetComponent(PropActiveListFactory);
_showTokenCount = ps.GetBoolean(PropShowTokenCount);
double relativeWordBeamWidth = ps.GetDouble(PropRelativeWordBeamWidth);
_growSkipInterval = ps.GetInt(PropGrowSkipInterval);
_wantEntryPruning = ps.GetBoolean(PropWantEntryPruning);
_logRelativeWordBeamWidth = LogMath.LinearToLog(relativeWordBeamWidth);
KeepAllTokens = true;
}
public override void NewProperties(PropertySheet ps)
{
base.NewProperties(ps);
_logMath = LogMath.GetLogMath();
Linguist = (Linguist.Linguist)ps.GetComponent(PropLinguist);
Pruner = (IPruner)ps.GetComponent(PropPruner);
Scorer = (IAcousticScorer)ps.GetComponent(PropScorer);
_activeListManager = (ActiveListManager)ps.GetComponent(PropActiveListManager);
_showTokenCount = ps.GetBoolean(PropShowTokenCount);
_growSkipInterval = ps.GetInt(PropGrowSkipInterval);
_checkStateOrder = ps.GetBoolean(PropCheckStateOrder);
_maxLatticeEdges = ps.GetInt(PropMaxLatticeEdges);
_acousticLookaheadFrames = ps.GetFloat(PropAcousticLookaheadFrames);
_relativeBeamWidth = _logMath.LinearToLog(ps.GetDouble(PropRelativeBeamWidth));
}
/// <summary>
/// Calls provided <see cref="IPruner"/> to prune entries. The
/// entries are passed to the <see cref="IPruner"/> in sorted (newest to
/// oldest <see cref="IndexSearcher"/>) order.
///
/// <para/><b>NOTE</b>: you must peridiocally call this, ideally
/// from the same background thread that opens new
/// searchers.
/// </summary>
public virtual void Prune(IPruner pruner)
{
UninterruptableMonitor.Enter(this);
try
{
// Cannot just pass searchers.values() to ArrayList ctor
// (not thread-safe since the values can change while
// ArrayList is init'ing itself); must instead iterate
// ourselves:
var trackers = _searchers.Values.Select(item => item.Value).ToList();
trackers.Sort();
var lastRecordTimeSec = 0.0;
double now = Time.NanoTime() / NANOS_PER_SEC;
foreach (var tracker in trackers)
{
double ageSec;
if (lastRecordTimeSec == 0.0)
{
ageSec = 0.0;
}
else
{
ageSec = now - lastRecordTimeSec;
}
// First tracker is always age 0.0 sec, since it's
// still "live"; second tracker's age (= seconds since
// it was "live") is now minus first tracker's
// recordTime, etc:
if (pruner.DoPrune(ageSec, tracker.Searcher))
{
//System.out.println("PRUNE version=" + tracker.version + " age=" + ageSec + " ms=" + Time.CurrentTimeMilliseconds());
Lazy <SearcherTracker> _;
_searchers.TryRemove(tracker.Version, out _);
tracker.Dispose();
}
lastRecordTimeSec = tracker.RecordTimeSec;
}
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
override public void newProperties(PropertySheet ps)
{
logMath = LogMath.getLogMath();
name = ps.InstanceName;
linguist = (Linguist.Linguist)ps.getComponent(PROP_LINGUIST);
pruner = (IPruner)ps.getComponent(PROP_PRUNER);
scorer = (IAcousticScorer)ps.getComponent(PROP_SCORER);
activeListFactory = (ActiveListFactory)ps.getComponent(PROP_ACTIVE_LIST_FACTORY);
_showTokenCount = ps.getBoolean(PROP_SHOW_TOKEN_COUNT);
double relativeWordBeamWidth = ps.getDouble(PROP_RELATIVE_WORD_BEAM_WIDTH);
growSkipInterval = ps.getInt(PROP_GROW_SKIP_INTERVAL);
wantEntryPruning = ps.getBoolean(PROP_WANT_ENTRY_PRUNING);
logRelativeWordBeamWidth = logMath.linearToLog(relativeWordBeamWidth);
this.keepAllTokens = true;
}
/**
* ///
* /// @param linguist
* /// @param pruner
* /// @param scorer
* /// @param activeListManager
* /// @param showTokenCount
* /// @param relativeWordBeamWidth
* /// @param growSkipInterval
* /// @param checkStateOrder
* /// @param buildWordLattice
* /// @param maxLatticeEdges
* /// @param acousticLookaheadFrames
* /// @param keepAllTokens
*/
public WordPruningBreadthFirstSearchManager(Linguist.Linguist linguist, IPruner pruner,
IAcousticScorer scorer, ActiveListManager activeListManager,
Boolean showTokenCount, double relativeWordBeamWidth,
int growSkipInterval,
Boolean checkStateOrder, Boolean buildWordLattice,
int maxLatticeEdges, float acousticLookaheadFrames,
Boolean keepAllTokens)
{
this.logMath = LogMath.getLogMath();
this.linguist = linguist;
this.pruner = pruner;
this.scorer = scorer;
this.activeListManager = activeListManager;
this._showTokenCount = showTokenCount;
this.growSkipInterval = growSkipInterval;
this._checkStateOrder = checkStateOrder;
this.buildWordLattice = buildWordLattice;
this.maxLatticeEdges = maxLatticeEdges;
this.acousticLookaheadFrames = acousticLookaheadFrames;
this.keepAllTokens = keepAllTokens;
this.relativeBeamWidth = logMath.linearToLog(relativeWordBeamWidth);
}
public WordPruningBreadthFirstSearchManager(Linguist.Linguist linguist, IPruner pruner,
IAcousticScorer scorer, ActiveListManager activeListManager,
Boolean showTokenCount, double relativeWordBeamWidth,
int growSkipInterval,
Boolean checkStateOrder, Boolean buildWordLattice,
int maxLatticeEdges, float acousticLookaheadFrames,
Boolean keepAllTokens)
{
_logMath = LogMath.GetLogMath();
Linguist = linguist;
Pruner = pruner;
Scorer = scorer;
_activeListManager = activeListManager;
_showTokenCount = showTokenCount;
_growSkipInterval = growSkipInterval;
_checkStateOrder = checkStateOrder;
BuildWordLattice = buildWordLattice;
_maxLatticeEdges = maxLatticeEdges;
_acousticLookaheadFrames = acousticLookaheadFrames;
KeepAllTokens = keepAllTokens;
_relativeBeamWidth = _logMath.LinearToLog(relativeWordBeamWidth);
}
private static bool FindSortingNetwork(int batchSize, IReadOnlyList <IComparatorNetwork> comparatorNets, IPruner pruner,
int heuristicPopulation, List <int> copySteps)
{
var comparatorsGenerator = new ComparatorsGenerator();
var sortingNetworksGenerator = new Generator();
var comparators =
comparatorsGenerator.GenerateComparators(Enumerable.Range(0, IComparatorNetwork.Inputs).ToArray());
var generatorPruner = new GeneratePruneInBatches(batchSize);
var stopWatch = Stopwatch.StartNew();
//comparatorNets ??= new List<IComparatorNetwork> {new ComparatorNetwork(new Comparator[1] {new Comparator(0, 1)})};
comparatorNets ??= new List <IComparatorNetwork>();
if (comparatorNets.Count == 0)
{
var firstNets = new List <IComparatorNetwork>();
for (var j = 0; j < comparators.Count; j++)
{
firstNets.Add(new ComparatorNetwork(new[] { comparators[j] }));
}
comparatorNets = firstNets;
}
var currentComparator = comparatorNets.Count > 0 ? comparatorNets[0].Comparators.Length : 0;
var rand = new Random();
for (var i = currentComparator; i <= IComparatorNetwork.NumComparators; i++)
{
Trace.WriteLine($"Adding Comparator {i}");
Trace.TraceInformation($"Generate");
var generateWatch = Stopwatch.StartNew();
if (comparatorNets.Count > batchSize)
{
var generatePruneWatch = Stopwatch.StartNew();
comparatorNets = generatorPruner.GeneratePrune(comparatorNets, comparators);
Trace.WriteLine($"Length after prune: {comparatorNets.Count}");
Trace.WriteLine($"Generate and Prune time {generatePruneWatch.Elapsed}");
if (heuristicPopulation > 0 && comparatorNets.Count > heuristicPopulation)
{
comparatorNets = HeuristicRemover.RemoveNetsWithMoreBadZeroes(comparatorNets, heuristicPopulation);
//comparatorNets = HeuristicRemover.RemoveNetsWithMoreOutputs(comparatorNets, heuristicPopulation);
}
}
else
{
//var newNets = sortingNetworksGenerator.Generate(comparatorNets, comparators).ToList();
//comparatorNets = newNets;
// First nets already generated
if (i != 1)
{
comparatorNets = sortingNetworksGenerator.Generate(comparatorNets, comparators).OrderBy(x => rand.Next()).ToList();
//comparatorNets = sortingNetworksGenerator.Generate(comparatorNets, comparators);
}
#if DEBUG
Trace.WriteLine($"Redundant number: {IComparatorNetwork.RedundantNumber:N}");
IComparatorNetwork.RedundantNumber = 0;
#endif
Trace.WriteLine($"Length after Generate: {comparatorNets.Count}");
Trace.WriteLine($"Generate time {generateWatch.Elapsed}");
var count = double.Parse(comparatorNets.Count.ToString());
Trace.TraceInformation($"Prune");
var pruneWatch = Stopwatch.StartNew();
if (IPruner.Threads > 1)
{
var splitNets = comparatorNets.SplitList(Math.Max((int)Math.Ceiling(count / IPruner.Threads), 10000))
.ToList();
comparatorNets = pruner.Prune(splitNets);
}
else
{
comparatorNets = pruner.Prune(comparatorNets);
}
if (heuristicPopulation > 0 && comparatorNets.Count > heuristicPopulation)
{
//comparatorNets = HeuristicRemover.RemoveNetsWithMoreBadZeroes(comparatorNets, heuristicPopulation);
comparatorNets = useBadZeroes ? HeuristicRemover.RemoveNetsWithMoreBadZeroes(comparatorNets, heuristicPopulation) :
HeuristicRemover.RemoveNetsWithMoreOutputs(comparatorNets, heuristicPopulation);
}
Trace.WriteLine($"Length after Prune: {comparatorNets.Count}");
Trace.WriteLine($"Prune time {pruneWatch.Elapsed}");
}
#if DEBUG
Trace.WriteLine($"Permutations tested: {IComparatorNetwork.TryPermutationCall}");
Trace.WriteLine($"Is subset dual: {IComparatorNetwork.IsSubsetDual}");
//Trace.WriteLine($"Try permutations call: {IComparatorNetwork.TryPermutationCall:N}");
IComparatorNetwork.IsSubset = 0;
IComparatorNetwork.IsSubsetDual = 0;
IComparatorNetwork.TryPermutationCall = 0;
#endif
Trace.TraceInformation($"Saving Nets");
var path =
$"SavedNetworks/nets_{IComparatorNetwork.Inputs}_{comparatorNets[0].Comparators.Length}_{DateTime.Now:yyyyMMddHHmmssfff}.json";
#if SAVEALL
comparatorNets.SaveToFile(path);
#endif
if (copySteps.Contains((int)i))
{
comparatorNets.SaveToFile(path);
}
Trace.WriteLine(string.Empty);
// Sorting network found
if (comparatorNets.Count == 1 && comparatorNets[0].IsSortingNetwork())
{
break;
}
GC.Collect();
GC.WaitForPendingFinalizers();
}
// Trace.WriteLine($"Subsume no check: {IComparatorNetwork.SubsumeNoCheck} ");
Trace.WriteLine($"Elapsed Time: {stopWatch.Elapsed} ");
#if DEBUG
Trace.WriteLine(string.Empty);
Debug.WriteLine($"Subsume total:{IComparatorNetwork.SubsumeTotal:N}");
Debug.WriteLine($"Subsume no check 1:{IComparatorNetwork.SubsumeNoCheck1:N}");
Debug.WriteLine($"Subsume no check 2:{IComparatorNetwork.SubsumeNoCheck2:N}");
Debug.WriteLine($"Output count bigger:{IComparatorNetwork.OutputCountBigger:N}");
Debug.WriteLine($"Subsume no check total:{IComparatorNetwork.SubsumeNoCheckTotal:N}");
Debug.WriteLine($"Subsume number:{IComparatorNetwork.SubsumeNumber:N}");
Debug.WriteLine($"Subsume succeed:{IComparatorNetwork.SubsumeSucceed:N}");
Debug.WriteLine($"Permutations performed:{IComparatorNetwork.PermutationsNumber:N}");
Debug.WriteLine($"Permutations walk:{IComparatorNetwork.PermutationsWalk:N}");
#endif
Trace.WriteLine(string.Empty);
PrintSortingNetworks(comparatorNets.Where(x => x.IsSortingNetwork2N()).ToList(), IComparatorNetwork.Inputs, heuristicPopulation.ToString());
return(comparatorNets.Any(x => x.IsSortingNetwork2N()));
}
public SearchEngine AddPruner(IPruner pruner)
{
pruners.Add(pruner);
return(this);
}
