static OneVsAllClassifier() { binaryIndex = new HashIndex <string>(); binaryIndex.Add(PosLabel); binaryIndex.Add(NegLabel); posIndex = binaryIndex.IndexOf(PosLabel); }
public void InitGroups(QueryProcessor processor, IIndex<long> emptyIndexContainer) { Debug.Assert(emptyIndexContainer.Count == 0); ITable child = BaseTable; // No groups, so make the entire child table the group, if (aggregateComposite == null || child.RowCount <= 1) { emptyIndexContainer.Add(0); emptyIndexContainer.Add(child.RowCount); } // Populate the index by the aggregate composite, else { // Create a resolver for the composite function IndexResolver resolver = processor.CreateResolver(child, aggregateComposite); // The groups state long groupPos = 0; long groupSize = 0; SqlObject[] lastComposite = null; // Scan over the child IRowCursor cursor = child.GetRowCursor(); while (cursor.MoveNext()) { RowId rowid = cursor.Current; // Get the group term SqlObject[] groupValue = resolver.GetValue(rowid); if (lastComposite == null) { lastComposite = groupValue; } else { int c = SqlObject.Compare(groupValue, lastComposite); // If group_val > the last composite, we are on a new group if (c > 0) { // New group, emptyIndexContainer.Add(groupPos); emptyIndexContainer.Add(groupSize); lastComposite = groupValue; groupPos = groupPos + groupSize; groupSize = 0; } else if (c < 0) { // This will happen if the child table is not sorted by the // composite expression. throw new ApplicationException("Aggregate child is not sorted correctly."); } } ++groupSize; } // Final group // (the below check probably not necessary since we already check for the // empty child so group size will always be >1 at this point). if (groupSize > 0) { emptyIndexContainer.Add(groupPos); emptyIndexContainer.Add(groupSize); } } // Set the group index childGroupsIndex = emptyIndexContainer; lookupCursor = BaseTable.GetRowCursor(); }
public void Add(TKey query, TValue value) { if (query == null || value == null) { return; } Debug.WriteLine($"Adding \"{value}\" for \"{query}\" query to forward index. Thread = {Thread.CurrentThread.ManagedThreadId}"); _forward.Add(query, value); }
protected virtual void SetUp() { index = new ConcurrentHashIndex <string>(); index.Add("The"); index.Add("Beast"); index2 = new ConcurrentHashIndex <string>(); index2.Add("Beauty"); index2.Add("And"); index2.Add("The"); index2.Add("Beast"); index3 = new ConcurrentHashIndex <string>(); index3.Add("Markov"); index3.Add("The"); index3.Add("Beast"); }
private void PopulateIndex(IIndex index, IEnumerable <KeyValuePair <long, JSONDocument> > store) { System.Threading.Tasks.Task.Factory.StartNew(() => { var enumerator = store.GetEnumerator(); while (enumerator.MoveNext()) { AttributeValue[] value; if (!index.IndexKey.TryGetValue(enumerator.Current.Value, out value)) { continue; } foreach (var attributeValue in value) { index.Add(attributeValue, enumerator.Current.Key, -1); } } ((IBPlusPersister)index).PersistAndCommitAll(); index.IsFunctional = true; persistanceManager.RegisterPersister((BPlusIndex)index); if (LoggerManager.Instance.IndexLogger != null) { LoggerManager.Instance.IndexLogger.Debug("BPlusIndex", "Index " + index.Name + " population complete, it is now available and ready to use"); } }); }
static void Main(string[] args) { try { /* * * index objects with attributes * * Still need a small adapter for Linq to XML though :< * */ countryList = new List <Country>(); countryIndex = new Index <Country>(); countryList.AddRange(new CountryXmlReader().Execute()); countryIndex.Add(countryList); Console.WriteLine("Added {0}", countryIndex.Count); Console.WriteLine("======================================"); SkipDemo(); PagingDemo(); } catch (Exception ex) { Console.WriteLine(ex); } finally { Console.WriteLine("[enter] to exit"); Console.ReadLine(); } }
/// <summary>The examples are assumed to be a list of RFVDatum.</summary> /// <remarks> /// The examples are assumed to be a list of RFVDatum. /// The datums are assumed to not contain the zeroes and then they are added to each instance. /// </remarks> public virtual NaiveBayesClassifier <L, F> TrainClassifier(GeneralDataset <L, F> examples, ICollection <F> featureSet) { int numFeatures = featureSet.Count; int[][] data = new int[][] { }; int[] labels = new int[examples.Size()]; labelIndex = new HashIndex <L>(); featureIndex = new HashIndex <F>(); foreach (F feat in featureSet) { featureIndex.Add(feat); } for (int d = 0; d < examples.Size(); d++) { RVFDatum <L, F> datum = examples.GetRVFDatum(d); ICounter <F> c = datum.AsFeaturesCounter(); foreach (F feature in c.KeySet()) { int fNo = featureIndex.IndexOf(feature); int value = (int)c.GetCount(feature); data[d][fNo] = value; } labelIndex.Add(datum.Label()); labels[d] = labelIndex.IndexOf(datum.Label()); } int numClasses = labelIndex.Size(); return(TrainClassifier(data, labels, numFeatures, numClasses, labelIndex, featureIndex)); }
public PasswordResponse Get() { //v1 /* * return new PasswordResponse * { * Password = Guid.NewGuid().ToString().Substring(0,6), * ApiVersion = "1.0", * ApiServer = Environment.MachineName * }; */ //v3 & 4 var response = new PasswordResponse { Password = GenerateRandomPassword(), ApiVersion = "4.0", ApiServer = Environment.MachineName }; _index.Add(new PasswordDetails(response)); return(response); }
private static void RunRectangles(IIndex <int, TRectangle, TPoint> index, IList <Tuple <int, TRectangle> > rectangles) { Run(index, rectangles, (list, data, random, i) => { var rect = data[i].Item2; rect.X += (float)random.NextDouble(-0.05, 0.05); rect.Y += (float)random.NextDouble(-0.05, 0.05); list.Add(Tuple.Create(i, rect)); }, (list, data, random, i) => list.Add(Tuple.Create(i, random.NextRectangle(Area, 16, 1024))), (ints, data) => { foreach (var item in data) { var rect = item.Item2; index.Add(rect, item.Item1); } }, (ints, update) => { var rect = update.Item2; index.Update(rect, Vector2.Zero, update.Item1); } ); }
public IExpressionRegistration Register <T>(Expression <Func <IResolver, T> > expression, object key = null, ILifetime lifetime = null, CompileMode?compileMode = null) { if (expression == null) { throw new ArgumentNullException("expression"); } var lt = lifetime ?? _defaultLifetimeFactory(); var cm = compileMode != null ? compileMode.Value : _defaultCompileMode; var registration = new ExpressionRegistration <T>(this, expression, lt, cm, key: key); _index.Add(registration); return(registration); }
public override bool Add(E e) { if (backingIndex.Contains(e)) { return(false); } return(spilloverIndex.Add(e)); }
public void Import(IIndex index, int count) { var reader = new DataDumpReader<Post>(); var xml = Path.Combine(path, "posts.xml"); index.Add(reader.Read(xml) .Where(p => p.PostTypeId == PostTypeId.Question) .Take(count)); }
// naClass.init(naWord, ttags); private int Add(AmbiguityClass a) { if (classes.Contains(a)) { return(classes.IndexOf(a)); } classes.Add(a); return(classes.IndexOf(a)); }
/// <summary>Add a new item to the index, with the specified position.</summary> /// <typeparam name="T"></typeparam> /// <param name="index">The index to add to.</param> /// <param name="point">The position of the item.</param> /// <param name="item">The item.</param> /// <exception cref="T:System.ArgumentException">The item is already stored in the index.</exception> /// <remarks> /// This will lead to the point being converted to an empty rectangle at the point's position, which will then be /// inserted, instead. /// </remarks> public static void Add <T>(this IIndex <T, TRectangle, TPoint> index, TPoint point, T item) { // Convert to rectangle, then add that. TRectangle bounds; bounds.X = point.X; bounds.Y = point.Y; bounds.Width = bounds.Height = 0; index.Add(bounds, item); }
public Db <T> AddData(T value) { var offset = _primaryIndex.Add(value); foreach (var(id, index) in _indices) { index.Add(value, offset); } return(this); }
private void OnForwardAdded(TKey query, IEnumerable <TValue> values) { if (query == null || values == null) { return; } foreach (var value in values) { Debug.WriteLine($"Adding \"{value}\" for \"{query}\" query to inverted index. Thread = {Thread.CurrentThread.ManagedThreadId}"); _inverted.Add(value, query); } }
static PascalTemplate() { //dates //location //workshop info //conference info //background symbol fieldIndices = new HashIndex <string>(); foreach (string field in fields) { fieldIndices.Add(field); } }
public void SetUp() { FullTextSearchIndex index = new FullTextSearchIndex(); index.Fields.Add("Title"); FullTextSearchIndex multipleFieldIndex = new FullTextSearchIndex(); multipleFieldIndex.Fields.Add("Title"); multipleFieldIndex.Fields.Add("Ingredients"); _index = index; _multipleFieldIndex = multipleFieldIndex; _record1 = new HashtableRecord(); _record1["Title"] = "Bolo de Chocolate"; _record1["Calories"] = 300; _record1["Ingredients"] = "3 colheres de açucar\n1 lata de nescau\nfermento"; _index.Add(_record1); _multipleFieldIndex.Add(_record1); DumpPostings(index.Postings); _record2 = new HashtableRecord(); _record2["Title"] = "Bolo de Açafrão"; _record2["Calories"] = 100; _record2["Ingredients"] = "10 folhas de açafrão\n1 colher de fermento em pó"; _index.Add(_record2); _multipleFieldIndex.Add(_record2); DumpPostings(index.Postings); _record3 = new HashtableRecord(); _record3["Title"] = "Torta de Chocolate"; _record3["Calories"] = 400; _record3["Ingredients"] = "1 lata de nescau\nchocolate granulado\naçucar"; _index.Add(_record3); _multipleFieldIndex.Add(_record3); DumpPostings(index.Postings); }
private static void RunPoints(IIndex <int, TRectangle, TPoint> index, IList <Tuple <int, TPoint> > points) { Run(index, points, (list, data, random, i) => list.Add(Tuple.Create(i, data[i].Item2 + random.NextVector(0.05f))), (list, data, random, i) => list.Add(Tuple.Create(i, random.NextVector(Area))), (ints, data) => { foreach (var item in data) { index.Add(item.Item2, item.Item1); } }, (ints, update) => index.Update(update.Item2, update.Item1) ); }
private void IndexFile(string filePath, IFileVersion version) { try { _eventReactor.React(EngineEvent.FileIndexingStarted, filePath); var encoding = FileEncoding.DetectFileEncoding(filePath); var words = _parserProvider.Provide(filePath).Parse(version, encoding); _index.Add(version, words); _eventReactor.React(EngineEvent.FileIndexingEnded, filePath); } catch (Exception exception) { _eventReactor.React(EngineEvent.FileIndexingEnded, filePath, exception); } }
private bool AddIndex(float[] feat, out int index) { index = -1; try { m_index.Add(feat, out index); if (index < 0) { m_logger.LogError($"Add index failed: {index}"); return(false); } return(true); } catch (Exception ex) { m_logger.LogError($"{nameof(AddIndex)} error: {ex.Message}\n{ex.StackTrace}"); return(false); } }
public void Add(string key, Stream data) { if (key == null) { throw new ArgumentNullException(nameof(key)); } if (data == null) { throw new ArgumentNullException(nameof(data)); } CheckDisposed(); if (_index.Contains(key)) { throw new DuplicateException($"An entry with the same key ({key}) already exists."); } var indexData = _storage.Append(data); _index.Add(key, indexData); }
public virtual void Train(IList <TaggedWord> sentence) { lex.Train(sentence, 1.0); string last = null; foreach (TaggedWord tagLabel in sentence) { string tag = tagLabel.Tag(); tagIndex.Add(tag); if (last == null) { initial.IncrementCount(tag); } else { ruleCounter.IncrementCount2D(last, tag); } last = tag; } }
public void Load(IIndex <long, ulong> index) { var fname = Path.Combine(settings.Folder, index.Name); if (File.Exists(fname)) { using (var stream = new FileStream(fname, FileMode.Open, FileAccess.Read)) using (var gzip = new GZipStream(stream, CompressionMode.Decompress)) using (var br = new BinaryReader(gzip)) { var c = br.ReadInt64(); for (long i = 0; i < c; ++i) { var k = br.ReadInt64(); var v = br.ReadUInt64(); index.Add(k, v); } } } }
protected internal virtual void Read(DataInputStream file) { try { int size = file.ReadInt(); index = new HashIndex <string>(); for (int i = 0; i < size; i++) { string tag = file.ReadUTF(); bool inClosed = file.ReadBoolean(); index.Add(tag); if (inClosed) { closed.Add(tag); } } } catch (IOException e) { Sharpen.Runtime.PrintStackTrace(e); } }
//try add if any error happen, then rollback private void TryAdd(string key, Stream data) { var indexData = _storage.Append(data); _index.Add(key, indexData); }
public void Add(string key, IndexData indexData) { WriteLock(() => _index.Add(key, indexData), $"Timeout {_timeout} expired to add key {key} to index"); }
public virtual void FinishTraining() { Timing.Tick("Counting characters..."); ClassicCounter <ChineseCharacterBasedLexicon.Symbol> charCounter = new ClassicCounter <ChineseCharacterBasedLexicon.Symbol>(); // first find all chars that occur only once foreach (IList <TaggedWord> labels in trainingSentences) { foreach (TaggedWord label in labels) { string word = label.Word(); if (word.Equals(LexiconConstants.Boundary)) { continue; } for (int j = 0; j < length; j++) { ChineseCharacterBasedLexicon.Symbol sym = ChineseCharacterBasedLexicon.Symbol.CannonicalSymbol(word[j]); charCounter.IncrementCount(sym); } charCounter.IncrementCount(ChineseCharacterBasedLexicon.Symbol.EndWord); } } ICollection <ChineseCharacterBasedLexicon.Symbol> singletons = Counters.KeysBelow(charCounter, 1.5); knownChars = Generics.NewHashSet(charCounter.KeySet()); Timing.Tick("Counting nGrams..."); GeneralizedCounter[] POSspecificCharNGrams = new GeneralizedCounter[ContextLength + 1]; for (int i = 0; i <= ContextLength; i++) { POSspecificCharNGrams[i] = new GeneralizedCounter(i + 2); } ClassicCounter <string> POSCounter = new ClassicCounter <string>(); IList <ISerializable> context = new List <ISerializable>(ContextLength + 1); foreach (IList <TaggedWord> words in trainingSentences) { foreach (TaggedWord taggedWord in words) { string word = taggedWord.Word(); string tag = taggedWord.Tag(); tagIndex.Add(tag); if (word.Equals(LexiconConstants.Boundary)) { continue; } POSCounter.IncrementCount(tag); for (int i_1 = 0; i_1 <= size; i_1++) { ChineseCharacterBasedLexicon.Symbol sym; ChineseCharacterBasedLexicon.Symbol unknownCharClass = null; context.Clear(); context.Add(tag); if (i_1 < size) { char thisCh = word[i_1]; sym = ChineseCharacterBasedLexicon.Symbol.CannonicalSymbol(thisCh); if (singletons.Contains(sym)) { unknownCharClass = UnknownCharClass(sym); charCounter.IncrementCount(unknownCharClass); } } else { sym = ChineseCharacterBasedLexicon.Symbol.EndWord; } POSspecificCharNGrams[0].IncrementCount(context, sym); // POS-specific 1-gram if (unknownCharClass != null) { POSspecificCharNGrams[0].IncrementCount(context, unknownCharClass); } // for unknown ch model // context is constructed incrementally: // tag prevChar prevPrevChar // this could be made faster using .sublist like in score for (int j = 1; j <= ContextLength; j++) { // poly grams if (i_1 - j < 0) { context.Add(ChineseCharacterBasedLexicon.Symbol.BeginWord); POSspecificCharNGrams[j].IncrementCount(context, sym); if (unknownCharClass != null) { POSspecificCharNGrams[j].IncrementCount(context, unknownCharClass); } // for unknown ch model break; } else { ChineseCharacterBasedLexicon.Symbol prev = ChineseCharacterBasedLexicon.Symbol.CannonicalSymbol(word[i_1 - j]); if (singletons.Contains(prev)) { context.Add(UnknownCharClass(prev)); } else { context.Add(prev); } POSspecificCharNGrams[j].IncrementCount(context, sym); if (unknownCharClass != null) { POSspecificCharNGrams[j].IncrementCount(context, unknownCharClass); } } } } } } // for unknown ch model POSDistribution = Distribution.GetDistribution(POSCounter); Timing.Tick("Creating character prior distribution..."); charDistributions = Generics.NewHashMap(); // charDistributions = Generics.newHashMap(); // 1.5 // charCounter.incrementCount(Symbol.UNKNOWN, singletons.size()); int numberOfKeys = charCounter.Size() + singletons.Count; Distribution <ChineseCharacterBasedLexicon.Symbol> prior = Distribution.GoodTuringSmoothedCounter(charCounter, numberOfKeys); charDistributions[Java.Util.Collections.EmptyList] = prior; for (int i_2 = 0; i_2 <= ContextLength; i_2++) { ICollection <KeyValuePair <IList <ISerializable>, ClassicCounter <ChineseCharacterBasedLexicon.Symbol> > > counterEntries = POSspecificCharNGrams[i_2].LowestLevelCounterEntrySet(); Timing.Tick("Creating " + counterEntries.Count + " character " + (i_2 + 1) + "-gram distributions..."); foreach (KeyValuePair <IList <ISerializable>, ClassicCounter <ChineseCharacterBasedLexicon.Symbol> > entry in counterEntries) { context = entry.Key; ClassicCounter <ChineseCharacterBasedLexicon.Symbol> c = entry.Value; Distribution <ChineseCharacterBasedLexicon.Symbol> thisPrior = charDistributions[context.SubList(0, context.Count - 1)]; double priorWeight = thisPrior.GetNumberOfKeys() / 200.0; Distribution <ChineseCharacterBasedLexicon.Symbol> newDist = Distribution.DynamicCounterWithDirichletPrior(c, thisPrior, priorWeight); charDistributions[context] = newDist; } } }
/// <exception cref="System.IO.IOException"/> private void GetFeatures(File file) { BufferedReader @in = new BufferedReader(new InputStreamReader(new FileInputStream(file), "GB18030")); string filename = file.GetName(); string singleFeatName = filename; if (singleFeatName.IndexOf('.') >= 0) { singleFeatName = Sharpen.Runtime.Substring(singleFeatName, 0, filename.LastIndexOf('.')); } ChineseMorphFeatureSets.FeatType featType = null; foreach (ChineseMorphFeatureSets.FeatType ft in ChineseMorphFeatureSets.FeatType.Values()) { if (filename.Contains(ft.ToString().ToLower())) { featType = ft; singleFeatName = Sharpen.Runtime.Substring(singleFeatName, 0, filename.IndexOf(ft.ToString().ToLower())); if (singleFeatName.EndsWith("_")) { singleFeatName = Sharpen.Runtime.Substring(singleFeatName, 0, singleFeatName.LastIndexOf('_')); } break; } } featIndex.Add(singleFeatName); string singleFeatIndexString = int.ToString(featIndex.IndexOf(singleFeatName)); ICollection <char> featureSet = Generics.NewHashSet(); string line; Pattern typedDoubleFeatPattern = Pattern.Compile("([A-Za-z]+)\\s+(.)\\s+(.)\\s*"); Pattern typedSingleFeatPattern = Pattern.Compile("([A-Za-z]+)\\s+(.)\\s*"); Pattern singleFeatPattern = Pattern.Compile("(.)(?:\\s+[0-9]+)?\\s*"); while ((line = @in.ReadLine()) != null) { if (line.Length == 0) { continue; } if (featType == null) { Matcher typedDoubleFeatMatcher = typedDoubleFeatPattern.Matcher(line); if (typedDoubleFeatMatcher.Matches()) { string featName = typedDoubleFeatMatcher.Group(1); featIndex.Add(featName); string featIndexString = int.ToString(featIndex.IndexOf(featName)); string prefixChar = typedDoubleFeatMatcher.Group(2); AddTypedFeature(featIndexString, prefixChar[0], true); string suffixChar = typedDoubleFeatMatcher.Group(3); AddTypedFeature(featIndexString, suffixChar[0], false); continue; } } Matcher typedSingleFeatMatcher = typedSingleFeatPattern.Matcher(line); if (typedSingleFeatMatcher.Matches()) { string featName = typedSingleFeatMatcher.Group(1); featIndex.Add(featName); string featIndexString = int.ToString(featIndex.IndexOf(featName)); string charString = typedSingleFeatMatcher.Group(2); switch (featType) { case ChineseMorphFeatureSets.FeatType.Prefix: { AddTypedFeature(featIndexString, charString[0], true); break; } case ChineseMorphFeatureSets.FeatType.Suffix: { AddTypedFeature(featIndexString, charString[0], false); break; } case ChineseMorphFeatureSets.FeatType.Singleton: { throw new Exception("ERROR: typed SINGLETON feature."); } } continue; } Matcher singleFeatMatcher = singleFeatPattern.Matcher(line); if (singleFeatMatcher.Matches()) { string charString = singleFeatMatcher.Group(); featureSet.Add(charString[0]); continue; } if (line.StartsWith("prefix") || line.StartsWith("suffix")) { if (featureSet.Count > 0) { Pair <ICollection <char>, ICollection <char> > p = affixFeatures[singleFeatIndexString]; if (p == null) { affixFeatures[singleFeatIndexString] = p = new Pair <ICollection <char>, ICollection <char> >(); } if (featType == ChineseMorphFeatureSets.FeatType.Prefix) { p.SetFirst(featureSet); } else { p.SetSecond(featureSet); } featureSet = Generics.NewHashSet(); } featType = ChineseMorphFeatureSets.FeatType.Prefix; if (line.StartsWith("prefix")) { featType = ChineseMorphFeatureSets.FeatType.Prefix; } else { if (line.StartsWith("suffix")) { featType = ChineseMorphFeatureSets.FeatType.Suffix; } } } } if (featureSet.Count > 0) { if (featType == ChineseMorphFeatureSets.FeatType.Singleton) { singletonFeatures[singleFeatIndexString] = featureSet; } else { Pair <ICollection <char>, ICollection <char> > p = affixFeatures[singleFeatIndexString]; if (p == null) { affixFeatures[singleFeatIndexString] = p = new Pair <ICollection <char>, ICollection <char> >(); } if (featType == ChineseMorphFeatureSets.FeatType.Prefix) { p.SetFirst(featureSet); } else { p.SetSecond(featureSet); } } } }