public void TestFileWithDifferentDelimiter() { KeyValuePair <List <List <string> >, string> fileInput = generateFileInput(AtLeast(100), " , ", true, true); Stream inputReader = new MemoryStream(fileInput.Value.getBytes(Encoding.UTF8)); FileDictionary dictionary = new FileDictionary(inputReader, " , "); List <List <string> > entries = fileInput.Key; IInputIterator inputIter = dictionary.GetEntryIterator(); assertTrue(inputIter.HasPayloads); BytesRef term; int count = 0; while ((term = inputIter.Next()) != null) { assertTrue(entries.size() > count); List <string> entry = entries[count]; assertTrue(entry.size() >= 2); // at least term and weight assertEquals(entry[0], term.Utf8ToString()); assertEquals(long.Parse(entry[1], CultureInfo.InvariantCulture), inputIter.Weight); if (entry.size() == 3) { assertEquals(entry[2], inputIter.Payload.Utf8ToString()); } else { assertEquals(inputIter.Payload.Length, 0); } count++; } assertEquals(count, entries.size()); }
internal FileIterator(FileDictionary outerInstance) { this.outerInstance = outerInstance; outerInstance.line = [email protected](); if (outerInstance.line == null) { outerInstance.done = true; IOUtils.Close(outerInstance.@in); } else { string[] fields = outerInstance.line.Split(new string[] { outerInstance.fieldDelimiter }, StringSplitOptions.RemoveEmptyEntries); if (fields.Length > 3) { throw new System.ArgumentException("More than 3 fields in one line"); } // term, weight, payload else if (fields.Length == 3) { hasPayloads = true; spare.CopyChars(fields[0]); ReadWeight(fields[1]); curPayload.CopyChars(fields[2]); } // term, weight else if (fields.Length == 2) { spare.CopyChars(fields[0]); ReadWeight(fields[1]); } // only term else { spare.CopyChars(fields[0]); curWeight = 1; } } }
public void TestFileWithOneEntry() { KeyValuePair <IList <IList <string> >, string> fileInput = generateFileInput(1, FileDictionary.DEFAULT_FIELD_DELIMITER, true, true); Stream inputReader = new MemoryStream(fileInput.Value.getBytes(Encoding.UTF8)); FileDictionary dictionary = new FileDictionary(inputReader); IList <IList <string> > entries = fileInput.Key; IInputEnumerator inputIter = dictionary.GetEntryEnumerator(); assertTrue(inputIter.HasPayloads); int count = 0; while (inputIter.MoveNext()) { assertTrue(entries.size() > count); IList <string> entry = entries[count]; assertTrue(entry.size() >= 2); // at least term and weight assertEquals(entry[0], inputIter.Current.Utf8ToString()); assertEquals(long.Parse(entry[1], CultureInfo.InvariantCulture), inputIter.Weight); if (entry.size() == 3) { assertEquals(entry[2], inputIter.Payload.Utf8ToString()); } else { assertEquals(inputIter.Payload.Length, 0); } count++; } assertEquals(count, entries.size()); }
public void TestFileWithTerm() { KeyValuePair<List<List<string>>, string> fileInput = generateFileInput(AtLeast(100), FileDictionary.DEFAULT_FIELD_DELIMITER, false, false); Stream inputReader = new MemoryStream(fileInput.Value.getBytes(Encoding.UTF8)); FileDictionary dictionary = new FileDictionary(inputReader); List<List<string>> entries = fileInput.Key; IInputIterator inputIter = dictionary.EntryIterator; assertFalse(inputIter.HasPayloads); BytesRef term; int count = 0; while ((term = inputIter.Next()) != null) { assertTrue(entries.size() > count); List<string> entry = entries[count]; assertTrue(entry.size() >= 1); // at least a term assertEquals(entry[0], term.Utf8ToString()); assertEquals(1, inputIter.Weight); assertNull(inputIter.Payload); count++; } assertEquals(count, entries.size()); }
public void TestFileWithTerm() { KeyValuePair <IList <IList <string> >, string> fileInput = generateFileInput(AtLeast(100), FileDictionary.DEFAULT_FIELD_DELIMITER, false, false); Stream inputReader = new MemoryStream(fileInput.Value.getBytes(Encoding.UTF8)); FileDictionary dictionary = new FileDictionary(inputReader); IList <IList <string> > entries = fileInput.Key; IInputEnumerator inputIter = dictionary.GetEntryEnumerator(); assertFalse(inputIter.HasPayloads); int count = 0; while (inputIter.MoveNext()) { assertTrue(entries.size() > count); IList <string> entry = entries[count]; assertTrue(entry.size() >= 1); // at least a term assertEquals(entry[0], inputIter.Current.Utf8ToString()); assertEquals(1, inputIter.Weight); assertNull(inputIter.Payload); count++; } assertEquals(count, entries.size()); }
public void TestFileWithWeight() { KeyValuePair <List <List <string> >, string> fileInput = generateFileInput(AtLeast(100), FileDictionary.DEFAULT_FIELD_DELIMITER, true, false); Stream inputReader = new MemoryStream(fileInput.Value.getBytes(Encoding.UTF8)); FileDictionary dictionary = new FileDictionary(inputReader); List <List <String> > entries = fileInput.Key; IInputIterator inputIter = dictionary.GetEntryIterator(); assertFalse(inputIter.HasPayloads); BytesRef term; int count = 0; while ((term = inputIter.Next()) != null) { assertTrue(entries.size() > count); List <String> entry = entries[count]; assertTrue(entry.size() >= 1); // at least a term assertEquals(entry[0], term.Utf8ToString()); assertEquals((entry.size() == 2) ? long.Parse(entry[1], CultureInfo.InvariantCulture) : 1, inputIter.Weight); assertNull(inputIter.Payload); count++; } assertEquals(count, entries.size()); }
internal FileIterator(FileDictionary outerInstance) { this.outerInstance = outerInstance; outerInstance.line = [email protected](); if (outerInstance.line == null) { outerInstance.done = true; IOUtils.Close(outerInstance.@in); } else { string[] fields = outerInstance.line.Split(outerInstance.fieldDelimiter, true); if (fields.Length > 3) { throw new System.ArgumentException("More than 3 fields in one line"); } // term, weight, payload else if (fields.Length == 3) { hasPayloads = true; spare.CopyChars(fields[0]); ReadWeight(fields[1]); curPayload.CopyChars(fields[2]); } // term, weight else if (fields.Length == 2) { spare.CopyChars(fields[0]); ReadWeight(fields[1]); } // only term else { spare.CopyChars(fields[0]); curWeight = 1; } } }
public void TestFileWithDifferentDelimiter() { KeyValuePair<List<List<string>>, string> fileInput = generateFileInput(AtLeast(100), " , ", true, true); Stream inputReader = new MemoryStream(fileInput.Value.getBytes(Encoding.UTF8)); FileDictionary dictionary = new FileDictionary(inputReader, " , "); List<List<string>> entries = fileInput.Key; IInputIterator inputIter = dictionary.EntryIterator; assertTrue(inputIter.HasPayloads); BytesRef term; int count = 0; while ((term = inputIter.Next()) != null) { assertTrue(entries.size() > count); List<string> entry = entries[count]; assertTrue(entry.size() >= 2); // at least term and weight assertEquals(entry[0], term.Utf8ToString()); assertEquals(long.Parse(entry[1]), inputIter.Weight); if (entry.size() == 3) { assertEquals(entry[2], inputIter.Payload.Utf8ToString()); } else { assertEquals(inputIter.Payload.Length, 0); } count++; } assertEquals(count, entries.size()); }