public void ExtractSearchTerms_ReturnsValidNumberOfSearchTermsWhenUnderscoreUsed()
{
List <string> parts = WordSplitter.ExtractSearchTerms("file_open_now");
Assert.AreEqual(1, parts.Count);
Assert.AreEqual("\"file_open_now\"", String.Join(" ", parts));
}
public void ExtractSearchTerms_ReturnsValidNumberOfSearchTermsWhenNumbersUsed()
{
List <string> parts = WordSplitter.ExtractSearchTerms("file324");
Assert.AreEqual(3, parts.Count);
Assert.AreEqual("file324 file 324", String.Join(" ", parts));
}
public void ExtractSearchTerms_ReturnsValidNumberOfSearchTermsWhenNumbersUsed2()
{
List <string> parts = WordSplitter.ExtractSearchTerms("Mp3Player");
Assert.AreEqual(4, parts.Count);
Assert.AreEqual("mp3player mp 3 player", String.Join(" ", parts));
}
public void ExtractSearchTerms_ReturnsValidNumberOfSearchTermsWhenBracketsUsed()
{
List <string> parts = WordSplitter.ExtractSearchTerms("file(new File())");
Assert.AreEqual(1, parts.Count);
Assert.AreEqual("\"file(new File())\"", String.Join(" ", parts));
}
public void ExtractSearchTerms_ReturnsValidNumberOfSearchTermsWhenUpperCasesUsed3()
{
List <string> parts = WordSplitter.ExtractSearchTerms("FileTXTOpenNow");
Assert.AreEqual(5, parts.Count);
Assert.AreEqual("filetxtopennow file txt open now", String.Join(" ", parts));
}
public void ExtractSearchTerms_ReturnsValidNumberOfSearchTermsWhenNoOperatortUsedWithQuotes()
{
List <string> parts = WordSplitter.ExtractSearchTerms("word \"words inside\"");
Assert.AreEqual(1, parts.Count);
Assert.AreEqual("\"word \"words inside\"\"", String.Join("*", parts));
}
public void ExtractSearchTerms_ReturnsValidNumberOfSearchTermsWhenEqualityOperatorUsed()
{
List <string> parts = WordSplitter.ExtractSearchTerms("file=new");
Assert.AreEqual(1, parts.Count);
Assert.AreEqual("\"file=new\"", String.Join(" ", parts));
}
public CriteriaBuilder Ext(string searchString, SimpleSearchCriteria searchCriteria = null)
{
Initialze(searchCriteria);
_searchCriteria.FileExtensions = WordSplitter.GetFileExtensions(searchString);
_searchCriteria.SearchByFileExtension = _searchCriteria.FileExtensions.Any();
return(this);
}
public void ExtractSearchTerms_ReturnsValidNumberOfSearchTermsWhenNoOperatortUsedWithoutQuotes()
{
List <string> parts = WordSplitter.ExtractSearchTerms("word -about");
Assert.AreEqual(2, parts.Count);
Assert.AreEqual("word*-about", String.Join("*", parts));
}
public void GlobalSetup()
{
splitter = WordSplitter.GetInstance();
allRanges = new List <WordRangeVisitor.Range>(32)
{
new WordRangeVisitor.Range(1, 1),
new WordRangeVisitor.Range(1, 2),
new WordRangeVisitor.Range(3, 4),
new WordRangeVisitor.Range(2, 4),
new WordRangeVisitor.Range(4, 5),
new WordRangeVisitor.Range(5, 6),
new WordRangeVisitor.Range(3, 5),
new WordRangeVisitor.Range(4, 6),
new WordRangeVisitor.Range(2, 2),
new WordRangeVisitor.Range(1, 3)
};
ranges1 = allRanges.GetRange(1, 1);
ranges2 = allRanges.GetRange(1, 2);
ranges3 = allRanges.GetRange(1, 3);
ranges4 = allRanges.GetRange(1, 4);
ranges5 = allRanges.GetRange(1, 5);
ranges6 = allRanges.GetRange(1, 6);
ranges7 = allRanges.GetRange(1, 7);
ranges8 = allRanges.GetRange(1, 8);
ranges9 = allRanges.GetRange(1, 9);
}
/// <summary>
/// It finds a substring from the actual value between a frirs word and a last word range.
/// Then it will walk into the tree and recurse through all the remaining steps or reutn null if no range.
/// </summary>
/// <param name="tree">The tree to walk into.</param>
/// <param name="value">The actual value of the node or null to get the root.</param>
/// <returns>Either null or the actual value that was found.</returns>
public override WalkList.WalkResult Walk(IParseTree tree, string value)
{
var actualValue = this.GetActualValue(tree, value);
string filteredValue;
if (tree.ChildCount == 1)
{
var child = tree.GetChild(0);
if (child is UserAgentParser.SingleVersionContext || child is UserAgentParser.SingleVersionWithCommasContext)
{
filteredValue = VersionSplitter.Instance.GetSplitRange(actualValue, this.firstWord, this.lastWord);
}
else
{
filteredValue = WordSplitter.GetInstance().GetSplitRange(actualValue, this.firstWord, this.lastWord);
}
}
else
{
filteredValue = WordSplitter.GetInstance().GetSplitRange(actualValue, this.firstWord, this.lastWord);
}
if (filteredValue is null)
{
return(null);
}
else
{
return(this.WalkNextStep(tree, filteredValue));
}
}
public void SimpleSearchCriteria_ToQueryStringCreatesValidQueryString_QuotedSearchTerm()
{
SearchCriteria simpleSearchCriteria = new SimpleSearchCriteria()
{
SearchByUsageType = true,
UsageTypes = new SortedSet <UsageType>()
{
UsageType.Definitions
},
SearchTerms = new SortedSet <string>(WordSplitter.ExtractSearchTerms("\"Class Simple\""))
};
string queryString = simpleSearchCriteria.ToQueryString();
Assert.AreEqual(queryString, "(" + SandoField.Name.ToString() + ":*Class?Simple*^3)", "Created query string is invalid!");
try
{
var parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_29, SandoField.Name.ToString(), new SimpleAnalyzer());
parser.SetAllowLeadingWildcard(true);
Query query = parser.Parse(queryString);
Assert.NotNull(query, "Generated query object is null!");
}
catch (Exception ex)
{
Assert.Fail(ex.Message);
}
}
public void NoEmptyWordPartsInResult()
{
var sut = new WordSplitter();
var result = sut.SplitToParts(new Word("abcdefaabbccddeedff"), 4).Count;
Assert.That(result, Is.EqualTo(9));
}
public void ExtractSearchTerms_ReturnsValidNumberOfSearchTermsWhenNoQuotesUsed()
{
List <string> parts = WordSplitter.ExtractSearchTerms("word words inside quotes another_word");
Assert.AreEqual(1, parts.Count);
Assert.AreEqual(String.Join("*", parts), "\"word words inside quotes another_word\"");
}
public void ExtractSearchTermsTestLeaveCompleteTerm()
{
var parts = WordSplitter.ExtractSearchTerms("_solutionEvents");
Assert.AreEqual(parts.Count, 1);
Assert.IsTrue(parts.Contains("\"_solutionEvents\""));
}
public static async Task <List <StringMetrics> > GetClosestNames(string id, int numberOfRecommendations)
{
return(await Task.Run(() =>
{
List <string> closest = new List <string>();
var twoDigit = Regex.Match(id, @"\d{2}").Value;
var fourDigit = Regex.Match(id, @"\d{4}").Value;
var dict = TG263Dictionary.Instance.GetDictionary();
var targetWords = WordSplitter.Split(id);
var commentPermutations = GetCommentedPermutations(id);
var commented = commentPermutations
.Where(c =>
{
var isOkay = TG263Dictionary.Instance.GetNameCompliance(c, false).GetAwaiter().GetResult();
return isOkay.IsNameOk;
})
.Select(c =>
{
var commentWords = WordSplitter.Split(c);
return new ConcreteStructureId()
{
StructureId = c,
StructIdWords = commentWords
};
}).ToList();
var concreteIds = dict.Where(d => d.StructureIds != null && d.StructureIds.Any())
.SelectMany(s => s.StructureIds)
.Concat(commented).ToList();
var candidates = concreteIds.Select(s =>
{
var strId = s.StructureId;
var words = s.StructIdWords.ToArray();
if (strId.Contains("00") && !string.IsNullOrEmpty(twoDigit))
{
strId = strId.Replace("00", twoDigit);
words = words.Select(w => w.Replace("00", twoDigit)).ToArray();
}
var metric = new StringMetrics()
{
Id = strId,
Dice = Dice.ComputeDistance(strId.ToUpper(), id.ToUpper()),
Levenshtein = Levenshtein.ComputeDistance(strId.ToUpper(), id.ToUpper())
};
return metric;
}).ToList();
//SORT
var ordered = candidates
.OrderBy(s => (s.Dice *s.Levenshtein));
return ordered.Take(numberOfRecommendations).ToList();
}));
}
public void CountOfUniqueLettersIsZero_ExceptionIsThrown()
{
Assert.Throws <ArgumentException> (() =>
{
var sut = new WordSplitter();
sut.SplitToParts(new Word("abc"), 0);
});
}
/// <summary>
/// The GetInstance.
/// </summary>
/// <returns>The <see cref="WordSplitter"/>.</returns>
public static WordSplitter GetInstance()
{
if (instance == null)
{
instance = new WordSplitter();
}
return(instance);
}
public CriteriaBuilder AddSearchString(string searchString, SimpleSearchCriteria searchCriteria = null)
{
Initialze(searchCriteria);
var terms = WordSplitter.ExtractSearchTerms(searchString).ToList();
SearchCriteriaReformer.ReformSearchCriteria(_searchCriteria);
_searchCriteria.SearchTerms = new SortedSet <string>(terms);
return(this);
}
public void WordSplitterRange()
{
string value = "one two/3 four-4 five(some more)";
// The '(' is one of the string terminators for this string splitter
Splitter splitter = WordSplitter.GetInstance();
// Single word
splitter.GetSplitRange(value, 0, 0).Should().BeNull();
splitter.GetSplitRange(value, 1, 1).Should().Be("one");
splitter.GetSplitRange(value, 2, 2).Should().Be("two");
splitter.GetSplitRange(value, 3, 3).Should().Be("3");
splitter.GetSplitRange(value, 4, 4).Should().Be("four");
splitter.GetSplitRange(value, 5, 5).Should().Be("4");
splitter.GetSplitRange(value, 6, 6).Should().Be("five");
splitter.GetSplitRange(value, 7, 7).Should().BeNull();
// First words
splitter.GetSplitRange(value, 1, 1).Should().Be("one");
splitter.GetSplitRange(value, 1, 2).Should().Be("one two");
splitter.GetSplitRange(value, 1, 3).Should().Be("one two/3");
splitter.GetSplitRange(value, 1, 4).Should().Be("one two/3 four");
splitter.GetSplitRange(value, 1, 5).Should().Be("one two/3 four-4");
splitter.GetSplitRange(value, 1, 6).Should().Be("one two/3 four-4 five");
splitter.GetSplitRange(value, 1, 7).Should().BeNull();
// Last words
splitter.GetSplitRange(value, 1, -1).Should().Be("one two/3 four-4 five");
splitter.GetSplitRange(value, 2, -1).Should().Be("two/3 four-4 five");
splitter.GetSplitRange(value, 3, -1).Should().Be("3 four-4 five");
splitter.GetSplitRange(value, 4, -1).Should().Be("four-4 five");
splitter.GetSplitRange(value, 5, -1).Should().Be("4 five");
splitter.GetSplitRange(value, 6, -1).Should().Be("five");
splitter.GetSplitRange(value, 7, -1).Should().BeNull();
// 2 word slices
splitter.GetSplitRange(value, 0, 1).Should().BeNull();
splitter.GetSplitRange(value, 1, 2).Should().Be("one two");
splitter.GetSplitRange(value, 2, 3).Should().Be("two/3");
splitter.GetSplitRange(value, 3, 4).Should().Be("3 four");
splitter.GetSplitRange(value, 4, 5).Should().Be("four-4");
splitter.GetSplitRange(value, 5, 6).Should().Be("4 five");
splitter.GetSplitRange(value, 6, 7).Should().BeNull();
// 3 word slices
splitter.GetSplitRange(value, 0, 2).Should().BeNull();
splitter.GetSplitRange(value, 1, 3).Should().Be("one two/3");
splitter.GetSplitRange(value, 2, 4).Should().Be("two/3 four");
splitter.GetSplitRange(value, 3, 5).Should().Be("3 four-4");
splitter.GetSplitRange(value, 4, 6).Should().Be("four-4 five");
splitter.GetSplitRange(value, 5, 7).Should().BeNull();
// Edge cases
splitter.GetSplitRange(value, 0, 0).Should().BeNull();
splitter.GetSplitRange(value, 0, -1).Should().BeNull();
splitter.GetSplitRange(value, -1, -1).Should().BeNull();
}
public void ParsingQuery()
{
bool invalidCharactersFound = WordSplitter.InvalidCharactersFound("session file info filetype:h");
Assert.IsFalse(invalidCharactersFound);
var terms = WordSplitter.ExtractSearchTerms("session file info filetype:h");
WordSplitter.GetFileExtensions("session file info filetype:h");
}
public void ExtractSearchTerms_ReturnsValidNumberOfSearchTermsWhenQuotesUsedInsideQuotes()
{
Assert.IsFalse(WordSplitter.InvalidCharactersFound("\"wordSplitter.ExtractWords(\"IInUnderscore\")\""));
Assert.IsFalse(WordSplitter.InvalidCharactersFound("\"Assert.IsNotNull(wordSplitter, \"Default word splitter should be used!\");\""));
Assert.IsNotNull(WordSplitter.ExtractSearchTerms("\"Assert.IsNotNull(wordSplitter, \"Default word splitter should be used!\");\""));
List <string> parts = WordSplitter.ExtractSearchTerms("\"wordSplitter.ExtractWords(\"IInUnderscore\")\"");
Assert.AreEqual(1, parts.Count);
}
public void BadCalls()
{
string value = "one two/3 four-4 five(some more)";
Splitter splitter = WordSplitter.GetInstance();
splitter.GetSplitRange(null, 1, 2).Should().BeNull();
splitter.GetSplitRange(value, -1, 2).Should().BeNull();
splitter.GetSplitRange(value, 1, -2).Should().BeNull();
splitter.GetSplitRange(value, 3, 2).Should().BeNull();
}
public void ExtractSearchTerms_ReturnsValidNumberOfSearchTermsWhenQuotesUsed()
{
List <string> parts = WordSplitter.ExtractSearchTerms("word \"words inside quotes\" another_word");
Assert.AreEqual(1, parts.Count);
Assert.AreEqual("\"word \"words inside quotes\" another_word\"", String.Join("*", parts));
parts = WordSplitter.ExtractSearchTerms("return \"..\\\\..\\\\Parser\"");
Assert.AreEqual(1, parts.Count);
Assert.AreEqual("\"return \"..\\\\..\\\\Parser\"\"", String.Join("*", parts));
}
public void ExtractSearchTerms_ContractFailsWhenSearchTermIsNull()
{
try
{
WordSplitter.ExtractSearchTerms(null);
}
catch
{
//contract exception catched here
}
Assert.True(contractFailed, "Contract should fail!");
}
public void ExtractSearchTerms_SlashesInsideQuotesInsideQuote()
{
//implicit quotes
List <string> parts = WordSplitter.ExtractSearchTerms("IsUnquotedLiteral(\"..\\Path\"))");
Assert.AreEqual(1, parts.Count);
Assert.AreEqual("\"IsUnquotedLiteral(\"..\\Path\"))\"", String.Join(" ", parts));
//explicit quotes
parts = WordSplitter.ExtractSearchTerms("\"IsUnquotedLiteral(\"..\\Path\"))\"");
Assert.AreEqual(1, parts.Count);
Assert.AreEqual("\"IsUnquotedLiteral(\"..\\Path\"))\"", String.Join(" ", parts));
}
public void Test001a_Split_VerifySplitWorks()
{
//Arrange
var l = "blabla kiki";
var ws = new WordSplitter(l);
//Act
var r = ws.Split();
//Assert
Assert.AreEqual(new List <string>()
{
"blabla", "kiki"
}, r);
}
public void WordSplitterOne()
{
string value = "word";
Splitter splitter = WordSplitter.GetInstance();
splitter.GetSingleSplit(value, -5).Should().BeNull();
splitter.GetSingleSplit(value, -1).Should().BeNull();
splitter.GetSingleSplit(value, 0).Should().BeNull();
splitter.GetSingleSplit(value, 1).Should().Be("word");
splitter.GetSingleSplit(value, 2).Should().BeNull();
splitter.GetFirstSplits(value, -5).Should().BeNull();
splitter.GetFirstSplits(value, -1).Should().BeNull();
splitter.GetFirstSplits(value, 0).Should().BeNull();
splitter.GetFirstSplits(value, 1).Should().Be("word");
splitter.GetFirstSplits(value, 2).Should().BeNull();
}
private void LoadDictionaryFromResources()
{
//LOAD REGEX PATTERNS AND TYPES
var structureNamesStringBlob = Properties.Resources.AllowedStructureNames;
string[] structureNamesLines = structureNamesStringBlob.Split(new string[] { "\r\n", "\n" }, StringSplitOptions.None);
tg263Structures = new List <TG263Structure>();
foreach (string l in structureNamesLines)
{
var keyvalue = l.Split('\t');
if (keyvalue.Length == 2)
{
var str = new TG263Structure()
{
StructureIdPattern = keyvalue[0].Trim(),
DICOMType = keyvalue[1].Trim().ToUpper(),
};
tg263Structures.Add(str);
}
}
var test = tg263Structures.ToList();
var append = new List <TG263Structure>();
//LOAD CONCRETE NAMES
for (int i = 0; i < tg263Structures.Count; i++)
{
var tg263 = tg263Structures[i];
var lines = Properties.Resources.StructureNamesConcrete.Split(new string[] { "\r\n", "\n" }, StringSplitOptions.None).ToList();
lines = lines.Select(l => l.Replace("xx", "00")).ToList();
var matches = lines.Where(l => (new Regex(tg263.StructureIdPattern, RegexOptions.IgnoreCase).IsMatch(l)));
if (matches.Any())
{
tg263.StructureIds = matches.Select(m =>
{
return(new ConcreteStructureId()
{
StructureId = m,
StructIdWords = WordSplitter.Split(m)
});
}).ToArray();
}
}
}
public void ExtractSearchTerms_FirstPartIsAlwaysOriginal()
{
const string term1 = "splitWord";
const string term2 = "splitAgain";
List <string> parts = WordSplitter.ExtractSearchTerms(term1);
Assert.AreEqual(3, parts.Count);
Assert.AreEqual(parts[0], term1.ToLower());
parts = WordSplitter.ExtractSearchTerms(term2);
Assert.AreEqual(3, parts.Count);
Assert.AreEqual(parts[0], term2.ToLower());
parts = WordSplitter.ExtractSearchTerms(term1 + " " + term2);
Assert.AreEqual(5, parts.Count);
Assert.AreEqual(parts[0], term1.ToLower());
Assert.AreEqual(parts[1], term2.ToLower());
}
//=====================================================================
/// <summary>
/// This is used to handle spell checking the given set of files in the background
/// </summary>
/// <param name="maxIssues">The maximum number of issues to report.</param>
/// <param name="spellCheckFiles">The files to spell check.</param>
/// <param name="codeAnalysisFiles">The code analysis dictionaries from each project that may be used in
/// the configurations used for spell checking.</param>
/// <param name="openDocuments">A list of documents open in the IDE. For these files, we'll get the
/// content from the editor if possible rather than the file on disk.</param>
private IEnumerable<FileMisspelling> SpellCheckFiles(int maxIssues,
IEnumerable<SpellCheckFileInfo> spellCheckFiles, Dictionary<string, List<string>> codeAnalysisFiles,
HashSet<string> openDocuments)
{
BindingList<FileMisspelling> issues = new BindingList<FileMisspelling>();
TextClassifier classifier;
List<string> cadFiles;
string documentText;
try
{
if(wordSplitter == null)
wordSplitter = new WordSplitter();
foreach(var file in spellCheckFiles)
{
cancellationTokenSource.Token.ThrowIfCancellationRequested();
// Get the code analysis files for the related project. These may be used to generate the
// spell checking configuration.
if(!codeAnalysisFiles.TryGetValue(file.ProjectFile, out cadFiles))
cadFiles = null;
wordSplitter.Configuration = file.GenerateConfiguration(cadFiles);
if(wordSplitter.Configuration != null)
{
// Create a dictionary for each configuration dictionary language ignoring any that are
// invalid and duplicates caused by missing languages which return the en-US dictionary.
var globalDictionaries = wordSplitter.Configuration.DictionaryLanguages.Select(l =>
GlobalDictionary.CreateGlobalDictionary(l, null,
wordSplitter.Configuration.AdditionalDictionaryFolders,
wordSplitter.Configuration.RecognizedWords)).Where(d => d != null).Distinct().ToList();
if(globalDictionaries.Any())
{
var dictionary = new SpellingDictionary(globalDictionaries,
wordSplitter.Configuration.IgnoredWords);
classifier = ClassifierFactory.GetClassifier(file.CanonicalName, wordSplitter.Configuration);
// If null, the file type is ignored
if(classifier != null)
{
// If open in an editor, use the current text from it if possible
if(openDocuments.Contains(file.CanonicalName))
{
documentText = GetDocumentText(file.CanonicalName);
if(documentText != null)
classifier.SetText(documentText);
}
// Switch to the UI thread to update the progress and then switch back to this one
ThreadHelper.Generic.Invoke(() =>
{
lblProgress.Text = "Spell checking " + file.Description;
});
foreach(var issue in this.GetMisspellingsInSpans(dictionary, classifier.Parse()))
{
issue.Dictionary = dictionary;
issue.ProjectName = Path.GetFileName(file.ProjectFile);
issue.Filename = file.Filename;
issue.CanonicalName = file.CanonicalName;
issue.LineNumber = classifier.GetLineNumber(issue.Span.Start);
issue.LineText = classifier[issue.LineNumber].Trim();
issues.Add(issue);
if(issues.Count >= maxIssues)
break;
}
}
}
}
if(issues.Count >= maxIssues)
break;
}
}
catch(OperationCanceledException)
{
System.Diagnostics.Debug.WriteLine("Spell checking process canceled");
}
finally
{
wordSplitter.Configuration = null;
}
return issues;
}
public void SetUp()
{
sut = new WordSplitter();
}
