public void ExtractSnippets() { const int snippetTokensCount = 12; var syntaxTree = CSharpSyntaxTree.ParseText(CommonTestData.SimpleProgramWithMethodAndProperty); var syntaxTreeRoot = syntaxTree.GetRoot(); var tokens = syntaxTreeRoot.GetTokens().Select(t => new CSharpToken(t)).ToList(); var extractor = new SnippetsExtractor(); var converter = new TokensKindsConverter(); var snippets = extractor.GetSnippets(tokens, snippetTokensCount, converter).ToList(); Assert.AreEqual(tokens.Count - snippetTokensCount + 1, snippets.Count); foreach (var snippet in snippets) { Assert.AreEqual(SnippetType.TokensKinds, snippet.SnippetType); Assert.AreEqual(snippetTokensCount, snippet.TokensCount); Console.WriteLine(snippet); } var lastTokens = tokens.TakeLast(snippetTokensCount).ToList(); var hasher = SnippetsExtractorOptions.Default.SequenceHasher; foreach (var token in lastTokens) { hasher.Enqueue(converter.Convert(token)); } Assert.AreEqual(hasher.GetCurrentHash(), snippets.Last().Hash); }
private async Task ExtractSnippetsFromSubmissionAsync(Submission submission) { logger.Information("Достаю сниппеты из решения {submissionId}, длина сниппетов: {tokensCount} токенов", submission.Id, configuration.SnippetTokensCount); var codeUnits = codeUnitsExtractor.Extract(submission.ProgramText); foreach (var codeUnit in codeUnits) { foreach (var tokenConverter in tokenConverters) { var snippets = snippetsExtractor.GetSnippets(codeUnit.Tokens, configuration.SnippetTokensCount, tokenConverter); foreach (var(index, snippet) in snippets.Enumerate()) { await snippetsRepo.AddSnippetOccurenceAsync(submission, snippet, codeUnit.FirstTokenIndex + index); } } } }