private string GetTokenizerText(string text)
{
StringBuilder result = new StringBuilder();
MyAnalyzer ma = new MyAnalyzer(AnalyzerFactory.stopWords);
Lucene.Net.Analysis.TokenStream ts = ma.TokenStream("", new System.IO.StringReader(text));
Lucene.Net.Analysis.Token token;
while ((token = ts.Next()) != null)
{
int len = token.TermLength();
char[] buff = token.TermBuffer();
if (len == 1)
{
if (buff[0] != MyFilter.Separator)
{
result.Append(buff, 0, 1);
}
}
else
{
result.Append(buff, 0, len);
}
}
ts.Close();
return(result.ToString());
}
private int GetTokenizerLength(string text)
{
int result = 0;
MyAnalyzer ma = new MyAnalyzer(AnalyzerFactory.stopWords);
Lucene.Net.Analysis.TokenStream ts = ma.TokenStream("", new System.IO.StringReader(text));
Lucene.Net.Analysis.Token token;
while ((token = ts.Next()) != null)
{
int len = token.TermLength();
if (len == 1)
{
char[] buff = token.TermBuffer();
if (buff[0] != MyFilter.Separator)
{
result++;
}
}
else
{
result += len;
}
}
ts.Close();
return(result);
}
public void URLSeriveTest()
{
string text = @"https://www.4guysfromrolla.com/articles/011211-1.aspx";
MyAnalyzer service = new MyAnalyzer();
string host = "4guysfromrolla";
var result = service.GetSiteResults(url: text, host: "");
}
private static void TestDescriptorIsExceptionSafeCore(DiagnosticDescriptor descriptor)
{
var localizableTitle = descriptor.Title;
var localizableMessage = descriptor.MessageFormat;
var localizableDescription = descriptor.Description;
// Verify exceptions from LocalizableResourceString don't go unhandled.
var title = localizableTitle.ToString();
var message = localizableMessage.ToString();
var description = localizableDescription.ToString();
// Verify exceptions from LocalizableResourceString are raised if OnException is set.
var exceptions = new List <Exception>();
var handler = new EventHandler <Exception>((sender, ex) => exceptions.Add(ex));
localizableTitle.OnException += handler;
localizableMessage.OnException += handler;
localizableDescription.OnException += handler;
// Access and evaluate localizable fields.
var unused1 = localizableTitle.ToString();
var unused2 = localizableMessage.ToString();
var unused3 = localizableDescription.ToString();
Assert.Equal(3, exceptions.Count);
// Verify DiagnosticAnalyzer.SupportedDiagnostics is also exception safe.
var analyzer = new MyAnalyzer(descriptor);
var exceptionDiagnostics = new List <Diagnostic>();
Action <Exception, DiagnosticAnalyzer, Diagnostic> onAnalyzerException = (ex, a, diag) =>
exceptionDiagnostics.Add(diag);
var analyzerManager = new AnalyzerManager(analyzer);
var analyzerExecutor = AnalyzerExecutor.CreateForSupportedDiagnostics(
onAnalyzerException,
analyzerManager
);
var descriptors = analyzerManager.GetSupportedDiagnosticDescriptors(
analyzer,
analyzerExecutor
);
Assert.Equal(1, descriptors.Length);
Assert.Equal(descriptor.Id, descriptors[0].Id);
// Access and evaluate localizable fields.
unused1 = descriptors[0].Title.ToString();
unused2 = descriptors[0].MessageFormat.ToString();
unused3 = descriptors[0].Description.ToString();
// Verify logged analyzer exception diagnostics.
Assert.Equal(3, exceptionDiagnostics.Count);
Assert.True(
exceptionDiagnostics.TrueForAll(AnalyzerExecutor.IsAnalyzerExceptionDiagnostic)
);
}
public void TextSeriveTest()
{
string text = @"The Visual Studio Customer Experience Improvement Program (VSCEIP) is designed
to help Microsoft improve Visual Studio over time. This program collects information about errors,
computer hardware, and how people use Visual Studio, without interrupting users in their tasks at the computer.
The information that's collected helps Microsoft identify which features to improve.
This document covers how to opt in or out of the VSCEIP.";
MyAnalyzer service = new MyAnalyzer();
var result = service.GetWordResults(text: text);
}
public void RunAnalyzersAfterGeneratingSource()
{
string text =
@"class C
{
}";
using (var directory = new DisposableDirectory(Temp))
{
var file = directory.CreateFile("c.cs");
file.WriteAllText(text);
int analyzerCalls = 0;
ImmutableArray <SyntaxTree> treesToAnalyze;
var analyzer = new MyAnalyzer(c =>
{
analyzerCalls++;
Assert.True(treesToAnalyze.IsDefault);
treesToAnalyze = ImmutableArray.CreateRange(c.Compilation.SyntaxTrees);
});
int generatorCalls = 0;
var generator = new MyGenerator(c =>
{
generatorCalls++;
c.AddCompilationUnit("__c", CSharpSyntaxTree.ParseText("class __C { }"));
});
var compiler = new MyCompiler(
baseDirectory: directory.Path,
args: new[] { "/nologo", "/preferreduilang:en", "/t:library", file.Path },
analyzers: ImmutableArray.Create <DiagnosticAnalyzer>(analyzer),
generators: ImmutableArray.Create <SourceGenerator>(generator));
var builder = new StringBuilder();
using (var writer = new StringWriter(builder))
{
compiler.Run(writer);
}
var output = builder.ToString();
// No errors from analyzer.
Assert.Equal("", output);
Assert.Equal(1, generatorCalls);
Assert.Equal(1, analyzerCalls);
Assert.Equal(2, treesToAnalyze.Length);
}
}
private void button1_Click(object sender, EventArgs e)
{
//TokenStream ts = new LetterDigitTokenizer(new System.IO.StringReader("Hiro's Shadow Scout"));
//TokenStream ts = new StandardTokenizer(new System.IO.StringReader("荷鲁斯之黑炎龙8"));
MyAnalyzer ma = new MyAnalyzer(new string[0]);
TokenStream ts = ma.TokenStream("", new StringReader("荷鲁斯之黑炎龙LV8"));
Token token;
while ((token = ts.Next()) != null)
{
this.richTextBox1.AppendText(token.TermText());
this.richTextBox1.AppendText(" ");
}
ts.Close();
}
public void RunAnalyzersAfterGeneratingSource()
{
string text =
@"class C
{
}";
using (var directory = new DisposableDirectory(Temp))
{
var file = directory.CreateFile("c.cs");
file.WriteAllText(text);
int analyzerCalls = 0;
ImmutableArray<SyntaxTree> treesToAnalyze;
var analyzer = new MyAnalyzer(c =>
{
analyzerCalls++;
Assert.True(treesToAnalyze.IsDefault);
treesToAnalyze = ImmutableArray.CreateRange(c.Compilation.SyntaxTrees);
});
int generatorCalls = 0;
var generator = new MyGenerator(c =>
{
generatorCalls++;
c.AddCompilationUnit("__c", CSharpSyntaxTree.ParseText("class __C { }"));
});
var compiler = new MyCompiler(
baseDirectory: directory.Path,
args: new[] { "/nologo", "/preferreduilang:en", "/t:library", file.Path },
analyzers: ImmutableArray.Create<DiagnosticAnalyzer>(analyzer),
generators: ImmutableArray.Create<SourceGenerator>(generator));
var builder = new StringBuilder();
using (var writer = new StringWriter(builder))
{
compiler.Run(writer);
}
var output = builder.ToString();
// No errors from analyzer.
Assert.Equal("", output);
Assert.Equal(1, generatorCalls);
Assert.Equal(1, analyzerCalls);
Assert.Equal(2, treesToAnalyze.Length);
}
}
public ActionResult Analyze(string webURL, string isEnabledAnalyze)
{
ViewMessage viewMessage = null;
if (string.IsNullOrWhiteSpace(isEnabledAnalyze))
{
if (!string.IsNullOrWhiteSpace(webURL))
{
Uri uriResult;
if (Uri.TryCreate(webURL, UriKind.Absolute, out uriResult))
{
var analyzer = new MyAnalyzer();
viewMessage = analyzer.GetSiteResults(url: webURL, host: uriResult.Host);
if (string.Compare(viewMessage.StatusID, "0") != 0)
{
return(View("Index", viewMessage));
}
return(View("AnalyzeSite", viewMessage));
}
else
{
// Text mode
var analyzer = new MyAnalyzer();
viewMessage = analyzer.GetWordResults(webURL);
return(View("AnalyzeText", viewMessage));
}
}
// SEO disabled
return(View("Index", new ViewMessage()
{
StatusID = "1",
Message = "Please submit text or URL."
}));
}
else
{
// SEO disabled
return(View("Index", new ViewMessage()
{
StatusID = "1",
Message = "SEO is Disabled.",
isEnabledAnalyzeChecked = isEnabledAnalyze != null
}));
}
}
private static void TestDescriptorIsExceptionSafeCore(DiagnosticDescriptor descriptor)
{
var localizableTitle = descriptor.Title;
var localizableMessage = descriptor.MessageFormat;
var localizableDescription = descriptor.Description;
// Verify exceptions from LocalizableResourceString don't go unhandled.
var title = localizableTitle.ToString();
var message = localizableMessage.ToString();
var description = localizableDescription.ToString();
// Verify exceptions from LocalizableResourceString are raised if OnException is set.
var exceptions = new List<Exception>();
var handler = new EventHandler<Exception>((sender, ex) => exceptions.Add(ex));
localizableTitle.OnException += handler;
localizableMessage.OnException += handler;
localizableDescription.OnException += handler;
// Access and evaluate localizable fields.
var unused1 = localizableTitle.ToString();
var unused2 = localizableMessage.ToString();
var unused3 = localizableDescription.ToString();
Assert.Equal(3, exceptions.Count);
// Verify DiagnosticAnalyzer.SupportedDiagnostics is also exception safe.
var analyzer = new MyAnalyzer(descriptor);
var exceptionDiagnostics = new List<Diagnostic>();
Action<Exception, DiagnosticAnalyzer, Diagnostic> onAnalyzerException = (ex, a, diag) => exceptionDiagnostics.Add(diag);
var analyzerExecutor = AnalyzerExecutor.CreateForSupportedDiagnostics(onAnalyzerException, AnalyzerManager.Instance);
var descriptors = AnalyzerManager.Instance.GetSupportedDiagnosticDescriptors(analyzer, analyzerExecutor);
Assert.Equal(1, descriptors.Length);
Assert.Equal(descriptor.Id, descriptors[0].Id);
// Access and evaluate localizable fields.
unused1 = descriptors[0].Title.ToString();
unused2 = descriptors[0].MessageFormat.ToString();
unused3 = descriptors[0].Description.ToString();
// Verify logged analyzer exception diagnostics.
Assert.Equal(3, exceptionDiagnostics.Count);
Assert.True(exceptionDiagnostics.TrueForAll(AnalyzerExecutor.IsAnalyzerExceptionDiagnostic));
}
static void Main(string[] args)
{
Stopwatch sw = Stopwatch.StartNew();
// инициализация - необходимо проводить один раз до обработки текстов
Console.Write("Initializing ... ");
ProcessorService.Initialize(MorphLang.RU | MorphLang.EN);
// инициализируются все используемые анализаторы
MyAnalyzer.Initialize();
Console.WriteLine("OK (by {0} ms), version {1}", (int)sw.ElapsedMilliseconds, ProcessorService.Version);
AnalysisResult ar = null;
if (args.Length > 0)
{
if (args[0] == "csv")
{
ClientApiSettings settings = new ClientApiSettings();
foreach (var file in Directory.GetFiles("Texts", "*.csv"))
{
using (var sr = new StreamReader(file))
{
var i = 1;
var line = sr.ReadLine();
while (line != null)
{
var data = line.Split(';', ' ', ';');
if (data.Length < 3)
{
Console.WriteLine("Ошибка формата csv. \r\n Формат\r\n Name;CaseId;DocumentId");
}
GlobalState.File = i + "_" + MakeValidFileName(data[0]) + ".txt";
var client = new PravoRu.DataLake.Arbitr.CaseCard.Api.Client.v1.FileClient(settings,
new HttpClientFactory());
DocumentPlainText response = null;
try
{
response = client.GetDocumentTextAsync(new DocumentFileRequest()
{
CaseId = Guid.Parse(data[1]),
IsBase64 = false,
DocumentId = Guid.Parse(data[2])
}).GetAwaiter().GetResult();
}
catch (Exception e)
{
Console.WriteLine(data[0] + "\t" + e.Message);
}
if (response == null)
{
line = sr.ReadLine();
continue;
}
File.WriteAllText(Path.Combine("Results", "Original_" + GlobalState.File), response.HtmlText);
// создаём экземпляр обычного процессора
using (Processor proc = ProcessorService.CreateSpecificProcessor(nameof(MyAnalyzer)))
{
// анализируем текст
ar = proc.Process(new SourceOfAnalysis(response.HtmlText));
try
{
PostExecute(ar);
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
Console.WriteLine("Обработан файл " + GlobalState.File);
line = sr.ReadLine();
i++;
}
}
}
}
if (args[0] == "txt")
{
foreach (var file in Directory.GetFiles("Texts", "*.txt"))
{
Console.WriteLine($"{file}------------------------------------");
string txt = File.ReadAllText(file);
GlobalState.File = new FileInfo(file).Name;
// создаём экземпляр обычного процессора
using (Processor proc = ProcessorService.CreateSpecificProcessor(nameof(MyAnalyzer)))
{
// анализируем текст
ar = proc.Process(new SourceOfAnalysis(txt));
try
{
PostExecute(ar);
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
}
}
}
sw.Stop();
Console.WriteLine("Over!(by {0} ms), version {1}", (int)sw.ElapsedMilliseconds, ProcessorService.Version);
}
