public static async Task AnalyseCodeInDirectory(string directoryName)
{
Console.WriteLine($"DEBUG: {nameof(AnalyserService)}.{nameof(AnalyseCodeInDirectory)} begin");
if (string.IsNullOrWhiteSpace(directoryName))
{
throw new ArgumentNullException(nameof(directoryName));
}
var project = DirectoryParser.Parse(directoryName);
var syntaxTrees = await Task.WhenAll(project.Documents.Select(d => d.GetSyntaxTreeAsync()));
var compilationRoots = syntaxTrees.Select(t => t.GetCompilationUnitRoot());
List <AnalyserItem> analyserItems = new List <AnalyserItem>();
foreach (var analyser in Analysers)
{
Console.WriteLine($"DEBUG: Starting {analyser.GetType().Name}");
foreach (var compilationRoot in compilationRoots)
{
analyser.Visit(compilationRoot);
}
Console.WriteLine($"DEBUG: Finished {analyser.GetType().Name}");
analyserItems.AddRange((analyser as IAnalyser).AnalyserItems);
}
await ConsoleReporter.Report(analyserItems);
Console.WriteLine($"DEBUG: {nameof(AnalyserService)}.{nameof(AnalyseCodeInDirectory)} end");
}
private void button1_Click(object sender, EventArgs e)
{
SetScanProperties();
int numberOfFiles = DirectoryParser.GetAllFiles(Folder, Recursive).Count;
backgroundWorker1.RunWorkerAsync(numberOfFiles);
}
private void StartScanForms(DoWorkEventArgs e)
{
if (String.IsNullOrWhiteSpace(Folder))
{
return;
}
Scan scan = new Scan
{
FilePath = Folder,
HashAlgorithm = Algorithm,
Time = DateTime.Now
};
try
{
scan.Insert();
}
catch (Exception ex)
{
}
List <string> allFiles = DirectoryParser.GetAllFiles(Folder, Recursive);
int nFiles = allFiles.Count;
int i = 0;
foreach (string file in allFiles)
{
if (backgroundWorker1.CancellationPending)
{
e.Cancel = true;
break;
}
else
{
try
{
FileScan.ScanFile(file, scan);
int progress = (int)Math.Ceiling((double)i / nFiles) * 100;
backgroundWorker1.ReportProgress(progress, (object)file);
i++;
}
catch (Exception ex)
{
continue;
}
}
}
}
public ApplicationContext ParseJavaFolder(string inputFolder, string outputFolder)
{
var ac = new ApplicationContext
{
InputDirectory = new DirectoryInfo(inputFolder),
OutputDirectory = new DirectoryInfo(outputFolder),
};
var parsedInputDirectory = DirectoryParser.Parse(ac.InputDirectory);
ac.RootDirectory = parsedInputDirectory;
return(ac);
}
static MetaParser()
{
_fileParsers = typeof(FileParser)
.Assembly.GetTypes()
.Where(t => t.IsSubclassOf(typeof(FileParser)) && !t.IsAbstract)
.Select(t => (FileParser)Activator.CreateInstance(t));
_directoryParser = new DirectoryParser();
#if DEBUG
foreach (var parser in _fileParsers)
{
Trace.WriteLine($"[{parser.GetType().Name}] {String.Join(", ", parser.GetSupportedFileExtensions())}");
}
#endif
}
private static void Icn()
{
string[] files = { "icnFiles.json", "icnEntites.json", "icnTree.json", "icnProject.json" };
IParser csParser = new DirectoryParser("");
IEnumerable <IFile> icnFiles;
EntityCollection icnEnitities;
Node <Entity> icnRoot;
if (!File.Exists(files[0]))
{
icnFiles = csParser.Read();
SaveClass(icnFiles, files[0]);
}
else
{
icnFiles = ReadClass <IEnumerable <DirectoryFile> >(files[0]);
}
if (!File.Exists(files[1]))
{
icnEnitities = Project.MakeEntities(icnFiles);
SaveClass(icnEnitities, files[1]);
}
else
{
icnEnitities = ReadClass <EntityCollection>(files[1]);
}
if (!File.Exists(files[2]))
{
icnRoot = Project.MakeTree(icnEnitities);
SaveClass(icnRoot, files[2]);
}
else
{
icnRoot = ReadClass <Node <Entity> >(files[2]);
}
Project project = new Project(icnEnitities, icnRoot, ProjectType.CSharp);
SaveClass(project, files[3]);
}
public static void StartScanConsole(string filePath, bool recursive, AvailableHashAlgorithms hashAlgorithm)
{
Scan scan = new Scan
{
FilePath = filePath,
HashAlgorithm = new FIMHashAlgorithm {
Id = (int)hashAlgorithm
},
Time = DateTime.Now
};
scan.Insert();
//insert scan in database
List <string> allFiles = DirectoryParser.GetAllFiles(filePath, recursive);
int nFiles = allFiles.Count;
int i = 0;
Console.WriteLine("Scanning ...");
foreach (string file in allFiles)
{
try
{
FileScan.ScanFile(file, scan);
Console.WriteLine(Math.Ceiling(((double)i / nFiles) * 100).ToString() + "%");
Console.SetCursorPosition(0, Console.CursorTop - 1);
System.Threading.Thread.Sleep(50);
i++;
}
catch (Exception ex)
{
Logger.Log(ex.Message, LogType.Error);
continue;
}
}
}
private void StartBackgroundScan()
{
int numberOfFiles = DirectoryParser.GetAllFiles(Folder, Recursive).Count;
backgroundWorker1.RunWorkerAsync(numberOfFiles);
}
public bool RefreshDatabase()
{
return(DirectoryParser.RefreshDatabase(db, config.GetValue <string>("musicFolderPath")));
}
private async Task ProcessWebDirectoryAsync(string name, WebDirectory webDirectory, CancellationToken cancellationToken)
{
if (Session.Parameters.ContainsKey(Constants.Parameters_GdIndex_RootId))
{
await Site.GoIndex.GdIndex.GdIndexParser.ParseIndex(HttpClient, webDirectory, string.Empty);
return;
}
if (!string.IsNullOrWhiteSpace(OpenDirectoryIndexerSettings.CommandLineOptions.UserAgent))
{
HttpClient.DefaultRequestHeaders.UserAgent.Clear();
HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd(OpenDirectoryIndexerSettings.CommandLineOptions.UserAgent);
}
HttpResponseMessage httpResponseMessage = await HttpClient.GetAsync(webDirectory.Url, cancellationToken);
string html = null;
if (httpResponseMessage.IsSuccessStatusCode)
{
SetRootUrl(httpResponseMessage);
html = await GetHtml(httpResponseMessage);
}
if (FirstRequest && !httpResponseMessage.IsSuccessStatusCode || httpResponseMessage.IsSuccessStatusCode && string.IsNullOrWhiteSpace(html) || html?.Contains("HTTP_USER_AGENT") == true)
{
Logger.Warn("First request fails, using Curl fallback User-Agent");
HttpClient.DefaultRequestHeaders.UserAgent.Clear();
HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd(Constants.UserAgent.Curl);
httpResponseMessage = await HttpClient.GetAsync(webDirectory.Url, cancellationToken);
if (httpResponseMessage.IsSuccessStatusCode)
{
SetRootUrl(httpResponseMessage);
html = await GetHtml(httpResponseMessage);
Logger.Warn("Yes, Curl User-Agent did the trick!");
}
}
if (FirstRequest && !httpResponseMessage.IsSuccessStatusCode || httpResponseMessage.IsSuccessStatusCode && string.IsNullOrWhiteSpace(html))
{
Logger.Warn("First request fails, using Chrome fallback User-Agent");
HttpClient.DefaultRequestHeaders.UserAgent.Clear();
HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd(Constants.UserAgent.Chrome);
httpResponseMessage = await HttpClient.GetAsync(webDirectory.Url, cancellationToken);
if (httpResponseMessage.IsSuccessStatusCode)
{
SetRootUrl(httpResponseMessage);
html = await GetHtml(httpResponseMessage);
Logger.Warn("Yes, Chrome User-Agent did the trick!");
}
}
if (!HttpClient.DefaultRequestHeaders.Contains("Referer"))
{
HttpClient.DefaultRequestHeaders.Add("Referer", webDirectory.Url);
}
bool calibreDetected = false;
string calibreVersionString = string.Empty;
if (httpResponseMessage.IsSuccessStatusCode)
{
FirstRequest = false;
List <string> serverHeaders = new List <string>();
if (httpResponseMessage.Headers.Contains("Server"))
{
serverHeaders = httpResponseMessage.Headers.GetValues("Server").ToList();
calibreDetected = serverHeaders.Any(h => h.Contains("calibre"));
}
if (calibreDetected)
{
string serverHeader = string.Join("/", serverHeaders);
calibreVersionString = serverHeader;
}
else
{
if (html == null)
{
html = await GetHtml(httpResponseMessage);
}
// UNTESTED (cannot find or down Calibre with this issue)
const string calibreVersionIdentifier = "CALIBRE_VERSION = \"";
calibreDetected = html?.Contains(calibreVersionIdentifier) == true;
if (calibreDetected)
{
int calibreVersionIdentifierStart = html.IndexOf(calibreVersionIdentifier);
calibreVersionString = html.Substring(calibreVersionIdentifierStart, html.IndexOf("\"", ++calibreVersionIdentifierStart));
}
}
}
if (calibreDetected)
{
Version calibreVersion = CalibreParser.ParseVersion(calibreVersionString);
Console.WriteLine($"Calibre {calibreVersion} detected! I will index it at max 100 books per 30 seconds, else it will break Calibre...");
Logger.Info($"Calibre {calibreVersion} detected! I will index it at max 100 books per 30 seconds, else it will break Calibre...");
await CalibreParser.ParseCalibre(HttpClient, httpResponseMessage.RequestMessage.RequestUri, webDirectory, calibreVersion, cancellationToken);
return;
}
if (httpResponseMessage.IsSuccessStatusCode && webDirectory.Url != httpResponseMessage.RequestMessage.RequestUri.ToString())
{
webDirectory.Url = httpResponseMessage.RequestMessage.RequestUri.ToString();
}
Uri originalUri = new Uri(webDirectory.Url);
Logger.Debug($"[{name}] Finish download '{webDirectory.Url}'");
// Process only same site
if (httpResponseMessage.RequestMessage.RequestUri.Host == Session.Root.Uri.Host)
{
int httpStatusCode = (int)httpResponseMessage.StatusCode;
if (!Session.HttpStatusCodes.ContainsKey(httpStatusCode))
{
Session.HttpStatusCodes[httpStatusCode] = 0;
}
Session.HttpStatusCodes[httpStatusCode]++;
if (httpResponseMessage.IsSuccessStatusCode)
{
if (html == null)
{
html = await GetHtml(httpResponseMessage);
}
Session.TotalHttpTraffic += html.Length;
WebDirectory parsedWebDirectory = await DirectoryParser.ParseHtml(webDirectory, html, HttpClient);
bool processSubdirectories = parsedWebDirectory.Parser != "DirectoryListingModel01";
AddProcessedWebDirectory(webDirectory, parsedWebDirectory, processSubdirectories);
}
else
{
httpResponseMessage.EnsureSuccessStatusCode();
}
}
else
{
Logger.Warn($"[{name}] Skipped result of '{webDirectory.Url}' which points to '{httpResponseMessage.RequestMessage.RequestUri}'");
Session.Skipped++;
}
}
private async Task WebDirectoryProcessor(ConcurrentQueue <WebDirectory> queue, string name, CancellationToken cancellationToken)
{
Logger.Debug($"Start [{name}]");
bool maxConnections = false;
do
{
Interlocked.Increment(ref RunningWebDirectoryThreads);
if (queue.TryDequeue(out WebDirectory webDirectory))
{
try
{
lock (WebDirectoryProcessorInfoLock)
{
WebDirectoryProcessorInfo[name] = webDirectory;
}
if (!Session.ProcessedUrls.Contains(webDirectory.Url))
{
Session.ProcessedUrls.Add(webDirectory.Url);
webDirectory.StartTime = DateTimeOffset.UtcNow;
Logger.Info($"[{name}] Begin processing {webDirectory.Url}");
if (Session.Root.Uri.Scheme == Constants.UriScheme.Ftp || Session.Root.Uri.Scheme == Constants.UriScheme.Ftps)
{
WebDirectory parsedWebDirectory = await FtpParser.ParseFtpAsync(name, webDirectory, OpenDirectoryIndexerSettings.Username, OpenDirectoryIndexerSettings.Password);
if (webDirectory?.CancellationReason == Constants.Ftp_Max_Connections)
{
webDirectory.CancellationReason = null;
maxConnections = true;
if (webDirectory.Name == Constants.Root)
{
webDirectory.Error = true;
Interlocked.Decrement(ref RunningWebDirectoryThreads);
throw new Exception("Error checking FTP because maximum connections reached");
}
// Requeue
Session.ProcessedUrls.Remove(webDirectory.Url);
queue.Enqueue(webDirectory);
try
{
await FtpParser.FtpClients[name].DisconnectAsync(cancellationToken);
lock (FtpParser.FtpClients)
{
FtpParser.FtpClients.Remove(name);
}
}
catch (Exception exFtpDisconnect)
{
Logger.Error(exFtpDisconnect, "Error disconnecting FTP connection.");
}
}
if (parsedWebDirectory != null)
{
DirectoryParser.CheckParsedResults(parsedWebDirectory);
AddProcessedWebDirectory(webDirectory, parsedWebDirectory);
}
}
else if (Session.Root.Uri.Host == Constants.GoogleDriveDomain)
{
string baseUrl = webDirectory.Url;
WebDirectory parsedWebDirectory = await GoogleDriveIndexer.IndexAsync(webDirectory);
parsedWebDirectory.Url = baseUrl;
AddProcessedWebDirectory(webDirectory, parsedWebDirectory);
}
else
{
if (Session.Root.Uri.Host == Constants.BlitzfilesTechDomain || SameHostAndDirectory(Session.Root.Uri, webDirectory.Uri))
{
Logger.Debug($"[{name}] Start download '{webDirectory.Url}'");
Session.TotalHttpRequests++;
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
cancellationTokenSource.CancelAfter(TimeSpan.FromMinutes(5));
Context pollyContext = new Context
{
{ "Processor", name },
{ "WebDirectory", webDirectory },
{ "CancellationTokenSource", cancellationTokenSource }
};
await RetryPolicy.ExecuteAsync(async (context, token) => { await ProcessWebDirectoryAsync(name, webDirectory, cancellationTokenSource.Token); }, pollyContext, cancellationTokenSource.Token);
}
else
{
Logger.Warn($"[{name}] Skipped result of '{webDirectory.Url}' because it is not the same host or path");
Session.Skipped++;
}
}
Logger.Info($"[{name}] Finished processing {webDirectory.Url}");
}
else
{
//Logger.Warn($"[{name}] Skip, already processed: {webDirectory.Uri}");
}
}
catch (Exception ex)
{
if (ex is TaskCanceledException taskCanceledException)
{
Session.Errors++;
webDirectory.Error = true;
if (!Session.UrlsWithErrors.Contains(webDirectory.Url))
{
Session.UrlsWithErrors.Add(webDirectory.Url);
}
if (webDirectory.ParentDirectory?.Url != null)
{
Logger.Error($"Skipped processing Url: '{webDirectory.Url}' from parent '{webDirectory.ParentDirectory.Url}'");
}
else
{
Logger.Error($"Skipped processing Url: '{webDirectory.Url}'");
Session.Root.Error = true;
}
}
else
{
Logger.Error(ex, $"Error processing Url: '{webDirectory.Url}' from parent '{webDirectory.ParentDirectory?.Url}'");
}
}
finally
{
lock (WebDirectoryProcessorInfoLock)
{
WebDirectoryProcessorInfo.Remove(name);
}
if (string.IsNullOrWhiteSpace(webDirectory.CancellationReason))
{
webDirectory.Finished = true;
webDirectory.FinishTime = DateTimeOffset.UtcNow;
}
}
}
Interlocked.Decrement(ref RunningWebDirectoryThreads);
// Needed, because of the TryDequeue, no waiting in ConcurrentQueue!
if (queue.IsEmpty)
{
// Don't hog the CPU when queue < threads
await Task.Delay(TimeSpan.FromMilliseconds(1000), cancellationToken);
}
else
{
await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationToken);
}
}while (!cancellationToken.IsCancellationRequested && (!queue.IsEmpty || RunningWebDirectoryThreads > 0) && !maxConnections);
Logger.Debug($"Finished [{name}]");
}
public static async Task <WebDirectory> ParseHtml(string html, string url = "http://localhost/")
{
return(await DirectoryParser.ParseHtml(new WebDirectory(null) { Url = url }, html));
}
private async Task WebDirectoryProcessor(ConcurrentQueue <WebDirectory> queue, string name, CancellationToken token)
{
Logger.Debug($"Start [{name}]");
do
{
Interlocked.Increment(ref RunningWebDirectoryThreads);
if (queue.TryDequeue(out WebDirectory webDirectory))
{
try
{
lock (WebDirectoryProcessorInfoLock)
{
WebDirectoryProcessorInfo[name] = webDirectory;
}
if (!Session.ProcessedUrls.Contains(webDirectory.Url))
{
Session.ProcessedUrls.Add(webDirectory.Url);
Logger.Info($"[{name}] Begin processing {webDirectory.Url}");
if (Session.Root.Uri.Scheme == "ftp")
{
WebDirectory parsedWebDirectory = await FtpParser.ParseFtpAsync(name, webDirectory);
AddProcessedWebDirectory(webDirectory, parsedWebDirectory);
}
else
if (Session.Root.Uri.Host == "drive.google.com")
{
string baseUrl = webDirectory.Url;
WebDirectory parsedWebDirectory = await GoogleDriveIndexer.IndexAsync(webDirectory);
parsedWebDirectory.Url = baseUrl;
AddProcessedWebDirectory(webDirectory, parsedWebDirectory);
}
else
{
if (webDirectory.Uri.Host == Session.Root.Uri.Host && webDirectory.Uri.LocalPath.StartsWith(Session.Root.Uri.LocalPath))
{
Logger.Debug($"[{name}] Start download '{webDirectory.Url}'");
Session.TotalHttpRequests++;
await RetryPolicy.ExecuteAsync(async() =>
{
webDirectory.StartTime = DateTimeOffset.UtcNow;
HttpResponseMessage httpResponseMessage = await HttpClient.GetAsync(webDirectory.Url);
string html = null;
if (httpResponseMessage.IsSuccessStatusCode)
{
html = await GetHtml(httpResponseMessage);
}
if (FirstRequest && !httpResponseMessage.IsSuccessStatusCode || httpResponseMessage.IsSuccessStatusCode && string.IsNullOrWhiteSpace(html))
{
Logger.Warn("First request fails, using Curl fallback User-Agent");
HttpClient.DefaultRequestHeaders.UserAgent.Clear();
HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent_Curl);
httpResponseMessage = await HttpClient.GetAsync(webDirectory.Url);
if (httpResponseMessage.IsSuccessStatusCode)
{
html = await GetHtml(httpResponseMessage);
Logger.Warn("Yes, this Curl User-Agent did the trick!");
}
}
if (FirstRequest && !httpResponseMessage.IsSuccessStatusCode || httpResponseMessage.IsSuccessStatusCode && string.IsNullOrWhiteSpace(html))
{
Logger.Warn("First request fails, using Chrome fallback User-Agent");
HttpClient.DefaultRequestHeaders.UserAgent.Clear();
HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent_Chrome);
httpResponseMessage = await HttpClient.GetAsync(webDirectory.Url);
if (httpResponseMessage.IsSuccessStatusCode)
{
html = await GetHtml(httpResponseMessage);
Logger.Warn("Yes, the Chrome User-Agent did the trick!");
}
}
bool calibreDetected = false;
string calibreVersionString = string.Empty;
if (httpResponseMessage.IsSuccessStatusCode)
{
FirstRequest = false;
List <string> serverHeaders = new List <string>();
if (httpResponseMessage.Headers.Contains("Server"))
{
serverHeaders = httpResponseMessage.Headers.GetValues("Server").ToList();
calibreDetected = serverHeaders.Any(h => h.Contains("calibre"));
}
if (calibreDetected)
{
string serverHeader = string.Join("/", serverHeaders);
calibreVersionString = serverHeader;
}
else
{
if (html == null)
{
html = await GetHtml(httpResponseMessage);
}
// UNTESTED (cannot find or down Calibre with this issue)
const string calibreVersionIdentifier = "CALIBRE_VERSION = \"";
calibreDetected = html?.Contains(calibreVersionIdentifier) == true;
if (calibreDetected)
{
int calibreVersionIdentifierStart = html.IndexOf(calibreVersionIdentifier);
calibreVersionString = html.Substring(calibreVersionIdentifierStart, html.IndexOf("\"", ++calibreVersionIdentifierStart));
}
}
}
if (calibreDetected)
{
Version calibreVersion = CalibreParser.ParseVersion(calibreVersionString);
Console.WriteLine($"Calibre {calibreVersion} detected! I will index it at max 100 books per 30 seconds, else it will break Calibre...");
Logger.Info($"Calibre {calibreVersion} detected! I will index it at max 100 books per 30 seconds, else it will break Calibre...");
await CalibreParser.ParseCalibre(HttpClient, httpResponseMessage.RequestMessage.RequestUri, webDirectory, calibreVersion);
return;
}
Uri originalUri = new Uri(webDirectory.Url);
Logger.Debug($"[{name}] Finish download '{webDirectory.Url}'");
// Process only same site
if (httpResponseMessage.RequestMessage.RequestUri.Host == Session.Root.Uri.Host)
{
int httpStatusCode = (int)httpResponseMessage.StatusCode;
if (!Session.HttpStatusCodes.ContainsKey(httpStatusCode))
{
Session.HttpStatusCodes[httpStatusCode] = 0;
}
Session.HttpStatusCodes[httpStatusCode]++;
if (httpResponseMessage.IsSuccessStatusCode)
{
if (html == null)
{
html = await GetHtml(httpResponseMessage);
}
Session.TotalHttpTraffic += html.Length;
WebDirectory parsedWebDirectory = await DirectoryParser.ParseHtml(webDirectory, html, HttpClient);
AddProcessedWebDirectory(webDirectory, parsedWebDirectory);
}
else
{
Session.Errors++;
webDirectory.Error = true;
if (!Session.UrlsWithErrors.Contains(webDirectory.Url))
{
Session.UrlsWithErrors.Add(webDirectory.Url);
}
httpResponseMessage.EnsureSuccessStatusCode();
}
}
else
{
Logger.Warn($"[{name}] Skipped result of '{webDirectory.Url}' which points to '{httpResponseMessage.RequestMessage.RequestUri}'");
Session.Skipped++;
}
});
}
else
{
Logger.Warn($"[{name}] Skipped result of '{webDirectory.Url}' because it is not the same host or path");
Session.Skipped++;
}
}
Logger.Info($"[{name}] Finished processing {webDirectory.Url}");
}
else
{
Logger.Warn($"[{name}] Skip, already processed: {webDirectory.Uri}");
}
}
catch (Exception ex)
{
Logger.Error(ex, $"Error processing Url: '{webDirectory.Url}' from parent '{webDirectory.ParentDirectory.Url}'");
Session.Errors++;
if (!Session.UrlsWithErrors.Contains(webDirectory.Url))
{
Session.UrlsWithErrors.Add(webDirectory.Url);
}
}
finally
{
lock (WebDirectoryProcessorInfoLock)
{
WebDirectoryProcessorInfo.Remove(name);
}
}
}
Interlocked.Decrement(ref RunningWebDirectoryThreads);
// Needed!
await Task.Delay(TimeSpan.FromMilliseconds(10));
}while (!token.IsCancellationRequested && (!queue.IsEmpty || RunningWebDirectoryThreads > 0));
Logger.Debug($"Finished [{name}]");
}
public static async Task <WebDirectory> ParseHtml(string html, string url = "http://localhost/", bool checkParents = true)
{
return(await DirectoryParser.ParseHtml(new WebDirectory(null) { Url = url }, html, checkParents : checkParents));
}
