private void App_Exit(object sender, ExitEventArgs e)
{
try
{
MutexList.RemoveAll();
}
catch (Exception ex)
{
LogWriter.Log(ex, "Impossible to remove all mutexes of the opened projects.");
}
try
{
NotifyIcon?.Dispose();
}
catch (Exception ex)
{
LogWriter.Log(ex, "Impossible to dispose the system tray icon.");
}
try
{
EncodingManager.StopAllEncodings();
}
catch (Exception ex)
{
LogWriter.Log(ex, "Impossible to cancel all encodings.");
}
try
{
UserSettings.Save();
}
catch (Exception ex)
{
LogWriter.Log(ex, "Impossible to save the user settings.");
}
try
{
if (_mutex != null && _accepted)
{
_mutex.ReleaseMutex();
_accepted = false;
}
}
catch (Exception ex)
{
LogWriter.Log(ex, "Impossible to release the single instance mutex.");
}
try
{
HotKeyCollection.Default.Dispose();
}
catch (Exception ex)
{
LogWriter.Log(ex, "Impossible to dispose the hotkeys.");
}
}
public Main()
{
_encodingManager = new EncodingManager();
var creationViewModel = new TaskCreationViewModel(_encodingManager);
_model = new MainWindowViewModel(_encodingManager.Model, creationViewModel)
{
CancelTaskCommand = new RelayCommand <EncodingTaskBase>(_encodingManager.CancelTask)
};
}
/// <summary>
/// Processes a FilesRow after crawling.
/// </summary>
/// <param name = "filesRow">The files row.</param>
/// <param name="webClient"></param>
/// <param name="actionManager"></param>
/// <param name="consoleManager"></param>
/// <param name="discoveryManager"></param>
/// <param name = "fileManager">The file manager.</param>
/// <param name = "fileManager">The file manager.</param>
/// <param name="memoryManager"></param>
/// <param name="ruleManager"></param>
/// <param name = "arachnodeDAO">The arachnode DAO.</param>
/// <param name = "imageManager">The image manager.</param>
public static void ProcessFile(ApplicationSettings applicationSettings, WebSettings webSettings, Crawler <TArachnodeDAO> crawler, ArachnodeDataSet.FilesRow filesRow, WebClient <TArachnodeDAO> webClient, Cache <TArachnodeDAO> cache, ActionManager <TArachnodeDAO> actionManager, ConsoleManager <TArachnodeDAO> consoleManager, CrawlerPeerManager <TArachnodeDAO> crawlerPeerManager, DiscoveryManager <TArachnodeDAO> discoveryManager, FileManager <TArachnodeDAO> fileManager, MemoryManager <TArachnodeDAO> memoryManager, RuleManager <TArachnodeDAO> ruleManager, IArachnodeDAO arachnodeDAO)
{
CacheManager <TArachnodeDAO> cacheManager = new CacheManager <TArachnodeDAO>(applicationSettings, webSettings);
CookieManager cookieManager = new CookieManager();;
CrawlRequestManager <TArachnodeDAO> crawlRequestManager = new CrawlRequestManager <TArachnodeDAO>(applicationSettings, webSettings, cache, consoleManager, discoveryManager);
DataTypeManager <TArachnodeDAO> dataTypeManager = new DataTypeManager <TArachnodeDAO>(applicationSettings, webSettings);
EncodingManager <TArachnodeDAO> encodingManager = new EncodingManager <TArachnodeDAO>(applicationSettings, webSettings);
PolitenessManager <TArachnodeDAO> politenessManager = new PolitenessManager <TArachnodeDAO>(applicationSettings, webSettings, cache);
ProxyManager <TArachnodeDAO> proxyManager = new ProxyManager <TArachnodeDAO>(applicationSettings, webSettings, consoleManager);
HtmlManager <TArachnodeDAO> htmlManager = new HtmlManager <TArachnodeDAO>(applicationSettings, webSettings, discoveryManager);
Crawl <TArachnodeDAO> crawl = new Crawl <TArachnodeDAO>(applicationSettings, webSettings, crawler, actionManager, consoleManager, cookieManager, crawlRequestManager, dataTypeManager, discoveryManager, encodingManager, htmlManager, politenessManager, proxyManager, ruleManager, true);
//create a CrawlRequest as this is what the internals of SiteCrawler.dll expect to operate on...
CrawlRequest <TArachnodeDAO> crawlRequest = new CrawlRequest <TArachnodeDAO>(new Discovery <TArachnodeDAO>(filesRow.AbsoluteUri), 1, UriClassificationType.Host, UriClassificationType.Host, 0, RenderType.None, RenderType.None);
crawlRequest.Crawl = crawl;
crawlRequest.Discovery.DiscoveryType = DiscoveryType.File;
crawlRequest.Discovery.ID = filesRow.ID;
crawlRequest.Data = filesRow.Source;
crawlRequest.ProcessData = true;
crawlRequest.WebClient = webClient;
crawlRequest.WebClient.HttpWebResponse.Headers.Clear();
//parse the ResponseHeaders from the FilesRow.ResponseHeaders string...
foreach (string responseHeader in filesRow.ResponseHeaders.Split("\r\n".ToCharArray(), StringSplitOptions.RemoveEmptyEntries))
{
string[] responseHeaderSplit = responseHeader.Split(":".ToCharArray());
string name = responseHeaderSplit[0];
string value = UserDefinedFunctions.ExtractResponseHeader(filesRow.ResponseHeaders, name, true).Value;
crawlRequest.WebClient.HttpWebResponse.Headers.Add(name, value);
}
//refresh the DataTypes in the DataTypeManager... (if necessary)...
if (dataTypeManager.AllowedDataTypes.Count == 0)
{
dataTypeManager.RefreshDataTypes();
}
crawlRequest.DataType = dataTypeManager.DetermineDataType(crawlRequest);
if (applicationSettings.InsertFiles)
{
crawlRequest.Discovery.ID = arachnodeDAO.InsertFile(crawlRequest.Parent.Uri.AbsoluteUri, crawlRequest.Discovery.Uri.AbsoluteUri, crawlRequest.WebClient.HttpWebResponse.Headers.ToString(), applicationSettings.InsertFileSource ? crawlRequest.Data : new byte[] { }, crawlRequest.DataType.FullTextIndexType, applicationSettings.ClassifyAbsoluteUris);
}
crawlRequest.ManagedDiscovery = fileManager.ManageFile(crawlRequest, crawlRequest.Discovery.ID.Value, crawlRequest.Discovery.Uri.AbsoluteUri, crawlRequest.Data, crawlRequest.DataType.FullTextIndexType, applicationSettings.ExtractFileMetaData, applicationSettings.InsertFileMetaData, applicationSettings.SaveDiscoveredFilesToDisk);
actionManager.PerformCrawlActions(crawlRequest, CrawlActionType.PostRequest, arachnodeDAO);
discoveryManager.CloseAndDisposeManagedDiscovery(crawlRequest, arachnodeDAO);
}
private void Window_Closing(object sender, System.ComponentModel.CancelEventArgs e)
{
//Remove all ItemViews of this window from the manager list.
NotificationBox?.RemoveAllViews();
_encoder = null;
GC.Collect();
//Display the encodings in the editor.
EncodingManager.MoveEncodingsToPopups();
}
protected override void OnStartup(StartupEventArgs e)
{
AppDomain.CurrentDomain.UnhandledException += UnhandledExceptionHandler;
Current.DispatcherUnhandledException += AppUnhandleExceptionHandler;
var ctx = new ProtocolContext();
var protocols = new List <IProtocol>();
protocols.AddRange(ctx.Protocols.Include("ProtocolStructures")
.Include("ProtocolCommands.CommandDatas")
.Include("Firmwares")
.ToList());
EncodingManager.LoadEncoder(protocols);
}
/// <summary>
/// 解码缓存字节为协议包
/// </summary>
private void Decode()
{
var package = ProtocolEncoder.Decode(_processBuffer.ToArray());
AsyncCleanBuffer(package);
if (!package.Finalized)
{
return;
}
package.ClientSource = ClientSource;
package.SetupProtocolData();
OnClientDecoded(package.ProtocolData);
EncodingManager.RunBuinessHandler(package);
}
private void Authentication()
{
var result = EncodingManager.Authentication(_processBuffer.ToArray());
AsyncCleanBuffer(result.Package);
if (result.ResultType != AuthenticationStatus.Authed)
{
AuthStatus = result.ResultType;
OnClientAuthenticaFailed();
return;
}
ClientSource = result.AuthedClientSource;
AuthStatus = AuthenticationStatus.Authed;
OnClientAuthentication();
}
/// <summary>
/// Initializes a new instance of the <see cref = "Crawl" /> class.
/// </summary>
/// <param name = "crawler">The crawler.</param>
/// <param name="actionManager"></param>
/// <param name="crawlRequestManager"></param>
/// <param name="discoveryManager"></param>
/// <param name="htmlManager"></param>
/// <param name="politenessManager"></param>
/// <param name="ruleManager"></param>
/// <param name = "processData">if set to <c>true</c> [process data].</param>
public Crawl(ApplicationSettings applicationSettings, WebSettings webSettings, Crawler <TArachnodeDAO> crawler, ActionManager <TArachnodeDAO> actionManager, ConsoleManager <TArachnodeDAO> consoleManager, CookieManager cookieManager, CrawlRequestManager <TArachnodeDAO> crawlRequestManager, DataTypeManager <TArachnodeDAO> dataTypeManager, DiscoveryManager <TArachnodeDAO> discoveryManager, EncodingManager <TArachnodeDAO> encodingManager, HtmlManager <TArachnodeDAO> htmlManager, PolitenessManager <TArachnodeDAO> politenessManager, ProxyManager <TArachnodeDAO> proxyManager, RuleManager <TArachnodeDAO> ruleManager, bool processData)
{
_applicationSettings = applicationSettings;
_webSettings = webSettings;
UncrawledCrawlRequests = new PriorityQueue <CrawlRequest <TArachnodeDAO> >();
UnassignedDiscoveries = new HashSet <string>();
_crawler = crawler;
_crawlInfo.MaximumCrawlDepth = 1;
_actionManager = actionManager;
_consoleManager = consoleManager;
_cookieManager = cookieManager;
_crawlRequestManager = crawlRequestManager;
_dataTypeManager = dataTypeManager;
_discoveryManager = discoveryManager;
_encodingManager = encodingManager;
_htmlManager = htmlManager;
_politenessManager = politenessManager;
_proxyManager = proxyManager;
_ruleManager = ruleManager;
_processData = processData;
_arachnodeDAO = (TArachnodeDAO)Activator.CreateInstance(typeof(TArachnodeDAO), _applicationSettings.ConnectionString, _applicationSettings, _webSettings, false, false);
_arachnodeDAO.ApplicationSettings = applicationSettings;
//_arachnodeDAO.OpenCommandConnections();
_dataManager = new DataManager <TArachnodeDAO>(_applicationSettings, _webSettings, _actionManager, _dataTypeManager, _discoveryManager, _ruleManager, _arachnodeDAO);
_fileManager = new FileManager <TArachnodeDAO>(_applicationSettings, _webSettings, _discoveryManager, _arachnodeDAO);
_imageManager = new ImageManager <TArachnodeDAO>(_applicationSettings, _webSettings, _discoveryManager, _arachnodeDAO);
_webClient = new WebClient <TArachnodeDAO>(_applicationSettings, _webSettings, _consoleManager, _cookieManager, _proxyManager);
_webPageManager = new WebPageManager <TArachnodeDAO>(_applicationSettings, _webSettings, _discoveryManager, _htmlManager, _arachnodeDAO);
}
public TaskCreationViewModel(EncodingManager encodingManager)
{
_encodingManager = encodingManager;
}
/// <summary>
/// Registers resources that classes will depend on
/// </summary>
/// <returns>Task.</returns>
protected override async Task RegisterResources(IProgress <double> progress)
{
await base.RegisterResources(progress).ConfigureAwait(false);
RegisterSingleInstance <IHttpResultFactory>(new HttpResultFactory(LogManager, FileSystemManager, JsonSerializer));
RegisterSingleInstance <IServerApplicationHost>(this);
RegisterSingleInstance <IServerApplicationPaths>(ApplicationPaths);
RegisterSingleInstance(ServerConfigurationManager);
LocalizationManager = new LocalizationManager(ServerConfigurationManager, FileSystemManager, JsonSerializer);
RegisterSingleInstance(LocalizationManager);
RegisterSingleInstance <IBlurayExaminer>(() => new BdInfoExaminer());
UserDataManager = new UserDataManager(LogManager);
RegisterSingleInstance(UserDataManager);
UserRepository = await GetUserRepository().ConfigureAwait(false);
RegisterSingleInstance(UserRepository);
DisplayPreferencesRepository = new SqliteDisplayPreferencesRepository(ApplicationPaths, JsonSerializer, LogManager);
RegisterSingleInstance(DisplayPreferencesRepository);
ItemRepository = new SqliteItemRepository(ApplicationPaths, JsonSerializer, LogManager);
RegisterSingleInstance(ItemRepository);
ProviderRepository = new SqliteProviderInfoRepository(ApplicationPaths, LogManager);
RegisterSingleInstance(ProviderRepository);
FileOrganizationRepository = await GetFileOrganizationRepository().ConfigureAwait(false);
RegisterSingleInstance(FileOrganizationRepository);
AuthenticationRepository = await GetAuthenticationRepository().ConfigureAwait(false);
RegisterSingleInstance(AuthenticationRepository);
//SyncRepository = await GetSyncRepository().ConfigureAwait(false);
//RegisterSingleInstance(SyncRepository);
UserManager = new UserManager(LogManager.GetLogger("UserManager"), ServerConfigurationManager, UserRepository, XmlSerializer, NetworkManager, () => ImageProcessor, () => DtoService, () => ConnectManager);
RegisterSingleInstance(UserManager);
LibraryManager = new LibraryManager(Logger, TaskManager, UserManager, ServerConfigurationManager, UserDataManager, () => LibraryMonitor, FileSystemManager, () => ProviderManager);
RegisterSingleInstance(LibraryManager);
var musicManager = new MusicManager(LibraryManager);
RegisterSingleInstance <IMusicManager>(new MusicManager(LibraryManager));
LibraryMonitor = new LibraryMonitor(LogManager, TaskManager, LibraryManager, ServerConfigurationManager, FileSystemManager);
RegisterSingleInstance(LibraryMonitor);
ProviderManager = new ProviderManager(HttpClient, ServerConfigurationManager, LibraryMonitor, LogManager, FileSystemManager);
RegisterSingleInstance(ProviderManager);
SeriesOrderManager = new SeriesOrderManager();
RegisterSingleInstance(SeriesOrderManager);
RegisterSingleInstance <ISearchEngine>(() => new SearchEngine(LogManager, LibraryManager, UserManager));
HttpServer = ServerFactory.CreateServer(this, LogManager, "Media Browser", WebApplicationName, "dashboard/index.html", _supportsNativeWebSocket);
RegisterSingleInstance(HttpServer, false);
progress.Report(10);
ServerManager = new ServerManager(this, JsonSerializer, LogManager.GetLogger("ServerManager"), ServerConfigurationManager);
RegisterSingleInstance(ServerManager);
var innerProgress = new ActionableProgress <double>();
innerProgress.RegisterAction(p => progress.Report((.75 * p) + 15));
await RegisterMediaEncoder(innerProgress).ConfigureAwait(false);
progress.Report(90);
ImageProcessor = new ImageProcessor(LogManager.GetLogger("ImageProcessor"), ServerConfigurationManager.ApplicationPaths, FileSystemManager, JsonSerializer, MediaEncoder);
RegisterSingleInstance(ImageProcessor);
SyncManager = new SyncManager(LibraryManager, SyncRepository, ImageProcessor, LogManager.GetLogger("SyncManager"));
RegisterSingleInstance(SyncManager);
DtoService = new DtoService(Logger, LibraryManager, UserDataManager, ItemRepository, ImageProcessor, ServerConfigurationManager, FileSystemManager, ProviderManager, () => ChannelManager, SyncManager);
RegisterSingleInstance(DtoService);
var encryptionManager = new EncryptionManager();
RegisterSingleInstance <IEncryptionManager>(encryptionManager);
ConnectManager = new ConnectManager(LogManager.GetLogger("Connect"), ApplicationPaths, JsonSerializer, encryptionManager, HttpClient, this, ServerConfigurationManager, UserManager, ProviderManager);
RegisterSingleInstance(ConnectManager);
DeviceManager = new DeviceManager(new DeviceRepository(ApplicationPaths, JsonSerializer), UserManager, FileSystemManager, LibraryMonitor, ConfigurationManager, LogManager.GetLogger("DeviceManager"));
RegisterSingleInstance(DeviceManager);
SessionManager = new SessionManager(UserDataManager, ServerConfigurationManager, Logger, UserRepository, LibraryManager, UserManager, musicManager, DtoService, ImageProcessor, ItemRepository, JsonSerializer, this, HttpClient, AuthenticationRepository, DeviceManager);
RegisterSingleInstance(SessionManager);
var newsService = new Server.Implementations.News.NewsService(ApplicationPaths, JsonSerializer);
RegisterSingleInstance <INewsService>(newsService);
var fileOrganizationService = new FileOrganizationService(TaskManager, FileOrganizationRepository, LogManager.GetLogger("FileOrganizationService"), LibraryMonitor, LibraryManager, ServerConfigurationManager, FileSystemManager, ProviderManager);
RegisterSingleInstance <IFileOrganizationService>(fileOrganizationService);
progress.Report(15);
ChannelManager = new ChannelManager(UserManager, DtoService, LibraryManager, Logger, ServerConfigurationManager, FileSystemManager, UserDataManager, JsonSerializer, LocalizationManager, HttpClient);
RegisterSingleInstance(ChannelManager);
TVSeriesManager = new TVSeriesManager(UserManager, UserDataManager, LibraryManager);
RegisterSingleInstance(TVSeriesManager);
var appThemeManager = new AppThemeManager(ApplicationPaths, FileSystemManager, JsonSerializer, Logger);
RegisterSingleInstance <IAppThemeManager>(appThemeManager);
var dlnaManager = new DlnaManager(XmlSerializer, FileSystemManager, ApplicationPaths, LogManager.GetLogger("Dlna"), JsonSerializer);
RegisterSingleInstance <IDlnaManager>(dlnaManager);
var connectionManager = new ConnectionManager(dlnaManager, ServerConfigurationManager, LogManager.GetLogger("UpnpConnectionManager"), HttpClient);
RegisterSingleInstance <IConnectionManager>(connectionManager);
CollectionManager = new CollectionManager(LibraryManager, FileSystemManager, LibraryMonitor, LogManager.GetLogger("CollectionManager"));
RegisterSingleInstance(CollectionManager);
var playlistManager = new PlaylistManager(LibraryManager, FileSystemManager, LibraryMonitor, LogManager.GetLogger("PlaylistManager"), UserManager);
RegisterSingleInstance <IPlaylistManager>(playlistManager);
LiveTvManager = new LiveTvManager(ServerConfigurationManager, FileSystemManager, Logger, ItemRepository, ImageProcessor, UserDataManager, DtoService, UserManager, LibraryManager, TaskManager, LocalizationManager, JsonSerializer);
RegisterSingleInstance(LiveTvManager);
UserViewManager = new UserViewManager(LibraryManager, LocalizationManager, FileSystemManager, UserManager, ChannelManager, LiveTvManager, ApplicationPaths, playlistManager);
RegisterSingleInstance(UserViewManager);
var contentDirectory = new ContentDirectory(dlnaManager, UserDataManager, ImageProcessor, LibraryManager, ServerConfigurationManager, UserManager, LogManager.GetLogger("UpnpContentDirectory"), HttpClient, LocalizationManager);
RegisterSingleInstance <IContentDirectory>(contentDirectory);
NotificationManager = new NotificationManager(LogManager, UserManager, ServerConfigurationManager);
RegisterSingleInstance(NotificationManager);
SubtitleManager = new SubtitleManager(LogManager.GetLogger("SubtitleManager"), FileSystemManager, LibraryMonitor, LibraryManager, ItemRepository);
RegisterSingleInstance(SubtitleManager);
ChapterManager = new ChapterManager(LibraryManager, LogManager.GetLogger("ChapterManager"), ServerConfigurationManager, ItemRepository);
RegisterSingleInstance(ChapterManager);
EncodingManager = new EncodingManager(FileSystemManager, Logger,
MediaEncoder, ChapterManager);
RegisterSingleInstance(EncodingManager);
var activityLogRepo = await GetActivityLogRepository().ConfigureAwait(false);
RegisterSingleInstance(activityLogRepo);
RegisterSingleInstance <IActivityManager>(new ActivityManager(LogManager.GetLogger("ActivityManager"), activityLogRepo, UserManager));
var authContext = new AuthorizationContext();
RegisterSingleInstance <IAuthorizationContext>(authContext);
RegisterSingleInstance <ISessionContext>(new SessionContext(UserManager, authContext, SessionManager));
RegisterSingleInstance <IAuthService>(new AuthService(UserManager, SessionManager, authContext, ServerConfigurationManager));
RegisterSingleInstance <ISubtitleEncoder>(new SubtitleEncoder(LibraryManager, LogManager.GetLogger("SubtitleEncoder"), ApplicationPaths, FileSystemManager, MediaEncoder, JsonSerializer));
var displayPreferencesTask = Task.Run(async() => await ConfigureDisplayPreferencesRepositories().ConfigureAwait(false));
var itemsTask = Task.Run(async() => await ConfigureItemRepositories().ConfigureAwait(false));
var userdataTask = Task.Run(async() => await ConfigureUserDataRepositories().ConfigureAwait(false));
await ConfigureNotificationsRepository().ConfigureAwait(false);
progress.Report(92);
await Task.WhenAll(itemsTask, displayPreferencesTask, userdataTask).ConfigureAwait(false);
progress.Report(100);
SetStaticProperties();
await((UserManager)UserManager).Initialize().ConfigureAwait(false);
SetKernelProperties();
}
private void ClearAll_Executed(object sender, ExecutedRoutedEventArgs e)
{
EncodingManager.RemoveFinishedEncodings();
}
/// <summary>
/// Registers resources that classes will depend on
/// </summary>
/// <returns>Task.</returns>
protected override async Task RegisterResources(IProgress <double> progress)
{
await base.RegisterResources(progress).ConfigureAwait(false);
RegisterSingleInstance <IHttpResultFactory>(new HttpResultFactory(LogManager, FileSystemManager, JsonSerializer));
RegisterSingleInstance <IServerApplicationHost>(this);
RegisterSingleInstance <IServerApplicationPaths>(ApplicationPaths);
RegisterSingleInstance(ServerConfigurationManager);
RegisterSingleInstance <IWebSocketServer>(() => new AlchemyServer(Logger));
RegisterSingleInstance <IBlurayExaminer>(() => new BdInfoExaminer());
UserDataManager = new UserDataManager(LogManager);
RegisterSingleInstance(UserDataManager);
UserRepository = await GetUserRepository().ConfigureAwait(false);
RegisterSingleInstance(UserRepository);
DisplayPreferencesRepository = new SqliteDisplayPreferencesRepository(ApplicationPaths, JsonSerializer, LogManager);
RegisterSingleInstance(DisplayPreferencesRepository);
ItemRepository = new SqliteItemRepository(ApplicationPaths, JsonSerializer, LogManager);
RegisterSingleInstance(ItemRepository);
ProviderRepository = new SqliteProviderInfoRepository(ApplicationPaths, LogManager);
RegisterSingleInstance(ProviderRepository);
FileOrganizationRepository = await GetFileOrganizationRepository().ConfigureAwait(false);
RegisterSingleInstance(FileOrganizationRepository);
UserManager = new UserManager(Logger, ServerConfigurationManager, UserRepository);
RegisterSingleInstance(UserManager);
LibraryManager = new LibraryManager(Logger, TaskManager, UserManager, ServerConfigurationManager, UserDataManager, () => LibraryMonitor, FileSystemManager, () => ProviderManager);
RegisterSingleInstance(LibraryManager);
LibraryMonitor = new LibraryMonitor(LogManager, TaskManager, LibraryManager, ServerConfigurationManager, FileSystemManager);
RegisterSingleInstance(LibraryMonitor);
ProviderManager = new ProviderManager(HttpClient, ServerConfigurationManager, LibraryMonitor, LogManager, FileSystemManager);
RegisterSingleInstance(ProviderManager);
RegisterSingleInstance <ISearchEngine>(() => new SearchEngine(LogManager, LibraryManager, UserManager));
SessionManager = new SessionManager(UserDataManager, ServerConfigurationManager, Logger, UserRepository, LibraryManager, UserManager);
RegisterSingleInstance(SessionManager);
HttpServer = ServerFactory.CreateServer(this, LogManager, "Media Browser", "mediabrowser", "dashboard/index.html");
RegisterSingleInstance(HttpServer, false);
progress.Report(10);
ServerManager = new ServerManager(this, JsonSerializer, Logger, ServerConfigurationManager);
RegisterSingleInstance(ServerManager);
LocalizationManager = new LocalizationManager(ServerConfigurationManager, FileSystemManager);
RegisterSingleInstance(LocalizationManager);
ImageProcessor = new ImageProcessor(Logger, ServerConfigurationManager.ApplicationPaths, FileSystemManager, JsonSerializer);
RegisterSingleInstance(ImageProcessor);
DtoService = new DtoService(Logger, LibraryManager, UserManager, UserDataManager, ItemRepository, ImageProcessor, ServerConfigurationManager, FileSystemManager, ProviderManager);
RegisterSingleInstance(DtoService);
var newsService = new Server.Implementations.News.NewsService(ApplicationPaths, JsonSerializer);
RegisterSingleInstance <INewsService>(newsService);
var fileOrganizationService = new FileOrganizationService(TaskManager, FileOrganizationRepository, Logger, LibraryMonitor, LibraryManager, ServerConfigurationManager, FileSystemManager, ProviderManager);
RegisterSingleInstance <IFileOrganizationService>(fileOrganizationService);
progress.Report(15);
var innerProgress = new ActionableProgress <double>();
innerProgress.RegisterAction(p => progress.Report((.75 * p) + 15));
await RegisterMediaEncoder(innerProgress).ConfigureAwait(false);
progress.Report(90);
EncodingManager = new EncodingManager(ServerConfigurationManager, FileSystemManager, Logger, ItemRepository,
MediaEncoder);
RegisterSingleInstance(EncodingManager);
var appThemeManager = new AppThemeManager(ApplicationPaths, FileSystemManager, JsonSerializer, Logger);
RegisterSingleInstance <IAppThemeManager>(appThemeManager);
LiveTvManager = new LiveTvManager(ServerConfigurationManager, FileSystemManager, Logger, ItemRepository, ImageProcessor, UserDataManager, DtoService, UserManager, LibraryManager, TaskManager);
RegisterSingleInstance(LiveTvManager);
var displayPreferencesTask = Task.Run(async() => await ConfigureDisplayPreferencesRepositories().ConfigureAwait(false));
var itemsTask = Task.Run(async() => await ConfigureItemRepositories().ConfigureAwait(false));
var userdataTask = Task.Run(async() => await ConfigureUserDataRepositories().ConfigureAwait(false));
await ConfigureNotificationsRepository().ConfigureAwait(false);
progress.Report(92);
await Task.WhenAll(itemsTask, displayPreferencesTask, userdataTask).ConfigureAwait(false);
progress.Report(100);
SetStaticProperties();
await((UserManager)UserManager).Initialize().ConfigureAwait(false);
SetKernelProperties();
}
/// <summary>
/// Initializes a new instance of the <see cref = "Crawler" /> class.
/// </summary>
public Crawler(ApplicationSettings applicationSettings, WebSettings webSettings, CrawlMode crawlMode, List <CrawlerPeer> crawlerPeers, List <DatabasePeer> databasePeers, bool enableRenderers)
{
Guid = Guid.NewGuid();
try
{
_applicationSettings = applicationSettings;
_webSettings = webSettings;
_arachnodeDAO = (TArachnodeDAO)Activator.CreateInstance(typeof(TArachnodeDAO), _applicationSettings.ConnectionString, _applicationSettings, _webSettings, true, true);
_applicationSettings = _arachnodeDAO.ApplicationSettings;
_consoleManager = new ConsoleManager <TArachnodeDAO>(_applicationSettings, _webSettings);
_consoleManager.OutputString("arachnode.net " + Assembly.GetExecutingAssembly().GetName().Version, ConsoleColor.Green, ConsoleColor.Gray);
_actionManager = new ActionManager <TArachnodeDAO>(_applicationSettings, _webSettings, _consoleManager);
_ruleManager = new RuleManager <TArachnodeDAO>(_applicationSettings, _webSettings, _consoleManager);
_memoryManager = new MemoryManager <TArachnodeDAO>(_applicationSettings, _webSettings);
_cacheManager = new CacheManager <TArachnodeDAO>(_applicationSettings, _webSettings);
_cookieManager = new CookieManager();
_cacheManager = new CacheManager <TArachnodeDAO>(_applicationSettings, _webSettings);
CrawlerPeers = crawlerPeers;
DatabasePeers = databasePeers;
_crawlerPeerManager = new CrawlerPeerManager <TArachnodeDAO>(_applicationSettings, _webSettings, CrawlerPeers, (TArachnodeDAO)Activator.CreateInstance(typeof(TArachnodeDAO), _applicationSettings.ConnectionString, _applicationSettings, _webSettings, true, true));
_databasePeerManager = new DatabasePeerManager <TArachnodeDAO>(_applicationSettings, _webSettings, DatabasePeers);
_cache = new Cache <TArachnodeDAO>(_applicationSettings, _webSettings, this, _actionManager, _cacheManager, _crawlerPeerManager, _memoryManager, _ruleManager);
_dataTypeManager = new DataTypeManager <TArachnodeDAO>(_applicationSettings, _webSettings);
_discoveryManager = new DiscoveryManager <TArachnodeDAO>(_applicationSettings, _webSettings, _cache, _actionManager, _cacheManager, _memoryManager, _ruleManager);
_crawlRequestManager = new CrawlRequestManager <TArachnodeDAO>(_applicationSettings, _webSettings, _cache, _consoleManager, _discoveryManager);
_encodingManager = new EncodingManager <TArachnodeDAO>(_applicationSettings, _webSettings);
_htmlManager = new HtmlManager <TArachnodeDAO>(_applicationSettings, _webSettings, _discoveryManager);
_politenessManager = new PolitenessManager <TArachnodeDAO>(_applicationSettings, _webSettings, _cache);
_proxyManager = new ProxyManager <TArachnodeDAO>(_applicationSettings, _webSettings, _consoleManager);
_reportingManager = new ReportingManager <TArachnodeDAO>(_applicationSettings, _webSettings, _consoleManager);
//create required directories...
if (!Directory.Exists(_applicationSettings.ConsoleOutputLogsDirectory))
{
Directory.CreateDirectory(_applicationSettings.ConsoleOutputLogsDirectory);
}
if (!Directory.Exists(_applicationSettings.DownloadedFilesDirectory))
{
Directory.CreateDirectory(_applicationSettings.DownloadedFilesDirectory);
}
if (!Directory.Exists(_applicationSettings.DownloadedImagesDirectory))
{
Directory.CreateDirectory(_applicationSettings.DownloadedImagesDirectory);
}
if (!Directory.Exists(_applicationSettings.DownloadedWebPagesDirectory))
{
Directory.CreateDirectory(_applicationSettings.DownloadedWebPagesDirectory);
}
QueryProcessor = new QueryProcessor <TArachnodeDAO>();
_consoleManager.OutputString("Crawler: Initializing Configuration/Database Connection.", ConsoleColor.White, ConsoleColor.Gray);
LoadCrawlActions(_arachnodeDAO);
LoadCrawlRules(_arachnodeDAO);
AreRenderersEnabled = enableRenderers;
Engine = new Engine <TArachnodeDAO>(_applicationSettings, _webSettings, this, _cache, _actionManager, _cacheManager, _consoleManager, _cookieManager, _crawlRequestManager, _dataTypeManager, _discoveryManager, _encodingManager, _htmlManager, _memoryManager, _politenessManager, _proxyManager, _reportingManager, _ruleManager, enableRenderers, (TArachnodeDAO)Activator.CreateInstance(typeof(TArachnodeDAO), _applicationSettings.ConnectionString, _applicationSettings, _webSettings, true, true));
CrawlMode = crawlMode;
/**/
if (CrawlerPeerManager != null && CrawlerPeerManager.CrawlerPeers != null && CrawlerPeerManager.CrawlerPeers.Count != 0)
{
ConsoleManager.OutputString("Crawler: Starting CrawlerPeerManager Server", ConsoleColor.White, ConsoleColor.Gray);
CrawlerPeerManager.StartServer(this, _arachnodeDAO);
_crawlerPeerManager.SendStatusMessageToCrawlerPeers(_arachnodeDAO);
}
/**/
if (Debugger.IsAttached)
{
_consoleManager.OutputString("Debugger: Attached - Expect Performance Degradation.", ConsoleColor.Yellow, ConsoleColor.Gray);
}
//update all core/components/managers with the updated ApplicationSettings...
#if DEMO
Engine.CrawlRequestCompleted += Engine_CrawlRequestCompleted;
_stopwatch.Start();
#endif
}
catch (InvalidConfigurationException invalidConfigurationException)
{
ProcessException(invalidConfigurationException);
throw new InvalidConfigurationException(invalidConfigurationException.ApplicationSettings, invalidConfigurationException.WebSettings, invalidConfigurationException.Message, InvalidConfigurationExceptionSeverity.Error);
}
catch (Exception exception)
{
ProcessException(exception);
throw new Exception(exception.Message, exception);
}
}
/// <summary>
/// Processes a WebPagesRow after crawling.
/// </summary>
/// <param name = "webPagesRow">The web pages row.</param>
/// <param name="webClient"></param>
/// <param name="actionManager"></param>
/// <param name="consoleManager"></param>
/// <param name="discoveryManager"></param>
/// <param name="memoryManager"></param>
/// <param name="ruleManager"></param>
/// <param name = "webPageManager">The web page manager.</param>
/// <param name = "arachnodeDAO">The arachnode DAO.</param>
/// <param name = "fileManager">The file manager.</param>
/// <param name = "imageManager">The image manager.</param>
public static void ProcessWebPage(ApplicationSettings applicationSettings, WebSettings webSettings, Crawler <TArachnodeDAO> crawler, ArachnodeDataSet.WebPagesRow webPagesRow, WebClient <TArachnodeDAO> webClient, Cache <TArachnodeDAO> cache, ActionManager <TArachnodeDAO> actionManager, ConsoleManager <TArachnodeDAO> consoleManager, CrawlerPeerManager <TArachnodeDAO> crawlerPeerManager, DiscoveryManager <TArachnodeDAO> discoveryManager, MemoryManager <TArachnodeDAO> memoryManager, RuleManager <TArachnodeDAO> ruleManager, WebPageManager <TArachnodeDAO> webPageManager, IArachnodeDAO arachnodeDAO)
{
CacheManager <TArachnodeDAO> cacheManager = new CacheManager <TArachnodeDAO>(applicationSettings, webSettings);
CookieManager cookieManager = new CookieManager();
CrawlRequestManager <TArachnodeDAO> crawlRequestManager = new CrawlRequestManager <TArachnodeDAO>(applicationSettings, webSettings, cache, consoleManager, discoveryManager);
DataTypeManager <TArachnodeDAO> dataTypeManager = new DataTypeManager <TArachnodeDAO>(applicationSettings, webSettings);
EncodingManager <TArachnodeDAO> encodingManager = new EncodingManager <TArachnodeDAO>(applicationSettings, webSettings);
PolitenessManager <TArachnodeDAO> politenessManager = new PolitenessManager <TArachnodeDAO>(applicationSettings, webSettings, cache);
ProxyManager <TArachnodeDAO> proxyManager = new ProxyManager <TArachnodeDAO>(applicationSettings, webSettings, consoleManager);
HtmlManager <TArachnodeDAO> htmlManager = new HtmlManager <TArachnodeDAO>(applicationSettings, webSettings, discoveryManager);
Crawl <TArachnodeDAO> crawl = new Crawl <TArachnodeDAO>(applicationSettings, webSettings, crawler, actionManager, consoleManager, cookieManager, crawlRequestManager, dataTypeManager, discoveryManager, encodingManager, htmlManager, politenessManager, proxyManager, ruleManager, true);
//create a CrawlRequest as this is what the internals of SiteCrawler.dll expect to operate on...
CrawlRequest <TArachnodeDAO> crawlRequest = new CrawlRequest <TArachnodeDAO>(new Discovery <TArachnodeDAO>(webPagesRow.AbsoluteUri), webPagesRow.CrawlDepth, UriClassificationType.Host, UriClassificationType.Host, 0, RenderType.None, RenderType.None);
crawlRequest.Crawl = crawl;
crawlRequest.Discovery.DiscoveryType = DiscoveryType.WebPage;
crawlRequest.Discovery.ID = webPagesRow.ID;
crawlRequest.Data = webPagesRow.Source;
crawlRequest.CurrentDepth = webPagesRow.CrawlDepth;
crawlRequest.Encoding = Encoding.GetEncoding(webPagesRow.CodePage);
crawlRequest.ProcessData = true;
crawlRequest.WebClient = webClient;
crawlRequest.WebClient.HttpWebResponse.Headers.Clear();
//parse the ResponseHeaders from the WebPagesRow.ResponseHeaders string...
foreach (string responseHeader in webPagesRow.ResponseHeaders.Split("\r\n".ToCharArray(), StringSplitOptions.RemoveEmptyEntries))
{
string[] responseHeaderSplit = responseHeader.Split(":".ToCharArray());
string name = responseHeaderSplit[0];
string value = UserDefinedFunctions.ExtractResponseHeader(webPagesRow.ResponseHeaders, name, true).Value;
crawlRequest.WebClient.HttpWebResponse.Headers.Add(name, value);
}
//refresh the DataTypes in the DataTypeManager... (if necessary)...
if (dataTypeManager.AllowedDataTypes.Count == 0)
{
dataTypeManager.RefreshDataTypes();
}
crawlRequest.DataType = dataTypeManager.DetermineDataType(crawlRequest);
//now, process the bytes...
encodingManager.ProcessCrawlRequest(crawlRequest, arachnodeDAO);
if (applicationSettings.InsertWebPages)
{
crawlRequest.Discovery.ID = arachnodeDAO.InsertWebPage(crawlRequest.Discovery.Uri.AbsoluteUri, crawlRequest.WebClient.HttpWebResponse.Headers.ToString(), applicationSettings.InsertWebPageSource ? crawlRequest.Data : new byte[] { }, crawlRequest.Encoding.CodePage, crawlRequest.DataType.FullTextIndexType, crawlRequest.CurrentDepth, applicationSettings.ClassifyAbsoluteUris);
}
crawlRequest.ManagedDiscovery = webPageManager.ManageWebPage(crawlRequest.Discovery.ID.Value, crawlRequest.Discovery.Uri.AbsoluteUri, crawlRequest.Data, crawlRequest.Encoding, crawlRequest.DataType.FullTextIndexType, applicationSettings.ExtractWebPageMetaData, applicationSettings.InsertWebPageMetaData, applicationSettings.SaveDiscoveredWebPagesToDisk);
//assigning FileAndImageDiscoveries isn't applicable because Files and Images need to be crawled to be properly classified... without classification we don't know whether they belong in dbo.Files or dbo.Images...
crawlRequestManager.ProcessEmailAddresses(crawlRequest, arachnodeDAO);
crawlRequestManager.ProcessHyperLinks(crawlRequest, arachnodeDAO);
actionManager.PerformCrawlActions(crawlRequest, CrawlActionType.PostRequest, arachnodeDAO);
discoveryManager.CloseAndDisposeManagedDiscovery(crawlRequest, arachnodeDAO);
}
