public void TestInit()
{
this.mockRequestData = new Mock <IRequestData>();
this.mockMetricsCollection = new Mock <IMetricsCollection>();
this.mockRequestData.Setup(rd => rd.MetricsCollection).Returns(this.mockMetricsCollection.Object);
this.discoveryManager = new DiscoveryManager(this.mockRequestData.Object);
}
void Start()
{
audioSource = GetComponent <AudioSource>();
questActive = questManager.GetComponent <QuestManager>();
chestBehavior = chestObjective.GetComponent <ChestBehavior>();
discovery = DiscoveryObjective.GetComponent <DiscoveryManager>();
}
private void Main_Load(object sender, EventArgs e)
{
try
{
_arachnodeDAO = new ArachnodeDAO(_applicationSettings.ConnectionString, _applicationSettings, _webSettings, true, true);
_actionManager = new ActionManager <ArachnodeDAO>(_applicationSettings, _webSettings, _consoleManager);
_consoleManager = new ConsoleManager <ArachnodeDAO>(_applicationSettings, _webSettings);
_memoryManager = new MemoryManager <ArachnodeDAO>(_applicationSettings, _webSettings);
_cacheManager = new CacheManager <ArachnodeDAO>(_applicationSettings, _webSettings);
_crawlerPeerManager = new CrawlerPeerManager <ArachnodeDAO>(_applicationSettings, _webSettings, null, _arachnodeDAO);
_cache = new Cache <ArachnodeDAO>(_applicationSettings, _webSettings, null, _actionManager, _cacheManager, _crawlerPeerManager, _memoryManager, _ruleManager);
_ruleManager = new RuleManager <ArachnodeDAO>(_applicationSettings, _webSettings, _consoleManager);
_discoveryManager = new DiscoveryManager <ArachnodeDAO>(_applicationSettings, _webSettings, _cache, _actionManager, _cacheManager, _memoryManager, _ruleManager);
nudWebPageID_ValueChanged(null, null);
nudFileID_ValueChanged(null, null);
nudImageID_ValueChanged(null, null);
}
catch (Exception exception)
{
MessageBox.Show(exception.Message + " ::" + exception.StackTrace, "Browser");
}
}
protected ADataManager(ApplicationSettings applicationSettings, WebSettings webSettings, ActionManager <TArachnodeDAO> actionManager, DataTypeManager <TArachnodeDAO> dataTypeManager, DiscoveryManager <TArachnodeDAO> discoveryManager, RuleManager <TArachnodeDAO> ruleManager, IArachnodeDAO arachnodeDAO) : base(applicationSettings, webSettings)
{
_dataTypeManager = dataTypeManager;
_discoveryManager = discoveryManager;
_ruleManager = ruleManager;
_actionManager = actionManager;
_arachnodeDAO = arachnodeDAO;
}
public void OnDeviceAdded(DiscoveryManager manager, ConnectableDevice device)
{
App.MainDispatcher.RunAsync(CoreDispatcherPriority.High, () =>
{
if (App.ApplicationModel.DiscoverredDevices.All(x => x.Id != device.Id))
{
App.ApplicationModel.AddDevice(device);
}
});
}
public MainWindow()
{
DiscoveryManager = new DiscoveryManager(OnAgentAdded);
UdpManager = new UdpManager();
_viewModel = new MainWindowViewModel(DiscoveryManager);
DataContext = _viewModel;
InitializeComponent();
}
public bool UnSubscribeOutputDevice(OutputSubscriptionRequest subReq)
{
var deviceInfo = DiscoveryManager.FindDeviceInfo(subReq.DeviceDescriptor.DeviceHandle);
if (deviceInfo == null)
{
return(false);
}
return(DescriptorManager.StopOutputDevice(deviceInfo));
}
private static void Start()
{
var privateKeyProvider = new PrivateKeyProvider(PrivateKey);
var config = new DiscoveryConfigurationProvider(new NetworkHelper(Logger));
var signer = new Signer();
var cryptoRandom = new CryptoRandom();
var configProvider = new ConfigurationProvider();
var nodeFactory = new NodeFactory();
var calculator = new NodeDistanceCalculator(config);
var nodeTable = new NodeTable(config, nodeFactory, new FileKeyStore(configProvider, new JsonSerializer(Logger), new AesEncrypter(configProvider, Logger), cryptoRandom, Logger), Logger, calculator);
var evictionManager = new EvictionManager(nodeTable, Logger);
var lifecycleFactory = new NodeLifecycleManagerFactory(nodeFactory, nodeTable, Logger, config, new DiscoveryMessageFactory(config), evictionManager);
var discoveryManager = new DiscoveryManager(Logger, config, lifecycleFactory, nodeFactory, nodeTable);
var nodesLocator = new NodesLocator(nodeTable, discoveryManager, config, Logger);
var discoveryMesageFactory = new DiscoveryMessageFactory(config);
var nodeIdResolver = new NodeIdResolver(signer);
var pingSerializer = new PingMessageSerializer(signer, privateKeyProvider, discoveryMesageFactory, nodeIdResolver, nodeFactory);
var pongSerializer = new PongMessageSerializer(signer, privateKeyProvider, discoveryMesageFactory, nodeIdResolver, nodeFactory);
var findNodeSerializer = new FindNodeMessageSerializer(signer, privateKeyProvider, discoveryMesageFactory, nodeIdResolver, nodeFactory);
var neighborsSerializer = new NeighborsMessageSerializer(signer, privateKeyProvider, discoveryMesageFactory, nodeIdResolver, nodeFactory);
var messageSerializationService = new MessageSerializationService();
messageSerializationService.Register(pingSerializer);
messageSerializationService.Register(pongSerializer);
messageSerializationService.Register(findNodeSerializer);
messageSerializationService.Register(neighborsSerializer);
//P2P initialization
IMessagePad eip8Pad = new Eip8MessagePad(cryptoRandom);
messageSerializationService.Register(new AuthEip8MessageSerializer(eip8Pad));
messageSerializationService.Register(new AckEip8MessageSerializer(eip8Pad));
messageSerializationService.Register(new HelloMessageSerializer());
messageSerializationService.Register(new DisconnectMessageSerializer());
messageSerializationService.Register(new Nethermind.Network.P2P.PingMessageSerializer());
messageSerializationService.Register(new Nethermind.Network.P2P.PongMessageSerializer());
messageSerializationService.Register(new StatusMessageSerializer());
IEciesCipher eciesCipher = new EciesCipher(cryptoRandom);
IEncryptionHandshakeService encryptionHandshakeService = new EncryptionHandshakeService(messageSerializationService, eciesCipher, cryptoRandom, signer, PrivateKey, Logger);
var p2pManager = new P2PManager(encryptionHandshakeService, Logger, messageSerializationService, Substitute.For <ISynchronizationManager>());
//Connect discovery with P2P
discoveryManager.RegisterDiscoveryListener(p2pManager);
_discoveryApp = new DiscoveryApp(config, nodesLocator, Logger, discoveryManager, nodeFactory, nodeTable, messageSerializationService, cryptoRandom);
_discoveryApp.Start(PrivateKey.PublicKey);
}
protected void Page_Load(object sender, EventArgs e)
{
EnableViewState = false;
try
{
if (Request.QueryString.Count == 2 && Request.QueryString.AllKeys[0] == "discoveryID" && Request.QueryString.AllKeys[1] == "absoluteUri")
{
ArachnodeDataSet.WebPagesRow webPagesRow = ArachnodeDAO.GetWebPage(Request.QueryString["discoveryID"]);
if (webPagesRow != null)
{
string source = null;
if (webPagesRow.Source == null || webPagesRow.Source.Length == 0)
{
string discoveryPath = DiscoveryManager.GetDiscoveryPath(ApplicationSettings.DownloadedWebPagesDirectory, webPagesRow.AbsoluteUri, webPagesRow.FullTextIndexType);
if (File.Exists(discoveryPath))
{
source = File.ReadAllText(discoveryPath, Encoding.GetEncoding(webPagesRow.CodePage));
}
else
{
uxLblException.Text = "The WebPage source was not found in the database or on disk.";
uxLblException.Visible = true;
return;
}
}
else
{
source = Encoding.GetEncoding(webPagesRow.CodePage).GetString(webPagesRow.Source);
}
//Request.Url.Scheme + "://" + Request.Url.Authority
//ANODET: Should this be a configuration setting? Perhaps - hotlinking isn't exactly polite, but does provide the best user experience. (Version 1.5)
uxLWebPage.Text = HtmlManager.CreateHtmlDocument(webPagesRow.AbsoluteUri, webPagesRow.FullTextIndexType, source, UriQualificationType.AbsoluteWhenDownloadedDiscoveryIsUnavailable, ArachnodeDAO, true).DocumentNode.OuterHtml;
}
else
{
uxLblException.Text = "The WebPage was not found in the database.";
uxLblException.Visible = true;
}
}
}
catch (Exception exception)
{
uxLblException.Text = exception.Message;
uxLblException.Visible = true;
ArachnodeDAO.InsertException(null, null, exception, false);
}
}
public void OnDeviceUpdated(DiscoveryManager manager, ConnectableDevice device)
{
App.MainDispatcher.RunAsync(CoreDispatcherPriority.High, () =>
{
if (App.ApplicationModel.DiscoverredDevices.Contains(device))
{
App.ApplicationModel.DiscoverredDevices.Remove(device);
App.ApplicationModel.DiscoverredDevices.Add(device);
device.OnPropertyChanged("ServiceNames");
}
});
}
public static void FindWardNodes()
{
var disco = new DiscoveryManager();
disco.DiscoveryAddressAdded += (sender, e) =>
{
if (WardNodeFound != null)
{
WardNodeFound(disco, new WebConfiguration(e.ServiceInfo.Address));
}
};
disco.Find(DiscoveryType.Zeroconf);
}
// Start is called before the first frame update
IEnumerator Start()
{
DiscoveryManager _discoveryManager = Director.GetManager <DiscoveryManager>();
while (_discoveryManager == null)
{
yield return(new WaitForEndOfFrame());
_discoveryManager = Director.GetManager <DiscoveryManager>();
}
selectedSecretText.enabled = false;
assignedDiscoveryText.text = "No secret selected";
_discoveryManager.OnNewDiscovery += NewDiscovery;
bookController.OnSecretClicked += SecretClicked;
}
private void SearchTvs()
{
listener = new DiscoveryManagerListener();
listener.Paired += (sender, o) =>
{
model.Connected = true;
model.SelectedDevice = new Independent <ConnectableDevice>(o as ConnectableDevice);
Dispatcher.RunAsync(CoreDispatcherPriority.High, () => Frame.Navigate(typeof(Sampler)));
};
DiscoveryManager.Init();
var discoveryManager = DiscoveryManager.GetInstance();
discoveryManager.AddListener(listener);
discoveryManager.PairingLevel = DiscoveryManager.PairingLevelEnum.On;
discoveryManager.Start();
}
public Main()
{
InitializeComponent();
_formText = Text;
_actionManager = new ActionManager <ArachnodeDAO>(_applicationSettings, _webSettings, _consoleManager);
_consoleManager = new ConsoleManager <ArachnodeDAO>(_applicationSettings, _webSettings);
_memoryManager = new MemoryManager <ArachnodeDAO>(_applicationSettings, _webSettings);
_cacheManager = new CacheManager <ArachnodeDAO>(_applicationSettings, _webSettings);
_crawlerPeerManager = new CrawlerPeerManager <ArachnodeDAO>(_applicationSettings, _webSettings, null, null);
_cache = new Cache <ArachnodeDAO>(_applicationSettings, _webSettings, null, _actionManager, _cacheManager, _crawlerPeerManager, _memoryManager, _ruleManager);
_ruleManager = new RuleManager <ArachnodeDAO>(_applicationSettings, _webSettings, _consoleManager);
_discoveryManager = new DiscoveryManager <ArachnodeDAO>(_applicationSettings, _webSettings, _cache, _actionManager, _cacheManager, _memoryManager, _ruleManager);
tbAbsoluteUri_KeyUp(this, new KeyEventArgs(Keys.Enter));
}
public void Setup()
{
NetworkConfig networkConfig = new NetworkConfig();
networkConfig.ExternalIp = IPAddress.Broadcast.ToString();
_masterNode = new Node(TestItem.PublicKeyA, IPAddress.Broadcast.ToString(), 30000, false);
DiscoveryConfig config = new DiscoveryConfig()
{
DiscoveryNewCycleWaitTime = 1
};
NodeDistanceCalculator distanceCalculator = new NodeDistanceCalculator(config);
_nodeTable = new NodeTable(
distanceCalculator,
config,
networkConfig,
LimboLogs.Instance);
DiscoveryMessageFactory messageFactory = new DiscoveryMessageFactory(Timestamper.Default);
EvictionManager evictionManager = new EvictionManager(_nodeTable, LimboLogs.Instance);
ITimerFactory timerFactory = Substitute.For <ITimerFactory>();
NodeStatsManager nodeStatsManager = new NodeStatsManager(timerFactory, LimboLogs.Instance);
NodeLifecycleManagerFactory managerFactory =
new NodeLifecycleManagerFactory(
_nodeTable,
messageFactory,
evictionManager,
nodeStatsManager,
config,
LimboLogs.Instance);
DiscoveryManager manager = new DiscoveryManager(
managerFactory,
_nodeTable,
new NetworkStorage(new MemDb(), LimboLogs.Instance),
config,
LimboLogs.Instance,
new IPResolver(networkConfig, LimboLogs.Instance));
_nodesLocator = new NodesLocator(_nodeTable, manager, config, LimboLogs.Instance);
}
/// <summary>
/// Initializes a new instance of the <see cref = "Crawl" /> class.
/// </summary>
/// <param name = "crawler">The crawler.</param>
/// <param name="actionManager"></param>
/// <param name="crawlRequestManager"></param>
/// <param name="discoveryManager"></param>
/// <param name="htmlManager"></param>
/// <param name="politenessManager"></param>
/// <param name="ruleManager"></param>
/// <param name = "processData">if set to <c>true</c> [process data].</param>
public Crawl(ApplicationSettings applicationSettings, WebSettings webSettings, Crawler <TArachnodeDAO> crawler, ActionManager <TArachnodeDAO> actionManager, ConsoleManager <TArachnodeDAO> consoleManager, CookieManager cookieManager, CrawlRequestManager <TArachnodeDAO> crawlRequestManager, DataTypeManager <TArachnodeDAO> dataTypeManager, DiscoveryManager <TArachnodeDAO> discoveryManager, EncodingManager <TArachnodeDAO> encodingManager, HtmlManager <TArachnodeDAO> htmlManager, PolitenessManager <TArachnodeDAO> politenessManager, ProxyManager <TArachnodeDAO> proxyManager, RuleManager <TArachnodeDAO> ruleManager, bool processData)
{
_applicationSettings = applicationSettings;
_webSettings = webSettings;
UncrawledCrawlRequests = new PriorityQueue <CrawlRequest <TArachnodeDAO> >();
UnassignedDiscoveries = new HashSet <string>();
_crawler = crawler;
_crawlInfo.MaximumCrawlDepth = 1;
_actionManager = actionManager;
_consoleManager = consoleManager;
_cookieManager = cookieManager;
_crawlRequestManager = crawlRequestManager;
_dataTypeManager = dataTypeManager;
_discoveryManager = discoveryManager;
_encodingManager = encodingManager;
_htmlManager = htmlManager;
_politenessManager = politenessManager;
_proxyManager = proxyManager;
_ruleManager = ruleManager;
_processData = processData;
_arachnodeDAO = (TArachnodeDAO)Activator.CreateInstance(typeof(TArachnodeDAO), _applicationSettings.ConnectionString, _applicationSettings, _webSettings, false, false);
_arachnodeDAO.ApplicationSettings = applicationSettings;
//_arachnodeDAO.OpenCommandConnections();
_dataManager = new DataManager <TArachnodeDAO>(_applicationSettings, _webSettings, _actionManager, _dataTypeManager, _discoveryManager, _ruleManager, _arachnodeDAO);
_fileManager = new FileManager <TArachnodeDAO>(_applicationSettings, _webSettings, _discoveryManager, _arachnodeDAO);
_imageManager = new ImageManager <TArachnodeDAO>(_applicationSettings, _webSettings, _discoveryManager, _arachnodeDAO);
_webClient = new WebClient <TArachnodeDAO>(_applicationSettings, _webSettings, _consoleManager, _cookieManager, _proxyManager);
_webPageManager = new WebPageManager <TArachnodeDAO>(_applicationSettings, _webSettings, _discoveryManager, _htmlManager, _arachnodeDAO);
}
/// <summary>
/// Process a range of FileID after crawling. Useful if crawled Files were not processed at crawl time according to desired ApplicationSettings configuration.
/// Calling this method DOES change the 'LastDiscovered' fields where applicable.
/// This method is not when crawling, rather during post-processing.
/// </summary>
/// <param name = "fileIDLowerBound"></param>
/// <param name = "fileIDUpperBound"></param>
public static void ProcessFiles(Crawler <TArachnodeDAO> crawler, long fileIDLowerBound, long fileIDUpperBound)
{
//do not assign the application settings. doing so will override the ApplicationSetting you set before calling this method...
TArachnodeDAO arachnodeDAO = (TArachnodeDAO)Activator.CreateInstance(typeof(TArachnodeDAO), crawler.ApplicationSettings.ConnectionString, crawler.ApplicationSettings, crawler.WebSettings, false, false);
ConsoleManager <TArachnodeDAO> consoleManager = new ConsoleManager <TArachnodeDAO>(crawler.ApplicationSettings, crawler.WebSettings);
ActionManager <TArachnodeDAO> actionManager = new ActionManager <TArachnodeDAO>(crawler.ApplicationSettings, crawler.WebSettings, consoleManager);
CookieManager cookieManager = new CookieManager();;
MemoryManager <TArachnodeDAO> memoryManager = new MemoryManager <TArachnodeDAO>(crawler.ApplicationSettings, crawler.WebSettings);
RuleManager <TArachnodeDAO> ruleManager = new RuleManager <TArachnodeDAO>(crawler.ApplicationSettings, crawler.WebSettings, consoleManager);
CacheManager <TArachnodeDAO> cacheManager = new CacheManager <TArachnodeDAO>(crawler.ApplicationSettings, crawler.WebSettings);
CrawlerPeerManager <TArachnodeDAO> crawlerPeerManager = new CrawlerPeerManager <TArachnodeDAO>(crawler.ApplicationSettings, crawler.WebSettings, null, arachnodeDAO);
Cache <TArachnodeDAO> cache = new Cache <TArachnodeDAO>(crawler.ApplicationSettings, crawler.WebSettings, crawler, actionManager, cacheManager, crawlerPeerManager, memoryManager, ruleManager);
DiscoveryManager <TArachnodeDAO> discoveryManager = new DiscoveryManager <TArachnodeDAO>(crawler.ApplicationSettings, crawler.WebSettings, cache, actionManager, cacheManager, memoryManager, ruleManager);
//load the CrawlActions, CrawlRules and EngineActions...
ruleManager.ProcessCrawlRules(crawler);
actionManager.ProcessCrawlActions(crawler);
actionManager.ProcessEngineActions(crawler);
//these three methods are called in the Engine.
UserDefinedFunctions.RefreshAllowedExtensions(true);
UserDefinedFunctions.RefreshAllowedSchemes(true);
UserDefinedFunctions.RefreshDisallowed();
//instantiate a WebClient to access the ResponseHeaders...
WebClient <TArachnodeDAO> webClient = new WebClient <TArachnodeDAO>(crawler.ApplicationSettings, crawler.WebSettings, consoleManager, cookieManager, new ProxyManager <TArachnodeDAO>(crawler.ApplicationSettings, crawler.WebSettings, consoleManager));
webClient.GetHttpWebResponse("http://google.com", "GET", null, null, null, null);
FileManager <TArachnodeDAO> fileManager = new FileManager <TArachnodeDAO>(crawler.ApplicationSettings, crawler.WebSettings, discoveryManager, arachnodeDAO);
for (long i = fileIDLowerBound; i <= fileIDUpperBound; i++)
{
ArachnodeDataSet.FilesRow filesRow = null;
try
{
//get the File from the database. we need the source data as we don't store this in the index.
//even though most of the fields are available in the Document, the File is the authoritative source, so we'll use that for all of the fields.
filesRow = arachnodeDAO.GetFile(i.ToString());
if (filesRow != null)
{
if (filesRow.Source == null || filesRow.Source.Length == 0)
{
if (File.Exists(discoveryManager.GetDiscoveryPath(crawler.ApplicationSettings.DownloadedFilesDirectory, filesRow.AbsoluteUri, filesRow.FullTextIndexType)))
{
filesRow.Source = File.ReadAllBytes(discoveryManager.GetDiscoveryPath(crawler.ApplicationSettings.DownloadedFilesDirectory, filesRow.AbsoluteUri, filesRow.FullTextIndexType));
}
else
{
Console.WriteLine("FileID: " + i + " was NOT processed successfully.");
if (OnFileProcessed != null)
{
OnFileProcessed.BeginInvoke(filesRow, "FileID: " + i + " was NOT processed successfully.", null, null);
}
}
}
ProcessFile(crawler.ApplicationSettings, crawler.WebSettings, crawler, filesRow, webClient, cache, actionManager, consoleManager, crawlerPeerManager, discoveryManager, fileManager, memoryManager, ruleManager, arachnodeDAO);
Console.WriteLine("FileID: " + i + " was processed successfully.");
if (OnFileProcessed != null)
{
OnFileProcessed.BeginInvoke(filesRow, "FileID: " + i + " was processed successfully.", null, null);
}
}
}
catch (Exception exception)
{
Console.WriteLine("FileID: " + i + " was NOT processed successfully.");
Console.WriteLine(exception.Message);
if (OnFileProcessed != null)
{
OnFileProcessed.BeginInvoke(filesRow, "FileID: " + i + " was NOT processed successfully.", null, null);
OnFileProcessed.BeginInvoke(filesRow, exception.Message, null, null);
}
arachnodeDAO.InsertException(null, null, exception, false);
}
}
//stop the CrawlActions, CrawlRules and EngineActions...
ruleManager.Stop();
actionManager.Stop();
}
/// <summary>
/// Processes a FilesRow after crawling.
/// </summary>
/// <param name = "filesRow">The files row.</param>
/// <param name="webClient"></param>
/// <param name="actionManager"></param>
/// <param name="consoleManager"></param>
/// <param name="discoveryManager"></param>
/// <param name = "fileManager">The file manager.</param>
/// <param name = "fileManager">The file manager.</param>
/// <param name="memoryManager"></param>
/// <param name="ruleManager"></param>
/// <param name = "arachnodeDAO">The arachnode DAO.</param>
/// <param name = "imageManager">The image manager.</param>
public static void ProcessFile(ApplicationSettings applicationSettings, WebSettings webSettings, Crawler <TArachnodeDAO> crawler, ArachnodeDataSet.FilesRow filesRow, WebClient <TArachnodeDAO> webClient, Cache <TArachnodeDAO> cache, ActionManager <TArachnodeDAO> actionManager, ConsoleManager <TArachnodeDAO> consoleManager, CrawlerPeerManager <TArachnodeDAO> crawlerPeerManager, DiscoveryManager <TArachnodeDAO> discoveryManager, FileManager <TArachnodeDAO> fileManager, MemoryManager <TArachnodeDAO> memoryManager, RuleManager <TArachnodeDAO> ruleManager, IArachnodeDAO arachnodeDAO)
{
CacheManager <TArachnodeDAO> cacheManager = new CacheManager <TArachnodeDAO>(applicationSettings, webSettings);
CookieManager cookieManager = new CookieManager();;
CrawlRequestManager <TArachnodeDAO> crawlRequestManager = new CrawlRequestManager <TArachnodeDAO>(applicationSettings, webSettings, cache, consoleManager, discoveryManager);
DataTypeManager <TArachnodeDAO> dataTypeManager = new DataTypeManager <TArachnodeDAO>(applicationSettings, webSettings);
EncodingManager <TArachnodeDAO> encodingManager = new EncodingManager <TArachnodeDAO>(applicationSettings, webSettings);
PolitenessManager <TArachnodeDAO> politenessManager = new PolitenessManager <TArachnodeDAO>(applicationSettings, webSettings, cache);
ProxyManager <TArachnodeDAO> proxyManager = new ProxyManager <TArachnodeDAO>(applicationSettings, webSettings, consoleManager);
HtmlManager <TArachnodeDAO> htmlManager = new HtmlManager <TArachnodeDAO>(applicationSettings, webSettings, discoveryManager);
Crawl <TArachnodeDAO> crawl = new Crawl <TArachnodeDAO>(applicationSettings, webSettings, crawler, actionManager, consoleManager, cookieManager, crawlRequestManager, dataTypeManager, discoveryManager, encodingManager, htmlManager, politenessManager, proxyManager, ruleManager, true);
//create a CrawlRequest as this is what the internals of SiteCrawler.dll expect to operate on...
CrawlRequest <TArachnodeDAO> crawlRequest = new CrawlRequest <TArachnodeDAO>(new Discovery <TArachnodeDAO>(filesRow.AbsoluteUri), 1, UriClassificationType.Host, UriClassificationType.Host, 0, RenderType.None, RenderType.None);
crawlRequest.Crawl = crawl;
crawlRequest.Discovery.DiscoveryType = DiscoveryType.File;
crawlRequest.Discovery.ID = filesRow.ID;
crawlRequest.Data = filesRow.Source;
crawlRequest.ProcessData = true;
crawlRequest.WebClient = webClient;
crawlRequest.WebClient.HttpWebResponse.Headers.Clear();
//parse the ResponseHeaders from the FilesRow.ResponseHeaders string...
foreach (string responseHeader in filesRow.ResponseHeaders.Split("\r\n".ToCharArray(), StringSplitOptions.RemoveEmptyEntries))
{
string[] responseHeaderSplit = responseHeader.Split(":".ToCharArray());
string name = responseHeaderSplit[0];
string value = UserDefinedFunctions.ExtractResponseHeader(filesRow.ResponseHeaders, name, true).Value;
crawlRequest.WebClient.HttpWebResponse.Headers.Add(name, value);
}
//refresh the DataTypes in the DataTypeManager... (if necessary)...
if (dataTypeManager.AllowedDataTypes.Count == 0)
{
dataTypeManager.RefreshDataTypes();
}
crawlRequest.DataType = dataTypeManager.DetermineDataType(crawlRequest);
if (applicationSettings.InsertFiles)
{
crawlRequest.Discovery.ID = arachnodeDAO.InsertFile(crawlRequest.Parent.Uri.AbsoluteUri, crawlRequest.Discovery.Uri.AbsoluteUri, crawlRequest.WebClient.HttpWebResponse.Headers.ToString(), applicationSettings.InsertFileSource ? crawlRequest.Data : new byte[] { }, crawlRequest.DataType.FullTextIndexType, applicationSettings.ClassifyAbsoluteUris);
}
crawlRequest.ManagedDiscovery = fileManager.ManageFile(crawlRequest, crawlRequest.Discovery.ID.Value, crawlRequest.Discovery.Uri.AbsoluteUri, crawlRequest.Data, crawlRequest.DataType.FullTextIndexType, applicationSettings.ExtractFileMetaData, applicationSettings.InsertFileMetaData, applicationSettings.SaveDiscoveredFilesToDisk);
actionManager.PerformCrawlActions(crawlRequest, CrawlActionType.PostRequest, arachnodeDAO);
discoveryManager.CloseAndDisposeManagedDiscovery(crawlRequest, arachnodeDAO);
}
/// <summary>
/// Returns executor Vs sources list
/// </summary>
private Dictionary <Tuple <Uri, string>, IEnumerable <string> > GetExecutorVsSourcesList(IMessageLogger logger)
{
var result = new Dictionary <Tuple <Uri, string>, IEnumerable <string> >();
var verifiedExtensionSourceMap = new Dictionary <string, IEnumerable <string> >();
// Validate the sources
foreach (var kvp in this.adapterSourceMap)
{
var verifiedSources = DiscoveryManager.GetValidSources(kvp.Value, logger);
if (verifiedSources.Any())
{
verifiedExtensionSourceMap.Add(kvp.Key, kvp.Value);
}
}
// No valid source is found => no executor vs source map
if (!verifiedExtensionSourceMap.Any())
{
return(result);
}
foreach (var kvp in verifiedExtensionSourceMap)
{
Dictionary <LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities>, IEnumerable <string> >
discovererToSourcesMap = DiscovererEnumerator.GetDiscovererToSourcesMap(
kvp.Key,
kvp.Value,
logger);
// Warning is logged by the inner layer
if (discovererToSourcesMap == null || discovererToSourcesMap.Count == 0)
{
continue;
}
foreach (var discoverer in discovererToSourcesMap.Keys)
{
var executorUri = discoverer.Metadata.DefaultExecutorUri;
if (executorUri == null)
{
string errorMessage = string.Format(
CultureInfo.CurrentUICulture,
CrossPlatEngineResources.IgnoringExecutorAsNoDefaultExecutorUriAttribute,
discoverer.Value);
logger.SendMessage(TestMessageLevel.Warning, errorMessage);
continue;
}
var executorUriExtensionTuple = new Tuple <Uri, string>(executorUri, kvp.Key);
if (!result.ContainsKey(executorUriExtensionTuple))
{
result.Add(executorUriExtensionTuple, discovererToSourcesMap[discoverer]);
}
else
{
string errorMessage = string.Format(
CultureInfo.CurrentUICulture,
CrossPlatEngineResources.DuplicateAdaptersFound,
executorUri,
discoverer.Value);
logger.SendMessage(TestMessageLevel.Warning, errorMessage);
}
}
}
return(result);
}
public void OnDeviceRemoved(DiscoveryManager manager, ConnectableDevice device)
{
throw new NotImplementedException();
}
public MainWindowViewModel(DiscoveryManager discoveryManager)
{
_discoveryManager = discoveryManager;
Agents = new ObservableCollection <ServiceAgent>();
}
/// <summary>
/// Initializes a new instance of the <see cref = "FileManager{TArachnodeDAO}" /> class.
/// </summary>
/// <param name = "arachnodeDAO">The arachnode DAO.</param>
protected AFileManager(ApplicationSettings applicationSettings, WebSettings webSettings, DiscoveryManager <TArachnodeDAO> discoveryManager, IArachnodeDAO arachnodeDAO) : base(applicationSettings, webSettings)
{
_discoveryManager = discoveryManager;
_arachnodeDAO = arachnodeDAO;
}
private void InitDiscovery()
{
if (_ctx.NodeStatsManager == null)
{
throw new StepDependencyException(nameof(_ctx.NodeStatsManager));
}
if (_ctx.Timestamper == null)
{
throw new StepDependencyException(nameof(_ctx.Timestamper));
}
if (_ctx.NodeKey == null)
{
throw new StepDependencyException(nameof(_ctx.NodeKey));
}
if (_ctx.CryptoRandom == null)
{
throw new StepDependencyException(nameof(_ctx.CryptoRandom));
}
if (!_ctx.Config <IInitConfig>().DiscoveryEnabled)
{
_ctx.DiscoveryApp = new NullDiscoveryApp();
return;
}
IDiscoveryConfig discoveryConfig = _ctx.Config <IDiscoveryConfig>();
SameKeyGenerator privateKeyProvider = new SameKeyGenerator(_ctx.NodeKey.Unprotect());
DiscoveryMessageFactory discoveryMessageFactory = new DiscoveryMessageFactory(_ctx.Timestamper);
NodeIdResolver nodeIdResolver = new NodeIdResolver(_ctx.EthereumEcdsa);
IPResolver ipResolver = new IPResolver(_networkConfig, _ctx.LogManager);
IDiscoveryMsgSerializersProvider msgSerializersProvider = new DiscoveryMsgSerializersProvider(
_ctx._messageSerializationService,
_ctx.EthereumEcdsa,
privateKeyProvider,
discoveryMessageFactory,
nodeIdResolver);
msgSerializersProvider.RegisterDiscoverySerializers();
NodeDistanceCalculator nodeDistanceCalculator = new NodeDistanceCalculator(discoveryConfig);
NodeTable nodeTable = new NodeTable(nodeDistanceCalculator, discoveryConfig, _networkConfig, _ctx.LogManager);
EvictionManager evictionManager = new EvictionManager(nodeTable, _ctx.LogManager);
NodeLifecycleManagerFactory nodeLifeCycleFactory = new NodeLifecycleManagerFactory(
nodeTable,
discoveryMessageFactory,
evictionManager,
_ctx.NodeStatsManager,
discoveryConfig,
_ctx.LogManager);
SimpleFilePublicKeyDb discoveryDb = new SimpleFilePublicKeyDb("DiscoveryDB", DiscoveryNodesDbPath.GetApplicationResourcePath(_ctx.Config <IInitConfig>().BaseDbPath), _ctx.LogManager);
NetworkStorage discoveryStorage = new NetworkStorage(
discoveryDb,
_ctx.LogManager);
DiscoveryManager discoveryManager = new DiscoveryManager(
nodeLifeCycleFactory,
nodeTable,
discoveryStorage,
discoveryConfig,
_ctx.LogManager,
ipResolver
);
NodesLocator nodesLocator = new NodesLocator(
nodeTable,
discoveryManager,
discoveryConfig,
_ctx.LogManager);
_ctx.DiscoveryApp = new DiscoveryApp(
nodesLocator,
discoveryManager,
nodeTable,
_ctx._messageSerializationService,
_ctx.CryptoRandom,
discoveryStorage,
_networkConfig,
discoveryConfig,
_ctx.Timestamper,
_ctx.LogManager);
_ctx.DiscoveryApp.Initialize(_ctx.NodeKey.PublicKey);
}
public void OnDiscoveryFailed(DiscoveryManager manager, ServiceCommandError error)
{
throw new NotImplementedException();
}
public DeviceReport GetOutputDeviceReport(DeviceDescriptor deviceDescriptor)
{
return(DiscoveryManager.FindDeviceInfo(deviceDescriptor.DeviceHandle).DeviceReport);
}
/// <summary>
/// Initializes a new instance of the <see cref = "Crawler" /> class.
/// </summary>
public Crawler(ApplicationSettings applicationSettings, WebSettings webSettings, CrawlMode crawlMode, List <CrawlerPeer> crawlerPeers, List <DatabasePeer> databasePeers, bool enableRenderers)
{
Guid = Guid.NewGuid();
try
{
_applicationSettings = applicationSettings;
_webSettings = webSettings;
_arachnodeDAO = (TArachnodeDAO)Activator.CreateInstance(typeof(TArachnodeDAO), _applicationSettings.ConnectionString, _applicationSettings, _webSettings, true, true);
_applicationSettings = _arachnodeDAO.ApplicationSettings;
_consoleManager = new ConsoleManager <TArachnodeDAO>(_applicationSettings, _webSettings);
_consoleManager.OutputString("arachnode.net " + Assembly.GetExecutingAssembly().GetName().Version, ConsoleColor.Green, ConsoleColor.Gray);
_actionManager = new ActionManager <TArachnodeDAO>(_applicationSettings, _webSettings, _consoleManager);
_ruleManager = new RuleManager <TArachnodeDAO>(_applicationSettings, _webSettings, _consoleManager);
_memoryManager = new MemoryManager <TArachnodeDAO>(_applicationSettings, _webSettings);
_cacheManager = new CacheManager <TArachnodeDAO>(_applicationSettings, _webSettings);
_cookieManager = new CookieManager();
_cacheManager = new CacheManager <TArachnodeDAO>(_applicationSettings, _webSettings);
CrawlerPeers = crawlerPeers;
DatabasePeers = databasePeers;
_crawlerPeerManager = new CrawlerPeerManager <TArachnodeDAO>(_applicationSettings, _webSettings, CrawlerPeers, (TArachnodeDAO)Activator.CreateInstance(typeof(TArachnodeDAO), _applicationSettings.ConnectionString, _applicationSettings, _webSettings, true, true));
_databasePeerManager = new DatabasePeerManager <TArachnodeDAO>(_applicationSettings, _webSettings, DatabasePeers);
_cache = new Cache <TArachnodeDAO>(_applicationSettings, _webSettings, this, _actionManager, _cacheManager, _crawlerPeerManager, _memoryManager, _ruleManager);
_dataTypeManager = new DataTypeManager <TArachnodeDAO>(_applicationSettings, _webSettings);
_discoveryManager = new DiscoveryManager <TArachnodeDAO>(_applicationSettings, _webSettings, _cache, _actionManager, _cacheManager, _memoryManager, _ruleManager);
_crawlRequestManager = new CrawlRequestManager <TArachnodeDAO>(_applicationSettings, _webSettings, _cache, _consoleManager, _discoveryManager);
_encodingManager = new EncodingManager <TArachnodeDAO>(_applicationSettings, _webSettings);
_htmlManager = new HtmlManager <TArachnodeDAO>(_applicationSettings, _webSettings, _discoveryManager);
_politenessManager = new PolitenessManager <TArachnodeDAO>(_applicationSettings, _webSettings, _cache);
_proxyManager = new ProxyManager <TArachnodeDAO>(_applicationSettings, _webSettings, _consoleManager);
_reportingManager = new ReportingManager <TArachnodeDAO>(_applicationSettings, _webSettings, _consoleManager);
//create required directories...
if (!Directory.Exists(_applicationSettings.ConsoleOutputLogsDirectory))
{
Directory.CreateDirectory(_applicationSettings.ConsoleOutputLogsDirectory);
}
if (!Directory.Exists(_applicationSettings.DownloadedFilesDirectory))
{
Directory.CreateDirectory(_applicationSettings.DownloadedFilesDirectory);
}
if (!Directory.Exists(_applicationSettings.DownloadedImagesDirectory))
{
Directory.CreateDirectory(_applicationSettings.DownloadedImagesDirectory);
}
if (!Directory.Exists(_applicationSettings.DownloadedWebPagesDirectory))
{
Directory.CreateDirectory(_applicationSettings.DownloadedWebPagesDirectory);
}
QueryProcessor = new QueryProcessor <TArachnodeDAO>();
_consoleManager.OutputString("Crawler: Initializing Configuration/Database Connection.", ConsoleColor.White, ConsoleColor.Gray);
LoadCrawlActions(_arachnodeDAO);
LoadCrawlRules(_arachnodeDAO);
AreRenderersEnabled = enableRenderers;
Engine = new Engine <TArachnodeDAO>(_applicationSettings, _webSettings, this, _cache, _actionManager, _cacheManager, _consoleManager, _cookieManager, _crawlRequestManager, _dataTypeManager, _discoveryManager, _encodingManager, _htmlManager, _memoryManager, _politenessManager, _proxyManager, _reportingManager, _ruleManager, enableRenderers, (TArachnodeDAO)Activator.CreateInstance(typeof(TArachnodeDAO), _applicationSettings.ConnectionString, _applicationSettings, _webSettings, true, true));
CrawlMode = crawlMode;
/**/
if (CrawlerPeerManager != null && CrawlerPeerManager.CrawlerPeers != null && CrawlerPeerManager.CrawlerPeers.Count != 0)
{
ConsoleManager.OutputString("Crawler: Starting CrawlerPeerManager Server", ConsoleColor.White, ConsoleColor.Gray);
CrawlerPeerManager.StartServer(this, _arachnodeDAO);
_crawlerPeerManager.SendStatusMessageToCrawlerPeers(_arachnodeDAO);
}
/**/
if (Debugger.IsAttached)
{
_consoleManager.OutputString("Debugger: Attached - Expect Performance Degradation.", ConsoleColor.Yellow, ConsoleColor.Gray);
}
//update all core/components/managers with the updated ApplicationSettings...
#if DEMO
Engine.CrawlRequestCompleted += Engine_CrawlRequestCompleted;
_stopwatch.Start();
#endif
}
catch (InvalidConfigurationException invalidConfigurationException)
{
ProcessException(invalidConfigurationException);
throw new InvalidConfigurationException(invalidConfigurationException.ApplicationSettings, invalidConfigurationException.WebSettings, invalidConfigurationException.Message, InvalidConfigurationExceptionSeverity.Error);
}
catch (Exception exception)
{
ProcessException(exception);
throw new Exception(exception.Message, exception);
}
}
public CoreEsp8266()
{
UdpManager = new UdpManager();
DiscoveryManager = new DiscoveryManager(UdpManager);
DescriptorManager = new DescriptorManager(UdpManager);
}
/// <summary>
/// Invoked when the application is launched normally by the end user. Other entry points
/// will be used such as when the application is launched to open a specific file.
/// </summary>
/// <param name="e">Details about the launch request and process.</param>
protected override void OnLaunched(LaunchActivatedEventArgs e)
{
#if DEBUG
if (System.Diagnostics.Debugger.IsAttached)
{
DebugSettings.EnableFrameRateCounter = true;
}
#endif
var rootFrame = Window.Current.Content as Frame;
// Do not repeat app initialization when the Window already has content,
// just ensure that the window is active
if (rootFrame == null)
{
// Create a Frame to act as the navigation context and navigate to the first page
rootFrame = new Frame();
rootFrame.NavigationFailed += OnNavigationFailed;
if (e.PreviousExecutionState == ApplicationExecutionState.Terminated)
{
//TODO: Load state from previously suspended application
}
// Place the frame in the current Window
Window.Current.Content = rootFrame;
}
Microsoft.VisualStudio.TestPlatform.TestExecutor.UnitTestClient.CreateDefaultUI();
// Ensure the current window is active
Window.Current.Activate();
//Microsoft.VisualStudio.TestPlatform.TestExecutor.UnitTestClient.Run(e.Arguments);
var dm = new DiscoveryManager(new RequestData()
{
IsTelemetryOptedIn = false,
ProtocolConfig = new ProtocolConfig
{
Version = 2
},
MetricsCollection = new NoOpMetricsCollection()
});
// Adapter output location
#if DEBUG
var source = "entrypoint\\test.harness.uwp.exe";
#else
var source = "test.harness.uwp.exe";
#endif
var adapterPath = Path.Combine(Package.Current.InstalledLocation.Path, "xunit.runner.visualstudio.uwp.testadapter.dll");
var hander = new MockTestDiscoveryEventHandler();
var criteria = new DiscoveryCriteria(new[] { "" }, long.MaxValue, string.Empty);;
criteria.AdapterSourceMap.Clear();
criteria.AdapterSourceMap.Add(adapterPath, new[] { source });
hander.OnDiscoveryCompleted += (sender, args) =>
{
// Run
var em1 = new ExecutionManager(new RequestData()
{
IsTelemetryOptedIn = false,
ProtocolConfig = new ProtocolConfig
{
Version = 2
},
MetricsCollection = new NoOpMetricsCollection()
});
// Run Specific tests
em1.Initialize(new[] { adapterPath });
var tec1 = new TestExecutionContext();
em1.StartTestRun(hander.TestCases, "", "", tec1, new MockTestCaseEventsHandler(), new MockRunEventsHandler());
};
dm.DiscoverTests(criteria, hander);
//Run All -- Not working yet
//var em = new ExecutionManager(new RequestData()
//{
// IsTelemetryOptedIn = false,
// ProtocolConfig = new ProtocolConfig
// {
// Version = 2
// },
// MetricsCollection = new NoOpMetricsCollection()
//});
// var tec = new TestExecutionContext();
// em.Initialize(new[] { adapterPath });
//em.StartTestRun(criteria.AdapterSourceMap, "", "", tec, new MockTestCaseEventsHandler(), new MockRunEventsHandler());
}
private void InitDiscovery()
{
if (!_initConfig.DiscoveryEnabled)
{
_discoveryApp = new NullDiscoveryApp();
return;
}
INetworkConfig networkConfig = _configProvider.GetConfig <INetworkConfig>();
networkConfig.MasterPort = _initConfig.DiscoveryPort;
var privateKeyProvider = new SameKeyGenerator(_nodeKey);
var discoveryMessageFactory = new DiscoveryMessageFactory(networkConfig, _timestamp);
var nodeIdResolver = new NodeIdResolver(_ecdsa);
IDiscoveryMsgSerializersProvider msgSerializersProvider = new DiscoveryMsgSerializersProvider(
_messageSerializationService,
_ecdsa,
privateKeyProvider,
discoveryMessageFactory,
nodeIdResolver);
msgSerializersProvider.RegisterDiscoverySerializers();
var nodeDistanceCalculator = new NodeDistanceCalculator(networkConfig);
var nodeTable = new NodeTable(
_keyStore,
nodeDistanceCalculator,
networkConfig,
_logManager);
var evictionManager = new EvictionManager(
nodeTable,
_logManager);
var nodeLifeCycleFactory = new NodeLifecycleManagerFactory(
nodeTable,
discoveryMessageFactory,
evictionManager,
_nodeStatsManager,
networkConfig,
_logManager);
var discoveryStorage = new NetworkStorage(
DiscoveryNodesDbPath,
networkConfig,
_logManager,
_perfService);
var discoveryManager = new DiscoveryManager(
nodeLifeCycleFactory,
nodeTable,
discoveryStorage,
networkConfig,
_logManager);
var nodesLocator = new NodesLocator(
nodeTable,
discoveryManager,
_configProvider,
_logManager);
_discoveryApp = new DiscoveryApp(
nodesLocator,
discoveryManager,
nodeTable,
_messageSerializationService,
_cryptoRandom,
discoveryStorage,
networkConfig,
_logManager, _perfService);
_discoveryApp.Initialize(_nodeKey.PublicKey);
}
public void TestInit()
{
this.discoveryManager = new DiscoveryManager();
}
