public Db4oHistoryService(Uri baseUri, bool resume) { m_Resume = resume; m_Db = Db4oEmbedded.OpenFile(Db4oEmbedded.NewConfiguration(), "NCrawlerHist_{0}.Yap".FormatWith(baseUri.GetHashCode())); ClearHistory(); }
/// <summary> /// Gets the local cache path mapping from uri /// </summary> /// <param name="uri">The URI of the resource to receive the data.</param> /// <returns></returns> private string GetCachePathFromUri(System.Uri uri) { string filename = uri.GetHashCode().ToString(); filename = filename.Substring(1, filename.Length - 1); return(System.IO.Path.Combine(_cachePath, filename + ".xml")); }
public EsentCrawlerHistoryService(Uri baseUri, bool resume) { m_Resume = resume; m_DatabaseFileName = Path.GetFullPath("NCrawlHist{0}\\Hist.edb".FormatWith(baseUri.GetHashCode())); if (!resume) { ClearHistory(); } m_EsentInstance = new EsentInstance(m_DatabaseFileName, (session, dbid) => { EsentTableDefinitions.CreateGlobalsTable(session, dbid); EsentTableDefinitions.CreateHistoryTable(session, dbid); }); // Get columns m_EsentInstance.Cursor((session, dbid) => { Api.JetGetColumnInfo(session, dbid, EsentTableDefinitions.GlobalsTableName, EsentTableDefinitions.GlobalsCountColumnName, out historyCountColumn); Api.JetGetColumnInfo(session, dbid, EsentTableDefinitions.HistoryTableName, EsentTableDefinitions.HistoryTableUrlColumnName, out historyUrlColumn); }); }
public virtual void Test22URL() { DeleteBase("test.url.odb"); var url1 = new Uri("http://google.com"); var url2 = new Uri("http://nprogramming.wordpress.com"); var h1 = url1.GetHashCode(); var h2 = url2.GetHashCode(); Println(h1 + " - " + h2); Println(url1.Host + " - " + url1.Port); Println(url2.Host + " - " + url2.Port); var odb = Open("test.url.odb"); odb.Store(url1); odb.Store(url2); odb.Close(); odb = Open("test.url.odb"); var query = odb.Query<Uri>(); var l = query.Execute<Uri>(); var first = l.FirstOrDefault(x => x.AbsoluteUri == "http://google.com/"); Assert.That(first, Is.Not.Null); var second = l.FirstOrDefault(x => x.AbsoluteUri == "http://nprogramming.wordpress.com/"); Assert.That(second, Is.Not.Null); odb.Close(); AssertEquals("Same HashCode Problem", 2, l.Count()); }
public EsentCrawlQueueService(Uri baseUri, bool resume) { m_DatabaseFileName = Path.GetFullPath("NCrawlQueue{0}\\Queue.edb".FormatWith(baseUri.GetHashCode())); if (!resume && File.Exists(m_DatabaseFileName)) { ClearQueue(); } m_EsentInstance = new EsentInstance(m_DatabaseFileName, (session, dbid) => { EsentTableDefinitions.CreateGlobalsTable(session, dbid); EsentTableDefinitions.CreateQueueTable(session, dbid); }); // Get columns m_EsentInstance.Cursor((session, dbid) => { Api.JetGetColumnInfo(session, dbid, EsentTableDefinitions.GlobalsTableName, EsentTableDefinitions.GlobalsCountColumnName, out queueCountColumn); Api.JetGetColumnInfo(session, dbid, EsentTableDefinitions.QueueTableName, EsentTableDefinitions.QueueTableDataColumnName, out dataColumn); }); }
private XmlReader getXmlReader(int key, Uri xmlUri) { int uriHashcode = xmlUri.GetHashCode(); if (m_xmlSourceETagDictionary.ContainsKey(uriHashcode)) { this.LogInfo("Dictionary contains key: {0}", uriHashcode); if (m_xmlSourceETagDictionary[uriHashcode] == key) { this.LogInfo("{0} matches {1}", m_xmlSourceETagDictionary[uriHashcode], key); return getXmlReader(uriHashcode, xmlUri, false); } else { this.LogInfo("{0} does not match {1}", m_xmlSourceETagDictionary[uriHashcode], key); m_xmlSourceETagDictionary[uriHashcode] = key; return getXmlReader(uriHashcode, xmlUri, true); } } else { this.LogInfo("Dictionary does not contain key: {0}", uriHashcode); m_xmlSourceETagDictionary[uriHashcode] = key; return getXmlReader(uriHashcode, xmlUri, true); } }
private void addXmlReader(int key, Uri uri) { XmlReader reader = createNewXmlReader(uri.OriginalString); int uriHashcode = uri.GetHashCode(); m_xmlReaderDictionary[uriHashcode] = reader; m_xmlSourceETagDictionary[uriHashcode] = key; }
public DbCrawlQueueService(Uri baseUri, bool resume) { m_GroupId = baseUri.GetHashCode(); if (!resume) { Clean(); } }
public SqLiteCrawlerHistoryService(Uri uri, bool resume) { m_GroupId = uri.GetHashCode(); if (!resume) { Clean(); } }
/// <summary> /// Initializes a new instance of the <see cref="EfCrawlQueueService"/> class. /// </summary> /// <param name="baseUri">Uri from which work is started.</param> /// <param name="resume">True to resume work, false otherwise.</param> public EfCrawlQueueService(Uri baseUri, bool resume) { this.groupId = baseUri.GetHashCode(); if (!resume) { this.CleanQueue(); } }
public Db4oHistoryService(Uri baseUri, bool resume) { m_Db = Db4oEmbedded.OpenFile(Db4oEmbedded.NewConfiguration(), "NCrawlerHist_{0}.Yap".FormatWith(baseUri.GetHashCode())); if (!resume) { m_Db.Query<StringWrapper>().ForEach(entry => m_Db.Delete(entry)); } }
internal CredentialCacheKey(System.Uri uriPrefix, string authType) { this.uriPrefix = uriPrefix; this.authType = authType; this.absPath = uriPrefix.AbsolutePath; this.absPath = this.absPath.Substring(0, this.absPath.LastIndexOf('/')); this.len = uriPrefix.AbsoluteUri.Length; this.hash = uriPrefix.GetHashCode() + authType.GetHashCode(); }
public Db4oQueueService(Uri baseUri, bool resume) { string fileName = Path.GetFullPath("NCrawlerQueue_{0}.Yap".FormatWith(baseUri.GetHashCode())); m_Db = Db4oEmbedded.OpenFile(Db4oEmbedded.NewConfiguration(), fileName); if (!resume) { ClearQueue(); } }
public Db4oQueueService(Uri baseUri, bool resume) { m_Db = Db4oEmbedded.OpenFile(Db4oEmbedded.NewConfiguration(), "NCrawlerQueue_{0}.Yap".FormatWith(baseUri.GetHashCode())); if (!resume) { m_Db.Query<CrawlerQueueEntry>().ForEach(entry => m_Db.Delete(entry)); } }
/// <summary> /// Initializes a new instance of the <see cref="EfCrawlerHistoryService"/> class. /// </summary> /// <param name="uri">Uri from which work is started.</param> /// <param name="resume">True to resume work, false otherwise.</param> public EfCrawlerHistoryService(Uri uri, bool resume) { this.resume = resume; this.groupId = uri.GetHashCode(); if (!this.resume) { this.CleanHistory(); } }
public static void Run() { Console.Out.WriteLine("Simple crawl demo using local database a storage"); var targetToCrawl = ConfigurationManager.AppSettings["CrawlTargetUrl"]; var maximumThreadCount = int.Parse(ConfigurationManager.AppSettings["MaximumThreadCount"]); var maximumCrawlDepth = int.Parse(ConfigurationManager.AppSettings["MaximumCrawlDepth"]); // Setup crawler to crawl http://ncrawler.codeplex.com // with 1 thread adhering to robot rules, and maximum depth // of 2 with 4 pipeline steps: // * Step 1 - The Html Processor, parses and extracts links, text and more from html // * Step 2 - Processes PDF files, extracting text // * Step 3 - Try to determine language based on page, based on text extraction, using google language detection // * Step 4 - Dump the information to the console, this is a custom step, see the DumperStep class DbServicesModule.Setup(true); using (Crawler c = new Crawler(new Uri(targetToCrawl), new WholeHtmlProcessor(), // Process html new DumperStep()) { // Custom step to visualize crawl MaximumThreadCount = maximumThreadCount, MaximumCrawlDepth = maximumCrawlDepth, ExcludeFilter = Program.ExtensionsToSkip, }) { AspectF.Define.Do<NCrawlerEntitiesDbServices>(e => { if (e.CrawlQueue.Any()) { var uri = new Uri(targetToCrawl); var groupId = uri.GetHashCode(); Console.Out.WriteLine("GroupId=" + groupId); e.ExecuteStoreCommand("Update CrawlQueue set Exclusion='false' where GroupId={0} and Exclusion='true'", groupId); //var exclusion = e.CrawlQueue.Where(m => m.Exclusion && m.GroupId == groupId).ToList(); //if (exclusion.Any()) //{ // Console.Out.WriteLine("Count with Exclusion=" + exclusion.Count); // exclusion.ForEach(m => m.Exclusion = false); //} ////foreach (var crawlQueue in e.CrawlQueue) ////{ //// crawlQueue.Exclusion = false; ////} //e.SaveChanges(); } }); // Begin crawl Console.Out.WriteLine(" Begin crawl"); c.Crawl(); } }
public CouchbaseNode(CouchbaseRestConfiguration config, Uri uri) { _config = config; Id = uri.GetHashCode(); Uri = uri; _builder = new UrlBuilder(uri); ErrorAge = DateTime.UtcNow; //The first time a request is made to a URI, the ServicePointManager //will create a ServicePoint to manage connections to a particular host ServicePointManager.FindServicePoint(Uri).SetTcpKeepAlive(true, 300, 30); }
protected BackgroundTransferRequest GetBackTXRequest(string src,string tag) { Uri src_uri = new Uri(src, UriKind.Absolute); Uri dst_uri = new Uri("shared\\transfers\\temp"+src_uri.GetHashCode()+".mp3", UriKind.Relative); BackgroundTransferRequest back_req = new BackgroundTransferRequest(src_uri, dst_uri); back_req.Tag = tag; bool use_cellular = false; IsolatedStorageSettings.ApplicationSettings.TryGetValue("use_cellular", out use_cellular); if (use_cellular) back_req.TransferPreferences = TransferPreferences.AllowCellularAndBattery; else back_req.TransferPreferences = TransferPreferences.AllowBattery; return back_req; }
public void GetImage(Uri uri, Action<byte[]> callback) { var fileName = cachePath + "Smeedee_img" + uri.GetHashCode(); if (File.Exists(fileName)) { var bytes = fileReader.ReadAllBytes(fileName); if (IsMissingImagePlaceholder(bytes)) bytes = null; callback(bytes); } else { serviceToCache.GetImage(uri, bytes => SaveAndCallback(bytes, fileName, callback)); } }
static int GetHashCode(IntPtr L) { try { ToLua.CheckArgsCount(L, 1); System.Uri obj = (System.Uri)ToLua.CheckObject <System.Uri>(L, 1); int o = obj.GetHashCode(); LuaDLL.lua_pushinteger(L, o); return(1); } catch (Exception e) { return(LuaDLL.toluaL_exception(L, e)); } }
public async Task<string> GetFeedXml(Uri uri) { var hash = uri.GetHashCode(); // get data folder var localFolder = ApplicationData.Current.LocalFolder; var filename = hash + ".xml"; var client = new HttpClient(); StorageFile file; try { file = await localFolder.GetFileAsync(filename); var msg = new HttpRequestMessage(HttpMethod.Head, uri); msg.Headers.IfModifiedSince = file.DateCreated; var resp = await client.SendRequestAsync(msg); if (resp.Headers.ContainsKey("Last-Modified")) { Debug.WriteLine(resp.Headers["Last-Modified"]); } } catch (FileNotFoundException) { } string response = await client.GetStringAsync(uri); file = await localFolder.CreateFileAsync(filename, CreationCollisionOption.ReplaceExisting); var stream = await file.OpenStreamForWriteAsync(); var writer = new StreamWriter(stream); writer.Write(response); writer.Flush(); writer.Dispose(); stream.Dispose(); // save to data folder return response; }
public bool HasXmlSourceChanged ( int eTag, Uri uri ) { int uriHashcode = uri.GetHashCode(); if (m_xmlSourceETagDictionary.ContainsKey(uriHashcode)) { if (m_xmlSourceETagDictionary[uriHashcode] == eTag) { this.LogInfo("Source has not changed. {0}. Count: {1}", eTag, m_xmlSourceETagDictionary.Count); return false; } else { this.LogInfo("Source has changed. {0}. Count: {1}", eTag, m_xmlSourceETagDictionary.Count); return true; } } else { this.LogInfo("Source has changed. {0}. Count: {1}", eTag, m_xmlSourceETagDictionary.Count); return true; } }
public static void CaptureScreen(Uri source, Dictionary<TimeSpan, object> captureList, double scale, CaptureWorkerDelegate finalWorkerPrimary, CaptureWorkerDelegate finalWorkerThumbnail) { var mutexLock = new Mutex(false, source.GetHashCode().ToString()); mutexLock.WaitOne(); var player = new MediaPlayer { Volume = 0, ScrubbingEnabled = true }; player.Open(source); player.Pause(); foreach (var pair in captureList) { var timeSpan = pair.Key; var state = pair.Value; player.Position = timeSpan; Thread.Sleep(1000); var width = player.NaturalVideoWidth; var height = player.NaturalVideoHeight; var rtb = new RenderTargetBitmap(width, height, 96, 96, PixelFormats.Pbgra32); var dv = new DrawingVisual(); using (DrawingContext dc = dv.RenderOpen()) dc.DrawVideo(player, new Rect(0, 0, width, height)); rtb.Render(dv); var frame = BitmapFrame.Create(rtb).GetCurrentValueAsFrozen(); if (finalWorkerPrimary != null) finalWorkerPrimary(frame as BitmapFrame, state); if (scale > 0 && finalWorkerThumbnail != null) { var thumbnailFrame = BitmapFrame.Create(new TransformedBitmap(frame as BitmapSource, new ScaleTransform(scale, scale))). GetCurrentValueAsFrozen(); var encoder = new JpegBitmapEncoder(); encoder.Frames.Add(thumbnailFrame as BitmapFrame); finalWorkerThumbnail(thumbnailFrame as BitmapFrame, state); } } player.Close(); mutexLock.ReleaseMutex(); }
private void ReadMessages(Uri heartBeatUrl, Uri configUrl) { if (this.stopEvent.WaitOne(0)) return; #if DEBUG_RETRY ThreadPool.QueueUserWorkItem(o => { Thread.Sleep(4000); CleanupRequests(); }); if (this.currentRetryCount > 0) throw new IOException(); #endif this.AbortRequests(); this.request = this.requestFactory.GetWebRequest(configUrl, configUrl.GetHashCode().ToString()); this.response = this.request.GetResponse(); // the url is supposed to send data indefinitely // the only way out of here is either calling Stop() or failing by an exception // (somehow stream.Dispose() hangs, so we'll not use the 'using' construct) var stream = this.response.GetResponseStream(); var reader = new StreamReader(stream, Encoding.UTF8, false); // TODO make the 10 seconds configurable (it if makes sense) using (this.heartbeat = new Heartbeat(heartBeatUrl, this.ConnectionTimeout, 10 * 1000, this.AbortRequests, this.Credentials)) { string line; var emptyCounter = 0; var messageBuilder = new StringBuilder(); while ((line = reader.ReadLine()) != null) { if (this.stopEvent.WaitOne(0)) return; // we're successfully reading the stream, so reset the retry counter this.currentRetryCount = 0; if (line.Length == 0) { emptyCounter++; // messages are separated by 3 empty lines if (emptyCounter == 3) { this.Trigger(messageBuilder.ToString()); messageBuilder.Length = 0; emptyCounter = 0; } } else { emptyCounter = 0; messageBuilder.Append(line); } } } if (log.IsErrorEnabled) log.Error("The infinite loop just finished, probably the server closed the connection without errors. (?)"); throw new IOException("Remote host closed the streaming connection"); }
private static string GetCacheFileName(System.Uri uri) { return(uri.GetHashCode().ToString()); }
public void UriHashCodes() { //Quick Test to see if how the Uri classes Hash Codes behave Uri test1 = new Uri("http://example.org/test#one"); Uri test2 = new Uri("http://example.org/test#two"); Uri test3 = new Uri("http://example.org/test#three"); Console.WriteLine("Three identical URIs with different Fragment IDs, .Net ignores the Fragments in creating Hash Codes"); Console.WriteLine("URI 1 has Hash Code " + test1.GetHashCode()); Console.WriteLine("URI 2 has Hash Code " + test2.GetHashCode()); Console.WriteLine("URI 3 has Hash Code " + test3.GetHashCode()); Assert.AreEqual(test1.GetHashCode(), test2.GetHashCode()); Assert.AreEqual(test2.GetHashCode(), test3.GetHashCode()); Assert.AreEqual(test1.GetHashCode(), test3.GetHashCode()); }
public TransformResponse Process () { XmlReader reader = m_navigator.ReadSubtree(); string xmlStylesheetHref = String.Empty; bool processWithEmbeddedPIStylsheet = false; do { switch (reader.NodeType) { case XmlNodeType.ProcessingInstruction: switch (reader.Name) { case "xml-stylesheet": string piValue = reader.Value; if (piValue.Contains("type=\"text/xsl\"") && piValue.Contains("href=")) { processWithEmbeddedPIStylsheet = true; xmlStylesheetHref = piValue.SubstringAfter("href=\"").SubstringBefore("\""); } Console.WriteLine("Stylesheet Href = {0}", xmlStylesheetHref); break; default: break; } break; case XmlNodeType.Element: if (reader.IsStartElement()) { switch (reader.Name) { case "service:compile": if (reader.HasAttributes) { bool USEMEMCACHE = false; bool SUCCESS = bool.TryParse(reader.GetAttribute("cache-result"), out USEMEMCACHE); Console.WriteLine(String.Format("Operation succeeded: {0}, Use memcached: {1}", SUCCESS, USEMEMCACHE)); } break; case "service:session": Console.WriteLine("service:session reached"); break; case "service:operation": case "my:page": Console.WriteLine("service:operation or my:page reached"); Uri baseXsltUri = new Uri(m_httpContext.Request.MapPath(xmlStylesheetHref)); m_xslTransformationManager.BaseXsltUri = baseXsltUri; string baseXslt = baseXsltUri.GetHashCode().ToString(); if (!m_xslTransformationManager.NamedXsltHashtable.ContainsKey(baseXslt)) { m_xslTransformationManager.AddTransformer(baseXslt, baseXsltUri); m_xslTransformationManager.NamedXsltHashtable.Add(baseXslt, baseXsltUri); } if (!m_xslTransformationManager.XmlSourceHashtable.ContainsKey(m_context.RequestXmlETag)) { Console.WriteLine(reader.ReadOuterXml()); using (MemoryStream stream = new MemoryStream(m_encoding.GetBytes(reader.ReadOuterXml().ToCharArray()))) { m_xslTransformationManager.AddXmlSource(m_context.RequestXmlETag.ToString(), (Stream)stream); } } TransformContext transformContext = new TransformContext(); transformContext.Context = m_transformContext; transformContext.HttpContext = m_httpContext; transformContext.XsltTransformationManager = m_xslTransformationManager; transformContext.XsltName = baseXslt; transformContext.Response = m_response; m_request.TransformContext = transformContext; m_response = m_transform.BeginTransformProcess(m_request); break; default: break; } } break; case XmlNodeType.EndElement: continue; case XmlNodeType.Text: case XmlNodeType.SignificantWhitespace: case XmlNodeType.Whitespace: break; case XmlNodeType.CDATA: break; case XmlNodeType.Comment: break; case XmlNodeType.DocumentType: break; case XmlNodeType.EntityReference: reader.ResolveEntity(); continue; case XmlNodeType.XmlDeclaration: case XmlNodeType.EndEntity: continue; default: break; //throw new InvalidOperationException(); } } while (reader.Read()); return m_response; }
internal CredentialCacheKey (Uri uriPrefix, string authType) { this.uriPrefix = uriPrefix; this.authType = authType; this.absPath = uriPrefix.AbsolutePath; this.absPath = absPath.Substring (0, absPath.LastIndexOf ('/')); this.len = uriPrefix.AbsoluteUri.Length; this.hash = uriPrefix.GetHashCode () + authType.GetHashCode (); }
/// <summary> /// Create a new <see cref="IHttpClient"/> instance for the given endpoint. /// </summary> /// <param name="endpoint">The endpoint for the client to retrieve.</param> /// <returns>An initialized <see cref="IHttpClient"/> instance.</returns> /// <seealso cref="IHttpClientFactory.CreateClient"/> public IHttpClient CreateClient(Uri endpoint) { if (endpoint == null) { throw new ArgumentNullException("endpoint"); } string resourceName; if (endpoint == WtaClient.SearchEndpoint) { resourceName = "MyTrails.Importer.Test.SampleData.trails.js"; } else if (endpoint.AbsoluteUri.Contains(WtaClient.TripReportsEndpointFormat)) { int resourceIndex = Math.Abs(endpoint.GetHashCode()) % TripReportResources.Length; resourceName = TripReportResources[resourceIndex]; } else { throw new NotSupportedException(string.Format(CultureInfo.InvariantCulture, "Endpoint not supported: {0}", WtaClient.SearchEndpoint)); } return new WtaDataFromEmbeddedResource(resourceName); }
/// <summary> /// Gets the local cache path mapping from uri /// </summary> /// <param name="uri">The URI of the resource to receive the data.</param> /// <returns></returns> private string GetCachePathFromUri(Uri uri) { string filename = uri.GetHashCode().ToString(); filename = filename.Substring(1, filename.Length - 1); return Path.Combine(_cachePath, filename + ".xml"); }
public void GetHashCodeShouldReturnCorrectValue() { // arrange var target = new UriComparer(); var uri = new Uri( "about:blank" ); var expected = uri.GetHashCode(); // act var actual = target.GetHashCode( uri ); // assert Assert.Equal( 0, target.GetHashCode( null ) ); Assert.Equal( expected, actual ); }
public void GetHashCodeTest () { Uri uri1 = new Uri ("http://www.contoso.com/index.htm#main"); Uri uri2 = new Uri ("http://www.contoso.com/index.htm#fragment"); Assert.AreEqual (uri1.GetHashCode (), uri2.GetHashCode (), "#1"); uri2 = new Uri ("http://www.contoso.com/index.htm?x=1"); Assert.IsTrue (uri1.GetHashCode () != uri2.GetHashCode (), "#2"); uri2 = new Uri ("http://www.contoso.com:80/index.htm"); Assert.AreEqual (uri1.GetHashCode (), uri2.GetHashCode (), "#3"); uri2 = new Uri ("http://www.contoso.com:8080/index.htm"); Assert.IsTrue (uri1.GetHashCode () != uri2.GetHashCode (), "#4"); }
public void RelativeGetHashCodeTest() { Uri uri1 = new Uri ("foo/bar", UriKind.Relative); Uri uri2 = new Uri ("foo/bar", UriKind.Relative); Uri uri3 = new Uri ("bar/man", UriKind.Relative); Assert.AreEqual (uri1.GetHashCode(), uri2.GetHashCode(), "#1"); Assert.IsTrue (uri1.GetHashCode() != uri3.GetHashCode(), "#2"); }
public void GetHashCodeTest () { Uri uri1 = new Uri ("http://www.contoso.com/index.htm#main"); Uri uri2 = new Uri ("http://www.contoso.com/index.htm#fragment"); AssertEquals ("#1", uri1.GetHashCode (), uri2.GetHashCode ()); uri2 = new Uri ("http://www.contoso.com/index.htm?x=1"); Assert ("#2", uri1.GetHashCode () != uri2.GetHashCode ()); uri2 = new Uri ("http://www.contoso.com:80/index.htm"); AssertEquals ("#3", uri1.GetHashCode (), uri2.GetHashCode ()); uri2 = new Uri ("http://www.contoso.com:8080/index.htm"); Assert ("#4", uri1.GetHashCode () != uri2.GetHashCode ()); }