コード例 #1
0
        public void WhenDocumentIsEmpty_GetTextsCount_ShouldReturnZeroCount()
        {
            htmlService.Setup(f => f.GetDocument(It.IsAny <Uri>())).Returns(mockDocumentProvider.GetHtmlDocument_Empty());

            IContentParser parser = new ContentParser(htmlService.Object);
            var            stats  = parser.GetDocumentStats(url);

            Assert.IsTrue(stats.TotalCount == 0, "Words count should be 0");
        }
コード例 #2
0
    // Use this for initialization
    void Start()
    {
        Manager <Player> .Add("Player", new Player());

        ContentParser.InsertReplace("%class%", "warrior");
        ContentParser.InsertReplace("%Class%", "Warrior");
        ItemManager.AddItem(new Item(0, new ItemDataStack("test", ItemType.A, new Stats(), ContentParser.ReplaceFromList("hello, %class%"), Resources.LoadAll <Sprite> ("Images/Items/items-3")[0], 100)));
        ItemManager.AddItem(new Item(1, new ItemDataVisual("test2", ItemType.A, new Stats(), "description2", Resources.LoadAll <Sprite> ("Images/Items/items-3")[1])));
    }
コード例 #3
0
 private void _FillQuestLog()
 {
     foreach (Quest quest in _playerObject.Player.QuestLog.Quests)
     {
         GameObject questObject = Instantiate(_questPrefab);
         questObject.transform.SetParent(gameObject.transform);
         questObject.GetComponentInChildren <Text> ().text = ContentParser.ReplaceFromList(quest.Name);
     }
 }
コード例 #4
0
 public BlogPostPublicAppService(
     IBlogRepository blogRepository,
     IBlogPostRepository blogPostRepository,
     ContentParser contentParser)
 {
     BlogRepository     = blogRepository;
     BlogPostRepository = blogPostRepository;
     ContentParser      = contentParser;
 }
コード例 #5
0
        public void WhenDocumentIsNotEmpty_GetImagesCount_ShouldReturnValidCount()
        {
            htmlService.Setup(f => f.GetDocument(It.IsAny <Uri>())).Returns(mockDocumentProvider.GetHtmlDocument_WithImages());

            IContentParser parser = new ContentParser(htmlService.Object);
            var            images = parser.GetImages(url);

            Assert.IsTrue(images.Count > 0, "Images count should not be 0");
        }
コード例 #6
0
ファイル: BlogReader.cs プロジェクト: cgourlay/website
 public BlogReader(IFileSystem fileSystem, 
     ContentParser parser,
     ContentConverter converter,
     ContentProcessor processor)
 {
     _fileSystem = fileSystem;
     _parser = parser;
     _converter = converter;
     _processor = processor;
 }
コード例 #7
0
        public void ParsingValidContentExample()
        {
            string        attributeContent = "test";
            OpenApiString example          = new OpenApiString(attributeContent);
            Dictionary <string, OpenApiExample> examples = new Dictionary <string, OpenApiExample>();

            string parsedExample = ContentParser.GetStringExampleFromContent(example, examples);

            Assert.AreEqual(attributeContent, parsedExample);
        }
コード例 #8
0
 public BlogReader(IFileSystem fileSystem,
                   ContentParser parser,
                   ContentConverter converter,
                   ContentProcessor processor)
 {
     _fileSystem = fileSystem;
     _parser     = parser;
     _converter  = converter;
     _processor  = processor;
 }
コード例 #9
0
        public void Should_throw_invalidpageexception_when_having_empty_lines()
        {
            var sampleData = "---" + Environment.NewLine +
                             " " + Environment.NewLine +
                             "title: Test" + Environment.NewLine +
                             "---";
            var parser = new ContentParser();

            Assert.Throws <InvalidPageException>(() => parser.ParsePost(sampleData));
        }
コード例 #10
0
    public async Task ParseAsync_ShouldWorkProperlyWithCorrectInputs(string content, int expectedLine)
    {
        _options.Value.AddWidget(testData.PollName, testData.WidgetName);
        contentParser = new ContentParser(_options);

        var widgets = await contentParser.ParseAsync(content);

        widgets.ShouldNotBeNull();
        widgets.Count.ShouldBe(expectedLine);
    }
コード例 #11
0
        public void CreatesHorizontalRule()
        {
            var html1 = ContentParser.ToHtml("---");
            var html2 = ContentParser.ToHtml("Some Text" + NewLine + "---");
            var html3 = ContentParser.ToHtml("Some Text" + NewLine + "---" + NewLine + "And even more");

            Assert.AreEqual(@"<hr />", html1);
            Assert.AreEqual(@"<p>Some Text</p><hr />", html2);
            Assert.AreEqual(@"<p>Some Text</p><hr /><p>And even more</p>", html3);
        }
コード例 #12
0
        public void WhenDocumentIsNotEmpty_GetTextsCount_ShouldReturnValidCount()
        {
            htmlService.Setup(f => f.GetDocument(It.IsAny <Uri>())).Returns(mockDocumentProvider.GetHtmlDocument_WithTexts());

            IContentParser parser = new ContentParser(htmlService.Object);
            var            stats  = parser.GetDocumentStats(url);

            Assert.IsNotNull(stats, "Html document stats not valid");
            Assert.IsTrue(stats.TotalCount > 0, "Words count should not be 0");
        }
コード例 #13
0
        public List <string> GetFilesNeededOnDiskBy(string absoluteName, EditorObjects.Parsing.TopLevelOrRecursive topLevelOrRecursive)
        {
            List <string> topLevelOnly   = null;
            bool          handledByCache = false;

            string standardized = FileManager.Standardize(absoluteName);

            if (filesNeededOnDisk.ContainsKey(standardized))
            {
                // compare dates:
                bool isOutOfDate = File.Exists(standardized) &&
                                   //File.GetLastWriteTime(standardized) > filesNeededOnDisk[standardized].LastWriteTime;
                                   // Do a != in case the user reverts a file
                                   File.GetLastWriteTime(standardized) != filesNeededOnDisk[standardized].LastWriteTime;

                if (!isOutOfDate)
                {
                    handledByCache = true;
                    topLevelOnly   = filesNeededOnDisk[standardized].References;
                }
            }

            if (!handledByCache)
            {
                // todo: do we want to change this to use
                topLevelOnly = ContentParser.GetFilesReferencedByAsset(absoluteName, TopLevelOrRecursive.TopLevel);
                PluginManager.GetFilesNeededOnDiskBy(absoluteName, TopLevelOrRecursive.TopLevel, topLevelOnly);
                // let's remove ../ if we can:
                for (int i = 0; i < topLevelOnly.Count; i++)
                {
                    topLevelOnly[i] = FlatRedBall.IO.FileManager.RemoveDotDotSlash(topLevelOnly[i]);
                }

                filesNeededOnDisk[standardized] = new FileReferenceInformation
                {
                    LastWriteTime = File.Exists(standardized) ? File.GetLastWriteTime(standardized) : DateTime.MinValue,
                    References    = topLevelOnly
                };
            }


            List <string> toReturn = new List <string>();

            toReturn.AddRange(topLevelOnly);

            if (topLevelOrRecursive == TopLevelOrRecursive.Recursive)
            {
                foreach (var item in topLevelOnly)
                {
                    toReturn.AddRange(GetFilesNeededOnDiskBy(item, TopLevelOrRecursive.Recursive));
                }
            }

            return(toReturn);
        }
コード例 #14
0
ファイル: Crawler.cs プロジェクト: Steve-Fenton/Katelyn
        private void AddLinksToQueueFor(QueueItem queueItem)
        {
            var request = new CrawlResult
            {
                Address       = queueItem.Address.AbsoluteUri,
                ParentAddress = queueItem.ParentAddress?.AbsoluteUri ?? string.Empty
            };

            var handler = new HttpClientHandler
            {
                AllowAutoRedirect = false
            };

            using (var httpClient = new HttpClient(handler))
            {
                var timer = CrawlTimer.Start();

                var response = httpClient.GetAsync(request.Address).Result;

                request.ContentType = response.Content.Headers.ContentType?.MediaType ?? "Unknown";
                request.StatusCode  = (int)response.StatusCode;

                if (request.StatusCode >= 400)
                {
                    request.Duration = timer.Stop();
                    _event.OnError(request, new Exception($"{response.StatusCode} ${response.ReasonPhrase}"));
                    return;
                }

                var content = GetContent(response);
                request.Duration = timer.Stop();

                ContentParser <Uri> contentParser = ParserFactory.GetLinkParser(_config, queueItem.ParentAddress, content, request.ContentType);
                request.Document = contentParser.Content;
                _event.OnDocumentLoaded(request);
                _requestQueue.AddRange(contentParser, queueItem.Address);

                var isError = false;
                ContentParser <string> searchParser = ParserFactory.GetSearchParser(_config, queueItem.ParentAddress, content, request.ContentType);

                foreach (string message in searchParser)
                {
                    isError = true;
                    _event.OnError(request, new Exception(message));
                }

                if (isError)
                {
                    return;
                }
            }

            _event.OnSuccess(request);
        }
コード例 #15
0
        public ContentParserTests()
        {
            _regexes = new Mock <IRegexes>();

            //could use auto mocking
            _regexes.SetupGet(x => x.AddedItemRegex).Returns(new Regex("test"));
            _regexes.SetupGet(x => x.UpdatedItemRegex).Returns(new Regex("test"));
            _regexes.SetupGet(x => x.RemovedItemRegex).Returns(new Regex("test"));

            _contentParser = new ContentParser(_regexes.Object);
        }
コード例 #16
0
        private static void OnPlayModeStateChanged(PlayModeStateChange state)
        {
            switch (state)
            {
            case PlayModeStateChange.EnteredEditMode:
                // call any queued code processing methods
                if (!QueuedAssetsToBeProcessed)
                {
                    break;
                }

                QueuedAssetsToBeProcessed = false;

                var importedAssets  = PostprocessBatch.ImportedAssets.ToArray();
                var deletedAssets   = PostprocessBatch.DeletedAssets.ToArray();
                var movedAssets     = PostprocessBatch.MovedAssets.ToArray();
                var movedFromAssets = PostprocessBatch.MovedFromAssets.ToArray();
                PostprocessBatch.Clear();

                ContentAssetProcessor.OnPostprocessAllAssets(importedAssets, deletedAssets, movedAssets, movedFromAssets);
                break;

            case PlayModeStateChange.ExitingEditMode:
                // generate bundles if necessary
                var config    = MasterConfig.GetInstance();
                var needBuild = config.AssetsNeedBuild;
                if (!needBuild)
                {
                    break;
                }

                var bundles = ContentObjectModel.GetInstance().AssetBundleObjects.Where(x => x.NeedBuild).ToList();
                if (bundles.Count > 0)
                {
                    try
                    {
                        ContentAssetProcessor.IgnoreChanges = true;
                        ContentParser.BuildAssetBundles(bundles);
                    }
                    finally
                    {
                        ContentAssetProcessor.IgnoreChanges = false;
                    }
                }
                break;

            case PlayModeStateChange.EnteredPlayMode:
                break;

            case PlayModeStateChange.ExitingPlayMode:
                break;
            }
        }
コード例 #17
0
    public async Task ParseAsync_ShouldWorkWithoutConfigOptions()
    {
        var content = @"**ABP Framework** is completely open source and developed in a community-driven manner.
                        [Widget Type=  ""Poll"" Code =""poll-name""]
                        Thanks _for_ *your * feedback.";

        contentParser = new ContentParser(_options);
        var widgets = await contentParser.ParseAsync(content);

        widgets.ShouldNotBeNull();
        widgets.Count.ShouldBe(1);//Ignored Widget
    }
コード例 #18
0
        public void CategoryParsing()
        {
            string test       = "[CATEGORY]";
            var    categories = new List <Category>();
            var    parser     = new ContentParser();

            parser.CategoryDiscovered += (sender, category) => categories.Add(category);

            parser.Parse(test);

            Assert.IsTrue(categories.Count == 1 && categories.First().Name == "CATEGORY");
        }
コード例 #19
0
        public void ProcessRequest(HttpContext context)
        {
            System.Collections.Generic.Dictionary <string, string> content = ContentParser.FromURL(context);

            // Start handling
            TokenHandle       handler = new TokenHandle(content, "prop");
            TokenHandleResult result  = handler.HandleResult;

            context.Response.ContentType = "application/json";
            context.Response.Write(result.ToString());
            context.Response.StatusCode = Convert.ToInt32(result.StatusCode);
        }
コード例 #20
0
        private void UpdateTemplate(bool forced)
        {
            string tempPath = new Uri(_baseUri, _config.Searching.TemplateUri).NormalizedPathAndQuery();
            ContentRecord record;
            ContentRecord.Builder update;
            if (_data.TryGetValue(TemplatePath, out record))
                update = record.ToBuilder();
            else
                update = _data.New(TemplatePath, DateTime.Now);

            ContentRecord template;
            if (_data.TryGetValue(tempPath, out template))
            {
                if (template.HasContentStoreId && (forced || template.HashOriginal != update.HashOriginal))
                {
                    update.SetContentType(template.ContentType);
                    update.SetHashOriginal(template.HashOriginal);
                    update.SetLastCrawled(template.LastCrawled);
                    update.SetLastValid(template.LastValid);
                    update.SetDateModified(DateTime.Now);
                    update.SetHttpStatus(template.HttpStatus);
                    update.ClearContentRedirect();
                    if (template.HasContentRedirect) update.SetContentRedirect(update.ContentRedirect);

                    ContentParser parser = new ContentParser(_data, _baseUri);
                    parser.RelativeUri = true;
                    parser.RewriteUri += uri => new Uri(uri.OriginalString);
                    Uri templateUri = new Uri(_baseUri, SearchTemplate.SearchPath);
                    parser.MakeRelativeUri = (s, d) => templateUri.MakeRelativeUri(d);
                    byte[] mapped = parser.ProcessFile(template, _data.ReadContent(template, true));

                    string templateHtml = CreateTemplate(Encoding.UTF8.GetString(mapped));

                    using (ITransactable trans = _data.WriteContent(update, Encoding.UTF8.GetBytes(templateHtml)))
                    {
                        _data.AddOrUpdate(TemplatePath, update.Build());
                        trans.Commit();
                    }
                }
            }

            if (!_data.TryGetValue(SearchCssPath, out record))
            {
                ContentRecord cssRecord = _data.New(SearchCssPath, DateTime.Now)
                    .SetContentType("text/css")
                    .SetHttpStatus(200)
                    .Build();

                _data.Add(cssRecord.ContentUri, cssRecord);
                _data.WriteContent(cssRecord, Encoding.UTF8.GetBytes(Properties.Resources.search_css));
            }
        }
コード例 #21
0
        private bool OpenVirtualFileModel(string virtualPath)
        {
            int p = virtualPath.IndexOf("::");

            if (p < 0)
            {
                return(false);
            }

            string container = virtualPath.Substring(0, p);
            string ext       = Path.GetExtension(container).ToLower();

            if (ext != ".swf" && ext != ".swc" && ext != ".ane")
            {
                return(false);
            }
            if (!File.Exists(container))
            {
                return(false);
            }

            string        fileName = Path.Combine(container, virtualPath.Substring(p + 2).Replace('.', Path.DirectorySeparatorChar));
            PathModel     path     = new PathModel(container, contextInstance);
            ContentParser parser   = new ContentParser(path.Path);

            parser.Run();
            AbcConverter.Convert(parser, path, contextInstance);

            if (path.HasFile(fileName))
            {
                FileModel model = path.GetFile(fileName);
                ASComplete.OpenVirtualFile(model);
                return(true);
            }
            int split = fileName.LastIndexOf(Path.DirectorySeparatorChar) + 1;

            fileName = fileName.Substring(0, split) + "package.as";
            if (path.HasFile(fileName))
            {
                FileModel model = path.GetFile(fileName);
                ASComplete.OpenVirtualFile(model);
                return(true);
            }
            fileName = fileName.Substring(0, split) + "toplevel.as";
            if (path.HasFile(fileName))
            {
                FileModel model = path.GetFile(fileName);
                ASComplete.OpenVirtualFile(model);
                return(true);
            }
            return(false);
        }
コード例 #22
0
        public void ContentParserMultipleBlackListTest()
        {
            // set a simple string containing the word to blacklist.
            string simpleString = @"[TIMESTAMP] [USERNAME] more than one word is blacklisted here.";

            // split with whitespace
            string[] testData = simpleString.Split(' ');

            // set the blacklist words to "more" and "word".
            string result = ContentParser.ParseConversationContent(testData, "more, word");

            Assert.AreEqual(@"**** than one **** is blacklisted here.", result);
        }
コード例 #23
0
        void OnViewReferencedFilesClick(object sender, EventArgs e)
        {
            List <string> allFiles =
                ContentParser.GetFilesReferencedByAsset(mLastFileName, TopLevelOrRecursive.Recursive);

            string message = "Referenced files:\n";

            foreach (string file in allFiles)
            {
                message += file + "\n";
            }

            MessageBox.Show(message);
        }
コード例 #24
0
        /// <summary>
        /// Called when GUI is to be rendered.
        /// </summary>
        public void OnGUI()
        {
            bool buildAssetBundles = EditorPrefs.GetBool("Delight_BuildAssetBundles");

            // rebuild all
            GUIContent rebuildAll = new GUIContent("Rebuild All", "Rebuilds all delight content (views, assets, etc).");

            if (GUILayout.Button(rebuildAll))
            {
                ContentParser.RebuildAll(buildAssetBundles);
            }

            var newBuildAssetBundles = EditorGUILayout.Toggle("Build Asset Bundles", EditorPrefs.GetBool("Delight_BuildAssetBundles"));

            if (newBuildAssetBundles != buildAssetBundles)
            {
                EditorPrefs.SetBool("Delight_BuildAssetBundles", newBuildAssetBundles);
            }

            bool disableAutoGenerateViews    = EditorPrefs.GetBool("Delight_DisableAutoGenerateViews");
            var  newDisableAutoGenerateViews = EditorGUILayout.Toggle("Disable view autogen", disableAutoGenerateViews);

            if (newDisableAutoGenerateViews != disableAutoGenerateViews)
            {
                EditorPrefs.SetBool("Delight_DisableAutoGenerateViews", newDisableAutoGenerateViews);
            }

            bool disableAutoGenerateHandlers    = EditorPrefs.GetBool("Delight_DisableAutoGenerateHandlers");
            var  newDisableAutoGenerateHandlers = EditorGUILayout.Toggle("Disable handler autogen", disableAutoGenerateHandlers);

            if (newDisableAutoGenerateHandlers != disableAutoGenerateHandlers)
            {
                EditorPrefs.SetBool("Delight_DisableAutoGenerateHandlers", newDisableAutoGenerateHandlers);
            }

            // open designer
            //GUIContent openDesigner = new GUIContent("Open Designer", "Opens delight designer.");
            //if (GUILayout.Button(openDesigner))
            //{
            //    EditorSceneManager.OpenScene("Assets/Delight/Scenes/DelightDesigner.unity");
            //}

            // TODO cleanup
            //GUIContent test = new GUIContent("Test", "");
            //if (GUILayout.Button(test))
            //{
            //    //var config = MasterConfig.GetInstance();
            //    //config.SaveConfig();
            //}
        }
コード例 #25
0
        //public void ProcessRequest(HttpContext context)
        //{
        //    context.Request.ContentType = "application/json";
        //    System.Collections.Generic.Dictionary<string, string> content = ContentParser.FromInputStream(context.Request.InputStream);

        //    string downloadUrl = @"http://*****:*****@"D:\Projects\WebServices\");
        //    DownloadResult result = downloader.Result;

        //    context.Response.ContentType = "application/json";
        //    context.Response.Write(result.ToString());
        //    context.Response.StatusCode = Convert.ToInt32(result.StatusCode);
        //}

        public override async Task ProcessRequestAsync(HttpContext context)
        {
            context.Request.ContentType = "application/json";
            System.Collections.Generic.Dictionary <string, string> content = ContentParser.FromInputStream(context.Request.InputStream);

            Downloader downloader = new Downloader(content, "bbb", @"D:\Projects\WebServices\WebServices\insurersinfo.ini");
            await downloader.Launch();

            DownloadResult result = downloader.Result;

            context.Response.ContentType = "application/json";
            context.Response.Write(result.ToString());
            context.Response.StatusCode = Convert.ToInt32(result.StatusCode);
        }
コード例 #26
0
    public async Task ParseAsync_ShouldWorkWithWrongPollName()
    {
        _options.Value.AddWidget(testData.PollName, testData.WidgetName);
        contentParser = new ContentParser(_options);

        var content = @"**ABP Framework** is completely open source and developed in a community-driven manner.
                        [Widget Type=   ""Poll"" PollWrongName =""poll-name""]
                        Thanks _for_ *your * feedback.";

        var widgets = await contentParser.ParseAsync(content);

        widgets.ShouldNotBeNull();
        widgets.Count.ShouldBe(3);
    }
コード例 #27
0
        public void ContentParserBlackListTest()
        {
            // set a simple string containing the word to blacklist.
            string simpleString = @"[TIMESTAMP] [USERNAME] one word are blacklisted here.";

            // split with whitespace
            string[] testData = simpleString.Split(' ');

            // set the blacklist word to "word".
            string result = ContentParser.ParseConversationContent(testData, "word");

            // check if string is as expected.
            Assert.AreEqual(@"one **** are blacklisted here.", result);
        }
コード例 #28
0
        public void ContentParserSpecialCharactersTest()
        {
            // set a complex string containing special characters.
            string simpleString = @"[TIMESTAMP] [USERNAME] & this < is % used / for \ testing@";

            // split with whitespace
            string[] testData = simpleString.Split(' ');

            // parse the splitted string.
            string result = ContentParser.ParseConversationContent(testData);

            // check if string is as expected.
            Assert.AreEqual(@"& this < is % used / for \ testing@", result);
        }
コード例 #29
0
        public void ContentParserSimpleTest()
        {
            // set a simple string with the given message format.
            string simpleString = "[TIMESTAMP] [USERNAME] this is used for testing";

            // split with whitespace
            string[] testData = simpleString.Split(' ');

            // parse the splitted string.
            string result = ContentParser.ParseConversationContent(testData);

            // check if string is as expected.
            Assert.AreEqual("this is used for testing", result);
        }
コード例 #30
0
 private int CountLinks(string directory, Predicate <Uri> test)
 {
     using (ContentStorage store = new ContentStorage(directory, true))
     {
         int           counter = 0;
         ContentParser parser  = new ContentParser(store, W3ExampleUrl);
         parser.VisitUri += u => { if (test(u))
                                   {
                                       counter++;
                                   }
         };
         parser.ProcessAll();
         return(counter);
     }
 }
コード例 #31
0
        private void Explore()
        {
            explored = true;

            if (parser != null)
            {
                return;
            }
            parser = new ContentParser(BackingPath);

            runner = new BackgroundWorker();
            runner.RunWorkerCompleted += new RunWorkerCompletedEventHandler(runner_ProcessEnded);
            runner.DoWork             += new DoWorkEventHandler(runner_DoWork);
            runner.RunWorkerAsync(parser);
        }
コード例 #32
0
        private void loadDataButton_Click(object sender, EventArgs e)
        {
            dataGridView.Enabled = true;

            if (dataGridView.Rows.Count > 1)
            {
                dataGridView.DataSource = null;
                dataGridView.Refresh();
            }

            FileReader reader = new FileReader();

            string [] results = reader.GetContent();

            if (results.Length > 1)
            {
                MessageBox.Show("Załadowano dane");
            }
            else
            {
                MessageBox.Show("Nie zaladowano danych. Spróbuj ponownie");
                dataGridView.DataSource = null;
                dataGridView.Refresh();
            }
            ContentParser contentParser = new ContentParser(results);

            content = contentParser.Parse();

            foreach (Measurement measurement in content)
            {
                table.Rows.Add(measurement.GetNumber(), measurement.GetValue());
            }
            dataGridView.DataSource = table;

            AVG      = calculations.AVG(content);
            Variance = calculations.Variance(content);
            Period   = ((1.0f / content.Count) * 10);

            avgBox.Text      = AVG.ToString();
            varianceBox.Text = Variance.ToString();
            periodBox.Text   = $"{Period.ToString()}";

            if (content.Count > 1)
            {
                graphButton.Enabled = true;
                saveButton.Enabled  = true;
            }
        }
コード例 #33
0
        public SiteCollector(string directory, string uriStart)
        {
            CrawlTime = DateTime.Now;
            _instanceId = Guid.NewGuid().ToUInt64();
            _baseUri = new Uri(new Uri(uriStart, UriKind.Absolute), "/");

            if (!Directory.Exists(directory))
                Directory.CreateDirectory(directory);

            _config = Config.ReadConfig(_baseUri, directory);
            _excluded = new PathExclusionList();
            _queue = new TextQueue(Path.Combine(directory, "workqueue.txt"), true);
            _data = new ContentStorage(directory, false);
            _parser = new ContentParser(_data, _baseUri);
            _parser.VisitUri += AddUri;

            AddUrlsFound = true;
            UpdateSearchTemplate = true;
            MaxCrawlAge = TimeSpan.MaxValue;
            AddUri(new Uri(uriStart, UriKind.Absolute));
        }
コード例 #34
0
ファイル: TopicReader.cs プロジェクト: cake-build/website
 public TopicReader(
     IFileSystem fileSystem,
     ContentParser contentParser,
     ContentConverter contentConverter,
     ContentProcessor contentProcessor)
 {
     if (fileSystem == null)
     {
         throw new ArgumentNullException(nameof(fileSystem));
     }
     if (contentParser == null)
     {
         throw new ArgumentNullException(nameof(contentParser));
     }
     if (contentConverter == null)
     {
         throw new ArgumentNullException(nameof(contentConverter));
     }
     _fileSystem = fileSystem;
     _contentParser = contentParser;
     _contentConverter = contentConverter;
     _contentProcessor = contentProcessor;
 }
コード例 #35
0
        private void RunOptimizer(Predicate<ContentRecord> filter)
        {
            Dictionary<string, string> values = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) {
                        { "site.uri", _baseUri.AbsoluteUri }
                    };

            foreach (var type in _config.DocumentTypes.Where(t => t.Optimizations != null))
            {
                ILookup<string, HttpCloneFileType> lookup = new[] {type}
                    .Concat(type.Aliases.SafeEnumeration())
                    .ToLookup(a => a.MimeType);

                ContentParser processor = new ContentParser(_content, _baseUri);
                processor.Reformat = CondenseHtml;
                processor.RelativeUri = true;
                processor.IndentChars = String.Empty;

                if (type.Type == ContentFormat.Html && _config.Searching != null && _config.Searching.FormXPath != null)
                    processor.RewriteXmlDocument += RewriteSearchForm;

                if (type.Type == ContentFormat.Html || type.Type == ContentFormat.Xml)
                {
                    new XmlRewriter(processor, type.Optimizations, values);
                }
                new RegexRewriter(processor, type.Optimizations, values);

                processor.ContextChanged +=
                    r =>
                    {
                        values["page.path"] = r.ContentUri;
                        values["page.uri"] = new Uri(_baseUri, r.ContentUri).AbsoluteUri;
                        values["page.mime"] = r.MimeType;
                    };
                processor.Process(x => (filter == null || filter(x)) && lookup.Contains(x.MimeType));
            }
        }
コード例 #36
0
            public XmlRewriter(ContentParser processor, HttpCloneOptimizations optimizations, IDictionary<string, string> namedValues)
            {
                _context = ContentRecord.DefaultInstance;
                _processor = processor;
                _optimizations = optimizations;
                _namedValues = namedValues;
                _elements = new Dictionary<XmlLightElement, object>();
                _replaces = new Dictionary<object, HttpCloneOptimizationReplace>();
                foreach (var rep in optimizations.AllItems())
                    foreach (var i in rep.ReplaceItem)
                        _replaces.Add(i, rep);

                _bytag = optimizations.AllItems()
                    .SelectMany(item => item.ReplaceItem.OfType<HttpCloneTag>())
                    .ToLookup(t => t.TagName);

                _xpaths = optimizations.AllItems()
                    .SelectMany(item => item.ReplaceItem.OfType<HttpCloneXPath>())
                    .ToArray();

                if (_bytag.Count > 0 || _xpaths.Length > 0)
                {
                    processor.ContextChanged += ContextChanged;
                    processor.RewriteElement += RewriteElement;
                    processor.RewriteXmlDocument += RewriteXmlDocument;
                }
            }
コード例 #37
0
 public ContentProcessor()
 {
     _contentParser = new ContentParser();
 }
コード例 #38
0
            public RegexRewriter(ContentParser processor, HttpCloneOptimizations optimizations, IDictionary<string, string> namedValues)
            {
                _processor = processor;
                _optimizations = optimizations;
                _namedValues = namedValues;

                _replaces = new Dictionary<object, HttpCloneOptimizationReplace>();
                foreach (var rep in optimizations.AllItems())
                    foreach (var i in rep.ReplaceItem)
                        _replaces.Add(i, rep);

                _matches = optimizations.AllItems()
                    .SelectMany(item => item.ReplaceItem.OfType<HttpCloneMatch>())
                    .ToArray();
                _expressions = _matches.Select(m => new Regex(m.Expression)).ToArray();

                if (_matches.Length > 0)
                {
                    _processor.RewriteContent += RewriteContent;
                    _processor.ContextChanged += ContextChanged;
                }
            }
コード例 #39
0
        public void ConvertTo(string target, ContentStorage writer)
        {
            Uri tgt = new Uri(target, UriKind.Absolute);
            ContentParser processor = new ContentParser(writer, _baseUri);

            processor.Reformat = Reformat;
            processor.RelativeUri = false;
            if (RebaseLinks)
            {
                processor.RewriteUri +=
                    uri =>
                        {
                            if (uri.IsSameHost(_baseUri))
                                return new Uri(tgt, uri.PathAndQuery);
                            return uri;
                        };
            }
            CopyTo(writer, processor.ProcessFile);
        }
コード例 #40
0
ファイル: TestCommands.cs プロジェクト: modulexcite/httpclone
 private int CountLinks(string directory, Predicate<Uri> test)
 {
     using (ContentStorage store = new ContentStorage(directory, true))
     {
         int counter = 0;
         ContentParser parser = new ContentParser(store, W3ExampleUrl);
         parser.VisitUri += u => { if (test(u)) counter++; };
         parser.ProcessAll();
         return counter;
     }
 }
コード例 #41
0
        public void Export(string directory)
        {
            Dictionary<string, string> renamed = GetFriendlyNames();
            Uri location = new Uri(directory, UriKind.Absolute);

            ContentParser parser = new ContentParser(_content, _baseUri);
            parser.RewriteElement +=
                e =>
                {
                    return e;
                };

            parser.RewriteUri +=
                uri =>
                {
                    string rename;
                    if (uri.IsSameHost(_baseUri))
                    {
                        if (renamed.TryGetValue(uri.NormalizedPathAndQuery(), out rename))
                        {
                            if (RebaseLinks)
                                return new Uri(location, rename.TrimStart('/', '\\'));
                            else
                                return new Uri(_baseUri, rename.TrimStart('/', '\\'));
                        }
                    }
                    return uri;
                };

            parser.RewriteAll = true;
            parser.RelativeUri = RebaseLinks;
            parser.Reformat = Reformat;
            parser.IndentChars = "  ";
            parser.ProcessAll(
                (r, b) =>
                {
                    string path;
                    if (renamed.TryGetValue(r.ContentUri, out path))
                    {
                        string file = Path.Combine(directory, path.Replace('/', '\\').TrimStart('\\'));
                        if (!Directory.Exists(Path.GetDirectoryName(file)))
                            Directory.CreateDirectory(Path.GetDirectoryName(file));
                        File.WriteAllBytes(file, b);
                    }
                }
            );
        }
コード例 #42
0
ファイル: TextFormatter.cs プロジェクト: TomGillen/Myre
 public static void SetFontSource(ContentManager fontContent)
 {
     fonts = new ContentParser<SpriteFont>(fontContent);
 }