public void TestWriteCsv()
        {
            MacroscopeJobMaster       JobMaster  = new MacroscopeJobMaster(MacroscopeConstants.RunTimeMode.LIVE);
            MacroscopeCsvErrorsReport ReportFile = new MacroscopeCsvErrorsReport();
            string Filename = string.Join(".", Path.GetTempFileName(), ".csv");

            ReportFile.WriteCsv(JobMaster: JobMaster, OutputFilename: Filename);
            Assert.IsTrue(File.Exists(Filename));
            File.Delete(Filename);
        }
예제 #2
0
        public void TestWriteXslx()
        {
            MacroscopeJobMaster           JobMaster = new MacroscopeJobMaster(MacroscopeConstants.RunTimeMode.LIVE);
            MacroscopeExcelOverviewReport Report    = new MacroscopeExcelOverviewReport();
            string Filename = string.Join(".", Path.GetTempFileName(), "xlsx");

            Report.WriteXslx(JobMaster: JobMaster, OutputFilename: Filename);
            Assert.IsTrue(File.Exists(Filename));
            File.Delete(Filename);
        }
        /**************************************************************************/

        public async Task <Image> LoadImageFromUri(MacroscopeJobMaster JobMaster, Uri TargetUri)
        {
            MacroscopeHttpTwoClient         Client   = JobMaster.GetHttpClient();
            MacroscopeHttpTwoClientResponse Response = null;
            Image LoadedImage = null;

            try
            {
                Response = await Client.Get(
                    TargetUri,
                    this.ConfigureHeadRequestHeadersCallback,
                    this.PostProcessRequestHttpHeadersCallback
                    );
            }
            catch (MacroscopeDocumentException ex)
            {
                this.DebugMsg(string.Format("MacroscopeDocumentException: {0}", ex.Message));
                this.DebugMsg(string.Format("MacroscopeDocumentException: {0}", TargetUri.ToString()));
            }
            catch (Exception ex)
            {
                this.DebugMsg(string.Format("Exception: {0}", ex.Message));
                this.DebugMsg(string.Format("Exception: {0}", TargetUri.ToString()));
            }

            if (Response != null)
            {
                try
                {
                    string ImageFilename = Path.GetTempFileName();
                    byte[] ByteData      = Response.GetContentAsBytes();

                    using (FileStream ImageStream = File.Create(ImageFilename))
                    {
                        foreach (byte b in ByteData)
                        {
                            ImageStream.WriteByte(b);
                        }
                        ImageStream.Close();
                    }

                    if (File.Exists(ImageFilename))
                    {
                        TemporaryFiles.Add(ImageFilename);
                        LoadedImage = Image.FromFile(ImageFilename);
                    }
                }
                catch (Exception ex)
                {
                    this.DebugMsg(string.Format("Exception: {0}", ex.Message));
                }
            }

            return(LoadedImage);
        }
 public void TestGenerateXmlSitemap()
 {
     foreach (string Url in this.Urls)
     {
         MacroscopeJobMaster          JobMaster        = new MacroscopeJobMaster(MacroscopeConstants.RunTimeMode.LIVE);
         MacroscopeDocumentCollection DocCollection    = new MacroscopeDocumentCollection(JobMaster: JobMaster);
         MacroscopeSitemapGenerator   SitemapGenerator = new MacroscopeSitemapGenerator(NewDocCollection: DocCollection);
         DocCollection.AddDocument(new MacroscopeDocument(JobMaster.SetStartUrl(Url: Url)));
         XmlDocument SitemapXML = SitemapGenerator.GenerateXmlSitemap(Host: new Uri(Url).Host);
         Assert.AreEqual("urlset", SitemapXML.DocumentElement.LocalName);
     }
 }
        public async Task TestQrDecode()
        {
            MacroscopeJobMaster       JobMaster   = new MacroscopeJobMaster(JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE);
            MacroscopeHttpImageLoader ImageLoader = new MacroscopeHttpImageLoader();
            Uri    ImageUri            = new Uri("https://nazuke.github.io/MacroscopeTestHarness/htdocs/qrcodes/seo-macroscope-home-page-qrcode.png");
            string QrCodeImageFilename = await ImageLoader.DownloadImageFromUriToFile(JobMaster : JobMaster, TargetUri : ImageUri);

            MacroscopeQrCodeAnalysis QrCodeAnalysis = new MacroscopeQrCodeAnalysis();
            string ResultText = QrCodeAnalysis.Decode(ImageFilename: QrCodeImageFilename);

            Assert.AreEqual("https://nazuke.github.io/SEOMacroscope/", ResultText);
        }
        /**************************************************************************/

        private void BuildWorksheetTelephoneNumbers(
            MacroscopeJobMaster JobMaster,
            XLWorkbook wb,
            string WorksheetLabel
            )
        {
            var ws = wb.Worksheets.Add(WorksheetLabel);

            int iRow    = 1;
            int iCol    = 1;
            int iColMax = 1;

            MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
            MacroscopeAllowedHosts       AllowedHosts  = JobMaster.GetAllowedHosts();

            {
                ws.Cell(iRow, iCol).Value = "Telephone Number";
                iCol++;

                ws.Cell(iRow, iCol).Value = "URL";
            }

            iColMax = iCol;

            iRow++;

            foreach (MacroscopeDocument msDoc in DocCollection.IterateDocuments())
            {
                if (msDoc.IsDocumentType(Type: MacroscopeConstants.DocumentType.HTML))
                {
                    Dictionary <string, string> TelephoneNumbers = msDoc.GetTelephoneNumbers();

                    foreach (string TelephoneNumber in TelephoneNumbers.Keys)
                    {
                        iCol = 1;

                        this.InsertAndFormatContentCell(ws, iRow, iCol, TelephoneNumber);

                        iCol++;

                        this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc);

                        iRow++;
                    }
                }
            }

            {
                var rangeData  = ws.Range(1, 1, iRow - 1, iColMax);
                var excelTable = rangeData.CreateTable();
            }
        }
        /**************************************************************************/

        public void WriteCsv(
            MacroscopeJobMaster JobMaster,
            MacroscopeCsvUriReport.OutputWorksheet SelectedOutputWorksheet,
            string OutputFilename
            )
        {
            try
            {
                using (StreamWriter writer = File.CreateText(OutputFilename))
                {
                    CsvWriter ws = new CsvWriter(writer);

                    switch (SelectedOutputWorksheet)
                    {
                    case MacroscopeCsvUriReport.OutputWorksheet.LINKS:
                        this.BuildWorksheetPageLinks(JobMaster, ws);
                        break;

                    case MacroscopeCsvUriReport.OutputWorksheet.HYPERLINKS:
                        this.BuildWorksheetPageHyperlinks(JobMaster, ws);
                        break;

                    case MacroscopeCsvUriReport.OutputWorksheet.URIS:
                        this.BuildWorksheetPageUriAnalysis(JobMaster, ws);
                        break;

                    case MacroscopeCsvUriReport.OutputWorksheet.ORPHANS:
                        this.BuildWorksheetPageOrphanedPages(JobMaster, ws);
                        break;

                    default:
                        break;
                    }
                }
            }
            catch (CsvHelperException)
            {
                MacroscopeSaveCsvFileException CannotSaveCsvFileException;
                CannotSaveCsvFileException = new MacroscopeSaveCsvFileException(
                    string.Format("Cannot write to CSV file at {0}", OutputFilename)
                    );
                throw CannotSaveCsvFileException;
            }
            catch (IOException)
            {
                MacroscopeSaveCsvFileException CannotSaveCsvFileException;
                CannotSaveCsvFileException = new MacroscopeSaveCsvFileException(
                    string.Format("Cannot write to CSV file at {0}", OutputFilename)
                    );
                throw CannotSaveCsvFileException;
            }
        }
예제 #8
0
        /**************************************************************************/

        private void BuildWorksheetPageRedirectsAudit(
            MacroscopeJobMaster JobMaster,
            CsvWriter ws
            )
        {
            MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
            MacroscopeAllowedHosts       AllowedHosts  = JobMaster.GetAllowedHosts();

            {
                ws.WriteField("Origin URL");
                ws.WriteField("Status Code");
                ws.WriteField("Status");
                ws.WriteField("Destination URL");

                ws.NextRecord();
            }

            foreach (string Url in DocCollection.DocumentKeys())
            {
                MacroscopeDocument msDoc = DocCollection.GetDocument(Url: Url);

                if (!msDoc.GetIsRedirect())
                {
                    continue;
                }

                string OriginURL      = msDoc.GetUrlRedirectFrom();
                string StatusCode     = (( int )msDoc.GetStatusCode()).ToString();
                string Status         = msDoc.GetStatusCode().ToString();
                string DestinationURL = msDoc.GetUrlRedirectTo();

                if (string.IsNullOrEmpty(OriginURL))
                {
                    continue;
                }

                if (string.IsNullOrEmpty(DestinationURL))
                {
                    continue;
                }

                this.InsertAndFormatUrlCell(ws, OriginURL);

                this.InsertAndFormatContentCell(ws, StatusCode);

                this.InsertAndFormatContentCell(ws, Status);

                this.InsertAndFormatUrlCell(ws, DestinationURL);

                ws.NextRecord();
            }
        }
        /**************************************************************************/

        private void BuildWorksheetEmailAddresses(
            MacroscopeJobMaster JobMaster,
            XLWorkbook wb,
            string WorksheetLabel
            )
        {
            var ws = wb.Worksheets.Add(WorksheetLabel);

            int iRow    = 1;
            int iCol    = 1;
            int iColMax = 1;

            MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
            MacroscopeAllowedHosts       AllowedHosts  = JobMaster.GetAllowedHosts();

            {
                ws.Cell(iRow, iCol).Value = "Email Address";
                iCol++;

                ws.Cell(iRow, iCol).Value = "URL";
            }

            iColMax = iCol;

            iRow++;

            foreach (MacroscopeDocument msDoc in DocCollection.IterateDocuments())
            {
                if (msDoc.GetIsHtml())
                {
                    Dictionary <string, string> EmailAddresses = msDoc.GetEmailAddresses();

                    foreach (string EmailAddress in EmailAddresses.Keys)
                    {
                        iCol = 1;

                        this.InsertAndFormatContentCell(ws, iRow, iCol, EmailAddress);

                        iCol++;

                        this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc);

                        iRow++;
                    }
                }
            }

            {
                var rangeData  = ws.Range(1, 1, iRow - 1, iColMax);
                var excelTable = rangeData.CreateTable();
            }
        }
        public void TestWriteXslx()
        {
            MacroscopeJobMaster JobMaster = new MacroscopeJobMaster(MacroscopeConstants.RunTimeMode.LIVE);
            MacroscopeDataExtractorCssSelectors DataExtractorCssSelectors = new MacroscopeDataExtractorCssSelectors(1);
            MacroscopeDataExtractorRegexes      DataExtractorRegexes      = new MacroscopeDataExtractorRegexes(1);
            MacroscopeDataExtractorXpaths       DataExtractorXpaths       = new MacroscopeDataExtractorXpaths(1);
            MacroscopeExcelDataExtractorReport  Report = new MacroscopeExcelDataExtractorReport(NewDataExtractorCssSelectors: DataExtractorCssSelectors, NewDataExtractorRegexes: DataExtractorRegexes, NewDataExtractorXpaths: DataExtractorXpaths);
            string Filename = string.Join(".", Path.GetTempFileName(), "xlsx");

            Report.WriteXslx(JobMaster: JobMaster, OutputFilename: Filename);
            Assert.IsTrue(File.Exists(Filename));
            File.Delete(Filename);
        }
예제 #11
0
        /**************************************************************************/

        private void BuildWorksheetListView(
            MacroscopeJobMaster JobMaster,
            XLWorkbook wb,
            string WorksheetLabel
            )
        {
            var ws = wb.Worksheets.Add(WorksheetLabel);

            int iRow    = 1;
            int iCol    = 0;
            int iColMax = 1;

            for (int i = 0; i < this.TargetListView.Columns.Count; i++)
            {
                string ColumnName = this.TargetListView.Columns[i].Text;

                iCol++;

                ws.Cell(iRow, iCol).Value = ColumnName;
            }

            for (int i = 1; i <= iCol; i++)
            {
                ws.Cell(iRow, i).Style.Font.SetBold();
            }

            iColMax = iCol;

            iRow++;

            for (int j = 0; j < this.TargetListView.Items.Count; j++)
            {
                iCol = 0;

                for (int k = 0; k < this.TargetListView.Items[j].SubItems.Count; k++)
                {
                    string CellValue = this.TargetListView.Items[j].SubItems[k].Text;

                    iCol++;

                    this.InsertAndFormatContentCell(ws, iRow, iCol, CellValue);
                }

                iRow++;
            }

            {
                var rangeData  = ws.Range(1, 1, iRow - 1, iColMax);
                var excelTable = rangeData.CreateTable();
            }
        }
예제 #12
0
        /**************************************************************************/

        private void BuildWorksheetPageGoodLinks(
            MacroscopeJobMaster JobMaster,
            CsvWriter ws
            )
        {
            MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
            MacroscopeAllowedHosts       AllowedHosts  = JobMaster.GetAllowedHosts();

            {
                ws.WriteField("Status Code");
                ws.WriteField("Status");
                ws.WriteField("Origin URL");
                ws.WriteField("Destination URL");

                ws.NextRecord();
            }

            foreach (string Url in DocCollection.DocumentKeys())
            {
                MacroscopeDocument     msDoc        = DocCollection.GetDocument(Url);
                MacroscopeHyperlinksIn HyperlinksIn = DocCollection.GetDocumentHyperlinksIn(Url);
                int    StatusCode = ( int )msDoc.GetStatusCode();
                string Status     = msDoc.GetStatusCode().ToString();

                if (
                    (StatusCode >= 200) &&
                    (StatusCode <= 299) &&
                    (HyperlinksIn != null))
                {
                    foreach (MacroscopeHyperlinkIn HyperlinkIn in HyperlinksIn.IterateLinks())
                    {
                        string OriginUrl = HyperlinkIn.GetSourceUrl();

                        if (
                            (OriginUrl != null) &&
                            (OriginUrl.Length > 0))
                        {
                            this.InsertAndFormatContentCell(ws, StatusCode.ToString());

                            this.InsertAndFormatContentCell(ws, Status);

                            this.InsertAndFormatUrlCell(ws, OriginUrl);

                            this.InsertAndFormatUrlCell(ws, msDoc);

                            ws.NextRecord();
                        }
                    }
                }
            }
        }
예제 #13
0
        /**************************************************************************/

        public MacroscopeJobMaster Load(string Pathname)
        {
            MacroscopeJobMaster JobMaster = null;

            if (File.Exists(Pathname))
            {
                Stream          openFileStream = File.OpenRead(Pathname);
                BinaryFormatter deserializer   = new BinaryFormatter();
                JobMaster = (MacroscopeJobMaster)deserializer.Deserialize(openFileStream);
                openFileStream.Close();
            }

            return(JobMaster);
        }
        public void TestJobMasterStartUrl()
        {
            MacroscopeJobMaster JobMaster;
            const string        StartUrl = "http://www.companyname.com/path/to/some/deep/folder/index.html";

            JobMaster = new MacroscopeJobMaster(
                JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
                TaskController: this
                );

            JobMaster.SetStartUrl(Url: StartUrl);

            Assert.AreEqual(StartUrl, JobMaster.GetStartUrl(), string.Format("FAIL: {0}", StartUrl));
        }
        /**************************************************************************/

        public void WriteCsv(
            MacroscopeJobMaster JobMaster,
            MacroscopeCsvPageMetadataReport.OutputWorksheet SelectedOutputWorksheet,
            string OutputFilename
            )
        {
            try
            {
                using (StreamWriter writer = File.CreateText(OutputFilename))
                {
                    CsvWriter ws = new CsvWriter(writer);

                    switch (SelectedOutputWorksheet)
                    {
                    case MacroscopeCsvPageMetadataReport.OutputWorksheet.TITLES:
                        this.BuildWorksheetPageTitles(JobMaster, ws);
                        break;

                    case MacroscopeCsvPageMetadataReport.OutputWorksheet.DESCRIPTIONS:
                        this.BuildWorksheetPageDescriptions(JobMaster, ws);
                        break;

                    case MacroscopeCsvPageMetadataReport.OutputWorksheet.KEYWORDS:
                        this.BuildWorksheetPageKeywords(JobMaster, ws);
                        break;

                    default:
                        this.BuildWorksheetPageTitles(JobMaster, ws);
                        break;
                    }
                }
            }
            catch (CsvWriterException)
            {
                MacroscopeSaveCsvFileException CannotSaveCsvFileException;
                CannotSaveCsvFileException = new MacroscopeSaveCsvFileException(
                    string.Format("Cannot write to CSV file at {0}", OutputFilename)
                    );
                throw CannotSaveCsvFileException;
            }
            catch (IOException)
            {
                MacroscopeSaveCsvFileException CannotSaveCsvFileException;
                CannotSaveCsvFileException = new MacroscopeSaveCsvFileException(
                    string.Format("Cannot write to CSV file at {0}", OutputFilename)
                    );
                throw CannotSaveCsvFileException;
            }
        }
        public async Task TestNHeadRequests()
        {
            MacroscopeJobMaster          JobMaster     = new MacroscopeJobMaster(JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE, TaskController: this);
            MacroscopeDocumentCollection DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);

            Assert.AreEqual(0, DocCollection.CountDocuments());

            foreach (string Url in this.Urls)
            {
                MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: Url);
                await msDoc.ExecuteHeadRequest();
            }

            Assert.AreEqual(this.MaxUrls, DocCollection.CountDocuments());
        }
 public void TestWriteCsv ()
 {
   MacroscopeJobMaster JobMaster = new MacroscopeJobMaster( MacroscopeConstants.RunTimeMode.LIVE );
   MacroscopeCsvContactDetailsReport ReportFile = new MacroscopeCsvContactDetailsReport();
   List<MacroscopeCsvContactDetailsReport.OutputWorksheet> SelectedOutputWorksheets = new List<MacroscopeCsvContactDetailsReport.OutputWorksheet>();
   SelectedOutputWorksheets.Add( MacroscopeCsvContactDetailsReport.OutputWorksheet.EMAIL );
   SelectedOutputWorksheets.Add( MacroscopeCsvContactDetailsReport.OutputWorksheet.TELEPHONE );
   foreach ( MacroscopeCsvContactDetailsReport.OutputWorksheet SelectedOutputWorksheet in SelectedOutputWorksheets )
   {
     string Filename = string.Join( ".", Path.GetTempFileName(), ".csv" );
     ReportFile.WriteCsv( JobMaster: JobMaster, SelectedOutputWorksheet: SelectedOutputWorksheet, OutputFilename: Filename );
     Assert.IsTrue( File.Exists( Filename ) );
     File.Delete( Filename );
   }
 }
예제 #18
0
        public async Task TestDetectLanguage()
        {
            MacroscopeJobMaster          JobMaster;
            MacroscopeDocumentCollection DocCollection;
            List <string> UrlList = new List <string>();

            UrlList.Add("https://nazuke.github.io/SEOMacroscope/");
            MacroscopePreferencesManager.SetDefaultValues();
            MacroscopePreferencesManager.SetDetectLanguage(Enabled: true);
            MacroscopePreferencesManager.SetRequestTimeout(Seconds: 10);

            JobMaster = new MacroscopeJobMaster(
                JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
                TaskController: this
                );

            DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);

            for (int i = 0; i < 10; i++)
            {
                foreach (string Url in UrlList)
                {
                    MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: Url);

                    Assert.IsNotNull(msDoc, string.Format("FAIL: {0}", Url));

                    bool ExecuteResult = await msDoc.Execute();

                    Assert.IsTrue(ExecuteResult, string.Format("FAIL: {0}", "Execute()"));

                    Assert.IsTrue(msDoc.IsDocumentType(Type: MacroscopeConstants.DocumentType.HTML), string.Format("FAIL: {0}", Url));

                    Assert.IsNotNull(msDoc.GetTitle(), string.Format("FAIL: {0}", msDoc.GetTitle()));

                    Assert.IsNotEmpty(msDoc.GetTitle(), string.Format("FAIL: {0}", msDoc.GetTitle()));

                    string LanguageTitle       = msDoc.GetTitleLanguage();
                    string LanguageDescription = msDoc.GetDescriptionLanguage();
                    string LanguageBodyText    = msDoc.GetDocumentTextLanguage();

                    Assert.AreEqual("en", LanguageTitle, string.Format("FAIL: {0} :: {1}", "LanguageTitle", LanguageTitle));

                    Assert.AreEqual("en", LanguageDescription, string.Format("FAIL: {0} :: {1}", "LanguageDescription", LanguageDescription));

                    Assert.AreEqual("en", LanguageBodyText, string.Format("FAIL: {0} :: {1}", "LanguageBodyText", LanguageBodyText));
                }
            }
        }
        public async Task TestDocumentCookiesAreSet()
        {
            MacroscopeJobMaster          JobMaster;
            MacroscopeDocumentCollection DocCollection;

            List <string> UrlList = new List <string>(2);

            UrlList.Add("https://httpbin.org/cookies/set?first=bongo&second=bongobongo");
            UrlList.Add("https://httpbin.org/cookies");
            UrlList.Add("https://httpbin.org/cookies");
            UrlList.Add("https://httpbin.org/cookies");
            UrlList.Add("https://httpbin.org/cookies");
            UrlList.Add("https://httpbin.org/cookies");
            UrlList.Add("https://httpbin.org/cookies");
            UrlList.Add("https://httpbin.org/cookies");
            UrlList.Add("https://httpbin.org/cookies");
            UrlList.Add("https://httpbin.org/cookies");
            UrlList.Add("https://httpbin.org/cookies");
            UrlList.Add("https://httpbin.org/cookies");
            UrlList.Add("https://httpbin.org/cookies");
            UrlList.Add("https://httpbin.org/cookies");
            UrlList.Add("https://httpbin.org/cookies");
            UrlList.Add("https://httpbin.org/cookies");

            JobMaster = new MacroscopeJobMaster(
                JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
                TaskController: this
                );

            DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);

            { // Set Cookies
                MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: UrlList[0]);
                bool ExecuteResult       = await msDoc.Execute();

                //Assert.IsTrue( ExecuteResult, string.Format( "FAIL: {0}", "Execute()" ) );
            }


            // Get Cookies
            for (int i = 1; i < UrlList.Count; i++)
            {
                MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: UrlList[i]);
                bool ExecuteResult       = await msDoc.Execute();

                Assert.IsTrue(ExecuteResult, string.Format("FAIL: {0}", "Execute()"));
            }
        }
예제 #20
0
        /**************************************************************************/

        public void WriteCsv(
            MacroscopeJobMaster JobMaster,
            MacroscopeCsvBrokenLinksReport.OutputWorksheet SelectedOutputWorksheet,
            string OutputFilename
            )
        {
            try
            {
                using (StreamWriter writer = File.CreateText(OutputFilename))
                {
                    CsvWriter ws = new CsvWriter(writer);

                    switch (SelectedOutputWorksheet)
                    {
                    case MacroscopeCsvBrokenLinksReport.OutputWorksheet.BROKEN_LINKS:
                        this.BuildWorksheetPageBrokenLinks(JobMaster, ws);
                        break;

                    case MacroscopeCsvBrokenLinksReport.OutputWorksheet.GOOD_LINKS:
                        this.BuildWorksheetPageGoodLinks(JobMaster, ws);
                        break;

                    case MacroscopeCsvBrokenLinksReport.OutputWorksheet.REDIRECTED_LINKS:
                        this.BuildWorksheetPageRedirectedLinks(JobMaster, ws);
                        break;

                    default:
                        break;
                    }
                }
            }
            catch (CsvWriterException)
            {
                MacroscopeSaveCsvFileException CannotSaveCsvFileException;
                CannotSaveCsvFileException = new MacroscopeSaveCsvFileException(
                    string.Format("Cannot write to CSV file at {0}", OutputFilename)
                    );
                throw CannotSaveCsvFileException;
            }
            catch (IOException)
            {
                MacroscopeSaveCsvFileException CannotSaveCsvFileException;
                CannotSaveCsvFileException = new MacroscopeSaveCsvFileException(
                    string.Format("Cannot write to CSV file at {0}", OutputFilename)
                    );
                throw CannotSaveCsvFileException;
            }
        }
예제 #21
0
        /**************************************************************************/

        public void WriteCsv(
            MacroscopeJobMaster JobMaster,
            MacroscopeCsvDataExtractorReport.OutputWorksheet SelectedOutputWorksheet,
            string OutputFilename
            )
        {
            try
            {
                using (StreamWriter writer = File.CreateText(OutputFilename))
                {
                    CsvWriter ws = new CsvWriter(writer);

                    switch (SelectedOutputWorksheet)
                    {
                    case MacroscopeCsvDataExtractorReport.OutputWorksheet.CSS_SELECTORS:
                        this.BuildWorksheetCssSelectors(JobMaster, ws);
                        break;

                    case MacroscopeCsvDataExtractorReport.OutputWorksheet.REGEXES:
                        this.BuildWorksheetRegularExpressions(JobMaster, ws);
                        break;

                    case MacroscopeCsvDataExtractorReport.OutputWorksheet.XPATHS:
                        this.BuildWorksheetXpaths(JobMaster, ws);
                        break;

                    default:
                        break;
                    }
                }
            }
            catch (CsvHelperException)
            {
                MacroscopeSaveCsvFileException CannotSaveCsvFileException;
                CannotSaveCsvFileException = new MacroscopeSaveCsvFileException(
                    string.Format("Cannot write to CSV file at {0}", OutputFilename)
                    );
                throw CannotSaveCsvFileException;
            }
            catch (IOException)
            {
                MacroscopeSaveCsvFileException CannotSaveCsvFileException;
                CannotSaveCsvFileException = new MacroscopeSaveCsvFileException(
                    string.Format("Cannot write to CSV file at {0}", OutputFilename)
                    );
                throw CannotSaveCsvFileException;
            }
        }
예제 #22
0
        /**************************************************************************/

        public async Task <bool> IsPdfUrl(MacroscopeJobMaster JobMaster, string Url)
        {
            bool   Result    = false;
            Uri    TargetUri = new Uri(Url);
            string MimeType  = await MacroscopeHttpUrlUtils.GetMimeTypeOfUrl(JobMaster : JobMaster, TargetUri : TargetUri);

            if (!string.IsNullOrEmpty(MimeType))
            {
                if (Regex.IsMatch(MimeType, "^application/pdf$", RegexOptions.IgnoreCase))
                {
                    Result = true;
                }
            }

            return(Result);
        }
        public void TestDuplicate()
        {
            const string StartUrl = "https://nazuke.github.io/SEOMacroscope/";
            const string DupeUrl  = "https://nazuke.github.io/SEOMacroscope/index.html";

            MacroscopeJobMaster JobMaster = new MacroscopeJobMaster(
                JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
                TaskController: this
                );

            MacroscopeDocumentCollection DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);

            Dictionary <string, Boolean> CrossCheckList = MacroscopeLevenshteinAnalysis.GetCrossCheckList(Capacity: DocCollection.CountDocuments());

            MacroscopeDocument msDoc          = DocCollection.CreateDocument(StartUrl);
            MacroscopeDocument msDocDifferent = DocCollection.CreateDocument(DupeUrl);

            msDoc.Execute();
            msDocDifferent.Execute();

            DocCollection.AddDocument(msDoc);
            DocCollection.AddDocument(msDocDifferent);

            DebugMsg(string.Format("msDoc: {0}", msDoc.GetStatusCode()));

            DebugMsg(string.Format("msDocDifferent: {0}", msDocDifferent.GetStatusCode()));

            for (int i = 1; i <= 100; i++)
            {
                MacroscopeLevenshteinAnalysis LevenshteinAnalysis = new MacroscopeLevenshteinAnalysis(
                    msDoc: msDoc,
                    SizeDifference: 64,
                    Threshold: 16,
                    CrossCheckList: CrossCheckList
                    );

                Dictionary <MacroscopeDocument, int> DocList = LevenshteinAnalysis.AnalyzeDocCollection(DocCollection: DocCollection);

                DebugMsg(string.Format("DocList: {0}", DocList.Count));

                foreach (MacroscopeDocument msDocAnalyzed in DocList.Keys)
                {
                    DebugMsg(string.Format("msDocAnalyzed: {0} => {1}", DocList[msDocAnalyzed], msDocAnalyzed.GetUrl()));
                    Assert.AreEqual(DocList[msDocAnalyzed], 0, string.Format("FAIL: {0} => {1}", DocList[msDocAnalyzed], msDocAnalyzed.GetUrl()));
                }
            }
        }
        public void TestWriteCsv()
        {
            MacroscopeJobMaster            JobMaster  = new MacroscopeJobMaster(MacroscopeConstants.RunTimeMode.LIVE);
            MacroscopeCsvBrokenLinksReport ReportFile = new MacroscopeCsvBrokenLinksReport();
            List <MacroscopeCsvBrokenLinksReport.OutputWorksheet> SelectedOutputWorksheets = new List <MacroscopeCsvBrokenLinksReport.OutputWorksheet>();

            SelectedOutputWorksheets.Add(MacroscopeCsvBrokenLinksReport.OutputWorksheet.BROKEN_LINKS);
            SelectedOutputWorksheets.Add(MacroscopeCsvBrokenLinksReport.OutputWorksheet.GOOD_LINKS);
            SelectedOutputWorksheets.Add(MacroscopeCsvBrokenLinksReport.OutputWorksheet.REDIRECTED_LINKS);
            foreach (MacroscopeCsvBrokenLinksReport.OutputWorksheet SelectedOutputWorksheet in SelectedOutputWorksheets)
            {
                string Filename = string.Join(".", Path.GetTempFileName(), ".csv");
                ReportFile.WriteCsv(JobMaster: JobMaster, SelectedOutputWorksheet: SelectedOutputWorksheet, OutputFilename: Filename);
                Assert.IsTrue(File.Exists(Filename));
                File.Delete(Filename);
            }
        }
 public void TestWriteSitemapXml()
 {
     foreach (string Url in this.Urls)
     {
         MacroscopeJobMaster          JobMaster        = new MacroscopeJobMaster(MacroscopeConstants.RunTimeMode.LIVE);
         MacroscopeDocumentCollection DocCollection    = new MacroscopeDocumentCollection(JobMaster: JobMaster);
         MacroscopeSitemapGenerator   SitemapGenerator = new MacroscopeSitemapGenerator(NewDocCollection: DocCollection);
         DocCollection.AddDocument(new MacroscopeDocument(JobMaster.SetStartUrl(Url: Url)));
         string Filename = string.Join(".", Path.GetTempFileName(), ".xml");
         SitemapGenerator.WriteSitemapXml(NewPath: Filename);
         Assert.IsTrue(File.Exists(Filename));
         if (File.Exists(Filename))
         {
             File.Delete(Filename);
         }
     }
 }
        public void TestWriteCsv()
        {
            MacroscopeJobMaster             JobMaster  = new MacroscopeJobMaster(MacroscopeConstants.RunTimeMode.LIVE);
            MacroscopeCsvPageMetadataReport ReportFile = new MacroscopeCsvPageMetadataReport();
            List <MacroscopeCsvPageMetadataReport.OutputWorksheet> SelectedOutputWorksheets = new List <MacroscopeCsvPageMetadataReport.OutputWorksheet>();

            SelectedOutputWorksheets.Add(MacroscopeCsvPageMetadataReport.OutputWorksheet.DESCRIPTIONS);
            SelectedOutputWorksheets.Add(MacroscopeCsvPageMetadataReport.OutputWorksheet.KEYWORDS);
            SelectedOutputWorksheets.Add(MacroscopeCsvPageMetadataReport.OutputWorksheet.TITLES);
            foreach (MacroscopeCsvPageMetadataReport.OutputWorksheet SelectedOutputWorksheet in SelectedOutputWorksheets)
            {
                string Filename = string.Join(".", Path.GetTempFileName(), ".csv");
                ReportFile.WriteCsv(JobMaster: JobMaster, SelectedOutputWorksheet: SelectedOutputWorksheet, OutputFilename: Filename);
                Assert.IsTrue(File.Exists(Filename));
                File.Delete(Filename);
            }
        }
        /**************************************************************************/

        public void WriteXslx(MacroscopeJobMaster JobMaster, string OutputFilename)
        {
            XLWorkbook Workbook = new XLWorkbook();
            Dictionary <string, int> DicTerms = null;

            if (this.ProgressForm != null)
            {
                const decimal MajorPercentageDivider = 4;

                for (int i = 0; i <= 3; i++)
                {
                    if (!this.ProgressForm.Cancelled())
                    {
                        this.ProgressForm.UpdatePercentages(
                            Title: "Processing Keywords",
                            Message: "Processing keyword terms collection:",
                            MajorPercentage: ((decimal)100 / MajorPercentageDivider) * (decimal)i + 1,
                            ProgressLabelMajor: "",
                            MinorPercentage: 0,
                            ProgressLabelMinor: ""
                            );

                        DicTerms = JobMaster.GetDocCollection().GetDeepKeywordAnalysisAsDictonary(Words: i + 1);
                        this.BuildWorksheetKeywordTerms(JobMaster, Workbook, string.Format("{0} Word Term", i + 1), DicTerms);
                    }
                }

                this.BuildWorksheetKeywordsPresence(JobMaster, Workbook, "Keywords Presence", JobMaster.GetDocCollection());

                if (!this.ProgressForm.Cancelled())
                {
                    this.SaveOutputFile(Workbook: Workbook, OutputFilename: OutputFilename);
                }
            }
            else
            {
                for (int i = 0; i <= 3; i++)
                {
                    DicTerms = JobMaster.GetDocCollection().GetDeepKeywordAnalysisAsDictonary(Words: i + 1);
                    this.BuildWorksheetKeywordTerms(JobMaster, Workbook, string.Format("{0} Word Term", i + 1), DicTerms);
                }

                this.SaveOutputFile(Workbook: Workbook, OutputFilename: OutputFilename);
            }
        }
        /** Process Data Extractors ***********************************************/

        private void ProcessGenericDataExtractors(
            string GenericText
            )
        {
            MacroscopeJobMaster JobMaster = this.DocCollection.GetJobMaster();

            {
                MacroscopeDataExtractorRegexes DataExtractor = JobMaster.GetDataExtractorRegexes();

                if ((DataExtractor != null) && (DataExtractor.IsEnabled()))
                {
                    this.ProcessGenericDataExtractorRegexes(
                        DataExtractor: DataExtractor,
                        GenericText: GenericText
                        );
                }
            }
        }
예제 #29
0
        /** Sessions Save Dialogue Boxes ******************************************/

        private void CallbackSaveSessionToFile(object sender, EventArgs e)
        {
            SaveFileDialog Dialog = new SaveFileDialog();

            Dialog.Filter           = "SEO Macroscope session files (*.seomacroscope)|*.seomacroscope";
            Dialog.FilterIndex      = 2;
            Dialog.RestoreDirectory = true;
            Dialog.DefaultExt       = "seomacroscope";
            Dialog.AddExtension     = true;
            Dialog.FileName         = "Session.seomacroscope";

            if (Dialog.ShowDialog() == DialogResult.OK)
            {
                string Pathname = Dialog.FileName;

                Cursor.Current = Cursors.WaitCursor;

                try
                {
                    MacroscopeSessionSaver SessionSaver = new MacroscopeSessionSaver();
                    MacroscopeJobMaster    JobMaster    = this.GetJobMaster();
                    SessionSaver.Save(JobMaster: JobMaster, Pathname: Pathname);

                    this.DialogueBoxFeedback("SEO Macroscope session", "The SEO Macroscope session file was saved successfully.");
                }
                catch (Exception ex)
                {
                    this.DialogueBoxError("Error", ex.Message);
                }
                finally
                {
                    Cursor.Current = Cursors.Default;
                }
            }
            else
            {
                this.DialogueBoxError("Error", "Could not open SEO Macroscope session file.");
            }

            if (Dialog != null)
            {
                Dialog.Dispose();
            }
        }
예제 #30
0
        /** -------------------------------------------------------------------- **/

        private async Task <byte[]> _LoadMemoryStreamFromUrl(MacroscopeJobMaster JobMaster, Uri TargetUri)
        {
            MacroscopeHttpTwoClient         Client   = JobMaster.GetHttpClient();
            MacroscopeHttpTwoClientResponse Response = null;

            byte[] ByteData = null;

            try
            {
                Response = await Client.Get(
                    TargetUri,
                    this.ConfigureHeadRequestHeadersCallback,
                    this.PostProcessRequestHttpHeadersCallback
                    );
            }
            catch (MacroscopeDocumentException ex)
            {
                this.DebugMsg(string.Format("MacroscopeDocumentException: {0}", ex.Message));
                this.DebugMsg(string.Format("MacroscopeDocumentException: {0}", TargetUri.ToString()));
            }
            catch (Exception ex)
            {
                this.DebugMsg(string.Format("Exception: {0}", ex.Message));
                this.DebugMsg(string.Format("Exception: {0}", TargetUri.ToString()));
            }

            if (Response != null)
            {
                try
                {
                    ByteData = Response.GetContentAsBytes();
                }
                catch (Exception ex)
                {
                    this.DebugMsg(string.Format("Exception: {0}", ex.Message));
                }
            }
            else
            {
                this.DebugMsg("NULL");
            }

            return(ByteData);
        }