public void TestWriteCsv()
{
MacroscopeJobMaster JobMaster = new MacroscopeJobMaster(MacroscopeConstants.RunTimeMode.LIVE);
MacroscopeCsvErrorsReport ReportFile = new MacroscopeCsvErrorsReport();
string Filename = string.Join(".", Path.GetTempFileName(), ".csv");
ReportFile.WriteCsv(JobMaster: JobMaster, OutputFilename: Filename);
Assert.IsTrue(File.Exists(Filename));
File.Delete(Filename);
}
public void TestWriteXslx()
{
MacroscopeJobMaster JobMaster = new MacroscopeJobMaster(MacroscopeConstants.RunTimeMode.LIVE);
MacroscopeExcelOverviewReport Report = new MacroscopeExcelOverviewReport();
string Filename = string.Join(".", Path.GetTempFileName(), "xlsx");
Report.WriteXslx(JobMaster: JobMaster, OutputFilename: Filename);
Assert.IsTrue(File.Exists(Filename));
File.Delete(Filename);
}
/**************************************************************************/
public async Task <Image> LoadImageFromUri(MacroscopeJobMaster JobMaster, Uri TargetUri)
{
MacroscopeHttpTwoClient Client = JobMaster.GetHttpClient();
MacroscopeHttpTwoClientResponse Response = null;
Image LoadedImage = null;
try
{
Response = await Client.Get(
TargetUri,
this.ConfigureHeadRequestHeadersCallback,
this.PostProcessRequestHttpHeadersCallback
);
}
catch (MacroscopeDocumentException ex)
{
this.DebugMsg(string.Format("MacroscopeDocumentException: {0}", ex.Message));
this.DebugMsg(string.Format("MacroscopeDocumentException: {0}", TargetUri.ToString()));
}
catch (Exception ex)
{
this.DebugMsg(string.Format("Exception: {0}", ex.Message));
this.DebugMsg(string.Format("Exception: {0}", TargetUri.ToString()));
}
if (Response != null)
{
try
{
string ImageFilename = Path.GetTempFileName();
byte[] ByteData = Response.GetContentAsBytes();
using (FileStream ImageStream = File.Create(ImageFilename))
{
foreach (byte b in ByteData)
{
ImageStream.WriteByte(b);
}
ImageStream.Close();
}
if (File.Exists(ImageFilename))
{
TemporaryFiles.Add(ImageFilename);
LoadedImage = Image.FromFile(ImageFilename);
}
}
catch (Exception ex)
{
this.DebugMsg(string.Format("Exception: {0}", ex.Message));
}
}
return(LoadedImage);
}
public void TestGenerateXmlSitemap()
{
foreach (string Url in this.Urls)
{
MacroscopeJobMaster JobMaster = new MacroscopeJobMaster(MacroscopeConstants.RunTimeMode.LIVE);
MacroscopeDocumentCollection DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);
MacroscopeSitemapGenerator SitemapGenerator = new MacroscopeSitemapGenerator(NewDocCollection: DocCollection);
DocCollection.AddDocument(new MacroscopeDocument(JobMaster.SetStartUrl(Url: Url)));
XmlDocument SitemapXML = SitemapGenerator.GenerateXmlSitemap(Host: new Uri(Url).Host);
Assert.AreEqual("urlset", SitemapXML.DocumentElement.LocalName);
}
}
public async Task TestQrDecode()
{
MacroscopeJobMaster JobMaster = new MacroscopeJobMaster(JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE);
MacroscopeHttpImageLoader ImageLoader = new MacroscopeHttpImageLoader();
Uri ImageUri = new Uri("https://nazuke.github.io/MacroscopeTestHarness/htdocs/qrcodes/seo-macroscope-home-page-qrcode.png");
string QrCodeImageFilename = await ImageLoader.DownloadImageFromUriToFile(JobMaster : JobMaster, TargetUri : ImageUri);
MacroscopeQrCodeAnalysis QrCodeAnalysis = new MacroscopeQrCodeAnalysis();
string ResultText = QrCodeAnalysis.Decode(ImageFilename: QrCodeImageFilename);
Assert.AreEqual("https://nazuke.github.io/SEOMacroscope/", ResultText);
}
/**************************************************************************/
private void BuildWorksheetTelephoneNumbers(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 1;
int iColMax = 1;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
{
ws.Cell(iRow, iCol).Value = "Telephone Number";
iCol++;
ws.Cell(iRow, iCol).Value = "URL";
}
iColMax = iCol;
iRow++;
foreach (MacroscopeDocument msDoc in DocCollection.IterateDocuments())
{
if (msDoc.IsDocumentType(Type: MacroscopeConstants.DocumentType.HTML))
{
Dictionary <string, string> TelephoneNumbers = msDoc.GetTelephoneNumbers();
foreach (string TelephoneNumber in TelephoneNumbers.Keys)
{
iCol = 1;
this.InsertAndFormatContentCell(ws, iRow, iCol, TelephoneNumber);
iCol++;
this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc);
iRow++;
}
}
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
/**************************************************************************/
public void WriteCsv(
MacroscopeJobMaster JobMaster,
MacroscopeCsvUriReport.OutputWorksheet SelectedOutputWorksheet,
string OutputFilename
)
{
try
{
using (StreamWriter writer = File.CreateText(OutputFilename))
{
CsvWriter ws = new CsvWriter(writer);
switch (SelectedOutputWorksheet)
{
case MacroscopeCsvUriReport.OutputWorksheet.LINKS:
this.BuildWorksheetPageLinks(JobMaster, ws);
break;
case MacroscopeCsvUriReport.OutputWorksheet.HYPERLINKS:
this.BuildWorksheetPageHyperlinks(JobMaster, ws);
break;
case MacroscopeCsvUriReport.OutputWorksheet.URIS:
this.BuildWorksheetPageUriAnalysis(JobMaster, ws);
break;
case MacroscopeCsvUriReport.OutputWorksheet.ORPHANS:
this.BuildWorksheetPageOrphanedPages(JobMaster, ws);
break;
default:
break;
}
}
}
catch (CsvHelperException)
{
MacroscopeSaveCsvFileException CannotSaveCsvFileException;
CannotSaveCsvFileException = new MacroscopeSaveCsvFileException(
string.Format("Cannot write to CSV file at {0}", OutputFilename)
);
throw CannotSaveCsvFileException;
}
catch (IOException)
{
MacroscopeSaveCsvFileException CannotSaveCsvFileException;
CannotSaveCsvFileException = new MacroscopeSaveCsvFileException(
string.Format("Cannot write to CSV file at {0}", OutputFilename)
);
throw CannotSaveCsvFileException;
}
}
/**************************************************************************/
private void BuildWorksheetPageRedirectsAudit(
MacroscopeJobMaster JobMaster,
CsvWriter ws
)
{
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
{
ws.WriteField("Origin URL");
ws.WriteField("Status Code");
ws.WriteField("Status");
ws.WriteField("Destination URL");
ws.NextRecord();
}
foreach (string Url in DocCollection.DocumentKeys())
{
MacroscopeDocument msDoc = DocCollection.GetDocument(Url: Url);
if (!msDoc.GetIsRedirect())
{
continue;
}
string OriginURL = msDoc.GetUrlRedirectFrom();
string StatusCode = (( int )msDoc.GetStatusCode()).ToString();
string Status = msDoc.GetStatusCode().ToString();
string DestinationURL = msDoc.GetUrlRedirectTo();
if (string.IsNullOrEmpty(OriginURL))
{
continue;
}
if (string.IsNullOrEmpty(DestinationURL))
{
continue;
}
this.InsertAndFormatUrlCell(ws, OriginURL);
this.InsertAndFormatContentCell(ws, StatusCode);
this.InsertAndFormatContentCell(ws, Status);
this.InsertAndFormatUrlCell(ws, DestinationURL);
ws.NextRecord();
}
}
/**************************************************************************/
private void BuildWorksheetEmailAddresses(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 1;
int iColMax = 1;
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
{
ws.Cell(iRow, iCol).Value = "Email Address";
iCol++;
ws.Cell(iRow, iCol).Value = "URL";
}
iColMax = iCol;
iRow++;
foreach (MacroscopeDocument msDoc in DocCollection.IterateDocuments())
{
if (msDoc.GetIsHtml())
{
Dictionary <string, string> EmailAddresses = msDoc.GetEmailAddresses();
foreach (string EmailAddress in EmailAddresses.Keys)
{
iCol = 1;
this.InsertAndFormatContentCell(ws, iRow, iCol, EmailAddress);
iCol++;
this.InsertAndFormatUrlCell(ws, iRow, iCol, msDoc);
iRow++;
}
}
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
public void TestWriteXslx()
{
MacroscopeJobMaster JobMaster = new MacroscopeJobMaster(MacroscopeConstants.RunTimeMode.LIVE);
MacroscopeDataExtractorCssSelectors DataExtractorCssSelectors = new MacroscopeDataExtractorCssSelectors(1);
MacroscopeDataExtractorRegexes DataExtractorRegexes = new MacroscopeDataExtractorRegexes(1);
MacroscopeDataExtractorXpaths DataExtractorXpaths = new MacroscopeDataExtractorXpaths(1);
MacroscopeExcelDataExtractorReport Report = new MacroscopeExcelDataExtractorReport(NewDataExtractorCssSelectors: DataExtractorCssSelectors, NewDataExtractorRegexes: DataExtractorRegexes, NewDataExtractorXpaths: DataExtractorXpaths);
string Filename = string.Join(".", Path.GetTempFileName(), "xlsx");
Report.WriteXslx(JobMaster: JobMaster, OutputFilename: Filename);
Assert.IsTrue(File.Exists(Filename));
File.Delete(Filename);
}
/**************************************************************************/
private void BuildWorksheetListView(
MacroscopeJobMaster JobMaster,
XLWorkbook wb,
string WorksheetLabel
)
{
var ws = wb.Worksheets.Add(WorksheetLabel);
int iRow = 1;
int iCol = 0;
int iColMax = 1;
for (int i = 0; i < this.TargetListView.Columns.Count; i++)
{
string ColumnName = this.TargetListView.Columns[i].Text;
iCol++;
ws.Cell(iRow, iCol).Value = ColumnName;
}
for (int i = 1; i <= iCol; i++)
{
ws.Cell(iRow, i).Style.Font.SetBold();
}
iColMax = iCol;
iRow++;
for (int j = 0; j < this.TargetListView.Items.Count; j++)
{
iCol = 0;
for (int k = 0; k < this.TargetListView.Items[j].SubItems.Count; k++)
{
string CellValue = this.TargetListView.Items[j].SubItems[k].Text;
iCol++;
this.InsertAndFormatContentCell(ws, iRow, iCol, CellValue);
}
iRow++;
}
{
var rangeData = ws.Range(1, 1, iRow - 1, iColMax);
var excelTable = rangeData.CreateTable();
}
}
/**************************************************************************/
private void BuildWorksheetPageGoodLinks(
MacroscopeJobMaster JobMaster,
CsvWriter ws
)
{
MacroscopeDocumentCollection DocCollection = JobMaster.GetDocCollection();
MacroscopeAllowedHosts AllowedHosts = JobMaster.GetAllowedHosts();
{
ws.WriteField("Status Code");
ws.WriteField("Status");
ws.WriteField("Origin URL");
ws.WriteField("Destination URL");
ws.NextRecord();
}
foreach (string Url in DocCollection.DocumentKeys())
{
MacroscopeDocument msDoc = DocCollection.GetDocument(Url);
MacroscopeHyperlinksIn HyperlinksIn = DocCollection.GetDocumentHyperlinksIn(Url);
int StatusCode = ( int )msDoc.GetStatusCode();
string Status = msDoc.GetStatusCode().ToString();
if (
(StatusCode >= 200) &&
(StatusCode <= 299) &&
(HyperlinksIn != null))
{
foreach (MacroscopeHyperlinkIn HyperlinkIn in HyperlinksIn.IterateLinks())
{
string OriginUrl = HyperlinkIn.GetSourceUrl();
if (
(OriginUrl != null) &&
(OriginUrl.Length > 0))
{
this.InsertAndFormatContentCell(ws, StatusCode.ToString());
this.InsertAndFormatContentCell(ws, Status);
this.InsertAndFormatUrlCell(ws, OriginUrl);
this.InsertAndFormatUrlCell(ws, msDoc);
ws.NextRecord();
}
}
}
}
}
/**************************************************************************/
public MacroscopeJobMaster Load(string Pathname)
{
MacroscopeJobMaster JobMaster = null;
if (File.Exists(Pathname))
{
Stream openFileStream = File.OpenRead(Pathname);
BinaryFormatter deserializer = new BinaryFormatter();
JobMaster = (MacroscopeJobMaster)deserializer.Deserialize(openFileStream);
openFileStream.Close();
}
return(JobMaster);
}
public void TestJobMasterStartUrl()
{
MacroscopeJobMaster JobMaster;
const string StartUrl = "http://www.companyname.com/path/to/some/deep/folder/index.html";
JobMaster = new MacroscopeJobMaster(
JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
TaskController: this
);
JobMaster.SetStartUrl(Url: StartUrl);
Assert.AreEqual(StartUrl, JobMaster.GetStartUrl(), string.Format("FAIL: {0}", StartUrl));
}
/**************************************************************************/
public void WriteCsv(
MacroscopeJobMaster JobMaster,
MacroscopeCsvPageMetadataReport.OutputWorksheet SelectedOutputWorksheet,
string OutputFilename
)
{
try
{
using (StreamWriter writer = File.CreateText(OutputFilename))
{
CsvWriter ws = new CsvWriter(writer);
switch (SelectedOutputWorksheet)
{
case MacroscopeCsvPageMetadataReport.OutputWorksheet.TITLES:
this.BuildWorksheetPageTitles(JobMaster, ws);
break;
case MacroscopeCsvPageMetadataReport.OutputWorksheet.DESCRIPTIONS:
this.BuildWorksheetPageDescriptions(JobMaster, ws);
break;
case MacroscopeCsvPageMetadataReport.OutputWorksheet.KEYWORDS:
this.BuildWorksheetPageKeywords(JobMaster, ws);
break;
default:
this.BuildWorksheetPageTitles(JobMaster, ws);
break;
}
}
}
catch (CsvWriterException)
{
MacroscopeSaveCsvFileException CannotSaveCsvFileException;
CannotSaveCsvFileException = new MacroscopeSaveCsvFileException(
string.Format("Cannot write to CSV file at {0}", OutputFilename)
);
throw CannotSaveCsvFileException;
}
catch (IOException)
{
MacroscopeSaveCsvFileException CannotSaveCsvFileException;
CannotSaveCsvFileException = new MacroscopeSaveCsvFileException(
string.Format("Cannot write to CSV file at {0}", OutputFilename)
);
throw CannotSaveCsvFileException;
}
}
public async Task TestNHeadRequests()
{
MacroscopeJobMaster JobMaster = new MacroscopeJobMaster(JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE, TaskController: this);
MacroscopeDocumentCollection DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);
Assert.AreEqual(0, DocCollection.CountDocuments());
foreach (string Url in this.Urls)
{
MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: Url);
await msDoc.ExecuteHeadRequest();
}
Assert.AreEqual(this.MaxUrls, DocCollection.CountDocuments());
}
public void TestWriteCsv ()
{
MacroscopeJobMaster JobMaster = new MacroscopeJobMaster( MacroscopeConstants.RunTimeMode.LIVE );
MacroscopeCsvContactDetailsReport ReportFile = new MacroscopeCsvContactDetailsReport();
List<MacroscopeCsvContactDetailsReport.OutputWorksheet> SelectedOutputWorksheets = new List<MacroscopeCsvContactDetailsReport.OutputWorksheet>();
SelectedOutputWorksheets.Add( MacroscopeCsvContactDetailsReport.OutputWorksheet.EMAIL );
SelectedOutputWorksheets.Add( MacroscopeCsvContactDetailsReport.OutputWorksheet.TELEPHONE );
foreach ( MacroscopeCsvContactDetailsReport.OutputWorksheet SelectedOutputWorksheet in SelectedOutputWorksheets )
{
string Filename = string.Join( ".", Path.GetTempFileName(), ".csv" );
ReportFile.WriteCsv( JobMaster: JobMaster, SelectedOutputWorksheet: SelectedOutputWorksheet, OutputFilename: Filename );
Assert.IsTrue( File.Exists( Filename ) );
File.Delete( Filename );
}
}
public async Task TestDetectLanguage()
{
MacroscopeJobMaster JobMaster;
MacroscopeDocumentCollection DocCollection;
List <string> UrlList = new List <string>();
UrlList.Add("https://nazuke.github.io/SEOMacroscope/");
MacroscopePreferencesManager.SetDefaultValues();
MacroscopePreferencesManager.SetDetectLanguage(Enabled: true);
MacroscopePreferencesManager.SetRequestTimeout(Seconds: 10);
JobMaster = new MacroscopeJobMaster(
JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
TaskController: this
);
DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);
for (int i = 0; i < 10; i++)
{
foreach (string Url in UrlList)
{
MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: Url);
Assert.IsNotNull(msDoc, string.Format("FAIL: {0}", Url));
bool ExecuteResult = await msDoc.Execute();
Assert.IsTrue(ExecuteResult, string.Format("FAIL: {0}", "Execute()"));
Assert.IsTrue(msDoc.IsDocumentType(Type: MacroscopeConstants.DocumentType.HTML), string.Format("FAIL: {0}", Url));
Assert.IsNotNull(msDoc.GetTitle(), string.Format("FAIL: {0}", msDoc.GetTitle()));
Assert.IsNotEmpty(msDoc.GetTitle(), string.Format("FAIL: {0}", msDoc.GetTitle()));
string LanguageTitle = msDoc.GetTitleLanguage();
string LanguageDescription = msDoc.GetDescriptionLanguage();
string LanguageBodyText = msDoc.GetDocumentTextLanguage();
Assert.AreEqual("en", LanguageTitle, string.Format("FAIL: {0} :: {1}", "LanguageTitle", LanguageTitle));
Assert.AreEqual("en", LanguageDescription, string.Format("FAIL: {0} :: {1}", "LanguageDescription", LanguageDescription));
Assert.AreEqual("en", LanguageBodyText, string.Format("FAIL: {0} :: {1}", "LanguageBodyText", LanguageBodyText));
}
}
}
public async Task TestDocumentCookiesAreSet()
{
MacroscopeJobMaster JobMaster;
MacroscopeDocumentCollection DocCollection;
List <string> UrlList = new List <string>(2);
UrlList.Add("https://httpbin.org/cookies/set?first=bongo&second=bongobongo");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
UrlList.Add("https://httpbin.org/cookies");
JobMaster = new MacroscopeJobMaster(
JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
TaskController: this
);
DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);
{ // Set Cookies
MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: UrlList[0]);
bool ExecuteResult = await msDoc.Execute();
//Assert.IsTrue( ExecuteResult, string.Format( "FAIL: {0}", "Execute()" ) );
}
// Get Cookies
for (int i = 1; i < UrlList.Count; i++)
{
MacroscopeDocument msDoc = DocCollection.CreateDocument(Url: UrlList[i]);
bool ExecuteResult = await msDoc.Execute();
Assert.IsTrue(ExecuteResult, string.Format("FAIL: {0}", "Execute()"));
}
}
/**************************************************************************/
public void WriteCsv(
MacroscopeJobMaster JobMaster,
MacroscopeCsvBrokenLinksReport.OutputWorksheet SelectedOutputWorksheet,
string OutputFilename
)
{
try
{
using (StreamWriter writer = File.CreateText(OutputFilename))
{
CsvWriter ws = new CsvWriter(writer);
switch (SelectedOutputWorksheet)
{
case MacroscopeCsvBrokenLinksReport.OutputWorksheet.BROKEN_LINKS:
this.BuildWorksheetPageBrokenLinks(JobMaster, ws);
break;
case MacroscopeCsvBrokenLinksReport.OutputWorksheet.GOOD_LINKS:
this.BuildWorksheetPageGoodLinks(JobMaster, ws);
break;
case MacroscopeCsvBrokenLinksReport.OutputWorksheet.REDIRECTED_LINKS:
this.BuildWorksheetPageRedirectedLinks(JobMaster, ws);
break;
default:
break;
}
}
}
catch (CsvWriterException)
{
MacroscopeSaveCsvFileException CannotSaveCsvFileException;
CannotSaveCsvFileException = new MacroscopeSaveCsvFileException(
string.Format("Cannot write to CSV file at {0}", OutputFilename)
);
throw CannotSaveCsvFileException;
}
catch (IOException)
{
MacroscopeSaveCsvFileException CannotSaveCsvFileException;
CannotSaveCsvFileException = new MacroscopeSaveCsvFileException(
string.Format("Cannot write to CSV file at {0}", OutputFilename)
);
throw CannotSaveCsvFileException;
}
}
/**************************************************************************/
public void WriteCsv(
MacroscopeJobMaster JobMaster,
MacroscopeCsvDataExtractorReport.OutputWorksheet SelectedOutputWorksheet,
string OutputFilename
)
{
try
{
using (StreamWriter writer = File.CreateText(OutputFilename))
{
CsvWriter ws = new CsvWriter(writer);
switch (SelectedOutputWorksheet)
{
case MacroscopeCsvDataExtractorReport.OutputWorksheet.CSS_SELECTORS:
this.BuildWorksheetCssSelectors(JobMaster, ws);
break;
case MacroscopeCsvDataExtractorReport.OutputWorksheet.REGEXES:
this.BuildWorksheetRegularExpressions(JobMaster, ws);
break;
case MacroscopeCsvDataExtractorReport.OutputWorksheet.XPATHS:
this.BuildWorksheetXpaths(JobMaster, ws);
break;
default:
break;
}
}
}
catch (CsvHelperException)
{
MacroscopeSaveCsvFileException CannotSaveCsvFileException;
CannotSaveCsvFileException = new MacroscopeSaveCsvFileException(
string.Format("Cannot write to CSV file at {0}", OutputFilename)
);
throw CannotSaveCsvFileException;
}
catch (IOException)
{
MacroscopeSaveCsvFileException CannotSaveCsvFileException;
CannotSaveCsvFileException = new MacroscopeSaveCsvFileException(
string.Format("Cannot write to CSV file at {0}", OutputFilename)
);
throw CannotSaveCsvFileException;
}
}
/**************************************************************************/
public async Task <bool> IsPdfUrl(MacroscopeJobMaster JobMaster, string Url)
{
bool Result = false;
Uri TargetUri = new Uri(Url);
string MimeType = await MacroscopeHttpUrlUtils.GetMimeTypeOfUrl(JobMaster : JobMaster, TargetUri : TargetUri);
if (!string.IsNullOrEmpty(MimeType))
{
if (Regex.IsMatch(MimeType, "^application/pdf$", RegexOptions.IgnoreCase))
{
Result = true;
}
}
return(Result);
}
public void TestDuplicate()
{
const string StartUrl = "https://nazuke.github.io/SEOMacroscope/";
const string DupeUrl = "https://nazuke.github.io/SEOMacroscope/index.html";
MacroscopeJobMaster JobMaster = new MacroscopeJobMaster(
JobRunTimeMode: MacroscopeConstants.RunTimeMode.LIVE,
TaskController: this
);
MacroscopeDocumentCollection DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);
Dictionary <string, Boolean> CrossCheckList = MacroscopeLevenshteinAnalysis.GetCrossCheckList(Capacity: DocCollection.CountDocuments());
MacroscopeDocument msDoc = DocCollection.CreateDocument(StartUrl);
MacroscopeDocument msDocDifferent = DocCollection.CreateDocument(DupeUrl);
msDoc.Execute();
msDocDifferent.Execute();
DocCollection.AddDocument(msDoc);
DocCollection.AddDocument(msDocDifferent);
DebugMsg(string.Format("msDoc: {0}", msDoc.GetStatusCode()));
DebugMsg(string.Format("msDocDifferent: {0}", msDocDifferent.GetStatusCode()));
for (int i = 1; i <= 100; i++)
{
MacroscopeLevenshteinAnalysis LevenshteinAnalysis = new MacroscopeLevenshteinAnalysis(
msDoc: msDoc,
SizeDifference: 64,
Threshold: 16,
CrossCheckList: CrossCheckList
);
Dictionary <MacroscopeDocument, int> DocList = LevenshteinAnalysis.AnalyzeDocCollection(DocCollection: DocCollection);
DebugMsg(string.Format("DocList: {0}", DocList.Count));
foreach (MacroscopeDocument msDocAnalyzed in DocList.Keys)
{
DebugMsg(string.Format("msDocAnalyzed: {0} => {1}", DocList[msDocAnalyzed], msDocAnalyzed.GetUrl()));
Assert.AreEqual(DocList[msDocAnalyzed], 0, string.Format("FAIL: {0} => {1}", DocList[msDocAnalyzed], msDocAnalyzed.GetUrl()));
}
}
}
public void TestWriteCsv()
{
MacroscopeJobMaster JobMaster = new MacroscopeJobMaster(MacroscopeConstants.RunTimeMode.LIVE);
MacroscopeCsvBrokenLinksReport ReportFile = new MacroscopeCsvBrokenLinksReport();
List <MacroscopeCsvBrokenLinksReport.OutputWorksheet> SelectedOutputWorksheets = new List <MacroscopeCsvBrokenLinksReport.OutputWorksheet>();
SelectedOutputWorksheets.Add(MacroscopeCsvBrokenLinksReport.OutputWorksheet.BROKEN_LINKS);
SelectedOutputWorksheets.Add(MacroscopeCsvBrokenLinksReport.OutputWorksheet.GOOD_LINKS);
SelectedOutputWorksheets.Add(MacroscopeCsvBrokenLinksReport.OutputWorksheet.REDIRECTED_LINKS);
foreach (MacroscopeCsvBrokenLinksReport.OutputWorksheet SelectedOutputWorksheet in SelectedOutputWorksheets)
{
string Filename = string.Join(".", Path.GetTempFileName(), ".csv");
ReportFile.WriteCsv(JobMaster: JobMaster, SelectedOutputWorksheet: SelectedOutputWorksheet, OutputFilename: Filename);
Assert.IsTrue(File.Exists(Filename));
File.Delete(Filename);
}
}
public void TestWriteSitemapXml()
{
foreach (string Url in this.Urls)
{
MacroscopeJobMaster JobMaster = new MacroscopeJobMaster(MacroscopeConstants.RunTimeMode.LIVE);
MacroscopeDocumentCollection DocCollection = new MacroscopeDocumentCollection(JobMaster: JobMaster);
MacroscopeSitemapGenerator SitemapGenerator = new MacroscopeSitemapGenerator(NewDocCollection: DocCollection);
DocCollection.AddDocument(new MacroscopeDocument(JobMaster.SetStartUrl(Url: Url)));
string Filename = string.Join(".", Path.GetTempFileName(), ".xml");
SitemapGenerator.WriteSitemapXml(NewPath: Filename);
Assert.IsTrue(File.Exists(Filename));
if (File.Exists(Filename))
{
File.Delete(Filename);
}
}
}
public void TestWriteCsv()
{
MacroscopeJobMaster JobMaster = new MacroscopeJobMaster(MacroscopeConstants.RunTimeMode.LIVE);
MacroscopeCsvPageMetadataReport ReportFile = new MacroscopeCsvPageMetadataReport();
List <MacroscopeCsvPageMetadataReport.OutputWorksheet> SelectedOutputWorksheets = new List <MacroscopeCsvPageMetadataReport.OutputWorksheet>();
SelectedOutputWorksheets.Add(MacroscopeCsvPageMetadataReport.OutputWorksheet.DESCRIPTIONS);
SelectedOutputWorksheets.Add(MacroscopeCsvPageMetadataReport.OutputWorksheet.KEYWORDS);
SelectedOutputWorksheets.Add(MacroscopeCsvPageMetadataReport.OutputWorksheet.TITLES);
foreach (MacroscopeCsvPageMetadataReport.OutputWorksheet SelectedOutputWorksheet in SelectedOutputWorksheets)
{
string Filename = string.Join(".", Path.GetTempFileName(), ".csv");
ReportFile.WriteCsv(JobMaster: JobMaster, SelectedOutputWorksheet: SelectedOutputWorksheet, OutputFilename: Filename);
Assert.IsTrue(File.Exists(Filename));
File.Delete(Filename);
}
}
/**************************************************************************/
public void WriteXslx(MacroscopeJobMaster JobMaster, string OutputFilename)
{
XLWorkbook Workbook = new XLWorkbook();
Dictionary <string, int> DicTerms = null;
if (this.ProgressForm != null)
{
const decimal MajorPercentageDivider = 4;
for (int i = 0; i <= 3; i++)
{
if (!this.ProgressForm.Cancelled())
{
this.ProgressForm.UpdatePercentages(
Title: "Processing Keywords",
Message: "Processing keyword terms collection:",
MajorPercentage: ((decimal)100 / MajorPercentageDivider) * (decimal)i + 1,
ProgressLabelMajor: "",
MinorPercentage: 0,
ProgressLabelMinor: ""
);
DicTerms = JobMaster.GetDocCollection().GetDeepKeywordAnalysisAsDictonary(Words: i + 1);
this.BuildWorksheetKeywordTerms(JobMaster, Workbook, string.Format("{0} Word Term", i + 1), DicTerms);
}
}
this.BuildWorksheetKeywordsPresence(JobMaster, Workbook, "Keywords Presence", JobMaster.GetDocCollection());
if (!this.ProgressForm.Cancelled())
{
this.SaveOutputFile(Workbook: Workbook, OutputFilename: OutputFilename);
}
}
else
{
for (int i = 0; i <= 3; i++)
{
DicTerms = JobMaster.GetDocCollection().GetDeepKeywordAnalysisAsDictonary(Words: i + 1);
this.BuildWorksheetKeywordTerms(JobMaster, Workbook, string.Format("{0} Word Term", i + 1), DicTerms);
}
this.SaveOutputFile(Workbook: Workbook, OutputFilename: OutputFilename);
}
}
/** Process Data Extractors ***********************************************/
private void ProcessGenericDataExtractors(
string GenericText
)
{
MacroscopeJobMaster JobMaster = this.DocCollection.GetJobMaster();
{
MacroscopeDataExtractorRegexes DataExtractor = JobMaster.GetDataExtractorRegexes();
if ((DataExtractor != null) && (DataExtractor.IsEnabled()))
{
this.ProcessGenericDataExtractorRegexes(
DataExtractor: DataExtractor,
GenericText: GenericText
);
}
}
}
/** Sessions Save Dialogue Boxes ******************************************/
private void CallbackSaveSessionToFile(object sender, EventArgs e)
{
SaveFileDialog Dialog = new SaveFileDialog();
Dialog.Filter = "SEO Macroscope session files (*.seomacroscope)|*.seomacroscope";
Dialog.FilterIndex = 2;
Dialog.RestoreDirectory = true;
Dialog.DefaultExt = "seomacroscope";
Dialog.AddExtension = true;
Dialog.FileName = "Session.seomacroscope";
if (Dialog.ShowDialog() == DialogResult.OK)
{
string Pathname = Dialog.FileName;
Cursor.Current = Cursors.WaitCursor;
try
{
MacroscopeSessionSaver SessionSaver = new MacroscopeSessionSaver();
MacroscopeJobMaster JobMaster = this.GetJobMaster();
SessionSaver.Save(JobMaster: JobMaster, Pathname: Pathname);
this.DialogueBoxFeedback("SEO Macroscope session", "The SEO Macroscope session file was saved successfully.");
}
catch (Exception ex)
{
this.DialogueBoxError("Error", ex.Message);
}
finally
{
Cursor.Current = Cursors.Default;
}
}
else
{
this.DialogueBoxError("Error", "Could not open SEO Macroscope session file.");
}
if (Dialog != null)
{
Dialog.Dispose();
}
}
/** -------------------------------------------------------------------- **/
private async Task <byte[]> _LoadMemoryStreamFromUrl(MacroscopeJobMaster JobMaster, Uri TargetUri)
{
MacroscopeHttpTwoClient Client = JobMaster.GetHttpClient();
MacroscopeHttpTwoClientResponse Response = null;
byte[] ByteData = null;
try
{
Response = await Client.Get(
TargetUri,
this.ConfigureHeadRequestHeadersCallback,
this.PostProcessRequestHttpHeadersCallback
);
}
catch (MacroscopeDocumentException ex)
{
this.DebugMsg(string.Format("MacroscopeDocumentException: {0}", ex.Message));
this.DebugMsg(string.Format("MacroscopeDocumentException: {0}", TargetUri.ToString()));
}
catch (Exception ex)
{
this.DebugMsg(string.Format("Exception: {0}", ex.Message));
this.DebugMsg(string.Format("Exception: {0}", TargetUri.ToString()));
}
if (Response != null)
{
try
{
ByteData = Response.GetContentAsBytes();
}
catch (Exception ex)
{
this.DebugMsg(string.Format("Exception: {0}", ex.Message));
}
}
else
{
this.DebugMsg("NULL");
}
return(ByteData);
}
