internal static bool DownloadAndSaveImage(XmlNode xmlNode)
{
var fileurl = string.Format("{0}{1}_1920x1080.jpg", Url, xmlNode.SelectSingleNode("urlBase").InnerText);
if (UrlsRetrieved.Contains(fileurl))
{
ConsoleWriter.WriteLine(2, "Already Dowloaded Image URL");
return(false);
}
var filePath = Path.Combine(Program.SavePath, GetFileName(xmlNode));
var tempfilename = Path.Combine(DownloadPath, Guid.NewGuid() + ".jpg");
var fileWebRequest = WebRequest.Create(fileurl);
using (var fileWebResponse = fileWebRequest.GetResponse())
{
using (var tempStream = File.Create(tempfilename))
{
var buffer = new byte[1024];
using (var fileStream = fileWebResponse.GetResponseStream())
{
int bytesRead;
do
{
bytesRead = fileStream.Read(buffer, 0, buffer.Length);
tempStream.Write(buffer, 0, bytesRead);
} while (bytesRead > 0);
}
}
}
ConsoleWriter.WriteLine(2, "Downloaded Image, Checking If Duplicate");
var newImage = false;
if (!ImageHashing.ImageInHash(tempfilename) && !File.Exists(filePath))
{
newImage = true;
ConsoleWriter.WriteLine(3, "Found New Image");
using (var srcImg = Image.FromFile(tempfilename))
{
ImagePropertyHandling.SetTitleOnImage(xmlNode, srcImg);
srcImg.Save(filePath);
}
ImageHashing.AddHash(filePath);
}
else
{
ConsoleWriter.WriteLine(3, "Identical Image Downloaded");
}
UrlsRetrieved.Add(fileurl);
File.Delete(tempfilename);
return(newImage);
}
internal static void Startup()
{
if (!Directory.Exists(Program.SavePath))
{
Directory.CreateDirectory(Program.SavePath);
}
if (!Directory.Exists(Program.AppData))
{
Directory.CreateDirectory(Program.AppData);
}
if (!Directory.Exists(BingInteractionAndParsing.DownloadPath))
{
Directory.CreateDirectory(BingInteractionAndParsing.DownloadPath);
}
if (!Directory.Exists(ImageHashing.HitogramPath))
{
Directory.CreateDirectory(ImageHashing.HitogramPath);
}
var logPath = Path.Combine(Program.SavePath, "Logs");
if (!Directory.Exists(logPath))
{
Directory.CreateDirectory(logPath);
}
ConsoleWriter.SetupLogWriter(Path.Combine(logPath, String.Format("Log-{0}.txt", DateTime.UtcNow.ToString("yyyy-MM-dd"))));
BingInteractionAndParsing.UrlsRetrieved.AddRange(Serializer.Deserialize <List <string> >(Path.Combine(Program.AppData, "urlsRetrieved.bin")));
BingInteractionAndParsing.Countries.AddRange(CultureInfo.GetCultures(CultureTypes.AllCultures).Where(x => x.Name.Contains("-") && x.Name.Length == 5));
ImageHashing.HistogramHashTable.AddRange(Serializer.Deserialize <List <HistogramHash> >(Path.Combine(Program.AppData, "imageHistogram.bin")));
ConsoleWriter.WriteLine("Have loaded {0} previous URLs", BingInteractionAndParsing.UrlsRetrieved.Count);
ConsoleWriter.WriteLine("Have loaded {0} countries", BingInteractionAndParsing.Countries.Count);
ConsoleWriter.WriteLine("Have loaded {0} previous hashes", ImageHashing.HistogramHashTable.Count);
ArchiveOldImages();
ImageHashing.ClearHash();
HashExistingImages();
if (ImageHashing.HistogramHashTable.Any())
{
Serializer.Serialize(ImageHashing.HistogramHashTable, Path.Combine(Program.AppData, "imageHistogram.bin"));
}
ClearLogFiles(logPath);
}
private static void HashExistingImages(int retryCount = 0)
{
try
{
foreach (var file in Directory.GetFiles(Program.SavePath, "*.jpg").Where(x => !ImageHashing.HaveFilePathInHashTable(x)))
{
ConsoleWriter.WriteLine("Hashing file: {0}", file);
ImageHashing.AddHash(file);
}
var preventArchiveDupes = bool.Parse(ConfigurationManager.AppSettings["PreventDuplicatesInArchive"]);
if (preventArchiveDupes)
{
var archivePath = Path.Combine(Program.SavePath, "Archive");
if (!Directory.Exists(archivePath))
{
Directory.CreateDirectory(archivePath);
}
foreach (var file in Directory.GetFiles(archivePath, "*.jpg").Where(x => !ImageHashing.HaveFilePathInHashTable(x)))
{
ConsoleWriter.WriteLine("Hashing file: {0}", file);
ImageHashing.AddHash(file);
}
}
}
catch (Exception)
{
if (retryCount < 5)
{
HashExistingImages(retryCount + 1);
}
else
{
throw;
}
}
ConsoleWriter.WriteLine("Now have {0} hashed images", ImageHashing.HistogramHashTable.Count);
}
