protected virtual async Task <bool> DownloadBinaryPost(TumblrPost downloadItem)
{
string url = Url(downloadItem);
if (!CheckIfFileExistsInDB(url))
{
string blogDownloadLocation = blog.DownloadLocation();
string fileName = FileName(downloadItem);
string fileLocation = FileLocation(blogDownloadLocation, fileName);
string fileLocationUrlList = FileLocationLocalized(blogDownloadLocation, downloadItem.TextFileLocation);
DateTime postDate = PostDate(downloadItem);
UpdateProgressQueueInformation(Resources.ProgressDownloadImage, fileName);
if (await DownloadBinaryFile(fileLocation, fileLocationUrlList, url))
{
SetFileDate(fileLocation, postDate);
UpdateBlogDB(downloadItem.DbType, fileName);
//TODO: Refactor
if (shellService.Settings.EnablePreview)
{
if (suffixes.Any(suffix => fileName.EndsWith(suffix)))
{
blog.LastDownloadedPhoto = Path.GetFullPath(fileLocation);
}
else
{
blog.LastDownloadedVideo = Path.GetFullPath(fileLocation);
}
}
return(true);
}
return(false);
}
return(true);
}
protected override bool CheckIfFileExistsInDirectory(string url)
{
string fileName = url.Split('/').Last();
Monitor.Enter(lockObjectDirectory);
string blogPath = blog.DownloadLocation();
if (Directory.EnumerateFiles(blogPath).Any(file => file.Contains(fileName)))
{
Monitor.Exit(lockObjectDirectory);
return(true);
}
Monitor.Exit(lockObjectDirectory);
return(false);
}
public void EnsureUniqueFolder(IBlog blog)
{
int number = 1;
string appendix = "";
while (BlogFiles.Any(b => b.DownloadLocation() == blog.DownloadLocation() + appendix) || Directory.Exists(blog.DownloadLocation() + appendix))
{
number++;
appendix = $"_{number}";
}
if (number != 1)
{
blog.FileDownloadLocation = blog.DownloadLocation() + appendix;
}
Directory.CreateDirectory(blog.DownloadLocation());
}
//public void UpdateBlogDB(string fileName)
//{
// lock (_lockObjectDb)
// {
// _files.Links.Add(fileName);
// }
//}
public bool CreateDataFolder()
{
if (string.IsNullOrEmpty(_blog.Name))
{
return(false);
}
var blogPath = _blog.DownloadLocation();
if (!Directory.Exists(blogPath))
{
Directory.CreateDirectory(blogPath);
}
return(true);
}
private async Task DownloadTextPostAsync(TumblrCrawlerData <T> crawlerData)
{
string blogDownloadLocation = blog.DownloadLocation();
string fileLocation = FileLocation(blogDownloadLocation, crawlerData.Filename);
await AppendToTextFileAsync(fileLocation, crawlerData.Data);
}
private bool CheckIfLinkRestored(TumblrPost downloadItem)
{
if (!blog.ForceRescan || blog.FilenameTemplate != "%f")
{
return(false);
}
lock (diskFilesLock)
{
if (diskFiles == null)
{
diskFiles = new HashSet <string>();
foreach (var item in Directory.EnumerateFiles(blog.DownloadLocation(), "*", SearchOption.TopDirectoryOnly))
{
if (!string.Equals(Path.GetExtension(item), ".json", StringComparison.OrdinalIgnoreCase))
{
diskFiles.Add(Path.GetFileName(item).ToLower());
}
}
}
var filename = downloadItem.Url.Split('/').Last().ToLower();
return(diskFiles.Contains(filename));
}
}
protected virtual async Task <bool> DownloadBinaryPost(TumblrPost downloadItem)
{
string url = Url(downloadItem);
if (!CheckIfFileExistsInDB(url))
{
string blogDownloadLocation = blog.DownloadLocation();
string fileLocationUrlList = FileLocationLocalized(blogDownloadLocation, downloadItem.TextFileLocation);
DateTime postDate = PostDate(downloadItem);
string fileName = FileName(downloadItem);
string fileLocation = FileLocation(blogDownloadLocation, fileName);
if (url.Contains("https://mega.nz/#"))
{
Uri link = new Uri(url);
Crawler.MegaLinkType linkType = Crawler.MegaLinkType.Single;
//Determines if the MEGA link is a folder or single file based on if the url is mega.nz/#! or mega.nz/#F
Regex regType = new Regex("(http[A-Za-z0-9_/:.]*mega.nz/#(.*)([A-Za-z0-9_]*))");
foreach (Match rmatch in regType.Matches(url))
{
string subStr = rmatch.Groups[2].Value[0].ToString();
if (subStr == "!")
{
linkType = Crawler.MegaLinkType.Single;
}
if (subStr == "F")
{
linkType = Crawler.MegaLinkType.Folder;
}
}
MegaApiClient client = new MegaApiClient();
client.LoginAnonymous();
switch (linkType)
{
case Crawler.MegaLinkType.Single:
INodeInfo nodeInfo = client.GetNodeFromLink(link);
fileName = nodeInfo.Name;
fileLocation = FileLocation(blogDownloadLocation, fileName);
UpdateProgressQueueInformation(Resources.ProgressDownloadImage, fileName);
if (await DownloadBinaryFile(fileLocation, fileLocationUrlList, url))
{
updateBlog(fileLocation, postDate, downloadItem, fileName);
return(true);
}
client.Logout();
return(false);
case Crawler.MegaLinkType.Folder:
//If the link is a folder, download all files from it.
IEnumerable <INode> nodes = client.GetNodesFromLink(link);
List <INode> allFiles = nodes.Where(n => n.Type == NodeType.File).ToList();
foreach (INode node in allFiles)
{
fileName = node.Name;
fileLocation = FileLocation(blogDownloadLocation, fileName);
UpdateProgressQueueInformation(Resources.ProgressDownloadImage, fileName);
if (await DownloadBinaryFile(fileLocation, fileLocationUrlList, url, node))
{
updateBlog(fileLocation, postDate, downloadItem, fileName);
}
}
client.Logout();
return(false);
default:
throw new ArgumentOutOfRangeException();
}
}
if (url.Contains("https://drive.google.com/"))
{
UserCredential credentials = Authenticate();
DriveService service = OpenService(credentials);
RequestInfo(service, url, blogDownloadLocation + "\\");
}
UpdateProgressQueueInformation(Resources.ProgressDownloadImage, fileName);
if (await DownloadBinaryFile(fileLocation, fileLocationUrlList, url))
{
updateBlog(fileLocation, postDate, downloadItem, fileName);
return(true);
}
return(false);
}
else
{
string fileName = FileName(downloadItem);
UpdateProgressQueueInformation(Resources.ProgressSkipFile, fileName);
}
return(true);
}
protected virtual async Task <bool> DownloadPhotoAsync(TumblrPost downloadItem)
{
string url = Url(downloadItem);
if (!(files.CheckIfFileExistsInDB(url) || blog.CheckIfBlogShouldCheckDirectory(GetCoreImageUrl(url))))
{
string blogDownloadLocation = blog.DownloadLocation();
string fileName = FileName(downloadItem);
string fileLocation = FileLocation(blogDownloadLocation, fileName);
string fileLocationUrlList = FileLocationLocalized(blogDownloadLocation, Resources.FileNamePhotos);
DateTime postDate = PostDate(downloadItem);
UpdateProgressQueueInformation(Resources.ProgressDownloadImage, fileName);
if (await DownloadBinaryFile(fileLocation, fileLocationUrlList, url))
{
SetFileDate(fileLocation, postDate);
UpdateBlogDB("DownloadedPhotos", fileName);
if (shellService.Settings.EnablePreview)
{
if (!fileName.EndsWith(".gif"))
{
blog.LastDownloadedPhoto = Path.GetFullPath(fileLocation);
}
else
{
blog.LastDownloadedVideo = Path.GetFullPath(fileLocation);
}
}
return(true);
}
return(false);
}
return(true);
}