public static string Process(DownloadItem downloadItem, string cachePath)
{
var regexMatch = Regex.Match(downloadItem.Url, MatchRegex);
var internalType = regexMatch.Groups["internalType"].Value;
var idType = regexMatch.Groups["idType"].Value;
var id = regexMatch.Groups["id"].Value;
string output = string.Empty;
switch (internalType)
{
case "movieMeterHandler":
output = Tools.ThirdParty.MovieMeterApiHandler.GenerateMovieMeterXml(downloadItem, idType, id);
break;
}
if (!string.IsNullOrEmpty(output))
{
File.WriteAllText(cachePath + ".txt.tmp",output, Encoding.UTF8);
Gzip.Compress(cachePath + ".txt.tmp", cachePath + ".txt.gz");
File.Delete(cachePath + ".txt.tmp");
}
return output;
}
private void btnLog_Click(object sender, System.EventArgs e)
{
DownloadItem di = new DownloadItem();
di.CookieContainer = new CookieContainer();
di.IgnoreCache = true;
di.Type = DownloadType.Html;
di.Url = "http://www.facebook.com";
}
public static bool CheckIfDownloadItemExistsInCache(DownloadItem downloadItem, bool populateReturn)
{
var path = GetPathFromUrl(downloadItem.Url, downloadItem.Section);
var checkExists = File.Exists(path);
if (checkExists)
{
switch (downloadItem.Type)
{
case DownloadType.Html:
downloadItem.Result = new HtmlResult { Success = true, Result = File.ReadAllText(path) };
break;
case DownloadType.Binary:
downloadItem.Result = new BinaryResult { Success = true, Result = path };
break;
}
}
return checkExists;
}
/// <summary>
/// Download Server Time for TheTVDB.
/// </summary>
/// <returns>
/// The current server time. Useful for updates.
/// </returns>
private static string DownloadServerTime()
{
var downloadItem = new DownloadItem
{
Url = "http://cache.thetvdb.com/api/Updates.php?type=none",
Type = DownloadType.Html
};
var html = new Html();
html.Get(downloadItem);
string serverTimeXml =
Downloader.ProcessDownload(
string.Format(string.Format("http://cache.thetvdb.com/api/Updates.php?type=none")),
DownloadType.Html,
Section.Tv);
var doc = new XmlDocument();
doc.LoadXml(serverTimeXml);
var timeValue = XRead.GetString(doc, "Time");
return timeValue;
}
/// <summary>
/// Gets the specified download item.
/// </summary>
/// <param name="downloadItem">The download item.</param>
public void Get(DownloadItem downloadItem)
{
Log.WriteToLog(LogSeverity.Info, downloadItem.ThreadID, string.Format("Processing HTML Started"), string.Format("{0}", downloadItem.Url));
downloadItem.Result = new HtmlResult();
downloadItem.Progress.Message = "Downloading " + downloadItem.Url;
var cachePath = WebCache.GetPathFromUrl(downloadItem.Url, downloadItem.Section);
Folders.CheckExists(cachePath, true);
if (!downloadItem.IgnoreCache && !downloadItem.Url.Contains("YNoCache"))
{
if (File.Exists(cachePath + ".txt.gz"))
{
Log.WriteToLog(LogSeverity.Info, downloadItem.ThreadID, string.Format(CultureInfo.CurrentCulture, "Url Found in Cache"), string.Format(CultureInfo.CurrentCulture, "{0}", downloadItem.Url));
downloadItem.Result = new HtmlResult
{
Result = Gzip.Decompress(cachePath + ".txt.gz"),
Success = true
};
return;
}
}
try
{
Log.WriteToLog(
LogSeverity.Info,
downloadItem.ThreadID,
string.Format(CultureInfo.CurrentCulture, "Downloading"),
string.Format(CultureInfo.CurrentCulture, "{0}", downloadItem.Url));
downloadItem.Url = downloadItem.Url.Replace(" ", "+").Replace("%20", "+");
var webClient = new WebClient
{
Proxy = null
};
webClient.Headers.Add("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)");
var outputString = webClient.DownloadString(downloadItem.Url);
var encode = Encoding.GetEncoding(1252);
if (downloadItem.Url.IndexOf("ofdb", StringComparison.CurrentCultureIgnoreCase) != -1 || downloadItem.Url.Contains("sratim") || downloadItem.Url.Contains("filmweb") || downloadItem.Url.Contains("allocine"))
{
encode = Encoding.GetEncoding("UTF-8");
}
if (downloadItem.Url.Contains("filmaffinity") || downloadItem.Url.Contains("filmdelta"))
{
encode = Encoding.GetEncoding("ISO-8859-1");
}
if (downloadItem.Url.Contains("kinopoisk"))
{
encode = Encoding.GetEncoding("windows-1251");
}
Log.WriteToLog(
LogSeverity.Info,
downloadItem.ThreadID,
string.Format(CultureInfo.CurrentCulture, "Download Complete. Saving to Cache"),
string.Format(CultureInfo.CurrentCulture, "{0}", downloadItem.Url));
File.WriteAllText(cachePath + ".txt.tmp", outputString, encode);
Gzip.Compress(cachePath + ".txt.tmp", cachePath + ".txt.gz");
File.Delete(cachePath + ".txt.tmp");
Log.WriteToLog(
LogSeverity.Info,
downloadItem.ThreadID,
string.Format(CultureInfo.CurrentCulture, "Process Complete."),
string.Format(CultureInfo.CurrentCulture, "{0}", downloadItem.Url));
downloadItem.Result.Result = outputString;
downloadItem.Result.Success = true;
return;
}
catch (Exception ex)
{
Log.WriteToLog(LogSeverity.Error, LoggerName.GeneralLog, string.Format("Download Html {0}", downloadItem.Url), ex.Message);
return;
}
}
/// <summary>
/// Gets the specified download item.
/// </summary>
/// <param name="downloadItem">The download item.</param>
public static void Get(DownloadItem downloadItem)
{
downloadItem.Result = new BinaryResult();
if (downloadItem.Url == null || !WebCache.CheckIfUrlIsTooLong(downloadItem.Url))
{
return;
}
if (!downloadItem.Url.ToLower().StartsWith("http://"))
{
downloadItem.Result.Success = true;
downloadItem.Result.Result = downloadItem.Url.ToLower();
return;
}
var path = WebCache.GetPathFromUrl(downloadItem.Url, downloadItem.Section);
var pathDir = path.Replace(Path.GetFileName(path), string.Empty);
if (!Directory.Exists(pathDir))
{
Directory.CreateDirectory(pathDir);
}
if (!downloadItem.IgnoreCache && File.Exists(path))
{
downloadItem.Result.Success = true;
downloadItem.Result.Result = path;
return;
}
if (File.Exists(path))
{
//TODO System.IO.IOException: file used by another process - possible exception
File.Delete(path);
}
// the URL to download the file from
var urlToReadFileFrom = downloadItem.Url;
// the path to write the file to
var filePathToWriteFileTo = path;
downloadItem.Progress.Percent = 0;
downloadItem.Progress.Message = string.Format("Connecting to {0}", urlToReadFileFrom);
HttpWebResponse response;
HttpWebRequest request;
bool error;
var count = 0;
do
{
try
{
long runningByteTotal = 0;
var url2 = new Uri(urlToReadFileFrom);
request = (HttpWebRequest)WebRequest.Create(url2);
request.MaximumAutomaticRedirections = 4;
request.UserAgent = Settings.Get.Web.UserAgent;
request.Timeout = 20000;
request.Credentials = CredentialCache.DefaultCredentials;
request.ReadWriteTimeout = 20000;
request.Proxy = null;
request.KeepAlive = false;
if (Settings.Get.Web.ProxyUse)
{
var proxy = new WebProxy(string.Format("{0}:{1}", Settings.Get.Web.ProxyIP, Settings.Get.Web.ProxyPort));
if (!string.IsNullOrEmpty(Settings.Get.Web.ProxyUserName) && !string.IsNullOrEmpty(Settings.Get.Web.ProxyPassword))
{
proxy.Credentials = new NetworkCredential(Settings.Get.Web.ProxyUserName, Settings.Get.Web.ProxyPassword);
}
request.Proxy = proxy;
}
else
{
request.Proxy = null;
}
request.KeepAlive = false;
using (response = (HttpWebResponse)request.GetResponse())
{
var size = response.ContentLength;
var streamRemote = response.GetResponseStream();
if (File.Exists(filePathToWriteFileTo) || size == -1)
{
return;
}
using (Stream streamLocal = new FileStream(
filePathToWriteFileTo,
FileMode.Create,
FileAccess.Write,
FileShare.None))
{
var byteBuffer = new byte[size];
downloadItem.Progress.Percent = 0;
downloadItem.Progress.Message = string.Format("Downloading {0}: 0%", urlToReadFileFrom);
string speed = "";
Stopwatch sw1 = new Stopwatch();
if (streamRemote != null)
{
int byteSize;
sw1.Start();
while ((byteSize = streamRemote.Read(byteBuffer, 0, byteBuffer.Length)) > 0)
{
sw1.Stop();
// write the bytes to the file system at the file path specified
streamLocal.Write(byteBuffer, 0, byteSize);
runningByteTotal += byteSize;
// calculate the progress out of a base "100"
var index = (double)runningByteTotal;
var total = (double)byteBuffer.Length;
var progressPercentageDouble = index / total;
var progressPercentageInteger = (int)(progressPercentageDouble * 100);
// calculate the average speed of the download
if (sw1.ElapsedMilliseconds != 0)
{
var speedbps = (double)(((runningByteTotal) / ((double)sw1.ElapsedMilliseconds / 1000)));
var speedkbps = (double)(((runningByteTotal) / ((double)sw1.ElapsedMilliseconds / 1000)) / 1024);
var speedmbps = (double)(((runningByteTotal) / ((double)sw1.ElapsedMilliseconds / 1000)) / (1024 * 1024));
if (speedmbps >= 1) speed = string.Format("{0:#.##} MBytes/s", speedmbps);
else if (speedkbps >= 1) speed = string.Format("{0:#.##} KBytes/s", speedkbps);
else if (speedbps >= 1) speed = string.Format("{0:#.##} Bytes/s", speedbps);
else speed = "";
}
downloadItem.Progress.Percent = progressPercentageInteger;
downloadItem.Progress.Message = string.Format(
"Downloading {1}: {0}% : {2}",
progressPercentageInteger,
urlToReadFileFrom,
speed);
sw1.Start();
}
}
// clean up the file stream
streamLocal.Close();
}
}
error = false;
}
catch (WebException ex)
{
if (ex.Status == WebExceptionStatus.ProtocolError)
{
if (urlToReadFileFrom.IndexOf("_thumb") > -1)
{
urlToReadFileFrom = urlToReadFileFrom.Replace("_thumb.jpg", ".jpg");
}
else
{
return;
}
}
downloadItem.Progress.Message = string.Format("Restarting Attempt: {0} - {1}", count + 1, urlToReadFileFrom);
downloadItem.Progress.Percent = 0;
error = true;
count++;
}
}
while (error && count < 5);
if (error)
{
try
{
if (File.Exists(path))
{
File.Delete(path);
}
}
catch
{
return;
}
return;
}
downloadItem.Result = new BinaryResult { Success = true, Result = path };
return;
}
public static string GenerateMovieMeterXml(DownloadItem downloadItem, string idType, string value)
{
var apiProxy = (IMMApi)XmlRpcProxyGen.Create(typeof(IMMApi));
const string imdb = "imdb";
const string search = "search";
var filmDetail = new FilmDetail();
if (idType == imdb)
{
value = apiProxy.RetrieveByImdb(getSessionKey(), value);
}
else if (idType == search)
{
var filmlist = apiProxy.Search(getSessionKey(), value);
if (filmlist.Length > 0)
{
value = filmlist[0].filmId;
}
else
{
return string.Empty;
}
}
filmDetail = apiProxy.RetrieveDetails(getSessionKey(), value.ToInt());
var movieMeter = new MovieMeter();
using (var stringWriter = new StringWriterWithEncoding(movieMeter.HtmlEncoding))
{
using (XmlWriter xmlWriter = XmlWriter.Create(stringWriter))
{
xmlWriter.WriteStartElement("moviemeter");
XWrite.WriteEnclosedElement(xmlWriter, "id", filmDetail.filmId);
XWrite.WriteEnclosedElement(xmlWriter, "imdbid", filmDetail.imdb);
XWrite.WriteEnclosedElement(xmlWriter, "title", filmDetail.title);
XWrite.WriteEnclosedElement(xmlWriter, "year", filmDetail.year);
foreach (var actor in filmDetail.actors)
{
XWrite.WriteEnclosedElement(xmlWriter, "actor", actor.name);
}
foreach (var genre in filmDetail.genres)
{
XWrite.WriteEnclosedElement(xmlWriter, "genre", genre);
}
XWrite.WriteEnclosedElement(xmlWriter, "plot", filmDetail.plot);
foreach (var actor in filmDetail.directors)
{
XWrite.WriteEnclosedElement(xmlWriter, "director", actor.name);
}
XWrite.WriteEnclosedElement(xmlWriter, "releasedate", filmDetail.dates_cinema[0].date);
foreach (var country in filmDetail.countries)
{
XWrite.WriteEnclosedElement(xmlWriter, "country", country.name, "code", country.iso_3166_1);
}
XWrite.WriteEnclosedElement(xmlWriter, "countries", filmDetail.countries_text);
XWrite.WriteEnclosedElement(xmlWriter, "rating", filmDetail.average);
xmlWriter.WriteEndElement();
}
return stringWriter.ToString();
}
}
/// <summary>
/// Gets the specified download item.
/// </summary>
/// <param name="downloadItem">The download item.</param>
public void Get(DownloadItem downloadItem)
{
Log.WriteToLog(LogSeverity.Info, downloadItem.ThreadID, string.Format("Processing HTML Started"), string.Format("{0}", downloadItem.Url));
downloadItem.Result = new HtmlResult();
downloadItem.Progress.Message = "Downloading " + downloadItem.Url;
var cachePath = WebCache.GetPathFromUrl(downloadItem.Url, downloadItem.Section);
Folders.CheckExists(cachePath, true);
if (!downloadItem.IgnoreCache && !downloadItem.Url.Contains("YNoCache"))
{
if (File.Exists(cachePath + ".txt.gz"))
{
Log.WriteToLog(LogSeverity.Info, downloadItem.ThreadID, string.Format(CultureInfo.CurrentCulture, "Url Found in Cache"), string.Format(CultureInfo.CurrentCulture, "{0}", downloadItem.Url));
downloadItem.Result = new HtmlResult
{
Result = Gzip.Decompress(cachePath + ".txt.gz"),
Success = true
};
return;
}
}
if (InternalHandlers.Check(downloadItem))
{
InternalHandlers.Process(downloadItem, cachePath);
return;
}
try
{
Log.WriteToLog(
LogSeverity.Info,
downloadItem.ThreadID,
string.Format(CultureInfo.CurrentCulture, "Downloading"),
string.Format(CultureInfo.CurrentCulture, "{0}", downloadItem.Url));
downloadItem.Url = downloadItem.Url.Replace(" ", "+").Replace("%20", "+");
var webClient = new WebClient
{
Proxy = null
};
if (Settings.Get.Web.EnableProxy)
{
webClient.Proxy = new WebProxy(Settings.Get.Web.ProxyUserName, (int)Settings.Get.Web.ProxyPort);
}
webClient.Headers.Add("user-agent", Settings.Get.Web.UserAgent);
var encode = Encoding.GetEncoding(1252);
foreach (var encoding in Settings.Get.Web.WebEncodings)
{
if (downloadItem.Url.Contains(encoding.Key))
{
encode = encoding.Value;
break;
}
}
webClient.Encoding = encode;
var outputString = webClient.DownloadString(downloadItem.Url);
Log.WriteToLog(
LogSeverity.Info,
downloadItem.ThreadID,
string.Format(CultureInfo.CurrentCulture, "Download Complete. Saving to Cache"),
string.Format(CultureInfo.CurrentCulture, "{0}", downloadItem.Url));
if (encode != Encoding.UTF8)
{
var origBytes = encode.GetBytes(outputString);
var newBytes = Encoding.Convert(encode, Encoding.UTF8, origBytes);
outputString = Encoding.UTF8.GetString(newBytes);
encode = Encoding.UTF8;
}
File.WriteAllText(cachePath + ".txt.tmp", outputString, encode);
Gzip.Compress(cachePath + ".txt.tmp", cachePath + ".txt.gz");
File.Delete(cachePath + ".txt.tmp");
Log.WriteToLog(
LogSeverity.Info,
downloadItem.ThreadID,
string.Format(CultureInfo.CurrentCulture, "Process Complete."),
string.Format(CultureInfo.CurrentCulture, "{0}", downloadItem.Url));
downloadItem.Result.Result = outputString;
downloadItem.Result.Success = true;
return;
}
catch (Exception ex)
{
Log.WriteToLog(LogSeverity.Error, LoggerName.GeneralLog, string.Format("Download Html {0}", downloadItem.Url), ex.Message);
return;
}
}
/// <summary>
/// Processes the season to the background queue.
/// </summary>
/// <param name="season">
/// The season.
/// </param>
private static void ProcessSeasonToBackground(Season season)
{
if (season.ContainsEpisodesWithFiles())
{
var current = GenerateOutput.AccessCurrentIOHandler() as IoInterface;
if (current == null)
{
return;
}
string bannerPath = current.GetSeasonBanner(season);
if (string.IsNullOrEmpty(bannerPath))
{
var downloadItem = new DownloadItem
{
Url = TvDBFactory.GetImageUrl(season.BannerUrl),
Type = DownloadType.Binary,
Section = Section.Tv
};
Downloader.AddToBackgroundQue(downloadItem);
}
else
{
season.BannerPath = bannerPath;
}
string posterPath = current.GetSeasonPoster(season);
if (string.IsNullOrEmpty(posterPath))
{
var downloadItem = new DownloadItem
{
Url = TvDBFactory.GetImageUrl(season.PosterUrl),
Type = DownloadType.Binary,
Section = Section.Tv
};
Downloader.AddToBackgroundQue(downloadItem);
}
else
{
season.PosterPath = posterPath;
}
string fanartPath = current.GetSeasonFanart(season);
if (string.IsNullOrEmpty(fanartPath))
{
var downloadItem = new DownloadItem
{
Url = TvDBFactory.GetImageUrl(season.FanartUrl),
Type = DownloadType.Binary,
Section = Section.Tv
};
Downloader.AddToBackgroundQue(downloadItem);
}
else
{
season.FanartPath = fanartPath;
}
}
}
/// <summary>
/// The process episode to background.
/// </summary>
/// <param name="episode">
/// The episode.
/// </param>
private static void ProcessEpisodeToBackground(Episode episode)
{
if (!string.IsNullOrEmpty(episode.CurrentFilenameAndPath))
{
var current = GenerateOutput.AccessCurrentIOHandler() as IoInterface;
if (current == null)
{
return;
}
string path = current.GetEpisodeScreenshot(episode);
if (string.IsNullOrEmpty(path))
{
var downloadItem = new DownloadItem
{
Url = TvDBFactory.GetImageUrl(episode.EpisodeScreenshotUrl),
Type = DownloadType.Binary,
Section = Section.Tv
};
Downloader.AddToBackgroundQue(downloadItem);
}
else
{
episode.EpisodeScreenshotPath = path;
}
}
}
/// <summary>
/// Gets images from the image block
/// </summary>
/// <param name="type">The ImageUsageType type.</param>
/// <param name="threadID">The thread MovieUniqueId.</param>
/// <param name="id">The tmDB id</param>
/// <returns>An image collection</returns>
public BindingList<ImageDetailsModel> GetImages(ImageUsageType type, int threadID, string id)
{
var xml = this.GetHtml("getInfo", threadID, id);
var output = new BindingList<ImageDetailsModel>();
var xmlDoc = XDocument.Parse(xml);
var tmdbType = string.Empty;
switch (type)
{
case ImageUsageType.Poster:
tmdbType = "poster";
break;
case ImageUsageType.Fanart:
tmdbType = "backdrop";
break;
}
var images = from m in xmlDoc.Descendants("image") where m.Attribute("type").Value == tmdbType select m;
foreach (var image in images)
{
if (image.Attribute("size").Value == "mid" || image.Attribute("size").Value == "poster")
{
var origional = new Uri(image.Attribute("url").Value);
var thumb = new Uri(origional.ToString().Replace("-poster", "-thumb"));
var width = image.Attribute("width").Value.ToInt();
var height = image.Attribute("height").Value.ToInt();
var downloadItem = new DownloadItem
{
Type = DownloadType.Binary,
Url = thumb.ToString(),
Section = Section.Movies
};
Downloader.AddToBackgroundQue(downloadItem);
output.Add(
new ImageDetailsModel
{
Height = height,
Width = width,
UriFull = origional,
UriThumb = thumb
});
}
}
return output;
}
/// <summary>
/// Processes the download.
/// </summary>
/// <param name="downloadItem">
/// The download item.
/// </param>
private static void ProcessDownload(DownloadItem downloadItem)
{
if (downloadItem.Type == DownloadType.Binary)
{
Binary.Get(downloadItem);
}
else if (downloadItem.Type == DownloadType.AppleBinary)
{
AppleBinary.Get(downloadItem);
}
else
{
var html = new Html();
html.Get(downloadItem);
}
}
/// <summary>
/// Downloads the specified URL.
/// </summary>
/// <param name="url">
/// The download URL.
/// </param>
/// <param name="type">
/// The download type.
/// </param>
/// <param name="section">
/// The section.
/// </param>
/// <param name="skipCache">
/// if set to <c>true</c> [skip cache].
/// </param>
/// <param name="cookieContainer">
/// The cookie container.
/// </param>
/// <returns>
/// The process download.
/// </returns>
public static string ProcessDownload(
string url,
DownloadType type,
Section section,
bool skipCache = false,
CookieContainer cookieContainer = null)
{
bool found = false;
if (url == null)
{
return string.Empty;
}
var item = new DownloadItem
{
Type = type,
Url = url,
Section = section,
IgnoreCache = skipCache,
CookieContainer = cookieContainer
};
try
{
lock (currentQueueLock)
{
currentQueue++;
}
Downloading.Add(url);
if (InBackgroundQue(url))
{
RemoveFromBackgroundQue(url);
}
}
catch (Exception ex)
{
Log.WriteToLog(LogSeverity.Error, 0, string.Format("Thread Download Setup {0}", url), ex.Message);
}
do
{
for (int i = 0; i < numThreads; i++)
{
if (!BackgroundWorkers[i].IsBusy)
{
try
{
item.ThreadID = i+1;
item.Progress = Progress[i];
BackgroundWorkers[i].RunWorkerAsync(item);
found = true;
do
{
Application.DoEvents();
Thread.Sleep(50);
}
while (BackgroundWorkers[i].IsBusy);
if (Downloading.Contains(url))
{
Downloading.Remove(url);
}
lock (currentQueueLock)
{
currentQueue--;
}
return item.Result.Result;
}
catch (Exception ex)
{
Log.WriteToLog(LogSeverity.Error, 0, string.Format("Thread {0} Downloading {1}", i, url), ex.Message);
}
}
}
Application.DoEvents();
Thread.Sleep(50);
}
while (found == false);
lock (currentQueueLock)
{
currentQueue--;
}
return null;
}
/// <summary>
/// Add downloaditem to background que
/// </summary>
/// <param name="downloadItem">
/// The download item.
/// </param>
public static void AddToBackgroundQue(DownloadItem downloadItem)
{
if (Get.Web.EnableAddToBackgroundQue == false)
{
return;
}
string path = WebCache.GetPathFromUrl(downloadItem.Url, downloadItem.Section);
if (!File.Exists(path))
{
lock (BackgroundDownloadQue)
{
BackgroundDownloadQue.Add(downloadItem);
}
}
}
/// <summary>
/// Add downloaditem to background que
/// </summary>
/// <param name="downloadItem">
/// The download item.
/// </param>
public static void AddToBackgroundQue(DownloadItem downloadItem, DownloadPriority downloadPriority = DownloadPriority.Normal)
{
if (Get.Web.EnableAddToBackgroundQue == false)
{
return;
}
string path = WebCache.GetPathFromUrl(downloadItem.Url, downloadItem.Section);
if (!File.Exists(path))
{
lock (BackgroundDownloadQue)
{
var check = (from d in BackgroundDownloadQue where d.Url == downloadItem.Url select d).SingleOrDefault();
if (check == null)
{
downloadItem.Priority = downloadPriority;
BackgroundDownloadQue.Add(downloadItem);
}
}
}
}
public static bool Check(DownloadItem downloadItem)
{
return Regex.IsMatch(downloadItem.Url, MatchRegex);
}
