/// <summary>
/// Upload a file to Azure Storage from a url
/// </summary>
/// <param name="fileName">File name</param>
/// <param name="url">Url of the file</param>
/// <param name="type"><see cref="ExportType"/></param>
/// <param name="forceReplace">Force replacing an existing file</param>
/// <returns>Uri of the uploaded file</returns>
public async Task <string> UploadFileFromUrlToAzureStorage(string fileName, string url, ExportType type, bool forceReplace = false)
{
try
{
if (!_initialized)
{
throw new Exception("Service is not initialized");
}
var blob = _container.GetBlockBlobReference($@"{type.ToFriendlyString()}/{fileName}");
if (forceReplace || !await blob.ExistsAsync())
{
var cookieContainer = new CookieContainer();
using (var handler = new HttpClientHandler {
CookieContainer = cookieContainer
})
using (var client = new HttpClient(handler)
{
Timeout = TimeSpan.FromSeconds(10)
})
using (var request = new HttpRequestMessage(HttpMethod.Get, url))
{
if (url.Contains("yts"))
{
// Sometimes YTS requires an authentication to access movies. Here is an already authenticated cookie valid until December 2017
cookieContainer.Add(new Uri(url),
new Cookie("PHPSESSID", "ncrmefe4s79la9d2mba0n538o6", "/", "yts.ag"));
cookieContainer.Add(new Uri(url),
new Cookie("__cfduid", "d6c81b283d74b436fec66f02bcb99c04d1481020053", "/", ".yts.ag"));
cookieContainer.Add(new Uri(url), new Cookie("bgb2", "1", "/", "yts.ag"));
cookieContainer.Add(new Uri(url),
new Cookie("uhh", "60a8c98fd72e7a79b731f1ea09a5d09d", "/", ".yts.ag"));
cookieContainer.Add(new Uri(url), new Cookie("uid", "2188423", "/", ".yts.ag"));
}
using (var response = await client.SendAsync(request))
{
response.EnsureSuccessStatusCode();
using (var contentStream = await response.Content.ReadAsStreamAsync())
{
// Create a directory under the root directory
var file = _container.GetBlockBlobReference($@"{type.ToFriendlyString()}/{fileName}");
await file.UploadFromStreamAsync(contentStream);
return(file.Uri.AbsoluteUri);
}
}
}
}
return(blob.Uri.AbsoluteUri);
}
catch (Exception)
{
return(string.Empty);
}
}
/// <summary>
/// Upload a file to Azure Storage from a url
/// </summary>
/// <param name="fileName">File name</param>
/// <param name="url">Url of the file</param>
/// <param name="type"><see cref="ExportType"/></param>
/// <param name="forceReplace">Force replacing an existing file</param>
/// <returns>Uri of the uploaded file</returns>
public async Task <string> UploadFileFromUrlToAzureStorage(string fileName, string url, ExportType type,
bool forceReplace = false)
{
try
{
if (!_initialized)
{
throw new Exception("Service is not initialized");
}
var blob = _container.GetBlockBlobReference($@"{type.ToFriendlyString()}/{fileName}");
if (forceReplace || !await blob.ExistsAsync().ConfigureAwait(false))
{
var cookieContainer = new CookieContainer();
using (var handler = new HttpClientHandler {
CookieContainer = cookieContainer
})
using (var client = new HttpClient(handler)
{
Timeout = TimeSpan.FromSeconds(10)
})
using (var request = new HttpRequestMessage(HttpMethod.Get, url))
{
if (url.Contains("yts"))
{
// Sometimes YTS requires an authentication to access movies. Here is an already authenticated cookie valid until December 2017
cookieContainer.Add(new Uri(url),
new Cookie("PHPSESSID", "9kp14954qk7pfa6kconsocg2a3", "/", "yts.am"));
cookieContainer.Add(new Uri(url),
new Cookie("__atuvs", "5a1b42230c1ac9da002", "/", "yts.am"));
cookieContainer.Add(new Uri(url),
new Cookie("__cfduid", "da06ef1392f710522307088b060b5826a1511735838", "/", ".yts.am"));
cookieContainer.Add(new Uri(url),
new Cookie("uhh", "5703d275ec7d989652c497b9f921dfcf", "/", ".yts.am"));
cookieContainer.Add(new Uri(url), new Cookie("uid", "3788520", "/", ".yts.am"));
}
using (var response = await client.SendAsync(request).ConfigureAwait(false))
{
response.EnsureSuccessStatusCode();
using (var contentStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))
{
var file = _container.GetBlockBlobReference($@"{type.ToFriendlyString()}/{fileName}");
if (blob.Name.Contains("background") ||
blob.Name.Contains("banner") ||
blob.Name.Contains("poster"))
{
try
{
using (var stream = new MemoryStream())
using (var image = Image.Load(contentStream))
{
if (blob.Name.Contains("background") || blob.Name.Contains("banner"))
{
image.Mutate(x => x
.Resize(new ResizeOptions
{
Mode = ResizeMode.Stretch,
Size = new Size(1280, 720),
Sampler = new BicubicResampler()
}));
}
if (blob.Name.Contains("poster"))
{
image.Mutate(x => x
.Resize(new ResizeOptions
{
Mode = ResizeMode.Stretch,
Size = new Size(400, 600),
Sampler = new BicubicResampler()
}));
}
image.SaveAsJpeg(stream, new JpegEncoder
{
Quality = 90
});
stream.Seek(0, SeekOrigin.Begin);
await file.UploadFromStreamAsync(stream).ConfigureAwait(false);
}
}
catch (Exception ex)
{
_loggingService.Telemetry.TrackException(ex);
await file.UploadFromStreamAsync(contentStream).ConfigureAwait(false);
}
}
else
{
await file.UploadFromStreamAsync(contentStream).ConfigureAwait(false);
}
return(file.Uri.AbsoluteUri);
}
}
}
}
return(blob.Uri.AbsoluteUri);
}
catch (Exception ex)
{
_loggingService.Telemetry.TrackException(ex);
return(string.Empty);
}
}
/// <summary>
/// Load an export
/// </summary>
/// <param name="exportType">Export to load</param>
/// <returns>Bson documents</returns>
public async Task <IEnumerable <BsonDocument> > LoadExport(ExportType exportType)
{
var export = new ConcurrentBag <BsonDocument>();
try
{
var loggingTraceBegin =
$@"Export {exportType.ToFriendlyString()} started at {DateTime.Now.ToString(
"dd/MM/yyyy HH:mm:ss.fff", CultureInfo.InvariantCulture)}";
_loggingService.Telemetry.TrackTrace(loggingTraceBegin);
if (exportType == ExportType.Shows)
{
using (var client = new RestClient(Constants.OriginalPopcornApi))
{
var request = new RestRequest("{segment}", Method.GET);
switch (exportType)
{
case ExportType.Shows:
request.AddUrlSegment("segment", "show");
break;
}
// Execute request
var response = await client.Execute(request);
// Load response into memory
using (var data = new MemoryStream(response.RawBytes))
using (var reader = new StreamReader(data, Encoding.UTF8))
{
string line;
// Read all response parts
while ((line = await reader.ReadLineAsync()) != null)
{
ConvertJsonToBsonDocument(line, export);
}
}
}
}
else if (exportType == ExportType.Movies)
{
var page = 1;
bool movieFound;
do
{
using (var client = new RestClient(Constants.YtsApiUrl))
{
var moviesByPageRequest = GetMoviesByPageRequest(page);
// Execute request
var movieShortResponse = await client.Execute(moviesByPageRequest);
var movieNode = JsonSerializer.Deserialize <MovieShortJsonNode>(movieShortResponse.RawBytes);
if (movieNode?.Data?.Movies == null || !movieNode.Data.Movies.Any())
{
movieFound = false;
}
else
{
movieFound = true;
page++;
await movieNode.Data.Movies.ParallelForEachAsync(async movie =>
{
try
{
using (var innerClient = new RestClient(Constants.YtsApiUrl))
{
var movieByIdRequest = GetMovieById(movie.Id);
var movieFullResponse =
await innerClient.Execute(movieByIdRequest);
var fullMovie =
JsonSerializer.Deserialize <MovieFullJsonNode>(
movieFullResponse.RawBytes);
ConvertJsonToBsonDocument(
JsonSerializer.ToJsonString(fullMovie.Data.Movie),
export);
}
}
catch (Exception ex)
{
_loggingService.Telemetry.TrackException(ex);
}
});
}
}
} while (movieFound);
}
var loggingTraceEnd =
$@"Export {export.Count} {exportType.ToFriendlyString()} ended at {DateTime.Now.ToString(
"dd/MM/yyyy HH:mm:ss.fff", CultureInfo.InvariantCulture)}";
_loggingService.Telemetry.TrackTrace(loggingTraceEnd);
return(export);
}
catch (Exception ex)
{
_loggingService.Telemetry.TrackException(ex);
return(export);
}
}
