private void SendEmail(UploadSet uploadSet, string subject, string body)
{
using (var httpClient = new HttpClient())
{
httpClient.Timeout = TimeSpan.FromMinutes(uploadSet.Setup.UploadTimeoutMinutes);
//httpClient.BaseAddress = new Uri(uploadSet.Setup.EmailEndpoint);
using (var content = new MultipartFormDataContent())
{
Console.WriteLine("Deactivating images");
Dictionary <string, string> svcMetadata = new Dictionary <string, string>();
svcMetadata.Clear();
var jsonData = "{" + String.Join(",", svcMetadata.Values) + "}";
content.Add(new StringContent("update"), "action");
content.Add(new StringContent(CurrentEnvironment.UserName), "transactionUser");
content.Add(new StringContent(jsonData), "images");
Logger.LogCustomEvent("sendEmail", svcMetadata);
/*
* //var response = httpClient.PostAsync(uploadSet.Setup.EmailEndPoint, content).Result;
*
* if (response.StatusCode != HttpStatusCode.OK)
* {
* var reason = response.ReasonPhrase;
* //response.Content.ReadAsStringAsync().Result;
* }
*/
}
}
}
private void MoveFiles(UploadSet uploadSet, string[] loadFiles, bool isError = false)
{
string eventName = isError ? "ArchivingInvalidItemImages" : "ArchivingImages";
//move files
if (!String.IsNullOrEmpty(uploadSet.Setup.ArchiveFolder))
{
Dictionary <string, string> filesToMove = new Dictionary <string, string>();
foreach (var item in loadFiles)
{
if (isError)
{
filesToMove.Add(item, item.ToErrorFolder(uploadSet.Setup.ArchiveFolder));
}
else
{
filesToMove.Add(item, item.ToDestinationFolder(uploadSet.Setup.ArchiveFolder));
}
}
Logger.LogCustomEvent(eventName, uploadSet.ToProperties());
MoveFiles(uploadSet.Setup.UserName, uploadSet.Setup.Password, uploadSet.Setup.FilePath, filesToMove);
}
}
public void Execute(IJobExecutionContext context)
{
try
{
List <UploadSet> fileSetsToUpload = new List <UploadSet>();
List <UploadInfo> filesToUpload = new List <UploadInfo>();
Logger.LogCustomEvent("UploadFile.Execute");
#region Find the files to upload
//Implement Idempotency - this job can be run multiple times without any negative effect.
//Files already uploaded will not be uploaded again even if the job executes multiple times.
foreach (SetupConfigurationElement item in JobConfiguration.Setups)
{
filesToUpload.Clear();
#region Find files for each setup
//get all files from the folder
AddSeqToFiles(item.UserName, item.Password, item.FilePath, "*." + item.FileExtension);
var filesInDirToUpload = GetFiles(item.ImagesPerCall).ToList();
//at this point we will either have new images or previously sucessfully uploaded images
//we need to include all the new ones and any previously successfully uploaded image that has been retaken
foreach (var file in filesInDirToUpload)
{
#region Add Files to Upload
FileInfo fi = new FileInfo(file);
UploadInfo ui = new UploadInfo
{
Id = Guid.NewGuid().ToString(),
FileName = file,
UploadEndPoint = item.UploadEndpoint,
Status = UploadSet.UploadStatus.NotStarted,
SourceMachineName = item.MachineName,
FileCreationTimeUtc = fi.LastWriteTimeUtc,
//mailEndPoint = item.UploadEndpoint,
};
filesToUpload.Add(ui);
#endregion
}
#endregion
//make sure the files to upload belong to a full set
var loadSets = filesToUpload.GroupBy(f => f.FileName.ToLoadNumber());
foreach (var set in loadSets)
{
fileSetsToUpload.Add(new UploadSet {
LoadNumber = set.Key, Setup = item, UploadInfoList = set.ToList(), IsRetake = set.First().IsRetake
});
}
}
#endregion
//exclude any files that are currently being uploaded
foreach (var file in UploadFileQueue.Instance.InProgress)
{
var existing = fileSetsToUpload.Where(f => f.LoadNumber.Equals(file, StringComparison.CurrentCultureIgnoreCase)).FirstOrDefault();
if (existing != null)
{
fileSetsToUpload.Remove(existing);
}
}
//queue the file uploads
foreach (var item in fileSetsToUpload)
{
Logger.LogCustomEvent("QueueUpload", item.ToProperties());
UploadFileQueue.Instance.Queue.Enqueue(item);
}
if (fileSetsToUpload.Count == 0)
{
Logger.LogCustomEvent("NoFilesToUpload");
Console.WriteLine("No file sets to upload: " + context.JobDetail.Description);
}
do
{
List <UploadSet> filesToProcess = new List <UploadSet>();
for (int i = 0; i < JobConfiguration.NumberOfThreads; i++)
{
UploadSet res = null;
if (UploadFileQueue.Instance.Queue.TryDequeue(out res))
{
filesToProcess.Add(res);
}
}
if (filesToProcess.Count() == 0)
{
break;
}
List <Task> tasks = new List <Task>();
try
{
foreach (var file in filesToProcess)
{
tasks.Add(Task.Factory.StartNew(() =>
{
if (loadNumber.Length > 0 && loadFiles.Count() == 0)
{
Console.WriteLine("No complete file for the load: " + loadNumber + " " + context.JobDetail.Description);
Console.WriteLine("move all photos for load " + loadNumber + " to error folder");
Logger.LogException(new Exception("No complete file for the load: " + loadNumber + " " + context.JobDetail.Description));
MoveFiles(file, loadFiles, true);
}
else if (loadNumber.Length > 0 && loadFiles.Count() > 0 && loadFiles.Count() != totalPhotos)
{
Console.WriteLine("file number on complete file is different from the total number of photos for load: " + loadNumber + " " + context.JobDetail.Description);
Console.WriteLine("move all photos for load " + loadNumber + " to error folder");
Logger.LogException(new Exception("file number on complete file is different from the total number of photos for load: " + loadNumber + " " + context.JobDetail.Description));
MoveFiles(file, loadFiles, true);
MoveFiles(file, completeFile, true);
}
else
{
UploadFileToServer(file);
}
}));
}
Task.WaitAll(tasks.ToArray());
}
catch (AggregateException excep)
{
foreach (var task in tasks)
{
if (task.Exception != null)
{
UploadInfo chunk = task.AsyncState as UploadInfo;
chunk.Status = UploadSet.UploadStatus.Failed;
}
}
}
}while (true);
}
catch (Exception ex)
{
//log exception
Logger.LogException(ex);
}
}
private void UploadFileToServer(UploadSet uploadSet)
{
var startTimeUtc = DateTime.UtcNow;
string deActivationErrorMessage = String.Empty;
try
{
UploadFileQueue.Instance.InProgress.Add(uploadSet.LoadNumber);
Logger.LogCustomEvent("ProcessUpload", uploadSet.ToProperties());
Console.WriteLine("Uploading load: " + uploadSet.LoadNumber + " " + DateTime.Now.ToString());
//get the item id
string itemId = GetItemId(uploadSet.LoadNumber); //TODO In case of PAS web service connectivity issue, don't move the files
#region validate
if (String.IsNullOrEmpty(JobConfiguration.UploadSource))
{
throw new Exception("UploadSource not found in Configuration for load: " + uploadSet.LoadNumber);
}
if (uploadSet.Setup == null)
{
throw new Exception("Configuration setup not found for load: " + uploadSet.LoadNumber);
}
if (String.IsNullOrEmpty(uploadSet.Setup.UploadEndpoint))
{
throw new Exception("UploadEndpoint not found in Configuration setup for load: " + uploadSet.LoadNumber);
}
if (String.IsNullOrEmpty(uploadSet.Setup.SubscriptionId))
{
throw new Exception("SubscriptionId not found in Configuration setup for load: " + uploadSet.LoadNumber);
}
if (!FilesExist(uploadSet.Setup.UserName, uploadSet.Setup.Password, uploadSet.Setup.FilePath, uploadSet.UploadInfoList.Select(u => u.FileName).ToList()))
{
throw new NonLogException("Files not found.");
}
if (String.IsNullOrEmpty(itemId))
{
List <string> filesToMove = new List <string>();
//take off _processed and add complete file
for (int i = 0; i < loadFiles.Count(); i++)
{
if (loadFiles[i].Contains("_processed"))
{
filesToMove.Add(loadFiles[i].Substring(0, loadFiles[i].IndexOf("_proessed")) + ".jpg");
}
else
{
filesToMove.Add(loadFiles[i]);
}
}
//add ready file
filesToMove.Add(completeFile[0]);
//Move to error folder
MoveFiles(uploadSet, filesToMove.ToArray(), true);
throw new NonLogException("Item Id not found for load: " + uploadSet.LoadNumber);
}
#endregion
//Do the actual upload
using (var httpClient = new HttpClient())
{
httpClient.Timeout = TimeSpan.FromMinutes(uploadSet.Setup.UploadTimeoutMinutes);
httpClient.BaseAddress = new Uri(uploadSet.Setup.UploadEndpoint);
httpClient.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", uploadSet.Setup.SubscriptionId);
httpClient.DefaultRequestHeaders.Add("Cache-Control", "no-cache");
List <string> itemPictureIds = new List <string>();
#region Get item pictures id for retakes
Logger.LogCustomEvent("GetItemId", uploadSet.ToProperties());
Console.WriteLine("Calling Get endpoint:" + uploadSet.Setup.GetEndpoint + " at:" + DateTime.Now.ToString());
var response = httpClient.GetAsync(uploadSet.Setup.GetEndpoint + "?item_id=" + itemId + "&stage=sale&active=true").Result;
if (response.StatusCode != HttpStatusCode.OK)
{
var reason = response.ReasonPhrase;
var errorMessage = response.Content.ReadAsStringAsync().Result;
string error = "Upload for load: " + uploadSet.LoadNumber + " failed to get Item Picture Ids for retaken images. Reason: " + reason + ". Error: " + errorMessage +
" endpoint=" + uploadSet.Setup.GetEndpoint + " item_id=" + itemId;
string subject = "Photo Booth Loading Error - Severity 2 - failed retrieve current photos";
//SendEmail(uploadSet, subject, error);
throw new Exception(error);
}
#endregion
Dictionary <string, string> svcMetadata = new Dictionary <string, string>();
Boolean saleStageOnly = true;
#region Deactivate images
if (processedPhotos == 0)
{
using (var content = new MultipartFormDataContent())
{
Console.WriteLine("Deactivating images");
svcMetadata.Clear();
var jsonResponse = response.Content.ReadAsStringAsync().Result;
//get item picture ids from Json
dynamic itemInfo = JObject.Parse(@jsonResponse);
foreach (var pair in itemInfo)
{
var metadata = pair.First.metadata.ToString();
var metadata_json = JObject.Parse(metadata);
var stages = metadata_json.stage;
var imageType = metadata_json.image_type;
string stagesString = stages.ToString();
string newStages = "";
if (stagesString.Contains("["))
{
if (imageType == "photo")
{
saleStageOnly = false;
newStages = stagesString.Replace("sale", "");
}
}
else
{
saleStageOnly = true;
}
if (saleStageOnly)
{
string id = pair.Path;
svcMetadata.Add(id, @"""" + id + @""":{""metadata"":{""active"":""false""}}");
}
else
{
string id = pair.Path;
svcMetadata.Add(id, @"""" + id + @""":{""metadata"":{""stage"":" + newStages + "}}");
}
}
if (svcMetadata.Count() > 0)
{
var jsonData = "{" + String.Join(",", svcMetadata.Values) + "}";
content.Add(new StringContent("update"), "action");
content.Add(new StringContent(CurrentEnvironment.UserName), "transactionUser");
content.Add(new StringContent(jsonData), "images");
Logger.LogCustomEvent("DeactiviatingImages", uploadSet.ToProperties());
Console.WriteLine("Calling deactive upload endpoint:" + uploadSet.Setup.UploadEndpoint + " at:" + DateTime.Now.ToString());
response = httpClient.PostAsync(uploadSet.Setup.UploadEndpoint, content).Result;
if (response.StatusCode != HttpStatusCode.OK)
{
var reason = response.ReasonPhrase;
deActivationErrorMessage = response.Content.ReadAsStringAsync().Result;
}
}
}
}
#endregion
using (var content = new MultipartFormDataContent())
{
#region Generate metadata
string uploadFileName = String.Empty;
svcMetadata.Clear();
foreach (var item in uploadSet.UploadInfoList)
{
int sequenceNumber;
fileOrder.TryGetValue(item.FileName.ToLower(), out sequenceNumber);
var imgData = new
{
item_id = itemId,
sequence_number = sequenceNumber.ToString(),
visibility = "public",
sync = "true",
usage = GetUsage(item.FileName)
};
var s = new JsonSerializer();
var sb = new StringBuilder();
using (var w = new StringWriter(sb))
{
s.Serialize(w, imgData);
}
uploadFileName = item.FileName.ToFileName().Replace(uploadSet.LoadNumber, itemId);
svcMetadata.Add(uploadFileName, @"""" + uploadFileName + @""":" + sb.ToString());
}
#endregion
string imagesJson = @"{" + String.Join(",", svcMetadata.Values) + "}";
content.Add(new StringContent(JobConfiguration.UploadSource), "source");
content.Add(new StringContent("upload"), "action");
content.Add(new StringContent(CurrentEnvironment.UserName), "transactionUser");
content.Add(new StringContent(imagesJson), "images");
Console.WriteLine(imagesJson);
//get image data
var imageData = GetImageData(uploadSet.Setup.UserName, uploadSet.Setup.Password, uploadSet.Setup.FilePath, uploadSet.UploadInfoList);
//mime type should be the same for all image
string mimeType = MimeMapping.GetMimeMapping(uploadSet.UploadInfoList.First().FileName);
for (int i = 0; i < imageData.Count; i++)
{
uploadFileName = uploadSet.UploadInfoList[i].FileName;
uploadFileName = uploadFileName.ToFileName().Replace(uploadSet.LoadNumber, itemId);
var streamContent = new ByteArrayContent(imageData[i]);
streamContent.Headers.Add("Content-Type", mimeType);
streamContent.Headers.Add("Content-Disposition", "form-data; name=\"file\"; filename=\"" + uploadFileName + "\"");
content.Add(streamContent, "files", uploadFileName);
}
Logger.LogCustomEvent("PostingImages", uploadSet.ToProperties());
Console.WriteLine("Calling upload endpoint:" + uploadSet.Setup.UploadEndpoint + " at:" + DateTime.Now.ToString());
response = httpClient.PostAsync(uploadSet.Setup.UploadEndpoint, content).Result;
if (response.StatusCode != HttpStatusCode.OK)
{
var reason = response.ReasonPhrase;
var errorMessage = response.Content.ReadAsStringAsync().Result;
string error = "Upload for load: " + uploadSet.LoadNumber + " failed. Reason: " + reason + ". Error: " + errorMessage +
" endpoint=" + uploadSet.Setup.UploadEndpoint;
string subject = "Photo Booth Loading Error - Severity 2 - failed upload photos";
//SendEmail(uploadSet, subject, error);
throw new Exception(error);
}
}
}
//Update status once uploaded
foreach (var item in uploadSet.UploadInfoList)
{
item.Status = UploadSet.UploadStatus.Successful;
}
Console.WriteLine("Upload " + uploadSet.LoadNumber + " successful " + DateTime.Now.ToString());
//add processed into file names.
if (processedPhotos + uploadSet.UploadInfoList.Count < totalPhotos)
{
for (int i = 0; i < processFiles.Count(); i++)
{
System.IO.File.Move(processFiles[i],
processFiles[i].Substring(0, processFiles[i].IndexOf(".jpg")) + "_processed" + ".jpg");
}
}
else
{
//rename photo files -taking off processed flag.
for (int i = 0; i < processedFiles.Count(); i++)
{
System.IO.File.Move(processedFiles[i],
processedFiles[i].Substring(0, processedFiles[i].IndexOf("_processed")) + ".jpg");
}
var filesToMove = new List <string>();
//take off _processed and add complete file
for (int i = 0; i < loadFiles.Count(); i++)
{
if (loadFiles[i].Contains("_processed"))
{
filesToMove.Add(loadFiles[i].Substring(0, loadFiles[i].IndexOf("_processed")) + ".jpg");
}
else
{
filesToMove.Add(loadFiles[i]);
}
}
//add ready file
filesToMove.Add(filesToMove[0].Substring(0, filesToMove[0].IndexOf(loadNumber)) + loadNumber + "_complete_" + totalPhotos + ".ready");
MoveFiles(uploadSet, filesToMove.ToArray());
}
}
catch (Exception ex)
{
foreach (var item in uploadSet.UploadInfoList)
{
//don't mark upload status failed if there is an error in archiving the files
if (ex.GetType() != typeof(FileArchiveException))
{
item.Status = UploadSet.UploadStatus.Failed;
}
item.ErrorMessage = ex.ToString();
//log exception
Logger.LogException(ex, item.ToProperties());
}
if (ex.GetType() == typeof(NonLogException))
{
uploadSet.CanTrack = false;
}
Console.WriteLine("Upload " + uploadSet.LoadNumber + " exception: " + ex.ToString());
}
finally
{
//Log if required
if (uploadSet.CanTrack)
{
//log progress
foreach (var item in uploadSet.UploadInfoList)
{
Logger.LogEvent(item.ToProperties());
}
//log dectivation
if (!String.IsNullOrEmpty(deActivationErrorMessage))
{
string subject = "Photo Booth Loading Error - Severity 2 - failed deactive photos";
string error = deActivationErrorMessage + " endpoint:" + uploadSet.Setup.UploadEndpoint + " content:" + uploadSet.ToProperties();
//SendEmail(uploadSet, subject, deActivationErrorMessage + );
Logger.LogException(new Exception(deActivationErrorMessage));
}
}
//remove from progress
string l = uploadSet.LoadNumber;
UploadFileQueue.Instance.InProgress.TryTake(out l);
}
}
