protected async override Task <ProcessResult> OnPolling(
Shared.Jobs.PollerJobParameters parameters,
string workingFolder)
{
Logger.DebugFormat("Downloaded file {0} to be converted to pdf", parameters.FileName);
var converter = Converters.FirstOrDefault(c => c.CanConvert(parameters.FileName));
if (converter == null)
{
Logger.InfoFormat("No converter for extension {0}", Path.GetExtension(parameters.FileName));
return(ProcessResult.Ok);
}
//Download file only if we have one converter that can generate pdf.
string pathToFile = await DownloadBlob(parameters.TenantId, parameters.JobId, parameters.FileName, workingFolder).ConfigureAwait(false);
string outFile = Path.Combine(workingFolder, Guid.NewGuid() + ".pdf");
if (!converter.Convert(pathToFile, outFile))
{
Logger.ErrorFormat("Error converting file {0} to pdf", pathToFile);
return(ProcessResult.Fail($"Error converting file {pathToFile} to pdf"));
}
await AddFormatToDocumentFromFile(
parameters.TenantId,
parameters.JobId,
new DocumentFormat(DocumentFormats.Pdf),
outFile,
new Dictionary <string, object>()).ConfigureAwait(false);
return(ProcessResult.Ok);
}
private bool CheckIfSomeJobCanStillProducePdfFormat(Shared.Jobs.QueuedJobInfo[] pendingJobs, String fileName, Int32 retryCount)
{
//We wait for the job to be generated only for the first 10 wait.
Boolean waitForNotGeneratedJob = retryCount < 10;
if (CheckIfQueueJobShouldStillBeExecuted(pendingJobs, "pdfConverter", allowNullJob: waitForNotGeneratedJob))
{
//PdfConverter did not run, need to wait
return(true);
}
var extension = Path.GetExtension(fileName);
if (!String.IsNullOrEmpty(extension) && officeExtensions.Contains(extension.Trim('.')))
{
//this is a file that can be converted with office.
if (CheckIfQueueJobShouldStillBeExecuted(pendingJobs, "office", allowNullJob: waitForNotGeneratedJob))
{
//The extension is an extension of office, but office queue still did not run.
return(true);
}
}
var jobThatStillWereNotRun = pendingJobs.Count(j => !j.Executed);
//we do not know if there are some more jobs that can produce pdf, we assume that no pdf format is present
//if this handle has no more job to run.
return(jobThatStillWereNotRun == 0);
}
private async Task <Int32> UploadAttachmentListToDocumentStore(PollerJobParameters parameters, string[] permittedExtension, string unzippingDirectory, IEnumerable <string> files)
{
Int32 uploadCount = 0;
foreach (string file in files)
{
var attachmentExtension = Path.GetExtension(file).Trim('.');
if (permittedExtension != null &&
!permittedExtension.Contains(attachmentExtension, StringComparer.OrdinalIgnoreCase))
{
Logger.DebugFormat("job: {0} File {1} attachment is discharded because extension {2} is not permitted",
parameters.JobId, file, attachmentExtension);
continue;
}
var relativeFileName = file.Substring(unzippingDirectory.Length);
await AddAttachmentToHandle(
parameters.TenantId,
parameters.JobId,
file,
"content_zip",
relativeFileName,
new Dictionary <string, object>() { }
);
uploadCount++;
}
return(uploadCount);
}
public bool CanConvert(string fileName)
{
var extension = Path.GetExtension(fileName);
return(extension.EndsWith("txt", StringComparison.OrdinalIgnoreCase) ||
extension.EndsWith("text", StringComparison.OrdinalIgnoreCase) ||
extension.EndsWith("log", StringComparison.OrdinalIgnoreCase));
}
protected String SanitizeFileNameForLength(String fileName)
{
if (fileName.Length > MaxFileNameLength)
{
//we need to clip length of the file.
var oriFileName = fileName;
var fileExtension = Path.GetExtension(fileName);
fileName = fileName.Substring(0, MaxFileNameLength - fileExtension.Length) + fileExtension;
Logger.InfoFormat("Original Filename {0} longer than 260 chars, it is truncated to {1}", oriFileName, fileName);
}
return(fileName);
}
private void SaveDocument(XComponent xComponent, string sourceFile, string destinationFile)
{
var propertyValues = new PropertyValue[2];
// Setting the flag for overwriting
propertyValues[1] = new PropertyValue {
Name = "Overwrite", Value = new Any(true)
};
//// Setting the filter name
propertyValues[0] = new PropertyValue
{
Name = "FilterName",
Value = new Any(ConvertExtensionToFilterType(Path.GetExtension(sourceFile)))
};
((XStorable)xComponent).storeToURL(destinationFile, propertyValues);
}
public void ConvertToPdf(string inputFile, string outputFile)
{
inputFile = SanitizeFileName(inputFile);
if (ConvertExtensionToFilterType(Path.GetExtension(inputFile)) == null)
{
throw new InvalidProgramException("Unknown file type for OpenOffice. File = " + inputFile);
}
StartOpenOffice();
//Get a ComponentContext
var xLocalContext = Bootstrap.bootstrap();
//Get MultiServiceFactory
var xRemoteFactory = (XMultiServiceFactory)xLocalContext.getServiceManager();
//Get a CompontLoader
var aLoader = (XComponentLoader)xRemoteFactory.createInstance("com.sun.star.frame.Desktop");
//Load the sourcefile
XComponent xComponent = null;
try
{
xComponent = InitDocument(aLoader, PathConverter(inputFile), "_blank");
//Wait for loading
while (xComponent == null)
{
Thread.Sleep(1000);
}
// save/export the document
SaveDocument(xComponent, inputFile, PathConverter(outputFile));
}
catch (Exception ex)
{
Logger.Error($"Error during conversion of file {inputFile}", ex);
throw;
}
finally
{
if (xComponent != null)
{
xComponent.dispose();
}
}
}
public FileNameWithExtension(string fileNameWithExtension)
{
if (String.IsNullOrWhiteSpace(fileNameWithExtension))
{
throw new ArgumentNullException("fileNameWithExtension");
}
fileNameWithExtension = fileNameWithExtension.Replace("\"", "");
FileName = Path.GetFileNameWithoutExtension(fileNameWithExtension);
Extension = Path.GetExtension(fileNameWithExtension);
if (fileNameWithExtension.StartsWith("."))
{
FileName = fileNameWithExtension;
Extension = "";
}
if (!String.IsNullOrWhiteSpace(Extension))
{
Extension = Extension.Remove(0, 1).ToLowerInvariant();
}
}
private string ProcessFile(string pathToFile, string workingFolder)
{
var extension = Path.GetExtension(pathToFile).ToLower();
if (extension == ".htmlzip" || extension == ".htmzip")
{
ZipFile.ExtractToDirectory(pathToFile, workingFolder);
Logger.DebugFormat("Extracted zip to {0}", workingFolder);
var htmlFile = Path.ChangeExtension(pathToFile, "html");
if (File.Exists(htmlFile))
{
Logger.DebugFormat("Html file is {0}", htmlFile);
return(htmlFile);
}
htmlFile = Path.ChangeExtension(pathToFile, "htm");
if (File.Exists(htmlFile))
{
Logger.DebugFormat("Html file is {0}", htmlFile);
return(htmlFile);
}
Logger.ErrorFormat("Invalid HTMLZIP file, name is {0} but corresponding html file not found after decompression", Path.GetFileName(pathToFile));
}
else if (extension == ".mht" || extension == ".mhtml")
{
MHTMLParser parser = new MHTMLParser(File.ReadAllText(pathToFile));
parser.OutputDirectory = workingFolder;
parser.DecodeImageData = false;
var html = parser.getHTMLText();
pathToFile = pathToFile + ".html";
File.WriteAllText(pathToFile, html);
}
return(pathToFile);
}
private Boolean IsUnzippedHtmlFile()
{
var fileExtension = Path.GetExtension(_inputFileName);
return(unzippedHtmlExtension.Any(s => s.Equals(fileExtension, StringComparison.OrdinalIgnoreCase)));
}
public string Download(BlobId blobId, string folder)
{
if (blobId == null)
{
throw new ArgumentNullException(nameof(blobId));
}
if (String.IsNullOrEmpty(folder))
{
throw new ArgumentNullException(nameof(folder));
}
if (!Directory.Exists(folder))
{
throw new ArgumentException($"folder {folder} does not exists", nameof(folder));
}
var descriptor = _blobDescriptorCollection.FindOneById(blobId);
if (descriptor == null)
{
throw new ArgumentException($"Descriptor for {blobId} not found in {_blobDescriptorCollection.CollectionNamespace.FullName}");
}
var localFileName = _directoryManager.GetFileNameFromBlobId(blobId);
if (!File.Exists(localFileName))
{
Logger.Error($"Blob {blobId} has descriptor, but blob file {localFileName} not found in the system.");
throw new ArgumentException($"Blob {blobId} not found");
}
var originalFileName = descriptor.FileNameWithExtension.ToString();
string destinationFileName = Path.Combine(folder, originalFileName);
Int32 uniqueId = 1;
while (File.Exists(destinationFileName))
{
destinationFileName = Path.Combine(folder, Path.GetFileNameWithoutExtension(originalFileName) + $" ({uniqueId++})") + Path.GetExtension(originalFileName);
}
File.Copy(localFileName, destinationFileName);
return(destinationFileName);
}
protected async override Task <ProcessResult> OnPolling(Shared.Jobs.PollerJobParameters parameters, string workingFolder)
{
string localFile = await DownloadBlob(
parameters.TenantId,
parameters.JobId,
parameters.FileName,
workingFolder);
String[] permittedExtension = null;
if (parameters.All.ContainsKey("extensions"))
{
var extensionsPermitted = parameters.All["extensions"];
if (extensionsPermitted != "*")
{
permittedExtension = extensionsPermitted.Split('|');
}
}
var extension = Path.GetExtension(localFile);
var unzippingDirectory = new DirectoryInfo(Path.Combine(workingFolder, Guid.NewGuid().ToString())).FullName;
if (!Directory.Exists(unzippingDirectory))
{
Directory.CreateDirectory(unzippingDirectory);
}
if (extension == ".zip")
{
//we can handle unzipping everything.
ZipFile.ExtractToDirectory(localFile, unzippingDirectory);
IEnumerable <String> files = Directory.EnumerateFiles(unzippingDirectory, "*.*", SearchOption.AllDirectories);
Int32 uploadCount = await UploadAttachmentListToDocumentStore(parameters, permittedExtension, unzippingDirectory, files);
Logger.DebugFormat("Uploaded {0} attachments", uploadCount);
}
else if (extension == ".eml")
{
using (var stream = File.Open(localFile, FileMode.Open, FileAccess.Read))
{
var message = MsgReader.Mime.Message.Load(stream);
var bodyPart = message.HtmlBody ?? message.TextBody;
String body = "";
if (bodyPart != null)
{
body = bodyPart.GetBodyAsText();
}
foreach (MsgReader.Mime.MessagePart attachment in message.Attachments.OfType <MsgReader.Mime.MessagePart>())
{
if (!String.IsNullOrEmpty(attachment.ContentId) &&
body.Contains(attachment.ContentId))
{
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Attachment cid {0} name {1} discharded because it is inline", attachment.ContentId, attachment.FileName);
continue;
}
}
String fileName = Path.Combine(unzippingDirectory, attachment.FileName);
File.WriteAllBytes(fileName, attachment.Body);
await AddAttachmentToHandle(
parameters.TenantId,
parameters.JobId,
fileName,
"attachment_email",
attachment.FileName,
new Dictionary <string, object>() { }
);
}
}
}
else if (extension == ".msg")
{
using (var stream = File.Open(localFile, FileMode.Open, FileAccess.Read))
using (var message = new Storage.Message(stream))
{
foreach (Storage.Attachment attachment in message.Attachments.OfType <Storage.Attachment>())
{
if (attachment.IsInline)
{
continue; //no need to uncompress inline attqach
}
String fileName = Path.Combine(unzippingDirectory, attachment.FileName);
File.WriteAllBytes(fileName, attachment.Data);
await AddAttachmentToHandle(
parameters.TenantId,
parameters.JobId,
fileName,
"attachment_email",
attachment.FileName,
new Dictionary <string, object>() { }
);
}
}
}
else if (extension == ".7z" || extension == ".7zip" || extension == ".rar")
{
//we can handle unzipping everything.
var extracted = _sevenZipExtractorFunctions.ExtractTo(localFile, unzippingDirectory);
Int32 uploadCount = await UploadAttachmentListToDocumentStore(parameters, permittedExtension, unzippingDirectory, extracted);
Logger.DebugFormat("Uploaded {0} attachments", uploadCount);
}
return(ProcessResult.Ok);
}
public bool CanConvert(string fileName)
{
var extension = Path.GetExtension(fileName);
return(supportedImageExtensions.Any(e => extension.EndsWith(e, StringComparison.OrdinalIgnoreCase)));
}