/// <summary> /// Initialize a new job, feed MS Excel format data into worker & executes worker /// </summary> /// <param name="dataSourceId">Should be a valid application id. If this is filled, then applicationName is optional.</param> /// <param name="dataSourceName">Should be a valid application name. If this is filled, then applicationId is optional.</param> /// <param name="processingBy">user name who is processing, we could get this from context, but for scenario like 'a126042, Deb'jyoti Das', let caller decide the user name.</param> /// <param name="excelFileName">Excel file name is used for just reference</param> /// <returns>Output xml. for details, read documentation.</returns> public StringBuilder ProcessSpreadSheet(int dataSourceId, string dataSourceName, string processingBy, string excelFileName) { Job job = null; try { string error = string.Empty; if ((string.IsNullOrEmpty(excelFileName)) || (!File.Exists(excelFileName))) { //we should never come here, this issue has been taken care in LocalFileSystemWatcher //job.TraceError("File {0} does not exist. Nothing to parse.", excelFileName); return(new StringBuilder()); } job = new Job(dataSourceId, dataSourceName, processingBy, excelFileName); Registry.Instance.Entries.Add(job.JobIdentifier, job); job.FileName = excelFileName; this.JobId = job.JobIdentifier.ToString(); if (!job.IsValid) { job.TraceError("Job could not be initialized, data source is not defined. Data source id ={0}, name = '{1}'", dataSourceId, dataSourceName); return(new StringBuilder()); } string onlyFileName = Path.GetFileName(job.FileName); job.Feed(); if (job.CsvRows.Count == 0) { //blank file job.TraceInformation(job.ErrorBlankInput); return(new StringBuilder()); } ExecuteWorkerManager(job); //Get the results from worker (data) job.Errors.AddRange(GetAllAttributeErrors(job.Rows)); //get all attribute errors if ((onlyFileName != null) && (onlyFileName.StartsWith(Constants.UnzippedFilePrefix))) { //1: orginal file //job.FileName = ZipFileWatcher.ExtractActualFileName(job.FileName); File.Delete(excelFileName);//we do not need as zip will be archived //2: zip file (this is important if consumer wants to refer the file (zip file exists) job.FileName = Registry.Instance.ZipFiles[ZipFileWatcher.ExtractUniqueId(onlyFileName)].ZipFileName; } if (!job.AbortRequested) { job.TraceInformation("Calling output writer '{0}'. Total processed rows:{1}", job.DataSource.OutputWriterTypeFullName, job.Rows.Count); Trace.Flush(); if (job.Errors.Count > 0) { new PostMan(job).Send(); } return(job.DataSource.OutputWriter.GetOutput()); } return(new StringBuilder()); } catch (BusinessException ex) { job.TraceInformation(ex.Message); //It is actually not an exception or error new PostMan(job, false).Send(PostMan.__warningStartTag + ex.Message + PostMan.__warningEndTag); return(new StringBuilder()); } catch (Exception ex) { string errorId = Guid.NewGuid().ToString(); string errorMessage = errorId + ex.ToString() + (ex.InnerException == null ? string.Empty : ex.InnerException.Message); job.TraceError("A server exception occurred. Please contact support. ErrorId = '{0}'. Server Time = '{1}'\n {2}", errorId, DateTime.Now, errorMessage); Trace.Flush(); return(new StringBuilder()); } }
public bool PrepareInput() { //int dataSourceId = 0; //int.TryParse(e.ApplicationParameters["DataSourceId"].ToString(), out dataSourceId); //if (dataSourceId == 0) // return; InputFileNameOnly = Path.GetFileNameWithoutExtension(InputFileName); //string InputFileExtension = Path.GetExtension(InputFileName); Keys = Cache.Instance.Bag[DataSourceId + ".keys"] as List <IdpeKey>; if (Keys == null) { Keys = DataSource.LoadKeys(DataSourceId); } OutputFolder = DataSource.GetOutputFolder(DataSourceId, Keys); ActualOutputFolder = OutputFolder; OutputFileName = DataSource.GetOutputFileName(DataSourceId, Keys, OutputFolder, InputFileNameOnly); string appWatchFilter = Keys.GetKeyValue(IdpeKeyTypes.WatchFilter); ZipInterfaceName = Keys.GetKeyValue(IdpeKeyTypes.ZipInterfaceName); if ((InputFileExtension.ToLower() == ".zip") || (InputFileExtension.ToLower() == ".rar") || (InputFileExtension.ToLower() == ".tar")) { OutputFolder = Path.Combine(EyediaCoreConfigurationSection.CurrentConfig.TempDirectory, Constants.IdpeBaseFolderName); OutputFolder = Path.Combine(OutputFolder, "RedirectedOutput"); OutputFolder = Path.Combine(OutputFolder, DateTime.Now.ToDBDateFormat()); OutputFolder = Path.Combine(OutputFolder, DataSourceId.ToString()); } if ((!string.IsNullOrEmpty(appWatchFilter)) && (appWatchFilter != Pullers.FileExtensionSupportAll)) { List <string> filters = new List <string>(); if (appWatchFilter.Contains("|")) { filters.AddRange(appWatchFilter.ToLower().Split("|".ToCharArray())); } else { filters.Add(appWatchFilter.ToLower()); } var filterOrNot = (from f in filters where f == InputFileExtension.ToLower() select f).SingleOrDefault(); if (filterOrNot == null) { if (!InputFileNameOnly.StartsWith(Constants.UnzippedFilePrefix)) { IdpeMessage warn = new IdpeMessage(IdpeMessageCodes.IDPE_FILE_TYPE_NOT_SUPPORTED); DataSource dataSource = new DataSource(DataSourceId, string.Empty); WithWarning = string.Format(warn.Message, dataSource.Name, appWatchFilter, Path.GetFileName(InputFileName)); ExtensionMethods.TraceInformation(WithWarning); new PostMan(dataSource).Send(PostMan.__warningStartTag + WithWarning + PostMan.__warningEndTag, "File Ignored"); return(false); } } } if (InputFileNameOnly.StartsWith(Constants.WCFFilePrefix)) { IsRequestFromWCF = true; JobId = InputFileNameOnly.Replace(Constants.WCFFilePrefix, ""); JobId = JobId.Replace(InputFileExtension, ""); } else if (InputFileNameOnly.StartsWith(Constants.UnzippedFilePrefix)) { ZipUniuqeId = ZipFileWatcher.ExtractUniqueId(InputFileNameOnly); OutputFolder = Path.Combine(OutputFolder, ZipUniuqeId); if (!Directory.Exists(OutputFolder)) { Directory.CreateDirectory(OutputFolder); } OutputFileName = Path.Combine(OutputFolder, InputFileNameOnly + Path.GetExtension(OutputFileName)); OutputFileName = ZipFileWatcher.ExtractActualFileName(OutputFileName); } return(true); }
/// <summary> /// Initialize a new job if jobId is NULL, feed CSV/FixedLength data into worker & executes worker /// </summary> /// <param name="dataSourceId">Should be a valid application id. If this is filled, then datasource name is optional.</param> /// <param name="dataSourceName">Should be a valid datasource name. If this is filled, then datasource id is optional.</param> /// <param name="processingBy">user name who is processing, we could get this from context, but for scenario like 'a126042, Deb'jyoti Das', let caller decide the user name.</param> /// <param name="fileName">fileName is used for just reference</param> /// <param name="jobId">When initial request comes from WCF, we already have jobId, else string.Empty</param> /// <param name="withError">Error message if you want to execute a dummy execution just to pass the error message to caller</param> /// <param name="withWarning">Warning message if you want to execute a dummy execution just to pass the error message to caller</param> /// <returns>Output StringBuilder. for details, read documentation.</returns> public StringBuilder ProcessJob(int dataSourceId, string dataSourceName, string processingBy, string fileName, string jobId, string withError, string withWarning) { Job job = null; try { string error = string.Empty; StringBuilder result = new StringBuilder(); if (string.IsNullOrEmpty(jobId)) { job = new Job(dataSourceId, dataSourceName, processingBy, fileName); if (job.Errors.Count > 0) { throw new BusinessException(job.Errors.ToLine("<br />")); } Registry.Instance.Entries.Add(job.JobIdentifier, job); } else { error = string.Format("There is no job found with id '{0}', or job might have expired.", jobId); if (!(Registry.Instance.Entries.ContainsKey(jobId))) { job.TraceInformation(error); } else { job = Registry.Instance.Entries[jobId] as Job; } if (job == null) { job.TraceInformation(error); return(result); } } job.PerformanceCounter.Start(job.JobIdentifier, JobPerformanceTaskNames.JobInit); job.IsWCFRequest = this.IsWCFRequest; this.JobId = job.JobIdentifier.ToString(); if (!job.IsValid) { error = string.Format("Job could not be initialized, application is not defined. Application id ={0}, name = '{1}'", dataSourceId, dataSourceName); job.TraceInformation(error); job.PerformanceCounter.Stop(job.JobIdentifier, JobPerformanceTaskNames.JobInit); return(result); } if ((!string.IsNullOrEmpty(withError)) || (!string.IsNullOrEmpty(withWarning))) { if (job.FileNameOnly.StartsWith(Constants.UnzippedFilePrefix)) { job.FileName = ZipFileWatcher.ExtractActualFileName(job.FileName); string originalFileName = Path.GetFileName(job.FileName); withError = withError.Replace(job.FileNameOnly, originalFileName); withWarning = withWarning.Replace(job.FileNameOnly, originalFileName); File.Delete(fileName);//we do not need as zip will be archived //this is required, as the consumer most of the time attaches the errored file in emails //as we deleted the actual file and job.FileName will never be retrieved. Hence set the actual-actual file, i.e. original zip file job.FileName = Registry.Instance.ZipFiles[ZipFileWatcher.ExtractUniqueId(job.FileNameOnly)].ZipFileName; } if (!string.IsNullOrEmpty(withError)) { withError = withError.Replace("APPNAME", job.DataSource.Name); job.Errors.Add(withError); } if (!string.IsNullOrEmpty(withWarning)) { withWarning = withWarning.Replace("APPNAME", job.DataSource.Name); job.Warnings.Add(withWarning); } Trace.Flush(); result = job.DataSource.OutputWriter.GetOutput(); job.PerformanceCounter.Stop(job.JobIdentifier, JobPerformanceTaskNames.JobInit); //job.Dispose(); return(result); } job.PerformanceCounter.Stop(job.JobIdentifier, JobPerformanceTaskNames.JobInit); job.Feed(); //Debugger.Break(); if (job.InputData.Rows.Count == 0) { job.IsFinished = true; //blank file job.TraceInformation(job.ErrorBlankInput); job.Errors.Add(job.ErrorBlankInput); //new PostMan(job, false).Send(PostMan.__errorStartTag + job.ErrorBlankInput + PostMan.__errorEndTag); return(new StringBuilder()); } job.PerformanceCounter.Start(job.JobIdentifier, JobPerformanceTaskNames.ExecuteWorkerManager); ExecuteWorkerManager(job); job.PerformanceCounter.Stop(job.JobIdentifier, JobPerformanceTaskNames.ExecuteWorkerManager); //Get the results from worker (data) job.Errors.AddRange(GetAllAttributeErrors(job.Rows)); //get all attribute errors if ((job.FileNameOnly != null) && (job.FileNameOnly.StartsWith(Constants.UnzippedFilePrefix))) { //1: orginal file //job.FileName = ZipFileWatcher.ExtractActualFileName(job.FileName); File.Delete(fileName);//we do not need as zip will be archived //2: zip file (this is important if consumer wants to refer the file (zip file exists) job.FileName = Registry.Instance.ZipFiles[ZipFileWatcher.ExtractUniqueId(job.FileNameOnly)].ZipFileName; } if (job.JobCompleted != null) { //from WCF - we dont have to process output, wcf function will anyway call it job.JobCompleted.Set(); IdpeKey key = job.DataSource.Key(IdpeKeyTypes.WcfCallsGenerateStandardOutput); bool boolValue = false; if (key != null) { boolValue = key.Value.ParseBool(); } if (boolValue) { if (!job.AbortRequested) { job.TraceInformation("Calling output writer '{0}'. Total processed rows:{1}", job.DataSource.OutputWriterTypeFullName, job.Rows.Count); Trace.Flush(); return(job.DataSource.OutputWriter.GetOutput()); } else { return(new StringBuilder(string.Format("Job '{0}' was aborted! See server log for more details.", job.JobIdentifier))); } } else { return(new StringBuilder()); } } else { if ((job.IsFinished) && (!job.IsErrored) && (job.Errors.Count > 0)) { new PostMan(job).Send(); } if ((!job.AbortRequested) && (job.Rows.Count > 0)) { job.TraceInformation("Calling output writer '{0}'. Total processed rows:{1}", job.DataSource.OutputWriterTypeFullName, job.Rows.Count); Trace.Flush(); job.PerformanceCounter.Start(job.JobIdentifier, JobPerformanceTaskNames.OutputWriter); result = job.DataSource.OutputWriter.GetOutput(); job.PerformanceCounter.Stop(job.JobIdentifier, JobPerformanceTaskNames.OutputWriter); return(result); } else { return(new StringBuilder()); } } } catch (BusinessException ex) { job.TraceInformation(ex.Message); //It is actually not an exception or error new PostMan(job, false).Send(PostMan.__warningStartTag + ex.Message + PostMan.__warningEndTag); return(new StringBuilder()); } catch (Exception ex) { string errorId = Guid.NewGuid().ToString(); string errorMessage = errorId + ex.ToString() + (ex.InnerException == null ? string.Empty : ex.InnerException.Message); job.TraceError(errorMessage); Trace.Flush(); if (job != null) { job.TraceError("A server exception occurred. Please contact support. ErrorId = '{0}'. Server Time = '{1}'", errorId, DateTime.Now); } if (job != null) { if (job.JobCompleted != null) { //from WCF - we dont have to process output, wcf function will anyway call it job.JobCompleted.Set(); } job.Dispose(); } return(new StringBuilder()); } }