public ServiceOfferingController(IServiceOfferingManager logicManager, IServiceTypeManager serviceTypeManager, IProviderManager providerManager, IProgramManager programManager, IFileProcessor fileProcessor)
{
if (logicManager == null)
{
throw new ArgumentNullException("logicManager");
}
if (serviceTypeManager == null)
{
throw new ArgumentNullException("serviceTypeManager");
}
if (providerManager == null)
{
throw new ArgumentNullException("providerManager");
}
if (programManager == null)
{
throw new ArgumentNullException("programManager");
}
if (fileProcessor == null)
{
throw new ArgumentNullException("fileProcessor");
}
LogicManager = logicManager;
ServiceTypeManager = serviceTypeManager;
ProviderManager = providerManager;
ProgramManager = programManager;
FileProcessor = fileProcessor;
}
/// <summary>
/// Construct a new FileProcessorHost
/// </summary>
/// <param name="processors"></param>
/// <remarks>Have to use array rather then IList due to Unity restrictions</remarks>
public FileProcessorHost(IFileProcessor[] processors)
{
if (processors == null)
{
throw new ArgumentNullException("processors");
}
this.processors = new List<IFileProcessor>(processors);
}
public FileDetailViewModel(string name, IDataService dataService, IViewThreadExecutor viewThreadExecutor, IFileProcessor fileProcessor, IAccTransactionValidator validator)
{
Name = name;
_dataService = dataService;
_viewThreadExecutor = viewThreadExecutor;
_fileProcessor = fileProcessor;
_validator = validator;
UploadCommand = new DelegateCommand(() => OnUploadData(), () => (Status==Status.NotUploaded) && (!IsUploading) );
ErrorList = new ObservableCollection<string>();
}
public FileProcessorWorker(IFileProcessor fileProcessor, string originalFilename)
: this()
{
this.fileProcessor = fileProcessor;
this.originalFilename = originalFilename;
if (fileProcessor is IThreadFileProcessor)
{
((IThreadFileProcessor)fileProcessor).Progress = new WorkerProgressCallback() { Worker = this };
}
}
public ServiceAttendanceController(IServiceAttendanceManager logicManager, IFileProcessor fileProcessor)
{
if (logicManager == null)
{
throw new ArgumentNullException("logicManager");
}
if (fileProcessor == null)
{
throw new ArgumentNullException("fileProcessor");
}
LogicManager = logicManager;
FileProcessor = fileProcessor;
}
public BitbucketService(
IBitbucketApiService apiService,
IRepository<BitbucketRepositoryDataRecord> repository,
IJobManager jobManager,
IFileProcessor fileProcessor,
IContentManager contentManager,
IEncryptionService encryptionService)
{
_apiService = apiService;
_jobManager = jobManager;
_repository = repository;
_fileProcessor = fileProcessor;
_contentManager = contentManager;
_encryptionService = encryptionService;
}
public IDictionary<string, DataSet> ProcessAllFiles(IFileProcessor fileProcessor)
{
if (fileProcessor == null)
{
throw new ArgumentNullException("fileProcessor");
}
if (!Directory.Exists(unprocessedFilesPath) && !Directory.Exists(processedFilesPath))
{
throw new BatchFileProcessorException("HELP!!!");
}
var allFiles = Directory.EnumerateFiles(unprocessedFilesPath);
successfullyProcessedFiles = new List<string>();
unsuccessfullyProcessedFiles = new List<string>();
var toReturn = new Dictionary<string, DataSet>();
int successfullyProcessed = 0;
foreach (var filename in allFiles)
{
var shortFilename = new FileInfo(filename).Name;
DataSet data = ProcessFile(filename, fileProcessor);
if (data.Tables.Count > 0)
{
successfullyProcessed++;
toReturn.Add(filename, data);
successfullyProcessedFiles.Add(filename);
}
else
{
unsuccessfullyProcessedFiles.Add(filename);
}
}
return toReturn;
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="processor">The file processor to use for file persistance.</param>
/// <param name="boundary">The boundary (delimiter) of the post.</param>
/// <param name="encoding">The encoder of the request stream.</param>
public FormStream(IFileProcessor processor, string boundary, Encoding encoding)
{
_defaultProcessor = processor;
_processor = processor;
_formContent = new MemoryStream();
_encoding = encoding;
_headerNeeded = true;
_position = 0;
_buffer = null;
_inField = false;
_inField = false;
_keepBackLength = boundary.Length + 6;
_fileError = false;
_previousFields = new Dictionary<string, string>();
BOUNDARY = _encoding.GetBytes(boundary);
EOF = _encoding.GetBytes(boundary + "--\r\n");
EOH = _encoding.GetBytes("\r\n\r\n");
CRLF = _encoding.GetBytes("\r\n");
ID_TAG = _encoding.GetBytes(DJUploadController.UPLOAD_ID_TAG);
DEFAULT_PARAMS_TAG = _encoding.GetBytes(DJUploadController.UPLOAD_DEFAULT_PARAMETER_TAG);
PARAMS_TAG = _encoding.GetBytes(DJFileUpload.UPLOAD_PARAMETER_TAG);
END_TAG = _encoding.GetBytes(DJFileUpload.UPLOAD_END_TAG);
}
/// <summary>
/// Checks the current field for actions (ID tag, processor parameters, etc).
/// </summary>
void CheckForActionFields()
{
byte[] bytes = _currentField.ToArray();
string statusKey = string.Empty;
int boundaryPos = IndexOf(bytes, BOUNDARY);
// Status ID field
if (!TryParseActionField(ID_TAG, bytes, out statusKey, boundaryPos))
{
string field = String.Empty;
// Processor parameters
if (TryParseActionField(DEFAULT_PARAMS_TAG, bytes, out field, boundaryPos))
{
// Switch the default processor
_defaultProcessor = UploadManager.Instance.DeserializeProcessor(field.Substring(DEFAULT_PARAMS_TAG.Length));
_processor = _defaultProcessor;
_currentField = new MemoryStream();
}
else if (TryParseActionField(PARAMS_TAG, bytes, out field, boundaryPos))
{
// Switch to the new processor
_processor = UploadManager.Instance.DeserializeProcessor(field.Substring(PARAMS_TAG.Length));
_currentField = new MemoryStream();
}
else if (TryParseActionField(END_TAG, bytes, out field, boundaryPos))
{
// Revert to the default processor
_processor = _defaultProcessor;
_currentField = new MemoryStream();
}
else
{
_currentFieldName = _currentFieldName.Replace("\"", String.Empty);
if (!_previousFields.ContainsKey(_currentFieldName))
{
try
{
string content;
if (boundaryPos > 0)
{
content = _encoding.GetString(bytes, 0, boundaryPos - 2);
}
else
{
content = _encoding.GetString(bytes, 0, bytes.Length - 2);
}
_previousFields.Add(_currentFieldName, content);
}
catch
{
_previousFields.Add(_currentFieldName, null);
}
}
}
}
else
{
// Save the status key
_statusKey = statusKey;
_currentField = new MemoryStream();
}
}
public DownloaderApplication(IFileOperations fileOperations, IStateSerializer serializer, IFileProcessor fileProcessor)
{
_fileOperations = fileOperations;
_serializer = serializer;
_fileProcessor = fileProcessor;
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="processor">File processor instance.</param>
public FileProcessorInitEventArgs(IFileProcessor processor)
{
_processor = processor;
}
/// <summary>
/// Serializes a processor to a string. The string is encrypted and validated.
/// Thanks to Dean Brettle for help with this http://www.brettle.com.
/// </summary>
/// <param name="processor">Processor to serialize.</param>
/// <returns>The serialized processor.</returns>
internal string SerializeProcessor(IFileProcessor processor)
{
SettingsStorageObject so = new SettingsStorageObject();
MemoryStream ms = new MemoryStream();
MemoryStream outStream = new MemoryStream();
BinaryFormatter bf = new BinaryFormatter();
bf.Serialize(ms, processor);
// Encrypt the serialized object
MemoryStream cipherTextStream = new MemoryStream();
SymmetricAlgorithm cipher = SymmetricAlgorithm.Create();
cipher.Mode = CipherMode.CBC;
cipher.Padding = PaddingMode.PKCS7;
cipher.Key = DefaultEncryptionKey;
CryptoStream cryptoStream = new CryptoStream(cipherTextStream, cipher.CreateEncryptor(), CryptoStreamMode.Write);
byte[] cryptoBytes = ms.ToArray();
cryptoStream.Write(cryptoBytes, 0, cryptoBytes.Length);
cryptoStream.Close();
so.CipherText = cipherTextStream.ToArray();
so.CipherIV = cipher.IV;
// Generate a hash for the encrypted data
KeyedHashAlgorithm kh = KeyedHashAlgorithm.Create();
kh.Key = DefaultValidationKey;
so.Hash = kh.ComputeHash(so.CipherText);
bf.Serialize(outStream, so);
return Convert.ToBase64String(outStream.ToArray());
}
/// <summary>
/// Ends the work.
/// </summary>
protected override void EndWork()
{
base.EndWork();
m_FileProcessor.Dispose();
m_FileProcessor = null;
}
static int Main(string[] args)
{
Init();
try
{
if (args.Length < 3)
{
Console.WriteLine("Input error");
return 1;
}
var multipartFlag = 0;
if (args.Length==4)
{
multipartFlag = int.Parse(args[3]);
}
switch (args[0].ToLower())
{
case "compress":
_fileProcessor = GetCompressor(multipartFlag);
break;
case "decompress":
_fileProcessor = GetDecompressor(multipartFlag);
break;
default:
Console.WriteLine("Input error");
return 1;
}
var input = new FileInfo(args[1]);
if (!input.Exists)
{
Console.WriteLine("Input file does not exists");
}
var output = new FileInfo(args[2]);
if (output.Exists)
{
File.WriteAllText(output.FullName, string.Empty);
}
_timer.Start();
_fileProcessor.Start(input,output);
_fileProcessor.Join();
_timer.Stop();
if (_fileProcessor.State == EFileProcessorState.Failed)
{
Console.WriteLine("File processing failed due to following error(s):");
if (_fileProcessor.Exceptions.Count != 0)
foreach (var exception in _fileProcessor.Exceptions)
{
Console.WriteLine(exception.Message);
}
return 1;
}
PrintInfo(input,output,_timer.Elapsed);
}
catch (Exception e)
{
Console.WriteLine("Exception occured in main thread: \n {0}",e.Message);
return 1;
}
return 0;
}
/// <summary>
/// Gets a new file processor from the upload manager.
/// </summary>
/// <returns>A file processor.</returns>
IFileProcessor GetProcessor()
{
_processor = UploadManager.Instance.GetProcessor();
return _processor;
}
public DataSet ProcessFile(string filename, IFileProcessor fileProcessor)
{
if (string.IsNullOrWhiteSpace(filename))
{
throw new ArgumentNullException("filename");
}
if (fileProcessor == null)
{
throw new ArgumentNullException("fileProcessor");
}
var fullFileName = Path.Combine(this.unprocessedFilesPath, filename);
if (!File.Exists(fullFileName))
{
throw new FileNotFoundException("File does not exist at location specified!", filename);
}
try
{
DataSet data = fileProcessor.ProcessFile(fullFileName);
if (data != null && data.Tables.Count > 0)
{
bool successfulMove = MoveFileToProcessedLocation(fullFileName);
if (successfulMove)
{
data.DataSetName = fullFileName;
Log.DebugFormat("Success");
return data;
}
else
{
Log.ErrorFormat("Unable to process file {0}", fullFileName);
return new DataSet();
}
}
else
{
bool successfulMove = MoveFileToErrorLocation(fullFileName);
if (successfulMove)
{
data.DataSetName = fullFileName;
Log.DebugFormat("Success");
return data;
}
else
{
Log.ErrorFormat("Unable to process file {0}", fullFileName);
return new DataSet();
}
}
}
catch (Exception ex)
{
Log.ErrorFormat("Unable to process file {0} {1}", filename, ex);
}
return new DataSet();
}
/// <summary>
/// constructor with compute node
/// </summary>
/// <param name="compute_node"></param>
//public FileSaturator(IComputeNode compute_node)
//{
// ComputeNode = compute_node;
// SaturationThreadCount = Math.Min(64,Environment.ProcessorCount * 4);
// //SaturationThreadCount = 1;
// SaturationFiles = new List<FileData>();
// LastFileDataRefresh = DateTime.UtcNow.AddDays(-1);
//}
public FileSaturator(IComputeNode compute_node, IFileProcessor file_processor, string catalog_name, string table_name)
{
ComputeNode = compute_node;
CatalogName = catalog_name;
TableName = table_name;
FileProcessor = file_processor;
//SaturationThreadCount = Math.Min(64,Environment.ProcessorCount * 4);
SaturationThreadCount = 1;
SaturationFiles = new List<FileData>();
LastFileDataRefresh = DateTime.UtcNow.AddDays(-1);
}
public ConsoleFileProcessorWorker(IFileProcessor fileProcessor, string originalFilename)
: base(fileProcessor, originalFilename)
{
ProgressChanged += ConsoleProgressChanged;
RunWorkerCompleted += ConsoleRunWorkerCompleted;
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="processor">The file processor to use for file persistance.</param>
/// <param name="boundary">The boundary (delimiter) of the post.</param>
/// <param name="encoding">The encoder of the request stream.</param>
public FormStream(IFileProcessor processor, string boundary, Encoding encoding)
{
_processor = processor;
_formContent = new MemoryStream();
_encoding = encoding;
_headerNeeded = true;
_position = 0;
_buffer = null;
_inField = false;
_inField = false;
_keepBackLength = boundary.Length + 6;
_fileError = false;
BOUNDARY = _encoding.GetBytes(boundary);
EOF = _encoding.GetBytes(boundary + "--\r\n");
EOH = _encoding.GetBytes("\r\n\r\n");
CRLF = _encoding.GetBytes("\r\n");
ID_TAG = _encoding.GetBytes(DJUploadController.UPLOAD_ID_TAG);
}
public void SetupTest()
{
_fileProcessor = new TextFileProcessor();
}
/// <summary>
/// Fires the ProcessorInit event.
/// </summary>
/// <param name="processor">File processor.</param>
public void OnProcessorInit(IFileProcessor processor)
{
if (ProcessorInit != null)
ProcessorInit(this, new FileProcessorInitEventArgs(processor));
}
/// <summary>
/// Processes all upload controls in a control collection when the module is not installed.
/// </summary>
/// <param name="cc">Control collection.</param>
/// <param name="defaultProcessor">The default processor.</param>
/// <param name="status">The upload status.</param>
void ProcessUploadControls(ControlCollection cc, IFileProcessor defaultProcessor, UploadStatus status)
{
foreach (Control c in cc)
{
System.Web.UI.WebControls.FileUpload fu = c as System.Web.UI.WebControls.FileUpload;
if (fu != null && fu.HasFile)
{
IFileProcessor controlProcessor = GetProcessorForControl(fu);
IFileProcessor processor = controlProcessor == null ? defaultProcessor : controlProcessor;
try
{
processor.StartNewFile(fu.FileName, fu.PostedFile.ContentType, null, null);
processor.Write(fu.FileBytes, 0, fu.FileBytes.Length);
processor.EndFile();
status.UploadedFiles.Add(new UploadedFile(fu.FileName, processor.GetIdentifier(), null));
}
catch(Exception ex)
{
status.ErrorFiles.Add(new UploadedFile(fu.FileName, processor.GetIdentifier(), null, ex));
}
}
if (c.HasControls())
{
ProcessUploadControls(c.Controls, defaultProcessor, status);
}
}
}
/// <summary>
/// Begins the work.
/// </summary>
protected override void BeginWork()
{
base.BeginWork();
try
{
m_Parameters = DocumentImportHelper.GetProfileBeo((string)BootParameters);
m_CounterForCorrelationId = 0;
InitializeConfigurationItems();
//?? need to percentage completion
m_PercenatgeCompletion = 100;
m_Documents = new List<RVWDocumentBEO>();
#region Get Dataset Details
if (m_Parameters != null && m_Parameters.DatasetDetails.FolderID > 0)
{
m_FileProcessor = FileProcessorFactory.CreateFileProcessor(
FileProcessorFactory.ExtractionChoices.CompoundFileExtraction);
m_Dataset = DataSetBO.GetDataSetDetailForDataSetId(m_Parameters.DatasetDetails.FolderID);
if (m_Dataset.Matter != null && m_Dataset.Matter.FolderID > 0)
{
var matterDetails = MatterDAO.GetMatterDetails(m_Dataset.Matter.FolderID.ToString());
if (matterDetails != null)
{
m_Dataset.Matter = matterDetails;
var searchServerDetails = ServerDAO.GetSearchServer(matterDetails.SearchServer.Id);
if (searchServerDetails != null)
{
m_Dataset.Matter.SearchServer = searchServerDetails;
}
}
}
else
throw new EVException().AddErrorCode(ErrorCodes.EDLoaderExtractionWorker_FailedToObtainMatterDetails); //?? need to set message in resource file
}
else
{
throw new EVException().AddErrorCode(ErrorCodes.EDLoaderExtractionWorker_ObtainDatasetDetailsFailure); //?? need to set message in resource file
}
#endregion
}
catch (Exception ex)
{
ex.Trace().Swallow();
}
}
private void ParseKVP(string[] kvp)
{
switch (kvp[0])
{
case "data source":
{
_datasource = kvp[1];
Console.WriteLine("Parsed {0} as: {1}", kvp[0], kvp[1]);
break;
}
case "initial catalog":
{
_database = kvp[1];
Console.WriteLine("Parsed {0} as: {1}", kvp[0], kvp[1]);
break;
}
case "file access":
{
if (Enum.TryParse<FileAccess>(kvp[1], out _fileAccess))
{
Console.WriteLine("Parsed {0} as: {1}", kvp[0], _fileAccess.ToString());
}
else
{
_fileAccess = FileAccess.Read;
Console.WriteLine("Unable to parse {0} defaulting to Read", kvp[0]);
}
break;
}
case "file type":
{
Type fileType = Type.GetType(kvp[1]);
Console.WriteLine("Parsed {0} as {1}", kvp[0], fileType.ToString());
if (fileType.GetInterface(typeof(IFileProcessor).ToString()) != null)
{
_fileProcessor = Activator.CreateInstance(fileType) as IFileProcessor;
}
break;
}
}
}
public FileReader(IFileProcessor fileProcessor)
{
_fileProcessor = fileProcessor;
}