Example #1
0
        protected async override Task RunInternal()
        {
            using (_importJobRepository.BeginOperation())
            {
                var          newFileStatus = FileStatus.Success;
                ImportedFile currentFile   = null;
                try
                {
                    currentFile = _importJobRepository
                                  .ImportedFilesQuery()
                                  .OrderByDescending(
                        f => f.OriginalFileName.StartsWith(_fileNameMatcher.AcctStatusReportNonEcaSqlLike))
                                  .ThenByDescending(f => f.OriginalFileName.StartsWith(_fileNameMatcher.AcctStatusReportSqlLike))
                                  .ThenByDescending(f => f.OriginalFileName.Contains(_fileNameMatcher.SytossClientInfoSqlLike))
                                  .ThenBy(f => f.FileCreateDate)
                                  .FirstOrDefault(f => f.FileState == FileState.Loaded &&
                                                  f.FileStatus == FileStatus.Success);

                    if (currentFile == null)
                    {
                        _interval *= 2;

                        if (_interval > 30000)
                        {
                            _interval = 30000;
                        }

                        return;
                    }

                    _interval = 300;

                    //retrieving steam of current file
                    var streamFile = _extractFileService.ExtractFile(currentFile.OriginalFileName);

                    if (_fileNameMatcher.IsFeesDataFile(currentFile.OriginalFileName))
                    {
                        newFileStatus = await ProcessFeesFileReport(currentFile, streamFile);
                    }
                    else if (_fileNameMatcher.IsAcctStatusReportNonEca(currentFile.OriginalFileName))
                    {
                        newFileStatus = await ProcessAcctStatusReportFileReport(currentFile,
                                                                                "AccountStatusReport",
                                                                                _fileNameMatcher.AcctStatusReportNonEcaSqlLike, streamFile);
                    }
                    else if (_fileNameMatcher.IsNavRegex(currentFile.OriginalFileName))
                    {
                        newFileStatus = await ProcessNavFileReport(currentFile, streamFile);
                    }
                    else if (_fileNameMatcher.IsAcctStatusReport(currentFile.OriginalFileName))
                    {
                        newFileStatus = await ProcessAcctStatusReportFileReport(currentFile,
                                                                                "ECAAccountStatusReport",
                                                                                _fileNameMatcher.AcctStatusReportSqlLike, streamFile);
                    }
                    else if (_fileNameMatcher.IsCashReport(currentFile.OriginalFileName))
                    {
                        newFileStatus = await ProcessCashReport(currentFile, streamFile);
                    }
                    else if (_fileNameMatcher.IsTradeAsReport(currentFile.OriginalFileName))
                    {
                        newFileStatus = await ProcessTradesAsReport(currentFile, streamFile);
                    }
                    else if (_fileNameMatcher.IsSytossClientInfo(currentFile.OriginalFileName))
                    {
                        newFileStatus = await ProcessSytossClientInfoReport(currentFile, streamFile);
                    }
                }
                catch (Exception ex)
                {
                    newFileStatus = FileStatus.Failed;
                    loggerException.Error(this.GetErrorLogMessage(ex));
                }
                finally
                {
                    if (currentFile != null)
                    {
                        currentFile.FileState  = FileState.Imported;
                        currentFile.FileStatus = newFileStatus;

                        _importJobRepository.UpdateImportedFile(currentFile);

                        _importJobRepository.SaveChanges();

                        if (newFileStatus == FileStatus.Failed)
                        {
                            Console.WriteLine($"ImportJob: Error while importing file {currentFile.OriginalFileName}");
                            logger.Log(LogLevel.Info,
                                       $"import$ImportJob: Error while importing file {currentFile.OriginalFileName}");

                            Thread.Sleep(1000 * 10);
                            // ReloadThread();
                        }
                        else
                        {
                            Console.WriteLine($"ImportJob: {currentFile.OriginalFileName} has been imported");
                            logger.Log(LogLevel.Info,
                                       $"import$ImportJob: {currentFile.OriginalFileName} has been imported");
                        }
                    }
                    else
                    {
                        logger.Log(LogLevel.Info,
                                   $"import$ImportJob: has not found new files for downloading. Waiting...");
                        Console.WriteLine("ImportJob: has not found new files for downloading. Waiting...");
                    }
                }
            }
        }
Example #2
0
        protected override async Task RunInternal()
        {
            SyncerInfoService.Log(GetType().Name, JobStatus.Started.ToString());

            using (_ftpJobRepository.BeginOperation())
            {
                var ftpCredentials = _ftpJobRepository.GetFtpCredentialList();

                if (!ftpCredentials.Any())
                {
                    GlobalLogger.LogInfo("Job has not found any ftp credentials. Waiting...", GetType().Name, true);
                }
                else
                {
                    var existingFiles     = _ftpJobRepository.ImportedFileList();
                    var existingFileNames = existingFiles.Select(ef => ef.OriginalFileName);

                    foreach (var cred in ftpCredentials)
                    {
                        var totalCount = 0;
                        List <(string fileName, string accountName, DateTime creationDate)> newFiles;
                        var accountNameList = cred.MasterAccounts.Select(m => m.AccountName).ToList();

                        try
                        {
                            newFiles = (await _ftpLoader.LoadFilesAsync(cred))
                                       .Where(f => f.creationDate >= _fromDate && accountNameList.Contains(f.accountName) && !existingFileNames.Contains(f.fileName))
                                       // order is important for file processing
                                       .OrderByDescending(f => _fileNameMatcher.IsAcctStatusReportNonEca(f.fileName))
                                       .ThenByDescending(f => _fileNameMatcher.IsAcctStatusReport(f.fileName))
                                       .ThenByDescending(f => f.creationDate)
                                       .ToList();
                        }
                        catch (Exception ex)
                        {
                            GlobalLogger.LogError($"Error while trying to load files from ftp: [{cred.FtpName}]", ex, GetType().Name, true);
                            return;
                        }

                        FilterFilesByMaxCreateDate(newFiles, existingFiles, _fileNameMatcher.AcctStatusReportNonEcaSqlLike, cred.MasterAccounts);
                        FilterFilesByMaxCreateDate(newFiles, existingFiles, _fileNameMatcher.AcctStatusReportSqlLike, cred.MasterAccounts);
                        FilterFilesByMaxCreateDate(newFiles, existingFiles, _fileNameMatcher.SytossClientInfoSqlLike, cred.MasterAccounts);

                        if (!newFiles.Any())
                        {
                            GlobalLogger.LogInfo($"Job has not found new files in [{cred.FtpName}]. Waiting...", GetType().Name, true);
                        }
                        else
                        {
                            GlobalLogger.LogInfo($"Job has found new: [{newFiles.Count()}] files on ftp: [{cred.FtpName}].", GetType().Name, true);

                            var transitFiles = _ftpJobRepository.GetTransitFileList();

                            foreach (var file in newFiles)
                            {
                                if (!existingFiles.All(f => f.OriginalFileName != file.fileName))
                                {
                                    continue;
                                }

                                var newFile = new ImportedFile()
                                {
                                    CreatedDate      = DateTime.UtcNow,
                                    ModifiedDate     = DateTime.UtcNow,
                                    OriginalFileName = file.fileName,
                                    FileCreateDate   = file.creationDate,
                                    MasterAccount    = cred.MasterAccounts.FirstOrDefault(a => a.AccountName == file.accountName),
                                    FtpCredential    = cred,
                                    FileState        = FileState.Registered,
                                    FileStatus       = FileStatus.Success,
                                    RegisteredDate   = DateTime.UtcNow
                                };

                                if (transitFiles.Any(f => f.AccountName == file.accountName && file.fileName.Contains(f.OriginalFileName)))
                                {
                                    newFile.FileUpload = new FileUpload();
                                }

                                _ftpJobRepository.AddImportedFile(newFile);

                                totalCount++;

                                GlobalLogger.LogInfo($"New file: {newFile.OriginalFileName} registered from ftp: [{cred.FtpName}].", GetType().Name, true);
                            }


                            try
                            {
                                _ftpJobRepository.SaveChanges();
                                GlobalLogger.LogInfo($"Amount registered: [{totalCount}] files in DB from ftp: [{cred.FtpName}].", GetType().Name, true);
                            }
                            catch (Exception ex)
                            {
                                GlobalLogger.LogError($"Error while trying to save registered files from ftp: [{cred.FtpName}]", ex, GetType().Name, true);
                                return;
                            }
                        }
                    }
                }
            }

            SyncerInfoService.Log(GetType().Name, JobStatus.Stopped.ToString());
        }
Example #3
0
        private async Task TryImportFile(ImportedFile currentFile)
        {
            var newFileStatus = FileStatus.Success;

            try
            {
                if (currentFile != null)
                {
                    GlobalLogger.LogInfo($"Start importing file: {currentFile.OriginalFileName} | Status: {currentFile.FileStatus}", GetType().Name, true);

                    var streamFile = _extractFileService.ExtractFile(currentFile.OriginalFileName);

                    if (_fileNameMatcher.IsFeesDataFile(currentFile.OriginalFileName))
                    {
                        newFileStatus = await ProcessFeesFileReport(currentFile, streamFile);
                    }
                    else if (_fileNameMatcher.IsAcctStatusReportNonEca(currentFile.OriginalFileName))
                    {
                        newFileStatus = await ProcessAcctStatusReportFileReport(
                            currentFile,
                            "AccountStatusReport",
                            _fileNameMatcher.AcctStatusReportNonEcaSqlLike,
                            streamFile);
                    }
                    else if (_fileNameMatcher.IsNavRegex(currentFile.OriginalFileName))
                    {
                        newFileStatus = await ProcessNavFileReport(currentFile, streamFile);
                    }
                    else if (_fileNameMatcher.IsAcctStatusReport(currentFile.OriginalFileName))
                    {
                        newFileStatus = await ProcessAcctStatusReportFileReport(
                            currentFile,
                            "ECAAccountStatusReport",
                            _fileNameMatcher.AcctStatusReportSqlLike,
                            streamFile);
                    }
                    else if (_fileNameMatcher.IsCashReport(currentFile.OriginalFileName))
                    {
                        newFileStatus = await ProcessCashReport(currentFile, streamFile);
                    }
                    else if (_fileNameMatcher.IsTradeAsReport(currentFile.OriginalFileName))
                    {
                        newFileStatus = await ProcessTradesAsReport(currentFile, streamFile);
                    }
                    else if (_fileNameMatcher.IsSytossClientInfo(currentFile.OriginalFileName))
                    {
                        newFileStatus = await ProcessSytossClientInfoReport(currentFile, streamFile);
                    }
                    else if (_fileNameMatcher.IsSytossOpenPositions(currentFile.OriginalFileName))
                    {
                        newFileStatus = await ProcessSytossOpenPositions(currentFile, streamFile);
                    }
                    else if (_fileNameMatcher.IsTradeExeReport(currentFile.OriginalFileName))
                    {
                        newFileStatus = await ProcessTradesExeReport(currentFile, streamFile);
                    }
                    else if (_fileNameMatcher.IsInterestAccruaReport(currentFile.OriginalFileName))
                    {
                        newFileStatus = await ProcessTradeInterestAccruaReport(currentFile, streamFile);
                    }
                    else if (_fileNameMatcher.IsTradeCommissionsDetReport(currentFile.OriginalFileName))
                    {
                        newFileStatus = await ProcessTradeCommissionsReport(currentFile, streamFile);
                    }
                }

                GlobalLogger.LogInfo($"File: {currentFile.OriginalFileName} has been imported.", GetType().Name, true);
            }
            catch (Exception ex)
            {
                newFileStatus = FileStatus.Failed;
                GlobalLogger.LogError($"Error while importing file {currentFile?.OriginalFileName}.", ex, GetType().Name, true);
            }
            finally
            {
                if (currentFile != null)
                {
                    currentFile.FileState  = FileState.Imported;
                    currentFile.FileStatus = newFileStatus;
                    if (newFileStatus == FileStatus.Success)
                    {
                        currentFile.ImportedDate = DateTime.UtcNow;
                    }

                    _importJobRepository.UpdateImportedFile(currentFile);
                    _importJobRepository.SaveChanges();
                }
                else
                {
                    GlobalLogger.LogInfo("Job has not found new files for importing. Waiting...", GetType().Name, true);
                }
            }
        }