Пример #1
0
        /// <summary>
        /// Executes the node.
        /// </summary>
        /// <param name="runtime"></param>
        /// <param name="scope"></param>
        /// <returns></returns>
        public override bool Execute(IFlowRuntimeService runtime, DataPinScope scope)
        {
            if (ftpServerService == null)
            {
                ftpServerService = CommonServiceLocator.ServiceLocator.Current.GetInstance <IFtpServerConfigurationService>();
            }

            var servername = scope.GetValue <string>(InPinServer);
            var server     = ftpServerService.GetByName(servername);

            if (serviceCache.Keys.Contains(server.Type.ToString()))
            {
                ftpService = serviceCache[server.Type.ToString()];
            }
            else
            {
                ftpService = CommonServiceLocator.ServiceLocator.Current.GetInstance <IFtpService>(server.Type.ToString());
                serviceCache.Add(server.Type.ToString(), ftpService);
            }
            var filename = scope.GetValue <string>(InPinFileName);
            var path     = scope.GetValue <string>(InPinPath);
            var file     = ftpService.DownloadFile(server, path + filename);

            scope.SetValue(OutPinFile, file);
            runtime.EnqueueNode(OutNodeSuccess, scope);

            return(true);
        }
Пример #2
0
        public IList <Vehicle> GetVehiclesListFromFeedFiles(IList <SiteFeed> dealersFeeds)
        {
            List <Vehicle> globalVehiclesList = new List <Vehicle>();

            foreach (var dealersFeed in dealersFeeds)
            {
                string lastModidiedFile = FtpService.LastModidiedFileInFolder(dealersFeed.FeedFolder);
                if (FtpService.DownloadFile(LocalPath, lastModidiedFile))
                {
                    IList <Vehicle> vehiclesFromCsv = dealersFeed.FeedProvider.ParseVehiclesFromCsv(File.ReadAllText(LocalPath), dealersFeed.SiteId);
                    globalVehiclesList.AddRange(vehiclesFromCsv);
                }
            }
            return(globalVehiclesList);
        }
        /// <summary>
        /// RunAsync
        /// Entry point for the igc extraction and storage
        /// </summary>
        public async Task <IList <string> > RunAsync(int?maxFilesTpProcess = null)
        {
            var lastProcessedFilename = _firestoreService.GetLastProcessedFile();
            var filesList             = _ftpService.GetFileList();

            // Remove files already processed from list
            if (!string.IsNullOrEmpty(lastProcessedFilename))
            {
                filesList.RemoveAll(o => isFileAlreadyProcessed(lastProcessedFilename, o));
            }


            // #### Process files ###
            var    processedFilesList      = new List <string>();
            var    processedItemCount      = 0;         // Keep track of processed items # so that we can regularly store the progress
            var    totalProcessedItemCount = 0;
            string lastProcessedFileName   = null;

            foreach (var f in filesList)
            {
                _logger.LogInformation($"Dealing with: {f}");

                // --- Get file from FTP ---
                var fileStream = _ftpService.DownloadFile(f);
                fileStream.Seek(0, SeekOrigin.Begin);

                // --- Unzip stream file content into stream ---
                var archive        = new ZipArchive(fileStream);
                var igcFile        = archive.Entries[0];
                var unzippedStream = igcFile.Open();

                // --- Retrieve flight date ---
                var isProcessingDone = false;
                var targetFolderName = "";
                await using (var igcStream = igcFile.Open())
                {
                    try
                    {
                        var igcHeader = _igcReaderService.GetHeaderFromStream(igcStream);
                        targetFolderName      = igcHeader.Date.ToString("yyyy_MM_dd") + "/";
                        isProcessingDone      = true;
                        lastProcessedFileName = f;
                        igcStream.Close();
                    }
                    catch (Exception e)
                    {
                        _logger.LogDebug($"Could not extract header from file:{e.Message}");
                        isProcessingDone = false;
                    }
                }

                if (isProcessingDone)
                {
                    // --- Store into a GCP bucket
                    _storageService.UploadToBucket(targetFolderName + igcFile.Name, unzippedStream);
                    processedFilesList.Add(f);
                }

                // --- Store progress ---
                totalProcessedItemCount++;
                processedItemCount++;
                if (processedItemCount >= _configuration.StoreProgressInterval)
                {
                    // Store progress in GCP Firestore so that we don't go over all files next time
                    _firestoreService.UpdateLastProcessedFile(f);
                    processedItemCount = 0;
                }

                // Clean up
                unzippedStream.Close();
                await unzippedStream.DisposeAsync();

                fileStream.Close();
                await fileStream.DisposeAsync();

                if (maxFilesTpProcess != null && totalProcessedItemCount == maxFilesTpProcess)
                {
                    break;
                }
            }
            // --- Store last processed file progress
            _firestoreService.UpdateLastProcessedFile(lastProcessedFileName);       // Update last processed file. Null or Empty won't be taken into account

            return(processedFilesList);
        }
Пример #4
0
 public void DownloadFile()
 {
     _ftp.DownloadFile("/filename.txt", "/local/filename.txt");
 }
Пример #5
0
        public void Start()
        {
            _logs.WriteLogLine("Start application");

            try
            {
                _logs.WriteLogLine($"FileSourceWithPath= {_configuration.FileSourceWithPath}");
                _logs.WriteLogLine($"DbConnectionString= {_configuration.DbConnectionString}");
                _logs.WriteLogLine($"FileLogWithPath= {_configuration.FileLogWithPath}");
                _logs.WriteLogLine($"FtpHost= {_configuration.FtpHost}");
                _logs.WriteLogLine($"FtpUsername= {_configuration.FtpUsername}");
                _logs.WriteLogLine($"FtpPassword= {_configuration.FtpPassword}");

                _ftpService.DownloadFile(_configuration.FtpFileNameWithPath, _configuration.FileSourceWithPath);
                var collBuffer = new List <string>();
                using (var file = new StreamReader(_configuration.FileSourceWithPath))
                {
                    string line;
                    bool   firstLine = true;
                    while ((line = file.ReadLine()) != null)
                    {
                        _logs.WriteLogLine(line, LogType.Debug);
                        if (firstLine)
                        {
                            firstLine = false;
                            if (_configuration.SkipFirstLine)
                            {
                                _logs.WriteLogLine("Line skipped", LogType.Debug);
                            }
                        }
                        else
                        {
                            collBuffer.Add(line);
                            if (collBuffer.Count >= _configuration.RowsNumberBuffer)
                            {
                                _dbHelper.InsertRows(collBuffer);
                                collBuffer.Clear();
                            }
                        }
                    }

                    if (collBuffer.Count != 0)
                    {
                        _dbHelper.InsertRows(collBuffer);
                    }
                }
            }
            catch (HelperException ex)
            {
                _logs.WriteLogLine($"From Helper: {ex.Message}", LogType.Error);
            }
            catch (FtpException ex)
            {
                _logs.WriteLogLine($"From FtpService: {ex.Message}", LogType.Error);
            }
            catch (DalException ex)
            {
                _logs.WriteLogLine($"From Dal: {ex.Message}", LogType.Error);
            }
            catch (Exception ex)
            {
                _logs.WriteLogLine($"{ex.Message}", LogType.Error);
            }

            _logs.WriteLogLine("End application");
        }