/// <summary>
/// Executes the node.
/// </summary>
/// <param name="runtime"></param>
/// <param name="scope"></param>
/// <returns></returns>
public override bool Execute(IFlowRuntimeService runtime, DataPinScope scope)
{
if (ftpServerService == null)
{
ftpServerService = CommonServiceLocator.ServiceLocator.Current.GetInstance <IFtpServerConfigurationService>();
}
var servername = scope.GetValue <string>(InPinServer);
var server = ftpServerService.GetByName(servername);
if (serviceCache.Keys.Contains(server.Type.ToString()))
{
ftpService = serviceCache[server.Type.ToString()];
}
else
{
ftpService = CommonServiceLocator.ServiceLocator.Current.GetInstance <IFtpService>(server.Type.ToString());
serviceCache.Add(server.Type.ToString(), ftpService);
}
var filename = scope.GetValue <string>(InPinFileName);
var path = scope.GetValue <string>(InPinPath);
var file = ftpService.DownloadFile(server, path + filename);
scope.SetValue(OutPinFile, file);
runtime.EnqueueNode(OutNodeSuccess, scope);
return(true);
}
public IList <Vehicle> GetVehiclesListFromFeedFiles(IList <SiteFeed> dealersFeeds)
{
List <Vehicle> globalVehiclesList = new List <Vehicle>();
foreach (var dealersFeed in dealersFeeds)
{
string lastModidiedFile = FtpService.LastModidiedFileInFolder(dealersFeed.FeedFolder);
if (FtpService.DownloadFile(LocalPath, lastModidiedFile))
{
IList <Vehicle> vehiclesFromCsv = dealersFeed.FeedProvider.ParseVehiclesFromCsv(File.ReadAllText(LocalPath), dealersFeed.SiteId);
globalVehiclesList.AddRange(vehiclesFromCsv);
}
}
return(globalVehiclesList);
}
/// <summary>
/// RunAsync
/// Entry point for the igc extraction and storage
/// </summary>
public async Task <IList <string> > RunAsync(int?maxFilesTpProcess = null)
{
var lastProcessedFilename = _firestoreService.GetLastProcessedFile();
var filesList = _ftpService.GetFileList();
// Remove files already processed from list
if (!string.IsNullOrEmpty(lastProcessedFilename))
{
filesList.RemoveAll(o => isFileAlreadyProcessed(lastProcessedFilename, o));
}
// #### Process files ###
var processedFilesList = new List <string>();
var processedItemCount = 0; // Keep track of processed items # so that we can regularly store the progress
var totalProcessedItemCount = 0;
string lastProcessedFileName = null;
foreach (var f in filesList)
{
_logger.LogInformation($"Dealing with: {f}");
// --- Get file from FTP ---
var fileStream = _ftpService.DownloadFile(f);
fileStream.Seek(0, SeekOrigin.Begin);
// --- Unzip stream file content into stream ---
var archive = new ZipArchive(fileStream);
var igcFile = archive.Entries[0];
var unzippedStream = igcFile.Open();
// --- Retrieve flight date ---
var isProcessingDone = false;
var targetFolderName = "";
await using (var igcStream = igcFile.Open())
{
try
{
var igcHeader = _igcReaderService.GetHeaderFromStream(igcStream);
targetFolderName = igcHeader.Date.ToString("yyyy_MM_dd") + "/";
isProcessingDone = true;
lastProcessedFileName = f;
igcStream.Close();
}
catch (Exception e)
{
_logger.LogDebug($"Could not extract header from file:{e.Message}");
isProcessingDone = false;
}
}
if (isProcessingDone)
{
// --- Store into a GCP bucket
_storageService.UploadToBucket(targetFolderName + igcFile.Name, unzippedStream);
processedFilesList.Add(f);
}
// --- Store progress ---
totalProcessedItemCount++;
processedItemCount++;
if (processedItemCount >= _configuration.StoreProgressInterval)
{
// Store progress in GCP Firestore so that we don't go over all files next time
_firestoreService.UpdateLastProcessedFile(f);
processedItemCount = 0;
}
// Clean up
unzippedStream.Close();
await unzippedStream.DisposeAsync();
fileStream.Close();
await fileStream.DisposeAsync();
if (maxFilesTpProcess != null && totalProcessedItemCount == maxFilesTpProcess)
{
break;
}
}
// --- Store last processed file progress
_firestoreService.UpdateLastProcessedFile(lastProcessedFileName); // Update last processed file. Null or Empty won't be taken into account
return(processedFilesList);
}
public void DownloadFile()
{
_ftp.DownloadFile("/filename.txt", "/local/filename.txt");
}
public void Start()
{
_logs.WriteLogLine("Start application");
try
{
_logs.WriteLogLine($"FileSourceWithPath= {_configuration.FileSourceWithPath}");
_logs.WriteLogLine($"DbConnectionString= {_configuration.DbConnectionString}");
_logs.WriteLogLine($"FileLogWithPath= {_configuration.FileLogWithPath}");
_logs.WriteLogLine($"FtpHost= {_configuration.FtpHost}");
_logs.WriteLogLine($"FtpUsername= {_configuration.FtpUsername}");
_logs.WriteLogLine($"FtpPassword= {_configuration.FtpPassword}");
_ftpService.DownloadFile(_configuration.FtpFileNameWithPath, _configuration.FileSourceWithPath);
var collBuffer = new List <string>();
using (var file = new StreamReader(_configuration.FileSourceWithPath))
{
string line;
bool firstLine = true;
while ((line = file.ReadLine()) != null)
{
_logs.WriteLogLine(line, LogType.Debug);
if (firstLine)
{
firstLine = false;
if (_configuration.SkipFirstLine)
{
_logs.WriteLogLine("Line skipped", LogType.Debug);
}
}
else
{
collBuffer.Add(line);
if (collBuffer.Count >= _configuration.RowsNumberBuffer)
{
_dbHelper.InsertRows(collBuffer);
collBuffer.Clear();
}
}
}
if (collBuffer.Count != 0)
{
_dbHelper.InsertRows(collBuffer);
}
}
}
catch (HelperException ex)
{
_logs.WriteLogLine($"From Helper: {ex.Message}", LogType.Error);
}
catch (FtpException ex)
{
_logs.WriteLogLine($"From FtpService: {ex.Message}", LogType.Error);
}
catch (DalException ex)
{
_logs.WriteLogLine($"From Dal: {ex.Message}", LogType.Error);
}
catch (Exception ex)
{
_logs.WriteLogLine($"{ex.Message}", LogType.Error);
}
_logs.WriteLogLine("End application");
}
