private static IProcessor GetProcessor(CollarFile file, GetTelonicsParametersForArgosDatesResult parameters)
{
switch (file.Format)
{
case 'E':
case 'F':
switch (parameters.CollarModel)
{
case "Gen3":
return(GetGen3Processor(parameters));
case "Gen4":
return(GetGen4Processor(parameters));
case "GPS8000":
return(GetGps8000Processor(parameters));
default:
throw new InvalidOperationException("Unsupported collar model '" + parameters.CollarModel +
"'. (supported models are Gen3, and Gen4)");
}
case 'G':
return(new DebevekProcessor());
default:
throw new InvalidOperationException("Unsupported CollarFile Format '" + file.Format +
"'. (supported formats are E,F,G)");
}
}
private static void ProcessDataLogFile(CollarFile file)
{
LogGeneralMessage(String.Format("Start local processing of file {0}", file.FileId));
var databaseFunctions = new AnimalMovementFunctions();
//FIXME: check that this is generic to all telonics processing and not just Argos files
databaseFunctions.ArgosFile_ClearProcessingResults(file.FileId);
var processor = new Gen4Processor(null);
var lines = processor.ProcessDataLog(file.Contents.ToArray());
//Add a newline, so that it exactly matches the direct output from TDC
var data = Encoding.UTF8.GetBytes(String.Join(Environment.NewLine, lines) + Environment.NewLine);
var filename = Path.GetFileNameWithoutExtension(file.FileName) + "_" + DateTime.Now.ToString("yyyyMMdd") + ".csv";
var fileLoader = new FileLoader(filename, data)
{
Project = file.Project,
Owner = file.ProjectInvestigator,
Collar = new Collar
{
CollarManufacturer = file.CollarManufacturer,
CollarId = file.CollarId
},
Status = file.Status,
ParentFileId = file.FileId,
AllowDuplicates = true
};
fileLoader.Load();
LogGeneralMessage("Finished local processing of file");
}
private static bool ProcessOnServer(CollarFile file, ArgosPlatform platform = null)
{
if (NeedTelonicsSoftware(file) && !HaveAccessToTelonicsSoftware())
{
if (OnDatabaseServer())
{
throw new InvalidOperationException("No access to Telonics software to process files.");
}
LogGeneralMessage(String.Format("Start processing file {0} on database", file.FileId));
var database = new AnimalMovementFunctions();
//FIXME: ProcessOnServer may be called with H and I (Teloncs files which need processing, but are not ArgosFiles)
//FIXME: If called by Animal Movement App Upload Form on client machine with H or I files, this code will fail
// if teloncs file but not argos
// database.TelonicsData_Process(file.FileId);
// if not telonics or not Argos then fail with warning
if (platform == null)
{
database.ArgosFile_Process(file.FileId);
}
else
{
database.ArgosFile_ProcessPlatform(file.FileId, platform.PlatformId);
}
LogGeneralMessage("Finished processing file on database");
return(true);
}
return(false);
}
/// <summary>
/// Validates and loads the file to the database
/// </summary>
/// <returns>Returns the newly created collar file, complete with database calculated fields</returns>
public CollarFile Load()
{
Validate();
if (FileFormatRequiresCollar && Collar == null)
{
Collar = FileCollar;
}
// The entity objects I got from the callers (i.e. Project, Owner, Collar)
// came from a foreign DataContext, so they cannot be used as associated
// entities with a new entity in this datacontext.
var file = new CollarFile
{
ProjectId = Project == null ? null : Project.ProjectId,
FileName = Path.GetFileName(FilePath),
CollarManufacturer = Collar == null ? null : Collar.CollarManufacturer,
CollarId = Collar == null ? null : Collar.CollarId,
Owner = Owner == null ? null : Owner.Login,
Status = Status,
Contents = Contents,
ParentFileId = ParentFileId,
ArgosDeploymentId = ArgosDeploymentId,
CollarParameterId = CollarParameterId
};
Database.CollarFiles.InsertOnSubmit(file);
Database.SubmitChanges();
//Linq TO SQL Insert with SPROC dos not set associations, and provides not partial methods to expand
file.LookupCollarFileFormat = Database.LookupCollarFileFormats.First(l => l.Code == file.Format);
return(file);
}
/// <summary>
/// Retrieves data for an Argos Platform from the Argos Web Server
/// </summary>
/// <param name="platform">The Argos Platform to retrieve from the server</param>
/// <param name="daysToRetrieve">The number of most recent days to retrieve from the server</param>
public static void DownloadArgosPlatform(ArgosPlatform platform, int?daysToRetrieve = null)
{
int daysSinceLastDownload;
if (daysToRetrieve.HasValue)
{
daysSinceLastDownload = daysToRetrieve.Value;
}
else
{
var database = new AnimalMovementDataContext();
var dateOfLastDownload = (from log in database.ArgosDownloads
where log.PlatformId == platform.PlatformId && log.FileId != null
orderby log.TimeStamp descending
select log.TimeStamp).FirstOrDefault();
daysSinceLastDownload = (DateTime.Now - dateOfLastDownload).Days;
}
var days = Math.Min(ArgosWebSite.MaxDays, daysSinceLastDownload);
if (days < ArgosWebSite.MinDays)
{
return;
}
var program = platform.ArgosProgram;
string errors;
var results = ArgosWebSite.GetCollar(program.UserName, program.Password, platform.PlatformId, days,
out errors);
CollarFile file = FileLoader.LoadPlatfrom(platform, days, results, errors);
if (file == null)
{
return;
}
FileProcessor.ProcessFile(file);
}
internal FileDetailsForm(CollarFile file)
{
InitializeComponent();
RestoreWindow();
File = file;
CurrentUser = Environment.UserDomainName + @"\" + Environment.UserName;
LoadDataContext();
SetUpControls();
}
private static void ProcessIdfFile(CollarFile file)
{
LogGeneralMessage(String.Format("Start local processing of file {0}", file.FileId));
var databaseViews = new AnimalMovementViews();
var idfLink = databaseViews.CollarParametersForIridiumDownload(file.FileId).FirstOrDefault();
if (idfLink == null)
{
LogGeneralMessage("No parameters found. Skipping file.");
return;
}
var database = new AnimalMovementDataContext();
var tpfFile = database.CollarParameterFiles.FirstOrDefault(f => f.FileId == idfLink.ParameterFileId);
if (tpfFile == null)
{
LogGeneralMessage("No collar parameter file found. Skipping file.");
return;
}
var collar = database.Collars.FirstOrDefault(
c => c.CollarManufacturer == idfLink.CollarManufacturer &&
(c.CollarId == idfLink.CollarId || c.CollarId == idfLink.CollarId.Substring(0, 6)));
if (collar == null)
{
LogGeneralMessage("No collar found. Skipping file.");
return;
}
var databaseFunctions = new AnimalMovementFunctions();
//FIXME: check that this is generic to all telonics processing and not just Argos files
databaseFunctions.ArgosFile_ClearProcessingResults(file.FileId);
databaseFunctions.CollarFile_FixOwnerOfIdfFile((file.FileId));
var processor = new Gen4Processor(tpfFile.Contents.ToArray());
var lines = processor.ProcessIdf(file.Contents.ToArray());
var data = Encoding.UTF8.GetBytes(String.Join(Environment.NewLine, lines) + Environment.NewLine);
var filename = Path.GetFileNameWithoutExtension(file.FileName) + "_" + DateTime.Now.ToString("yyyyMMdd") + ".csv";
var fileLoader = new FileLoader(filename, data)
{
Project = null,
Owner = tpfFile.ProjectInvestigator, //Must match result from databaseFunctions.CollarFile_FixOwnerOfIdfFile
Collar = collar,
Status = file.Status,
ParentFileId = file.FileId,
AllowDuplicates = false
};
fileLoader.Load();
LogGeneralMessage("Finished local processing of file");
}
internal static void exceptionHandler(Exception ex, CollarFile file, ArgosPlatform platform)
{
if (file == null)
{
ReportException(
String.Format("Unexpected Exception ({0}) when Loading an unknown collar file",
ex.Message));
}
else
{
ReportException(
String.Format("Unexpected Exception ({0}) when Loading collar file (id: {1}, name:{2}",
ex.Message, file.FileId, file.FileName));
}
}
private static void ProcessParameterSet(CollarFile file, ArgosFile argos, DateTime first, DateTime last,
IEnumerable <ArgosTransmission> transmissions,
GetTelonicsParametersForArgosDatesResult parameters)
{
LogGeneralMessage(String.Format(" Start processing collar {0}/{1}", parameters.CollarManufacturer,
parameters.CollarId));
var start = parameters.StartDate ?? DateTime.MinValue;
if (start < first)
{
start = first;
}
var end = parameters.EndDate ?? DateTime.MaxValue;
if (last < end)
{
end = last;
}
var processor = GetProcessor(file, parameters);
var transmissionSubset = transmissions.Where(t => start <= t.DateTime && t.DateTime <= end);
var lines = processor.ProcessTransmissions(transmissionSubset, argos);
var data = Encoding.UTF8.GetBytes(String.Join(Environment.NewLine, lines) + Environment.NewLine);
var filename = Path.GetFileNameWithoutExtension(file.FileName) + "_" + parameters.CollarId + ".csv";
var fileLoader = new FileLoader(filename, data)
{
Project = file.Project,
Owner = file.ProjectInvestigator,
Collar = new Collar {
CollarManufacturer = parameters.CollarManufacturer,
CollarId = parameters.CollarId
},
Status = file.Status,
ParentFileId = file.FileId,
ArgosDeploymentId = parameters.DeploymentId,
CollarParameterId = parameters.ParameterId,
AllowDuplicates = true
};
fileLoader.Load();
var message =
String.Format(
" Successfully added Argos {0} transmissions from {1:g} to {2:g} to Collar {3}/{4}",
parameters.PlatformId, start, end,
parameters.CollarManufacturer, parameters.CollarId);
LogGeneralMessage(message);
}
private static void HandleException(Exception ex, CollarFile file, ArgosPlatform platform)
{
if (file == null)
{
Console.WriteLine("Processor exception handler called without a file: {0}", ex.Message);
return;
}
if (file.LookupCollarFileFormat.ArgosData != 'Y')
{
Console.WriteLine("Processor exception handler called with a Non-Argos Data file({1}): {0}", ex.Message, file.FileId);
return;
}
var db = new AnimalMovementDataContext();
db.ArgosFileProcessingIssues_Insert(file.FileId, ex.Message, platform == null ? null : platform.PlatformId,
null, null, null, null);
}
private static ArgosFile GetArgosFile(CollarFile file)
{
switch (file.Format)
{
case 'E':
return(new ArgosEmailFile(file.Contents.ToArray()));
case 'F':
return(new ArgosAwsFile(file.Contents.ToArray()));
case 'G':
return(new DebevekFile(file.Contents.ToArray()));
default:
throw new InvalidOperationException("Unsupported file format: " + file.Format +
" (supported formats are E,F,G)");
}
}
/// <summary>
/// Retrieves data for an Argos Program from the Argos Web Server
/// </summary>
/// <param name="program">The Argos Program (contains Platforms) to retrieve from the server</param>
/// <param name="daysToRetrieve">The number of most recent days to retrieve from the server</param>
public static void DownloadArgosProgram(ArgosProgram program, int?daysToRetrieve = null)
{
int daysSinceLastDownload;
if (daysToRetrieve.HasValue)
{
daysSinceLastDownload = daysToRetrieve.Value;
}
else
{
var database = new AnimalMovementDataContext();
var dateOfLastDownload = (from log in database.ArgosDownloads
where log.ProgramId == program.ProgramId && log.FileId != null
orderby log.TimeStamp descending
select log.TimeStamp).FirstOrDefault();
//Problem: using Days always truncates.
// If I download at 1am on day 1, and then 11pm on day 2, (1.9 days -> 1day),
// I will miss any data created between 1am and 11pm on day 1.
// However, rounding up may cause a lot of duplication,
// (i.e. 6:01am on day1 and then 6:02am on day2, will need to download 2 days to get the extra minute)
// by default we should round up to make sure that all data is obtained. However, since this is
// typically called on a scheduled task at the same time (+/- download/processing time) everyday.
// I will check if we are close to an even day, and then round tword that day
var timespan = DateTime.Now - dateOfLastDownload;
int extraDay = timespan.Hours == 0 && timespan.Minutes < 5 ? 0 : 1;
daysSinceLastDownload = timespan.Days + extraDay;
}
var days = Math.Min(ArgosWebSite.MaxDays, daysSinceLastDownload);
if (days < ArgosWebSite.MinDays)
{
return;
}
string errors;
var results = ArgosWebSite.GetProgram(program.UserName, program.Password, program.ProgramId, days,
out errors);
CollarFile file = FileLoader.LoadProgram(program, days, results, errors);
if (file == null)
{
return;
}
FileProcessor.ProcessFile(file);
}
private static void ProcessFile(CollarFile file, ArgosFile argos, ArgosPlatform platform)
{
LogGeneralMessage(String.Format("Start local processing of file {0}", file.FileId));
var databaseFunctions = new AnimalMovementFunctions();
if (platform == null)
{
databaseFunctions.ArgosFile_ClearProcessingResults(file.FileId);
if (IsArgosAwsFileIncomplete(argos as ArgosAwsFile) ?? false)
{
LogIssueForFile(file.FileId,
"The Argos server could not return all the data requested and this file is incomplete.");
}
}
else
{
databaseFunctions.ArgosFile_UnProcessPlatform(file.FileId, platform.PlatformId);
}
var transmissionsByPlatform = from transmission in argos.GetTransmissions()
group transmission by transmission.PlatformId
into transmissions
where platform == null || transmissions.Key == platform.PlatformId
select transmissions;
foreach (var transmissionSet in transmissionsByPlatform)
{
try
{
ProcessTransmissions(file, argos, transmissionSet);
}
catch (Exception ex)
{
var message = String.Format("ERROR {0} adding Argos {1} transmissions",
ex.Message, transmissionSet.Key);
LogIssueForFile(file.FileId, message, transmissionSet.Key);
}
}
LogGeneralMessage("Finished local processing of file");
}
/// <summary>
/// Full or partial processing of Telonics data in an Argos file
/// </summary>
/// <param name="file">The collar file with Argos data to processes. Must not be null</param>
/// <param name="platform">The Argos platform in the file to process.
/// If this is null, then all the platforms will be processed.</param>
public static void ProcessFile(CollarFile file, ArgosPlatform platform = null)
{
if (file == null)
{
throw new ArgumentNullException("file", "No collar file was provided to process.");
}
if (!ProcessOnServer(file, platform))
{
if (file.Format == 'H')
{
ProcessDataLogFile(file);
}
else if (file.Format == 'I')
{
ProcessIdfFile(file);
}
else
{
ProcessFile(file, GetArgosFile(file), platform);
}
}
}
/// <summary>
/// Loads and logs the download data for an Argos Platform
/// </summary>
/// <param name="platform">The Argos PlatformId that this data is for</param>
/// <param name="days">The number of days that are included in this data set</param>
/// <param name="results">The data returned from the Argos Web Server (may be null)</param>
/// <param name="errors">Any errors returned by the Argos Web Server (may be null)</param>
/// <returns>Returns the newly created collar file, complete with database calculated fields</returns>
/// <remarks>Only one of results and errors can be non-null.</remarks>
internal static CollarFile LoadPlatfrom(ArgosPlatform platform, int days,
ArgosWebSite.ArgosWebResult results, string errors)
{
//if results is null, then errors should be non-null (database rule, insert will fail if false)
CollarFile file = null;
var database = new AnimalMovementDataContext();
//Linq to SQL wraps the changes in a transaction so file will not be created if log cannot be written
if (results != null)
{
file = new CollarFile
{
Owner = platform.ArgosProgram.Manager,
FileName = "platform_" + platform.PlatformId + "_" + DateTime.Now.ToString("yyyyMMdd") + ".aws",
Status = 'A',
Contents = results.ToBytes()
};
database.CollarFiles.InsertOnSubmit(file);
}
//Linq to SQL does not return the PK (timestamp) of the new ArgosDownload object, so don't use it in this data context
var log = new ArgosDownload
{
PlatformId = platform.PlatformId,
CollarFile = file,
Days = days,
ErrorMessage = errors
};
database.ArgosDownloads.InsertOnSubmit(log);
database.SubmitChanges();
//Linq TO SQL Insert with SPROC dos not set associations, and provides not partial methods to expand
if (file != null)
{
file.LookupCollarFileFormat = database.LookupCollarFileFormats.First(l => l.Code == file.Format);
}
return(file);
}
private static bool NeedTelonicsSoftware(CollarFile file)
{
var database = new AnimalMovementViews();
return(database.FileHasGen4Data(file.FileId) ?? false);
}
/// <summary>
/// Full or partially processes telonics data in argos files. This program is designed to run regularly
/// as a scheduled task (with no arguments) as well as being called by the database with a single fileid
/// argument (to fully process that file), or with two arguments, platformid and fileid (to partially
/// process that file for just the single platform). When a file is fully or partially processed, it
/// clears any prior processing results and issues, to prevent duplicates.
/// Any processing errors are written to the database, any other errors are written to the console.
/// </summary>
/// <param name="args">
/// If there are no args then ask the database for a list of files that need processing and process them all.
/// If there is only one argument and that argument is the login (domain\username) of a project
/// investigator in the database, then process all the files that belong to the PI or his projects and
/// need processing.
/// If the argument matches /np[latform], or '/na[rgos]', then the remaining files will be fully processed
/// If the argument matches /nf[ile], then the next argos platform specified will not be used to
/// partially process the previously specified file
/// If the argument matches /a[rgos]:xxx or /p[latform]:xxx or xxx and xxx is a platform id in the
/// database, then the previously specified file (or the next file if no has been specified) will be
/// partially processed for only this platform.
/// if the argument matches /f[ile]:xxx or xxx, and xxx is a fileid in the database, then the file will
/// be partially processed if an argos platform has been provided as a prior argument, otherwise it will
/// be fully processed.
/// In the unlikely event that an argument matches a valid platform and a valid file, then the tie will go
/// to the file.
/// Any other argument is ignored with a warning
/// Note:
/// A project investigator login will be accepted as an argument in any position, but is only
/// meaningful as the first and only argument
/// If you want to partially process a file, you must provide the Argos platform first
/// Examples:
/// To fully process multiple files
/// ArgosProcessor,exe f1 f2 f3 ...
/// To partially process multiple files with for one Argos platform
/// ArgosProcessor,exe p1 f1 f2 f3 ...
/// To partially process one file for multiple Argos platforms
/// ArgosProcessor,exe p1 f1 p2 p3 ...
/// To partially process multiple files with multiple Argos platforms
/// ArgosProcessor,exe p1 f1 p2 p3 /nf p4 f2 p5 p6 ...
/// To fully process files f1 and f2 and partially process files f3 and f4
/// ArgosProcessor,exe f1 f2 p1 f3 p2 /nf p3 f4 p4 ...
/// or ArgosProcessor,exe p1 f3 p2 /nf p3 f4 p4 /np f1 f2 ...
/// </param>
/// <remarks>
/// If a copy of TDC.exe is available from this program (as specified in the config file),
/// then the processor will be make use of it to process the files locally and send the
/// results to the database, otherwise the processor will request that the database
/// invoke the ArgosProcessor on the server.
/// </remarks>
private static void Main(string[] args)
{
try
{
if (args.Length == 0)
{
FileProcessor.ProcessAll(HandleException);
}
else
{
ArgosPlatform platform = null;
CollarFile file = null;
ProjectInvestigator pi = null;
foreach (var arg in args)
{
if (ClearPlatform(arg))
{
platform = null;
}
else
{
if (ClearCollarFile(arg))
{
file = null;
}
else
{
var fileArg = GetCollarFile(arg);
if (fileArg != null)
{
file = fileArg;
}
else
{
var platformArg = GetPlatform(arg);
if (platformArg != null)
{
platform = platformArg;
}
else
{
var piArg = GetProjectInvestigator(arg);
if (piArg != null)
{
pi = piArg;
}
else
{
Console.WriteLine("Unhandled argument: {0}", arg);
}
}
}
}
}
if (args.Length == 1 && pi != null)
{
FileProcessor.ProcessAll(HandleException, pi);
}
if (file != null)
{
try
{
FileProcessor.ProcessFile(file, platform);
}
catch (Exception ex)
{
HandleException(ex, file, platform);
}
}
}
}
}
catch (Exception ex)
{
Console.WriteLine("Unhandled Exception: {0}", ex.Message);
}
}
private static void ProcessTransmissions(CollarFile file, ArgosFile argos,
IGrouping <string, ArgosTransmission> group)
{
LogGeneralMessage(String.Format(" Start processing transmissions for Argos Id {0}", group.Key));
var platformId = group.Key;
var transmissions = group.ToList();
var first = transmissions.Min(t => t.DateTime);
var last = transmissions.Max(t => t.DateTime);
var databaseViews = new AnimalMovementViews();
var parameterSets =
databaseViews.GetTelonicsParametersForArgosDates(platformId, first, last)
.OrderBy(c => c.StartDate)
.ToList();
if (parameterSets.Count == 0)
{
var msg = String.Format("No Collar for ArgosId {0} from {1:g} to {2:g}",
platformId, first, last);
LogIssueForFile(file.FileId, msg, platformId, null, null, first, last);
return;
}
if (parameterSets[0].StartDate != null && first < parameterSets[0].StartDate)
{
var msg = String.Format("No Collar for ArgosId {0} from {1:g} to {2:g}",
platformId, first, parameterSets[0].StartDate);
LogIssueForFile(file.FileId, msg, platformId, null, null, first, parameterSets[0].StartDate);
}
int lastIndex = parameterSets.Count - 1;
if (parameterSets[lastIndex].EndDate != null && parameterSets[lastIndex].EndDate < last)
{
var msg = String.Format("No Collar for ArgosId {0} from {1:g} to {2:g}",
platformId, parameterSets[lastIndex].EndDate, last);
LogIssueForFile(file.FileId, msg, platformId, null, null, parameterSets[lastIndex].EndDate, last);
}
foreach (var parameterSet in parameterSets)
{
if (parameterSet.ParameterId == null ||
(parameterSet.CollarModel == "Gen3" && parameterSet.Gen3Period == null) ||
(parameterSet.CollarModel == "Gen4" && parameterSet.Format == null))
{
var start = parameterSet.StartDate ?? first;
var end = parameterSet.EndDate ?? last;
var msg = String.Format("No Telonics Parameters for Collar {0}/{3} from {1:g} to {2:g}",
parameterSet.CollarManufacturer, start, end, parameterSet.CollarId);
LogIssueForFile(file.FileId, msg, platformId, parameterSet.CollarManufacturer, parameterSet.CollarId, start, end);
continue;
}
try
{
ProcessParameterSet(file, argos, first, last, transmissions, parameterSet);
}
catch (Exception ex)
{
var message = String.Format(
"ERROR {0} adding Argos {1} transmissions from {2:g} to {3:g} to Collar {4}/{5}",
ex.Message, parameterSet.PlatformId, first, last,
parameterSet.CollarManufacturer, parameterSet.CollarId);
LogIssueForFile(file.FileId, message, parameterSet.PlatformId, parameterSet.CollarManufacturer,
parameterSet.CollarId, first, last);
}
}
LogGeneralMessage(" Finished processing transmissions");
}
