public override IEnumerable <Hero> Parse(Localization localization)
{
Stopwatch time = new Stopwatch();
var failedParsedHeroes = new List <(string CHeroId, Exception Exception)>();
int currentCount = 0;
ParsedData.Clear();
Console.WriteLine($"Parsing {Name}...");
time.Start();
Parser.HotsBuild = App.HotsBuild;
Parser.Localization = localization;
HashSet <string[]> heroes = Parser.Items;
// parse all the heroes
Console.Write($"\r{currentCount,6} / {heroes.Count} total {Name}");
Parallel.ForEach(heroes, new ParallelOptions {
MaxDegreeOfParallelism = App.MaxParallelism
}, hero =>
{
try
{
ParsedData.GetOrAdd(hero[0], Parser.GetInstance().Parse(hero));
}
catch (Exception ex)
{
failedParsedHeroes.Add((hero[0], ex));
}
private void WriteFile()
{
using (var write = new StreamWriter(_newFilePath, true, Encoding.UTF8))
{
write.Write(ParsedData.ToString());
}
ParsedData.Clear();
ParsedData = new StringBuilder();
GC.Collect();
GC.WaitForPendingFinalizers();
}
public override bool Parse(string szData)
{
if (szData == null)
{
throw new ArgumentNullException(nameof(szData));
}
bool fResult = true;
AirblyResponse ar = JsonConvert.DeserializeObject <AirblyResponse>(szData);
ParsedData.Clear();
ParsedData.Columns.Add(new DataColumn(KnownColumnNames.DATE, typeof(DateTime)));
ParsedData.Columns.Add(new DataColumn(KnownColumnNames.ALT, typeof(Int32)));
ParsedData.Columns.Add(new DataColumn(KnownColumnNames.LON, typeof(double)));
ParsedData.Columns.Add(new DataColumn(KnownColumnNames.LAT, typeof(double)));
ParsedData.Columns.Add(new DataColumn(KnownColumnNames.COMMENT, typeof(string)));
ParsedData.Columns.Add(new DataColumn(KnownColumnNames.PITCH, typeof(Int32)));
ParsedData.Columns.Add(new DataColumn(KnownColumnNames.ROLL, typeof(Int32)));
// ParsedData.Columns.Add(new DataColumn(KnownColumnNames.SPEED, typeof(double)));
List <AirblyPoint> lstPoints = new List <AirblyPoint>(ar.points.Values);
lstPoints.Sort((ap1, ap2) => { return(ap1.happenedAt.CompareTo(ap2.happenedAt)); });
foreach (AirblyPoint ap in lstPoints)
{
DataRow dr = ParsedData.NewRow();
dr[KnownColumnNames.DATE] = ap.Timestamp;
dr[KnownColumnNames.ALT] = ap.altitude;
dr[KnownColumnNames.LAT] = ap.latitude;
dr[KnownColumnNames.LON] = ap.longitude;
dr[KnownColumnNames.COMMENT] = ap.type;
dr[KnownColumnNames.PITCH] = ap.pitch;
dr[KnownColumnNames.ROLL] = ap.roll;
ParsedData.Rows.Add(dr);
}
return(fResult);
}
/// <summary>
/// Parses GPX-based flight data
/// </summary>
/// <param name="szData">The string of flight data</param>
/// <returns>True for success</returns>
public override bool Parse(string szData)
{
if (String.IsNullOrEmpty(szData))
{
return(false);
}
StringBuilder sbErr = new StringBuilder();
ParsedData.Clear();
Boolean fResult = true;
byte[] bytes = Encoding.UTF8.GetBytes(szData);
MemoryStream stream = null;
try
{
stream = new MemoryStream(bytes);
using (StreamReader sr = new StreamReader(stream))
{
stream = null;
XDocument xml = XDocument.Load(sr);
GPXPathRoot root = FindRoot(xml);
if (root == null)
{
return(false);
}
if (root.elements != null)
{
int iRow = 0;
bool fHasAlt = false;
bool fHasDate = false;
bool fHasLatLon = false;
bool fHasSpeed = false;
List <Position> lst = new List <Position>();
foreach (XElement e in root.elements)
{
foreach (XElement coord in e.Descendants(root.xnamespace + "trkpt"))
{
XAttribute xLat = null;
XAttribute xLon = null;
XElement xAlt = null;
XElement xTime = null;
XElement xSpeed = null;
XElement xBadElfSpeed = null;
xLat = coord.Attribute("lat");
xLon = coord.Attribute("lon");
xAlt = coord.Descendants(root.xnamespace + "ele").FirstOrDefault();
xTime = coord.Descendants(root.xnamespace + "time").FirstOrDefault();
xSpeed = SpeedElement(coord, root);
fHasAlt = (xAlt != null);
fHasDate = (xTime != null);
fHasSpeed = (xSpeed != null || xBadElfSpeed != null);
fHasLatLon = (xLat != null && xLon != null);
if (!fHasAlt && !fHasDate && !fHasSpeed && !fHasLatLon)
{
throw new MyFlightbookException(Resources.FlightData.errGPXNoPath);
}
if (fHasLatLon)
{
try
{
Position samp = new Position(Convert.ToDouble(xLat.Value, System.Globalization.CultureInfo.InvariantCulture), Convert.ToDouble(xLon.Value, System.Globalization.CultureInfo.InvariantCulture));
if (fHasAlt)
{
samp.Altitude = (Int32)Convert.ToDouble(xAlt.Value, System.Globalization.CultureInfo.InvariantCulture);
}
if (fHasDate)
{
samp.Timestamp = xTime.Value.ParseUTCDate();
}
if (fHasSpeed)
{
samp.Speed = Convert.ToDouble(xSpeed.Value, System.Globalization.CultureInfo.InvariantCulture);
}
lst.Add(samp);
}
catch (Exception ex) when(ex is FormatException)
{
fResult = false;
sbErr.AppendFormat(CultureInfo.CurrentCulture, Resources.FlightData.errGPXBadRow, iRow);
sbErr.Append("\r\n");
}
catch (Exception ex)
{
MyFlightbookException.NotifyAdminException(new MyFlightbookException("Unknown error in ParseFlightDataGPX", ex));
throw;
}
}
iRow++;
}
}
// Derive speed and put it into a data table
if (!fHasSpeed)
{
Position.DeriveSpeed(lst);
}
ToDataTable(lst);
}
else
{
throw new MyFlightbookException(Resources.FlightData.errGPXNoPath);
}
}
}
catch (System.Xml.XmlException ex)
{
sbErr.Append(Resources.FlightData.errGeneric + ex.Message);
fResult = false;
}
catch (MyFlightbookException ex)
{
sbErr.Append(Resources.FlightData.errGeneric + ex.Message);
fResult = false;
}
finally
{
stream?.Dispose();
}
ErrorString = sbErr.ToString();
return(fResult);
}
public override bool Parse(string szData)
{
if (szData == null)
{
throw new ArgumentNullException("szData");
}
Boolean fResult = true;
StringBuilder sbErr = new StringBuilder();
/*
* GGA - essential fix data which provide 3D location and accuracy data.
*
* $GPGGA,123519,4807.038,N,01131.000,E,1,08,0.9,545.4,M,46.9,M,,*47
*
* Where:
* GGA Global Positioning System Fix Data
* 123519 Fix taken at 12:35:19 UTC
* 4807.038,N Latitude 48 deg 07.038' N
* 01131.000,E Longitude 11 deg 31.000' E
* 1 Fix quality: 0 = invalid
* 1 = GPS fix (SPS)
* 2 = DGPS fix
* 3 = PPS fix
* 4 = Real Time Kinematic
* 5 = Float RTK
* 6 = estimated (dead reckoning) (2.3 feature)
* 7 = Manual input mode
* 8 = Simulation mode
* 08 Number of satellites being tracked
* 0.9 Horizontal dilution of position
* 545.4,M Altitude, Meters, above mean sea level
* 46.9,M Height of geoid (mean sea level) above WGS84
* ellipsoid
* (empty field) time in seconds since last DGPS update
* (empty field) DGPS station ID number
* 47 the checksum data, always begins with *
*
*
* RMC - NMEA has its own version of essential gps pvt (position, velocity, time) data. It is called RMC, The Recommended Minimum, which will look similar to:
*
* $GPRMC,123519,A,4807.038,N,01131.000,E,022.4,084.4,230394,003.1,W*6A
*
* Where:
* RMC Recommended Minimum sentence C
* 123519 Fix taken at 12:35:19 UTC
* A Status A=active or V=Void.
* 4807.038,N Latitude 48 deg 07.038' N
* 01131.000,E Longitude 11 deg 31.000' E
* 022.4 Speed over the ground in knots
* 084.4 Track angle in degrees True
* 230394 Date - 23rd of March 1994
* 003.1,W Magnetic Variation
* 6A The checksum data, always begins with *
*
* */
char[] wordSep = { ',' };
char[] lineSep = { '\n' };
ParsedData.Clear();
try
{
string[] rgSentences = szData.Split(lineSep, StringSplitOptions.RemoveEmptyEntries);
double alt = 0.0;
ParsedData.Columns.Add(new DataColumn(KnownColumnNames.DATE, typeof(DateTime)));
ParsedData.Columns.Add(new DataColumn(KnownColumnNames.ALT, typeof(Int32)));
ParsedData.Columns.Add(new DataColumn(KnownColumnNames.LON, typeof(double)));
ParsedData.Columns.Add(new DataColumn(KnownColumnNames.LAT, typeof(double)));
ParsedData.Columns.Add(new DataColumn(KnownColumnNames.SPEED, typeof(double)));
CultureInfo ci = System.Globalization.CultureInfo.InvariantCulture;
int iSentence = 0;
foreach (string sentence in rgSentences)
{
try
{
string[] rgWords = sentence.Split(wordSep, StringSplitOptions.None);
if (String.Compare(rgWords[0], "$GPGGA", StringComparison.OrdinalIgnoreCase) == 0)
{
if (String.Compare(rgWords[6], "0", StringComparison.Ordinal) != 0) // is this valid?
{
alt = Convert.ToDouble(rgWords[9], CultureInfo.InvariantCulture) * ConversionFactors.FeetPerMeter;
}
}
else if (String.Compare(rgWords[0], "$GPRMC", StringComparison.Ordinal) == 0) // actual sample
{
string szTime = rgWords[1];
string szDate = rgWords[9];
int year2digit = Convert.ToInt16(szDate.Substring(4), CultureInfo.InvariantCulture);
int curYear = DateTime.Now.Year;
int year20thCentury = 1900 + year2digit;
int year21stCentury = 2000 + year2digit;
int year = (Math.Abs(curYear - year20thCentury) < Math.Abs(curYear - year21stCentury)) ? year20thCentury : year21stCentury;
DataRow dr = ParsedData.NewRow();
DateTime dt = new DateTime(year,
Convert.ToInt16(szDate.Substring(2, 2), ci),
Convert.ToInt16(szDate.Substring(0, 2), ci),
Convert.ToInt16(szTime.Substring(0, 2), ci),
Convert.ToInt16(szTime.Substring(2, 2), ci),
Convert.ToInt16(szTime.Substring(4, 2), ci),
DateTimeKind.Utc);
dr[KnownColumnNames.DATE] = dt;
dr[KnownColumnNames.ALT] = alt;
dr[KnownColumnNames.LAT] = (Convert.ToDouble(rgWords[3].Substring(0, 2), ci) + (Convert.ToDouble(rgWords[3].Substring(2), ci) / 60.0)) * ((String.Compare(rgWords[4], "N", StringComparison.OrdinalIgnoreCase) == 0) ? 1 : -1);
dr[KnownColumnNames.LON] = (Convert.ToDouble(rgWords[5].Substring(0, 3), ci) + (Convert.ToDouble(rgWords[5].Substring(3), ci) / 60.0)) * ((String.Compare(rgWords[6], "E", StringComparison.OrdinalIgnoreCase) == 0) ? 1 : -1);
dr[KnownColumnNames.SPEED] = Convert.ToDouble(rgWords[7], ci);
ParsedData.Rows.Add(dr);
}
}
catch (System.FormatException ex)
{
sbErr.AppendFormat(CultureInfo.CurrentCulture, Resources.FlightData.errInRow, iSentence, ex.Message);
fResult = false;
}
catch (System.IndexOutOfRangeException ex)
{
sbErr.AppendFormat(CultureInfo.CurrentCulture, Resources.FlightData.errInRow, iSentence, ex.Message);
fResult = false;
}
catch (Exception ex)
{
MyFlightbookException.NotifyAdminException(ex);
throw;
}
iSentence++;
}
}
catch (Exception ex)
{
MyFlightbookException.NotifyAdminException(ex);
throw;
}
ErrorString = sbErr.ToString();
return(fResult);
}
/// <summary>
/// Parses the passed IGC data to the passed data table
/// </summary>
/// <param name="szData">The data to parse.</param>
/// <returns>True for success</returns>
public override bool Parse(string szData)
{
ParsedData.Clear();
MatchCollection mcDate = null;
try
{
// check for valid IGC data
if (String.IsNullOrEmpty(szData))
{
throw new InvalidDataException("No data to parse");
}
if (!CanParse(szData))
{
throw new InvalidDataException("Data to parse is not IGC format");
}
// Get the base date, utc.
mcDate = rIGCDate.Matches(szData);
if (mcDate == null || mcDate.Count > 0 && mcDate[0].Groups.Count == 3)
{
throw new InvalidDataException("IGC Data has no date field as required");
}
}
catch (InvalidDataException ex)
{
ErrorString = ex.Message;
return(false);
}
GroupCollection gcDate = mcDate[0].Groups;
int day = Convert.ToInt32(gcDate["day"].Value, CultureInfo.InvariantCulture);
int month = Convert.ToInt32(gcDate["month"].Value, CultureInfo.InvariantCulture);
int year = Convert.ToInt32(gcDate["year"].Value, CultureInfo.InvariantCulture);
int yearNow = DateTime.Now.Year;
int millenium = (yearNow / 1000) * 1000;
// fix up a two digit year to be whichever is latest but less than or equal to this year
year += (year + millenium) <= yearNow ? millenium : millenium - 100;
DateTime dtBase = new DateTime(year, month, day, 0, 0, 0, DateTimeKind.Utc);
int hLast = int.MinValue;
int dayOffset = 0;
List <Position> lst = new List <Position>();
MatchCollection mc = rIGCPosition.Matches(szData);
foreach (Match match in mc)
{
if (match.Groups.Count == 13)
{
GroupCollection gc = match.Groups;
int h = Convert.ToInt32(gc["hrs"].Value, CultureInfo.InvariantCulture);
int m = Convert.ToInt32(gc["min"].Value, CultureInfo.InvariantCulture);
int s = Convert.ToInt32(gc["sec"].Value, CultureInfo.InvariantCulture);
int latd = Convert.ToInt32(gc["latd"].Value, CultureInfo.InvariantCulture);
int latm = Convert.ToInt32(gc["latm"].Value, CultureInfo.InvariantCulture); // = Minutes x 1000 (MMmmm)
int latsign = (gc["latns"].Value.CompareOrdinal("S") == 0) ? -1 : 1;
int lond = Convert.ToInt32(gc["lond"].Value, CultureInfo.InvariantCulture);
int lonm = Convert.ToInt32(gc["lonm"].Value, CultureInfo.InvariantCulture);
int lonsign = (gc["lonew"].Value.CompareOrdinal("W") == 0) ? -1 : 1;
bool fHasAlt = gc["val"].Value.CompareOrdinal("A") == 0;
int palt = Convert.ToInt32(gc["palt"].Value, CultureInfo.InvariantCulture);
int gpsalt = Convert.ToInt32(gc["gpsalt"].Value, CultureInfo.InvariantCulture);
// check for the flight going past midnight.
if (h < hLast)
{
dayOffset++;
}
hLast = h;
// Create the data sample
LatLong llSample = new LatLong(latsign * (latd + latm / 60000.0), lonsign * (lond + lonm / 60000.0));
DateTime dtSample = dtBase.AddSeconds(h * 3600 + m * 60 + s);
int altSample = fHasAlt ? palt : 0;
lst.Add(new Position(llSample, altSample, dtSample));
}
}
Position.DeriveSpeed(lst);
ToDataTable(lst);
return(true);
}
/// <summary>
/// Parses the items and returns a collection in ascending order.
/// </summary>
/// <param name="localization"></param>
/// <returns></returns>
public virtual IEnumerable <T> Parse(Localization localization)
{
Stopwatch time = new Stopwatch();
ParsedData.Clear();
Console.WriteLine($"Parsing {Name} data...");
time.Start();
int currentCount = 0;
HashSet <string[]> items = Parser.Items;
IEnumerable <string[]> generalItems = items.Where(x => x.Length == 1);
IEnumerable <string[]> mapItems = items.Where(x => x.Length > 1);
Console.Write($"\r{currentCount,6} / {items.Count} total {Name}");
try
{
Parallel.ForEach(generalItems, new ParallelOptions {
MaxDegreeOfParallelism = App.MaxParallelism
}, item =>
{
T parsedItem = Parser.GetInstance().Parse(item);
ParsedData.GetOrAdd(parsedItem.Id, parsedItem);
Console.Write($"\r{Interlocked.Increment(ref currentCount),6} / {items.Count} total {Name}");
});
// check if there are any
if (mapItems.Any())
{
// group them up by the map name id
IEnumerable <IGrouping <string, string[]> > mapItemsGroup = mapItems.GroupBy(x => x.ElementAtOrDefault(1));
foreach (IGrouping <string, string[]> mapItemGroup in mapItemsGroup)
{
Parser.LoadMapData(mapItemGroup.Key);
Parallel.ForEach(mapItemGroup, new ParallelOptions {
MaxDegreeOfParallelism = App.MaxParallelism
}, mapItem =>
{
T parsedMapItem = Parser.GetInstance().Parse(mapItem);
ParsedData.GetOrAdd(parsedMapItem.Id, parsedMapItem);
Console.Write($"\r{Interlocked.Increment(ref currentCount),6} / {items.Count} total {Name}");
});
Parser.RestoreGameData();
}
}
}
catch (Exception ex)
{
Console.ForegroundColor = ConsoleColor.Red;
App.WriteExceptionLog($"{Name.Where(x => !char.IsWhiteSpace(x))}", ex);
Console.WriteLine();
Console.WriteLine($"Failed to parse {Name}");
Console.WriteLine(ex);
Console.WriteLine();
Console.ResetColor();
Environment.Exit(1);
}
finally
{
Console.Write($"\r{currentCount,6} / {items.Count} total {Name}");
}
time.Stop();
Console.WriteLine();
Console.WriteLine($"Finished in {time.Elapsed.TotalSeconds} seconds");
Console.WriteLine();
return(ParsedData.Values.OrderBy(x => x.Id));
}