public static void BuildVSXCatalog(string directory)
{
string[] files = Directory.GetFiles(directory);
int i = 0;
var db = new StarCatalogContext();
foreach (string file in files)
{
StreamReader stream = File.OpenText(file);
//int found = 0;
//int notfound = 0;
while (!stream.EndOfStream)
{
string entry = stream.ReadLine();
D_VSXEntry md = new D_VSXEntry();
FixedLengthReader flr = new FixedLengthReader(entry);
flr.read(md);
db.VSXEntries.Add(md);
//string key = FindPPMXKey(md.RADeg, md.DEDeg);
//if (key != null)
//{
// found++;
// _ppmdb.Put(WriteOptions.Default, key+".vsxinfo", entry);
//}
//else
//{
// notfound++;
// // Just use the closes match.
// key = ConvertToKey(md.RADeg, md.DEDeg)[1];
// _ppmdb.Put(WriteOptions.Default, key + ".vsxinfo", entry);
//}
//if ((found+notfound)%100 == 0)
// Console.Write(String.Format("\rFound: {0} Not Found: {1}", found, notfound));
if (i++ % 1000 == 0)
{
db.SaveChanges();
db.Dispose();
db = new StarCatalogContext();
}
}
Console.WriteLine();
Console.WriteLine("Completed loading catalog: " + file);
}
}
public static void GetFrameInfo(string directory, StarCatalogContext db)
{
string[] files = Directory.GetFiles(directory);
foreach (string file in files)
{
string key = Path.GetFileName(file).Replace(".fit", "");
if (db.Frames.Where(x => x.File == key).Count() == 0)
{
D_Frame frame = new D_Frame()
{
File = key
};
db.Frames.Add(frame);
Fits f = new Fits(file);
BasicHDU[] hdus = f.Read();
DateTime date = File.GetCreationTime(file);
double jd = Math.Round(DateTimeToJulian(date), 5);
for (int i = 0; i < hdus.Length; i += 1)
{
Header hdr = hdus[i].Header;
date = DateTime.Parse(hdr.GetStringValue("DATE-OBS") + "Z");
jd = Math.Round(DateTimeToJulian(date), 5);
frame.Time = jd;
frame.Duration = (int)hdr.GetDoubleValue("EXPOSURE");
frame.ISO = (int)hdr.GetDoubleValue("GAIN");
frame.LocLat = GlobalVariables.LocLat;
frame.LocLng = GlobalVariables.LocLng;
frame.Camera = hdr.GetStringValue("INSTRUME");
frame.Reported = 0;
}
}
}
db.SaveChanges();
}
public static D_VSXEntry GetVSXEntry(double ra, double dec)
{
Slice encodedstar;
using (var db = new StarCatalogContext())
{
var result = db.VSXEntries.Where(x => Math.Abs((double)x.RADeg - (float)ra) < 0.001 && Math.Abs((double)x.DEDeg - (float)dec) < 0.001).FirstOrDefault();
return(result);
//if (_ppmdb.TryGet(ReadOptions.Default, ppmx.Substring(0, 15) + ".vsxinfo", out encodedstar))
//{
// D_VSXEntry md = new D_VSXEntry();
// FixedLengthReader flr = new FixedLengthReader(encodedstar.ToString());
// flr.read(md);
// return md;
//}
//else
// return null;
}
}
public static string GetAUID(D_VSXEntry entry)
{
using (var db = new StarCatalogContext())
{
if (entry.AUID == null)
{
var html = GetHTML("http://www.aavso.org/vsx/index.php?view=detail.top&oid=" + entry.OID.Trim());
string pattern = @"\d+-\D+-\d+";
Regex rgx = new Regex(pattern, RegexOptions.IgnoreCase);
MatchCollection matches = rgx.Matches(html);
if (matches.Count > 0)
{
db.VSXEntries.Attach(entry);
entry.AUID = matches[0].Value;
}
db.SaveChanges();
}
return(entry.AUID);
}
}
public string LoadChart(string directory, StarCatalogContext db)
{
TextReader reader = File.OpenText(directory + "\\chart.csv");
var csv = new CsvReader(reader);
csv.Configuration.Delimiter = "\t";
csv.ReadHeader();
string chartname = csv.FieldHeaders.GetValue(0).ToString();
if (db.ChartEntries.Where(x => x.Chart == chartname).Count() == 0)
{
csv.Read();
do
{
D_ChartEntry entry = new D_ChartEntry()
{
Chart = chartname,
AUID = csv.GetField(0).ToString(),
Label = csv.GetField(3).ToString(),
Comments = csv.GetField(6).ToString()
};
// prune off degree character
entry.J2000_RA = GetCoord(csv.GetField(1).ToString());
entry.J2000_DEC = GetCoord(csv.GetField(2).ToString());
entry.VMag = GetValue(csv.GetField(4).ToString());
entry.BVColor = GetValue(csv.GetField(5).ToString());
db.ChartEntries.Add(entry);
}while (csv.Read());
db.SaveChanges();
}
csv.Dispose();
return(chartname);
}
static void Main(string[] args)
{
if (GlobalVariables.ParseCommands(args) == false)
{
return;
}
if (GlobalVariables.BuildStarDatabase == true)
{
StarCatalog.BuildPPMCatalog(GlobalVariables.PPMCatalogSource);
StarCatalog.BuildVSXCatalog(GlobalVariables.VSXCatalogSource);
return;
}
using (var db = new StarCatalogContext())
{
if (GlobalVariables.LoadFrames == true)
{
FrameManager.GetFrameInfo(GlobalVariables.InputDirectory, db);
return;
}
if (GlobalVariables.LoadResults == true)
{
string chartname =
new ChartManager().LoadChart(GlobalVariables.InputDirectory, db);
new SampleManager().LoadStarField(GlobalVariables.InputDirectory, chartname, db);
return;
}
var field = StarField.LoadStarField(GlobalVariables.Chart, db);
// Apply quality filter.
field.Stars = field.Stars.OrderByDescending(x => x.AvgInstrumentalMag).Where(x => x.MinSNR > GlobalVariables.MinSNR && x.Flags < 16).ToList();
field.DoPhotometricReduction();
// Output results
StringBuilder candidates = new StringBuilder();
candidates.AppendLine(String.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12}", "Star", "RMag", "C-VMag", "C-VMag-E", "VMag", "C-ColorIndex", "ColorIndex", "Label", "Known Variable", "Uncertainty", "Score", "Flags", "URL"));
StringBuilder timeseries = new StringBuilder();
timeseries.AppendLine(String.Format("{0},{1},{2},{3},{4},{5}", "Score", "Star", "ObsDate", "VMag", "TFAMag", "Uncertainty"));
StringBuilder aavsolog = new StringBuilder();
aavsolog.AppendLine("#TYPE=EXTENDED");
aavsolog.AppendLine("#OBSCODE=" + GlobalVariables.ObsCode);
aavsolog.AppendLine("#SOFTWARE=VSA V" + Assembly.GetEntryAssembly().GetName().Version);
aavsolog.AppendLine("#DELIM=,");
aavsolog.AppendLine("#DATE=JD");
aavsolog.AppendLine("#OBSTYPE=DSLR");
aavsolog.AppendLine("#NAME,DATE,MAG,MERR,FILT,TRANS,MTYPE,CNAME,CMAG,KNAME,KMAG,AMASS,GROUP,CHART,NOTES");
Console.WriteLine("Outputing results.");
foreach (Star star in field.Stars)
{
D_VSXEntry entry = StarCatalog.GetVSXEntry((double)star.RA, (double)star.DEC);
string variabletype = "";
if (entry != null && entry.VFlag < 2)
{
Console.WriteLine("Found variable star: " + entry.Name);
variabletype = "http://www.aavso.org/vsx/index.php?view=detail.top&oid=" + entry.OID.Trim();
string ensemble = "Custom ensemble of " + (star.Comparables.Count() - 1).ToString() + " due to low chart count.";
if (field.ChartComparables.Count > 1)
{
ensemble = field.ChartComparables[1].Label;
for (int f = 2; f < field.ChartComparables.Count; f++)
{
ensemble += "/" + field.ChartComparables[f].Label;
}
}
// Query to get auid.
string auid = StarCatalog.GetAUID(entry);
if (auid != null)
{
aavsolog.AppendLine(String.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},{14}",
auid,
Math.Round(field.Frames[0].Time, 4),
Math.Round(star.VMag, 3).ToString().PadLeft(7, ' '),
Math.Round(star.e_VMag, 3),
"V",
"YES",
"STD",
"ENSEMBLE",
"na",
field.ChartComparables[0].Label.Replace("[CHECK]", "").Trim(),
Math.Round(field.ChartComparables[0].VMag, 3).ToString().PadLeft(7, ' '),
"na",
"na",
field.Frames[0].Chart,
String.Format("MAG=avg {0} img|MERR=StdDe|ENSEMBLE={1}", star.Samples.Count(), ensemble)));
}
}
string label = "";
if (field.ChartComparables.Contains(star))
{
label = star.Label;
}
candidates.Append(String.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11}",
star.Name,
star.CatalogEntry.Rmag,
star.VMag,
star.e_VMag,
star.CatalogEntry.Vmag,
star.CalculatedColorIndex,
star.ColorIndex,
label,
variabletype,
star.EnsembleError,
star.Score,
Convert.ToString(star.Flags, 2).PadLeft(5, '0')));
// Output the calculated differential magnitude with the comparable star.
int i = 0;
foreach (SampleData sample in star.Samples)
{
timeseries.AppendLine(String.Format("{0},{1},{2},{3},{4},{5}", star.Score, star.Name, field.Frames[i].Time, sample.ApparentVMag - star.VMag, sample.TFAVMag - star.VMag, sample.Uncertainty));
i++;
}
candidates.Append(String.Format(",http://simbad.u-strasbg.fr/simbad/sim-coo?Coord={0}%20{1}%20{2}%20{3}%20{4}%20{5}&Radius=2&Radius.unit=arcsec&",
star.Name.Substring(0, 2), star.Name.Substring(2, 2), star.Name.Substring(4, 4), star.Name.Substring(8, 3), star.Name.Substring(11, 2), star.Name.Substring(13, 2)));
candidates.AppendLine();
}
File.WriteAllText(GlobalVariables.GetOutput(".\\AAVSO\\" + field.Frames[0].Chart + ".txt"), aavsolog.ToString());
File.WriteAllText(GlobalVariables.GetOutput(".\\candidates_out.csv"), candidates.ToString());
File.WriteAllText(GlobalVariables.GetOutput(".\\timeseries_out.csv"), timeseries.ToString());
}
}
/// <summary>
/// Load star-field specified in file
/// </summary>
/// <param name="filename">CSV file for star-field to load</param>
/// <returns></returns>
public static StarField LoadStarField(string chart, StarCatalogContext db)
{
StarField field = new StarField()
{
Stars = new List <Star>(),
Frames = db.Frames.Where(x => x.Chart == chart).OrderBy(x => x.Time).ToList(),
ChartComparables = new List <Star>()
};
Console.WriteLine("Loading chart entries.");
field.ChartEntries = db.ChartEntries.Where(x => x.Chart == chart).ToList();
Console.WriteLine("Loading frames.");
int framenumber = 1;
foreach (D_Frame frame in field.Frames)
{
// Get all samples for frame.
List <F_Sample> samples = db.Samples.Where(x => x.ID_Frame == frame.ID).ToList();
foreach (F_Sample sample in samples)
{
var star = field.Stars.Where(x => x.ID == sample.ID_Star).FirstOrDefault();
if (star == null)
{
var d_star = db.Stars.Where(x => x.ID == sample.ID_Star).First();
if (d_star != null)
{
star = new Star()
{
ID = d_star.ID,
Name = d_star.Nameplate,
RA = (decimal)d_star.J2000_RA,
DEC = (decimal)d_star.J2000_DEC,
OID = d_star.OID,
ValidColorIndex = true,
Samples = new List <SampleData>()
};
field.Stars.Add(star);
}
else
{
continue; // skip star.
}
}
var data = new SampleData()
{
ImgX = (decimal)sample.ImgX,
ImgY = (decimal)sample.ImgY,
ApetureSize = sample.Aperture,
InstrumentalVFlux = (decimal)sample.FluxV,
InstrumentalBFlux = (decimal)sample.FluxB,
SNR = (decimal)sample.SNR,
Flag = sample.FlagV
};
if (sample.SNR > 0)
{
data.Uncertainty = (decimal)Math.Abs((sample.FluxV + sample.FluxB / sample.SNR).Mag() - sample.FluxV.Mag());
}
else
{
data.Uncertainty = -1;
}
// Calculated fields
if (data.InstrumentalBFlux <= 0 || data.InstrumentalVFlux <= 0)
{
star.ValidColorIndex = false;
}
else
{
data.InstrumentalBMag = data.InstrumentalBFlux.Mag();
data.InstrumentalVMag = data.InstrumentalVFlux.Mag();
}
star.Flags |= sample.FlagB;
star.Samples.Add(data);
}
// Add place-holder samples for stars not containing the right sample count.
foreach (Star star in field.Stars)
{
if (star.Samples.Count < framenumber)
{
star.Samples.Add(new SampleData());
}
}
framenumber++;
}
foreach (Star star in field.Stars)
{
if (star.Samples.Count > 0)
{
star.EnsembleError = star.Samples.Average(x => x.Uncertainty);
}
star.MinSNR = star.Samples.Min(x => x.SNR);
// Load star catalog
star.CatalogEntry = StarCatalog.GetPPMEntry(star.Name);
star.Flags = Math.Max(star.Flags, star.Samples.Max(x => x.Flag));
// Calculate color index.
if (star.CatalogEntry.Vmag > 0 && star.CatalogEntry.Bmag > 0)
{
star.ValidCatalogMag = true;
star.ColorIndex = star.CatalogEntry.Bmag - star.CatalogEntry.Vmag;
}
if (star.ValidColorIndex == true)
{
star.AvgInstrumentalMag = star.Samples.Average(x => x.InstrumentalVMag);
star.InstrumentalColorIndex = star.Samples.Average(x => x.InstrumentalBMag) - star.Samples.Average(x => x.InstrumentalVMag);
}
}
var eligible = field.Stars.Where(x => x.Flags == 0 && x.ValidColorIndex == true).ToList();
foreach (D_ChartEntry entry in field.ChartEntries)
{
double ra = entry.J2000_RA;
double dec = entry.J2000_DEC;
var star = eligible.Where(x => Math.Abs((double)x.RA - ra) < 0.001 && Math.Abs((double)x.DEC - dec) < 0.001).FirstOrDefault();
if (star != null)
{
if (star.MinSNR > 0 && star.Flags == 0 && star.OID == -1)
{
star.ColorIndex = (decimal)entry.BVColor;
star.CatalogEntry.Vmag = (decimal)entry.VMag;
star.Label = entry.Label;
field.ChartComparables.Add(star);
}
}
}
if (field.ChartComparables.Count > 0)
{
field.ChartComparables[0].Label = "[CHECK] " + field.ChartComparables[0].Label;
}
if (field.ChartComparables.Count < GlobalVariables.Comparables)
{
Console.WriteLine("Adding extra stars due to low chart count.");
var eligiblestars = field.Stars
.Where(x => x.MinSNR > GlobalVariables.MinSNR)
.Where(x => field.ChartComparables.Contains(x) == false)
.Where(x => x.Flags == 0 && x.OID == -1 && x.ValidCatalogMag == true && x.ValidColorIndex == true)
.ToList();
var builder = new ChartBuilder(field, eligiblestars, field.ChartComparables.ToList());
field.ChartComparables = builder.Solve();
foreach (Star star in field.ChartComparables)
{
if (star.Label == null)
{
star.Label = star.CatalogEntry.PPMX.Trim();
}
}
}
if (field.ChartComparables.Count >= GlobalVariables.Comparables)
{
Console.WriteLine("Finding best comparable fit.");
var eligiblestars = field.ChartComparables.Skip(1).ToList();
var builder = new ChartBuilder(field, eligiblestars, new List <Star>()
{
field.ChartComparables[0]
});
field.ChartComparables = builder.Solve();
}
return(field);
}
/// <summary>
/// Load star-field specified in file
/// </summary>
/// <param name="filename">CSV file for star-field to load</param>
/// <returns></returns>
public void LoadStarField(string directory, string chartname, StarCatalogContext db)
{
// Load and parse csv.
TextReader reader = File.OpenText(directory + "\\series_g.csv");
var csv = new CsvReader(reader);
csv.Configuration.Delimiter = ";";
csv.ReadHeader();
// Load headers
int fluxsample = 6;
while (fluxsample < csv.FieldHeaders.Count())
{
string filename = csv.FieldHeaders.GetValue(fluxsample++).ToString();
var match = db.Frames.Where(x => filename.Contains(x.File)).First();
match.Chart = chartname;
frames.Add(match);
}
db.SaveChanges();
csv.Read();
csv.Read();
csv.Read();
int aperture = GetAperture(csv.GetField(6));
csv.Read();
// Pre-load all existing sample data for frames.
List <F_Sample> existingsamples = new List <F_Sample>();
foreach (D_Frame d_frame in frames)
{
existingsamples.AddRange(db.Samples.Where(x => x.ID_Frame == d_frame.ID).ToList());
}
int starcount = 0;
Console.WriteLine();
do
{
// Create star if it doesn't exist.
string name = csv.GetField(0);
D_Star star = db.Stars.Where(x => x.Nameplate == name).FirstOrDefault();
if (star == null)
{
star = new D_Star()
{
Nameplate = name,
J2000_RA = csv.GetField <double>(1),
J2000_DEC = csv.GetField <double>(2)
};
D_VSXEntry entry = StarCatalog.GetVSXEntry(star.J2000_RA, star.J2000_DEC);
if (entry != null)
{
star.OID = Convert.ToInt32(entry.OID);
}
else
{
star.OID = -1;
}
db.Stars.Add(star);
}
stars.Add(star);
/////
// Load flux samples.
string parsedflux;
fluxsample = 6;
int frame = 0;
int flag = csv.GetField <int>(3);
while (csv.TryGetField(fluxsample++, out parsedflux))
{
if (existingsamples.Where(x => x.ID_Frame == frames[frame].ID && x.ID_Star == star.ID).Count() == 0)
{
double flux = 0.0;
double.TryParse(parsedflux, out flux);
F_Sample sample = new F_Sample()
{
Star = star,
ID_Frame = frames[frame].ID,
Aperture = aperture,
FluxV = flux
};
// Save flux measurement as instrumental mag.
samples.Add(sample);
frame++;
}
}
Console.Write("\rStars loaded: " + ++starcount);
}while (csv.Read());
// Save stars
db.SaveChanges();
// Supplement and persist sample data.
if (samples.Count > 0)
{
foreach (F_Sample sample in samples)
{
// Set star ID.
sample.ID_Star = sample.Star.ID;
db.Samples.Add(sample);
}
LoadInstrumentalColorIndex(directory);
LoadSNR(directory);
db.SaveChanges();
}
csv.Dispose();
}
