static void Main(string[] args)
{
Thread.CurrentThread.CurrentCulture = new CultureInfo("en-US");
Stopwatch stopwatch = new Stopwatch();
DateTime startDate;
var endDate = DateTime.Today.GetBusinessDay(-1);
var options = new CommandLineOptions();
if (!Parser.Default.ParseArguments(args, options))
Environment.Exit(Parser.DefaultExitCodeFail);
DateTime.TryParse(String.Join(".", options.startDate), out startDate);
var logging = new Logging(options.LogFile);
Trace.WriteLine(options.ToString());
var config = new ConfigBase();
config.Load(options.Settings);
if (!options.NoDownload)
{
IDataSource dataSource = new Bloomberg();
dataSource.Connect();
HashSet<string> shareNames = new HashSet<string>();
// get specifications
var sheet = new Sheet();
sheet.Download(new string[] { config.GetValue("sheetCode"), config.GetValue("shareNames") });
var shareIDs = sheet.toShares();
shareNames.UnionWith(
dataSource.GetTickers(shareIDs.ToList())
);
sheet.Download(new string[] { config.GetValue("sheetCode"), config.GetValue("indices") });
var indexNames = sheet.toShares();
sheet.Download(new string[] { config.GetValue("sheetCode"), config.GetValue("fields") });
var fields = new List<Field>();
sheet.toFields<Field>(fields);
//download index compositions
if (indexNames != null && indexNames.Count() > 0)
{
//obtain components of indices
var names = dataSource.DownloadMultipleComponents(indexNames.ToList(), "INDX_MEMBERS");
//convert tickers -> BB IDs
shareNames.UnionWith(dataSource.GetTickers(names));
}
LocalDisk disk = new LocalDisk();
disk.SetPath(options.Dir);
//delete data for shares-reload and shares-delete
{
sheet.Download(new string[] { config.GetValue("sheetCode"), config.GetValue("shares-reload") });
var sharesReload = dataSource.GetTickers(sheet.toShares().ToList());
sheet.Download(new string[] { config.GetValue("sheetCode"), config.GetValue("shares-delete") });
var sharesDelete = dataSource.GetTickers(sheet.toShares().ToList());
foreach (var item in sharesDelete.Concat(sharesReload))
disk.DeleteDirectory(item.StripOfIllegalCharacters());
//delete shares-delete names from list of downloadable shares
foreach (var item in sharesDelete)
{
if (shareNames.Contains(item))
shareNames.Remove(item);
}
}
//download and save data
stopwatch.Start();
{
var shares = new SharesBatch(shareNames.ToList(), fields, dataSource, disk, startDate, endDate);
shares.PerformOperations();
Trace.Write("Processing Individual: ");
foreach (var shareName in shareNames)
{
Share share = new Share(name: shareName, fields: fields, dataSource: dataSource, fileAccess: disk, startDate: startDate, endDate: endDate);
share.DoWork();
}
}
dataSource.Disconnect();
//download fieldInfo
{
if (shareNames.Count() > 0)
{
dataSource.Connect(dataType: "//blp/apiflds");
disk.SetPath(options.FieldInfoDir);
Share share = new Share(name: shareNames.First(), fields: fields, dataSource: dataSource, fileAccess: disk, startDate: startDate, endDate: endDate);
share.DoWorkFieldInfo();
dataSource.Disconnect();
}
}
stopwatch.Stop();
Trace.WriteLine("Time spent downloading from BB: " + stopwatch.Elapsed.ToString());
}
//upload data via SQL connection
if (!options.NoUpload)
{
stopwatch.Restart();
{
LocalDisk disk = new LocalDisk();
disk.SetPath(options.Dir);
var database = new MySQL(config.GetValue("sqlIP"), config.GetValue("sqlUser"), config.GetValue("sqlPass"), config.GetValue("sqlDB"), disk);
database.DoWork();
}
{
LocalDisk disk = new LocalDisk();
disk.SetPath(options.FieldInfoDir);
var database = new MySQL(config.GetValue("sqlIP"), config.GetValue("sqlUser"), config.GetValue("sqlPass"), config.GetValue("sqlDB"), disk);
database.DoWorkFieldInfo();
}
stopwatch.Stop();
Trace.WriteLine("Time spent uploading: " + stopwatch.Elapsed.ToString());
logging.Close();
{
Console.WriteLine("Executing long job, you can force exit program, it will continue executing on server");
LocalDisk disk = new LocalDisk();
disk.SetPath(options.Dir);
var database = new MySQL(config.GetValue("sqlIP"), config.GetValue("sqlUser"), config.GetValue("sqlPass"), config.GetValue("sqlDB"), disk);
database.executeScript();
}
}
}
private void SharesNewOld(IEnumerable<IField> fields)
{
sharesNew = new List<string>();
sharesOld = new List<string>();
foreach (var shareName in shareNames)
{
var share = new Share(shareName, fields, dataSource, fileAccess);
if (!share.ShareExists())
sharesNew.Add(shareName);
}
sharesOld = (from s in shareNames
where !this.sharesNew.Contains(s)
select s).ToList();
}
// check specific share for last update - for all historical fields. Extend to all shares, where the same conditions are met. Download
private void DownloadOldWithSameLastUpdateDate()
{
if (oldFields == null || oldFields.Count()==0)
return;
var oldFieldsReference = (from f in oldFields
where f.requestType == "ReferenceDataRequest"
select f).ToList();
oldFieldsReference.Sort();
foreach (var f in FieldBlocks(oldFieldsReference))
{
if (f != null && f.Count() > 0)
this.DownloadNew(sharesOld.ToList(), f, null);
}
var oldFieldsHistorical = (from f in oldFields
where f.requestType == "HistoricalDataRequest"
select f).ToList();
oldFieldsHistorical.Sort();
Share share = new Share(sharesOld.First(), fields, dataSource, fileAccess);
DateTime oldestUpdate = this.endDate;
foreach (var f in oldFieldsHistorical)
{
var update = share.CheckLatest(f);
if (update!=null)
{
if (update < oldestUpdate)
oldestUpdate = update.Value;
}
}
if (oldestUpdate <= this.startDate)
return;
foreach (var f in FieldBlocks(oldFieldsHistorical))
{
if (f != null && f.Count() > 0)
this.DownloadNew(sharesOld.ToList(), f, oldestUpdate);
}
}
private void FieldsNewOld()
{
newFields = new List<IField>();
oldFields = new List<IField>();
if (sharesOld == null || sharesOld.Count() == 0)
return;
Share share = new Share(sharesOld.First(), fields, dataSource, fileAccess);
foreach (var f in fields)
{
if (!share.FieldExists(f))
newFields.Add(f);
else
oldFields.Add(f);
}
newFields.Sort();
oldFields.Sort();
}
private void DownloadNew(List<string> shares, IEnumerable<IField> fields, DateTime? startDate)
{
Console.ForegroundColor = ConsoleColor.White;
Trace.Write("\nFields: ");
Console.ForegroundColor = ConsoleColor.Gray;
Trace.Write(fields.ToExtendedString());
Console.ForegroundColor = ConsoleColor.White;
Trace.Write(" Shares: ");
Console.ForegroundColor = ConsoleColor.Gray;
var equities = from s in shares
select s;
foreach (var shareBlock in this.ShareBlocks(shares))
{
var output = dataSource.DownloadData(shareBlock.ToList(), fields.ToList(), startDate: startDate.HasValue ? startDate.Value : this.startDate, endDate: endDate);
var enumerator = output.GetEnumerator();
foreach (var s in shareBlock)
{
var share = new Share(" ", fields, dataSource, fileAccess, startDate: startDate.HasValue ? startDate.Value : this.startDate, endDate: endDate);
foreach (var f in fields)
{
enumerator.MoveNext();
var field = enumerator.Current;
share.name = field.Item1;
share.InjectDownloaded(f, field.Item2);
}
share.FieldsToKeep(this.fields);
share.DoWork();
}
}
}