private static void ScheduleCherriesBatchProcessTask(bool runPredictaImport, bool runEodIndecesImport, int[] idExchanges, string[] exchanges, Dictionary <string, string> config)
{ // Thread process
if (runPredictaImport)
{
PredictaImport.Execute(new string[] { Constants.Last, Constants.Securities, Constants.Prices, Constants.Indeces }, config);
BatchCalculations.Execute(config, new int[] { Constants.PredictaExchangeId });
}
if (runEodIndecesImport)
{
EODPricesImport.Execute(new string[] { Constants.Last, Constants.Indeces }, config, new string[] { Constants.EODIndecesImportFilenamePrefix }, Constants.IndecesExchangeCode);
BatchCalculations.Execute(config, new int[] { Constants.IndecesExchangeCode });
}
if (idExchanges.Length > 0)
{
EODSecuritiesImport.Execute(config, exchanges);
EODPricesImport.Execute(new string[] { Constants.Last, Constants.Prices }, config, exchanges, 0);
BatchCalculations.Execute(config, idExchanges);
}
}
public static void Main(string[] args)
{
Dictionary <string, string> config = new Dictionary <string, string>();
var path = Environment.CurrentDirectory;
config[Constants.DB_ConnectionString] = Convert.ToString(ConfigurationManager.ConnectionStrings[Constants.DB_ConnectionString]);
config[Constants.DB_ConnectionString_Quandl] = Convert.ToString(ConfigurationManager.ConnectionStrings[Constants.DB_ConnectionString_Quandl]);
config[Constants.SourceDir] = path + ConfigurationManager.AppSettings[Constants.SourceDir];
config[Constants.ProcessedDir] = path + ConfigurationManager.AppSettings[Constants.ProcessedDir];
config[Constants.RejectedDir] = path + ConfigurationManager.AppSettings[Constants.RejectedDir];
config[Constants.TempDir] = path + ConfigurationManager.AppSettings[Constants.TempDir];
config[Constants.EODPricesFTPServerURL] = ConfigurationManager.AppSettings[Constants.EODPricesFTPServerURL];
config[Constants.EODPricesFTPUserName] = ConfigurationManager.AppSettings[Constants.EODPricesFTPUserName];
config[Constants.EODPricesFTPPassword] = ConfigurationManager.AppSettings[Constants.EODPricesFTPPassword];
config[Constants.EODSecuritiesFTPServerURL] = ConfigurationManager.AppSettings[Constants.EODSecuritiesFTPServerURL];
config[Constants.EODSecuritiesFTPUserName] = ConfigurationManager.AppSettings[Constants.EODSecuritiesFTPUserName];
config[Constants.EODSecuritiesFTPPassword] = ConfigurationManager.AppSettings[Constants.EODSecuritiesFTPPassword];
config[Constants.PredictaFTPServerURL] = ConfigurationManager.AppSettings[Constants.PredictaFTPServerURL];
config[Constants.PredictaFTPUserName] = ConfigurationManager.AppSettings[Constants.PredictaFTPUserName];
config[Constants.PredictaFTPPassword] = ConfigurationManager.AppSettings[Constants.PredictaFTPPassword];
config[Constants.Exchanges] = ConfigurationManager.AppSettings[Constants.Exchanges];
config[Constants.EODExchanges] = ConfigurationManager.AppSettings[Constants.EODExchanges];
config[Constants.QuandlExchanges] = ConfigurationManager.AppSettings[Constants.QuandlExchanges];
config[Constants.IntrinioExchanges] = ConfigurationManager.AppSettings[Constants.IntrinioExchanges];
config[Constants.ChooseSecuritiesForExchange] = ConfigurationManager.AppSettings[Constants.ChooseSecuritiesForExchange];
config[Constants.SchedulerSecondsToSleep] = ConfigurationManager.AppSettings[Constants.SchedulerSecondsToSleep];
config[Constants.DoDeleteFiles] = ConfigurationManager.AppSettings[Constants.DoDeleteFiles];
// Quandl securities
config[Constants.QuandlAPIkey] = ConfigurationManager.AppSettings[Constants.QuandlAPIkey];
config[Constants.QuandlSecuritiesFTPServerURL] = ConfigurationManager.AppSettings[Constants.QuandlSecuritiesFTPServerURL];
config[Constants.QuandlSecuritiesFTPUserName] = ConfigurationManager.AppSettings[Constants.QuandlSecuritiesFTPUserName];
config[Constants.QuandlSecuritiesFTPPassword] = ConfigurationManager.AppSettings[Constants.QuandlSecuritiesFTPPassword];
// Quandl prices
config[Constants.QuandlHistPricesFTPServerURL] = ConfigurationManager.AppSettings[Constants.QuandlHistPricesFTPServerURL];
config[Constants.QuandlPricesFTPServerURL] = ConfigurationManager.AppSettings[Constants.QuandlPricesFTPServerURL];
config[Constants.QuandlPricesFTPUserName] = ConfigurationManager.AppSettings[Constants.QuandlPricesFTPUserName];
config[Constants.QuandlPricesFTPPassword] = ConfigurationManager.AppSettings[Constants.QuandlPricesFTPPassword];
// Intrinio securities
config[Constants.IntrinioAPIkey] = ConfigurationManager.AppSettings[Constants.IntrinioAPIkey];
config[Constants.IntrinioSecuritiesInitFTPServerURL] = ConfigurationManager.AppSettings[Constants.IntrinioSecuritiesInitFTPServerURL];
config[Constants.IntrinioSecuritiesInitFTPServerURL_TASE] = ConfigurationManager.AppSettings[Constants.IntrinioSecuritiesInitFTPServerURL_TASE];
config[Constants.IntrinioSecuritiesFTPServerURL_start] = ConfigurationManager.AppSettings[Constants.IntrinioSecuritiesFTPServerURL_start];
config[Constants.IntrinioSecuritiesFTPServerURL_end] = ConfigurationManager.AppSettings[Constants.IntrinioSecuritiesFTPServerURL_end];
config[Constants.IntrinioSecuritiesFTPUserName] = ConfigurationManager.AppSettings[Constants.IntrinioSecuritiesFTPUserName];
config[Constants.IntrinioSecuritiesFTPPassword] = ConfigurationManager.AppSettings[Constants.IntrinioSecuritiesFTPPassword];
// Intrinio prices
config[Constants.IntrinioHistPricesFTPServerURL] = ConfigurationManager.AppSettings[Constants.IntrinioHistPricesFTPServerURL];
config[Constants.IntrinioPricesFTPServerURL] = ConfigurationManager.AppSettings[Constants.IntrinioPricesFTPServerURL];
config[Constants.IntrinioPricesFTPUserName] = ConfigurationManager.AppSettings[Constants.IntrinioPricesFTPUserName];
config[Constants.IntrinioPricesFTPPassword] = ConfigurationManager.AppSettings[Constants.IntrinioPricesFTPPassword];
string[] eodExchanges = null;
try
{
eodExchanges = config[Constants.EODExchanges].Split(new char[] { ',' });
}
catch
{
}
string[] exchanges = null;
try
{
exchanges = config[Constants.Exchanges].Split(new char[] { ',' });
}
catch
{
}
if (!Directory.Exists(config[Constants.SourceDir]))
{
Directory.CreateDirectory(config[Constants.SourceDir]);
}
if (!Directory.Exists(config[Constants.ProcessedDir]))
{
Directory.CreateDirectory(config[Constants.ProcessedDir]);
}
if (!Directory.Exists(config[Constants.RejectedDir]))
{
Directory.CreateDirectory(config[Constants.RejectedDir]);
}
if (!Directory.Exists(config[Constants.TempDir]))
{
Directory.CreateDirectory(config[Constants.TempDir]);
}
//LR : added to run from the source code
//args = new string[] { "-import", "eod", "2017/07/01", "2017/07/31", "-i" };
//args = new string[] { "-import", "eod", "2017/08/17", "2017/08/23", "-p" };
//args = new string[] { "-import", "eod", "-daily", "-s" };
//args = new string[] { "-import", "eod", "-daily", "-i" };
//args = new string[] { "-import", "eod", "-daily", "-i" };
//args = new string[] { "-import", "eod", "-last", "-i" };
//args = new string[] { "-calc" };
//args = new string[] { "-import", "eod", "-s" };
//args = new string[] { "-import", "predicta", "-daily", "-s" };
//args = new string[] { "-import", "predicta", "2017/08/03", "2017/08/15", "-i" };
//args = new string[] { "-import", "predicta", "2017/08/10", "2017/08/20", "-p" };
if (args[0].ToLower() == Constants.Import && args[1].ToLower() == Constants.EOD)
{
string[] eodImportArgs = new string[args.Length - 2];
for (int i = 2, i1 = 0; i < args.Length;)
{
eodImportArgs[i1++] = args[i++];
}
if (eodImportArgs.Contains(Constants.Securities))
{
EODSecuritiesImport.Execute(config, eodExchanges);
}
if (eodImportArgs.Contains(Constants.Prices))
{
EODPricesImport.Execute(eodImportArgs, config, eodExchanges, 0);
}
if (eodImportArgs.Contains(Constants.Indeces))
{
EODPricesImport.Execute(eodImportArgs, config, new string[] { Constants.EODIndecesImportFilenamePrefix }, Constants.IndecesExchangeCode);
}
}
else if (args[0].ToLower() == Constants.Import && args[1].ToLower() == Constants.Predicta)
{
string[] predictaImportArgs = new string[args.Length - 2];
for (int i = 2, i1 = 0; i < args.Length;)
{
predictaImportArgs[i1++] = args[i++];
}
PredictaImport.Execute(predictaImportArgs, config);
}
else if (args[0].ToLower() == Constants.Import && args[1].ToLower() == Constants.Quandl)
{
// string array of exchanges (Symbols) to download from Quandl
string[] qndlExchanges = null;
try
{
qndlExchanges = config[Constants.QuandlExchanges].Split(new char[] { ',' });
}
catch
{
}
// int array of exchanges id (DB table)
int[] exchangeID = new int[qndlExchanges.Length];
using (SqlConnection conn = new SqlConnection(config[Constants.DB_ConnectionString_Quandl]))
{
conn.Open();
for (int i = 0; i < qndlExchanges.Length; i++)
{
SqlCommand command = new SqlCommand("select idStockExchange from tblSel_StockExchanges where strSymbol = '" + qndlExchanges[i] + "'", conn);
exchangeID[i] = Convert.ToInt32(command.ExecuteScalar());
}
}
string[] quandlImportArgs = new string[args.Length - 2];
for (int i = 2, i1 = 0; i < args.Length;)
{
quandlImportArgs[i1++] = args[i++];
}
if (quandlImportArgs.Contains(Constants.Securities))
{
QuandlSecuritiesImport.Execute(config);
}
//if (quandlImportArgs.Contains(Constants.Prices))
// QuandlPricesImport.Execute(config, exchangeID);
//if (quandlImportArgs.Contains(Constants.HistPrices))
// QuandlHistPricesImport.Execute(config, exchangeID);
// OR -- temporary hardcoded call
if (quandlImportArgs.Contains(Constants.HistPrices) || quandlImportArgs.Contains(Constants.Prices))
{
string wichPrices;
if (quandlImportArgs.Contains(Constants.Prices))
{
wichPrices = Constants.Prices;
}
else
{
wichPrices = Constants.HistPrices;
}
QuoteMediaPricesImport.Execute(config, exchangeID, wichPrices);
}
//if (eodImportArgs.Contains(Constants.Prices))
// EODPricesImport.Execute(eodImportArgs, config, eodExchanges, 0);
//if (eodImportArgs.Contains(Constants.Indeces))
// EODPricesImport.Execute(eodImportArgs, config, new string[] { Constants.EODIndecesImportFilenamePrefix }, Constants.IndecesExchangeCode);
}
else if (args[0].ToLower() == Constants.Import && args[1].ToLower() == Constants.Intrinio)
{
// string array of exchanges (Symbols) to download from 'stock_exchanges.csv'
//string[] intriExchanges = null;
//try
//{
// intriExchanges = config[Constants.IntrinioExchanges].Split(new char[] { ',' });
//}
//catch
//{
//}
// OR
string[] intriExchanges = config[Constants.IntrinioExchanges].Split(new char[] { ',' });
// int array of exchanges id (DB table)
int[] exchangeID = new int[intriExchanges.Length];
using (SqlConnection conn = new SqlConnection(config[Constants.DB_ConnectionString_Quandl]))
{
conn.Open();
for (int i = 0; i < intriExchanges.Length; i++)
{
SqlCommand command = new SqlCommand("select idStockExchange from tblSel_StockExchanges where strSymbol = '" + intriExchanges[i] + "'", conn);
exchangeID[i] = Convert.ToInt32(command.ExecuteScalar());
}
}
string[] intriImportArgs = new string[args.Length - 2];
for (int i = 2, i1 = 0; i < args.Length;)
{
intriImportArgs[i1++] = args[i++];
}
// ************** Securities import: initial and fundamental ******************
if (intriImportArgs.Contains(Constants.Securities))
{
////IntrinioSecuritiesImport.Execute(config, exchangeID);
////return;
// Initial import
if (intriImportArgs[0].ToLower() == Constants.Initial)
{
// ***************************************************
// USCOMP - US composite - securities import
IntrinioSecuritiesInitialImport.Execute(config); // WARNING!!! SKIP USCOMP securities import if testing TASE !!!!!!!!!!!!!!
// Historical prices import for NEW securities
List <string> secWithDividOrSplit = new List <string>();
IntrinioHistPricesImport.Execute(config, exchangeID, secWithDividOrSplit, false, true);
// ***************************************************
// TASE securities import
IntrinioSecuritiesInitialImport.Execute(config, true);
// Historical prices import for NEW securities
int[] exchangeID_tase = new int[1] {
1
};
secWithDividOrSplit.Clear();
IntrinioHistPricesImport.Execute(config, exchangeID_tase, secWithDividOrSplit, true, true);
}
//************ FUNDAMENTALS *******************************
////!! Uncomment 2 lines when fundamentals are ok !! else // Fundamental import: only USCOMP
//// !!!!!!!!! IntrinioSecuritiesImport.Execute(config, exchangeID); // DO NOT RUN BEFORE CHECKING WE HAVE DATA IN calls
//************************************************************
}
// **************** TEST QUANDL *******************************
//if (quandlImportArgs.Contains(Constants.Prices))
// QuandlPricesImport.Execute(config, exchangeID);
//if (quandlImportArgs.Contains(Constants.HistPrices))
// QuandlHistPricesImport.Execute(config, exchangeID);
// ************************************************************
//**************** ORIGINAL -- UNCOMMENT AFTER WEEKEND RUN *************
// ************** Prices import: Historical
if (intriImportArgs.Contains(Constants.HistPrices)) // Historical prices import: USA/TASE/USA Indices
{
List <string> secWithDividOrSplit = new List <string>();
// USA exchanges
IntrinioHistPricesImport.Execute(config, exchangeID, secWithDividOrSplit); // WARNING!!! SKIP USA exchanges historical import if testing 'TASE exchange' or 'USA indices prices' !!!!!
//*******************************************************************************
// TASE exchange
int[] exchangeID_tase = new int[1] {
1
};
secWithDividOrSplit.Clear();
IntrinioHistPricesImport.Execute(config, exchangeID_tase, secWithDividOrSplit, true);
//*******************************************************************************
// USA indices prices - HISTORICAL
IntrinioHistIndicesPricesImport.Execute(config);
}
//*******************************************************************************
////////// CUSTOM HISTORY - WEEKEND RUN - USA HISTORICAL !!!!!!!!!!!!!!!!!
////////if (intriImportArgs.Contains(Constants.HistPrices)) // Historical prices import: USA/TASE/USA Indices
////////{
//////// List<string> secWithDividOrSplit = new List<string>();
//////// // USA exchanges
//////// IntrinioHistPricesImport.Execute(config, exchangeID, secWithDividOrSplit); // WARNING!!! SKIP USA exchanges historical import if testing 'TASE exchange' or 'USA indices prices' !!!!!
//////// //*******************************************************************************
//////// //// TASE exchange
//////// //int[] exchangeID_tase = new int[1] { 1 };
//////// //secWithDividOrSplit.Clear();
//////// //IntrinioHistPricesImport.Execute(config, exchangeID_tase, secWithDividOrSplit, true);
//////// //*******************************************************************************
//////// //// USA indices prices - HISTORICAL
//////// //IntrinioHistIndicesPricesImport.Execute(config);
////////}
else if (intriImportArgs.Contains(Constants.Prices)) // Daily prices import: USA/TASE/USA Indices
{
// USA exchanges
List <string> secWithDividOrSplit = new List <string>();
IntrinioPricesImport.Execute(intriImportArgs, config, exchangeID, secWithDividOrSplit); // USCOMP prices import // WARNING!!! SKIP USCOMP securities import if testing TASE !!!!!!!!!!!!!!
if (secWithDividOrSplit.Count > 0)
{
// Import historical prices for securities with divident or/and split
IntrinioHistPricesImport.Execute(config, exchangeID, secWithDividOrSplit);
}
// TASE exchange
int[] exchangeID_tase = new int[1] {
1
};
secWithDividOrSplit.Clear();
IntrinioPricesImport.Execute(intriImportArgs, config, exchangeID_tase, secWithDividOrSplit, true); // TASE prices import
if (secWithDividOrSplit.Count > 0)
{
// Import historical prices for securities with divident or/and split
IntrinioHistPricesImport.Execute(config, exchangeID_tase, secWithDividOrSplit, true);
}
// USA indices prices - DAILY
IntrinioHistIndicesPricesImport.Execute(config, true);
// TASE USD/ILS exchange rate 9001 import
IntrinioPredictaImport.Execute(intriImportArgs, config);
}
}
//////else if (args[0].ToLower() == Constants.CalcIntrinio)
//////{
////// string[] intriExchanges = (config[Constants.IntrinioExchanges]).Split(new char[] { ',' });
////// // int array of exchanges id (DB table)
////// int[] exchangeID = new int[intriExchanges.Length];
////// using (SqlConnection conn = new SqlConnection(config[Constants.DB_ConnectionString_Quandl]))
////// {
////// conn.Open();
////// for (int i = 0; i < intriExchanges.Length; i++)
////// {
////// SqlCommand command = new SqlCommand("select idStockExchange from tblSel_StockExchanges where strSymbol = '" + intriExchanges[i] + "'", conn);
////// exchangeID[i] = Convert.ToInt32(command.ExecuteScalar());
////// }
////// }
////// BatchCalculationsIntrinio_CUSTOM.Execute(config, exchangeID);
//////}
//**************** ORIGINAL -- UNCOMMENT AFTER WEEKEND RUN *************
else if (args[0].ToLower() == Constants.CalcIntrinio)
{
//// USA ONLY!!!!!!!!! string[] intriExchanges = (config[Constants.IntrinioExchanges]).Split(new char[] { ',' });
//// TASE ONLY!!!!!!!!!string[] intriExchanges = ("TASE").Split(new char[] { ',' });
string[] intriExchanges = (config[Constants.IntrinioExchanges] + ",TASE").Split(new char[] { ',' });
// int array of exchanges id (DB table)
int[] exchangeID = new int[intriExchanges.Length];
using (SqlConnection conn = new SqlConnection(config[Constants.DB_ConnectionString_Quandl]))
{
conn.Open();
for (int i = 0; i < intriExchanges.Length; i++)
{
SqlCommand command = new SqlCommand("select idStockExchange from tblSel_StockExchanges where strSymbol = '" + intriExchanges[i] + "'", conn);
exchangeID[i] = Convert.ToInt32(command.ExecuteScalar());
}
}
BatchCalculationsIntrinio.Execute(config, exchangeID);
}
//***********************************************************************
else if (args[0].ToLower() == Constants.Calc)
{
int[] idExchanges = new int[exchanges.Length];
using (SqlConnection conn = new SqlConnection(config[Constants.DB_ConnectionString]))
{
conn.Open();
for (int i = 0; i < exchanges.Length; i++)
{
SqlCommand command = new SqlCommand("select idStockExchange from tblSel_StockExchanges where strSymbol = '" + exchanges[i] + "'", conn);
idExchanges[i] = Convert.ToInt32(command.ExecuteScalar());
}
}
BatchCalculations.Execute(config, idExchanges);
}
else if (args[0].ToLower() == Constants.Scheduler)
{
for (; ;)
{
int scheduleIntervalSecs = Convert.ToInt32(config[Constants.SchedulerSecondsToSleep]);
bool runPredictaImport = false;
bool runEodIndecesImport = false;
List <string> lsExchanges = new List <string>();
List <int> idExchanges = new List <int>();
using (SqlConnection conn = new SqlConnection(config[Constants.DB_ConnectionString]))
{
conn.Open();
SqlCommand command = new SqlCommand(
"select t.idStockExchange, e.strSymbol, t.scheduleTime from tbl_ExchangesTiming t left outer join tblSel_StockExchanges e on t.idStockExchange = e.idStockExchange",
conn);
SqlDataReader reader = command.ExecuteReader();
while (reader.Read())
{
DateTime scheduleTime1 = Convert.ToDateTime(reader[2]);
DateTime scheduleTime2 = new DateTime(DateTime.Today.Year, DateTime.Today.Month, DateTime.Today.Day, scheduleTime1.Hour, scheduleTime1.Minute, scheduleTime1.Second);
if (DateTime.Now.AddSeconds(-scheduleIntervalSecs) < scheduleTime2 && DateTime.Now.AddSeconds(60) > scheduleTime2)
{
int exchangeId = Convert.ToInt32(reader[0]);
if (exchangeId == Constants.PredictaExchangeId)
{
runPredictaImport = true;
}
else if (exchangeId == Constants.IndecesExchangeCode)
{
runEodIndecesImport = true;
}
else
{
idExchanges.Add(exchangeId);
lsExchanges.Add(Convert.ToString(reader[1]));
}
}
}
}
if (runEodIndecesImport || runPredictaImport || idExchanges.Count > 0)
{ // Main scheduler
ThreadStart starter = delegate { ScheduleCherriesBatchProcessTask(runPredictaImport, runEodIndecesImport, idExchanges.ToArray(), lsExchanges.ToArray(), config); };
Thread thread = new Thread(starter);
thread.Start();
}
Thread.Sleep(scheduleIntervalSecs * 1000);
}
}
}