public void Parse(string url)
{
MasterIndex masterIndex = AddMasterIndex(url);
Stream masterIndexStream = Ftp.GetZippedFile(url);
ParseMasterIndexStream(masterIndex.MasterIndexId, masterIndexStream);
}
public void ProcessFullIndex(ushort year, ushort quarter)
{
string key = year.ToString("0000") + quarter.ToString("00");
if (tasks.ContainsKey(key))
{
return;
}
Task task = Task.Factory.StartNew(() =>
{
logger.Info($"ProcessFullIndex - Init proces for year {year}, quarter {quarter}");
try
{
MasterIndex index;
Quarter q = (Quarter)quarter;
logger.Info($"ProcessFullIndex - Getting full index from DB");
index = GetFullIndexFromDB(year, q);
if (index != null && index.IsComplete)
{
return(index);
}
logger.Info($"ProcessFullIndex - Downloading full index from Web");
string content;
if (GetFullIndexFromWeb(year, q, out content))
{
if (index == null)
{
logger.Info($"ProcessFullIndex - Creating new index");
index = new MasterIndex();
index.Quarter = q;
index.Year = year;
}
logger.Info($"ProcessFullIndex - Parsing index");
IList <IndexEntry> entries = parser.ParseMasterIndex(content);
index.TotalLines = entries.Count;
logger.Info($"ProcessFullIndex - Updating total lines");
edgarFilesRepo.Update(index, "TotalLines");
logger.Info($"ProcessFullIndex - Saving entries to DB");
SaveIndexEntriesToDB(index, entries);
logger.Info($"ProcessFullIndex - End successful");
return(index);
}
else
{
throw new ApplicationException($"It wasn't able to retrieve index (year={year}, quarter={quarter}");
}
}
catch (Exception ex)
{
logger.Fatal("ProcessFullIndex - Exception: " + ex.Message, ex);
throw ex;
}
}, TaskCreationOptions.LongRunning).ContinueWith(ProcessFullIndexWhenFinish, key);
tasks.Add(key, task);
}
private void SaveIndexEntriesToDB(MasterIndex index, IList <IndexEntry> entries)
{
edgarFilesBulkRepo.SaveIndexEntries(index, entries);
long dbRowsCopied = edgarFilesRepo.GetIndexEntriesCount(index);
index.ProcessedLines = dbRowsCopied;
edgarFilesRepo.Update(index, "ProcessedLines");
if (index.ProcessedLines == index.TotalLines)
{
index.IsComplete = true;
edgarFilesRepo.Update(index, "IsComplete");
}
}
private MasterIndex AddMasterIndex(string url)
{
MasterIndex masterIndex = _db.MasterIndexes.FirstOrDefault(m => m.Url == url);
if (masterIndex == null)
{
masterIndex = GetMasterIndexFromUrl(url);
Console.WriteLine($"Adding new MasterIndex for {masterIndex.Name}");
_db.MasterIndexes.Add(masterIndex);
_db.SaveChanges();
}
else
{
Console.WriteLine($"MasterIndex {masterIndex.Name} already exists, continuing.");
}
return(masterIndex);
}
public long SaveIndexEntries(MasterIndex index, IList <IndexEntry> entries)
{
log.Info("SaveIndexEntries - Saving data to datatable");
string tableName = "IndexEntries";
DataTable dt = GetEmptyDataTable(tableName);
foreach (IndexEntry entry in entries)
{
DataRow dr = dt.NewRow();
dr["CIK"] = entry.CIK;
dr["FormTypeId"] = entry.FormTypeId;
dr["DateFiled"] = entry.DateFiled;
dr["RelativeURL"] = entry.RelativeURL;
dr["MasterIndexId"] = index.Id;
//dr["MasterDailyIndex_Id"] = ?????;
dt.Rows.Add(dr);
}
log.Info($"SaveIndexEntries - Datatable has {dt.Rows.Count} rows, starting bulk copy");
long rowsCopied = BulkCopy(tableName, dt);
log.Info($"SaveIndexEntries - Bulk copy ended ok");
return(rowsCopied);
}
public IHttpActionResult GetDailyIndex(ushort year, ushort quarter, uint date)
{
MasterIndex index = indexService.ProcessDailyIndex(year, quarter, date);
return(Ok(index));
}
public ActionResult <MasterIndex> GetDailyIndex(ushort year, ushort quarter, uint date)
{
MasterIndex index = indexesParser.ProcessDailyIndex(year, quarter, date);
return(Ok(index));
}
public long GetIndexEntriesCount(MasterIndex index)
{
return(Context.IndexEntries.Where(entry => entry.MasterIndexId == index.Id).Count());
}
public void Update(MasterIndex index, string property)
{
Context.Entry <MasterIndex>(index).Property(property).IsModified = true;
Context.SaveChanges();
}
public void Add(MasterIndex index)
{
Context.MasterIndexes.Add(index);
Context.SaveChanges();
}
public void Test_MasterIndex_Ctor()
{
MasterIndex mi = new MasterIndex();
return;
}
