/// <summary>
/// Get subqueries for all the child nodes in a configuration based off the query results
/// </summary>
/// <param name="config"></param>
/// <param name="queryResults"></param>
/// <returns></returns>
public static ConcurrentQueue <VistaQuery> getSubQueriesFromResults(ExtractorConfiguration config, QueryResults queryResults)
{
ConcurrentQueue <VistaQuery> subQueries = new ConcurrentQueue <VistaQuery>();
if (queryResults.DdrResults[0].Rows.Count == 0)
{
return(subQueries);
}
String queryResultsSiteCode = Convert.ToString((Int16)queryResults.DdrResults[0].Rows[0]["SiteCode"]);
foreach (TreeNode <QueryConfiguration> child in config.QueryConfigurations.RootNode.Children)
{
foreach (String ien in queryResults.SubQueryIens["/" + child.Value.File]) // TODO - make sure we're checking against correct format (/45/45.06 vs 45.06
{
VistaQuery current = new VistaQuery(config, child.Value);
current.MaxRecords = ""; // set blank because there is no looping in subfiles
current.IENS = ien;
current.StartIen = current.From = child.Value.From; // ticket #9 bug fix line - didn't uncover in testing because 1) weren't specifiying start IEN 2) not specifying start IEN on ExtractorConfiguration object which is where the property was being copied from. Get it instead directly from TreeNode
current.SiteCode = queryResultsSiteCode;
current.IsSubFileQuery = true;
subQueries.Enqueue(current);
}
}
return(subQueries);
}
//void queryKeyValue(ConcurrentQueue<VistaQuery> queries, ConcurrentDictionary<String, Dictionary<String, String>> resultsBag)
//{
// //ConcurrentQueue<VistaQuery> jobs = (ConcurrentQueue<VistaQuery>)queries;
// VistaQuery current = null;
// while (queries.TryDequeue(out current))
// {
// try
// {
// Dictionary<String, String> keyValResult = VistaDaoUtils.toDictFromDdrGetsEntry(_dao.ddrGetsEntry(current.SiteCode, current.VistaFile, current.IENS, current.Fields, current.Flags));
// resultsBag[current.IENS] = keyValResult;
// }
// catch (Exception)
// {
// // todo - create placeholder for exceptions!
// }
// }
//}
/// <summary>
/// Threaded WP query function
/// </summary>
/// <param name="allQueries">ConcurrentQueue of VistaQuery</param>
void queryWpOrComputed(ConcurrentQueue <VistaQuery> allQueries)
{
//ConcurrentQueue<VistaQuery> all = (ConcurrentQueue<VistaQuery>)allQueries;
VistaQuery myQuery = null;
while (allQueries.TryDequeue(out myQuery))
{
try
{
String[] result = _dao.ddrGetsEntry(myQuery.SiteCode, myQuery.VistaFile, myQuery.IENS, myQuery.Fields, myQuery.Flags);
String adjustedResult = DataTableUtils.addGetsEntryToDdrResult(myQuery.From, result, myQuery);
_wpOrComputedResults.TryAdd(myQuery.IENS, adjustedResult);
}
catch (Exception exc)
{
_exceptionBag.Add(exc);
try
{
if (myQuery.QueryErrorCount < 3)
{
myQuery.QueryErrorCount++;
allQueries.Enqueue(myQuery); // re-queue if error but don't try forever - probably a unrecoverable issue if multiple failures on same query
}
}
catch (Exception) { /* guess have no choice but to swallow... */ }
}
}
}
internal void threadedQuery(ConcurrentQueue <VistaQuery> concurrentQueueOfQueries)
{
//ConcurrentQueue<VistaQuery> queue = (ConcurrentQueue<VistaQuery>)concurrentQueueOfQueries;
VistaQuery currentQuery = null;
while (concurrentQueueOfQueries.TryDequeue(out currentQuery))
{
try
{
QueryResults result = query(currentQuery);
_resultsBag.Add(result);
}
catch (Exception exc)
{
_exceptionBag.Add(new downstream.domain.exception.DownstreamException("Problem with query:" + Environment.NewLine + currentQuery.ToString() + exc.ToString()));
try
{
if (currentQuery.QueryErrorCount < 3)
{
currentQuery.QueryErrorCount++;
concurrentQueueOfQueries.Enqueue(currentQuery); // re-queue if error but don't try forever - probably a unrecoverable issue if multiple failures on same query
}
}
catch (Exception) { /* guess have no choice but to swallow... */ }
}
}
}
/// <summary>
/// Build a queue of VistaQuery for a single subfile of an ExtractorConfiguration
/// </summary>
/// <param name="config"></param>
/// <param name="queryResults"></param>
/// <param name="subfile">This call's subfile will be of the format "/2.98"</param>
/// <returns></returns>
public static ConcurrentQueue <VistaQuery> getSubQueriesFromResultsBySubfile(ExtractorConfiguration config, QueryResults queryResults, String subfile)
{
ConcurrentQueue <VistaQuery> subQueries = new ConcurrentQueue <VistaQuery>();
if (queryResults.DdrResults[0].Rows.Count == 0)
{
return(subQueries);
}
String queryResultsSiteCode = Convert.ToString((Int16)queryResults.DdrResults[0].Rows[0]["SiteCode"]);
TreeNode <QueryConfiguration> queryConfig = config.QueryConfigurations.search(new QueryConfiguration()
{
File = subfile.Substring(1)
}); // search for non fully qualified file name
if (queryConfig == null || queryConfig.Value == null)
{
throw new ArgumentException("The specified subfile ({0}) doesn't exist in the configuration tree!", subfile);
}
foreach (String ien in queryResults.SubQueryIens[subfile]) // TODO - make sure we're checking against correct format (/45/45.06 vs 45.06
{
VistaQuery current = new VistaQuery(config, queryConfig.Value);
current.MaxRecords = ""; // set blank because there is no looping in subfiles
current.IENS = ien;
current.StartIen = current.From = queryConfig.Value.From; // ticket #9 bug fix line - didn't uncover in testing because 1) weren't specifiying start IEN 2) not specifying start IEN on ExtractorConfiguration object which is where the property was being copied from. Get it instead directly from TreeNode
current.SiteCode = queryResultsSiteCode;
current.IsSubFileQuery = true;
subQueries.Enqueue(current);
}
return(subQueries);
}
void consolidateKeyValueTables(QueryResults topLevelResults, VistaQuery query)
{
// we might have an _KEYVALUE tables in our collection of results - check for those and, if present, combine them all in one table
if (!String.IsNullOrEmpty(query.WP_Or_Computed_Fields) && !String.IsNullOrEmpty(query.Gets_Alignment))
{
// ok - so we know the configuration specified we should query to create a KEYVALUE table - did we get any results?
bool foundOne = false;
foreach (QueryResults result in _resultsBag)
{
if (result.DdrResults.Count > 1) // when the config specifies _KEYVALUE tables, we will have more than one DataTable in our QueryResults
{
foundOne = true;
break;
}
}
if (foundOne)
{
DataTable containerForAll = DataTableUtils.generateVerticalDataTable(query.VistaFile, true);
topLevelResults.DdrResults.Add(containerForAll);
foreach (QueryResults result in _resultsBag)
{
if (result.DdrResults.Count > 1 && result.DdrResults[1] != null && result.DdrResults[1].Rows.Count > 0)
{
DataTableUtils.addResultsToTable(containerForAll, result.DdrResults[1]);
}
}
}
}
}
public override string ToString()
{
StringBuilder sb = new StringBuilder();
sb.AppendLine("***** Start DownstreamException String *****");
if (!String.IsNullOrEmpty(Message))
{
sb.AppendLine("Message: " + Message);
}
if (InnerException != null)
{
sb.AppendLine("Inner Exception: " + InnerException.ToString());
}
if (Extractor != null)
{
sb.AppendLine("Extractor: " + Extractor.ToString());
}
if (VistaQuery != null)
{
sb.AppendLine("Vista Query: " + VistaQuery.ToString());
}
if (ExtractorConfiguration != null)
{
sb.AppendLine("Extractor Config: " + ExtractorConfiguration.ToString());
}
if (MessageTO != null)
{
sb.AppendLine("MessageTO Object: " + MessageTO.ToString());
}
sb.AppendLine("***** End DownstreamException String *****");
return(sb.ToString());
}
public static DataTable toKeyValTableFromDdrGetsEntry(VistaQuery query, Dictionary <String, String> keyVals, DateTime extractionTimestamp)
{
DataTable keyValTable = generateVerticalDataTable(query.VistaFile, query.IsSubFileQuery);
int currentIndex = query.IsSubFileQuery ? 4 : 3;
foreach (String key in keyVals.Keys)
{
object[] values = new object[5 + (query.IsSubFileQuery ? 1 : 0)]; // 5 columns if not subfile query, add 1 if is subfile query
if (query.IsSubFileQuery)
{
String ienString = StringUtils.convertIensToIen(query.IENS); // since this comes from DDR GETS ENTRY, the query will contain the IENS
values[0] = (ienString).Substring((ienString).IndexOf('_') + 1); // split the string nLevel_(n-1)Level_..._1Level in to two pieces and take the second - this gets rid of current record's IEN piece
values[1] = ienString;
values[2] = query.SiteCode;
values[3] = extractionTimestamp;
}
else
{
values[0] = StringUtils.convertIensToIen(query.IENS); // since this comes from DDR GETS ENTRY, the query will contain the IENS - we'll only have one comma piece if not subfile
values[1] = query.SiteCode;
values[2] = extractionTimestamp;
}
values[currentIndex] = key;
values[currentIndex + 1] = keyVals[key];
keyValTable.Rows.Add(values);
}
return(keyValTable);
}
public QueryResults queryWithDepth(ExtractorConfiguration config, VistaQuery topLevelQuery)
{
_configForQueryWithDepth = config;
// IEN seeding
VistaIenSeeder seeder = new VistaIenSeeder(_dao);
if (String.Equals(topLevelQuery.From, "0") && seeder.needsSeeding(topLevelQuery.VistaFile)) // if we're starting at beginning of file and file is being seeded
{
_report.addDebug("This file requires IEN seeding!");
topLevelQuery.From = seeder.getSeed(topLevelQuery.SiteCode, topLevelQuery.VistaFile);
if (String.IsNullOrEmpty(topLevelQuery.From)) // some sites/files have no records - getSeed will return an empty string in those cases
{
throw new NoDataInSiteAndFileException();
}
}
// end IEN seeding
// special DD stuff for 63 file tree
if (LabChemUtils.containsLabChemConfig(config))
{
setupLabChem(config, topLevelQuery);
}
// end 63 file tree setup
QueryResults topLevel = this.query(topLevelQuery);
if (topLevel.SubQueryIens != null && topLevel.SubQueryIens.Count > 0)
{
foreach (String subFile in topLevel.SubQueryIens.Keys)
{
ConcurrentQueue <VistaQuery> subFileQueryQueue = DataTableUtils.getSubQueriesFromResultsBySubfile(config, topLevel, subFile);
queryMultiple(config, subFileQueryQueue, topLevel);
}
}
return(topLevel);
}
/// <summary>
/// This is a helper function for ticket #16 - the least obtrusive solution seemed to be simply moving the identifier boolean values back out to the the
/// end of the DDR strings and placing the WP fields at the end of the regular fields and then using the existing functions to build the DataTable without modification
/// </summary>
public static void adjustDdrResultsWithWpAndIdentifiedFiles(VistaQuery query, String[] ddrResults)
{
String[] ddrFields = query.Fields.Split(semicolonDelim);
String[] wpFields = query.WP_Or_Computed_Fields.Split(semicolonDelim);
String[] identifiedFiles = query.IdentifiedFiles.Split(semicolonDelim);
// Query: fields = ".01;.03;1", identifiedFiles = "/2.06;/2.98", wp_or_computed = "5;8"
// .01 .03 1 /2.06 /2.98 5 8
// sample result coming in: PATIENT,ONE^3010101^123 MAIN ST^0^10[FS]This is some long text I had to get via a WP field call[FS]This is another big long text field
// which should be transformed to
// .01 .03 1 5 8 /2.06 /2.98
// PATIENT,ONE^3010101^123 MAIN ST^This is some long text I had to get via a WP field call^This is another big long text field^0^10
int firstIdentifierIndex = ddrFields.Length + 1; // + 1 to get past fields + IEN
int fistWpOrCompIndex = firstIdentifierIndex + identifiedFiles.Length; // then add the number of identified fields
String[] identifierPlaceholders = new String[identifiedFiles.Length];
String[] wpPlaceholders = new String[wpFields.Length];
for (int i = 0; i < ddrResults.Length; i++)
{
ddrResults[i] = ddrResults[i].Replace(FIELD_SEPARATOR, "^"); // replace FS with carat
ddrResults[i] = ddrResults[i].Replace("^", "^"); // replace identifier delim with carat
String[] allPieces = ddrResults[i].Split(caratDelim);
// put the identifier flags and WP fields in placeholder arrays
for (int j = 0; j < identifiedFiles.Length; j++)
{
identifierPlaceholders[j] = allPieces[firstIdentifierIndex + j];
}
for (int k = 0; k < wpFields.Length; k++)
{
wpPlaceholders[k] = allPieces[fistWpOrCompIndex + k];
}
// finally shift the values around in the string and rebuild the string with carat delimiters
StringBuilder sb = new StringBuilder();
for (int n = 0; n < ddrFields.Length + 1; n++) // loop through regular DDR values = don't forget to add + 1 for IEN!
{
sb.Append(allPieces[n]);
sb.Append("^");
}
for (int k = 0; k < wpFields.Length; k++) // then add the WP fields from the placeholders
{
sb.Append(wpPlaceholders[k]);
sb.Append("^");
}
for (int j = 0; j < identifiedFiles.Length; j++)
{
sb.Append(identifierPlaceholders[j]);
sb.Append("^");
}
sb.Remove(sb.Length - 1, 1); // remove trailing carat
ddrResults[i] = sb.ToString(); // adjusted DDR result ready to roll!
}
// done massaging! the ddrResults should be fixed and toQueryResults function below should now work for the adjusted ddrResults!
}
/// <summary>
/// Downstream extractor constructor. Any of the domain specific arguments can be null
/// </summary>
/// <param name="extractor"></param>
/// <param name="extractorConfig"></param>
/// <param name="vistaQuery"></param>
/// <param name="messageTO"></param>
/// <param name="message"></param>
/// <param name="inner"></param>
public DownstreamException(Extractor extractor, ExtractorConfiguration extractorConfig, VistaQuery vistaQuery,
MessageTO messageTO, string message, Exception inner)
: base(message, inner)
{
Extractor = extractor;
ExtractorConfiguration = extractorConfig;
VistaQuery = vistaQuery;
MessageTO = messageTO;
}
public QueryResults query(VistaQuery query)
{
// if we're querying unpacked, we should add the "WID" field to our fields string so DDR LISTER will return the identifier values
String[] ddrResults = _dao.ddrLister(query.SiteCode, query.VistaFile, query.IENS, query.Fields, query.Flags, query.MaxRecords, query.From, query.Part, query.XREF, query.Screen, query.Identifier);
// special 63.04 handling
if (String.Equals(query.VistaFile, "63.04"))
{
_labChemIens.Add(query.IENS.Replace(",", "")); // add the file 63 IEN to this collection for ticket #76
//return new LabChemUtils(_labChemDataDictionary).parseLabChemDdrResults(query, ddrResults);
return(new LabChemUtils().parseLabChemDdrResults(query, ddrResults));
}
// end special 63.04 handling
// first check to see if we should fetch key/val (aka vertical) results
DataTable verticalResults = null;
if (!String.IsNullOrEmpty(query.WP_Or_Computed_Fields) && !String.IsNullOrEmpty(query.Gets_Alignment)) // KEY/VAL SUB QUERIES
{
verticalResults = getVerticalResultsForQueries(query, ddrResults, query.IENS);
} // if not looking for vertical, do we need to fetch WP or other large fields and add them to DDR?
else if (!String.IsNullOrEmpty(query.WP_Or_Computed_Fields)) // WP fields
{
ddrResults = addWpOrComputed(query, ddrResults);
}
QueryResults qr = null;
// if we are fetching WP fields AND this configuration isn't building a key value table AND there are subfiles then we need to call our super special method in DataTableUtils! ticket #16
if (!String.IsNullOrEmpty(query.WP_Or_Computed_Fields) && String.IsNullOrEmpty(query.Gets_Alignment) && !String.IsNullOrEmpty(query.IdentifiedFiles))
{
DataTableUtils.adjustDdrResultsWithWpAndIdentifiedFiles(query, ddrResults);
qr = DataTableUtils.toQueryResultsFromDdr(query, ddrResults); // ddrResults is "fixed" by adjust function
}
else // for most cases, just building this up without special logic above
{
qr = DataTableUtils.toQueryResultsFromDdr(query, ddrResults);
}
// ugh - this seems ugly and hackish doing this out of process from the subqueries for WP fields above... oh, well, seems ok for now at least
if (!String.IsNullOrEmpty(query.WP_Or_Computed_Fields) && _exceptionBag != null)
{
foreach (Exception e in _exceptionBag)
{
_report.Exceptions.Add(e);
}
}
// did we have any key/val queries? if so, add the table to our results
if (verticalResults != null)
{
qr.DdrResults.Add(verticalResults);
}
return(qr); // DataTableUtils.toQueryResultsFromDdr(query, ddrResults);
}
internal void setupLabChem(ExtractorConfiguration config, VistaQuery topLevelQuery)
{
if (!(_dao is MdoVistaDao))
{
throw new ConfigurationException("File 63 tree with LAB CHEM can only be extracted with a local MdoVistaDao Vista DAO impl configuration");
}
//_labChemDataDictionary = VistaDaoUtils.getLabChemFieldsDynamic(topLevelQuery.SiteCode,
// (gov.va.medora.mdo.dao.AbstractConnection)gov.va.medora.mdo.domain.pool.connection.ConnectionPools.getInstance().checkOutAlive(topLevelQuery.SiteCode));
_labChemIens = new List <String>(); // use this to keep track of file 63 IENs that have data in 63.04 for purposes of ticket #76
}
static void extractWithStopIen(ExtractorConfiguration config, String startIen, String stopIen)
{
VistaQuery query = new VistaQuery(config, config.QueryConfigurations.RootNode.Value);
query.StartIen = query.From = startIen;
query.SiteCode = config.SiteCode;
VistaService svc = new VistaService();
svc.LastIen = stopIen;
svc.testExecute(new domain.reporting.ExtractorReport("TestID"), query, config, new FileDao(false));
}
public QueryResults parseLabChemDdrResults(VistaQuery query, String[] results)
{
Dictionary <String, IList <String> > testsByIens = parseLabChemDdrResultsAndSetCommentFlags(query, results);
Dictionary <String, String> commentsByIens = VistaDaoUtils.getCommentsForIens(query.SiteCode, this.IensWithComment);
mergeLabChemCommentsWithParsedResults(commentsByIens, testsByIens);
return(new QueryResults()
{
StringResult = buildResults(testsByIens)
});
}
/// <summary>
/// This method should only be called from a test! It is used to bypass all the server startup, get last IEN, etc. code
/// </summary>
/// <param name="report"></param>
/// <param name="query"></param>
/// <param name="config"></param>
/// <param name="fileDao"></param>
internal void testExecute(ExtractorReport report, VistaQuery query, ExtractorConfiguration config, FileDao fileDao)
{
_report = report;
_vistaQuery = query;
_config = config;
_fileDao = fileDao;
_report.setConfiguration(_config);
_vistaDao = new VistaDaoImpl();
// call this to make sure setup is the same
if (!checkSiteForWork())
{
throw new ConfigurationErrorsException("Test execute checkSiteForWork returned false");
}
// do this for loop in execute
_server = new Server(true);
_server.startListener();
_serviceState = new VistaServiceState();
_serviceState.Status = ServiceStatus.RUNNING;
this.extract();
}
public QueryResults queryWithDepth(ExtractorConfiguration config, VistaQuery topLevelQuery)
{
throw new NotImplementedException();
}
public Dictionary <String, IList <String> > parseLabChemDdrResultsAndSetCommentFlags(VistaQuery query, String[] results)
{
Dictionary <String, IList <String> > testsByIens = new Dictionary <String, IList <String> >();
String parentIen = query.IENS.Replace(",", "");
//StringBuilder sb = new StringBuilder();
for (int i = 0; i < results.Length; i++) // String s in subResults)
{
String currentIen = results[i].Split(new char[] { '^' })[0];
// comment was written out first so let's get that
String[] ddrPartAndIds = results[i].Split(new String[] { "^" }, StringSplitOptions.None);
int firstTilde = ddrPartAndIds[1].IndexOf('~');
String commentFlag = "";
String restOfIdentifier = "";
if (firstTilde > 0)
{
commentFlag = ddrPartAndIds[1].Substring(0, firstTilde);
restOfIdentifier = ddrPartAndIds[1].Substring(firstTilde + 1);
}
else // no tests!!! only have comments flag
{
commentFlag = ddrPartAndIds[1].Trim();
restOfIdentifier = "";
}
if (!String.Equals(commentFlag, "0"))
{
this.IensWithComment.Add(String.Concat(",", currentIen, query.IENS));
}
// now that we've obtained the comment, let's remove the flag for comment for getAllValsFromDelimited doesn't need to be modified
results[i] = String.Concat(ddrPartAndIds[0], "^", restOfIdentifier);
String[] multipleLines = getAllValsFromDelimited(results[i]);
IList <String> tests = new List <String>();
for (int j = 0; j < multipleLines.Length; j++)
{
String recordWithoutIen = multipleLines[j].Substring(multipleLines[j].IndexOf('^') + 1); // remove the IEN from the DDR results string
multipleLines[j] = parentIen + FIELD_DELIM + String.Concat(parentIen, "_", currentIen) + FIELD_DELIM + query.SiteCode + FIELD_DELIM + System.DateTime.Now.ToString() + FIELD_DELIM + recordWithoutIen.Replace('^', FIELD_DELIM);
multipleLines[j] = String.Concat(multipleLines[j], FIELD_DELIM); // add one more field delimiter in prep for comment field
//sb.Append(multipleLines[j]);
//sb.Append(RECORD_DELIM);
tests.Add(multipleLines[j]);
}
testsByIens.Add(String.Concat(",", currentIen, query.IENS), tests); // e.g. ,6899697.918861,277, -> UREA NITROGEN, WBC, RBC, test 4, etc...
}
return(testsByIens);
}
protected override void run()
{
try
{
MessageTO jobResponse = null;
if (String.Equals("false", ConfigurationManager.AppSettings[config.AppConfigSettingsConstants.EnforceCallback], StringComparison.CurrentCultureIgnoreCase))
{
jobResponse = _client.sendNewJobRequest("SansCallback", 0);
}
else // default if config item doesn't exist or it set to anything but false
{
jobResponse = _client.sendNewJobRequest(_server.SocketContainer.HostName, _server.SocketContainer.ListeningPort);
}
_client.disconnect();
// check job stack isn't empty
if (jobResponse != null && jobResponse.MessageType == MessageTypeTO.NewJobResponse &&
!String.IsNullOrEmpty(jobResponse.Message) && jobResponse.Message.Contains("No more jobs"))
{
_report.addInfo("The Orchestrator service reported there are no more jobs on the work stack. Exiting normally...");
return;
}
// then check we received a valid response
if (jobResponse == null || jobResponse.MessageType != MessageTypeTO.NewJobResponse ||
jobResponse.Configuration == null || !jobResponse.Configuration.isCompleteConfiguration())
{
throw new DownstreamException(null, null, null, jobResponse, "Invalid new job response!", null);
}
_report.addInfo("Successfully connected and registered with orchestrator and received a valid job!");
_config = jobResponse.Configuration;
_report.setConfiguration(_config);
_report.BatchId = jobResponse.Message; // we have the batch ID now - set it on the report for easy tracing
_extractor = jobResponse.Extractor;
// now that we have our config, we can create our file DAO
_fileDao = new FileDaoFactory().getFileDao();
_fileDao.setExtractsDirectory(_config.SiteCode, jobResponse.Message); // messsage should contain correct directory name
// Create the top level query that will drive any subqueries
_vistaQuery = new VistaQuery(_config, _config.QueryConfigurations.RootNode.Value);
if (checkSiteForWork())
{
_report.addDebug(_config.SiteCode + " appears to have work! Starting extraction job...");
extract();
}
else
{
_report.addDebug("Site appears to have no work!");
}
// finally, try and notify orchestrator we have finished
sendUnlockRequest(MessageTypeTO.JobCompletedRequest, _client);
}
catch (Exception exc)
{
_report.Errored = true;
_report.addException(exc);
if (_config != null) // if this was set then we know we received a job
{
sendUnlockRequest(MessageTypeTO.JobErrorRequest, _client);
}
}
finally
{
try
{
_client.disconnect();
_server.stopListener();
}
catch (Exception) { /* nothing we can do if these error */ }
GC.Collect();
}
}
// per #30 - need to validate every single record
internal static IList <domain.reporting.Exceptional> validateDdrResults(VistaQuery vq, ref String[] ddrResults)
{
IList <domain.reporting.Exceptional> exceptions = new List <domain.reporting.Exceptional>();
if (ddrResults == null || ddrResults.Length <= 0)
{
return(exceptions); // valid DDR results - just no data!
}
IList <String> destination = new List <String>();
String[] fields = vq.Fields.Split(semicolonDelim);
// identified files, wp fields
bool hasIdentifiedFiles = !String.IsNullOrEmpty(vq.IdentifiedFiles);
bool hasWpOrComputed = (!String.IsNullOrEmpty(vq.WP_Or_Computed_Fields) && String.IsNullOrEmpty(vq.Gets_Alignment)); // WP_OR_COMPUTED should have a value AND GETS_ALIGNMENT should not
String[] identifiedFiles = new String[0];
if (hasIdentifiedFiles)
{
identifiedFiles = vq.IdentifiedFiles.Split(semicolonDelim);
}
String[] wpOrComputed = new String[0];
if (hasWpOrComputed)
{
wpOrComputed = vq.WP_Or_Computed_Fields.Split(semicolonDelim);
}
for (int i = 0; i < ddrResults.Length; i++)
{
if (hasWpOrComputed && hasIdentifiedFiles) // special case! should have already called adjustDdrResults on this result set so we can check these ones separately
{
if (ddrResults[i].Split(caratDelim).Length != fields.Length + 1 + identifiedFiles.Length + wpOrComputed.Length)
{
exceptions.Add(new domain.reporting.Exceptional()
{
Code = domain.reporting.ErrorCode.INVALIDLY_FORMED_RECORD, Message = "Field count + WP field count + identifier field count of adjusted result did not match: " + ddrResults[i]
});
continue;
}
else
{
destination.Add(ddrResults[i]);
continue;
}
}
if (ddrResults[i].Split(caratDelim).Length < (fields.Length + 1 + (hasIdentifiedFiles ? 1 : 0))) // +1 for IEN - if we have identified files, an extra carat is returned with the DDR results
{
exceptions.Add(new domain.reporting.Exceptional()
{
Code = domain.reporting.ErrorCode.INVALIDLY_FORMED_RECORD, Message = "Field count did not match: " + ddrResults[i]
});
continue;
}
// now check identified files
if (hasIdentifiedFiles)
{
ddrResults[i] = ddrResults[i].Replace("^", "^");
if (ddrResults[i].Split(caratDelim).Length != (fields.Length + identifiedFiles.Length + 1)) // + 1 for IEN
{
exceptions.Add(new domain.reporting.Exceptional()
{
Code = domain.reporting.ErrorCode.INVALIDLY_FORMED_RECORD, Message = "Field count + identifier count did not match: " + ddrResults[i]
});
continue;
}
}
// now check WP fields
if (hasWpOrComputed)
{
ddrResults[i] = ddrResults[i].Replace(FIELD_SEPARATOR, "^");
if (ddrResults[i].Split(caratDelim).Length != (fields.Length + identifiedFiles.Length + wpOrComputed.Length + 1)) // + 1 for IEN - identified files length will just be zero is none are specified
{
exceptions.Add(new domain.reporting.Exceptional()
{
Code = domain.reporting.ErrorCode.INVALIDLY_FORMED_RECORD, Message = "Field count + WP field count did not match: " + ddrResults[i]
});
continue;
}
}
// finally... done with all our checks! add to destination which we will set the original String[] to when done with loop
destination.Add(ddrResults[i]);
}
if (destination.Count != ddrResults.Length)
{
exceptions.Add(new domain.reporting.Exceptional()
{
Code = domain.reporting.ErrorCode.INFORMATIONAL, Message = "It appears " + (ddrResults.Length - destination.Count) + " out of " + ddrResults.Length + " records were not valid!"
});
//System.Console.WriteLine("Looks like we skipped a record!");
}
ddrResults = destination.ToArray();
return(exceptions);
}
internal static string addGetsEntryToDdrResult(string parentDdrResult, string[] getsEntryResult, VistaQuery query)
{
// build up string to append: FS[.08]{0}FS[25]{1}FS[1001]{2}
StringBuilder strWithPlaceHolders = new StringBuilder();
String[] wpFields = query.WP_Or_Computed_Fields.Split(new char[] { ';' });
for (int i = 0; i < wpFields.Length; i++)
{
strWithPlaceHolders.Append(DataTableUtils.FIELD_SEPARATOR);
//strWithPlaceHolders.Append("[" + wpFields[i] + "]"); // used only for debugging purposes
strWithPlaceHolders.Append("{" + i.ToString() + "}");
}
String resultToAppend = strWithPlaceHolders.ToString();
String[] replacementPieces = new String[wpFields.Length];
int currentIdx = 0;
for (int i = 0; i < getsEntryResult.Length; i++)
{
String line = getsEntryResult[i];
String[] pieces = line.Split(new char[] { '^' });
if (pieces.Length < 4) // DDR GETS ENTRY results with data look like: 405.2^2^.08^This is some data
{
continue;
}
if (String.Equals("[WORD PROCESSING]", pieces[3], StringComparison.CurrentCultureIgnoreCase))
{
StringBuilder sb = new StringBuilder();
while (!String.Equals(getsEntryResult[++i], "$$END$$", StringComparison.CurrentCultureIgnoreCase))
{
sb.AppendLine(getsEntryResult[i]);
}
replacementPieces[currentIdx++] = sb.ToString();
}
else
{
replacementPieces[currentIdx++] = pieces[3];
}
}
return(String.Concat(parentDdrResult, String.Format(resultToAppend, replacementPieces)));
}
public void queryMultiple(ExtractorConfiguration config, ConcurrentQueue <VistaQuery> queriesToExecute, QueryResults topLevelResults)
{
while (queriesToExecute.Count > 0)
{
bool runningLabChem = false;
if (queriesToExecute != null && queriesToExecute.Count > 0 && String.Equals(queriesToExecute.First().VistaFile, "63.04"))
{
runningLabChem = true; // only set this true when we are actually executing lab chem queries
}
// first setup table to combine all results from multi-threaded queries
VistaQuery trash = null;
queriesToExecute.TryPeek(out trash);
DataTable subfileTable = DataTableUtils.generateVistaQueryDataTable(
trash.VistaFile, trash.Fields.Split(new char[] { ';' }), true, trash.WP_Or_Computed_Fields.Split(new char[] { ';' }));
// now multi-threaded query
query(queriesToExecute);
Dictionary <String, ConcurrentQueue <VistaQuery> > levelQueues = new Dictionary <string, ConcurrentQueue <VistaQuery> >();
// now we need to go through each query result and add the IENS to the level queue
foreach (QueryResults result in _resultsBag)
{
if (runningLabChem)
{
continue; // nothing to do here - just continue
}
// we didn't receive any results for this subfile even though the parent query indicated there was data - TBD - should we log???
if (result.DdrResults[0].Rows.Count <= 0)
{
_exceptionBag.Add(new domain.exception.DownstreamException(String.Format("No results found when querying a subfile despite indicator in parent file {0}", result.DdrResults[0].TableName)));
continue;
}
if (result.SubQueryIens == null)
{
continue;
}
foreach (String key in result.SubQueryIens.Keys)
{
if (!levelQueues.ContainsKey(key)) // we can't set this up before entering the loop because not all query results will have data/sub-query IENS
{
levelQueues.Add(key, new ConcurrentQueue <VistaQuery>());
}
ConcurrentQueue <VistaQuery> subQueries = DataTableUtils.getSubQueriesFromResultsBySubfile(config, result, key);
foreach (VistaQuery subQuery in subQueries)
{
levelQueues[key].Enqueue(subQuery);
}
}
}
// done setting up level queues
// special 63.04 handling
if (runningLabChem)
{
StringBuilder combined63x04Data = new StringBuilder();
foreach (QueryResults result in _resultsBag)
{
combined63x04Data.Append(result.StringResult);
}
topLevelResults.StringResult = combined63x04Data.ToString();
String[] temp = new String[_labChemIens.Count];
_labChemIens.CopyTo(temp, 0);
topLevelResults.LabChemIens = temp.ToList();
_labChemIens = null;
return; // don't want to run through code below!
}
// end special 63.04 handling
foreach (QueryResults result in _resultsBag)
{
DataTableUtils.addResultsToTable(subfileTable, result.DdrResults[0]);
}
foreach (Exception error in _exceptionBag)
{
_report.addException(error);
_report.addDebug("Found " + _exceptionBag.Count + " various errors while querying subfiles for this batch");
}
// we might have an _KEYVALUE tables in our collection of results - check for those and, if present, combine them all in one table
consolidateKeyValueTables(topLevelResults, trash);
// then add our table to top level (but only if we have records)
if (subfileTable != null && subfileTable.Rows.Count > 0)
{
topLevelResults.DdrResults.Add(subfileTable);
}
// let's log a summary of this level query
_report.addInfo("Found " + subfileTable.Rows.Count + " records for subfile " + subfileTable.TableName);
// finally recurse through subfiles of this subfile
foreach (String subFile in levelQueues.Keys)
{
if (levelQueues[subFile] != null && levelQueues[subFile].Count > 0)
{
_report.addInfo("Starting query for " + levelQueues[subFile].Count + " subfile records for file " + subFile + " based on identifier results from parent query");
}
else
{
_report.addInfo(subFile + " does not appear to have any data for this IEN range");
}
queryMultiple(config, levelQueues[subFile], topLevelResults);
}
}
}
// currently this is only being used for file 63.3 - because the results are nested and 63.3 doesn't typically contain
// a lot of records in the subfile path, i'm removing all the multithreading crap since it caused an issue with the extracted data
internal DataTable getVerticalResultsForQueries(VistaQuery topLevelQuery, String[] ddrResults, String parentRecordIENS)
{
DataTable verticalResults = DataTableUtils.generateVerticalDataTable(topLevelQuery.VistaFile, topLevelQuery.IsSubFileQuery);
if (ddrResults == null || ddrResults.Length == 0)
{
return(verticalResults);
}
// build queue of queries and prepare threadsafe placeholder for results
// _verticalResults = new ConcurrentDictionary<string, Dictionary<string, string>>();
Dictionary <String, Dictionary <String, String> > resultsDict = new Dictionary <String, Dictionary <String, String> >();
ConcurrentQueue <VistaQuery> verticalQueries = new ConcurrentQueue <VistaQuery>();
foreach (String ddrResult in ddrResults)
{
String correctedIensString = parentRecordIENS;
if (!String.IsNullOrEmpty(correctedIensString))
{
if (correctedIensString.StartsWith(","))
{
correctedIensString = correctedIensString.Substring(1, correctedIensString.Length - 1); // remove leading comma so IENS string below is builts correctly
}
}
correctedIensString = String.Concat(ddrResult.Split(new char[] { '^' })[0], ",", correctedIensString);// IEN from current ddr result + , + IENS string from top level query
VistaQuery vq = new VistaQuery()
{
Fields = topLevelQuery.WP_Or_Computed_Fields,
Flags = "IN",
IENS = correctedIensString,
SiteCode = topLevelQuery.SiteCode,
VistaFile = topLevelQuery.VistaFile, // same vista file as specified in top level query
};
Dictionary <String, String> result = VistaDaoUtils.toDictFromDdrGetsEntry(_dao.ddrGetsEntry(vq.SiteCode, vq.VistaFile, vq.IENS, vq.Fields, vq.Flags));
resultsDict.Add(correctedIensString, result);
// verticalQueries.Enqueue(vq);
// _verticalResults.TryAdd(correctedIensString, new Dictionary<string, string>()); // we'll have multiple results per IENS
}
// multi-thread queries
//IList<Task> verticalTasks = new List<Task>();
//for (int i = 0; i < _subQueryWorkers; i++)
//{
// Task workerTask = new Task(() => queryKeyValue(verticalQueries));
// verticalTasks.Add(workerTask);
// workerTask.Start();
//}
//foreach (Task t in verticalTasks)
//{
// t.Wait();
//}
// put all our results in to a single data table
DateTime batchTimestamp = DateTime.Now; // we give all our Key/Val the same timestamp
foreach (String key in resultsDict.Keys)
{
// DataTableUtils.toKeyValTableFromDdrGetsEntry only needs the subfile info, IENS, sitecode, and timestamp
VistaQuery reconstructedQuery = new VistaQuery()
{
IENS = key,
SiteCode = topLevelQuery.SiteCode,
IsSubFileQuery = topLevelQuery.IsSubFileQuery
};
DataTable currentBatch = DataTableUtils.toKeyValTableFromDdrGetsEntry(reconstructedQuery, resultsDict[key], batchTimestamp);
DataTableUtils.addResultsToTable(verticalResults, currentBatch);
}
// done! return vertical/key-val datatable
return(verticalResults);
}
public static QueryResults toQueryResultsFromDdr(VistaQuery query, String[] ddrResults)
{
QueryResults result = new QueryResults();
IList <domain.reporting.Exceptional> errors = validateDdrResults(query, ref ddrResults);
if (errors.Count > 0)
{
// no longer throwing an exception if zero results - a few cases found where the end of the file returns some kkoky
//if (ddrResults.Length == 0) // if all of the records were marked invalid, i guess we should just fail big
//{
// throw new DownstreamException(String.Format("Those DDR results appear to be invalid: {0}", ddrResults[0])) { VistaQuery = query };
//}
//else // otherwise just set our exceptions on the query results and we'll handle them further up the stack
//{
foreach (domain.reporting.Exceptional e in errors)
{
result.Exceptionals.Add(e);
}
//}
}
// toQueryResults assumes the ddrResults string contains the WP/Computed fields already (i.e. added elsewhere to each line) - when we added those results, we used the FS character.
// we need to swap that character back out with a carat so the loop below can parse the results more easily
String[] wpAndComputed = new String[0];
if (!String.IsNullOrEmpty(query.WP_Or_Computed_Fields) && String.IsNullOrEmpty(query.Gets_Alignment))
{
wpAndComputed = query.WP_Or_Computed_Fields.Split(semicolonDelim);
for (int i = 0; i < ddrResults.Length; i++)
{
ddrResults[i] = ddrResults[i].Replace(FIELD_SEPARATOR, "^");
}
}
String[] fields = query.Fields.Split(semicolonDelim);
DataTable currentTable = generateVistaQueryDataTable(query.VistaFile, fields, query.IsSubFileQuery, wpAndComputed);
result.DdrResults.Add(currentTable);
if (ddrResults == null || ddrResults.Length == 0)
{
return(result);
}
// Prepare to process our results
Int16 intSiteCode = Convert.ToInt16(query.SiteCode);
DateTime retrievalTime = DateTime.Now;
bool hasIdentifiedFiles = !String.IsNullOrEmpty(query.IdentifiedFiles);
String iensString = "";
String[] identifiedFiles = null;
int identifiedFilesStartIndex = 0;
if (hasIdentifiedFiles)
{
iensString = query.IENS;
result.SubQueryIens = new Dictionary <String, IList <String> >();
identifiedFiles = query.IdentifiedFiles.Split(semicolonDelim);
identifiedFilesStartIndex = query.Fields.Split(semicolonDelim).Length + 1 + wpAndComputed.Length; // we'll start looking at the resolved file indicators here - per ticket #16, we moved the identifier flags out to the end of the string so we need to go past the WP fields now, too
foreach (String identifiedFile in identifiedFiles)
{
result.SubQueryIens.Add(identifiedFile, new List <String>()); // only building our container objects here
}
}
String pIenStr = ""; // string for DataTable - looks like third_second_first per db requirements
if (query.IsSubFileQuery)
{
pIenStr = StringUtils.convertIensToIen(query.IENS);
}
// Push each valid ddrresult into our datatable
String enDdiolChar = "^";
String carat = "^";
foreach (string ddrResult in ddrResults)
{
// Break our string up into it's parts
string[] ddrValues = ddrResult.Replace(enDdiolChar, carat).Split(caratDelim);
object[] rowVals = new object[result.DdrResults[0].Columns.Count]; // generateVistaQueryDataTable already determined the correct number of columns
if (ddrValues.Length < fields.Length + 1) // +1 for the IEN - found some weird random cases where differing data seems to make it's way in to results - let's just ignore it for now
{
logging.Log.LOG("Found a malformed record in this query result! We're ignoring for now but may need to look at it later: " + ddrResult);
continue;
}
int currentColumnIndex = 3;
if (query.IsSubFileQuery)
{
rowVals[0] = pIenStr; // SECONDPARENTIEN_FIRSTPARENTIEN <-- converted to format by StringUtils function above
rowVals[1] = String.Concat(ddrValues[0], "_", pIenStr); // nthLevel_n-1Level_..._firstLevel
rowVals[2] = intSiteCode;
rowVals[3] = retrievalTime;
currentColumnIndex = 4;
}
else
{
rowVals[0] = ddrValues[0];
rowVals[1] = intSiteCode;
rowVals[2] = retrievalTime;
}
for (int i = 1; i < (query.IsSubFileQuery ? currentTable.Columns.Count - 3 : currentTable.Columns.Count - 2); i++) // already stored IEN so start at 1, number of columns minus extra added
{
rowVals[currentColumnIndex] = ddrValues[i];
currentColumnIndex++;
}
result.DdrResults[0].Rows.Add(rowVals);
// now take care of identified files
if (hasIdentifiedFiles)
{
for (int i = 0; i < identifiedFiles.Length; i++)
{
if (!String.Equals("0", ddrValues[identifiedFilesStartIndex + i]))
{
if (String.IsNullOrEmpty(iensString)) // if we don't already have an IENS string started, use a leading comma ',' character
{
result.SubQueryIens[identifiedFiles[i]].Add("," + ddrValues[0] + ","); // first child subfile - use parent's IEN in column 0
}
else // already started an IENS string (i.e. in a subfile) - concatenate the current IEN and a comma ',' character
{
result.SubQueryIens[identifiedFiles[i]].Add("," + ddrValues[0] + iensString); // add the IEN to the correct queue - if we're in a subfile, the IEN is the second column!!!
}
}
}
}
}
return(result);
}
public String[] addWpOrComputed(VistaQuery topLevelQuery, String[] ddrResults)
{
// create queue of queries
ConcurrentQueue <VistaQuery> wpOrComputedQueries = new ConcurrentQueue <VistaQuery>();
IList <String> iensStrings = new List <String>(); // we'll use this as a holding location for all our queries so we don't have to build the IENS string twice
foreach (String ddrResult in ddrResults)
{
String correctedIensString = topLevelQuery.IENS;
if (!String.IsNullOrEmpty(correctedIensString))
{
if (correctedIensString.StartsWith(","))
{
correctedIensString = correctedIensString.Substring(1, correctedIensString.Length - 1); // remove leading comma so IENS string below is builts correctly
}
}
correctedIensString = String.Concat(ddrResult.Split(new char[] { '^' })[0], ",", correctedIensString);// IEN from current ddr result + , + IENS string from top level query
iensStrings.Add(correctedIensString);
VistaQuery wpOrComputedQuery = new VistaQuery()
{
WP_Or_Computed_Fields = topLevelQuery.WP_Or_Computed_Fields, // we copy this because DataTableUtils.addGetsEntryToDdrResult looks for it there!!!
Fields = topLevelQuery.WP_Or_Computed_Fields,
VistaFile = topLevelQuery.VistaFile,
SiteCode = topLevelQuery.SiteCode,
IENS = correctedIensString,
Flags = "I",
From = ddrResult // NOTE - we're hijacking this field since it's not used in the DDR GETS ENTRY call to store the current DDR result for the multi-threaded queries
};
wpOrComputedQueries.Enqueue(wpOrComputedQuery);
}
_exceptionBag = new ConcurrentBag <Exception>();
_wpOrComputedResults = new ConcurrentDictionary <string, string>(); // create new before each kickoff
IList <Task> wpOrComputedTasks = new List <Task>();
//IList<Thread> wpOrComputedThreads = new List<Thread>();
for (int i = 0; i < _subQueryWorkers; i++)
{
Task wpOrComputedTask = new Task(() => queryWpOrComputed(wpOrComputedQueries));
wpOrComputedTasks.Add(wpOrComputedTask);
wpOrComputedTask.Start();
//Thread wpOrComputedThread = new Thread(new ParameterizedThreadStart(this.queryWpOrComputed));
//wpOrComputedThreads.Add(wpOrComputedThread);
//wpOrComputedThread.Start(wpOrComputedQueries);
}
foreach (Task t in wpOrComputedTasks)
{
t.Wait();
}
//foreach (Thread t in wpOrComputedThreads)
//{
// t.Join();
//}
for (int i = 0; i < iensStrings.Count; i++) // go through each IENS string and get value from subqueries dictionary
{
if (_wpOrComputedResults.ContainsKey(iensStrings[i]))
{
ddrResults[i] = _wpOrComputedResults[iensStrings[i]]; // replace DDR result with corrected - we do it this way to preserve the order of the original DDR query results
}
}
return(ddrResults);
}
internal static String vistaQueryToString(VistaQuery vq)
{
StringBuilder sb = new StringBuilder();
return(sb.ToString());
}
