/// <inheritdoc/>
public IQuerySyntaxHelper GetQuerySyntaxHelper()
{
if (_cachedQuerySyntaxHelper == null)
{
_cachedQuerySyntaxHelper = ExternalCohortTable.GetQuerySyntaxHelper();
}
return(_cachedQuerySyntaxHelper);
}
/// <inheritdoc/>
public void ReverseAnonymiseDataTable(DataTable toProcess, IDataLoadEventListener listener, bool allowCaching)
{
int haveWarnedAboutTop1AlreadyCount = 10;
var syntax = ExternalCohortTable.GetQuerySyntaxHelper();
string privateIdentifier = syntax.GetRuntimeName(GetPrivateIdentifier());
string releaseIdentifier = syntax.GetRuntimeName(GetReleaseIdentifier());
//if we don't want to support caching or there is no cached value yet
if (!allowCaching || _releaseToPrivateKeyDictionary == null)
{
DataTable map = FetchEntireCohort();
Stopwatch sw = new Stopwatch();
sw.Start();
//dictionary of released values (for the cohort) back to private values
_releaseToPrivateKeyDictionary = new Dictionary <string, string>();
foreach (DataRow r in map.Rows)
{
if (_releaseToPrivateKeyDictionary.Keys.Contains(r[releaseIdentifier]))
{
if (haveWarnedAboutTop1AlreadyCount > 0)
{
haveWarnedAboutTop1AlreadyCount--;
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Top 1-ing will occur for release identifier " + r[releaseIdentifier] + " because it maps to multiple private identifiers"));
}
else
{
if (haveWarnedAboutTop1AlreadyCount == 0)
{
haveWarnedAboutTop1AlreadyCount = -1;
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Top 1-ing error message disabled due to flood of messages"));
}
}
}
else
{
_releaseToPrivateKeyDictionary.Add(r[releaseIdentifier].ToString().Trim(), r[privateIdentifier].ToString().Trim());
}
_reverseAnonymiseProgressFetchingMap++;
if (_reverseAnonymiseProgressFetchingMap % 500 == 0)
{
listener.OnProgress(this, new ProgressEventArgs("Assembling Release Map Dictionary", new ProgressMeasurement(_reverseAnonymiseProgressFetchingMap, ProgressType.Records), sw.Elapsed));
}
}
listener.OnProgress(this, new ProgressEventArgs("Assembling Release Map Dictionary", new ProgressMeasurement(_reverseAnonymiseProgressFetchingMap, ProgressType.Records), sw.Elapsed));
}
int nullsFound = 0;
int substitutions = 0;
Stopwatch sw2 = new Stopwatch();
sw2.Start();
//fix values
foreach (DataRow row in toProcess.Rows)
{
try
{
object value = row[releaseIdentifier];
if (value == null || value == DBNull.Value)
{
nullsFound++;
continue;
}
row[releaseIdentifier] = _releaseToPrivateKeyDictionary[value.ToString().Trim()].Trim();//swap release value for private value (reversing the anonymisation)
substitutions++;
_reverseAnonymiseProgressReversing++;
if (_reverseAnonymiseProgressReversing % 500 == 0)
{
listener.OnProgress(this, new ProgressEventArgs("Substituting Release Identifiers For Private Identifiers", new ProgressMeasurement(_reverseAnonymiseProgressReversing, ProgressType.Records), sw2.Elapsed));
}
}
catch (KeyNotFoundException e)
{
throw new Exception("Could not find private identifier (" + privateIdentifier + ") for the release identifier (" + releaseIdentifier + ") with value '" + row[releaseIdentifier] + "' in cohort with cohortDefinitionID " + OriginID, e);
}
}
//final value
listener.OnProgress(this, new ProgressEventArgs("Substituting Release Identifiers For Private Identifiers", new ProgressMeasurement(_reverseAnonymiseProgressReversing, ProgressType.Records), sw2.Elapsed));
if (nullsFound > 0)
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "Found " + nullsFound + " null release identifiers amongst the " + toProcess.Rows.Count + " rows of the input data table (on which we were attempting to reverse annonymise)"));
}
listener.OnNotify(this, new NotifyEventArgs(substitutions > 0?ProgressEventType.Information : ProgressEventType.Error, "Substituted " + substitutions + " release identifiers for private identifiers in input data table (input data table contained " + toProcess.Rows.Count + " rows)"));
toProcess.Columns[releaseIdentifier].ColumnName = privateIdentifier;
}
