private static bool RunNHSNumberTests()
{
Console.WriteLine();
Console.WriteLine("Running NHSNumber validation test");
OpenPseudonymiser.Crypto crypto = new Crypto();
bool success = true;
string processedNHSNumber = "";
// A call to ProcessNHSNumber will strip all non numeric characters from the string
processedNHSNumber = crypto.ProcessNHSNumber("4505577104");
// A call to the static "IsValidNHSNumber" method will return a true if the string passes the NHS number validation checksum as described here:
// http://www.datadictionary.nhs.uk/data_dictionary/attributes/n/nhs_number_de.asp
success = success & (NHSNumberValidator.IsValidNHSNumber(processedNHSNumber));
Console.WriteLine("NHSNumber validation test 1: " + success);
processedNHSNumber = crypto.ProcessNHSNumber("1");
success = success & !(NHSNumberValidator.IsValidNHSNumber(processedNHSNumber));
Console.WriteLine("NHSNumber validation test 2: " + success);
processedNHSNumber = crypto.ProcessNHSNumber("fish");
success = success & !(NHSNumberValidator.IsValidNHSNumber(processedNHSNumber));
Console.WriteLine("NHSNumber validation test 3: " + success);
processedNHSNumber = crypto.ProcessNHSNumber("1267716276817687612");
success = success & !(NHSNumberValidator.IsValidNHSNumber(processedNHSNumber));
Console.WriteLine("NHSNumber validation test 4: " + success);
processedNHSNumber = crypto.ProcessNHSNumber("450 557 7104");
success = success & (NHSNumberValidator.IsValidNHSNumber(processedNHSNumber));
Console.WriteLine("NHSNumber validation test 5: " + success);
processedNHSNumber = crypto.ProcessNHSNumber("450-557-7104");
success = success & (NHSNumberValidator.IsValidNHSNumber(processedNHSNumber));
Console.WriteLine("NHSNumber validation test 6: " + success);
Console.WriteLine("NHSNumber validation test: " + success);
return(success);
}
public static bool IsValidNHSNumber(string inputString)
{
return(NHSNumberValidator.IsValidNHSNumber(inputString));
}
private bool MainWork(string filename, int indexOfNHSNumber, Dispatcher dispatcher, ref bool wasCancelled)
{
_validNHSNumCount = 0;
_inValidNHSNumCount = 0;
_missingNHSNumCount = 0;
_rowsProcessed = 0;
// the thing that gets the digest..
Crypto crypto = new Crypto();
if (usingEncryptedSalt)
{
crypto.SetEncryptedSalt(File.ReadAllBytes(encryptedSaltFile));
}
else
{
crypto.SetPlainTextSalt(_salt);
}
FileStream fileStream = new FileStream(filename, FileMode.Open, FileAccess.Read);
FileStream writeStream;
try
{
writeStream = new FileStream(_outputFolder + "\\" + "OpenPseudonymised_" + _outputFileNameOnly, FileMode.Create, FileAccess.Write);
}
catch (IOException)
{
ErrorProgressDelegate error = new ErrorProgressDelegate(ErrorProgressText);
dispatcher.BeginInvoke(error, "OpenPseudonymiser cannot create the Output file. Is the output file already open?");
return(false);
}
_inputFileStreamLength = fileStream.Length;
long totalCharsRead = 0;
SortedList <string, int> inputFields;
SortedList <int, string> outputFields;
GetInputAndOutputFields(out inputFields, out outputFields);
WriteRunLogFile(inputFields, outputFields);
using (StreamWriter streamWriter = new StreamWriter(writeStream))
{
using (StreamReader streamReader = new StreamReader(fileStream))
{
// 'read off' the first line, these are the input columns headings
string[] inputHeadings = streamReader.ReadLine().Split(',');
// write the first line as the selected column headings
string lineToWrite = "Digest,";
foreach (int key in outputFields.Keys) // keys in the output are indexes (opposite to input SortedList)
{
lineToWrite += outputFields[key] + ",";
}
// Do we want to do any NHSNumber checksum validation?
if (_performNHSNumberValidation)
{
lineToWrite += "validNHS,";
}
// strip trailing comma
lineToWrite = lineToWrite.Substring(0, lineToWrite.Length - 1);
streamWriter.WriteLine(lineToWrite);
// We have no way of knowing how many lines there are in the file without opening the whole thing, which would kill the app.
// So we will use a fixed size buffer and manually look for lines inside it.
int _bufferSize = 16384;
char[] readBuffer = new char[_bufferSize];
StringBuilder workingBuffer = new StringBuilder(); // where we will store the left over stuff after we split the read buffer into lines
// read into the buffer
long charsRead = streamReader.Read(readBuffer, 0, _bufferSize);
totalCharsRead += charsRead;
while (charsRead > 0)
{
if (worker.CancellationPending)
{
wasCancelled = true;
//display cancellation message
CancelProgressDelegate canceller = new CancelProgressDelegate(CancelProgressText);
dispatcher.BeginInvoke(canceller, _rowsProcessed);
return(true);
}
// put the stuff we just read from the file into our working buffer
workingBuffer.Append(readBuffer);
// slice the workingBuffer into lines
string[] linesArray = workingBuffer.ToString().Split('\n');
// process all the lines EXCEPT THE LAST ONE in the lines array (the last one is likely to be incomplete)
for (int i = 0; i < (linesArray.Length - 1); i++)
{
string line = linesArray[i];
// the line should have the same number of columns as the ColumnCollection, if not then up the jagged lines count, and skip processing
string[] lineColumns = line.Split(',');
// if we get a jagged result here (i.e. length of columns does not match the headers) then try a split using quote delimited data
if (lineColumns.Length != ColumnCollection.Count)
{
// thanks to http://stackoverflow.com/questions/3776458/split-a-comma-separated-string-with-both-quoted-and-unquoted-strings
// for this bit of regex
Regex csvSplit = new Regex("(?:^|,)(\"(?:[^\"]+|\"\")*\"|[^,]*)", RegexOptions.Compiled);
int jag = 0;
lineColumns = new string[csvSplit.Matches(line).Count];
foreach (Match match in csvSplit.Matches(line))
{
lineColumns[jag] = match.Value.TrimStart(',');
jag++;
}
}
// if we're still jagged then we can't do much other than skip processing this line and increment the jagged counter
if (lineColumns.Length != ColumnCollection.Count)
{
_jaggedLines++;
}
else
{
// get the columns for crypting using the inputFields, since this is a sorted list we always get the indexes from aphabetically ordered keys
SortedList <string, string> hashNameValueCollection = new SortedList <string, string>();
// first column is the digest
foreach (string key in inputFields.Keys)
{
string theData = lineColumns[inputFields[key]];
// we always process the one they selected as NHSNumber ..
if (_performNHSNumberValidation)
{
string nhskey = inputHeadings[indexOfNHSNumber - 1];
if (nhskey == key)
{
theData = crypto.ProcessNHSNumber(theData);
}
}
hashNameValueCollection.Add(key, theData);
}
string digest = crypto.GetDigest(hashNameValueCollection);
string validNHS = "";
lineToWrite = digest + ",";
// output the rest of the columns in the output list
foreach (int key in outputFields.Keys) // keys in the output are indexes (opposite to input SortedList)
{
// Look for column heading that is a date..
if (_processDateColumns.Contains(outputFields[key]))
{
lineToWrite += crypto.RoundDownDate(lineColumns[key]) + ",";
}
else
{
lineToWrite += lineColumns[key] + ",";
}
}
// last column is the NHS Validation (if requested)
if (_performNHSNumberValidation)
{
// find the NHSNumber in the list of input columns and validate it
string key = inputHeadings[indexOfNHSNumber - 1];
{
string trimmedNHSNumber = lineColumns[indexOfNHSNumber - 1].Trim();
// trimmed data is length < 1 so we call this missing NHS Number
if (trimmedNHSNumber.Length < 1)
{
validNHS = "-1";
_missingNHSNumCount++;
}
else
{
// we have data for the NHS field, is it valid?
string cleanedNHSNumber = crypto.ProcessNHSNumber(trimmedNHSNumber);
if (NHSNumberValidator.IsValidNHSNumber(cleanedNHSNumber))
{
validNHS = "1";
_validNHSNumCount++;
}
else
{
validNHS = "0";
_inValidNHSNumCount++;
}
}
lineToWrite += validNHS + ",";
}
}
// we're done writing the output line now. Strip trailing comma.
lineToWrite = lineToWrite.Substring(0, lineToWrite.Length - 1);
// some files have a double line break at the end of the lines, remove this.
lineToWrite = lineToWrite.Replace(Environment.NewLine, "").Replace("\r\n", "").Replace("\n", "").Replace("\r", "");
streamWriter.WriteLine(lineToWrite);
}
}
_rowsProcessed += linesArray.Length - 1;
// set the working buffer to be the last line, so the next pass can concatonate
string lastLine = linesArray[linesArray.Length - 1];
workingBuffer = new StringBuilder(lastLine);
UpdateProgressDelegate update = new UpdateProgressDelegate(UpdateProgressText);
dispatcher.BeginInvoke(update, totalCharsRead, _inputFileStreamLength, _rowsProcessed, _validNHSNumCount, _inValidNHSNumCount, _missingNHSNumCount);
// empty the readbuffer, or the last read will only partially fill it, and we'll have some old data in the tail
readBuffer = new char[_bufferSize];
// read the next lot
charsRead = streamReader.Read(readBuffer, 0, _bufferSize);
totalCharsRead += charsRead;
}
}
}
return(false);
}
