/// <Summary>Base for Importing V1.</Summary>
/// <remarks>Need list of people to merge into.</remarks>
protected ImportV1Base(TallyJ2dContext db, ImportFile file, XmlDocument xmlDoc, List<Person> people, Action<Person> addPerson, ILogHelper logHelper)
{
_db = db;
_file = file;
_xmlDoc = xmlDoc;
_people = people;
AddPerson = addPerson;
_logHelper = logHelper;
}
public ImportV1Election(TallyJ2dContext db, ImportFile file, XmlDocument xml,
Election election,
Location location,
Action<Ballot> storeBallot,
Action<Vote> storeVote,
List<Person> people, Action<Person> addPerson,
Action<ResultSummary> storeResultSummary,
ILogHelper logHelper
)
: base(db, file, xml, people, addPerson, logHelper)
{
_election = election;
_location = location;
_storeBallot = storeBallot;
_storeVote = storeVote;
_storeResultSummary = storeResultSummary;
}
public void SkipDuplicates()
{
var fakeImportFile = new ImportFile();
var fakes = new ImportFakes();
var xmlDoc = new XmlDocument();
var fakeDataContext = new FakeDataContext();
xmlDoc.LoadXml("<Community><Person LName='Accorti' FName='Pónt' AKAName='Paul'></Person><Person LName='Accorti' FName='Pónt' AKAName='Paul'></Person></Community>");
var model = new ImportV1Community(fakeDataContext, fakeImportFile, xmlDoc, fakes.People, fakes.AddPersonToDb, fakes.LogHelper);
model.Process();
fakes.People.Count.ShouldEqual(1);
var people = fakes.People[0];
people.LastName.ShouldEqual("Accorti");
people.FirstName.ShouldEqual("Pónt");
people.OtherNames.ShouldEqual("Paul");
}
/// <Summary>Save file</Summary>
public override Stream GetWriteStream(UploadedFile file)
{
//
// Not working... file.ServerLocation is empty, however, the HttpRequest has the file contents
//
//
//
//
//
//
var longLength = file.ContentLength;
var contentLength = longLength > Int32.MaxValue ? 0 : (int)longLength;
var buffer = new byte[contentLength];
var start = 0;
using (var fileStream = File.OpenRead(file.ServerLocation))
{
int count;
while ((count = fileStream.Read(buffer, start, contentLength - start)) > 0)
{
start += count;
}
}
var record = new ImportFile
{
Contents = buffer,
FileSize = contentLength,
OriginalFileName = file.ClientName,
UploadTime = DateTime.Now,
ProcessingStatus = "Imported"
};
Db.ImportFiles.Add(record);
Db.SaveChanges();
return null;
}
public string ProcessUpload(out int rowId)
{
rowId = 0;
var httpRequest = HttpContext.Current.Request;
var inputStream = httpRequest.InputStream;
if (inputStream.Length == 0)
{
return "No file received";
}
var name = HttpUtility.UrlDecode(httpRequest.Headers["X-File-Name"].DefaultTo("unknown name"));
var fileSize = (int)inputStream.Length;
var record = new ImportFile
{
ElectionGuid = UserSession.CurrentElectionGuid,
Contents = new byte[fileSize],
FileSize = fileSize,
OriginalFileName = name,
UploadTime = DateTime.Now,
FileType = FileTypeCsv,
ProcessingStatus = "Uploaded"
};
var numWritten = inputStream.Read(record.Contents, 0, fileSize);
if (numWritten != fileSize)
{
return "Read {0}. Should be {1}.".FilledWith(numWritten, fileSize);
}
new ImportHelper().ExtraProcessingIfMultipartEncoded(record);
Db.ImportFiles.Add(record);
Db.SaveChanges();
rowId = record.C_RowId;
new LogHelper().Add("Uploaded file #" + record.C_RowId);
return "";
}
public ImportV1Community(TallyJ2dContext db, ImportFile file, XmlDocument xml, List<Person> people, Action<Person> addPerson, ILogHelper logHelper)
: base(db, file, xml, people, addPerson, logHelper)
{
}
public void Basic_Import_Test()
{
var fakeImportFile = new ImportFile();
var fakes = new ImportFakes();
var xmlDoc = new XmlDocument();
var fakeDataContext = new FakeDataContext();
xmlDoc.LoadXml(Resources.SampleElection);
var electionGuid = Guid.NewGuid();
var election = new Election {ElectionGuid = electionGuid};
var location = new Location {LocationGuid = Guid.NewGuid()};
var model = new ImportV1Election(fakeDataContext, fakeImportFile, xmlDoc, election, location,
fakes.AddBallotToDb, fakes.AddVoteToDb,
fakes.People, fakes.AddPersonToDb, fakes.AddResultSummaryToDb, fakes.LogHelper);
model.Process();
election.Name.ShouldEqual("(Imported) Sample LSA Election");
election.DateOfElection.ShouldEqual(new DateTime(2011, 4, 20));
election.ElectionType.ShouldEqual(ElectionTypeEnum.Lsa);
election.ElectionMode.ShouldEqual(ElectionModeEnum.Normal);
election.IsSingleNameElection.ShouldEqual(false);
election.NumberExtra.ShouldEqual(0);
election.NumberToElect.ShouldEqual(9);
election.ShowAsTest.ShouldEqual(true, "Imported elections are marked as Test");
election.TallyStatus.ShouldEqual(ElectionTallyStatusEnum.Reviewing, "Imported elections set to Review mode");
fakes.ResultSummaries.Count.ShouldEqual(1);
var resultSummary = fakes.ResultSummaries[0];
resultSummary.DroppedOffBallots.ShouldEqual(1);
resultSummary.MailedInBallots.ShouldEqual(10);
resultSummary.CalledInBallots.ShouldEqual(0);
resultSummary.InPersonBallots.ShouldEqual(17);
resultSummary.NumEligibleToVote.ShouldEqual(51);
fakes.Ballots.Count.ShouldEqual(28);
var ballot1 = fakes.Ballots[0];
ballot1.StatusCode.ShouldEqual(BallotStatusEnum.Ok);
var votes1 = fakes.Votes.Where(v => v.BallotGuid == ballot1.BallotGuid).ToList();
votes1.Count.ShouldEqual(9);
var vote1 = votes1[0];
vote1.StatusCode.ShouldEqual(VoteHelper.VoteStatusCode.Ok);
var matchingPerson = fakes.People.Where(p => p.PersonGuid == vote1.PersonGuid).ToList();
matchingPerson.Count.ShouldEqual(1);
vote1.PersonCombinedInfo.ShouldEqual(matchingPerson[0].CombinedInfo);
var ballot4 = fakes.Ballots[3];
ballot4.ComputerCode.ShouldEqual("A");
ballot4.BallotNumAtComputer.ShouldEqual(4);
var votes4 = fakes.Votes.Where(v => v.BallotGuid == ballot4.BallotGuid).ToList();
votes4.Count.ShouldEqual(9);
var vote4_9 = votes4[8];
vote4_9.StatusCode.ShouldEqual(VoteHelper.VoteStatusCode.Ok);
vote4_9.InvalidReasonGuid.ShouldEqual(IneligibleReasonEnum.Unreadable_Writing_illegible);
var ballot11 = fakes.Ballots[10];
ballot11.ComputerCode.ShouldEqual("A");
ballot11.BallotNumAtComputer.ShouldEqual(11);
ballot11.StatusCode.ShouldEqual(BallotStatusEnum.TooMany);
var votes11 = fakes.Votes.Where(v => v.BallotGuid == ballot11.BallotGuid).ToList();
votes11.Count.ShouldEqual(10);
}
private static XmlDocument GetXmlDoc(ImportFile file)
{
var xml = new XmlDocument();
xml.Load(new MemoryStream(file.Contents));
return xml;
}
public void ExtraProcessingIfMultipartEncoded(ImportFile record)
{
const string multipartDividerPrefix = "-----------------------------";
foreach (var codePage in Encodings.Select(encoding => encoding.Key))
{
var textReader = new StringReader(record.Contents.AsString(codePage));
var line = textReader.ReadLine();
if (line == null)
{
textReader.Dispose();
continue;
}
if (!line.StartsWith(multipartDividerPrefix))
{
textReader.Dispose();
continue;
}
// this file is encoded...
//line1 "-----------------------------7dc1372120770"
//line2 "Content-Disposition: form-data; name=\"qqfile\"; filename=\"C:\\Temp\\sampleCommunity.csv\""
//line3 "Content-Type: application/vnd.ms-excel"
//line4 ""
//line5 "Given,Surname,Other,ID,Group,Email,Phone"
line = textReader.ReadLine();
try
{
var split = line.Split(new[] { "filename=" }, StringSplitOptions.None);
record.OriginalFileName = Path.GetFileName(split[1].Replace("\"", ""));
}
catch
{
// swallow it and move on
}
textReader.ReadLine(); // 3
var lines = new List<string>();
line = textReader.ReadLine();
while (line != null)
{
if (!line.StartsWith(multipartDividerPrefix))
{
lines.Add(line);
}
line = textReader.ReadLine();
}
if (lines.Count == 0)
{
textReader.Dispose();
continue;
}
record.Contents = Encoding.GetEncoding(codePage).GetBytes(lines.JoinedAsString("\r\n", false));
record.CodePage = codePage;
return;
}
}
public ActionResult DeleteFile(int id)
{
var targetFile = new ImportFile { C_RowId = id, ElectionGuid = UserSession.CurrentElectionGuid };
Db.ImportFiles.Attach(targetFile);
Db.ImportFiles.Remove(targetFile);
Db.SaveChanges();
new LogHelper().Add("Deleted file #" + id);
return GetUploadList();
}
public string ProcessUpload(out int rowId)
{
rowId = 0;
var httpRequest = HttpContext.Current.Request;
var inputStream = httpRequest.InputStream;
if (inputStream.Length == 0)
{
return "No file received";
}
var name = HttpUtility.UrlDecode(httpRequest.Headers["X-File-Name"].DefaultTo("unknown name"));
var fileSize = (int)inputStream.Length;
var record = new ImportFile
{
ElectionGuid = UserSession.CurrentElectionGuid,
Contents = new byte[fileSize],
FileSize = fileSize,
OriginalFileName = name,
UploadTime = DateTime.Now,
ProcessingStatus = "Uploaded"
};
var numWritten = inputStream.Read(record.Contents, 0, fileSize);
if (numWritten != fileSize)
{
return "Read {0}. Should be {1}.".FilledWith(numWritten, fileSize);
}
new ImportHelper().ExtraProcessingIfMultipartEncoded(record);
try
{
var xml = GetXmlDoc(record);
if (xml == null || xml.DocumentElement == null)
{
return "Invalid Xml file";
}
if (xml.DocumentElement.Name == "Community")
{
record.FileType = FileTypeV1Community;
}
else if (xml.DocumentElement.Name == "Election")
{
record.FileType = FileTypeV1Election;
}
else
{
return "Unexpected Xml file";
}
}
catch (Exception e)
{
return e.Message;
}
Db.ImportFiles.Add(record);
Db.SaveChanges();
rowId = record.C_RowId;
new LogHelper().Add("Uploaded file #" + record.C_RowId);
return "";
}
private JsonResult ReadFields(ImportFile importFile)
{
var textReader = new StringReader(importFile.Contents.AsString(importFile.CodePage));
var csv = new CsvReader(textReader, true) { SkipEmptyLines = true };
var csvHeaders = csv.GetFieldHeaders();
//mapping: csv->db,csv->db
var currentMappings =
importFile.ColumnsToRead.DefaultTo("").SplitWithString(",").Select(s => s.SplitWithString("->"));
var dbFields = DbFieldsList;
const int numSampleLinesWanted = 5;
var numSampleLinesFound = numSampleLinesWanted;
var sampleValues = new Dictionary<string, List<string>>();
for (var i = 0; i < numSampleLinesFound; i++)
{
if (csv.EndOfStream)
{
numSampleLinesFound = i;
break;
}
foreach (var csvHeader in csvHeaders)
{
if (i == 0)
{
if (sampleValues.ContainsKey(csvHeader))
{
// ignore second column with same title
continue;
}
sampleValues.Add(csvHeader, new List<string> { csv[i, csvHeader] });
}
else
{
sampleValues[csvHeader].Add(csv[i, csvHeader]);
}
}
}
return new
{
possible = dbFields,
csvFields = csvHeaders.Select(header => new
{
field = header,
map = currentMappings.Where(cs => cs[0] == header)
.Select(cs => cs[1]).SingleOrDefault().DefaultTo(""),
sample = sampleValues[header]
})
}.AsJsonResult();
}
