public void SerializeDeserializeWithEnumTest()
{
TestSensusServiceHelper service1 = new TestSensusServiceHelper();
SensusServiceHelper.Initialize(() => service1);
ActivityDatum datum = new ActivityDatum(DateTimeOffset.UtcNow,
Activities.OnBicycle,
ActivityPhase.Starting,
ActivityState.Inactive,
1);
Protocol protocol = new Protocol("test");
AnonymizedJsonContractResolver anonymizer = new AnonymizedJsonContractResolver(protocol);
string serializedJSON = datum.GetJSON(anonymizer, false);
ActivityDatum deserializedDatum = Datum.FromJSON(serializedJSON) as ActivityDatum;
Assert.AreEqual(datum.BuildId, deserializedDatum.BuildId);
Assert.AreEqual(datum.DeviceId, deserializedDatum.DeviceId);
Assert.AreEqual(datum.Id, deserializedDatum.Id);
Assert.AreEqual(datum.ProtocolId, deserializedDatum.ProtocolId);
Assert.AreEqual(datum.Timestamp, deserializedDatum.Timestamp);
Assert.AreEqual(datum.Activity, deserializedDatum.Activity);
Assert.AreEqual(datum.Confidence, deserializedDatum.Confidence);
Assert.AreEqual(datum.ActivityStarting, deserializedDatum.ActivityStarting);
Assert.AreEqual(datum.Phase, deserializedDatum.Phase);
Assert.AreEqual(datum.State, deserializedDatum.State);
}
public override async Task <T> GetDatum <T>(string datumKey, CancellationToken cancellationToken)
{
AmazonS3Client s3 = null;
try
{
s3 = InitializeS3();
Stream responseStream = (await s3.GetObjectAsync(_bucket, datumKey, cancellationToken)).ResponseStream;
T datum = null;
using (StreamReader reader = new StreamReader(responseStream))
{
string json = reader.ReadToEnd().Trim().Trim('[', ']'); // there will only be one datum in the array, so trim the braces and deserialize the datum.
json = SensusServiceHelper.Get().ConvertJsonForCrossPlatform(json);
datum = Datum.FromJSON(json) as T;
}
return(datum);
}
catch (Exception ex)
{
string message = "Failed to get datum from Amazon S3: " + ex.Message;
SensusServiceHelper.Get().Logger.Log(message, LoggingLevel.Normal, GetType());
throw new Exception(message);
}
finally
{
DisposeS3(s3);
}
}
public override async Task<T> GetDatumAsync<T>(string datumKey, CancellationToken cancellationToken)
{
AmazonS3Client s3 = null;
try
{
s3 = await CreateS3ClientAsync();
Stream responseStream = (await s3.GetObjectAsync(_bucket, datumKey, cancellationToken)).ResponseStream;
T datum = null;
using (StreamReader reader = new StreamReader(responseStream))
{
string datumJSON = reader.ReadToEnd().Trim();
datumJSON = SensusServiceHelper.Get().ConvertJsonForCrossPlatform(datumJSON);
datum = Datum.FromJSON(datumJSON) as T;
}
return datum;
}
catch (Exception ex)
{
string message = "Failed to get datum from Amazon S3: " + ex.Message;
SensusServiceHelper.Get().Logger.Log(message, LoggingLevel.Normal, GetType());
throw new Exception(message);
}
finally
{
DisposeS3(s3);
}
}
public override List <Datum> GetDataForRemoteDataStore(CancellationToken cancellationToken, Action <double> progressCallback)
{
lock (_locker)
{
CloseFile();
// get local data from all files
List <Datum> localData = new List <Datum>();
foreach (string path in Directory.GetFiles(StorageDirectory))
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
try
{
using (StreamReader file = new StreamReader(path))
{
string line;
while (!cancellationToken.IsCancellationRequested && (line = file.ReadLine()) != null)
{
if (!string.IsNullOrWhiteSpace(line))
{
localData.Add(Datum.FromJSON(line));
if (progressCallback != null && _numDataStoredInFiles >= 10 && (localData.Count % (_numDataStoredInFiles / 10)) == 0)
{
progressCallback(localData.Count / (double)_numDataStoredInFiles);
}
}
}
file.Close();
}
}
catch (Exception ex)
{
SensusServiceHelper.Get().Logger.Log("Exception while reading local data store for transfer to remote: " + ex.Message, LoggingLevel.Normal, GetType());
}
}
if (cancellationToken.IsCancellationRequested)
{
SensusServiceHelper.Get().Logger.Log("Canceled retrieval of local data for remote data store.", LoggingLevel.Normal, GetType());
}
// reinitialize file if we're running
if (Running)
{
InitializeFile();
}
return(localData);
}
}
protected override List <Datum> CommitData(List <Datum> data, CancellationToken cancellationToken)
{
List <Datum> committed = new List <Datum>();
lock (_data)
foreach (Datum datum in data)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
// all locally stored data, whether on disk or in RAM, should be anonymized as required
// by the protocol. convert datum to/from JSON in order to apply anonymization.
try
{
string json = datum.GetJSON(Protocol.JsonAnonymizer);
try
{
Datum anonymizedDatum = Datum.FromJSON(json);
try
{
_data.Add(anonymizedDatum);
committed.Add(datum);
}
catch (Exception ex)
{
SensusServiceHelper.Get().Logger.Log("Failed to add datum to collection: " + ex.Message, LoggingLevel.Normal, GetType());
}
}
catch (Exception ex)
{
SensusServiceHelper.Get().Logger.Log("Failed to get datum from JSON: " + ex.Message, LoggingLevel.Normal, GetType());
}
}
catch (Exception ex)
{
SensusServiceHelper.Get().Logger.Log("Failed to get JSON from datum: " + ex.Message, LoggingLevel.Normal, GetType());
}
}
return(committed);
}
public void SerializeDeserializeTest()
{
TestSensusServiceHelper service1 = new TestSensusServiceHelper();
SensusServiceHelper.Initialize(() => service1);
LocationDatum datum = new LocationDatum(DateTimeOffset.UtcNow, 0.5, 75.5, -35.5);
Protocol protocol = new Protocol("test");
AnonymizedJsonContractResolver anonymizer = new AnonymizedJsonContractResolver(protocol);
string serializedJSON = datum.GetJSON(anonymizer, false);
LocationDatum deserializedDatum = Datum.FromJSON(serializedJSON) as LocationDatum;
Assert.AreEqual(datum.Accuracy, deserializedDatum.Accuracy);
Assert.AreEqual(datum.BuildId, deserializedDatum.BuildId);
Assert.AreEqual(datum.DeviceId, deserializedDatum.DeviceId);
Assert.AreEqual(datum.Id, deserializedDatum.Id);
Assert.AreEqual(datum.Latitude, deserializedDatum.Latitude);
Assert.AreEqual(datum.Longitude, deserializedDatum.Longitude);
Assert.AreEqual(datum.ProtocolId, deserializedDatum.ProtocolId);
Assert.AreEqual(datum.Timestamp, deserializedDatum.Timestamp);
}
public override Task <List <Datum> > CommitAsync(IEnumerable <Datum> data, CancellationToken cancellationToken)
{
return(Task.Run(() =>
{
List <Datum> committedData = new List <Datum>();
lock (_data)
{
foreach (Datum datum in data)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
// all locally stored data, whether on disk or in RAM, should be anonymized as required
// by the protocol. convert datum to/from JSON in order to apply anonymization.
try
{
string anonymizedDatumJSON = datum.GetJSON(Protocol.JsonAnonymizer, false);
Datum anonymizedDatum = Datum.FromJSON(anonymizedDatumJSON);
_data.Add(anonymizedDatum);
MostRecentSuccessfulCommitTime = DateTime.Now;
committedData.Add(anonymizedDatum);
}
catch (Exception ex)
{
SensusServiceHelper.Get().Logger.Log("Failed to add anonymized datum: " + ex.Message, LoggingLevel.Normal, GetType());
}
}
}
CheckSizeAndCommitToRemote(cancellationToken);
return committedData;
}));
}
public override void CommitToRemote(CancellationToken cancellationToken)
{
lock (_storageDirectoryLocker)
{
string[] pathsToCommit = Directory.GetFiles(StorageDirectory);
// get path for uncommitted data
WriteToNewPath();
string uncommittedDataPath = _path;
// reset _path for standard commits
WriteToNewPath();
using (StreamWriter uncommittedDataFile = new StreamWriter(uncommittedDataPath))
{
foreach (string pathToCommit in pathsToCommit)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
// wrap in try-catch to ensure that we process all files
try
{
using (StreamReader fileToCommit = new StreamReader(pathToCommit))
{
// commit data in small batches. all data will end up in the uncommitted file, in the batch object, or
// in the remote data store.
HashSet <Datum> batch = new HashSet <Datum>();
string datumJSON;
while ((datumJSON = fileToCommit.ReadLine()) != null)
{
// if we have been canceled, dump the rest of the file into the uncommitted data file.
if (cancellationToken.IsCancellationRequested)
{
uncommittedDataFile.WriteLine(datumJSON);
}
else
{
// wrap in try-catch to ensure that we process all lines
try
{
batch.Add(Datum.FromJSON(datumJSON));
}
catch (Exception ex)
{
SensusServiceHelper.Get().Logger.Log("Failed to add datum to batch from JSON: " + ex.Message, LoggingLevel.Normal, GetType());
}
if (batch.Count >= 50000)
{
CommitAndReleaseBatchToRemote(batch, cancellationToken, uncommittedDataFile);
}
}
}
// commit partial batch
if (batch.Count > 0)
{
CommitAndReleaseBatchToRemote(batch, cancellationToken, uncommittedDataFile);
}
}
// we've read all lines in the file and either committed them to the remote data store or written them
// to the uncommitted data file. we can delete the current path.
File.Delete(pathToCommit);
}
catch (Exception ex)
{
SensusServiceHelper.Get().Logger.Log("Failed to commit: " + ex.Message, LoggingLevel.Normal, GetType());
}
}
}
}
}
public override void ClearDataCommittedToRemoteDataStore(List <Datum> dataCommittedToRemote)
{
lock (_locker)
{
CloseFile();
SensusServiceHelper.Get().Logger.Log("Received " + dataCommittedToRemote.Count + " remote-committed data elements to clear.", LoggingLevel.Debug, GetType());
HashSet <Datum> hashDataCommittedToRemote = new HashSet <Datum>(dataCommittedToRemote); // for quick access via hashing
// clear remote-committed data from all local files
foreach (string path in Directory.GetFiles(StorageDirectory))
{
SensusServiceHelper.Get().Logger.Log("Clearing remote-committed data from \"" + path + "\".", LoggingLevel.Debug, GetType());
string uncommittedDataPath = Path.GetTempFileName();
int uncommittedDataCount = 0;
using (StreamWriter uncommittedDataFile = new StreamWriter(uncommittedDataPath))
using (StreamReader file = new StreamReader(path))
{
string line;
while ((line = file.ReadLine()) != null)
{
Datum datum = Datum.FromJSON(line);
if (hashDataCommittedToRemote.Contains(datum))
{
--_numDataStoredInFiles;
}
else
{
uncommittedDataFile.WriteLine(datum.GetJSON(Protocol.JsonAnonymizer)); // need to pass in the anonymizer, since the user might have selected an anonymization option between the time that the datum was written to file and the time of execution of the current line of code.
++uncommittedDataCount;
}
}
uncommittedDataFile.Close();
file.Close();
}
if (uncommittedDataCount == 0) // all data in local file were committed to remote data store -- delete local and filtered files
{
SensusServiceHelper.Get().Logger.Log("Cleared all data from local file. Deleting file.", LoggingLevel.Debug, GetType());
File.Delete(path);
File.Delete(uncommittedDataPath);
}
else // data from local file were not committed to the remote data store -- move filtered path to local path and retry sending to remote store next time
{
SensusServiceHelper.Get().Logger.Log(uncommittedDataCount + " data elements in local file were not committed to remote data store.", LoggingLevel.Debug, GetType());
File.Delete(path);
File.Move(uncommittedDataPath, path);
}
}
// reinitialize file if we're running
if (Running)
{
InitializeFile();
}
SensusServiceHelper.Get().Logger.Log("Finished clearing remote-committed data elements.", LoggingLevel.Verbose, GetType());
}
}