public override async Task WriteDatumAsync(Datum datum, CancellationToken cancellationToken)
{
AmazonS3Client s3 = null;
try
{
s3 = await CreateS3ClientAsync();
string datumJSON = datum.GetJSON(Protocol.JsonAnonymizer, true);
byte[] datumJsonBytes = Encoding.UTF8.GetBytes(datumJSON);
MemoryStream dataStream = new MemoryStream(datumJsonBytes);
await PutAsync(s3, dataStream, GetDatumKey(datum), "application/json", cancellationToken);
}
finally
{
DisposeS3(s3);
}
}
public override Task WriteDatumAsync(Datum datum, CancellationToken cancellationToken)
{
return(Task.Run(async() =>
{
AmazonS3Client s3 = null;
try
{
s3 = InitializeS3();
string datumJSON = datum.GetJSON(Protocol.JsonAnonymizer, true);
byte[] datumJsonBytes = Encoding.UTF8.GetBytes(datumJSON);
MemoryStream dataStream = new MemoryStream();
dataStream.Write(datumJsonBytes, 0, datumJsonBytes.Length);
dataStream.Position = 0;
await Put(s3, dataStream, GetDatumKey(datum), "application/json", cancellationToken);
}
finally
{
DisposeS3(s3);
}
}));
}
public override void WriteDatum(Datum datum, CancellationToken cancellationToken)
{
if (!Running)
{
return;
}
lock (_dataBuffer)
{
_dataBuffer.Add(datum);
_totalDataBuffered++;
_dataHaveBeenBuffered.Set();
// start the long-running task for writing data to file. also check the status of the task after
// it has been created and restart the task if it stops due to cancellation, fault, or completion.
if (_writeBufferedDataToFileTask == null ||
_writeBufferedDataToFileTask.Status == TaskStatus.Canceled ||
_writeBufferedDataToFileTask.Status == TaskStatus.Faulted ||
_writeBufferedDataToFileTask.Status == TaskStatus.RanToCompletion)
{
_writeBufferedDataToFileTask = Task.Run(async() =>
{
try
{
while (Running)
{
// wait for the signal to from data from the buffer to the file
_dataHaveBeenBuffered.WaitOne();
// write data. be sure to acquire locks in the same order as in Flush.
bool checkSize = false;
bool startNewFile = false;
lock (_toWriteBuffer)
{
// copy the current data to the buffer to write, and clear the current buffer. we use
// the intermediary buffer to free up the data buffer lock as quickly as possible for
// callers to WriteDatum. all probes call WriteDatum, so we need to accommodate
// potentially hundreds of samples per second.
lock (_dataBuffer)
{
_toWriteBuffer.AddRange(_dataBuffer);
_dataBuffer.Clear();
}
// write each datum from the intermediary buffer to disk.
for (int i = 0; i < _toWriteBuffer.Count;)
{
Datum datumToWrite = _toWriteBuffer[i];
bool datumWritten = false;
lock (_fileLocker)
{
// it's possible to stop the datastore and dispose the file before entering this lock, in
// which case we won't have a file to write to. check the file.
if (_currentFile != null)
{
#region write JSON for datum to file
string datumJSON = null;
try
{
datumJSON = datumToWrite.GetJSON(Protocol.JsonAnonymizer, false);
}
catch (Exception ex)
{
SensusException.Report("Failed to get JSON for datum.", ex);
}
if (datumJSON != null)
{
try
{
byte[] datumJsonBytes = Encoding.UTF8.GetBytes((_totalDataWrittenToCurrentFile == 0 ? "" : ",") + Environment.NewLine + datumJSON);
_currentFile.Write(datumJsonBytes, 0, datumJsonBytes.Length);
_totalDataWrittenToCurrentFile++;
_totalDataWritten++;
datumWritten = true;
// periodically check the size of the current file
if (_totalDataWrittenToCurrentFile % DATA_WRITES_PER_SIZE_CHECK == 0)
{
checkSize = true;
}
}
catch (Exception writeException)
{
SensusException.Report("Exception while writing datum JSON bytes to file: " + writeException.Message, writeException);
startNewFile = true;
break;
}
}
#endregion
}
}
if (datumWritten)
{
_toWriteBuffer.RemoveAt(i);
}
else
{
i++;
}
}
}
if (checkSize)
{
// must do the check within the lock, since other callers might be trying to close the file and null the path.
lock (_fileLocker)
{
if (SensusServiceHelper.GetFileSizeMB(_currentPath) >= MAX_FILE_SIZE_MB)
{
startNewFile = true;
}
}
}
if (startNewFile)
{
await StartNewFileAsync(cancellationToken);
}
if (checkSize)
{
await WriteToRemoteIfTooLargeAsync(cancellationToken);
}
_bufferedDataHaveBeenWrittenToFile.Set();
}
}
catch (Exception writeTaskException)
{
SensusException.Report("Exception while writing buffered data to file: " + writeTaskException.Message, writeTaskException);
}
});
}
}
}
public override void ClearDataCommittedToRemoteDataStore(List <Datum> dataCommittedToRemote)
{
lock (_locker)
{
CloseFile();
SensusServiceHelper.Get().Logger.Log("Received " + dataCommittedToRemote.Count + " remote-committed data elements to clear.", LoggingLevel.Debug, GetType());
HashSet <Datum> hashDataCommittedToRemote = new HashSet <Datum>(dataCommittedToRemote); // for quick access via hashing
// clear remote-committed data from all local files
foreach (string path in Directory.GetFiles(StorageDirectory))
{
SensusServiceHelper.Get().Logger.Log("Clearing remote-committed data from \"" + path + "\".", LoggingLevel.Debug, GetType());
string uncommittedDataPath = Path.GetTempFileName();
int uncommittedDataCount = 0;
using (StreamWriter uncommittedDataFile = new StreamWriter(uncommittedDataPath))
using (StreamReader file = new StreamReader(path))
{
string line;
while ((line = file.ReadLine()) != null)
{
Datum datum = Datum.FromJSON(line);
if (hashDataCommittedToRemote.Contains(datum))
{
--_numDataStoredInFiles;
}
else
{
uncommittedDataFile.WriteLine(datum.GetJSON(Protocol.JsonAnonymizer)); // need to pass in the anonymizer, since the user might have selected an anonymization option between the time that the datum was written to file and the time of execution of the current line of code.
++uncommittedDataCount;
}
}
uncommittedDataFile.Close();
file.Close();
}
if (uncommittedDataCount == 0) // all data in local file were committed to remote data store -- delete local and filtered files
{
SensusServiceHelper.Get().Logger.Log("Cleared all data from local file. Deleting file.", LoggingLevel.Debug, GetType());
File.Delete(path);
File.Delete(uncommittedDataPath);
}
else // data from local file were not committed to the remote data store -- move filtered path to local path and retry sending to remote store next time
{
SensusServiceHelper.Get().Logger.Log(uncommittedDataCount + " data elements in local file were not committed to remote data store.", LoggingLevel.Debug, GetType());
File.Delete(path);
File.Move(uncommittedDataPath, path);
}
}
// reinitialize file if we're running
if (Running)
{
InitializeFile();
}
SensusServiceHelper.Get().Logger.Log("Finished clearing remote-committed data elements.", LoggingLevel.Verbose, GetType());
}
}
public override void WriteDatum(Datum datum, CancellationToken cancellationToken)
{
if (!Running)
{
return;
}
lock (_storeBuffer)
{
_storeBuffer.Add(datum);
_totalDataBuffered++;
_checkForBufferedData.Set();
// start the long-running task for writing data to file. also check the status of the task after
// it has been created and restart the task if it stops for some reason.
if (_writeToFileTask == null ||
_writeToFileTask.Status == TaskStatus.Canceled ||
_writeToFileTask.Status == TaskStatus.Faulted ||
_writeToFileTask.Status == TaskStatus.RanToCompletion)
{
_writeToFileTask = Task.Run(async() =>
{
try
{
while (Running)
{
// wait for the signal to check for and write data
_checkForBufferedData.WaitOne();
bool checkSize = false;
// be sure to acquire the locks in the same order as done in Flush.
lock (_toWriteBuffer)
{
// copy the current data to the buffer to write, and clear the current buffer.
lock (_storeBuffer)
{
_toWriteBuffer.AddRange(_storeBuffer);
_storeBuffer.Clear();
}
// write each datum
for (int i = 0; i < _toWriteBuffer.Count;)
{
Datum datumToWrite = _toWriteBuffer[i];
bool datumWritten = false;
// lock the file so that we can safely write the current datum to it
lock (_locker)
{
// it's possible to stop the datastore before entering this lock, in which case we won't
// have a file to write to. check for a running data store here.
if (_file != null)
{
#region write JSON for datum to file
string datumJSON = null;
try
{
datumJSON = datumToWrite.GetJSON(Protocol.JsonAnonymizer, false);
}
catch (Exception ex)
{
SensusException.Report("Failed to get JSON for datum.", ex);
}
if (datumJSON != null)
{
try
{
byte[] datumJsonBytes = Encoding.UTF8.GetBytes((_dataWrittenToCurrentFile == 0 ? "" : ",") + Environment.NewLine + datumJSON);
_file.Write(datumJsonBytes, 0, datumJsonBytes.Length);
_dataWrittenToCurrentFile++;
_totalDataWritten++;
datumWritten = true;
if (_dataWrittenToCurrentFile % DATA_WRITES_PER_SIZE_CHECK == 0)
{
checkSize = true;
}
}
catch (Exception writeException)
{
SensusException.Report("Failed to write datum JSON bytes to file.", writeException);
#region something went wrong with file write...switch to a new file in the hope that it will work better.
try
{
CloseFile();
}
catch (Exception closeException)
{
SensusException.Report("Failed to close file after failing to write it.", closeException);
}
try
{
OpenFile();
}
catch (Exception openException)
{
SensusException.Report("Failed to open new file after failing to write the previous one.", openException);
}
#endregion
}
}
#endregion
}
}
if (datumWritten)
{
_toWriteBuffer.RemoveAt(i);
}
else
{
i++;
}
}
}
#region periodically check the size of the current file and the entire local data store
if (checkSize)
{
// must do the check within the lock, since other callers might be trying to close the file and null the path.
lock (_locker)
{
if (SensusServiceHelper.GetFileSizeMB(_path) >= MAX_FILE_SIZE_MB)
{
CloseFile();
OpenFile();
}
}
// write the local data to remote if the overall size has grown too large
await WriteToRemoteIfTooLargeAsync(cancellationToken);
}
#endregion
_finishedCheckingForBufferedData.Set();
}
}
catch (Exception taskException)
{
SensusException.Report("Local data store write task threw exception.", taskException);
}
});
}
}
}
public override Task <bool> WriteDatumAsync(Datum datum, CancellationToken cancellationToken)
{
return(Task.Run(async() =>
{
bool written = false;
// only 1 write operation at once
lock (_locker)
{
// it's possible to stop the datastore before entering this lock, in which case we won't
// have a file to write to. check for a running data store here.
if (!Running)
{
return written;
}
// get anonymized JSON for datum
string datumJSON = null;
try
{
datumJSON = datum.GetJSON(Protocol.JsonAnonymizer, false);
}
catch (Exception ex)
{
SensusException.Report("Failed to get JSON for datum.", ex);
}
// write JSON to file
if (datumJSON != null)
{
try
{
byte[] datumJsonBytes = Encoding.UTF8.GetBytes((_dataWrittenToCurrentFile == 0 ? "" : ",") + Environment.NewLine + datumJSON);
_file.Write(datumJsonBytes, 0, datumJsonBytes.Length);
_bytesWrittenToCurrentFile += datumJsonBytes.Length;
_dataWrittenToCurrentFile++;
_totalDataWritten++;
written = true;
}
catch (Exception writeException)
{
SensusException.Report("Failed to write datum JSON bytes to file.", writeException);
// something went wrong with file write...switch to a new file in the hope that it will work better.
try
{
CloseFile();
}
catch (Exception closeException)
{
SensusException.Report("Failed to close file after failing to write it.", closeException);
}
try
{
OpenFile();
}
catch (Exception openException)
{
SensusException.Report("Failed to open new file after failing to write the previous one.", openException);
}
}
}
}
// every so often, check the sizes of the file and data store
if ((_dataWrittenToCurrentFile % DATA_WRITES_PER_SIZE_CHECK) == 0)
{
// switch to a new file if the current one has grown too large
if (SensusServiceHelper.GetFileSizeMB(_path) >= MAX_FILE_SIZE_MB)
{
CloseFile();
OpenFile();
}
// write the local data to remote if the overall size has grown too large
await WriteToRemoteIfTooLargeAsync(cancellationToken);
}
return written;
}));
}