/// <summary>
/// Inserts object into database if it doesn't already exist
/// </summary>
/// <returns>True if the object was inserted, otherwise false</returns>
public static async Task <bool> Upload(
IId obj,
IDataApiClient dataApiClient,
bool overwriteIfExists = false)
{
if (overwriteIfExists)
{
await dataApiClient.ReplaceAsync(obj, obj.Id);
return(true);
}
var exists = await dataApiClient.ExistsAsync(obj.GetType().Name, obj.Id);
if (!exists)
{
await dataApiClient.InsertAsync(obj, obj.Id);
}
return(!exists);
}
public static async Task <DataReference> TransferBinaryData(IDataApiClient dataApiClient, byte[] data, string id = null)
{
if (id == null)
{
id = IdGenerator.Sha1HashFromByteArray(data);
}
var dataType = nameof(DataBlob);
if (await dataApiClient.ExistsAsync <DataBlob>(id))
{
return(new DataReference(dataType, id));
}
var dataBlob = new DataBlob(id, new byte[0]);
id = await dataApiClient.CreateSubmission(dataBlob, x => x.Data, id);
await dataApiClient.TransferSubmissionData(dataType, id, data);
return(new DataReference(dataType, id));
}
public static async Task <DataReference> TransferFile(IDataApiClient dataApiClient, string filePath, string id = null)
{
if (id == null)
{
id = IdGenerator.Sha1HashFromFile(filePath);
}
var dataType = nameof(DataBlob);
if (await dataApiClient.ExistsAsync <DataBlob>(id))
{
return(new DataReference(dataType, id));
}
var dataBlob = new DataBlob(id, new byte[0], Path.GetFileName(filePath));
id = await dataApiClient.CreateSubmission(dataBlob, x => x.Data, id);
using (var fileStream = File.OpenRead(filePath))
await dataApiClient.TransferSubmissionData(dataType, id, fileStream);
return(new DataReference(dataType, id));
}
public static async Task AddToDataProject(
IDataApiClient dataApiClient,
string dataProjectId,
string dataType,
string id,
List <DataReference> derivedData = null,
string filename = null)
{
if (!await dataApiClient.ExistsAsync <DataProject>(dataProjectId))
{
throw new KeyNotFoundException($"No data project with ID '{dataProjectId}' exists");
}
var dataProjectUploadInfo = new DataProjectUploadInfo(
dataApiClient.LoggedInUsername,
DateTime.UtcNow,
dataProjectId,
new DataReference(dataType, id),
derivedData,
filename);
await dataApiClient.InsertAsync(dataProjectUploadInfo, dataProjectUploadInfo.Id);
}
private async Task Run(CancellationToken cancellationToken)
{
try
{
await dataProcessingServiceLogger.Log(new DataProcessingServiceLog($"{nameof(Distributor)} started", null));
await dataProcessingServiceLogger.Log(new DataProcessingServiceLog(
$"{nameof(Distributor)} running for types '{processorDatabase.InputTypes.Aggregate((a, b) => a + ", " + b)}'", null));
}
catch (Exception e)
{
Console.WriteLine(e);
}
try
{
while (!cancellationToken.IsCancellationRequested)
{
if (dataApiClient.IsAvailable())
{
if (!dataApiClient.IsLoggedIn && !dataApiClient.RetryLogin().IsAuthenticated)
{
cancellationToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(60));
continue;
}
try
{
var subscriptionNotifications = await dataApiClient.GetSubscribedObjects();
foreach (var subscriptionNotification in subscriptionNotifications)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
var dataType = subscriptionNotification.DataType;
if (subscriptionNotification.ModificationType != DataModificationType.Deleted)
{
var exists = await dataApiClient.ExistsAsync(
dataType,
subscriptionNotification.DataObjectId);
if (!exists)
{
await MarkAsProcessed(subscriptionNotification.Id);
continue;
}
}
try
{
var typeProcessors = processorDatabase.GetForType(dataType);
switch (subscriptionNotification.ModificationType)
{
case DataModificationType.Created:
case DataModificationType.Replaced:
var typeObject = await LoadObject(subscriptionNotification);
await ApplyProcessorsToObject(
subscriptionNotification.ModificationType,
dataType,
subscriptionNotification.DataObjectId,
typeObject,
typeProcessors);
break;
case DataModificationType.Deleted:
await ApplyProcessorsToObject(
subscriptionNotification.ModificationType,
dataType,
subscriptionNotification.DataObjectId,
null,
typeProcessors);
break;
default:
throw new ArgumentOutOfRangeException();
}
await MarkAsProcessed(subscriptionNotification.Id);
}
catch (Exception e)
{
var logEntry = new DataProcessingServiceLog(
$"Processing of '{dataType}' with ID '{subscriptionNotification.DataObjectId}' failed: {e.Message}",
new ExecutionSummaryLogEntryDetails(
DataProcessingServiceExecutorType.Processor,
"Unknown",
0.To(Unit.Second),
isError: true,
isWorkDone: false,
inputDataObjectId: subscriptionNotification.DataObjectId,
inputTypeName: dataType));
await dataProcessingServiceLogger.Log(logEntry);
}
}
}
catch (Exception e)
{
var logEntry = new DataProcessingServiceLog(
$"Processing of subscription notifications failed: {e.InnermostException().Message}",
new CrashLogEntryDetails(nameof(Distributor), e.InnermostException().Message));
await dataProcessingServiceLogger.Log(logEntry);
}
}
pollNowEventHandle.Reset();
WaitHandle.WaitAny(new[] { cancellationToken.WaitHandle, pollNowEventHandle }, PollInterval);
}
}
catch (Exception e)
{
await dataProcessingServiceLogger.Log(new DataProcessingServiceLog(
$"Distributor crashed: {e.InnermostException().Message}",
new CrashLogEntryDetails(nameof(Distributor), e.InnermostException().Message)));
}
try
{
await dataProcessingServiceLogger.Log(new DataProcessingServiceLog($"{nameof(Distributor)} stopped", null));
}
catch
{
// Ignore logging errors
}
}