/// <summary> /// Inserts object into database if it doesn't already exist /// </summary> /// <returns>True if the object was inserted, otherwise false</returns> public static async Task <bool> Upload( IId obj, IDataApiClient dataApiClient, bool overwriteIfExists = false) { if (overwriteIfExists) { await dataApiClient.ReplaceAsync(obj, obj.Id); return(true); } var exists = await dataApiClient.ExistsAsync(obj.GetType().Name, obj.Id); if (!exists) { await dataApiClient.InsertAsync(obj, obj.Id); } return(!exists); }
public static async Task <DataReference> TransferBinaryData(IDataApiClient dataApiClient, byte[] data, string id = null) { if (id == null) { id = IdGenerator.Sha1HashFromByteArray(data); } var dataType = nameof(DataBlob); if (await dataApiClient.ExistsAsync <DataBlob>(id)) { return(new DataReference(dataType, id)); } var dataBlob = new DataBlob(id, new byte[0]); id = await dataApiClient.CreateSubmission(dataBlob, x => x.Data, id); await dataApiClient.TransferSubmissionData(dataType, id, data); return(new DataReference(dataType, id)); }
public static async Task <DataReference> TransferFile(IDataApiClient dataApiClient, string filePath, string id = null) { if (id == null) { id = IdGenerator.Sha1HashFromFile(filePath); } var dataType = nameof(DataBlob); if (await dataApiClient.ExistsAsync <DataBlob>(id)) { return(new DataReference(dataType, id)); } var dataBlob = new DataBlob(id, new byte[0], Path.GetFileName(filePath)); id = await dataApiClient.CreateSubmission(dataBlob, x => x.Data, id); using (var fileStream = File.OpenRead(filePath)) await dataApiClient.TransferSubmissionData(dataType, id, fileStream); return(new DataReference(dataType, id)); }
public static async Task AddToDataProject( IDataApiClient dataApiClient, string dataProjectId, string dataType, string id, List <DataReference> derivedData = null, string filename = null) { if (!await dataApiClient.ExistsAsync <DataProject>(dataProjectId)) { throw new KeyNotFoundException($"No data project with ID '{dataProjectId}' exists"); } var dataProjectUploadInfo = new DataProjectUploadInfo( dataApiClient.LoggedInUsername, DateTime.UtcNow, dataProjectId, new DataReference(dataType, id), derivedData, filename); await dataApiClient.InsertAsync(dataProjectUploadInfo, dataProjectUploadInfo.Id); }
private async Task Run(CancellationToken cancellationToken) { try { await dataProcessingServiceLogger.Log(new DataProcessingServiceLog($"{nameof(Distributor)} started", null)); await dataProcessingServiceLogger.Log(new DataProcessingServiceLog( $"{nameof(Distributor)} running for types '{processorDatabase.InputTypes.Aggregate((a, b) => a + ", " + b)}'", null)); } catch (Exception e) { Console.WriteLine(e); } try { while (!cancellationToken.IsCancellationRequested) { if (dataApiClient.IsAvailable()) { if (!dataApiClient.IsLoggedIn && !dataApiClient.RetryLogin().IsAuthenticated) { cancellationToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(60)); continue; } try { var subscriptionNotifications = await dataApiClient.GetSubscribedObjects(); foreach (var subscriptionNotification in subscriptionNotifications) { if (cancellationToken.IsCancellationRequested) { break; } var dataType = subscriptionNotification.DataType; if (subscriptionNotification.ModificationType != DataModificationType.Deleted) { var exists = await dataApiClient.ExistsAsync( dataType, subscriptionNotification.DataObjectId); if (!exists) { await MarkAsProcessed(subscriptionNotification.Id); continue; } } try { var typeProcessors = processorDatabase.GetForType(dataType); switch (subscriptionNotification.ModificationType) { case DataModificationType.Created: case DataModificationType.Replaced: var typeObject = await LoadObject(subscriptionNotification); await ApplyProcessorsToObject( subscriptionNotification.ModificationType, dataType, subscriptionNotification.DataObjectId, typeObject, typeProcessors); break; case DataModificationType.Deleted: await ApplyProcessorsToObject( subscriptionNotification.ModificationType, dataType, subscriptionNotification.DataObjectId, null, typeProcessors); break; default: throw new ArgumentOutOfRangeException(); } await MarkAsProcessed(subscriptionNotification.Id); } catch (Exception e) { var logEntry = new DataProcessingServiceLog( $"Processing of '{dataType}' with ID '{subscriptionNotification.DataObjectId}' failed: {e.Message}", new ExecutionSummaryLogEntryDetails( DataProcessingServiceExecutorType.Processor, "Unknown", 0.To(Unit.Second), isError: true, isWorkDone: false, inputDataObjectId: subscriptionNotification.DataObjectId, inputTypeName: dataType)); await dataProcessingServiceLogger.Log(logEntry); } } } catch (Exception e) { var logEntry = new DataProcessingServiceLog( $"Processing of subscription notifications failed: {e.InnermostException().Message}", new CrashLogEntryDetails(nameof(Distributor), e.InnermostException().Message)); await dataProcessingServiceLogger.Log(logEntry); } } pollNowEventHandle.Reset(); WaitHandle.WaitAny(new[] { cancellationToken.WaitHandle, pollNowEventHandle }, PollInterval); } } catch (Exception e) { await dataProcessingServiceLogger.Log(new DataProcessingServiceLog( $"Distributor crashed: {e.InnermostException().Message}", new CrashLogEntryDetails(nameof(Distributor), e.InnermostException().Message))); } try { await dataProcessingServiceLogger.Log(new DataProcessingServiceLog($"{nameof(Distributor)} stopped", null)); } catch { // Ignore logging errors } }