/// <summary> /// Constructs an uploader object. /// </summary> /// <param name="batchConfig">Optional; The batching configuration that controls the buffer size.</param> /// <param name="loggingServiceBaseAddress">Optional; The address of a custom HTTP logging service. When null, the join service address is used.</param> /// <param name="httpClient">Optional; The custom <see cref="IHttpClient"/> object to handle HTTP requests.</param> /// <param name="developmentMode">If true, enables additional logging and disables batching.</param> public EventUploader(BatchingConfiguration batchConfig = null, string loggingServiceBaseAddress = null, IHttpClient httpClient = null, bool developmentMode = false) : base(batchConfig, developmentMode) { this.loggingServiceBaseAddress = loggingServiceBaseAddress ?? ServiceConstants.JoinAddress; this.httpClient = httpClient ?? new UploaderHttpClient(); // override ReferenceResolver is supplied if (this.batchConfig.ReferenceResolver == null) { this.eventSerializer = ev => JsonConvert.SerializeObject(new ExperimentalUnitFragment { Key = ev.Key, Value = ev }); } else { this.eventSerializer = ev => JsonConvert.SerializeObject( new ExperimentalUnitFragment { Key = ev.Key, Value = ev }, Formatting.None, new JsonSerializerSettings { ReferenceResolverProvider = () => this.batchConfig.ReferenceResolver }); } }
/// <summary> /// Constructs an uploader object. /// </summary> /// <param name="batchConfig">Optional; The batching configuration that controls the buffer size.</param> public BaseEventUploader(BatchingConfiguration batchConfig = null, bool developmentMode = false) { this.cancellationTokenSource = new CancellationTokenSource(); this.cancellationToken = this.cancellationTokenSource.Token; this.batchConfig = batchConfig ?? new BatchingConfiguration(developmentMode); this.eventSource = new TransformBlock <IEvent, TTransformedEvent>( ev => TransformEventInternal(ev), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = this.batchConfig.MaxDegreeOfSerializationParallelism, BoundedCapacity = this.batchConfig.MaxUploadQueueCapacity }); this.eventObserver = this.eventSource.AsObserver(); this.eventProcessor = new ActionBlock <IList <TTransformedEvent> > ( (Func <IList <TTransformedEvent>, Task>) this.UploadTransformedEventsInternal, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 16, // with the heavy number of cores out there, the memory can otherwise easily overflow BoundedCapacity = this.batchConfig.MaxUploadQueueCapacity, } ); this.eventUnsubscriber = this.eventSource.AsObservable() .Window(this.batchConfig.MaxDuration) .Select(w => w.Buffer(this.batchConfig.MaxEventCount, this.batchConfig.MaxBufferSizeInBytes, this.MeasureTransformedEventInternal)) .SelectMany(buffer => buffer) .Subscribe(this.eventProcessor.AsObserver()); this.random = new Random(0); }
/// <summary> /// Constructs an uploader object. /// </summary> /// <param name="eventHubConnectionString">The Azure Stream Analytics connection string.</param> /// <param name="batchConfig">Optional; The batching configuration to used when uploading data.</param> /// <param name="developmentMode">If true, enables additional logging and disables batching.</param> public EventUploaderASA ( string eventHubConnectionString, BatchingConfiguration batchConfig = null, bool developmentMode = false ) : base(batchConfig, developmentMode) { this.connectionString = eventHubConnectionString; var builder = new ServiceBusConnectionStringBuilder(this.connectionString) { TransportType = TransportType.Amqp, }; var eventHubInputName = builder.EntityPath; builder.EntityPath = null; if (this.batchConfig.ReUseTcpConnection) { this.client = EventHubClient.CreateFromConnectionString(builder.ToString(), eventHubInputName); } else { var factory = MessagingFactory.CreateFromConnectionString(builder.ToString()); this.client = factory.CreateEventHubClient(eventHubInputName); } }