public static IEnumerable <IGeoCell> Stratch(IFetchRequest request, Array mask = null) { IEnumerable <IGeoCell> streched = null; switch (request.Domain.SpatialRegionType) { case SpatialRegionSpecification.Points: streched = BatchProcessingUtils.StretchCells(request.Domain.Lats, request.Domain.Lats, request.Domain.Lons, request.Domain.Lons, request.Domain.TimeRegion, mask); break; case SpatialRegionSpecification.Cells: streched = BatchProcessingUtils.StretchCells(request.Domain.Lats, request.Domain.Lats2, request.Domain.Lons, request.Domain.Lons2, request.Domain.TimeRegion, mask); break; case SpatialRegionSpecification.PointGrid: streched = BatchProcessingUtils.StretchGrid(request.Domain.Lats, request.Domain.Lons, request.Domain.TimeRegion, mask); break; case SpatialRegionSpecification.CellGrid: streched = BatchProcessingUtils.StretchCellGrid(request.Domain.Lats, request.Domain.Lons, request.Domain.TimeRegion, mask); break; default: throw new NotImplementedException(); } return(streched); }
public static DataSet CreateRequestBlobDataSet(string dsUri, IFetchRequest request) { var sd = GetRequestSchemaAndData(request); var ds = AzureBlobDataSet.CreateSetWithSmallData(dsUri, sd.Item1, sd.Item2); return(ds); }
public static IFetchRequest <TOriginatingEntity, TNestedFetch> ThenFetchMany < TOriginatingEntity, TFetchOn, TNestedFetch>( this IFetchRequest <TOriginatingEntity, TFetchOn> source, Expression <Func <TFetchOn, IEnumerable <TNestedFetch> > > path) { return(GetCurrentProvider().ThenFetchMany(source, path)); }
/// <summary>Performs request for climate parameter</summary> /// <param name="request">Request to perform</param> /// <param name="progressReport">Handler to call periodically with information about request</param> /// <param name="cacheFileName">Optional file name with Dmitrov dataset (CSV or NetCDF) to cache results. If specified file exists DataSet from it is returned and /// no actual request is made</param> /// <returns>Task that results in DataSet with climate parameter values, optional uncertainty and provenance information</returns> public static Task <DataSet> FetchAsync(IFetchRequest request, Action <FetchStatus> progressReport = null, string cacheFileName = null) { if (String.IsNullOrEmpty(cacheFileName)) { return(Instance.FetchAsync(request, progressReport)); } else { if (File.Exists(cacheFileName)) { try { return(Task.FromResult <DataSet>(DataSet.Open(cacheFileName))); // Try to open cached data set } catch (Exception exc) { Trace.WriteLine(String.Format("Cached response is not found at {0}. Performing request...", cacheFileName)); } } return(Instance.FetchAsync(request, progressReport).ContinueWith <DataSet>(t => { var result = t.Result; try { result = result.Clone(cacheFileName); } catch (Exception exc2) { Trace.WriteLine(String.Format("Error writing cached response to {0}: {1}", cacheFileName, exc2.Message)); } return result; }, TaskContinuationOptions.OnlyOnRanToCompletion)); } }
public static IFetchRequest <TQueried, TRelated> ThenEagerFetch <TQueried, TFetch, TRelated>( this IFetchRequest <TQueried, TFetch> query, Expression <Func <TFetch, TRelated> > relatedObjectSelector) { var impl = query as FetchRequest <TQueried, TFetch>; var fetch = EagerFetchingExtensionMethods.ThenFetch(impl.NhFetchRequest, relatedObjectSelector); return(new FetchRequest <TQueried, TRelated>(fetch)); }
public RequestContext(IFetchRequest request, IFetchEngine engine, IDictionary <string, string> ds2fc, DataSet storage, ushort id) : base(storage) { this.id = id; this.engine = engine; this.request = request; this.ds2fc = ds2fc; }
/// <summary> /// Eager-loads a second-level projection of the specified queryable, /// referencing a mapped child of the first eager-loaded child. /// </summary> /// <typeparam name="T"></typeparam> /// <typeparam name="TRel">The type of the rel.</typeparam> /// <typeparam name="TRel2">The type of the rel2.</typeparam> /// <param name="queryable">The queryable.</param> /// <param name="expression">The expression.</param> /// <returns></returns> public static IFetchRequest <T, TRel2> ThenFetchOk <T, TRel, TRel2>( this IFetchRequest <T, TRel> queryable, Expression <Func <TRel, TRel2> > expression) => queryable is QueryableFetchHelper <T, TRel>? FetchHelper.CreateNonNH <T, TRel2>(queryable) : FetchHelper.Create(queryable.ThenFetch(expression));
public IFetchRequest <TOriginalEntity, TNestedFetch> ThenFetchMany <TOriginalEntity, TFetchOn, TNestedFetch>( IFetchRequest <TOriginalEntity, TFetchOn> source, Expression <Func <TFetchOn, IEnumerable <TNestedFetch> > > path) { var fetchRequest = (NHibernateFetchRequest <TOriginalEntity, TFetchOn>)source; var fetchQuery = fetchRequest.Queryable.ThenFetchMany(path); return(new NHibernateFetchRequest <TOriginalEntity, TNestedFetch>(fetchQuery)); }
/// <summary> /// Initializes a new instance of the <see cref="T:MailKit.Net.Imap.ImapEvent.MessageNew"/> class. /// </summary> /// <remarks> /// Initializes a new instance of the <see cref="T:MailKit.Net.Imap.ImapEvent.MessageNew"/> class. /// </remarks> /// <param name="request">The fetch request to use when new messages arrive.</param> /// <exception cref="ArgumentNullException"> /// <paramref name="request"/> is <c>null</c>. /// </exception> public MessageNew(IFetchRequest request) : base("MessageNew", true) { if (request == null) { throw new ArgumentNullException(nameof(request)); } this.request = request; }
ThenFetchMany <TOriginatingEntity, TFetchOn, TNestedFetch>( IFetchRequest <TOriginatingEntity, TFetchOn> source, Expression <Func <TFetchOn, IEnumerable <TNestedFetch> > > path) { var fetchRequest = (FetchRequest <IQueryable <TOriginatingEntity>, TOriginatingEntity, TFetchOn>)source; return(new FetchRequest <IQueryable <TOriginatingEntity>, TOriginatingEntity, TNestedFetch>(fetchRequest .Queryable)); }
/// <summary> /// Retrieve data /// </summary> /// <typeparam name="TResult">Data type</typeparam> /// <param name="request">Fetch request</param> /// <returns>Result</returns> public Task <TResult> Retrieve <TResult>(IFetchRequest <TResult> request) where TResult : class { if (handlers.TryGetValue(request.GetType(), out var handler)) { return(handler.DynamicInvoke(request) as Task <TResult>); } throw new FetchHandlerNotFoundException(request.GetType()); }
private bool CreateBlobWithJob(IFetchRequest request, string hash, double allowedCreationTimeSec) { var requestBlob = resultsContainer.GetPageBlobReference(ResultBlobName(hash)); while (true) { try { JobManagerTrace.TraceVerbose("Trying to create blob ({0})", hash); requestBlob.Create(0, accessCondition: AccessCondition.GenerateIfNotExistsCondition()); JobManagerTrace.TraceVerbose("Blob {0} successfully created", hash); //loading request into blob string blobUri = ResultDataSetUri(hash, true).ToString(); JobManagerTrace.TraceVerbose("Filling blob with request. uri = {0}", blobUri); RequestDataSetFormat.CreateRequestBlobDataSet(blobUri, request).Dispose(); return(true); } catch (StorageException e) { // Blob already exist - probably someone already works with it if (e.RequestInformation.HttpStatusCode == 412 /* Procondition Failed*/) { JobManagerTrace.TraceVerbose("Can't create blob {0}. It is already exists", hash); try { requestBlob.FetchAttributes(); } catch (StorageException) { JobManagerTrace.TraceWarning("Can't get modification time of blob {0}. Retrying", hash); continue; } double allowdSeconds = allowedCreationTimeSec; if ((DateTime.UtcNow - requestBlob.Properties.LastModified.Value).TotalSeconds > allowdSeconds) { JobManagerTrace.TraceWarning("Job blob {0} exists but there are no job records in the job table exists longer than permitted time ({1}). deleting it", hash, allowdSeconds); requestBlob.DeleteIfExists(); } else { return(false); } } else { throw; } } } }
/// <summary> /// Process the request with a data source specified by its definition /// </summary> /// <param name="request">A request to process</param> /// <param name="dataSource">A data source definition to process the request with</param> /// <returns></returns> private async Task <IFetchResponseWithProvenance> ProcessWithDataSourceAsync(IFetchRequest request, ExtendedDataSourceDefinition dataSource) { traceSource.TraceEvent(TraceEventType.Information, 7, string.Format("One data source {0} serves the request", dataSource.Name)); if (!dataSource.IsFederated) //local { return(await ProcessWithLocalDataSourceAsync(request, dataSource)); } else //federated { return(await ProcessWithFederatedDataSourceAsync(request, dataSource)); } }
private static IFetchRequest TranslateRequestIntoDsNamings(IFetchRequest request, ExtendedDataSourceDefinition dataSource) { string dsName; if (dataSource.EnvToDsMapping.TryGetValue(request.EnvironmentVariableName, out dsName)) { return(new FetchRequest(dsName, request.Domain, request.ReproducibilityTimestamp, request.ParticularDataSource)); } else { return(request); } }
/// <summary> /// Eager-loads a second-level projection of the specified queryable, /// referencing a mapped child of the first eager-loaded child. /// </summary> public static IFetchRequest <T, TRel2> ThenEagerlyFetch <T, TRel, TRel2>( this IFetchRequest <T, TRel> queryable, Expression <Func <TRel, TRel2> > expression) { if (queryable is QueryableFetchHelper <T, TRel> ) { return(FetchHelper.CreateNonNH <T, TRel2>(queryable)); } else { return(FetchHelper.Create(queryable.ThenFetch(expression))); } }
/// <summary>Translates request from current data source namespace to FC namespace</summary> /// <param name="r">Request with local namings</param> /// <returns>Request with FC namings</returns> public IFetchRequest TranslateRequestToFC(IFetchRequest r) { string fcName; if (!ds2fc.TryGetValue(r.EnvironmentVariableName, out fcName)) { return(r); } else { return(new FetchRequest(fcName, r.Domain, r.ReproducibilityTimestamp, r.ParticularDataSource)); } }
/// <summary> /// Applies intelligent fetching logic (competition by uncertainties, returning the results from the data sources with lowest uncertainty) for a specified request among specified data sources /// </summary> /// <param name="request"></param> /// <param name="dataSources"></param> /// <returns></returns> private async Task <IFetchResponseWithProvenance> IntelligentFetchingAsync(IFetchRequest request, ExtendedDataSourceDefinition[] dataSources) { traceSource.TraceEvent(TraceEventType.Information, 4, string.Format("{0} data sources serve the request", dataSources.Length)); var localDataSources = dataSources.Where(d => !d.IsFederated); var federatedDataSources = dataSources.Where(d => d.IsFederated); var localResultTask = ProcessWithLocalDataSourcesAsync(request, localDataSources.ToArray()); var federatedResultsTask = ProcessWithFederatedDataSourcesAsync(request, federatedDataSources); List <Array> provenances = new List <Array>(dataSources.Length); List <Array> uncertainties = new List <Array>(dataSources.Length); List <Array> values = new List <Array>(dataSources.Length); var localResult = await localResultTask; if (localResult != null) { provenances.Add(localResult.Provenance); uncertainties.Add(localResult.Uncertainty); values.Add(localResult.Values); } var federatedResults = await federatedResultsTask; foreach (var remoteServiceResult in federatedResults) { provenances.Add(remoteServiceResult.Item2.Provenance); uncertainties.Add(remoteServiceResult.Item2.Uncertainty); values.Add(remoteServiceResult.Item2.Values); } var mergedResults = MergeResult(provenances, uncertainties, values); if (mergedResults == null) { throw new Exception("Request can't be processed as there are no data sources which are able to process it. Check your service configurations (especially variables mappings)"); } //int[] dsIDsWithNans = GetIDsOfDataSourcesWithNanInValues(mergedResults.Provenance, mergedResults.Uncertatinty, mergedResults.Values); //if (dsIDsWithNans.Length > 0) //{ // dataSources = dataSources.Where(ds => !dsIDsWithNans.Contains(ds.ID)).ToArray(); //filtering out data sources that returned NAN. One more loop pass for uncertainty competition round //} return(new FetchResponseWithProvenance(request, mergedResults.Values, mergedResults.Uncertatinty, mergedResults.Provenance)); }
/// <summary> /// Produces the mean values array for the request supplied, applying the optional boolean mask to the request /// </summary> /// <param name="request">A spatio-temporal region to produce a mean values for</param> /// <param name="mask">An optional boolean array (the same dimensions as a result array) containing false values for the nodes where the mean values calculation can be omitted</param> /// <returns></returns> public async Task <Array> AggregateAsync(IRequestContext context, Array mask = null) { IFetchRequest request = context.Request; var name = context.Request.EnvironmentVariableName; IEnumerable <IGeoCell> cells = RequestToBatchAdapter.Stratch(request, mask); IEnumerable <ICellRequest> requests = cells.Select(c => new NameAnnotatedGeoCell(c, name)); double[] strechedResults = await component.AggregateCellsBatchAsync(requests); Array res = RequestToBatchAdapter.Fold(strechedResults, request, mask); return(res); }
public async Task <IFetchResponseWithProvenance> PerformRequestAsync(IFetchRequest request) { string error; if (!request.Domain.IsContentValid(out error)) { throw new ArgumentException(error); } var config = configProvider.GetConfiguration(request.ReproducibilityTimestamp); string envVariableName = request.EnvironmentVariableName; string[] dataSourcesName = request.ParticularDataSource; traceSource.TraceEvent(TraceEventType.Start, 1, string.Format("Request started: {0},{1},{2}", envVariableName, (dataSourcesName == null) ? ("*") : (dataSourcesName.Aggregate(string.Empty, (acc, ds) => acc + " " + ds)), request.ReproducibilityTimestamp)); var dataSources = (dataSourcesName == null) ? config.DataSources.Where(d => d.ProvidedVariables.Contains(envVariableName)).ToArray() : config.DataSources.Where(d => dataSourcesName.Contains(d.Name) && d.ProvidedVariables.Contains(envVariableName)).ToArray(); try { if (dataSources.Length == 0) { traceSource.TraceEvent(TraceEventType.Error, 2, "No data sources found for serving the request"); throw new InvalidOperationException("No data sources found for serving the request"); } else if (dataSources.Length == 1) // One data source - invoke it directly, no further merging { return(await ProcessWithDataSourceAsync(request, dataSources[0])); } else // Full intelligent fetching pipeline { return(await IntelligentFetchingAsync(request, dataSources)); } } catch (Exception exc) { traceSource.TraceEvent(TraceEventType.Error, 999, string.Format("Request failed: {0}", exc.ToString())); traceSource.TraceData(TraceEventType.Error, 999, exc); throw exc; } }
/// <summary>Creates and fill dataset with completed request</summary> /// <param name="dsUri">Dmitrov URI to create dataset at</param> /// <param name="request">FetchClimate2 request</param> /// <param name="values">Array of computed values</param> /// <param name="provenance">Array of provenance IDs (may be null)</param> /// <param name="uncertainty">Array of uncertainties</param> /// <returns></returns> public static DataSet CreateCompletedRequestDataSet(string dsUri, IFetchRequest request, Array values, Array provenance, Array uncertainty) { var sd = GetRequestSchemaAndData(request); var ds = DataSet.Open(dsUri); if (sd.Item1.Metadata != null) { foreach (var m in sd.Item1.Metadata) { ds.Metadata[m.Key] = m.Value.Value; } } foreach (var v in sd.Item1.Variables) { Array data; if (v.Name == RequestDataSetFormat.ValuesVariableName) { data = values; } else if (v.Name == RequestDataSetFormat.ProvenanceVariableName) { data = provenance; } else if (v.Name == RequestDataSetFormat.UncertaintyVariableName) { data = uncertainty; } else { data = sd.Item2[v.Name]; } if (data != null) { ds.AddVariable(v.Type, v.Name, data, v.Dimensions); } } ds.Commit(); return(ds); }
public static int GetPartitionsCount(IFetchRequest request, int minPtsPerPartition, int maxPtsPerPartition, int freeWorkers = 0) { if (minPtsPerPartition < 1) { minPtsPerPartition = 1; } if (maxPtsPerPartition < 1) { maxPtsPerPartition = 1; } int[] shape = request.Domain.GetDataArrayShape(); long totalPts = 1; foreach (var i in shape) { totalPts *= i; } int desiredPartsCount = (int)Math.Min(Math.Max((totalPts + maxPtsPerPartition - 1) / maxPtsPerPartition, Math.Min(freeWorkers, totalPts / minPtsPerPartition)), shape[0]); if (shape.Length == 1) { int desiredSize = (int)((totalPts / desiredPartsCount + 511) / 512) * 512; if (totalPts % desiredSize == 0) { return((int)(totalPts / desiredSize)); } else { return((int)(totalPts / desiredSize) + 1); } } else { return(desiredPartsCount); } }
public abstract IRequestContext CopyWithNewRequest(IFetchRequest request);
public override IRequestContext CopyWithNewRequest(IFetchRequest request) { return(new DependentRequestContext(request, engine, ds2fc, Storage, id)); }
public static Array Fold(double[] strached, IFetchRequest request, Array mask = null) { return(BatchProcessingUtils.Fold(strached, request.Domain.GetDataArrayShape(), mask)); }
public StandaloneRequestContext(IFetchRequest request, IFetchEngine engine, IDictionary <string, string> ds2fc, DataSet storage, ushort id) : base(request, engine, ds2fc, storage, id) { }
public static IFetchRequest <TQueried, TRelated> ThenFetchMany <TQueried, TFetch, TRelated>( this IFetchRequest <TQueried, TFetch> query, Expression <Func <TFetch, IEnumerable <TRelated> > > relatedObjectSelector) { return(FetchingProvider.ThenFetchMany(query, relatedObjectSelector)); }
public static IFetchRequest <TQueried, TRelated> ThenFetch <TQueried, TFetch, TRelated>(this IFetchRequest <TQueried, TFetch> query, Expression <Func <TFetch, TRelated> > relatedObjectSelector) { return(query.ThenEagerFetch(relatedObjectSelector)); }
public FetchResponse(IFetchRequest request, Array values, Array uncertainty) { this.request = request; this.values = values; this.uncertainty = uncertainty; }
public IFetchRequest <TQueried, TRelated> ThenFetchMany <TQueried, TFetch, TRelated>(IFetchRequest <TQueried, TFetch> query, Expression <Func <TFetch, IEnumerable <TRelated> > > relatedObjectSelector) { var impl = query as NhFetch <TQueried, TFetch>; var fetch = EagerFetchingExtensionMethods.ThenFetchMany(impl.NhFetchRequest, relatedObjectSelector); return(new NhFetch <TQueried, TRelated>(fetch)); }
public IFetchRequest <TQueried, TRelated> ThenFetchMany <TQueried, TFetch, TRelated>(IFetchRequest <TQueried, TFetch> query, Expression <Func <TFetch, IEnumerable <TRelated> > > relatedObjectSelector) { throw new NotImplementedException(); }