public static IEnumerable <IGeoCell> Stratch(IFetchRequest request, Array mask = null)
{
IEnumerable <IGeoCell> streched = null;
switch (request.Domain.SpatialRegionType)
{
case SpatialRegionSpecification.Points:
streched = BatchProcessingUtils.StretchCells(request.Domain.Lats, request.Domain.Lats, request.Domain.Lons, request.Domain.Lons, request.Domain.TimeRegion, mask);
break;
case SpatialRegionSpecification.Cells:
streched = BatchProcessingUtils.StretchCells(request.Domain.Lats, request.Domain.Lats2, request.Domain.Lons, request.Domain.Lons2, request.Domain.TimeRegion, mask);
break;
case SpatialRegionSpecification.PointGrid:
streched = BatchProcessingUtils.StretchGrid(request.Domain.Lats, request.Domain.Lons, request.Domain.TimeRegion, mask);
break;
case SpatialRegionSpecification.CellGrid:
streched = BatchProcessingUtils.StretchCellGrid(request.Domain.Lats, request.Domain.Lons, request.Domain.TimeRegion, mask);
break;
default:
throw new NotImplementedException();
}
return(streched);
}
public static DataSet CreateRequestBlobDataSet(string dsUri, IFetchRequest request)
{
var sd = GetRequestSchemaAndData(request);
var ds = AzureBlobDataSet.CreateSetWithSmallData(dsUri, sd.Item1, sd.Item2);
return(ds);
}
public static IFetchRequest <TOriginatingEntity, TNestedFetch> ThenFetchMany <
TOriginatingEntity, TFetchOn, TNestedFetch>(
this IFetchRequest <TOriginatingEntity, TFetchOn> source,
Expression <Func <TFetchOn, IEnumerable <TNestedFetch> > > path)
{
return(GetCurrentProvider().ThenFetchMany(source, path));
}
/// <summary>Performs request for climate parameter</summary>
/// <param name="request">Request to perform</param>
/// <param name="progressReport">Handler to call periodically with information about request</param>
/// <param name="cacheFileName">Optional file name with Dmitrov dataset (CSV or NetCDF) to cache results. If specified file exists DataSet from it is returned and
/// no actual request is made</param>
/// <returns>Task that results in DataSet with climate parameter values, optional uncertainty and provenance information</returns>
public static Task <DataSet> FetchAsync(IFetchRequest request, Action <FetchStatus> progressReport = null, string cacheFileName = null)
{
if (String.IsNullOrEmpty(cacheFileName))
{
return(Instance.FetchAsync(request, progressReport));
}
else
{
if (File.Exists(cacheFileName))
{
try
{
return(Task.FromResult <DataSet>(DataSet.Open(cacheFileName))); // Try to open cached data set
}
catch (Exception exc)
{
Trace.WriteLine(String.Format("Cached response is not found at {0}. Performing request...", cacheFileName));
}
}
return(Instance.FetchAsync(request, progressReport).ContinueWith <DataSet>(t => {
var result = t.Result;
try
{
result = result.Clone(cacheFileName);
}
catch (Exception exc2)
{
Trace.WriteLine(String.Format("Error writing cached response to {0}: {1}", cacheFileName, exc2.Message));
}
return result;
}, TaskContinuationOptions.OnlyOnRanToCompletion));
}
}
public static IFetchRequest <TQueried, TRelated> ThenEagerFetch <TQueried, TFetch, TRelated>(
this IFetchRequest <TQueried, TFetch> query, Expression <Func <TFetch, TRelated> > relatedObjectSelector)
{
var impl = query as FetchRequest <TQueried, TFetch>;
var fetch = EagerFetchingExtensionMethods.ThenFetch(impl.NhFetchRequest, relatedObjectSelector);
return(new FetchRequest <TQueried, TRelated>(fetch));
}
public RequestContext(IFetchRequest request, IFetchEngine engine, IDictionary <string, string> ds2fc, DataSet storage, ushort id)
: base(storage)
{
this.id = id;
this.engine = engine;
this.request = request;
this.ds2fc = ds2fc;
}
/// <summary>
/// Eager-loads a second-level projection of the specified queryable,
/// referencing a mapped child of the first eager-loaded child.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <typeparam name="TRel">The type of the rel.</typeparam>
/// <typeparam name="TRel2">The type of the rel2.</typeparam>
/// <param name="queryable">The queryable.</param>
/// <param name="expression">The expression.</param>
/// <returns></returns>
public static IFetchRequest <T, TRel2> ThenFetchOk <T, TRel, TRel2>(
this IFetchRequest <T, TRel> queryable,
Expression <Func <TRel, TRel2> > expression)
=>
queryable is QueryableFetchHelper <T, TRel>?
FetchHelper.CreateNonNH <T, TRel2>(queryable)
:
FetchHelper.Create(queryable.ThenFetch(expression));
public IFetchRequest <TOriginalEntity, TNestedFetch> ThenFetchMany <TOriginalEntity, TFetchOn, TNestedFetch>(
IFetchRequest <TOriginalEntity, TFetchOn> source,
Expression <Func <TFetchOn, IEnumerable <TNestedFetch> > > path)
{
var fetchRequest = (NHibernateFetchRequest <TOriginalEntity, TFetchOn>)source;
var fetchQuery = fetchRequest.Queryable.ThenFetchMany(path);
return(new NHibernateFetchRequest <TOriginalEntity, TNestedFetch>(fetchQuery));
}
/// <summary>
/// Initializes a new instance of the <see cref="T:MailKit.Net.Imap.ImapEvent.MessageNew"/> class.
/// </summary>
/// <remarks>
/// Initializes a new instance of the <see cref="T:MailKit.Net.Imap.ImapEvent.MessageNew"/> class.
/// </remarks>
/// <param name="request">The fetch request to use when new messages arrive.</param>
/// <exception cref="ArgumentNullException">
/// <paramref name="request"/> is <c>null</c>.
/// </exception>
public MessageNew(IFetchRequest request) : base("MessageNew", true)
{
if (request == null)
{
throw new ArgumentNullException(nameof(request));
}
this.request = request;
}
ThenFetchMany <TOriginatingEntity, TFetchOn, TNestedFetch>(
IFetchRequest <TOriginatingEntity, TFetchOn> source,
Expression <Func <TFetchOn, IEnumerable <TNestedFetch> > > path)
{
var fetchRequest = (FetchRequest <IQueryable <TOriginatingEntity>, TOriginatingEntity, TFetchOn>)source;
return(new FetchRequest <IQueryable <TOriginatingEntity>, TOriginatingEntity, TNestedFetch>(fetchRequest
.Queryable));
}
/// <summary>
/// Retrieve data
/// </summary>
/// <typeparam name="TResult">Data type</typeparam>
/// <param name="request">Fetch request</param>
/// <returns>Result</returns>
public Task <TResult> Retrieve <TResult>(IFetchRequest <TResult> request)
where TResult : class
{
if (handlers.TryGetValue(request.GetType(), out var handler))
{
return(handler.DynamicInvoke(request) as Task <TResult>);
}
throw new FetchHandlerNotFoundException(request.GetType());
}
private bool CreateBlobWithJob(IFetchRequest request, string hash, double allowedCreationTimeSec)
{
var requestBlob = resultsContainer.GetPageBlobReference(ResultBlobName(hash));
while (true)
{
try
{
JobManagerTrace.TraceVerbose("Trying to create blob ({0})", hash);
requestBlob.Create(0, accessCondition: AccessCondition.GenerateIfNotExistsCondition());
JobManagerTrace.TraceVerbose("Blob {0} successfully created", hash);
//loading request into blob
string blobUri = ResultDataSetUri(hash, true).ToString();
JobManagerTrace.TraceVerbose("Filling blob with request. uri = {0}", blobUri);
RequestDataSetFormat.CreateRequestBlobDataSet(blobUri, request).Dispose();
return(true);
}
catch (StorageException e)
{
// Blob already exist - probably someone already works with it
if (e.RequestInformation.HttpStatusCode == 412 /* Procondition Failed*/)
{
JobManagerTrace.TraceVerbose("Can't create blob {0}. It is already exists", hash);
try
{
requestBlob.FetchAttributes();
}
catch (StorageException)
{
JobManagerTrace.TraceWarning("Can't get modification time of blob {0}. Retrying", hash);
continue;
}
double allowdSeconds = allowedCreationTimeSec;
if ((DateTime.UtcNow - requestBlob.Properties.LastModified.Value).TotalSeconds > allowdSeconds)
{
JobManagerTrace.TraceWarning("Job blob {0} exists but there are no job records in the job table exists longer than permitted time ({1}). deleting it", hash, allowdSeconds);
requestBlob.DeleteIfExists();
}
else
{
return(false);
}
}
else
{
throw;
}
}
}
}
/// <summary>
/// Process the request with a data source specified by its definition
/// </summary>
/// <param name="request">A request to process</param>
/// <param name="dataSource">A data source definition to process the request with</param>
/// <returns></returns>
private async Task <IFetchResponseWithProvenance> ProcessWithDataSourceAsync(IFetchRequest request, ExtendedDataSourceDefinition dataSource)
{
traceSource.TraceEvent(TraceEventType.Information, 7, string.Format("One data source {0} serves the request", dataSource.Name));
if (!dataSource.IsFederated) //local
{
return(await ProcessWithLocalDataSourceAsync(request, dataSource));
}
else //federated
{
return(await ProcessWithFederatedDataSourceAsync(request, dataSource));
}
}
private static IFetchRequest TranslateRequestIntoDsNamings(IFetchRequest request, ExtendedDataSourceDefinition dataSource)
{
string dsName;
if (dataSource.EnvToDsMapping.TryGetValue(request.EnvironmentVariableName, out dsName))
{
return(new FetchRequest(dsName, request.Domain, request.ReproducibilityTimestamp, request.ParticularDataSource));
}
else
{
return(request);
}
}
/// <summary>
/// Eager-loads a second-level projection of the specified queryable,
/// referencing a mapped child of the first eager-loaded child.
/// </summary>
public static IFetchRequest <T, TRel2> ThenEagerlyFetch <T, TRel, TRel2>(
this IFetchRequest <T, TRel> queryable,
Expression <Func <TRel, TRel2> > expression)
{
if (queryable is QueryableFetchHelper <T, TRel> )
{
return(FetchHelper.CreateNonNH <T, TRel2>(queryable));
}
else
{
return(FetchHelper.Create(queryable.ThenFetch(expression)));
}
}
/// <summary>Translates request from current data source namespace to FC namespace</summary>
/// <param name="r">Request with local namings</param>
/// <returns>Request with FC namings</returns>
public IFetchRequest TranslateRequestToFC(IFetchRequest r)
{
string fcName;
if (!ds2fc.TryGetValue(r.EnvironmentVariableName, out fcName))
{
return(r);
}
else
{
return(new FetchRequest(fcName, r.Domain, r.ReproducibilityTimestamp, r.ParticularDataSource));
}
}
/// <summary>
/// Applies intelligent fetching logic (competition by uncertainties, returning the results from the data sources with lowest uncertainty) for a specified request among specified data sources
/// </summary>
/// <param name="request"></param>
/// <param name="dataSources"></param>
/// <returns></returns>
private async Task <IFetchResponseWithProvenance> IntelligentFetchingAsync(IFetchRequest request, ExtendedDataSourceDefinition[] dataSources)
{
traceSource.TraceEvent(TraceEventType.Information, 4, string.Format("{0} data sources serve the request", dataSources.Length));
var localDataSources = dataSources.Where(d => !d.IsFederated);
var federatedDataSources = dataSources.Where(d => d.IsFederated);
var localResultTask = ProcessWithLocalDataSourcesAsync(request, localDataSources.ToArray());
var federatedResultsTask = ProcessWithFederatedDataSourcesAsync(request, federatedDataSources);
List <Array> provenances = new List <Array>(dataSources.Length);
List <Array> uncertainties = new List <Array>(dataSources.Length);
List <Array> values = new List <Array>(dataSources.Length);
var localResult = await localResultTask;
if (localResult != null)
{
provenances.Add(localResult.Provenance);
uncertainties.Add(localResult.Uncertainty);
values.Add(localResult.Values);
}
var federatedResults = await federatedResultsTask;
foreach (var remoteServiceResult in federatedResults)
{
provenances.Add(remoteServiceResult.Item2.Provenance);
uncertainties.Add(remoteServiceResult.Item2.Uncertainty);
values.Add(remoteServiceResult.Item2.Values);
}
var mergedResults = MergeResult(provenances, uncertainties, values);
if (mergedResults == null)
{
throw new Exception("Request can't be processed as there are no data sources which are able to process it. Check your service configurations (especially variables mappings)");
}
//int[] dsIDsWithNans = GetIDsOfDataSourcesWithNanInValues(mergedResults.Provenance, mergedResults.Uncertatinty, mergedResults.Values);
//if (dsIDsWithNans.Length > 0)
//{
// dataSources = dataSources.Where(ds => !dsIDsWithNans.Contains(ds.ID)).ToArray(); //filtering out data sources that returned NAN. One more loop pass for uncertainty competition round
//}
return(new FetchResponseWithProvenance(request,
mergedResults.Values,
mergedResults.Uncertatinty,
mergedResults.Provenance));
}
/// <summary>
/// Produces the mean values array for the request supplied, applying the optional boolean mask to the request
/// </summary>
/// <param name="request">A spatio-temporal region to produce a mean values for</param>
/// <param name="mask">An optional boolean array (the same dimensions as a result array) containing false values for the nodes where the mean values calculation can be omitted</param>
/// <returns></returns>
public async Task <Array> AggregateAsync(IRequestContext context, Array mask = null)
{
IFetchRequest request = context.Request;
var name = context.Request.EnvironmentVariableName;
IEnumerable <IGeoCell> cells = RequestToBatchAdapter.Stratch(request, mask);
IEnumerable <ICellRequest> requests = cells.Select(c => new NameAnnotatedGeoCell(c, name));
double[] strechedResults = await component.AggregateCellsBatchAsync(requests);
Array res = RequestToBatchAdapter.Fold(strechedResults, request, mask);
return(res);
}
public async Task <IFetchResponseWithProvenance> PerformRequestAsync(IFetchRequest request)
{
string error;
if (!request.Domain.IsContentValid(out error))
{
throw new ArgumentException(error);
}
var config = configProvider.GetConfiguration(request.ReproducibilityTimestamp);
string envVariableName = request.EnvironmentVariableName;
string[] dataSourcesName = request.ParticularDataSource;
traceSource.TraceEvent(TraceEventType.Start, 1, string.Format("Request started: {0},{1},{2}",
envVariableName, (dataSourcesName == null) ? ("*") : (dataSourcesName.Aggregate(string.Empty, (acc, ds) => acc + " " + ds)), request.ReproducibilityTimestamp));
var dataSources = (dataSourcesName == null) ?
config.DataSources.Where(d => d.ProvidedVariables.Contains(envVariableName)).ToArray() :
config.DataSources.Where(d => dataSourcesName.Contains(d.Name) && d.ProvidedVariables.Contains(envVariableName)).ToArray();
try
{
if (dataSources.Length == 0)
{
traceSource.TraceEvent(TraceEventType.Error, 2, "No data sources found for serving the request");
throw new InvalidOperationException("No data sources found for serving the request");
}
else if (dataSources.Length == 1) // One data source - invoke it directly, no further merging
{
return(await ProcessWithDataSourceAsync(request, dataSources[0]));
}
else // Full intelligent fetching pipeline
{
return(await IntelligentFetchingAsync(request, dataSources));
}
}
catch (Exception exc)
{
traceSource.TraceEvent(TraceEventType.Error, 999, string.Format("Request failed: {0}", exc.ToString()));
traceSource.TraceData(TraceEventType.Error, 999, exc);
throw exc;
}
}
/// <summary>Creates and fill dataset with completed request</summary>
/// <param name="dsUri">Dmitrov URI to create dataset at</param>
/// <param name="request">FetchClimate2 request</param>
/// <param name="values">Array of computed values</param>
/// <param name="provenance">Array of provenance IDs (may be null)</param>
/// <param name="uncertainty">Array of uncertainties</param>
/// <returns></returns>
public static DataSet CreateCompletedRequestDataSet(string dsUri, IFetchRequest request, Array values, Array provenance, Array uncertainty)
{
var sd = GetRequestSchemaAndData(request);
var ds = DataSet.Open(dsUri);
if (sd.Item1.Metadata != null)
{
foreach (var m in sd.Item1.Metadata)
{
ds.Metadata[m.Key] = m.Value.Value;
}
}
foreach (var v in sd.Item1.Variables)
{
Array data;
if (v.Name == RequestDataSetFormat.ValuesVariableName)
{
data = values;
}
else if (v.Name == RequestDataSetFormat.ProvenanceVariableName)
{
data = provenance;
}
else if (v.Name == RequestDataSetFormat.UncertaintyVariableName)
{
data = uncertainty;
}
else
{
data = sd.Item2[v.Name];
}
if (data != null)
{
ds.AddVariable(v.Type, v.Name, data, v.Dimensions);
}
}
ds.Commit();
return(ds);
}
public static int GetPartitionsCount(IFetchRequest request, int minPtsPerPartition, int maxPtsPerPartition, int freeWorkers = 0)
{
if (minPtsPerPartition < 1)
{
minPtsPerPartition = 1;
}
if (maxPtsPerPartition < 1)
{
maxPtsPerPartition = 1;
}
int[] shape = request.Domain.GetDataArrayShape();
long totalPts = 1;
foreach (var i in shape)
{
totalPts *= i;
}
int desiredPartsCount = (int)Math.Min(Math.Max((totalPts + maxPtsPerPartition - 1) / maxPtsPerPartition, Math.Min(freeWorkers, totalPts / minPtsPerPartition)), shape[0]);
if (shape.Length == 1)
{
int desiredSize = (int)((totalPts / desiredPartsCount + 511) / 512) * 512;
if (totalPts % desiredSize == 0)
{
return((int)(totalPts / desiredSize));
}
else
{
return((int)(totalPts / desiredSize) + 1);
}
}
else
{
return(desiredPartsCount);
}
}
public abstract IRequestContext CopyWithNewRequest(IFetchRequest request);
public override IRequestContext CopyWithNewRequest(IFetchRequest request)
{
return(new DependentRequestContext(request, engine, ds2fc, Storage, id));
}
public static Array Fold(double[] strached, IFetchRequest request, Array mask = null)
{
return(BatchProcessingUtils.Fold(strached, request.Domain.GetDataArrayShape(), mask));
}
public StandaloneRequestContext(IFetchRequest request, IFetchEngine engine, IDictionary <string, string> ds2fc, DataSet storage, ushort id)
: base(request, engine, ds2fc, storage, id)
{
}
public static IFetchRequest <TQueried, TRelated> ThenFetchMany <TQueried, TFetch, TRelated>(
this IFetchRequest <TQueried, TFetch> query,
Expression <Func <TFetch, IEnumerable <TRelated> > > relatedObjectSelector)
{
return(FetchingProvider.ThenFetchMany(query, relatedObjectSelector));
}
public static IFetchRequest <TQueried, TRelated> ThenFetch <TQueried, TFetch, TRelated>(this IFetchRequest <TQueried, TFetch> query, Expression <Func <TFetch, TRelated> > relatedObjectSelector)
{
return(query.ThenEagerFetch(relatedObjectSelector));
}
public FetchResponse(IFetchRequest request, Array values, Array uncertainty)
{
this.request = request;
this.values = values;
this.uncertainty = uncertainty;
}
public IFetchRequest <TQueried, TRelated> ThenFetchMany <TQueried, TFetch, TRelated>(IFetchRequest <TQueried, TFetch> query, Expression <Func <TFetch, IEnumerable <TRelated> > > relatedObjectSelector)
{
var impl = query as NhFetch <TQueried, TFetch>;
var fetch = EagerFetchingExtensionMethods.ThenFetchMany(impl.NhFetchRequest, relatedObjectSelector);
return(new NhFetch <TQueried, TRelated>(fetch));
}
public IFetchRequest <TQueried, TRelated> ThenFetchMany <TQueried, TFetch, TRelated>(IFetchRequest <TQueried, TFetch> query, Expression <Func <TFetch, IEnumerable <TRelated> > > relatedObjectSelector)
{
throw new NotImplementedException();
}
