public IJob TakeJob(string industry) { var jobIdsQuery = CreateJobQuery(industry) .Select(record => record.Id) .Take(50); if (jobIdsQuery.Count() == 0) return null; var jobIds = jobIdsQuery.ToArray(); var lockFile = _lockFileResolve.Value; var jobNumber = 0; while (jobNumber < jobIds.Length && !lockFile.TryAcquire("Job - " + industry + jobIds[jobNumber])) { jobNumber++; } // We couldn't find any open jobs in the first 50, not looking further if (jobNumber == jobIds.Length) return null; var jobRecord = _repository.Get(jobIds[jobNumber]); var job = new Job(industry, jobRecord.ContextDefinion); _jobReferences[job] = new JobReference { Id = jobRecord.Id, LockFile = lockFile }; return job; }
// [END build_service] /// <summary> /// Pages through the results of an arbitrary Bigquery request. /// </summary> /// <param name="bigquery">The bigquery service.</param> /// <param name="jobRef">The job whose rows will be fetched.</param> /// <param name="rowsPerPage">How many rows to fetch in each http request?</param> /// <returns>An IEnumerable of rows.</returns> // [START paging] public static IEnumerable<TableRow> GetRows(BigqueryService bigquery, JobReference jobRef, long? rowsPerPage = null) { var request = new JobsResource.GetQueryResultsRequest( bigquery, jobRef.ProjectId,jobRef.JobId); request.MaxResults = rowsPerPage; do { var response = request.Execute(); if (response.Rows != null) foreach (var row in response.Rows) yield return row; request.PageToken = response.PageToken; } while (!String.IsNullOrEmpty(request.PageToken)); }
/// <inheritdoc /> public override BigqueryResult GetQueryResults(JobReference jobReference, GetQueryResultsOptions options = null) { GaxRestPreconditions.CheckNotNull(jobReference, nameof(jobReference)); Func <GetQueryResultsRequest> requestProvider = () => { var request = Service.Jobs.GetQueryResults(jobReference.ProjectId, jobReference.JobId); options?.ModifyRequest(request); return(request); }; var firstResponse = requestProvider().Execute(); return(new BigqueryResult(this, firstResponse, requestProvider)); }
public IJob TakeOnlyJob(string industry) { var lockFile = _lockResolve.Value; if (lockFile == null || !lockFile.TryAcquire("Only Job - " + industry)) return null; var jobRecord = CreateJobQuery(industry) .Take(1) .FirstOrDefault(); if (jobRecord == null) return null; var job = new Job(industry, jobRecord.ContextDefinion); _jobReferences[job] = new JobReference { Id = jobRecord.Id, LockFile = lockFile }; return job; }
internal override BigQueryResults GetQueryResults(JobReference jobReference, TableReference tableReference, GetQueryResultsOptions options) { GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference)); DateTime start = Clock.GetCurrentDateTimeUtc(); while (true) { // This will throw if the query has timed out. var response = GetRawQueryResults(jobReference, options, start); if (response.JobComplete == true) { return(new BigQueryResults(this, response, tableReference, options)); } } }
void UploadData(Table table) { Job job = new Job(); var config = new JobConfiguration(); var configLoad = new JobConfigurationLoad { Schema = table.Schema, DestinationTable = table.TableReference, Encoding = "ISO-8859-1", CreateDisposition = "CREATE_IF_NEEDED", WriteDisposition = "", FieldDelimiter = ",", AllowJaggedRows = true, SourceFormat = "CSV" }; config.Load = configLoad; job.Configuration = config; var jobId = "---" + Environment.TickCount; var jobRef = new JobReference { JobId = jobId, ProjectId = connection.ProjectId }; job.JobReference = jobRef; using ( Stream stream = Assembly.GetExecutingAssembly() .GetManifestResourceStream( $"DevExpress.DataAccess.BigQuery.Tests.Tests.{table.TableReference.TableId}.csv")) { var insertMediaUpload = new JobsResource.InsertMediaUpload(connection.Service, job, job.JobReference.ProjectId, stream, "application/octet-stream"); insertMediaUpload.Upload(); } while (true) { Job job1 = connection.Service.Jobs.Get(connection.ProjectId, jobId).Execute(); if (job1.Status.State.Equals("DONE")) { break; } Thread.Sleep(5000); } }
internal override async Task <BigQueryResults> GetQueryResultsAsync(JobReference jobReference, TableReference tableReference, GetQueryResultsOptions options, CancellationToken cancellationToken) { GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference)); DateTime start = Clock.GetCurrentDateTimeUtc(); while (true) { // This will throw if the query has timed out. var response = await GetRawQueryResultsAsync(jobReference, options, start, cancellationToken).ConfigureAwait(false); if (response.JobComplete == true) { return(new BigQueryResults(this, response, tableReference, options)); } } }
internal override BigQueryResults GetQueryResults(JobReference jobReference, TableReference tableReference, GetQueryResultsOptions options) { GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference)); DateTime start = Clock.GetCurrentDateTimeUtc(); while (true) { // This will throw if the query has timed out. Otherwise, the RPC will have a timeout, // and the server won't return an incomplete job before that timeout. So although this // looks like a tight loop without any delay, the delay is on the server side. var response = GetRawQueryResults(jobReference, options, start); if (response.JobComplete == true) { return(new BigQueryResults(this, response, tableReference, options)); } } }
public void PublicConstructor() { var rows = new List <BigQueryRow>(); var schema = new TableSchema(); var jobReference = new JobReference { ProjectId = "project", JobId = "job" }; var tableReference = new TableReference { ProjectId = "project", DatasetId = "dataset", TableId = "table" }; var nextPageToken = "token"; var page = new BigQueryPage(rows, schema, jobReference, tableReference, nextPageToken); Assert.Same(rows, page.Rows); Assert.Same(schema, page.Schema); Assert.Same(jobReference, page.JobReference); Assert.Same(tableReference, page.TableReference); Assert.Same(nextPageToken, page.NextPageToken); }
/// <inheritdoc /> public override BigqueryJob PollJob(JobReference jobReference, PollJobOptions options = null) { GaxRestPreconditions.CheckNotNull(jobReference, nameof(jobReference)); options?.Validate(); DateTimeOffset?deadline = options?.GetEffectiveDeadline() ?? DateTimeOffset.MaxValue; long maxRequests = options?.MaxRequests ?? long.MaxValue; TimeSpan interval = options?.Interval ?? TimeSpan.FromSeconds(1); for (long i = 0; i < maxRequests && DateTimeOffset.UtcNow < deadline; i++) { var job = GetJob(jobReference); if (job.State == JobState.Done) { return(job); } Thread.Sleep(interval); } throw new TimeoutException($"Job {jobReference.JobId} did not complete in time."); }
internal override BigQueryResults GetQueryResults(JobReference jobReference, TableReference tableReference, GetQueryResultsOptions options) { GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference)); GaxPreconditions.CheckNotNull(tableReference, nameof(tableReference)); // This validates the options before we make any RPCs var listRowsOptions = options?.ToListRowsOptions(); DateTime start = Clock.GetCurrentDateTimeUtc(); while (true) { // This will throw if the query has timed out. var request = CreateGetQueryResultsRequest(jobReference, options, start); var response = request.Execute(); if (response.JobComplete == true) { return(new BigQueryResults(this, response, tableReference, listRowsOptions)); } } }
private JobsResource.GetQueryResultsRequest CreateGetQueryResultsRequest(JobReference jobReference, GetQueryResultsOptions options, DateTime loopStart) { var timeSoFar = Clock.GetCurrentDateTimeUtc() - loopStart; var timeout = options?.Timeout ?? GetQueryResultsOptions.DefaultTimeout; var timeRemainingMs = (long)(timeout - timeSoFar).TotalMilliseconds; if (timeRemainingMs < 1) { // TODO: Check this is correct throw new TimeoutException("Query timed out"); } var requestTimeoutMs = Math.Min(timeRemainingMs, s_maxGetQueryResultsRequestTimeout); var request = Service.Jobs.GetQueryResults(jobReference.ProjectId, jobReference.JobId); request.Location = jobReference.Location; request.TimeoutMs = requestTimeoutMs; options?.ModifyRequest(request); RetryHandler.MarkAsRetriable(request); return(request); }
public IJob TakeJob(string industry) { var jobIdsQuery = CreateJobQuery(industry) .Select(record => record.Id) .Take(50); if (!jobIdsQuery.Any()) { return(null); } var jobIds = jobIdsQuery.ToArray(); var lockFile = _lockResolve.Value; if (lockFile == null) { return(null); } var jobNumber = 0; while (jobNumber < jobIds.Length && !lockFile.TryAcquire("Job - " + industry + jobIds[jobNumber])) { jobNumber++; } // We couldn't find any open jobs in the first 50, not looking further if (jobNumber == jobIds.Length) { return(null); } var jobRecord = _repository.Get(jobIds[jobNumber]); var job = new Job(industry, jobRecord.ContextDefinion); _jobReferences[job] = new JobReference { Id = jobRecord.Id, LockFile = lockFile }; return(job); }
// [END build_service] /// <summary> /// Pages through the results of an arbitrary Bigquery request. /// </summary> /// <param name="jobRef">The job whose rows will be fetched.</param> /// <param name="rowsPerPage">How many rows to fetch in each http /// request?</param> /// <returns>An IEnumerable of rows.</returns> // [START paging] public IEnumerable <TableRow> GetRows(JobReference jobRef, long?rowsPerPage = null) { BigqueryService bigquery = CreateAuthorizedClient(); var request = new JobsResource.GetQueryResultsRequest( bigquery, jobRef.ProjectId, jobRef.JobId); request.MaxResults = rowsPerPage; do { var response = request.Execute(); if (response.Rows != null) { foreach (var row in response.Rows) { yield return(row); } } request.PageToken = response.PageToken; } while (!String.IsNullOrEmpty(request.PageToken)); }
private JobsResource.GetQueryResultsRequest CreateGetQueryResultsRequest(JobReference jobReference, GetQueryResultsOptions options, DateTime loopStart) { var timeSoFar = Clock.GetCurrentDateTimeUtc() - loopStart; var timeout = options?.Timeout ?? GetQueryResultsOptions.DefaultTimeout; var timeRemainingMs = (long)(timeout - timeSoFar).TotalMilliseconds; if (timeRemainingMs < 1) { // TODO: Check this is correct throw new TimeoutException("Query timed out"); } var requestTimeoutMs = Math.Min(timeRemainingMs, s_maxGetQueryResultsRequestTimeout); var request = Service.Jobs.GetQueryResults(jobReference.ProjectId, jobReference.JobId); // We never use the results within the first response; instead, we're just checking that the job has // completed and using the statistics and schema from it. request.MaxResults = 0; request.ModifyRequest += _versionHeaderAction; request.TimeoutMs = requestTimeoutMs; options?.ModifyRequest(request); return(request); }
public void InternalConstructor() { var nextPageToken = "token"; var schema = new TableSchema(); var row = new BigQueryRow(new TableRow(), schema, schema.IndexFieldNames()); var rawPage = new Page <BigQueryRow>(new List <BigQueryRow> { row }, nextPageToken); var jobReference = new JobReference { ProjectId = "project", JobId = "job" }; var tableReference = new TableReference { ProjectId = "project", DatasetId = "dataset", TableId = "table" }; var page = new BigQueryPage(rawPage, schema, jobReference, tableReference); Assert.Equal(new[] { row }, page.Rows); Assert.Same(schema, page.Schema); Assert.Same(jobReference, page.JobReference); Assert.Same(tableReference, page.TableReference); Assert.Same(nextPageToken, page.NextPageToken); }
private QueryResponse(BigQueryContext context, string query, TimeSpan executionTime, bool isDynamic, IRowsParser rowsParser, IList <TableRow> rows, TableSchema schema, bool?cacheHit, string eTag, string kind, string pageToken, long?totalBytesProcessed, ulong?totalRows, bool?jobComplete, JobReference jobReference) { _context = context; _isDynamic = isDynamic; _rowsParser = rowsParser; Rows = rows == null ? new T[0] : rowsParser.Parse <T>( context.fallbacks, context.IsConvertResultUtcToLocalTime, schema, rows, isDynamic).ToArray(); TableFieldSchemas = (schema == null) ? new TableFieldSchema[0] : schema.Fields; Query = query; ExecutionTime = executionTime; CacheHit = cacheHit; ETag = eTag; Kind = kind; PageToken = pageToken; TotalBytesProcessed = totalBytesProcessed; TotalRows = totalRows; TotalBytesProcessedFormatted = totalBytesProcessed.ToHumanReadableSize(); _jobComplete = jobComplete.GetValueOrDefault(false); _jobReference = jobReference; }
internal override Task <GetQueryResultsResponse> GetRawQueryResultsAsync(JobReference jobReference, GetQueryResultsOptions options, DateTime?timeoutBase, CancellationToken cancellationToken) { var request = CreateGetQueryResultsRequest(jobReference, options, timeoutBase ?? Clock.GetCurrentDateTimeUtc()); return(request.ExecuteAsync(cancellationToken)); }
/// <summary> /// Asynchronously cancels the specified job. /// </summary> /// <param name="jobReference">A fully-qualified identifier for the job. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <param name="cancellationToken">The token to monitor for cancellation requests.</param> /// <returns>A task representing the asynchronous operation. When complete, the result is /// the final state of the job.</returns> public virtual Task <BigQueryJob> CancelJobAsync(JobReference jobReference, CancelJobOptions options = null, CancellationToken cancellationToken = default(CancellationToken)) => throw new NotImplementedException();
/// <summary> /// Cancels the specified job. /// </summary> /// <param name="jobReference">A fully-qualified identifier for the job. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <returns>The final state of the job.</returns> public virtual BigQueryJob CancelJob(JobReference jobReference, CancelJobOptions options = null) => throw new NotImplementedException();
/// <summary> /// Asynchronously polls the specified job for completion. /// </summary> /// <param name="jobReference">A fully-qualified identifier for the job. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <param name="pollSettings">The settings to control how often and long the job is fetched before timing out if it is still incomplete. May be null, in which case defaults will be supplied.</param> /// <param name="cancellationToken">The token to monitor for cancellation requests.</param> /// <returns>A task representing the asynchronous operation. When complete, the result is /// the completed job.</returns> public virtual Task <BigQueryJob> PollJobUntilCompletedAsync(JobReference jobReference, GetJobOptions options = null, PollSettings pollSettings = null, CancellationToken cancellationToken = default(CancellationToken)) => throw new NotImplementedException();
/// <summary> /// Polls the specified job for completion. /// </summary> /// <param name="jobReference">A fully-qualified identifier for the job. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <param name="pollSettings">The settings to control how often and long the job is fetched before timing out if it is still incomplete. May be null, in which case defaults will be supplied.</param> /// <returns>The completed job.</returns> public virtual BigQueryJob PollJobUntilCompleted(JobReference jobReference, GetJobOptions options = null, PollSettings pollSettings = null) => throw new NotImplementedException();
/// <summary> /// Retrieves a job. /// </summary> /// <param name="jobReference">A fully-qualified identifier for the job. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <returns>The retrieved job.</returns> public virtual BigqueryJob GetJob(JobReference jobReference, GetJobOptions options = null) { throw new NotImplementedException(); }
/// <summary> /// "Raw" version of GetQueryResultsAsync, with no translation to BigQueryResults. /// </summary> /// <param name="jobReference">A fully-qualified identifier for the job. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <param name="timeoutBase">A base value to use when applying a timeout, or null to use the current date/time.</param> /// <param name="cancellationToken">The token to monitor for cancellation requests.</param> /// <returns>The results of the query.</returns> internal virtual Task <GetQueryResultsResponse> GetRawQueryResultsAsync(JobReference jobReference, GetQueryResultsOptions options, DateTime?timeoutBase, CancellationToken cancellationToken) => throw new NotImplementedException();
/// <summary> /// Retrieves the results of the specified job, which must be a query job. /// </summary> /// <remarks> /// <para> /// This operation will only complete when the specified query has completed. /// </para> /// </remarks> /// <param name="jobReference">A fully-qualified identifier for the job. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <returns>The results of the query.</returns> public virtual BigQueryResults GetQueryResults(JobReference jobReference, GetQueryResultsOptions options = null) => throw new NotImplementedException();
internal override GetQueryResultsResponse GetRawQueryResults(JobReference jobReference, GetQueryResultsOptions options, DateTime?timeoutBase) { var request = CreateGetQueryResultsRequest(jobReference, options, timeoutBase ?? Clock.GetCurrentDateTimeUtc()); return(request.Execute()); }
private static Job GetJob(JobReference reference) => new Job { JobReference = reference };
/// <inheritdoc /> public override Task <BigQueryResults> PollQueryUntilCompletedAsync(JobReference jobReference, GetQueryResultsOptions options = null, PollSettings pollSettings = null, CancellationToken cancellationToken = default(CancellationToken)) { GaxPreconditions.CheckNotNull(jobReference, nameof(jobReference)); return(Polling.PollRepeatedlyAsync(ignoredDeadline => GetQueryResultsAsync(jobReference, options, cancellationToken), job => job.Completed, Clock, Scheduler, pollSettings ?? s_defaultPollSettings, cancellationToken)); }
private TestRunContext CreateTestRunContext() { string releaseUri = null; string releaseEnvironmentUri = null; string teamProject = _executionContext.Variables.System_TeamProject; string owner = _executionContext.Variables.Build_RequestedFor; string buildUri = _executionContext.Variables.Build_BuildUri; int buildId = _executionContext.Variables.Build_BuildId ?? 0; string pullRequestTargetBranchName = _executionContext.Variables.System_PullRequest_TargetBranch; string stageName = _executionContext.Variables.System_StageName; string phaseName = _executionContext.Variables.System_PhaseName; string jobName = _executionContext.Variables.System_JobName; int stageAttempt = _executionContext.Variables.System_StageAttempt ?? 0; int phaseAttempt = _executionContext.Variables.System_PhaseAttempt ?? 0; int jobAttempt = _executionContext.Variables.System_JobAttempt ?? 0; //Temporary fix to support publish in RM scenarios where there might not be a valid Build ID associated. //TODO: Make a cleaner fix after TCM User Story 401703 is completed. if (buildId == 0) { _platform = _configuration = null; } if (!string.IsNullOrWhiteSpace(_executionContext.Variables.Release_ReleaseUri)) { releaseUri = _executionContext.Variables.Release_ReleaseUri; releaseEnvironmentUri = _executionContext.Variables.Release_ReleaseEnvironmentUri; } // If runName is not provided by the task, then create runName from testRunner name and buildId. string runName = String.IsNullOrWhiteSpace(_runTitle) ? String.Format("{0}_TestResults_{1}", _testRunner, buildId) : _runTitle; StageReference stageReference = new StageReference() { StageName = stageName, Attempt = Convert.ToInt32(stageAttempt) }; PhaseReference phaseReference = new PhaseReference() { PhaseName = phaseName, Attempt = Convert.ToInt32(phaseAttempt) }; JobReference jobReference = new JobReference() { JobName = jobName, Attempt = Convert.ToInt32(jobAttempt) }; PipelineReference pipelineReference = new PipelineReference() { PipelineId = buildId, StageReference = stageReference, PhaseReference = phaseReference, JobReference = jobReference }; TestRunContext testRunContext = new TestRunContext( owner: owner, platform: _platform, configuration: _configuration, buildId: buildId, buildUri: buildUri, releaseUri: releaseUri, releaseEnvironmentUri: releaseEnvironmentUri, runName: runName, testRunSystem: _testRunSystem, buildAttachmentProcessor: new CodeCoverageBuildAttachmentProcessor(), targetBranchName: pullRequestTargetBranchName, pipelineReference: pipelineReference ); return(testRunContext); }
internal virtual Task <BigQueryResults> GetQueryResultsAsync(JobReference jobReference, TableReference tableReference, GetQueryResultsOptions options, CancellationToken cancellationToken) => throw new NotImplementedException();
/// <summary> /// Asynchronously retrieves the results of the specified job, which must be a query job. /// </summary> /// <remarks> /// <para> /// This operation will only complete when the specified query has completed. /// </para> /// </remarks> /// <param name="jobReference">A fully-qualified identifier for the job. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <param name="cancellationToken">The token to monitor for cancellation requests.</param> /// <returns>A task representing the asynchronous operation. When complete, the result is /// the results of the query.</returns> public virtual Task <BigQueryResults> GetQueryResultsAsync(JobReference jobReference, GetQueryResultsOptions options = null, CancellationToken cancellationToken = default) => throw new NotImplementedException();
internal static JobReference ConvertBQJobInfoToJobRef(this BQJobInfo jobInfo) { JobReference jobRef = new JobReference(); jobRef.JobId = jobInfo.JobId; jobRef.ProjectId = jobRef.ProjectId; return jobRef; }
public IJob TakeOnlyJob(string industry) { var lockFile = _lockFileResolve.Value; if (!lockFile.TryAcquire("Only Job - " + industry)) return null; var jobRecord = _repository.Table .Where(record => record.Industry == industry) .OrderBy(record => record.Id) .Take(1) .FirstOrDefault(); if (jobRecord == null) return null; var job = new Job(industry, jobRecord.ContextDefinion); _jobReferences[job] = new JobReference { Id = jobRecord.Id, LockFile = lockFile }; return job; }
/// <summary> /// "Raw" version of GetQueryResults, with no translation to BigQueryResults. /// </summary> /// <param name="jobReference">A fully-qualified identifier for the job. Must not be null.</param> /// <param name="options">The options for the operation. May be null, in which case defaults will be supplied.</param> /// <param name="timeoutBase">A base value to use when applying a timeout, or null to use the current date/time.</param> /// <returns>The results of the query.</returns> internal virtual GetQueryResultsResponse GetRawQueryResults(JobReference jobReference, GetQueryResultsOptions options, DateTime?timeoutBase) => throw new NotImplementedException();
// Note - these methods are not part of the regular "pattern", so are not in the GetQueryResults region above. // We want to remove them, if the underlying GetQueryResultsResponse starts including the table reference. // These methods allow us to call GetQueryResultsAsync from BigQueryJob without fetching the job again. internal virtual BigQueryResults GetQueryResults(JobReference jobReference, TableReference tableReference, GetQueryResultsOptions options) => throw new NotImplementedException();
void UploadData(Table table) { Job job = new Job(); var config = new JobConfiguration(); var configLoad = new JobConfigurationLoad { Schema = table.Schema, DestinationTable = table.TableReference, Encoding = "ISO-8859-1", CreateDisposition = "CREATE_IF_NEEDED", WriteDisposition = "", FieldDelimiter = ",", AllowJaggedRows = true, SourceFormat = "CSV" }; config.Load = configLoad; job.Configuration = config; var jobId = "---" + Environment.TickCount; var jobRef = new JobReference { JobId = jobId, ProjectId = connection.ProjectId }; job.JobReference = jobRef; using ( Stream stream = Assembly.GetExecutingAssembly() .GetManifestResourceStream(string.Format("DevExpress.DataAccess.BigQuery.Tests.{0}.csv", table.TableReference.TableId))) { var insertMediaUpload = new JobsResource.InsertMediaUpload(connection.Service, job, job.JobReference.ProjectId, stream, "application/octet-stream"); insertMediaUpload.Upload(); } while (true) { Job job1 = connection.Service.Jobs.Get(connection.ProjectId, jobId).Execute(); if (job1.Status.State.Equals("DONE")) { break; } Thread.Sleep(5000); } }