private static async Task <List <NewTreeItem> > CopyDirectoryIntoTree(GitHubClient github, long repoId, string repoDirectory, string path = "")
{
List <NewTreeItem> result = new List <NewTreeItem>();
string fullPath = MiscUtils.MapPath(Path.Join("wwwroot", path));
await Directory.GetFiles(fullPath).ToList().ForEachAsync(async file =>
{
NewBlob newBlob = new NewBlob
{
Content = Convert.ToBase64String(await File.ReadAllBytesAsync(file)),
Encoding = EncodingType.Base64
};
BlobReference newBlobCreated = await github.Git.Blob.Create(repoId, newBlob);
string newBlobSha = newBlobCreated.Sha;
result.Add(new NewTreeItem
{
Mode = "100644",
Type = TreeType.Blob,
Sha = newBlobSha,
Path = Path.Join(repoDirectory, MiscUtils.UnmapPath(file))
});
});
await Directory.GetDirectories(fullPath).ToList().ForEachAsync(async directory =>
{
string unmappedPath = MiscUtils.UnmapPath(directory);
if (unmappedPath == "webwebResources")
{
return;
}
List <NewTreeItem> directoryContents = await CopyDirectoryIntoTree(github, repoId, repoDirectory, unmappedPath);
result = result.Concat(directoryContents).ToList();
});
return(result);
}
/*
* If ContinueOnError argument is 'true', return the exception else nope.
*/
private void DeleteBlobFromAzureStorage(CodeActivityContext context)
{
try {
AzureParamHelper azureStorageHelper = new AzureParamHelper(StorageConnectionString.Get(context), StorageBlobContainer.Get(context));
IsDeleted.Set(context, AzureHelper.DeleteBlobFromAzureStorage(BlobReference.Get(context)));
} catch (Exception ex) {
if (Convert.ToBoolean((int)ContinueOnError))
{
}
else
{
throw ex;
}
}
}
public void BlobValid(BlobReference blobReference, bool valid)
{
}
public void RetrievedBlob(BlobReference blobReference)
{
}
/// <summary>
/// Writes contents to a specified blob in the specified GitHub repository.
/// </summary>
/// <param name="appConfig">The application configuration object which contains values
/// for connecting to the specified GitHub repository.</param>
/// <param name="privateKey">The RSA private key of a registered GitHub app installed in the specified repository.</param>
/// <returns>A task.</returns>
public static async Task WriteToRepositoryAsync(ApplicationConfig appConfig, string privateKey)
{
if (appConfig == null)
{
throw new ArgumentNullException(nameof(appConfig), "Parameter cannot be null");
}
if (string.IsNullOrEmpty(privateKey))
{
throw new ArgumentNullException(nameof(privateKey), "Parameter cannot be null or empty");
}
var gitHubClient = GitHubClientFactory.GetGitHubClient(appConfig, privateKey);
// Get repo references
var references = await gitHubClient.Git.Reference.GetAll(appConfig.GitHubOrganization, appConfig.GitHubRepoName);
// Check if the working branch is in the refs
var workingBranch = references.Where(reference => reference.Ref == $"refs/heads/{appConfig.WorkingBranch}").FirstOrDefault();
// Check if branch already exists
if (workingBranch == null)
{
// Working branch does not exist, so branch off from the reference branch
var refBranch = references.Where(reference => reference.Ref == $"refs/heads/{appConfig.ReferenceBranch}").FirstOrDefault();
// Create new branch; exception will throw if branch already exists
workingBranch = await gitHubClient.Git.Reference.Create(appConfig.GitHubOrganization, appConfig.GitHubRepoName,
new NewReference($"refs/heads/{appConfig.WorkingBranch}", refBranch.Object.Sha));
}
// Get reference of the working branch
var workingReference = await gitHubClient.Git.Reference.Get(appConfig.GitHubOrganization,
appConfig.GitHubRepoName,
workingBranch.Ref);
// Get the latest commit of this branch
var latestCommit = await gitHubClient.Git.Commit.Get(appConfig.GitHubOrganization,
appConfig.GitHubRepoName,
workingReference.Object.Sha);
// Create blob
NewBlob blob = new NewBlob {
Encoding = EncodingType.Utf8, Content = appConfig.FileContent
};
BlobReference blobRef = await gitHubClient.Git.Blob.Create(appConfig.GitHubOrganization,
appConfig.GitHubRepoName,
blob);
// Create new Tree
var tree = new NewTree {
BaseTree = latestCommit.Tree.Sha
};
var treeMode = (int)appConfig.TreeItemMode;
// Add items based on blobs
tree.Tree.Add(new NewTreeItem
{
Path = appConfig.FileContentPath,
Mode = treeMode.ToString(),
Type = TreeType.Blob,
Sha = blobRef.Sha
});
var newTree = await gitHubClient.Git.Tree.Create(appConfig.GitHubOrganization,
appConfig.GitHubRepoName,
tree);
// Create a commit
var newCommit = new NewCommit(appConfig.CommitMessage,
newTree.Sha,
workingReference.Object.Sha);
var commit = await gitHubClient.Git.Commit.Create(appConfig.GitHubOrganization,
appConfig.GitHubRepoName,
newCommit);
// Push the commit
await gitHubClient.Git.Reference.Update(appConfig.GitHubOrganization,
appConfig.GitHubRepoName,
workingBranch.Ref,
new ReferenceUpdate(commit.Sha));
}
/// <summary>
/// Alternative for UploadAndReferenceBlobAsync that uses WalkBlocksAsync instead of the synchronous WalkBlocks.
/// Also utilizes the AsyncHttpRetryHelper to mitigate transient exceptions.
/// </summary>
public static Task UploadAndReferenceBlobWithRetriesAsync(
this IBlobStoreHttpClient client,
BlobIdentifier blobId,
Stream stream,
BlobReference reference,
Context context,
CancellationToken cts)
{
Contract.Requires(stream != null);
var attempt = 0;
return(AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
bool blobUploaded = false;
stream.Position = 0;
attempt++;
await VsoHash.WalkBlocksAsync(
stream,
blockActionSemaphore: null,
multiBlocksInParallel: true,
singleBlockCallback:
async(block, blockLength, blockHash) =>
{
await client.PutSingleBlockBlobAndReferenceAsync(
blobId, block, blockLength, reference, cts).ConfigureAwait(false);
blobUploaded = true;
},
multiBlockCallback:
(block, blockLength, blockHash, isFinalBlock) =>
client.PutBlobBlockAsync(blobId, block, blockLength, cts),
multiBlockSealCallback: async blobIdWithBlocks =>
{
var failedRefs = await client.TryReferenceWithBlocksAsync(
new Dictionary <BlobIdentifierWithBlocks, IEnumerable <BlobReference> >
{
{ blobIdWithBlocks, new[] { reference } }
},
cancellationToken: cts).ConfigureAwait(false);
blobUploaded = !failedRefs.Any();
}
);
if (!blobUploaded)
{
throw new AsyncHttpRetryHelper.RetryableException($"Could not upload blob on attempt {attempt}.");
}
},
maxRetries: 5,
tracer: new AppTraceSourceContextAdapter(context, "BlobStoreHttpClient", SourceLevels.All),
canRetryDelegate: exception =>
{
// HACK HACK: This is an additional layer of retries specifically to catch the SSL exceptions seen in DM.
// Newer versions of Artifact packages have this retry automatically, but the packages deployed with the M119.0 release do not.
// Once the next release is completed, these retries can be removed.
if (exception is HttpRequestException && exception.InnerException is WebException)
{
return true;
}
while (exception != null)
{
if (exception is TimeoutException)
{
return true;
}
exception = exception.InnerException;
}
return false;
},
cancellationToken: cts,
continueOnCapturedContext: false,
context: context.Id.ToString()));
}
public async Task Apply(string owner, string repository, GhStandardFileSet fileSet)
{
Repository repo = await _client.Repository.Get(owner, repository);
// Determine if auto-content branch exists
string checkBranch = repo.DefaultBranch;
var branchName = _arguments.BranchName;
try
{
await _client.Repository.Branch.Get(repo.Id, branchName);
checkBranch = branchName;
Log.Information("{Repository}: branch '{BranchName}' exists, using that as base instead of '{DefaultBranch}'", repo.FullName, branchName, repo.DefaultBranch);
}
catch (NotFoundException)
{
}
// Diff files
List <string> upToDate = new List <string>();
Dictionary <string, byte[]> files = fileSet.GetFiles().ToDictionary(s => s.path, s => s.value);
{
foreach ((string path, byte[] value) in files)
{
try
{
byte[] existing = await _client.Repository.Content.GetRawContentByRef(owner, repository, path, checkBranch);
Utility.NormalizeNewlines(ref existing);
if (existing.SequenceEqual(value))
{
upToDate.Add(path);
}
}
catch (NotFoundException)
{
}
}
if (upToDate.Count == files.Count)
{
Log.Information("{Repository}: is up-to-date", repo.FullName);
return;
}
}
foreach (string path in files.Keys.Except(upToDate))
{
Log.Information("{Repository}: '{path}' is outdated", repo.FullName, path);
}
NewTree newTree = new NewTree();
foreach ((string path, byte[] value) in files)
{
BlobReference newBlob = await _client.Git.Blob.Create(repo.Id, new NewBlob
{
Content = Convert.ToBase64String(value),
Encoding = EncodingType.Base64
});
newTree.Tree.Add(new NewTreeItem
{
Type = TreeType.Blob,
Mode = "100644",
Path = path,
Sha = newBlob.Sha
});
}
Reference headReference = await _client.Git.Reference.Get(repo.Id, $"heads/{repo.DefaultBranch}");
string headCommit = headReference.Object.Sha;
TreeResponse previousTree = await _client.Git.Tree.Get(repo.Id, headReference.Ref);
newTree.BaseTree = previousTree.Sha;
TreeResponse newTreeResponse = await _client.Git.Tree.Create(repo.Id, newTree);
Commit createdCommit = await _client.Git.Commit.Create(repo.Id, new NewCommit("Updating standard content files for repository", newTreeResponse.Sha, headCommit)
{
Author = new Committer(_arguments.CommitAuthor, _arguments.CommitEmail, DateTimeOffset.UtcNow)
});
Reference existingBranch = null;
try
{
existingBranch = await _client.Git.Reference.Get(repo.Id, _branchNameRef);
}
catch (NotFoundException)
{
}
if (existingBranch != null)
{
Log.Information("{Repository}: Force-pushing to '{BranchName}'", repo.FullName, branchName);
// Update / force-push
await _client.Git.Reference.Update(repo.Id, _branchNameRef, new ReferenceUpdate(createdCommit.Sha, true));
}
else
{
Log.Information("{Repository}: Creating '{BranchName}'", repo.FullName, branchName);
// Create
await _client.Git.Reference.Create(repo.Id, new NewReference(_branchNameRef, createdCommit.Sha));
// Create PR
PullRequest newPr = await _client.Repository.PullRequest.Create(repo.Id, new NewPullRequest("Auto: Updating standardized files", branchName, repo.DefaultBranch));
Log.Information("{Repository}: PR created #{PrNumber} - {Title}", repo.FullName, newPr.Number, newPr.Title);
}
}
public async Task DeploySite()
{
StatusUpdate?.Invoke("Starting deploy...", StatusType.Status);
if (_githubToken == string.Empty)
{
// TODO: Send an error message
StatusUpdate?.Invoke("No GitHub token configured.", StatusType.Error);
return;
}
if (_repoName == string.Empty)
{
StatusUpdate?.Invoke("No repository configured.", StatusType.Error);
return;
}
string[] splitRepoName = _repoName.Split('/');
if (splitRepoName.Length != 2)
{
StatusUpdate?.Invoke("Repository name format invalid.", StatusType.Error);
return;
}
if (_repoDestination == string.Empty)
{
StatusUpdate?.Invoke("No destination configured.", StatusType.Error);
return;
}
StatusUpdate?.Invoke("Getting repository...", StatusType.Status);
long repoId = (await _github.Repository.Get(splitRepoName[0], splitRepoName[1])).Id;
string headMasterRef = "heads/" + _repoBranch;
Reference masterReference;
try
{
masterReference = await _github.Git.Reference.Get(repoId, headMasterRef);
}
catch (Exception ex)
{
if (ex is NotFoundException)
{
// This probably means the branch hasn't been created yet
// https://github.com/octokit/octokit.net/issues/1098
StatusUpdate?.Invoke("Creating branch " + _repoBranch + "...", StatusType.Status);
Reference master = await _github.Git.Reference.Get(repoId, "heads/master");
masterReference = await _github.Git.Reference.Create(repoId,
new NewReference("refs/" + headMasterRef, master.Object.Sha));
}
else
{
throw;
}
}
StatusUpdate?.Invoke("Getting current commit...", StatusType.Status);
GitHubCommit currentCommit = (await _github.Repository.Commit.GetAll(repoId)).First();
string currentTreeSha = currentCommit.Commit.Tree.Sha;
NewTree newTree = new NewTree();
if (_repoDirectory != string.Empty)
{
TreeResponse currentTree = await _github.Git.Tree.Get(repoId, currentTreeSha);
// Copy the current tree to the new tree
StatusUpdate?.Invoke("Copying current tree to new tree...", StatusType.Status);
newTree = (await CopySubtree(_github, currentTree, repoId, _repoDirectory)).Tree;
}
// Now, create all of the files in the tree
StatusUpdate?.Invoke("Uploading compiled pages and adding to tree...", StatusType.Status);
await _context.CompiledPages.ToList().ForEachAsync(async page =>
{
NewBlob newBlob = new NewBlob
{
Content = MiscUtils.Base64Encode(page.Contents),
Encoding = EncodingType.Base64
};
BlobReference newBlobCreated = await _github.Git.Blob.Create(repoId, newBlob);
string newBlobSha = newBlobCreated.Sha;
newTree.Tree.Add(new NewTreeItem
{
Mode = "100644",
Type = TreeType.Blob,
Sha = newBlobSha,
Path = Path.Join(_repoDirectory, page.Title + ".html")
});
});
// Upload all of the files from the wwwroot directory
StatusUpdate?.Invoke("Uploading other files and adding to tree...", StatusType.Status);
(await CopyDirectoryIntoTree(_github, repoId, _repoDirectory)).ForEach(item => { newTree.Tree.Add(item); });
StatusUpdate?.Invoke("Creating commit...", StatusType.Status);
string newTreeSha = (await _github.Git.Tree.Create(repoId, newTree)).Sha;
NewCommit newCommit = new NewCommit("Automated deploy from webweb", newTreeSha, masterReference.Object.Sha);
Commit createdCommit = await _github.Git.Commit.Create(repoId, newCommit);
await _github.Git.Reference.Update(repoId, headMasterRef, new ReferenceUpdate(createdCommit.Sha));
StatusUpdate?.Invoke("Deploy complete.", StatusType.Success);
}
public PageCreated(Guid crawlId, Guid siteId, Guid pageId, string url, BlobReference blobReference, DateTime timeOfCreation)
: base(crawlId)
{
SiteId = siteId;
PageId = pageId;
Url = url;
BlobReference = blobReference;
TimeOfCreation = timeOfCreation;
}
public PageContentChanged(Guid siteId, Guid pageId, BlobReference blobReference, DateTime timeOfChange)
: base(siteId)
{
PageId = pageId;
BlobReference = blobReference;
TimeOfChange = timeOfChange;
}
protected override async Task <IEnumerable <IDocument> > ExecuteConfigAsync(
IDocument input,
IExecutionContext context,
IMetadata values)
{
// See http://www.levibotelho.com/development/commit-a-file-with-the-github-api/
// Set up the client
GitHubClient github = new GitHubClient(new ProductHeaderValue("Statiq"), GitHubClient.GitHubApiUrl);
if (values.TryGetValue(Token, out string token))
{
github.Credentials = new Credentials(token);
}
else if (values.TryGetValue(Username, out string username) && values.TryGetValue(Password, out string password))
{
github.Credentials = new Credentials(username, password);
}
else
{
throw new ExecutionException("Could not determine GitHub credentials");
}
if (!values.TryGetValue(Owner, out string owner) ||
string.IsNullOrEmpty(owner) ||
!values.TryGetValue(Name, out string name) ||
string.IsNullOrEmpty(name))
{
throw new ExecutionException("Invalid repository owner or name");
}
// Get the current head tree
string branch = values.GetString(Branch, DefaultBranch);
Reference reference = await github.Git.Reference.Get(owner, name, "heads/" + branch);
Commit commit = await github.Git.Commit.Get(owner, name, reference.Object.Sha);
// Iterate the output path, adding new tree items
// Don't reference a base tree so that any items not reflected in the new tree will be deleted
NormalizedPath sourcePath = values.GetPath(SourcePath, context.FileSystem.GetOutputPath());
NewTree newTree = new NewTree();
foreach (string outputFile in Directory.GetFiles(sourcePath.FullPath, "*", SearchOption.AllDirectories))
{
// Upload the blob
BlobReference blob = await github.Git.Blob.Create(owner, name, new NewBlob
{
Content = Convert.ToBase64String(await File.ReadAllBytesAsync(outputFile)),
Encoding = EncodingType.Base64
});
// Add the new blob to the tree
string relativePath = Path.GetRelativePath(sourcePath.FullPath, outputFile).Replace("\\", "/");
newTree.Tree.Add(new NewTreeItem
{
Path = relativePath,
Mode = "100644",
Type = TreeType.Blob,
Sha = blob.Sha
});
}
// Create the new tree
TreeResponse newTreeResponse = await github.Git.Tree.Create(owner, name, newTree);
// Create the commit
Commit newCommit = await github.Git.Commit.Create(owner, name, new NewCommit($"Deployment from Statiq", newTreeResponse.Sha, commit.Sha));
// Update the head ref
await github.Git.Reference.Update(owner, name, reference.Ref, new ReferenceUpdate(newCommit.Sha, true));
return(await input.YieldAsync());
}
/*
* If ContinueOnError argument is 'true', return the exception else nope.
*/
private void UploadToAzureStorage(CodeActivityContext context)
{
try {
AzureParamHelper azureStorageHelper = new AzureParamHelper(StorageConnectionString.Get(context), StorageBlobContainer.Get(context));
BlobUri.Set(context, AzureHelper.UploadToAzureStorage(SourceFile.Get(context), BlobReference.Get(context), Convert.ToBoolean((int)DeleteSourceAfterUpload)));
} catch (Exception ex) {
if (Convert.ToBoolean((int)ContinueOnError))
{
}
else
{
throw ex;
}
}
}
public void BlobValid(BlobReference blobReference, bool valid)
{
Console.WriteLine(valid
? $"Valid blob: {blobReference.Name}"
: $"Invalid blob: {blobReference.Name}");
}
public void RetrievedBlob(BlobReference blobReference)
{
Console.WriteLine($"Restored blob: {blobReference.Name}");
}
public void StoredBlob(BlobReference blobReference)
{
Console.WriteLine($"Stored blob: {blobReference.Name}");
}
public void StoredBlob(BlobReference blobReference)
{
}
public async Task WriteToRepositoryAsync(ApplicationConfig appConfig, string privateKey)
{
if (appConfig == null)
{
throw new ArgumentNullException(nameof(appConfig), "Parameter cannot be null");
}
try
{
string token = await GitHubAuthService.GetGithubAppTokenAsync(appConfig, privateKey);
// Pass the JWT as a Bearer token to Octokit.net
var finalClient = new GitHubClient(new ProductHeaderValue(appConfig.GitHubAppName))
{
Credentials = new Credentials(token, AuthenticationType.Bearer)
};
// Get repo references
var references = await finalClient.Git.Reference.GetAll(appConfig.GitHubOrganization, appConfig.GitHubRepoName);
// Check if the working branch is in the refs
var workingBranch = references.Where(reference => reference.Ref == $"refs/heads/{appConfig.WorkingBranch}").FirstOrDefault();
// Check if branch already exists.
if (workingBranch == null)
{
// Working branch does not exist so branch off reference branch
var refBranch = references.Where(reference => reference.Ref == $"refs/heads/{appConfig.ReferenceBranch}").FirstOrDefault();
// Exception will throw if branch already exists
var newBranch = await finalClient.Git.Reference.Create(appConfig.GitHubOrganization, appConfig.GitHubRepoName,
new NewReference($"refs/heads/{appConfig.WorkingBranch}", refBranch.Object.Sha));
// create file
var createChangeSet = await finalClient.Repository.Content.CreateFile(
appConfig.GitHubOrganization,
appConfig.GitHubRepoName,
appConfig.FileContentPath,
new CreateFileRequest(appConfig.CommitMessage,
appConfig.FileContent,
appConfig.WorkingBranch));
}
else
{
// Get reference of the working branch
var masterReference = await finalClient.Git.Reference.Get(appConfig.GitHubOrganization,
appConfig.GitHubRepoName, workingBranch.Ref);
// Get the latest commit of this branch
var latestCommit = await finalClient.Git.Commit.Get(appConfig.GitHubOrganization, appConfig.GitHubRepoName,
masterReference.Object.Sha);
// Create blob
NewBlob blob = new NewBlob {
Encoding = EncodingType.Utf8, Content = appConfig.FileContent
};
BlobReference blobRef =
await finalClient.Git.Blob.Create(appConfig.GitHubOrganization, appConfig.GitHubRepoName, blob);
// Create new Tree
var tree = new NewTree {
BaseTree = latestCommit.Tree.Sha
};
// Add items based on blobs
tree.Tree.Add(new NewTreeItem
{
Path = appConfig.FileContentPath, Mode = appConfig.TreeItemMode.ToString(), Type = TreeType.Blob, Sha = blobRef.Sha
});
var newTree = await finalClient.Git.Tree.Create(appConfig.GitHubOrganization, appConfig.GitHubRepoName, tree);
// Create commit
var newCommit = new NewCommit(appConfig.CommitMessage, newTree.Sha,
masterReference.Object.Sha);
var commit =
await finalClient.Git.Commit.Create(appConfig.GitHubOrganization, appConfig.GitHubRepoName, newCommit);
// Push the commit
await finalClient.Git.Reference.Update(appConfig.GitHubOrganization, appConfig.GitHubRepoName, workingBranch.Ref,
new ReferenceUpdate(commit.Sha));
}
// Create PR
var pullRequest = await finalClient.Repository.PullRequest.Create(appConfig.GitHubOrganization,
appConfig.GitHubAppName,
new NewPullRequest(appConfig.PullRequestTitle, appConfig.WorkingBranch, appConfig.ReferenceBranch) { Body = appConfig.PullRequestBody });
// Add reviewers
var teamMembers = appConfig.Reviewers;
var reviewersResult = await finalClient.Repository.PullRequest.ReviewRequest.Create(appConfig.GitHubOrganization,
appConfig.GitHubRepoName, pullRequest.Number,
new PullRequestReviewRequest(teamMembers.AsReadOnly(), null));
// Add label
var issueUpdate = new IssueUpdate();
issueUpdate.AddAssignee(appConfig.PullRequestAssignee);
issueUpdate.AddLabel(appConfig.PullRequestLabel);
// Update the PR with the relevant info
await finalClient.Issue.Update(appConfig.GitHubOrganization, appConfig.GitHubRepoName, pullRequest.Number,
issueUpdate);
}
catch (Exception ex)
{
throw ex;
}
}
async Task InvokeBatchExecutionService(string accountNameInput, string accountKeyInput, string containerInput, string inputBlobName, string inputFileName, OutputObject outputObj)
{
// Upload file to Blob if pathFile given
if (isHasInput && input_radio_File.Checked)
{
accountNameInput = outputObj.AccountName;
if (string.IsNullOrEmpty(accountNameInput))
{
//Console.WriteLine("Upload Error");
ShowError("Please enter Account Name");
return;
}
accountKeyInput = outputObj.AccountKey;
if (string.IsNullOrEmpty(accountKeyInput))
{
//Console.WriteLine("Upload Error");
ShowError("Please enter Account Key");
return;
}
containerInput = outputObj.Container;
if (string.IsNullOrEmpty(containerInput))
{
//Console.WriteLine("Upload Error");
ShowError("Please enter Container Name");
return;
}
if (string.IsNullOrEmpty(_fileUpload.FileName))
{
//Console.WriteLine("Upload Error");
ShowError("Please choose input file");
return;
}
inputBlobName = inputFileName;
//ScriptManager.RegisterStartupScript(Page, Page.GetType(), "disable", "$(\"#btnclear\").prop(\"disabled\",true);", true);
bool uploadresult = await uploadBigFile(accountNameInput, accountKeyInput, containerInput, inputFileName);
//$("#btnclear").prop("disabled",true)
if (!uploadresult)
{
//Console.WriteLine("Upload Error");
ShowError("Upload Error");
return;
}
}
// First collect and fill in the URI and access key for your web service endpoint.
// These are available on your service's API help page.
var endpointUri = paramObj.Url;
string accessKey = Crypto.DecryptStringAES(paramObj.APIKey);
// Create an Azure Machine Learning runtime client for this endpoint
var runtimeClient = new RuntimeClient(endpointUri, accessKey);
// Define the request information for your batch job. This information can contain:
// -- A reference to the AzureBlob containing the input for your job run
// -- A set of values for global parameters defined as part of your experiment and service
// -- A set of output blob locations that allow you to redirect the job's results
// NOTE: This sample is applicable, as is, for a service with explicit input port and
// potential global parameters. Also, we choose to also demo how you could override the
// location of one of the output blobs that could be generated by your service. You might
// need to tweak these features to adjust the sample to your service.
//
// All of these properties of a BatchJobRequest shown below can be optional, depending on
// your service, so it is not required to specify all with any request. If you do not want to
// use any of the parameters, a null value should be passed in its place.
// Define the reference to the blob containing your input data. You can refer to this blob by its
// connection string / container / blob name values; alternatively, we also support references
// based on a blob SAS URI
string ext = ".csv";//inputBlobName.Substring(inputBlobName.LastIndexOf("."));
BlobReference inputBlob;
if (isHasInput)
{
inputBlob = BlobReference.CreateFromConnectionStringData(
connectionString: string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", accountNameInput, accountKeyInput),
containerName: containerInput,
blobName: inputBlobName);
ext = inputBlobName.Substring(inputBlobName.LastIndexOf("."));
}
else
{
inputBlob = null;
}
// If desired, one can override the location where the job outputs are to be stored, by passing in
// the storage account details and name of the blob where we want the output to be redirected to.
var outputLocations = new Dictionary <string, BlobReference>();
foreach (var keyvalue in outputObj.NodeOutputName)
{
outputLocations.Add(
keyvalue.Key,
BlobReference.CreateFromConnectionStringData(
connectionString: string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", outputObj.AccountName, outputObj.AccountKey),
containerName: outputObj.Container,
blobName: !outputObj.isAddTime ? keyvalue.Value + "_" + DateTime.Now.ToString("MMddyy_hhmmss") + ext : keyvalue.Value + ext
));
}
;
// If applicable, you can also set the global parameters for your service
var globalParameters = new Dictionary <string, string>();
foreach (var global in paramObj.listGlobalParameter)
{
string columnValue = "";
var control = FindControl(global.Name);
if (control is TextBox)
{
TextBox txt = control as TextBox;
if (txt.Text != "")
{
columnValue = txt.Text;
}
}
else if (control is DropDownList)
{
DropDownList lb = control as DropDownList;
if (lb.SelectedIndex != -1)
{
columnValue = lb.SelectedValue;
}
}
if (control is RadioButtonList)
{
RadioButtonList ct = control as RadioButtonList;
if (ct.SelectedIndex != -1)
{
columnValue = ct.SelectedValue;
}
}
globalParameters.Add(global.Name, columnValue);
}
var jobRequest = new BatchJobRequest
{
Input = inputBlob,
GlobalParameters = globalParameters,
Outputs = outputLocations
};
try
{
// Register the batch job with the system, which will grant you access to a job object
BatchJob job = await runtimeClient.RegisterBatchJobAsync(jobRequest);
AddJobIdCookie(job.Id, job.CreatedAt.ToLocalTime().ToString());
// Start the job to allow it to be scheduled in the running queue
await job.StartAsync();
//ScriptManager.RegisterStartupScript(Page, typeof(Page), "OpenWindow", "window.open('Status.aspx?jobid=" + job.Id + "');", true);
Response.Redirect("Status.aspx?jobid=" + job.Id);
// Wait for the job's completion and handle the output
//BatchJobStatus jobStatus = await job.WaitForCompletionAsync();
//while (job.CheckStatus().JobState != JobState.Finished && job.CheckStatus().JobState != JobState.Failed)
//{
// Console.WriteLine(job.Id + ":" + job.CreatedAt.ToLocalTime() + job.CheckStatus().JobState);
//}
//BatchJobStatus jobStatus = job.CheckStatus();
////job.CreatedAt
//if (jobStatus.JobState == JobState.Finished)
//{
// // Process job outputs
// //Console.WriteLine(@"Job {0} has completed successfully and returned {1} outputs", job.Id, jobStatus.Results.Count);
// lblJobIdSuccess.Text = job.Id;
// ScriptManager.RegisterStartupScript(Page, Page.GetType(), "CompleteSuccess", "$('#CompleteSuccess').modal();", true);
// foreach (var output in jobStatus.Results)
// {
// //Console.WriteLine(@"\t{0}: {1}", output.Key, output.Value.AbsoluteUri);
// Response.Redirect(output.Value.AbsoluteUri);
// }
//}
//else if (jobStatus.JobState == JobState.Failed)
//{
// // Handle job failure
// //Console.WriteLine(@"Job {0} has failed with this error: {1}", job.Id, jobStatus.Details);
// txtresultModal.Text = jobStatus.Details;
// ScriptManager.RegisterStartupScript(Page, Page.GetType(), "failModal", "$('#failModal').modal();", true);
//}
}
catch (ArgumentException ex)
{
//Console.WriteLine("Argument {0} is invalid: {1}", aex.ParamName, aex.Message);
//txtresultModal.Text = ex.Message;
//ScriptManager.RegisterStartupScript(Page, Page.GetType(), "failModal", "$('#failModal').modal();", true);
ShowError(ex.Message);
}
catch (RuntimeException runtimeError)
{
//Console.WriteLine("Runtime error occurred: {0} - {1}", runtimeError.ErrorCode, runtimeError.Message);
//Console.WriteLine("Error details:");
string error = "";
foreach (var errorDetails in runtimeError.Details)
{
error += string.Format("\t{0} - {1}", errorDetails.Code, errorDetails.Message);
}
//txtresultModal.Text = error;
//ScriptManager.RegisterStartupScript(Page, Page.GetType(), "failModal", "$('#failModal').modal();", true);
ShowError(error);
}
catch (Exception ex)
{
//Console.WriteLine("Unexpected error occurred: {0} - {1}", ex.GetType().Name, ex.Message);
//txtresultModal.Text = ex.Message;
//ScriptManager.RegisterStartupScript(Page, Page.GetType(), "failModal", "$('#failModal').modal();", true);
ShowError(ex.Message);
}
}
