/// <summary> /// Clean usage database. /// </summary> public static void CleanUsageDatabase() { StorageUtilities.DeleteAllRows( server: Server, database: UsageDatabase, tableNames: string.Format(CultureInfo.InvariantCulture, "[{0}].[UsageRecords]", DefaultUsageSchemaName)); }
/// <summary> /// Clean events database. /// </summary> public static void CleanEventsDatabase() { StorageUtilities.DeleteAllRows( server: Server, database: EventsDatabase, tableNames: string.Format(CultureInfo.InvariantCulture, "[{0}].[Events]", DefaultEventSchemaName)); }
private FileInfo ResizeVhdFile(long FileSizeBefore, long FileSize) { CheckOS(); WriteWarning("The VHD file needs to be resized. During the resizing process, the cmdlet will temporarily create a resized file in the same directory as the provided VHD/VHDX file."); try { ManagementScope scope = new ManagementScope(@"\root\virtualization\V2"); long FullFileSize = FileSize + 512; using (ManagementObject imageManagementService = StorageUtilities.GetImageManagementService(scope)) { CreateBackUp(vhdFileToBeUploaded.FullName); WriteVerbose("Resizing VHD file"); using (ManagementBaseObject inParams = imageManagementService.GetMethodParameters("ResizeVirtualHardDisk")) { inParams["Path"] = vhdFileToBeUploaded.FullName; inParams["MaxInternalSize"] = FileSize; using (ManagementBaseObject outParams = imageManagementService.InvokeMethod( "ResizeVirtualHardDisk", inParams, null)) { ManagementPath path = new ManagementPath((string)outParams["Job"]); ManagementObject job = new ManagementObject(path); string jobStatus = (string)job["JobStatus"]; ushort percentComplete = (ushort)job["PercentComplete"]; while (jobStatus == "Job is running" && percentComplete < 100) { Program.SyncOutput.ProgressHyperV(percentComplete, "Resizing VHD"); Thread.Sleep(1000); job.Get(); jobStatus = (string)job["JobStatus"]; percentComplete = (ushort)job["PercentComplete"]; } Program.SyncOutput.ProgressHyperV(percentComplete, "Resizing VHD"); WmiUtilities.ValidateOutput(outParams, scope); } } WriteVerbose("Resized " + vhdFileToBeUploaded + " from " + FileSizeBefore + " bytes to " + FullFileSize + " bytes."); temporaryFileCreated = true; return(new FileInfo(vhdFileToBeUploaded.FullName)); } } catch (System.Management.ManagementException ex) { if (ex.Message == "Invalid namespace ") { Exception outputEx = new Exception("Failed to resize VHD file. Hyper-V Platform is not found.\nFollow this link to enable Hyper-V or resize file manually: https://aka.ms/usingAdd-AzVhd"); ThrowTerminatingError(new ErrorRecord( outputEx, "Hyper-V is unavailable", ErrorCategory.InvalidOperation, null)); } throw ex; } }
internal static async Task ProcessReportFileAsync(string reportFileContent, ILogger log) { var transcriptionReportFile = JsonConvert.DeserializeObject <TranscriptionReportFile>(reportFileContent); var failedTranscriptions = transcriptionReportFile.Details. Where(detail => !string.IsNullOrEmpty(detail.Status) && detail.Status.Equals("Failed", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrEmpty(detail.SourceUrl)); foreach (var failedTranscription in failedTranscriptions) { var fileName = StorageUtilities.GetFileNameFromUri(new Uri(failedTranscription.SourceUrl)); var message = $"Transcription \"{fileName}\" failed with error \"{failedTranscription.ErrorKind}\" and message \"{failedTranscription.ErrorMessage}\""; log.LogError(message); var errorTxtname = fileName + ".txt"; await StorageUtilities.WriteTextFileToBlobAsync( FetchTranscriptionEnvironmentVariables.AzureWebJobsStorage, message, FetchTranscriptionEnvironmentVariables.ErrorReportOutputContainer, errorTxtname, log).ConfigureAwait(false); } }
private async void SearchSavedFolders(string searchString = "") { List <DocumentFolder> lstDocFolders = new List <DocumentFolder>(); string RootFolderExternalStorage = StorageUtilities.GetInstance().GetAppExternalStorageFolderName(); PermissionStatus readPermission = await PermissionUtilities.GetInstance().RequestExtrenalStorageReadPermission(); if (Directory.Exists(RootFolderExternalStorage) && readPermission == PermissionStatus.Granted) { string[] folderNamesList = Directory.GetDirectories(RootFolderExternalStorage); foreach (string folderName in folderNamesList) { DocumentFolder docFolder = new DocumentFolder(); DirectoryInfo dir = new DirectoryInfo(folderName); docFolder.FolderName = dir.Name; docFolder.FolderPath = dir.FullName; lstDocFolders.Add(docFolder); } //--For Search: lstDocFolders = lstDocFolders.FindAll(d => d.FolderName.ToLower().Contains(searchString.ToLower())); _lstFolders = new ObservableCollection <DocumentFolder>(lstDocFolders); lstFolders.ItemsSource = _lstFolders; lstFolders.SelectedItem = null; } }
private void OnExecute() { var dateClassifier = new DateClassifier(); if (WorkingDirectory == null) { WorkingDirectory = Directory.GetCurrentDirectory(); } var storageUtilities = new StorageUtilities(); var manager = new StorageManager(dateClassifier, storageUtilities); var dateClassifierArgs = new DateClassifierArgs { Interval = Interval, DirectoryPath = WorkingDirectory, RecursiveMode = Recursive ? RecursiveMode.RootFolder : RecursiveMode.None, UseMultipleClassifiers = UseMultipleClassifiers }; if (MoveToRoot) { Console.WriteLine("Moving files to root"); storageUtilities.MoveFilesToRoot(dateClassifierArgs); return; } manager.OrganizeDirectory(dateClassifierArgs); }
public bool IsValidServiceBusMessage(Message message) { if (message == null) { Logger.LogError($"Message {nameof(message)} is null."); return(false); } var messageBody = Encoding.UTF8.GetString(message.Body); try { var serviceBusMessage = JsonConvert.DeserializeObject <ServiceBusMessage>(messageBody); if (serviceBusMessage.EventType.Contains("BlobCreate", StringComparison.OrdinalIgnoreCase) && StorageUtilities.GetContainerNameFromUri(serviceBusMessage.Data.Url).Equals(StartTranscriptionEnvironmentVariables.AudioInputContainer, StringComparison.Ordinal)) { return(true); } } catch (JsonSerializationException e) { Logger.LogError($"Exception {e.Message} while parsing message {messageBody} - message will be ignored."); return(false); } return(false); }
public static PSStorageService Create(StorageManagementClient client, StorageServicePropertiesOperationContext account) { var cloudStorageAccount = StorageUtilities.GenerateCloudStorageAccount(client, account.StorageAccountName); return(new PSStorageService(account, new AzureStorageContext(cloudStorageAccount))); }
public bool Save(FlyScene scene) { try { string directory = SaveFolder; Directory.CreateDirectory(directory); string fullPath = Path.Combine(directory, "test.flys"); using (var stream = new MemoryStream()) { var context = new BinarySerializationContext(stream); Save(scene, context); stream.Flush(); StorageUtilities.CopyToFile(stream, fullPath); } Console.WriteLine("Saved!"); return(true); } catch (Exception ex) { Console.WriteLine(ex.Message); return(false); } }
internal static async Task ProcessFailedTranscriptionAsync(BatchClient client, Transcription transcription, Guid transcriptionId, string jobName, ILogger log) { log.LogInformation($"Got failed transcription for job {jobName}"); var report = "Unknown Error."; var errorTxtname = jobName + ".txt"; var transcriptionFiles = await client.GetTranscriptionFilesAsync(transcriptionId).ConfigureAwait(false); var reportFile = transcriptionFiles.Values.Where(t => t.Kind == TranscriptionFileKind.TranscriptionReport).FirstOrDefault(); var reportFileContent = string.Empty; if (reportFile?.Links?.ContentUrl != null) { using var webClient = new WebClient(); reportFileContent = webClient.DownloadString(reportFile.Links.ContentUrl); report = reportFileContent; } else if (transcription.Properties?.Error?.Message != null) { report = transcription.Properties.Error.Message; report += string.IsNullOrEmpty(transcription.Properties.Error.Code) ? string.Empty : $" (Error: {transcription.Properties.Error.Code})."; } var errorOutputContainer = FetchTranscriptionEnvironmentVariables.ErrorReportOutputContainer; await StorageUtilities.WriteTextFileToBlobAsync(FetchTranscriptionEnvironmentVariables.AzureWebJobsStorage, report, errorOutputContainer, errorTxtname, log).ConfigureAwait(false); if (!string.IsNullOrEmpty(reportFileContent)) { await ProcessReportFileAsync(reportFileContent, log).ConfigureAwait(false); } await client.DeleteTranscriptionAsync(transcriptionId).ConfigureAwait(false); }
private static void TrackWorkCompleted(WorkEntity workTrackingEntity, System.Collections.Generic.Dictionary <Vector3, int> vertexCounts) { workTrackingEntity.MetadataBase64 = SerializationUtilities.EncodeMetadataToBase64(vertexCounts); workTrackingEntity.CompletedTime = DateTime.UtcNow; StorageUtilities.UpdateWorkCompletedMetadata(TableClient, workTrackingEntity); }
public static PSStorageService Create(StorageManagementClient client, StorageServicePropertiesOperationContext account) { return(new PSStorageService(account, new LazyAzureStorageContext((s) => { return StorageUtilities.GenerateCloudStorageAccount(client, account.StorageAccountName); }, account.StorageAccountName) as AzureStorageContext)); }
private void CheckForComplete(SlicingOptions options, CubeManager manager) { int expectedResults = options.TextureSliceX * options.TextureSliceY; if (StorageUtilities.GetWorkCompletedCount(TableClient, options.CloudResultPath, options.CloudResultContainer) != expectedResults) { return; } var workResults = StorageUtilities.GetWorkCompletedMetadata(TableClient, options.CloudResultPath, options.CloudResultContainer); // Write metadata CubeMetadata metadata = new CubeMetadata(options.CubeGrid) { WorldBounds = manager.ObjInstance.Size, VirtualWorldBounds = options.ForceCubicalCubes ? manager.ObjInstance.CubicalSize : manager.ObjInstance.Size, VertexCount = manager.ObjInstance.VertexList.Count }; // Configure texture slicing metadata if (!string.IsNullOrEmpty(options.Texture) && (options.TextureSliceX + options.TextureSliceY) > 2) { metadata.TextureSetSize = new Vector2(options.TextureSliceX, options.TextureSliceY); } else { metadata.TextureSetSize = new Vector2(1, 1); } var resultsList = workResults.Select(w => SerializationUtilities.DecodeMetadataFromBase64( Texture.GetCubeListFromTextureTile(options.TextureSliceY, options.TextureSliceX, w.TextureTileX, w.TextureTileY, manager.ObjInstance), w.MetadataBase64)); foreach (var result in resultsList) { foreach (var cube in result.Keys) { metadata.CubeExists[cube.X, cube.Y, cube.Z] = result[cube]; } } // Write out some json metadata string metadataPath = Path.Combine(outputPath, "metadata.json"); if (File.Exists(metadataPath)) { File.Delete(metadataPath); } string metadataString = JsonConvert.SerializeObject(metadata); File.WriteAllText(metadataPath, metadataString); StorageUtilities.UpdateSetCompleted(TableClient, options.SetKey); }
public static PSStorageAccount Create(StorageModels.StorageAccount storageAccount, IStorageManagementClient client) { var result = new PSStorageAccount(storageAccount); var credentials = StorageUtilities.GenerateStorageCredentials(new ARMStorageProvider(client), result.ResourceGroupName, result.StorageAccountName); CloudStorageAccount account = new CloudStorageAccount(credentials, storageAccount.PrimaryEndpoints.Blob, storageAccount.PrimaryEndpoints.Queue, storageAccount.PrimaryEndpoints.Table, null); result.Context = new AzureStorageContext(account); return(result); }
public void GetContainerFromSasTest() { var containerName = StorageUtilities.GetContainerNameFromUri(new Uri(TestSasUri)); var fileName = StorageUtilities.GetFileNameFromUri(new Uri(TestSasUri)); var fileNameWithoutExtension = StorageUtilities.GetFileNameWithoutExtension(fileName); Assert.AreEqual("testContainer", containerName); Assert.AreEqual("testfolder/test.wav", fileName); Assert.AreEqual("testfolder/test", fileNameWithoutExtension); }
private static WorkEntity TrackWorkStarted(DateTime startTime, SlicingOptions slicingOptions) { var workTrackingEntity = new WorkEntity(slicingOptions.CloudResultPath, slicingOptions.CloudResultContainer, slicingOptions.TextureTile.X, slicingOptions.TextureTile.Y) { StartTime = startTime, }; StorageUtilities.InsertWorkStartedMetadata(TableClient, workTrackingEntity); return(workTrackingEntity); }
private async Task VerifySourceDataAsync(SlicingOptions slicingOptions, CancellationToken cancellationToken) { if (!File.Exists(slicingOptions.Obj)) { await StorageUtilities.DownloadBlobAsync(BlobClient, slicingOptions.Obj, slicingOptions.CloudObjPath, cancellationToken).ConfigureAwait(false); } if (!File.Exists(slicingOptions.Texture)) { await StorageUtilities.DownloadBlobAsync(BlobClient, slicingOptions.Texture, slicingOptions.CloudTexturePath, cancellationToken).ConfigureAwait(false); } }
protected string GetStorageKey() { string storageKey = string.Empty; if (!string.IsNullOrEmpty(StorageAccountName)) { var storageCredentials = StorageUtilities.GenerateStorageCredentials(this.StorageClient, this.ResourceGroupName, this.StorageAccountName); storageKey = storageCredentials.ExportBase64EncodedKey(); } return(storageKey); }
public static CloudStorageAccount GetCloudStorageAccount(this AzureSubscription subscription) { if (subscription == null) { return(null); } using (var storageClient = AzureSession.ClientFactory.CreateClient <StorageManagementClient>(subscription, AzureEnvironment.Endpoint.ServiceManagement)) { return(StorageUtilities.GenerateCloudStorageAccount( storageClient, subscription.GetProperty(AzureSubscription.Property.StorageAccount))); } }
private async void UploadButton_Click(object sender, RoutedEventArgs e) { var imageFile = await StorageUtilities.PickSingleImage(); if (imageFile == null) { return; } ImageFile = imageFile; Hide(); }
/// <summary> /// Make sure the storage account key is set. /// If user doesn't specify it in command line, we try to resolve the key for the user given the storage account. /// </summary> /// <param name="storageAccount">The storage account to list the key.</param> private void InitializeStorageAccountKey(StorageAccount storageAccount) { if (string.IsNullOrEmpty(this.StorageAccountKey)) { if (storageAccount == null) { throw new Exception(string.Format(CultureInfo.InvariantCulture, Properties.Resources.DiagnosticsExtensionFailedToListKeyForNoStorageAccount, this.StorageAccountName)); } var psStorageAccount = new PSStorageAccount(storageAccount); var credentials = StorageUtilities.GenerateStorageCredentials(this.StorageClient, psStorageAccount.ResourceGroupName, psStorageAccount.StorageAccountName); this.StorageAccountKey = credentials.ExportBase64EncodedKey(); } }
private async Task WriteFailedJobLogToStorageAsync(IEnumerable <ServiceBusMessage> serviceBusMessages, string errorMessage, string jobName) { var errorOutputContainer = StartTranscriptionEnvironmentVariables.ErrorReportOutputContainer; var errorFileName = jobName + ".txt"; await StorageUtilities.WriteTextFileToBlobAsync(StartTranscriptionEnvironmentVariables.AzureWebJobsStorage, errorMessage, errorOutputContainer, errorFileName, Logger).ConfigureAwait(false); foreach (var message in serviceBusMessages) { var fileName = StorageUtilities.GetFileNameFromUri(message.Data.Url); errorFileName = fileName + ".txt"; await StorageUtilities.WriteTextFileToBlobAsync(StartTranscriptionEnvironmentVariables.AzureWebJobsStorage, errorMessage, errorOutputContainer, errorFileName, Logger).ConfigureAwait(false); } }
private async void btnSave_Clicked(object sender, EventArgs e) { try { PermissionStatus status = await PermissionUtilities.GetInstance().RequestExtrenalStorageWritePermission(); if (status == PermissionStatus.Granted) { string FolderNameInput = string.Empty; if (string.IsNullOrEmpty(_ParentFolderName)) //Direct camera on from Landing Page { FolderNameInput = await DisplayPromptAsync("Name", "What Document is this?", initialValue : "SampleDoc", keyboard : Keyboard.Default); if (string.IsNullOrEmpty(FolderNameInput)) { FolderNameInput = DateTime.Now.ToFileTime().ToString(); } _ParentFolderName = FolderNameInput; } string PublicExternalFolderPath = StorageUtilities.GetInstance().GetAppExternalStorageFolderName(); //1st time case if (!Directory.Exists(PublicExternalFolderPath)) { Directory.CreateDirectory(PublicExternalFolderPath); } string SaveDirectory = Path.Combine(PublicExternalFolderPath, _ParentFolderName); if (!Directory.Exists(SaveDirectory)) { Directory.CreateDirectory(SaveDirectory); } string saveFileName = DateTime.Now.ToFileTime().ToString(); string SaveFilePath = Path.Combine(SaveDirectory, saveFileName + ".jpg"); File.Copy(_EditFile, SaveFilePath, true); GC.Collect(); await Navigation.PushAsync(new DocumentLandingPage()); } } catch (Exception ex) { await DisplayAlert("Error", ex.Message, "OK"); } }
public void CreateDefaultMongoStorage() { var configurationBuilder = new ConfigurationBuilder(); configurationBuilder.AddJsonFile("appsettings.json"); var configuration = configurationBuilder.Build(); var storage = StorageUtilities.CreateStorage("DotnetSpider.Mongo.MongoEntityStorage, DotnetSpider.Mongo", configuration) as MongoEntityStorage; Assert.NotNull(storage); Assert.Equal("mongodb://*****:*****@localhost/test", storage.ConnectionString); }
private void UploadResultData(SlicingOptions slicingOptions) { var files = Directory.GetFiles(outputPath, "*", SearchOption.AllDirectories); foreach (var file in files) { StorageUtilities.UploadBlob( BlobClient, file, Path.Combine(slicingOptions.CloudResultPath, file.Replace(outputPath, string.Empty).TrimStart(new char[] { '\\', '/' })).Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), slicingOptions.CloudResultContainer); File.Delete(file); } }
public static PSStorageAccount Create(StorageModels.StorageAccount storageAccount, IStorageManagementClient client) { var result = new PSStorageAccount(storageAccount); result.Context = new LazyAzureStorageContext((s) => { var credentials = StorageUtilities.GenerateStorageCredentials(new ARMStorageProvider(client), result.ResourceGroupName, s); return(new CloudStorageAccount(credentials, ARMStorageService.GetUri(storageAccount.PrimaryEndpoints.Blob), ARMStorageService.GetUri(storageAccount.PrimaryEndpoints.Queue), ARMStorageService.GetUri(storageAccount.PrimaryEndpoints.Table), ARMStorageService.GetUri(storageAccount.PrimaryEndpoints.File))); }, result.StorageAccountName) as AzureStorageContext; return(result); }
public override void ExecuteCmdlet() { CloudStorageAccount account; if (Context != null) { account = Context.StorageAccount; } else { account = StorageUtilities.GenerateCloudStorageAccount(new ARMStorageProvider(StorageClient), ResourceGroupName, Name); } // Clear the current storage account for both SM and RM GeneralUtilities.ClearCurrentStorageAccount(true); DefaultContext.SetCurrentStorageAccount(account.ToString(true)); WriteObject(account.Credentials.AccountName); }
public void CreateDefaultSqlServerStorage() { var configurationBuilder = new ConfigurationBuilder(); configurationBuilder.AddJsonFile("appsettings.json"); var configuration = configurationBuilder.Build(); var storage = StorageUtilities.CreateStorage("DotnetSpider.DataFlow.SqlServerEntityStorage, DotnetSpider", configuration) as SqlServerEntityStorage; Assert.NotNull(storage); Assert.Equal(StorageMode.InsertIgnoreDuplicate, storage.Mode); Assert.Equal("Database='mysql3';Data Source=localhost;password=1qazZAQ!;User ID=root;Port=3308;", storage.ConnectionString); Assert.False(storage.IgnoreCase); Assert.Equal(1000, storage.RetryTimes); Assert.True(storage.UseTransaction); }
protected override void ProcessRecord() { CloudStorageAccount account; if (Context != null) { account = Context.StorageAccount; } else { account = StorageUtilities.GenerateCloudStorageAccount(StorageClient, ResourceGroupName, StorageAccountName); } // Clear the current storage account for both SM and RM GeneralUtilities.ClearCurrentStorageAccount(true); DefaultContext.SetCurrentStorageAccount(account.ToString(true)); WriteObject(account.Credentials.AccountName); }
/// <summary> /// Initialize the storage account key if it's not specified. /// It can be defined in multiple places, we only take the one with higher precedence. And the precedence is: /// 1. The one we try to resolve within current subscription /// 2. The one defined in PrivateConfig in the configuration file /// </summary> public static string InitializeStorageAccountKey(IStorageManagementClient storageClient, string storageAccountName = null, string configurationPath = null) { string storageAccountKey = null; StorageAccount storageAccount = null; if (TryGetStorageAccount(storageClient, storageAccountName, out storageAccount)) { // Help user retrieve the storage account key var psStorageAccount = new PSStorageAccount(storageAccount); var credentials = StorageUtilities.GenerateStorageCredentials(storageClient, psStorageAccount.ResourceGroupName, psStorageAccount.StorageAccountName); storageAccountKey = credentials.ExportBase64EncodedKey(); } else { // Use the one defined in PrivateConfig storageAccountKey = GetStorageAccountInfoFromPrivateConfig(configurationPath, PrivConfKeyAttr); } return(storageAccountKey); }