/// <summary>
/// Clean usage database.
/// </summary>
public static void CleanUsageDatabase()
{
StorageUtilities.DeleteAllRows(
server: Server,
database: UsageDatabase,
tableNames: string.Format(CultureInfo.InvariantCulture, "[{0}].[UsageRecords]", DefaultUsageSchemaName));
}
/// <summary>
/// Clean events database.
/// </summary>
public static void CleanEventsDatabase()
{
StorageUtilities.DeleteAllRows(
server: Server,
database: EventsDatabase,
tableNames: string.Format(CultureInfo.InvariantCulture, "[{0}].[Events]", DefaultEventSchemaName));
}
private FileInfo ResizeVhdFile(long FileSizeBefore, long FileSize)
{
CheckOS();
WriteWarning("The VHD file needs to be resized. During the resizing process, the cmdlet will temporarily create a resized file in the same directory as the provided VHD/VHDX file.");
try
{
ManagementScope scope = new ManagementScope(@"\root\virtualization\V2");
long FullFileSize = FileSize + 512;
using (ManagementObject imageManagementService =
StorageUtilities.GetImageManagementService(scope))
{
CreateBackUp(vhdFileToBeUploaded.FullName);
WriteVerbose("Resizing VHD file");
using (ManagementBaseObject inParams =
imageManagementService.GetMethodParameters("ResizeVirtualHardDisk"))
{
inParams["Path"] = vhdFileToBeUploaded.FullName;
inParams["MaxInternalSize"] = FileSize;
using (ManagementBaseObject outParams = imageManagementService.InvokeMethod(
"ResizeVirtualHardDisk", inParams, null))
{
ManagementPath path = new ManagementPath((string)outParams["Job"]);
ManagementObject job = new ManagementObject(path);
string jobStatus = (string)job["JobStatus"];
ushort percentComplete = (ushort)job["PercentComplete"];
while (jobStatus == "Job is running" && percentComplete < 100)
{
Program.SyncOutput.ProgressHyperV(percentComplete, "Resizing VHD");
Thread.Sleep(1000);
job.Get();
jobStatus = (string)job["JobStatus"];
percentComplete = (ushort)job["PercentComplete"];
}
Program.SyncOutput.ProgressHyperV(percentComplete, "Resizing VHD");
WmiUtilities.ValidateOutput(outParams, scope);
}
}
WriteVerbose("Resized " + vhdFileToBeUploaded + " from " + FileSizeBefore + " bytes to " + FullFileSize + " bytes.");
temporaryFileCreated = true;
return(new FileInfo(vhdFileToBeUploaded.FullName));
}
}
catch (System.Management.ManagementException ex)
{
if (ex.Message == "Invalid namespace ")
{
Exception outputEx = new Exception("Failed to resize VHD file. Hyper-V Platform is not found.\nFollow this link to enable Hyper-V or resize file manually: https://aka.ms/usingAdd-AzVhd");
ThrowTerminatingError(new ErrorRecord(
outputEx,
"Hyper-V is unavailable",
ErrorCategory.InvalidOperation,
null));
}
throw ex;
}
}
internal static async Task ProcessReportFileAsync(string reportFileContent, ILogger log)
{
var transcriptionReportFile = JsonConvert.DeserializeObject <TranscriptionReportFile>(reportFileContent);
var failedTranscriptions = transcriptionReportFile.Details.
Where(detail => !string.IsNullOrEmpty(detail.Status) &&
detail.Status.Equals("Failed", StringComparison.OrdinalIgnoreCase) &&
!string.IsNullOrEmpty(detail.SourceUrl));
foreach (var failedTranscription in failedTranscriptions)
{
var fileName = StorageUtilities.GetFileNameFromUri(new Uri(failedTranscription.SourceUrl));
var message = $"Transcription \"{fileName}\" failed with error \"{failedTranscription.ErrorKind}\" and message \"{failedTranscription.ErrorMessage}\"";
log.LogError(message);
var errorTxtname = fileName + ".txt";
await StorageUtilities.WriteTextFileToBlobAsync(
FetchTranscriptionEnvironmentVariables.AzureWebJobsStorage,
message,
FetchTranscriptionEnvironmentVariables.ErrorReportOutputContainer,
errorTxtname,
log).ConfigureAwait(false);
}
}
private async void SearchSavedFolders(string searchString = "")
{
List <DocumentFolder> lstDocFolders = new List <DocumentFolder>();
string RootFolderExternalStorage = StorageUtilities.GetInstance().GetAppExternalStorageFolderName();
PermissionStatus readPermission = await PermissionUtilities.GetInstance().RequestExtrenalStorageReadPermission();
if (Directory.Exists(RootFolderExternalStorage) && readPermission == PermissionStatus.Granted)
{
string[] folderNamesList = Directory.GetDirectories(RootFolderExternalStorage);
foreach (string folderName in folderNamesList)
{
DocumentFolder docFolder = new DocumentFolder();
DirectoryInfo dir = new DirectoryInfo(folderName);
docFolder.FolderName = dir.Name;
docFolder.FolderPath = dir.FullName;
lstDocFolders.Add(docFolder);
}
//--For Search:
lstDocFolders = lstDocFolders.FindAll(d => d.FolderName.ToLower().Contains(searchString.ToLower()));
_lstFolders = new ObservableCollection <DocumentFolder>(lstDocFolders);
lstFolders.ItemsSource = _lstFolders;
lstFolders.SelectedItem = null;
}
}
private void OnExecute()
{
var dateClassifier = new DateClassifier();
if (WorkingDirectory == null)
{
WorkingDirectory = Directory.GetCurrentDirectory();
}
var storageUtilities = new StorageUtilities();
var manager = new StorageManager(dateClassifier, storageUtilities);
var dateClassifierArgs = new DateClassifierArgs
{
Interval = Interval,
DirectoryPath = WorkingDirectory,
RecursiveMode = Recursive ? RecursiveMode.RootFolder : RecursiveMode.None,
UseMultipleClassifiers = UseMultipleClassifiers
};
if (MoveToRoot)
{
Console.WriteLine("Moving files to root");
storageUtilities.MoveFilesToRoot(dateClassifierArgs);
return;
}
manager.OrganizeDirectory(dateClassifierArgs);
}
public bool IsValidServiceBusMessage(Message message)
{
if (message == null)
{
Logger.LogError($"Message {nameof(message)} is null.");
return(false);
}
var messageBody = Encoding.UTF8.GetString(message.Body);
try
{
var serviceBusMessage = JsonConvert.DeserializeObject <ServiceBusMessage>(messageBody);
if (serviceBusMessage.EventType.Contains("BlobCreate", StringComparison.OrdinalIgnoreCase) &&
StorageUtilities.GetContainerNameFromUri(serviceBusMessage.Data.Url).Equals(StartTranscriptionEnvironmentVariables.AudioInputContainer, StringComparison.Ordinal))
{
return(true);
}
}
catch (JsonSerializationException e)
{
Logger.LogError($"Exception {e.Message} while parsing message {messageBody} - message will be ignored.");
return(false);
}
return(false);
}
public static PSStorageService Create(StorageManagementClient client,
StorageServicePropertiesOperationContext account)
{
var cloudStorageAccount = StorageUtilities.GenerateCloudStorageAccount(client, account.StorageAccountName);
return(new PSStorageService(account, new AzureStorageContext(cloudStorageAccount)));
}
public bool Save(FlyScene scene)
{
try
{
string directory = SaveFolder;
Directory.CreateDirectory(directory);
string fullPath = Path.Combine(directory, "test.flys");
using (var stream = new MemoryStream())
{
var context = new BinarySerializationContext(stream);
Save(scene, context);
stream.Flush();
StorageUtilities.CopyToFile(stream, fullPath);
}
Console.WriteLine("Saved!");
return(true);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
return(false);
}
}
internal static async Task ProcessFailedTranscriptionAsync(BatchClient client, Transcription transcription, Guid transcriptionId, string jobName, ILogger log)
{
log.LogInformation($"Got failed transcription for job {jobName}");
var report = "Unknown Error.";
var errorTxtname = jobName + ".txt";
var transcriptionFiles = await client.GetTranscriptionFilesAsync(transcriptionId).ConfigureAwait(false);
var reportFile = transcriptionFiles.Values.Where(t => t.Kind == TranscriptionFileKind.TranscriptionReport).FirstOrDefault();
var reportFileContent = string.Empty;
if (reportFile?.Links?.ContentUrl != null)
{
using var webClient = new WebClient();
reportFileContent = webClient.DownloadString(reportFile.Links.ContentUrl);
report = reportFileContent;
}
else if (transcription.Properties?.Error?.Message != null)
{
report = transcription.Properties.Error.Message;
report += string.IsNullOrEmpty(transcription.Properties.Error.Code) ? string.Empty : $" (Error: {transcription.Properties.Error.Code}).";
}
var errorOutputContainer = FetchTranscriptionEnvironmentVariables.ErrorReportOutputContainer;
await StorageUtilities.WriteTextFileToBlobAsync(FetchTranscriptionEnvironmentVariables.AzureWebJobsStorage, report, errorOutputContainer, errorTxtname, log).ConfigureAwait(false);
if (!string.IsNullOrEmpty(reportFileContent))
{
await ProcessReportFileAsync(reportFileContent, log).ConfigureAwait(false);
}
await client.DeleteTranscriptionAsync(transcriptionId).ConfigureAwait(false);
}
private static void TrackWorkCompleted(WorkEntity workTrackingEntity, System.Collections.Generic.Dictionary <Vector3, int> vertexCounts)
{
workTrackingEntity.MetadataBase64 = SerializationUtilities.EncodeMetadataToBase64(vertexCounts);
workTrackingEntity.CompletedTime = DateTime.UtcNow;
StorageUtilities.UpdateWorkCompletedMetadata(TableClient, workTrackingEntity);
}
public static PSStorageService Create(StorageManagementClient client,
StorageServicePropertiesOperationContext account)
{
return(new PSStorageService(account, new LazyAzureStorageContext((s) =>
{
return StorageUtilities.GenerateCloudStorageAccount(client, account.StorageAccountName);
}, account.StorageAccountName) as AzureStorageContext));
}
private void CheckForComplete(SlicingOptions options, CubeManager manager)
{
int expectedResults = options.TextureSliceX * options.TextureSliceY;
if (StorageUtilities.GetWorkCompletedCount(TableClient, options.CloudResultPath, options.CloudResultContainer) != expectedResults)
{
return;
}
var workResults = StorageUtilities.GetWorkCompletedMetadata(TableClient, options.CloudResultPath, options.CloudResultContainer);
// Write metadata
CubeMetadata metadata = new CubeMetadata(options.CubeGrid)
{
WorldBounds = manager.ObjInstance.Size,
VirtualWorldBounds = options.ForceCubicalCubes ? manager.ObjInstance.CubicalSize : manager.ObjInstance.Size,
VertexCount = manager.ObjInstance.VertexList.Count
};
// Configure texture slicing metadata
if (!string.IsNullOrEmpty(options.Texture) && (options.TextureSliceX + options.TextureSliceY) > 2)
{
metadata.TextureSetSize = new Vector2(options.TextureSliceX, options.TextureSliceY);
}
else
{
metadata.TextureSetSize = new Vector2(1, 1);
}
var resultsList = workResults.Select(w =>
SerializationUtilities.DecodeMetadataFromBase64(
Texture.GetCubeListFromTextureTile(options.TextureSliceY, options.TextureSliceX, w.TextureTileX, w.TextureTileY, manager.ObjInstance),
w.MetadataBase64));
foreach (var result in resultsList)
{
foreach (var cube in result.Keys)
{
metadata.CubeExists[cube.X, cube.Y, cube.Z] = result[cube];
}
}
// Write out some json metadata
string metadataPath = Path.Combine(outputPath, "metadata.json");
if (File.Exists(metadataPath))
{
File.Delete(metadataPath);
}
string metadataString = JsonConvert.SerializeObject(metadata);
File.WriteAllText(metadataPath, metadataString);
StorageUtilities.UpdateSetCompleted(TableClient, options.SetKey);
}
public static PSStorageAccount Create(StorageModels.StorageAccount storageAccount, IStorageManagementClient client)
{
var result = new PSStorageAccount(storageAccount);
var credentials = StorageUtilities.GenerateStorageCredentials(new ARMStorageProvider(client), result.ResourceGroupName, result.StorageAccountName);
CloudStorageAccount account = new CloudStorageAccount(credentials,
storageAccount.PrimaryEndpoints.Blob, storageAccount.PrimaryEndpoints.Queue, storageAccount.PrimaryEndpoints.Table, null);
result.Context = new AzureStorageContext(account);
return(result);
}
public void GetContainerFromSasTest()
{
var containerName = StorageUtilities.GetContainerNameFromUri(new Uri(TestSasUri));
var fileName = StorageUtilities.GetFileNameFromUri(new Uri(TestSasUri));
var fileNameWithoutExtension = StorageUtilities.GetFileNameWithoutExtension(fileName);
Assert.AreEqual("testContainer", containerName);
Assert.AreEqual("testfolder/test.wav", fileName);
Assert.AreEqual("testfolder/test", fileNameWithoutExtension);
}
private static WorkEntity TrackWorkStarted(DateTime startTime, SlicingOptions slicingOptions)
{
var workTrackingEntity = new WorkEntity(slicingOptions.CloudResultPath, slicingOptions.CloudResultContainer, slicingOptions.TextureTile.X, slicingOptions.TextureTile.Y)
{
StartTime = startTime,
};
StorageUtilities.InsertWorkStartedMetadata(TableClient, workTrackingEntity);
return(workTrackingEntity);
}
private async Task VerifySourceDataAsync(SlicingOptions slicingOptions, CancellationToken cancellationToken)
{
if (!File.Exists(slicingOptions.Obj))
{
await StorageUtilities.DownloadBlobAsync(BlobClient, slicingOptions.Obj, slicingOptions.CloudObjPath, cancellationToken).ConfigureAwait(false);
}
if (!File.Exists(slicingOptions.Texture))
{
await StorageUtilities.DownloadBlobAsync(BlobClient, slicingOptions.Texture, slicingOptions.CloudTexturePath, cancellationToken).ConfigureAwait(false);
}
}
protected string GetStorageKey()
{
string storageKey = string.Empty;
if (!string.IsNullOrEmpty(StorageAccountName))
{
var storageCredentials = StorageUtilities.GenerateStorageCredentials(this.StorageClient, this.ResourceGroupName, this.StorageAccountName);
storageKey = storageCredentials.ExportBase64EncodedKey();
}
return(storageKey);
}
public static CloudStorageAccount GetCloudStorageAccount(this AzureSubscription subscription)
{
if (subscription == null)
{
return(null);
}
using (var storageClient = AzureSession.ClientFactory.CreateClient <StorageManagementClient>(subscription, AzureEnvironment.Endpoint.ServiceManagement))
{
return(StorageUtilities.GenerateCloudStorageAccount(
storageClient, subscription.GetProperty(AzureSubscription.Property.StorageAccount)));
}
}
private async void UploadButton_Click(object sender, RoutedEventArgs e)
{
var imageFile = await StorageUtilities.PickSingleImage();
if (imageFile == null)
{
return;
}
ImageFile = imageFile;
Hide();
}
/// <summary>
/// Make sure the storage account key is set.
/// If user doesn't specify it in command line, we try to resolve the key for the user given the storage account.
/// </summary>
/// <param name="storageAccount">The storage account to list the key.</param>
private void InitializeStorageAccountKey(StorageAccount storageAccount)
{
if (string.IsNullOrEmpty(this.StorageAccountKey))
{
if (storageAccount == null)
{
throw new Exception(string.Format(CultureInfo.InvariantCulture, Properties.Resources.DiagnosticsExtensionFailedToListKeyForNoStorageAccount, this.StorageAccountName));
}
var psStorageAccount = new PSStorageAccount(storageAccount);
var credentials = StorageUtilities.GenerateStorageCredentials(this.StorageClient, psStorageAccount.ResourceGroupName, psStorageAccount.StorageAccountName);
this.StorageAccountKey = credentials.ExportBase64EncodedKey();
}
}
private async Task WriteFailedJobLogToStorageAsync(IEnumerable <ServiceBusMessage> serviceBusMessages, string errorMessage, string jobName)
{
var errorOutputContainer = StartTranscriptionEnvironmentVariables.ErrorReportOutputContainer;
var errorFileName = jobName + ".txt";
await StorageUtilities.WriteTextFileToBlobAsync(StartTranscriptionEnvironmentVariables.AzureWebJobsStorage, errorMessage, errorOutputContainer, errorFileName, Logger).ConfigureAwait(false);
foreach (var message in serviceBusMessages)
{
var fileName = StorageUtilities.GetFileNameFromUri(message.Data.Url);
errorFileName = fileName + ".txt";
await StorageUtilities.WriteTextFileToBlobAsync(StartTranscriptionEnvironmentVariables.AzureWebJobsStorage, errorMessage, errorOutputContainer, errorFileName, Logger).ConfigureAwait(false);
}
}
private async void btnSave_Clicked(object sender, EventArgs e)
{
try
{
PermissionStatus status = await PermissionUtilities.GetInstance().RequestExtrenalStorageWritePermission();
if (status == PermissionStatus.Granted)
{
string FolderNameInput = string.Empty;
if (string.IsNullOrEmpty(_ParentFolderName)) //Direct camera on from Landing Page
{
FolderNameInput = await DisplayPromptAsync("Name", "What Document is this?",
initialValue : "SampleDoc", keyboard : Keyboard.Default);
if (string.IsNullOrEmpty(FolderNameInput))
{
FolderNameInput = DateTime.Now.ToFileTime().ToString();
}
_ParentFolderName = FolderNameInput;
}
string PublicExternalFolderPath = StorageUtilities.GetInstance().GetAppExternalStorageFolderName();
//1st time case
if (!Directory.Exists(PublicExternalFolderPath))
{
Directory.CreateDirectory(PublicExternalFolderPath);
}
string SaveDirectory = Path.Combine(PublicExternalFolderPath, _ParentFolderName);
if (!Directory.Exists(SaveDirectory))
{
Directory.CreateDirectory(SaveDirectory);
}
string saveFileName = DateTime.Now.ToFileTime().ToString();
string SaveFilePath = Path.Combine(SaveDirectory, saveFileName + ".jpg");
File.Copy(_EditFile, SaveFilePath, true);
GC.Collect();
await Navigation.PushAsync(new DocumentLandingPage());
}
}
catch (Exception ex)
{
await DisplayAlert("Error", ex.Message, "OK");
}
}
public void CreateDefaultMongoStorage()
{
var configurationBuilder = new ConfigurationBuilder();
configurationBuilder.AddJsonFile("appsettings.json");
var configuration = configurationBuilder.Build();
var storage =
StorageUtilities.CreateStorage("DotnetSpider.Mongo.MongoEntityStorage, DotnetSpider.Mongo",
configuration) as MongoEntityStorage;
Assert.NotNull(storage);
Assert.Equal("mongodb://*****:*****@localhost/test",
storage.ConnectionString);
}
private void UploadResultData(SlicingOptions slicingOptions)
{
var files = Directory.GetFiles(outputPath, "*", SearchOption.AllDirectories);
foreach (var file in files)
{
StorageUtilities.UploadBlob(
BlobClient,
file,
Path.Combine(slicingOptions.CloudResultPath, file.Replace(outputPath, string.Empty).TrimStart(new char[] { '\\', '/' })).Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar),
slicingOptions.CloudResultContainer);
File.Delete(file);
}
}
public static PSStorageAccount Create(StorageModels.StorageAccount storageAccount, IStorageManagementClient client)
{
var result = new PSStorageAccount(storageAccount);
result.Context = new LazyAzureStorageContext((s) =>
{
var credentials = StorageUtilities.GenerateStorageCredentials(new ARMStorageProvider(client), result.ResourceGroupName, s);
return(new CloudStorageAccount(credentials,
ARMStorageService.GetUri(storageAccount.PrimaryEndpoints.Blob),
ARMStorageService.GetUri(storageAccount.PrimaryEndpoints.Queue),
ARMStorageService.GetUri(storageAccount.PrimaryEndpoints.Table),
ARMStorageService.GetUri(storageAccount.PrimaryEndpoints.File)));
}, result.StorageAccountName) as AzureStorageContext;
return(result);
}
public override void ExecuteCmdlet()
{
CloudStorageAccount account;
if (Context != null)
{
account = Context.StorageAccount;
}
else
{
account = StorageUtilities.GenerateCloudStorageAccount(new ARMStorageProvider(StorageClient), ResourceGroupName, Name);
}
// Clear the current storage account for both SM and RM
GeneralUtilities.ClearCurrentStorageAccount(true);
DefaultContext.SetCurrentStorageAccount(account.ToString(true));
WriteObject(account.Credentials.AccountName);
}
public void CreateDefaultSqlServerStorage()
{
var configurationBuilder = new ConfigurationBuilder();
configurationBuilder.AddJsonFile("appsettings.json");
var configuration = configurationBuilder.Build();
var storage =
StorageUtilities.CreateStorage("DotnetSpider.DataFlow.SqlServerEntityStorage, DotnetSpider",
configuration) as SqlServerEntityStorage;
Assert.NotNull(storage);
Assert.Equal(StorageMode.InsertIgnoreDuplicate, storage.Mode);
Assert.Equal("Database='mysql3';Data Source=localhost;password=1qazZAQ!;User ID=root;Port=3308;",
storage.ConnectionString);
Assert.False(storage.IgnoreCase);
Assert.Equal(1000, storage.RetryTimes);
Assert.True(storage.UseTransaction);
}
protected override void ProcessRecord()
{
CloudStorageAccount account;
if (Context != null)
{
account = Context.StorageAccount;
}
else
{
account = StorageUtilities.GenerateCloudStorageAccount(StorageClient, ResourceGroupName, StorageAccountName);
}
// Clear the current storage account for both SM and RM
GeneralUtilities.ClearCurrentStorageAccount(true);
DefaultContext.SetCurrentStorageAccount(account.ToString(true));
WriteObject(account.Credentials.AccountName);
}
/// <summary>
/// Initialize the storage account key if it's not specified.
/// It can be defined in multiple places, we only take the one with higher precedence. And the precedence is:
/// 1. The one we try to resolve within current subscription
/// 2. The one defined in PrivateConfig in the configuration file
/// </summary>
public static string InitializeStorageAccountKey(IStorageManagementClient storageClient, string storageAccountName = null, string configurationPath = null)
{
string storageAccountKey = null;
StorageAccount storageAccount = null;
if (TryGetStorageAccount(storageClient, storageAccountName, out storageAccount))
{
// Help user retrieve the storage account key
var psStorageAccount = new PSStorageAccount(storageAccount);
var credentials = StorageUtilities.GenerateStorageCredentials(storageClient, psStorageAccount.ResourceGroupName, psStorageAccount.StorageAccountName);
storageAccountKey = credentials.ExportBase64EncodedKey();
}
else
{
// Use the one defined in PrivateConfig
storageAccountKey = GetStorageAccountInfoFromPrivateConfig(configurationPath, PrivConfKeyAttr);
}
return(storageAccountKey);
}
