static Microsoft.WindowsAzure.Storage.Table.CloudTable GetAzureTable() { Microsoft.WindowsAzure.Storage.CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageAccount")); //create cloud table client Microsoft.WindowsAzure.Storage.Table.CloudTableClient tableClient = storageAccount.CreateCloudTableClient(); //get cloud table Microsoft.WindowsAzure.Storage.Table.CloudTable table = tableClient.GetTableReference("customers"); //cleanup table.DeleteIfExists(); table.CreateIfNotExists(); return(table); }
public static Task <TResult> ResolveCreate <TResult>(this StorageException exception, Microsoft.WindowsAzure.Storage.Table.CloudTable table, Func <TResult> retry, Func <ExtendedErrorInformationCodes, string, TResult> onFailure = default, Func <TResult> onAlreadyExists = default, AzureStorageDriver.RetryDelegate onTimeout = default) { return(exception.ParseStorageException( onEntityAlreadyExists: (msg) => onAlreadyExists().AsTask(), onNotFound: async(msg) => // IsProblemTableDoesNotExist { try { await table.CreateIfNotExistsAsync(); return retry(); } catch (StorageException createEx) { // Catch bug with azure storage table client library where // if two resources attempt to create the table at the same // time one gets a precondtion failed error. System.Threading.Thread.Sleep(1000); return retry(); } catch (Exception) { throw; }; }, onTimeout: async(msg) => // IsProblemTimeout { if (onTimeout.IsDefaultOrNull()) { onTimeout = AzureStorageDriver.GetRetryDelegate(); } bool shouldRetry = false; await onTimeout(exception.RequestInformation.HttpStatusCode, exception, () => { shouldRetry = true; return 1.AsTask(); }); if (shouldRetry) { return retry(); } throw exception; }, onDefaultCallback: (error, msg) => onFailure(error, msg).AsTask())); }
/// <summary> /// Sets the connection to Azure /// Creates the table if it does not exist /// </summary> public BaseDataServiceContext() { storageCredentials = new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials( Sample.Azure.Common.Setting.SettingService.CloudStorageAccountName, Sample.Azure.Common.Setting.SettingService.CloudStorageKey); storageAccount = new Microsoft.WindowsAzure.Storage.CloudStorageAccount(storageCredentials, new Uri(Sample.Azure.Common.Setting.SettingService.CloudStorageBlobEndPoint), new Uri(Sample.Azure.Common.Setting.SettingService.CloudStorageQueueEndPoint), new Uri(Sample.Azure.Common.Setting.SettingService.CloudStorageTableEndPoint), new Uri(Sample.Azure.Common.Setting.SettingService.CloudStorageFileEndPoint)); tableClient = storageAccount.CreateCloudTableClient(); cloudTable = tableClient.GetTableReference(this.TableName); if (!tableCreatedList.Contains(this.TableName)) { cloudTable.CreateIfNotExists(); tableCreatedList.Add(this.TableName); } }
static void Main(string[] args) { Console.ForegroundColor = ConsoleColor.Cyan; Console.WriteLine("Checking Configuration Values..."); Console.ResetColor(); Console.WriteLine("--------------------------------"); // Check for valid directories if (string.IsNullOrEmpty(Config.LOCAL_LOCATION_FILES_SOURCE_DOCUMENTS)) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("Local directory for source documents is blank."); Console.ReadLine(); Environment.Exit(0); } if (!Directory.Exists(Config.LOCAL_LOCATION_FILES_SOURCE_DOCUMENTS)) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("AppConfig - Local directory for SOURCE documents: {0} does not exist.", Config.LOCAL_LOCATION_FILES_SOURCE_DOCUMENTS); Console.ReadLine(); Environment.Exit(0); } if (!Directory.Exists(Config.LOCAL_LOCATION_FILES_PROCESSED_OUTPUTS)) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("AppConfig - Local directory for PROCESSED documents: {0} does not exist.", Config.LOCAL_LOCATION_FILES_PROCESSED_OUTPUTS); Console.ReadLine(); Environment.Exit(0); } if (!Config.USE_COGNITIVE_SERVICES_V2 && !Config.USE_COGNITIVE_SERVICES_V3) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("AppConfig - You must have either Cognitive Services V2 or V3 enabled in the Config file."); Console.ReadLine(); Environment.Exit(0); } if (string.IsNullOrEmpty(Config.COGNITIVE_SERVICES_KEY) || string.IsNullOrEmpty(Config.COGNITIVE_SERVICES_REGION_URI)) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("AppConfig - COGNITIVE_SERVICES_KEY or COGNITIVE_SERVICES_REGION is empty."); Console.ReadLine(); Environment.Exit(0); } Console.WriteLine("Use Cognitive Services Bing Entity Search: " + Config.USE_COGNITIVE_SERVICES_BING_ENTITY_SEARCH); Console.WriteLine("Use Azure Blob Storage: " + Config.USE_AZURE_BLOB_STORAGE); Console.WriteLine("Use Azure Table Storage: " + Config.USE_AZURE_TABLE_STORAGE); Console.WriteLine("Use CosmosDB Storage: " + Config.USE_COSMOSDB_STORAGE); Console.WriteLine("--------------------------------"); Console.WriteLine(string.Empty); Dictionary <string, Exception> errors = new Dictionary <string, Exception>(); Dictionary <string, int> longDocuments = new Dictionary <string, int>(); Dictionary <string, Tuple <string, string, string> > processedTrainingFiles = new Dictionary <string, Tuple <string, string, string> >(1700); var scoringTableEntities = new List <Microsoft.Azure.CosmosDB.Table.DynamicTableEntity>(); var topThreeClassificationNamesDictionary = new Dictionary <string, List <string> >(); var topThreeClassificationProbabilitiesDictionary = new Dictionary <string, List <double> >(); Console.ForegroundColor = ConsoleColor.Cyan; Console.WriteLine("Extracting content from documents..."); Console.ResetColor(); // List of types of extensions var fileTypes = new List <Tuple <string, string> >(); var currentFilesDirectory = string.Empty; var filePath = string.Empty; currentFilesDirectory = Config.LOCAL_LOCATION_FILES_SOURCE_DOCUMENTS; var docExt = new List <string> { ".DOC", ".DOCX", ".DOTX", ".DOT" }; var files = Util.DirectoryTraverseForFiles(currentFilesDirectory).ToList(); try { Console.WriteLine("--------------------------------"); Console.WriteLine("Processing Files..."); // 2) Process Files for (int fileNum = 0; fileNum != files.Count; fileNum++) { // Cognitive Services API Calls var cognitiveServicesApiCalls = new CognitiveServicesApiCalls(); // Retrieve the file path filePath = files[fileNum]; // Retrieve the directory, file name & extension var categoryAndFileNames = filePath.Replace(currentFilesDirectory, string.Empty) .Split(new string[] { "\\" }, StringSplitOptions.None); var originalDocumentExtension = Path.GetExtension(filePath).ToUpper(); // Clean up file names var category = categoryAndFileNames[0]; var cleanCategory = category.Replace(" ", "_").Replace(".", "_"); var fileName = Path.GetFileName(filePath); var cleanFileName = fileName.Replace(" ", "_").Replace(".", "_"); var fileTotalOcr = string.Empty; var keyPhraseString = string.Empty; var entitiesString = string.Empty; var pages = 0; var uri = string.Empty; var documentType = "Unknown"; //type of document (i.e. PDF, Word, Excel etc.) var documentSizeInBytes = 0L; Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine("Processing file {0} : ID={1} [{2} of {3}]", fileName, cleanFileName, files.IndexOf(filePath) + 1, files.Count); Console.ResetColor(); // Group file types based on the required processing fileTypes.Add(new Tuple <string, string>(fileName, originalDocumentExtension)); var wordDocuments = new List <string> { ".DOC", ".DOCX", ".DOTX", ".DOT" }; var excelDocuments = new List <string> { ".XLS", ".XLSX", ".XLT", ".XLTX" }; var htmlDocuments = new List <string> { ".HTM", ".HTML", ".SHTML" }; // Hold the values for the image pages List <MemoryStream> imageStreams = new List <MemoryStream>(); // Hold the values for the OCR from pages List <string> imagePagesOcr = new List <string>(); // Process/read document MemoryStream documentStream = new MemoryStream(); using (var file = File.OpenRead(filePath)) { documentSizeInBytes = new System.IO.FileInfo(filePath).Length; var pdfDocumentPartFileNames = new List <string>(); // PDF Files if (originalDocumentExtension == ".PDF") { documentType = "PDF"; // Setup PDFReader for unethical reading PdfReader.unethicalreading = true; // Setup PDF part cache location var directoryCategoryPdfCache = Config.LOCAL_LOCATION_FILES_PROCESSED_OUTPUTS + @"\PDFCache\" + cleanCategory + @"\"; var directoryFilePdfCache = directoryCategoryPdfCache + cleanFileName; System.IO.Directory.CreateDirectory(directoryCategoryPdfCache.ToLower()); System.IO.Directory.CreateDirectory(directoryFilePdfCache.ToLower()); Console.WriteLine("\tCracking Document into Pages..."); using (PdfReader pdfReader = new PdfReader(filePath)) { for (int pagenumber = 1; pagenumber <= pdfReader.NumberOfPages; pagenumber++) { iTextSharp.text.Document iTextDocument = new iTextSharp.text.Document(); var fullFilePdfPartName = directoryFilePdfCache + @"\" + cleanFileName + (pagenumber) + ".pdf"; pdfDocumentPartFileNames.Add(fullFilePdfPartName); PdfCopy copy = new PdfCopy(iTextDocument, new FileStream(fullFilePdfPartName, FileMode.Create)); iTextDocument.Open(); copy.AddPage(copy.GetImportedPage(pdfReader, pagenumber)); iTextDocument.Close(); } } Aspose.Pdf.Document document = new Aspose.Pdf.Document(file); document.Save(documentStream); pages = document.Pages.Count; Console.WriteLine(string.Format("\tPages: {0}", pages)); //imageStreams.Add(document.ConvertPageToPNGMemoryStream(document.Pages[1])); } // Excel Files else if (excelDocuments.Contains(originalDocumentExtension)) { documentType = "Excel"; Aspose.Cells.Workbook workBook = new Aspose.Cells.Workbook(file); pages = workBook.Worksheets.Count; workBook.Save(documentStream, Aspose.Cells.SaveFormat.Pdf); } // HTML & Word Documents else if (htmlDocuments.Contains(originalDocumentExtension) || wordDocuments.Contains(originalDocumentExtension)) { documentType = "Html"; Aspose.Words.Document document = new Aspose.Words.Document(file); pages = document.PageCount; document.Save(documentStream, Aspose.Words.SaveFormat.Pdf); } // Convert any documents into images if (originalDocumentExtension == ".PDF") { foreach (var pdfDcoumentFileName in pdfDocumentPartFileNames) { var pdfDocument = new Aspose.Pdf.Document(pdfDcoumentFileName); var pdfDocumentPage = pdfDocument.Pages[1]; try { imageStreams.Add(pdfDocument.ConvertPageToPNGMemoryStream(pdfDocumentPage)); } catch (Exception e) { Console.WriteLine("!!! ERROR !!!: Converting PDF To Image - " + pdfDcoumentFileName + " ||| " + e.ToString()); } } } // Word Documents else if (wordDocuments.Contains(originalDocumentExtension.ToUpper())) { documentType = "Word"; Microsoft.Office.Interop.Word.Application wordApp = new Microsoft.Office.Interop.Word.Application(); var wordDocument = new Microsoft.Office.Interop.Word.Document(); object missing = System.Type.Missing; wordDocument = wordApp.Documents.Open(filePath); foreach (Microsoft.Office.Interop.Word.Window window in wordDocument.Windows) { foreach (Microsoft.Office.Interop.Word.Pane pane in window.Panes) { // set the pages pages = pane.Pages.Count; for (var i = 1; i <= pane.Pages.Count; i++) { var bits = pane.Pages[i].EnhMetaFileBits; try { using (var ms = new MemoryStream((byte[])(bits))) { var imageStream = new MemoryStream(); var image = System.Drawing.Image.FromStream(ms); image.Save(imageStream, ImageFormat.Png); //ms.Position = 0; imageStreams.Add(imageStream); } } catch (System.Exception ex) { var error = ex.Message; throw (ex); } } } } wordDocument.Close(Type.Missing, Type.Missing, Type.Missing); wordApp.Quit(Type.Missing, Type.Missing, Type.Missing); } else if (documentStream != null && imageStreams.Count == 0) { var pdfDocument = new Aspose.Pdf.Document(documentStream); // Done here as Aspose workaround for free version for (int pageNum = 0; pageNum != pdfDocument.Pages.Count; pageNum++) { if (pageNum == 4) { // Track documents over 4 pages longDocuments.Add(fileName, pdfDocument.Pages.Count); break; } // PageCollection starts at 1, not 0 index imageStreams.Add(pdfDocument.ConvertPageToPNGMemoryStream(pdfDocument.Pages[pageNum + 1])); } } } // EOF reading file Console.WriteLine("\tConverting Pages to Images..."); for (int i = 0; i != imageStreams.Count; i++) { //Console.WriteLine(string.Format("\tProcessing Image {0} of {1}", (i + 1), imageStreams.Count)); var imageStream = imageStreams[i]; // Cognitive Services Requirement: Convert to MB, Computer Vision images max out at 4 MB // Fine for individual page documents var megaBytes = Util.ConvertBytesToMegabytes(imageStream.Length); do { var reducedImage = Util.ReduceImageQuality(imageStream); imageStream = reducedImage; }while( Util.ConvertBytesToMegabytes(imageStream.Length) > 3.99 ); // Setup Image store location var directoryCategory = Config.LOCAL_LOCATION_FILES_PROCESSED_OUTPUTS + @"\" + cleanCategory + @"\"; var directoryFile = directoryCategory + cleanFileName; var fullFileImageName = directoryFile + @"\" + cleanFileName + (i + 1) + ".png"; System.IO.Directory.CreateDirectory(directoryCategory.ToLower()); System.IO.Directory.CreateDirectory(directoryFile.ToLower()); // Save the image to local image folder (cache) using (Bitmap image = (Bitmap)System.Drawing.Image.FromStream(imageStream)) { var resizedImage = Util.ResizeImageForCognitiveOCR(image); resizedImage.Save(fullFileImageName, ImageFormat.Png); } var basePath = cleanCategory + @"\" + cleanFileName + @"\" + cleanFileName + (i + 1); // Set up cloud path for JSON content var cloudImagePath = (basePath + ".png").ToLower(); var cloudOcrPath = (basePath + ".json").ToLower(); // Use API that passes image binary directly var ocrResult = CognitiveServices.VisionOCRResultBatchReadFromImageAsync(fullFileImageName, "v2.1").Result; //var ocrResult = CognitiveServices.OCRResultBatchRead(imageUrl, "v2.1").Result; cognitiveServicesApiCalls.ApiCallCount++; if (Config.USE_AZURE_BLOB_STORAGE) { // Azure Storage Objects Init var cloudStorageBloblClient = AzureStorage.BlobStorageAccount.CreateCloudBlobClient(); var enrichmentContainer = cloudStorageBloblClient.GetContainerReference(Config.STORAGE_TABLE_AND_CONTAINER_NAMES.ToLower()); uri = enrichmentContainer.StorageUri.PrimaryUri + @"/" + cleanCategory + @"/" + cleanFileName + @"/"; var trainingImage = enrichmentContainer.GetBlockBlobReference(cloudImagePath); var trainingImageOcr = enrichmentContainer.GetBlockBlobReference(cloudOcrPath); // Send PNG image to Blob Storage using (var fs = new FileStream(fullFileImageName, FileMode.Open)) { trainingImage.UploadFromStream(fs); } // Retrieve OCR keyPhraseResult and upload JSON to cloud // Uses API based on cloud storage var imageUrl = trainingImage.Uri.AbsoluteUri + Config.STORAGE_ACCOUNT_TEMP_SAS_KEY; // Upload the JSON response to the blob containers trainingImageOcr.UploadText(ocrResult.Item1); } // Write JSON to local disk var jsonFileName = basePath + ".json"; System.IO.File.WriteAllText(Config.LOCAL_LOCATION_FILES_PROCESSED_OUTPUTS + @"\" + jsonFileName, ocrResult.Item1); var ocrString = ocrResult.Item2.ToString(); imagePagesOcr.Add(ocrString); // Console.WriteLine("Number of OCR Regions Found - " + ocrResult.Item2.regions.Count); } var ocrPhrases = new List <KeyValuePair <string, string> >(); foreach (var ocrItem in imagePagesOcr) { // remove the trial items from Aspose var tempOcrItem = ocrItem. Replace("Evaluation Only. Created with Aspose.PDF. Copyright 2002-2019 Aspose Pty Ltd.", string.Empty). Replace("Evaluation Only. Created with Aspose.PDF. Copyright 2002-2018 Aspose Pty Ltd.", string.Empty). Replace("Evaluation Only. Created with Aspose.PDF", string.Empty). Replace("Evaluation Only. Created with Aspose.Words", string.Empty). Replace("Evaluation Only. Created with Aspose.Cells", string.Empty). Replace("Created with Aspose.Cells for .NET.Copyright 2003 - 2018", string.Empty). Replace("Copyright 2002-2019 Aspose Pty Ltd.", string.Empty). Replace("Copyright 2002-2018 Aspose Pty Ltd.", string.Empty). Replace("Aspose Pty Ltd.", string.Empty). Replace("Created with Aspose.", string.Empty). Replace("Copyright 2002-2018.", string.Empty). Replace("Copyright 2003-2018.", string.Empty). Replace("Copyright 2002-2018", string.Empty). Replace("Copyright 2003-2018", string.Empty). Replace("spose Pty Ltd.", string.Empty). Replace("Pty Ltd.", string.Empty). Replace("Evaluation With A", string.Empty). Replace("Evaluation With %002-2018 A", string.Empty). Replace("Evaluation Only.", string.Empty). Replace(". Aspose Pty ", string.Empty). Replace("Evaluatqoh With %002-2018 A", string.Empty). Replace("Created with Aspose.Cells for .NET.Copyright 2003 - 2018 A", string.Empty). Replace("Evaluation Only• created 18 Aspose Pty", string.Empty). Replace("Evaluation Only. Created with Aspose.PDF", string.Empty). Replace("with Aspose. PDF", string.Empty). Replace("Aspose.PDF", string.Empty); // add to main OCR file fileTotalOcr += tempOcrItem + System.Environment.NewLine; // Add to phrases to process (english) if (tempOcrItem.Length > 5000) { var tempOcrItemOne = tempOcrItem.Substring(0, 5000); var tempOcrItemTwo = tempOcrItem.Substring(5000, (tempOcrItem.Length - 5000)); // Add both ocrPhrases.Add(new KeyValuePair <string, string>("en", tempOcrItemOne)); ocrPhrases.Add(new KeyValuePair <string, string>("en", tempOcrItemTwo)); } else { ocrPhrases.Add(new KeyValuePair <string, string>("en", tempOcrItem)); } } List <string> keyPhrasesV2 = new List <string>(); List <string> entitiesV2 = new List <string>(); var piiResultV2 = new PIIResult(); string distinctKeyPhraseString = string.Empty; string distinctEntitiesString = string.Empty; if (Config.USE_COGNITIVE_SERVICES_V2) { // Key Phrases - V2 Console.WriteLine("\tKey Phrases V2..."); var keyPhraseResult = CognitiveServices.TextAnalyticsKeyPhrasesAndEntities(ocrPhrases, ref cognitiveServicesApiCalls); keyPhrasesV2 = keyPhraseResult.Item1.Documents.SelectMany(i => i.KeyPhrases).Where(a => Helpers.IsEntity(a)).ToList(); distinctKeyPhraseString = string.Join(" ;;;; ", keyPhrasesV2.Distinct().ToArray()); keyPhraseString = string.Join(" ;;;; ", keyPhrasesV2.ToArray()); // Entities - V2 Console.WriteLine("\tEntities V2..."); var entitiesRecords = keyPhraseResult.Item2.Documents.SelectMany(i => i.Entities).Where(a => Helpers.IsEntity(a.Name)).ToList(); entitiesV2 = entitiesRecords.Select(i => i.Name.Replace(System.Environment.NewLine, string.Empty).Trim()).ToList(); distinctEntitiesString = string.Join(" ;;;; ", entitiesV2.Distinct().ToArray()); entitiesString = string.Join(" ;;;; ", entitiesV2.ToArray()); // PII Result - V2 Console.WriteLine("\tPII Information V2..."); piiResultV2 = CognitiveServices.TextAnalyticsPIIResultV2(ocrPhrases, ref cognitiveServicesApiCalls); } List <string> keyPhrasesV3 = new List <string>(); List <CognitiveServiceClasses.Entities.Entity> entitiesV3 = new List <CognitiveServiceClasses.Entities.Entity>(); List <CognitiveServiceClasses.PII.Entity> piiResultV3 = new List <CognitiveServiceClasses.PII.Entity>(); SentimentV3Response sentimentV3Prediction = new SentimentV3Response(); if (Config.USE_COGNITIVE_SERVICES_V3) { // Key Phrases - V3 Console.WriteLine("\tKey Phrases V3..."); var textAnalyticsV3KeyPhrasesPrediction = CognitiveServices.TextAnalyticsKeyPhrasesV3PreviewAsync(ocrPhrases).Result; keyPhrasesV3 = textAnalyticsV3KeyPhrasesPrediction.documents.SelectMany(a => a.keyPhrases).ToList(); cognitiveServicesApiCalls.ApiCallV3Count++; // Entities - V3 Console.WriteLine("\tEntities V3..."); var textAnalyticsV3EntitiesPrediction = CognitiveServices.TextAnalyticsEntitiesV3PreviewAsync(ocrPhrases).Result; entitiesV3 = textAnalyticsV3EntitiesPrediction.documents.SelectMany(a => a.entities).ToList(); cognitiveServicesApiCalls.ApiCallV3Count++; // PIIs - V3 Console.WriteLine("\tPIIs V3..."); var textAnalyticsV3PIIPrediction = CognitiveServices.TextAnalyticsPIIV3PreviewAsync(ocrPhrases).Result; piiResultV3 = textAnalyticsV3PIIPrediction.documents.SelectMany(a => a.entities).ToList(); cognitiveServicesApiCalls.ApiCallV3Count++; // Sentiment Analysis - V3 Console.WriteLine("\tSentiment Analysis V3..."); var textAnalyticsInput = new TextAnalyticsInput() { Id = "1", Text = fileTotalOcr.Length > 5100 ? fileTotalOcr.Substring(0, 5100) : fileTotalOcr }; var textAnalyticsInputs = new List <TextAnalyticsInput> { textAnalyticsInput }; sentimentV3Prediction = CognitiveServices.TextAnalyticsSentimentAnalysisV3PreviewAsync(textAnalyticsInputs).Result; cognitiveServicesApiCalls.ApiCallV3Count++; } List <BingEntityData> entityTaxonyResult = new List <BingEntityData>(); if (Config.USE_COGNITIVE_SERVICES_BING_ENTITY_SEARCH) { Console.WriteLine("\tRetrieving Bing Entitites..."); entityTaxonyResult = CognitiveServices.BingEntities(entitiesV2); Console.WriteLine("\tFinished Retrieving Bing Entitites."); } // Add up OCR pages & Process Key Entities processedTrainingFiles.Add(cleanFileName, new Tuple <string, string, string>(cleanCategory, fileTotalOcr, keyPhraseString)); // Azure Table Storage // Note: most attributes are truncated due to limitations for Azure Table Storage // For large documents use: CosmosDb or blob storage if (Config.USE_AZURE_TABLE_STORAGE) { var tableStorageClient = AzureStorage.BlobStorageAccount.CreateCloudTableClient(); Microsoft.WindowsAzure.Storage.Table.CloudTable table = tableStorageClient.GetTableReference(Config.STORAGE_TABLE_AND_CONTAINER_NAMES); table.CreateIfNotExists(); Util.WriteToBlobStorageTable(table, cleanCategory, cleanFileName, fileTotalOcr, keyPhraseString, distinctKeyPhraseString, entitiesString, distinctEntitiesString, pages, uri, documentType, documentSizeInBytes, piiResultV2, piiResultV3, entityTaxonyResult, sentimentV3Prediction, cognitiveServicesApiCalls); Console.WriteLine("\tPersisted: Azure Table Storage"); } if (Config.USE_COSMOSDB_STORAGE) { // Cosmos DB - Documents Client DocumentClient documentClient = new DocumentClient(Config.COSMOSDB_DOCUMENTS_URI, Config.COSMOSDB_DOCUMENTS_KEY); //// CosmosDB SQL API Util.WriteToCosmosDbStorageSQLApi(documentClient, cleanCategory, cleanFileName, fileTotalOcr, keyPhrasesV2, keyPhrasesV3, entitiesV2, entitiesV3, pages, uri, documentType, documentSizeInBytes, piiResultV2, piiResultV3, entityTaxonyResult, sentimentV3Prediction, cognitiveServicesApiCalls); Console.WriteLine("\tPersisted: CosmosDB - SQL API"); } //// CosmosDB SQL API Util.WriteToLocalStorage(cleanCategory, cleanFileName, fileTotalOcr, keyPhrasesV2, keyPhrasesV3, entitiesV2, entitiesV3, pages, uri, documentType, documentSizeInBytes, piiResultV2, piiResultV3, entityTaxonyResult, sentimentV3Prediction, cognitiveServicesApiCalls); Console.WriteLine("\tPersisted: Local Storage"); } ; // EOF for loop } catch (Exception e) { errors.Add(filePath, e); Console.WriteLine("!!! ERROR !!!: " + e.ToString()); } // Write File // Util.WriteCsvFile(processedTrainingFiles); // Distribution of various file types (by extension) var fileTypeCounts = fileTypes.GroupBy(a => a.Item2). Select(group => new { Extension = group.Key.ToUpper(), Count = group.Count() }).OrderByDescending(o => o.Count); // Print out errors file File.Delete(Config.LOCAL_LOCATION_FILES_SOURCE_DOCUMENTS + "Errors.txt"); using (TextWriter tw = new StreamWriter(Config.LOCAL_LOCATION_FILES_SOURCE_DOCUMENTS + "Errors.txt")) { foreach (var error in errors) { tw.WriteLine(error.Key); } } Console.WriteLine("Number of errors: " + errors.Count); Console.ReadLine(); }
public static Microsoft.WindowsAzure.Storage.Table.TableResult GetRandomTableResult() { //try //{ Microsoft.WindowsAzure.Storage.Table.TableResult retreiveResult = null; XmlDocument document = new XmlDocument(); document.Load(@"C:\Users\Administrator\source\repos\ConsoleApp5\ConsoleApp5\bin\Debug\Test.xml"); var appSettings = System.Configuration.ConfigurationManager.AppSettings; if (appSettings.Count == 0) { Console.WriteLine("The appsettings is empty\nEnter a valid ConnectionString in AppConfig file"); Console.WriteLine("Press a key to exit.."); Console.ReadLine(); } else { foreach (var key in appSettings.AllKeys) { Microsoft.WindowsAzure.Storage.CloudStorageAccount storageAccount = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(System.Configuration.ConfigurationManager.AppSettings[key]); Microsoft.WindowsAzure.Storage.Table.CloudTableClient tableClient = storageAccount.CreateCloudTableClient(); Microsoft.WindowsAzure.Storage.Table.CloudTable table = tableClient.GetTableReference("CustomerInformation"); //table.CreateIfNotExists(); /*foreach (Json_File file in json_Files) * { * Microsoft.WindowsAzure.Storage.Table.TableOperation insertOperation = Microsoft.WindowsAzure.Storage.Table.TableOperation.Insert(file); * table.Execute(insertOperation); * } */ if (table.Exists()) { Console.WriteLine("The table exists.."); //Microsoft.WindowsAzure.Storage.Table.EntityProperty.CreateEntityPropertyFromObject Dictionary <int, string> Partition_Keys = new Dictionary <int, string>(); int num = 1; Microsoft.WindowsAzure.Storage.Table.TableContinuationToken token = null; var entities = new List <Json_File>(); do { var queryResult = table.ExecuteQuerySegmented(new Microsoft.WindowsAzure.Storage.Table.TableQuery <Json_File>(), token); entities.AddRange(queryResult.Results); token = queryResult.ContinuationToken; } while (token != null); foreach (var file in entities) { Partition_Keys.Add(num, file.PartitionKey); num++; } int size = Partition_Keys.Count; Random random = new Random(); int random_number = random.Next(1, size + 1); string RandomPartitionKey = Partition_Keys[random_number]; Microsoft.WindowsAzure.Storage.Table.TableContinuationToken continuationToken = null; var Entities = new List <Json_File>(); do { var rangeQueryResult = table .ExecuteQuerySegmented(new Microsoft.WindowsAzure.Storage.Table.TableQuery <Json_File>() .Where(Microsoft.WindowsAzure.Storage.Table.TableQuery .GenerateFilterCondition("PartitionKey", Microsoft.WindowsAzure.Storage.Table.QueryComparisons.Equal, RandomPartitionKey)), continuationToken); Entities.AddRange(rangeQueryResult.Results); continuationToken = rangeQueryResult.ContinuationToken; } while (continuationToken != null); Dictionary <int, string> Row_Keys = new Dictionary <int, string>(); int number = 1; foreach (var file in Entities) { Row_Keys.Add(number, file.RowKey); number++; } int sub_size = Row_Keys.Count; Random rand = new Random(); int rand_number = rand.Next(1, sub_size + 1); string RandomRowKey = Row_Keys[rand_number]; Microsoft.WindowsAzure.Storage.Table.TableOperation retreiveOperation = Microsoft.WindowsAzure.Storage.Table.TableOperation.Retrieve <Json_File>(RandomPartitionKey, RandomRowKey); retreiveResult = table.Execute(retreiveOperation); } else { Console.WriteLine("Table doesn't exists\nPress a key to exit...."); Console.ReadLine(); } } } return(retreiveResult); /*catch (Exception ex) * { * Console.WriteLine(ex.Message); * Console.WriteLine(ex.StackTrace); * Console.WriteLine(ex.InnerException); * Console.WriteLine(ex.Source); * Console.WriteLine("Press a key to exit..."); * Console.ReadLine(); * } */ }
static TimController() { Microsoft.WindowsAzure.ServiceRuntime.RoleEnvironment.Changed += RoleEnvironment_Changed; Trace.TraceInformation("[TRACE] Entering TimController::TimController() static initializer..."); string strDatabaseConnectionString = Microsoft.WindowsAzure.CloudConfigurationManager.GetSetting("InfloDatabaseConnectionString"); if (strDatabaseConnectionString == null) { Trace.TraceError("Unable to retrieve database connection string"); } else if (strDatabaseConnectionString.Length <= 0) { Trace.TraceError("Database connection string empty"); } else //connect to the database { srInfloDbContext = new InfloDbContext(strDatabaseConnectionString); } string strStorageAccountConnectionString = Microsoft.WindowsAzure.CloudConfigurationManager.GetSetting("StorageAccountConnectionString"); if (strStorageAccountConnectionString == null) { Trace.TraceError("Unable to retrieve storage account connection string"); } else if (strStorageAccountConnectionString.Length <= 0) { Trace.TraceError("Storage account connection string empty"); } else //connect to the cloud storage account { Trace.TraceInformation("[INFO] Retrieved StorageAccountConnectionString for TimController\n{0}", strStorageAccountConnectionString); try { srStorageAccount = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(strStorageAccountConnectionString); } catch (Exception e) { Trace.TraceError("Exception occurred when parsing storage account connection string\n{0}\n{1}", strStorageAccountConnectionString, e.Message); } if (srStorageAccount != null) { srCloudTableClient = srStorageAccount.CreateCloudTableClient(); srTimV2VTable = srCloudTableClient.GetTableReference(srTimV2VTableName); try { srTimV2VTable.CreateIfNotExists(); } catch (Exception e) { Trace.TraceError("Exception occurred when creating table for V2V TIMs\n{0}", e.Message); srTimV2VTable = null; } } } AlertSearchRadius = GetAlertSearchDistance(); QWarnAheadDistance = GetQWarnAheadDistance(); srInfloDbContext.Configuration_Roadway.Load(); srInfloDbContext.Configuration_RoadwayLinks.Load(); srInfloDbContext.Configuration_RoadwayMileMarkers.Load(); srInfloDbContext.TMEOutput_QWARNMessage_CV.Load(); srInfloDbContext.TMEOutput_SPDHARMMessage_CV.Load(); Trace.TraceInformation("[TRACE] Exiting TimController::TimController() static initializer..."); return; }