public IActionResult GetSavedDiscovery([FromQuery] string filter) { string entityAsJson = ""; try { _logger.LogInformation("CPAPI: Get"); // Deserialize the filter DiscoveryFilters oFilter = new DiscoveryFilters(); if (filter != null && filter.Length > 0) { _logger.LogDebug("Deserializing filter of length: " + filter.Length); oFilter = JsonConvert.DeserializeObject <DiscoveryFilters>(filter); } string storageAccountConnectionString = Utils.GetSecretOrEnvVar(ConfigurationProperties.AzureStorageAccountConnectionString, Configuration, _logger).Trim(); // validate tika base address if (storageAccountConnectionString == "") { _logger.LogWarning("Azure storage account connection string not set"); return(StatusCode((int)System.Net.HttpStatusCode.InternalServerError)); } else { _logger.LogDebug("Azure storage account connection string loaded"); } // Process the records CloudStorageAccount account = CloudStorageAccount.Parse(storageAccountConnectionString); // Create the table client. //log.Info("Creating cloud table client"); CloudTableClient tableClient = account.CreateCloudTableClient(); // Create the table if it doesn't exist. //log.Info("Getting table reference"); CloudTable table = tableClient.GetTableReference("stlpdiscovery"); Task tCreate = table.CreateIfNotExistsAsync(); tCreate.Wait(); // Create a default query TableQuery <DiscoveryEntity> query = new TableQuery <DiscoveryEntity>(); if (oFilter.filters.Count > 0) { string combinedFilter = ""; foreach (DiscoveryFilter entry in oFilter.filters) { string cleanFilterPKey = Utils.CleanTableKey(entry.Value); string pkquery = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, cleanFilterPKey); combinedFilter = pkquery; } // Create final combined query query = new TableQuery <DiscoveryEntity>().Where(combinedFilter); } List <DiscoveryEntity> spfileEntities = new List <DiscoveryEntity>(); TableContinuationToken token = null; var runningQuery = new TableQuery <DiscoveryEntity>() { FilterString = query.FilterString, SelectColumns = query.SelectColumns }; do { runningQuery.TakeCount = query.TakeCount - spfileEntities.Count; Task <TableQuerySegment <DiscoveryEntity> > tSeg = table.ExecuteQuerySegmentedAsync <DiscoveryEntity>(runningQuery, token); tSeg.Wait(); token = tSeg.Result.ContinuationToken; spfileEntities.AddRange(tSeg.Result); } while (token != null && (query.TakeCount == null || spfileEntities.Count < query.TakeCount.Value)); //!ct.IsCancellationRequested && //no sorting //spfileEntities.Sort((x, y) => String.Compare(x.Label, y.Label)); entityAsJson = JsonConvert.SerializeObject(spfileEntities, Formatting.Indented); } catch (Exception ex) { string exceptionMsg = "Discovery batch status GET exception: " + ex.Message; //log.Info("Exception occurred extracting text from uploaded file \r\nError: " + ex.Message); if (ex.InnerException != null) { exceptionMsg = exceptionMsg + "[" + ex.InnerException.Message + "]"; } _logger.LogError(exceptionMsg); return(StatusCode((int)System.Net.HttpStatusCode.InternalServerError)); } ObjectResult result = new ObjectResult(entityAsJson); return(result); }
public IActionResult Get([FromQuery] string filter) { string entityAsJson = ""; try { _logger.LogInformation("CPAPI: Get RecordAssociations By Discovery Filter"); // Deserialize the ontology filter DiscoveryFilters oFilter = new DiscoveryFilters(); if (filter != null && filter.Length > 0) { _logger.LogDebug("Deserializing record association keyphrase filter of length: " + filter.Length); oFilter = JsonConvert.DeserializeObject <DiscoveryFilters>(filter); } // Create the table if it doesn't exist. //log.Info("Getting table reference"); CloudTable table = Utils.GetCloudTable("stlprecordassociationkeyphrasesreverse", _logger); // Create a default query TableQuery <RecordAssociationKeyPhraseReverseEntity> query = new TableQuery <RecordAssociationKeyPhraseReverseEntity>(); if (oFilter.filters.Count > 0) { string combinedFilter = ""; // ***************************** // Check for Keyphrase filters // ***************************** foreach (DiscoveryFilter dfilter in oFilter.filters) { // Check for keyphrase filters if (dfilter.Type == DiscoveryFilterType.keyphrase) { string cleanFilterPKey = Utils.CleanTableKey(dfilter.Value); string pkquery = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, cleanFilterPKey); if (combinedFilter != "") { combinedFilter = TableQuery.CombineFilters(combinedFilter, TableOperators.Or, pkquery); } else { combinedFilter = pkquery; } } } // Create final combined query query = new TableQuery <RecordAssociationKeyPhraseReverseEntity>().Where(combinedFilter); } List <RecordAssociationKeyPhraseReverseEntity> recordAssociationEntities = new List <RecordAssociationKeyPhraseReverseEntity>(); TableContinuationToken token = null; var runningQuery = new TableQuery <RecordAssociationKeyPhraseReverseEntity>() { FilterString = query.FilterString, SelectColumns = query.SelectColumns }; do { runningQuery.TakeCount = query.TakeCount - recordAssociationEntities.Count; Task <TableQuerySegment <RecordAssociationKeyPhraseReverseEntity> > tSeg = table.ExecuteQuerySegmentedAsync <RecordAssociationKeyPhraseReverseEntity>(runningQuery, token); tSeg.Wait(); token = tSeg.Result.ContinuationToken; recordAssociationEntities.AddRange(tSeg.Result); } while (token != null && (query.TakeCount == null || recordAssociationEntities.Count < query.TakeCount.Value) && recordAssociationEntities.Count < 500); //!ct.IsCancellationRequested && recordAssociationEntities.Sort((x, y) => String.Compare(x.RowKey, y.RowKey)); // ********************************** // Check for file extension filters // ********************************** List <string> filetypeFilters = new List <string>(); foreach (DiscoveryFilter dfilter in oFilter.filters) { if (dfilter.Type == DiscoveryFilterType.filetype) { filetypeFilters.Add(dfilter.Value); } } if (filetypeFilters.Count > 0) { // Remove any file types not in the filter list for (int i = recordAssociationEntities.Count - 1; i >= 0; i--) { string fileExtension = System.IO.Path.GetExtension(recordAssociationEntities[i].ItemUri); if (fileExtension != null) { if (!filetypeFilters.Exists(e => e.Equals(fileExtension))) { // Remove the item recordAssociationEntities.Remove(recordAssociationEntities[i]); } } } } // ********************************** // Check for system filters // ********************************** List <string> systemFilters = new List <string>(); foreach (DiscoveryFilter dfilter in oFilter.filters) { if (dfilter.Type == DiscoveryFilterType.system) { systemFilters.Add(dfilter.Value.Replace("|", "/")); } } if (systemFilters.Count > 0) { // Remove any files not in the filtered system(s) for (int i = recordAssociationEntities.Count - 1; i >= 0; i--) { string itemUri = recordAssociationEntities[i].ItemUri; bool isItemInSystem = false; foreach (string system in systemFilters) { // Check if the item Uri starts with the system Uri if (itemUri.StartsWith(system)) { isItemInSystem = true; break; } } if (!isItemInSystem) { // Remove the item recordAssociationEntities.Remove(recordAssociationEntities[i]); } } } entityAsJson = JsonConvert.SerializeObject(recordAssociationEntities, Formatting.Indented); } catch (Exception ex) { string exceptionMsg = "Record Association By Discovery Filter GET exception: " + ex.Message; //log.Info("Exception occurred extracting text from uploaded file \r\nError: " + ex.Message); if (ex.InnerException != null) { exceptionMsg = exceptionMsg + "[" + ex.InnerException.Message + "]"; } _logger.LogError(exceptionMsg); return(StatusCode((int)System.Net.HttpStatusCode.InternalServerError)); } ObjectResult result = new ObjectResult(entityAsJson); return(result); }