public IActionResult GetSavedDiscovery([FromQuery] string filter)
{
string entityAsJson = "";
try
{
_logger.LogInformation("CPAPI: Get");
// Deserialize the filter
DiscoveryFilters oFilter = new DiscoveryFilters();
if (filter != null && filter.Length > 0)
{
_logger.LogDebug("Deserializing filter of length: " + filter.Length);
oFilter = JsonConvert.DeserializeObject <DiscoveryFilters>(filter);
}
string storageAccountConnectionString = Utils.GetSecretOrEnvVar(ConfigurationProperties.AzureStorageAccountConnectionString, Configuration, _logger).Trim();
// validate tika base address
if (storageAccountConnectionString == "")
{
_logger.LogWarning("Azure storage account connection string not set");
return(StatusCode((int)System.Net.HttpStatusCode.InternalServerError));
}
else
{
_logger.LogDebug("Azure storage account connection string loaded");
}
// Process the records
CloudStorageAccount account = CloudStorageAccount.Parse(storageAccountConnectionString);
// Create the table client.
//log.Info("Creating cloud table client");
CloudTableClient tableClient = account.CreateCloudTableClient();
// Create the table if it doesn't exist.
//log.Info("Getting table reference");
CloudTable table = tableClient.GetTableReference("stlpdiscovery");
Task tCreate = table.CreateIfNotExistsAsync();
tCreate.Wait();
// Create a default query
TableQuery <DiscoveryEntity> query = new TableQuery <DiscoveryEntity>();
if (oFilter.filters.Count > 0)
{
string combinedFilter = "";
foreach (DiscoveryFilter entry in oFilter.filters)
{
string cleanFilterPKey = Utils.CleanTableKey(entry.Value);
string pkquery = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, cleanFilterPKey);
combinedFilter = pkquery;
}
// Create final combined query
query = new TableQuery <DiscoveryEntity>().Where(combinedFilter);
}
List <DiscoveryEntity> spfileEntities = new List <DiscoveryEntity>();
TableContinuationToken token = null;
var runningQuery = new TableQuery <DiscoveryEntity>()
{
FilterString = query.FilterString,
SelectColumns = query.SelectColumns
};
do
{
runningQuery.TakeCount = query.TakeCount - spfileEntities.Count;
Task <TableQuerySegment <DiscoveryEntity> > tSeg = table.ExecuteQuerySegmentedAsync <DiscoveryEntity>(runningQuery, token);
tSeg.Wait();
token = tSeg.Result.ContinuationToken;
spfileEntities.AddRange(tSeg.Result);
} while (token != null && (query.TakeCount == null || spfileEntities.Count < query.TakeCount.Value)); //!ct.IsCancellationRequested &&
//no sorting
//spfileEntities.Sort((x, y) => String.Compare(x.Label, y.Label));
entityAsJson = JsonConvert.SerializeObject(spfileEntities, Formatting.Indented);
}
catch (Exception ex)
{
string exceptionMsg = "Discovery batch status GET exception: " + ex.Message;
//log.Info("Exception occurred extracting text from uploaded file \r\nError: " + ex.Message);
if (ex.InnerException != null)
{
exceptionMsg = exceptionMsg + "[" + ex.InnerException.Message + "]";
}
_logger.LogError(exceptionMsg);
return(StatusCode((int)System.Net.HttpStatusCode.InternalServerError));
}
ObjectResult result = new ObjectResult(entityAsJson);
return(result);
}
public IActionResult Get([FromQuery] string filter)
{
string entityAsJson = "";
try
{
_logger.LogInformation("CPAPI: Get RecordAssociations By Discovery Filter");
// Deserialize the ontology filter
DiscoveryFilters oFilter = new DiscoveryFilters();
if (filter != null && filter.Length > 0)
{
_logger.LogDebug("Deserializing record association keyphrase filter of length: " + filter.Length);
oFilter = JsonConvert.DeserializeObject <DiscoveryFilters>(filter);
}
// Create the table if it doesn't exist.
//log.Info("Getting table reference");
CloudTable table = Utils.GetCloudTable("stlprecordassociationkeyphrasesreverse", _logger);
// Create a default query
TableQuery <RecordAssociationKeyPhraseReverseEntity> query = new TableQuery <RecordAssociationKeyPhraseReverseEntity>();
if (oFilter.filters.Count > 0)
{
string combinedFilter = "";
// *****************************
// Check for Keyphrase filters
// *****************************
foreach (DiscoveryFilter dfilter in oFilter.filters)
{
// Check for keyphrase filters
if (dfilter.Type == DiscoveryFilterType.keyphrase)
{
string cleanFilterPKey = Utils.CleanTableKey(dfilter.Value);
string pkquery = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, cleanFilterPKey);
if (combinedFilter != "")
{
combinedFilter = TableQuery.CombineFilters(combinedFilter, TableOperators.Or, pkquery);
}
else
{
combinedFilter = pkquery;
}
}
}
// Create final combined query
query = new TableQuery <RecordAssociationKeyPhraseReverseEntity>().Where(combinedFilter);
}
List <RecordAssociationKeyPhraseReverseEntity> recordAssociationEntities = new List <RecordAssociationKeyPhraseReverseEntity>();
TableContinuationToken token = null;
var runningQuery = new TableQuery <RecordAssociationKeyPhraseReverseEntity>()
{
FilterString = query.FilterString,
SelectColumns = query.SelectColumns
};
do
{
runningQuery.TakeCount = query.TakeCount - recordAssociationEntities.Count;
Task <TableQuerySegment <RecordAssociationKeyPhraseReverseEntity> > tSeg = table.ExecuteQuerySegmentedAsync <RecordAssociationKeyPhraseReverseEntity>(runningQuery, token);
tSeg.Wait();
token = tSeg.Result.ContinuationToken;
recordAssociationEntities.AddRange(tSeg.Result);
} while (token != null && (query.TakeCount == null || recordAssociationEntities.Count < query.TakeCount.Value) && recordAssociationEntities.Count < 500); //!ct.IsCancellationRequested &&
recordAssociationEntities.Sort((x, y) => String.Compare(x.RowKey, y.RowKey));
// **********************************
// Check for file extension filters
// **********************************
List <string> filetypeFilters = new List <string>();
foreach (DiscoveryFilter dfilter in oFilter.filters)
{
if (dfilter.Type == DiscoveryFilterType.filetype)
{
filetypeFilters.Add(dfilter.Value);
}
}
if (filetypeFilters.Count > 0)
{
// Remove any file types not in the filter list
for (int i = recordAssociationEntities.Count - 1; i >= 0; i--)
{
string fileExtension = System.IO.Path.GetExtension(recordAssociationEntities[i].ItemUri);
if (fileExtension != null)
{
if (!filetypeFilters.Exists(e => e.Equals(fileExtension)))
{
// Remove the item
recordAssociationEntities.Remove(recordAssociationEntities[i]);
}
}
}
}
// **********************************
// Check for system filters
// **********************************
List <string> systemFilters = new List <string>();
foreach (DiscoveryFilter dfilter in oFilter.filters)
{
if (dfilter.Type == DiscoveryFilterType.system)
{
systemFilters.Add(dfilter.Value.Replace("|", "/"));
}
}
if (systemFilters.Count > 0)
{
// Remove any files not in the filtered system(s)
for (int i = recordAssociationEntities.Count - 1; i >= 0; i--)
{
string itemUri = recordAssociationEntities[i].ItemUri;
bool isItemInSystem = false;
foreach (string system in systemFilters)
{
// Check if the item Uri starts with the system Uri
if (itemUri.StartsWith(system))
{
isItemInSystem = true;
break;
}
}
if (!isItemInSystem)
{
// Remove the item
recordAssociationEntities.Remove(recordAssociationEntities[i]);
}
}
}
entityAsJson = JsonConvert.SerializeObject(recordAssociationEntities, Formatting.Indented);
}
catch (Exception ex)
{
string exceptionMsg = "Record Association By Discovery Filter GET exception: " + ex.Message;
//log.Info("Exception occurred extracting text from uploaded file \r\nError: " + ex.Message);
if (ex.InnerException != null)
{
exceptionMsg = exceptionMsg + "[" + ex.InnerException.Message + "]";
}
_logger.LogError(exceptionMsg);
return(StatusCode((int)System.Net.HttpStatusCode.InternalServerError));
}
ObjectResult result = new ObjectResult(entityAsJson);
return(result);
}
