internal static List <POCO.Files.CPFile> GetFilesAssociatedToRecordForBatchProcessing(DataConfig dataConfig, POCO.System system, POCO.Record record)
{
List <POCO.Files.CPFile> filesToProcess = new List <POCO.Files.CPFile>();
// Create the filter
string recordUri = record.RowKey;
if (!recordUri.EndsWith("|"))
{
recordUri += "|";
} // Standardise the url to end in a pipe '|' to help match files via partial Uri
List <DataFactory.Filter> filters = new List <DataFactory.Filter>();
DataFactory.Filter pkGTEFilter = new DataFactory.Filter("PartitionKey", recordUri, "ge");
DataFactory.Filter pkLTFilter = new DataFactory.Filter("PartitionKey", Utils.GetLessThanFilter(recordUri), "lt");
filters.Add(pkGTEFilter);
filters.Add(pkLTFilter);
// Check the system type
string thisPageId = string.Empty;
string nextPageId = string.Empty;
switch (system.Type)
{
case POCO.SystemType.SharePointOnline:
case POCO.SystemType.SharePointTeam:
case POCO.SystemType.SharePointOneDrive:
{
do
{
List <POCO.SPFile> spfiles = DataFactory.SharePointOnline.GetFiles(dataConfig, filters, thisPageId, 1000, out nextPageId);
// Convert to ICPFile format
foreach (POCO.SPFile fileToConvert in spfiles)
{
// Convert to ICPFile compatible object
POCO.Files.SPFile file = JsonConvert.DeserializeObject <POCO.Files.SPFile>(JsonConvert.SerializeObject(fileToConvert));
filesToProcess.Add(file);
}
thisPageId = nextPageId;
}while (nextPageId != string.Empty);
break;
}
case POCO.SystemType.GoogleDrive:
{
List <POCO.Files.GoogleFile> googlefiles = DataFactory.Google.GetFiles(dataConfig, filters);
// Convert to ICPFile format
foreach (POCO.Files.GoogleFile fileToConvert in googlefiles)
{
// Convert to ICPFile compatible object
POCO.Files.GoogleFile file = JsonConvert.DeserializeObject <POCO.Files.GoogleFile>(JsonConvert.SerializeObject(fileToConvert));
filesToProcess.Add(file);
}
break;
}
case POCO.SystemType.NTFSShare:
List <POCO.NTFSFile> ntfsFiles = DataFactory.NTFS.GetFiles(dataConfig, filters);
// Convert to ICPFile format
foreach (POCO.NTFSFile fileToConvert in ntfsFiles)
{
// Convert to ICPFile compatible object
POCO.Files.NTFSFile ntfsfile = JsonConvert.DeserializeObject <POCO.Files.NTFSFile>(JsonConvert.SerializeObject(fileToConvert));
filesToProcess.Add(ntfsfile);
}
break;
case POCO.SystemType.Basecamp:
// Get the Basecamp files
// URL format of the basecamp project is: https://<basecamp api>/<account number>/projects/<project id>.json
// all data for a project is in "buckets": https://<basecamp api>/<account number>/buckets/<project id>/<type of bucket>/<id>.json
// any data matching should use the buckets/<project id>>
string basecampFilesUri = record.RecordUri.Replace("projects", "buckets");
if (basecampFilesUri.ToLower().EndsWith(".json"))
{
basecampFilesUri = basecampFilesUri.Substring(0, basecampFilesUri.Length - 5);
}
if (!basecampFilesUri.EndsWith("|"))
{
basecampFilesUri += "|";
}
List <DataFactory.Filter> basecampFilters = new List <DataFactory.Filter>();
string pk = Utils.CleanTableKey(basecampFilesUri);
DataFactory.Filter basecampPK = new DataFactory.Filter("PartitionKey", pk, "eq");
basecampFilters.Add(basecampPK);
//DataFactory.Filter basecampPKLT = new DataFactory.Filter("PartitionKey", Utils.GetLessThanFilter(pk), "lt");
//basecampFilters.Add(basecampPKGE);
//basecampFilters.Add(basecampPKLT);
DataFactory.DataConfig basecampDataConfig = dataConfig;
List <POCO.BasecampDocumentEntity> basecampFiles = DataFactory.BasecampProject.GetFiles(basecampDataConfig, basecampFilters);
// Convert to ICPFile format
foreach (POCO.BasecampDocumentEntity fileToConvert in basecampFiles)
{
// Convert to ICPFile compatible object
//BasecampFile basefile = JsonConvert.DeserializeObject<BasecampFile>(JsonConvert.SerializeObject(fileToConvert));
POCO.Files.BasecampFile bf = new POCO.Files.BasecampFile();
bf.BatchGuid = Guid.Empty;
bf.BatchStatus = null;
bf.CreationTime = fileToConvert.TimeCreated;
bf.FolderUri = fileToConvert.PartitionKey;
bf.ItemUri = fileToConvert.ServerRelativeUrl;
bf.JsonFileProcessResult = null;
bf.LastModifiedTime = fileToConvert.TimeLastModified;
bf.Name = fileToConvert.Name;
bf.OrganisationId = Guid.Empty;
bf.PartitionKey = fileToConvert.PartitionKey;
bf.RowKey = fileToConvert.RowKey;
bf.SiteUrl = fileToConvert.ServerRelativeUrl;
bf.SourceFileName = fileToConvert.Name;
bf.SourceRelativeUrl = fileToConvert.ServerRelativeUrl;
bf.Title = fileToConvert.Title;
bf.UniqueId = fileToConvert.UniqueId;
filesToProcess.Add(bf);
}
break;
default:
{
throw new NotImplementedException("Unknown system type: " + system.Type);
}
}
return(filesToProcess);
}
public IActionResult GetByRecordAssociationFilter([FromQuery] string filter)
{
string entityAsJson = "";
List <POCO.RecordAssociationSubjectObject> subjectObjectEntities = new List <POCO.RecordAssociationSubjectObject>();
try
{
_logger.LogInformation("CPAPI: Get By Record Association Filter");
// Deserialize the ontology filter
Controllers.RecordAssociationFilter oFilter = new Controllers.RecordAssociationFilter();
if (filter != null && filter.Length > 0)
{
_logger.LogDebug("Deserializing Record Association filter of length: " + filter.Length);
oFilter = JsonConvert.DeserializeObject <Controllers.RecordAssociationFilter>(filter);
}
// Check if a filter has been supplied
if (oFilter.recordassociations.Count == 0)
{
// Nothing to filter by - return invalid filter
return(StatusCode((int)System.Net.HttpStatusCode.NoContent));
}
// Create the filters for the datafactory
List <DataFactory.Filter> filters = new List <DataFactory.Filter>();
if (oFilter.recordassociations.Count > 0)
{
string combinedFilter = "";
foreach (RecordAssociation rif in oFilter.recordassociations)
{
string cleanFilterPKey = Utils.CleanTableKey(rif.recorduri); // Utils.CleanTableKey(rif.recorduri);
string cleanFilterRKey = Utils.CleanTableKey(rif.recordassociationuri);
if (cleanFilterPKey != "")
{
DataFactory.Filter pkfilt = new DataFactory.Filter("PartitionKey", cleanFilterPKey, "eq");
filters.Add(pkfilt);
}
// Check if an item key has been provided
if (cleanFilterRKey != "")
{
DataFactory.Filter rkfilt = new DataFactory.Filter("RowKey", cleanFilterRKey, "eq");
filters.Add(rkfilt);
}
}
}
else
{
_logger.LogInformation("RecordKeyPhraseFilter query BLANK");
}
DataFactory.DataConfig dataCfg = Utils.GetDataConfig();
List <POCO.RecordAssociationSubjectObject> subjectobject = DataFactory.Record.GetRecordAssociationSubjectObject(dataCfg, filters);
//CloudTable table = Utils.GetCloudTableNoCreate("stlprecordassociationsubjectobject", _logger);
//// Create a default query
//TableQuery<SubjectObjectEntity> query = new TableQuery<SubjectObjectEntity>();
//string combinedFilter = "";
//if (oFilter.recordassociations.Count > 0)
//{
// foreach (string rafilter in oFilter.recordassociations)
// {
// string cleanFilterPKey = Utils.CleanTableKey(rafilter);
// string pkquery = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, cleanFilterPKey);
// if (combinedFilter != "")
// {
// combinedFilter = TableQuery.CombineFilters(combinedFilter, TableOperators.Or, pkquery);
// }
// else
// {
// combinedFilter = pkquery;
// }
// }
//}
//// Create final combined query
//query = new TableQuery<SubjectObjectEntity>().Where(combinedFilter);
//List<SubjectObjectEntity> SubjectObjectEntityEntities = new List<SubjectObjectEntity>();
//TableContinuationToken token = null;
//var runningQuery = new TableQuery<SubjectObjectEntity>()
//{
// FilterString = query.FilterString,
// SelectColumns = query.SelectColumns
//};
//do
//{
// runningQuery.TakeCount = query.TakeCount - SubjectObjectEntityEntities.Count;
// Task<TableQuerySegment<SubjectObjectEntity>> tSeg = table.ExecuteQuerySegmentedAsync<SubjectObjectEntity>(runningQuery, token);
// tSeg.Wait();
// token = tSeg.Result.ContinuationToken;
// SubjectObjectEntityEntities.AddRange(tSeg.Result);
//} while (token != null && (query.TakeCount == null || SubjectObjectEntityEntities.Count < query.TakeCount.Value) && SubjectObjectEntityEntities.Count < 1000); //!ct.IsCancellationRequested &&
// Return only distinct entrys
List <string> SubjectObjectEntityKeys = new List <string>();
foreach (POCO.RecordAssociationSubjectObject ne in subjectobject)
{
if (!SubjectObjectEntityKeys.Contains(ne.RowKey))
{
// Add the entity to the output
// and the key
subjectObjectEntities.Add(ne);
SubjectObjectEntityKeys.Add(ne.RowKey);
}
}
subjectObjectEntities.Sort((x, y) => String.Compare(x.RowKey, y.RowKey));
// Serialize
entityAsJson = JsonConvert.SerializeObject(subjectObjectEntities, Formatting.Indented);
}
catch (Exception ex)
{
string exceptionMsg = "Subject Object GET BY RECORD FILTER exception: " + ex.Message;
//log.Info("Exception occurred extracting text from uploaded file \r\nError: " + ex.Message);
if (ex.InnerException != null)
{
exceptionMsg = exceptionMsg + "[" + ex.InnerException.Message + "]";
}
_logger.LogError(exceptionMsg);
return(StatusCode((int)System.Net.HttpStatusCode.InternalServerError));
}
ObjectResult result = new ObjectResult(entityAsJson);
return(result);
}
public static void LogEventService(DataFactory.DataConfig providerConfig, string serviceModule, string serviceEvent)
{
// Store the table name suffix
string tableNameSuffix = DateTime.UtcNow.ToString(Utils.TableSuffixDateFormatYM);
// Create the data object
DateTime currentDateTime = DateTime.UtcNow;
string logid = Guid.NewGuid().ToString();
POCO.LogServiceEvent log = new POCO.LogServiceEvent();
log.PartitionKey = Utils.CleanTableKey(serviceModule);
log.RowKey = Utils.CleanTableKey(logid);
log.LogDateTime = currentDateTime.ToUniversalTime();
log.LogId = logid;
log.NodeIp = string.Empty;
log.NodeName = string.Empty;
log.ServiceEvent = serviceEvent;
log.ServiceModule = serviceModule;
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
// Cycle the tables each month to manage size
string tableName = AzureTableNames.DiagnosticsFileProcessing + tableNameSuffix;
AzureLogServiceEvent az = new AzureLogServiceEvent(log);
CloudTable table = Utils.GetCloudTable(providerConfig, tableName);
TableOperation operation = TableOperation.InsertOrReplace(az);
Task tUpdate = table.ExecuteAsync(operation);
tUpdate.Wait();
break;
case "internal.mongodb":
string mongoTableName = MongoTableNames.DiagnosticsFileProcessing + tableNameSuffix;
IMongoCollection <MongoLogServiceEvent> collection = Utils.GetMongoCollection <MongoLogServiceEvent>(providerConfig, mongoTableName);
// Create the data object
MongoLogServiceEvent mongoObject = new MongoLogServiceEvent();
mongoObject.PartitionKey = Utils.CleanTableKey(serviceModule);
mongoObject.RowKey = Utils.CleanTableKey(logid);
mongoObject.LogDateTime = currentDateTime.ToUniversalTime();
mongoObject.LogId = logid;
mongoObject.NodeIp = string.Empty;
mongoObject.NodeName = string.Empty;
mongoObject.ServiceEvent = serviceEvent;
mongoObject.ServiceModule = serviceModule;
// Insert
collection.InsertOne(mongoObject);
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
//TODO return id of new object if supported
return;
}
public IActionResult GetByFilter([FromQuery] string filter)
{
string entityAsJson = "";
try
{
_logger.LogInformation("CPAPI: Get");
// Deserialize the ontology filter
OntologyFilter oFilter = new OntologyFilter();
if (filter != null && filter.Length > 0)
{
_logger.LogDebug("Deserializing ontology filter of length: " + filter.Length);
oFilter = JsonConvert.DeserializeObject <OntologyFilter>(filter);
}
List <DataFactory.Filter> filters = new List <DataFactory.Filter>();
foreach (string of in oFilter.ontology)
{
string cleanFilterPKey = Utils.CleanTableKey(of);
DataFactory.Filter pkfilt = new DataFactory.Filter("PartitionKey", cleanFilterPKey, "eq");
filters.Add(pkfilt);
}
//CloudTable table = Utils.GetCloudTable("stlpontology", _logger);
//// Create a default query
//TableQuery<OntologyEntity> query = new TableQuery<OntologyEntity>();
//if (oFilter.ontology.Count>0)
//{
// string combinedFilter = "";
// foreach(string of in oFilter.ontology)
// {
// string cleanFilterPKey = Utils.CleanTableKey(of);
// string pkquery = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, cleanFilterPKey);
// if (combinedFilter!="")
// {
// combinedFilter = TableQuery.CombineFilters(combinedFilter, TableOperators.Or, pkquery);
// }
// else
// {
// combinedFilter = pkquery;
// }
// }
// // Create final combined query
// query = new TableQuery<OntologyEntity>().Where(combinedFilter);
//}
//List<OntologyEntity> ontologyEntities = new List<OntologyEntity>();
//TableContinuationToken token = null;
//var runningQuery = new TableQuery<OntologyEntity>()
//{
// FilterString = query.FilterString,
// SelectColumns = query.SelectColumns
//};
//do
//{
// runningQuery.TakeCount = query.TakeCount - ontologyEntities.Count;
// Task<TableQuerySegment<OntologyEntity>> tSeg = table.ExecuteQuerySegmentedAsync<OntologyEntity>(runningQuery, token);
// tSeg.Wait();
// token = tSeg.Result.ContinuationToken;
// ontologyEntities.AddRange(tSeg.Result);
//} while (token != null && (query.TakeCount == null || ontologyEntities.Count < query.TakeCount.Value)); //!ct.IsCancellationRequested &&
DataFactory.DataConfig datacfg = Utils.GetDataConfig();
List <POCO.Ontology> ontologyEntities = DataFactory.Ontology.GetOntology(datacfg, filters);
ontologyEntities.Sort((x, y) => String.Compare(x.Label, y.Label));
entityAsJson = JsonConvert.SerializeObject(ontologyEntities, Formatting.Indented);
}
catch (Exception ex)
{
string exceptionMsg = "Ontology GET exception: " + ex.Message;
//log.Info("Exception occurred extracting text from uploaded file \r\nError: " + ex.Message);
if (ex.InnerException != null)
{
exceptionMsg = exceptionMsg + "[" + ex.InnerException.Message + "]";
}
_logger.LogError(exceptionMsg);
return(StatusCode((int)System.Net.HttpStatusCode.InternalServerError));
}
ObjectResult result = new ObjectResult(entityAsJson);
return(result);
}
public IActionResult GetByFileProcessExceptionFilter([FromQuery] string filter)
{
string entityAsJson = "";
try
{
_logger.LogInformation("CPAPI: GetByFileProcessExceptionFilter");
// Deserialize the data filter
FileExceptionFilter oFilter = new FileExceptionFilter();
if (filter != null && filter.Length > 0)
{
string filterDecoded = System.Net.WebUtility.HtmlDecode(filter);
filterDecoded = System.Net.WebUtility.UrlDecode(filterDecoded);
_logger.LogDebug("Deserializing filter of length: " + filterDecoded.Length);
oFilter = JsonConvert.DeserializeObject <FileExceptionFilter>(filterDecoded);
}
// Validate the filter
if (string.IsNullOrEmpty(oFilter.systemuri) ||
string.IsNullOrEmpty(oFilter.exceptiontype))
{
_logger.LogWarning("GetByFileProcessExceptionFilter: systemuri or exceptiontype is blank");
return(StatusCode((int)System.Net.HttpStatusCode.BadRequest));
}
//Guid systemGuid = Guid.NewGuid();
//if (!Guid.TryParse(oFilter.systemid, out systemGuid))
//{
// _logger.LogWarning("GetByFileProcessExceptionFilter: systemid is invalid - expected Guid");
// return StatusCode((int)System.Net.HttpStatusCode.BadRequest);
//}
if (oFilter.exceptiontype.Length > 50)
{
_logger.LogWarning("GetByFileProcessExceptionFilter: exception type is invalid - too long");
return(StatusCode((int)System.Net.HttpStatusCode.BadRequest));
}
// Create system filter
List <DataFactory.Filter> filters = new List <DataFactory.Filter>();
DataFactory.Filter f1 = new DataFactory.Filter("PartitionKey", oFilter.systemuri, "eq");
filters.Add(f1);
DataFactory.DataConfig dataConfig = Utils.GetDataConfig();
List <POCO.System> systems = DataFactory.System.GetSystems(dataConfig, filters);
if (systems.Count != 1)
{
_logger.LogWarning("GetByFileProcessExceptionFilter: system count expected 1 instead of: " + systems.Count.ToString());
return(StatusCode((int)System.Net.HttpStatusCode.BadRequest));
}
// Get the data
List <POCO.FileProcessException> fileexceptions = DataFactory.Log.GetFileProcessException(dataConfig, systems[0], oFilter.exceptiontype);
// Convert to JSON and return
entityAsJson = JsonConvert.SerializeObject(fileexceptions);
}
catch (Exception ex)
{
string exceptionMsg = "GetByFileProcessExceptionFilter: exception: " + ex.Message;
//log.Info("Exception occurred extracting text from uploaded file \r\nError: " + ex.Message);
if (ex.InnerException != null)
{
exceptionMsg = exceptionMsg + "[" + ex.InnerException.Message + "]";
}
_logger.LogError(exceptionMsg);
return(StatusCode((int)System.Net.HttpStatusCode.InternalServerError));
}
ObjectResult result = new ObjectResult(entityAsJson);
return(result);
}
public IActionResult GetByReverseFilter([FromQuery] string filter)
{
string entityAsJson = "";
List <string> keyPhrases = new List <string>();
try
{
_logger.LogInformation("CPAPI: Get By Reverse KeyPhrase Filter");
// Deserialize the ontology filter
KeyPhraseFilter oFilter = new KeyPhraseFilter();
if (filter != null && filter.Length > 0)
{
_logger.LogDebug("Deserializing KeyPhrase filter of length: " + filter.Length);
oFilter = JsonConvert.DeserializeObject <KeyPhraseFilter>(filter);
}
List <string> wordsToFind = new List <string>();
// Create a default query
List <DataFactory.Filter> filters = new List <DataFactory.Filter>();
if (oFilter.keyphrases.Count > 0)
{
//string combinedFilter = "";
foreach (string rif in oFilter.keyphrases)
{
string cleanFilterPKey = Utils.CleanTableKey(rif);
//string cleanFilterRKey = Utils.CleanTableKey(rif);
DataFactory.Filter pkFilterGE = new DataFactory.Filter("PartitionKey", cleanFilterPKey, "ge");
DataFactory.Filter pkFilterLT = new DataFactory.Filter("PartitionKey", Utils.GetLessThanFilter(cleanFilterPKey), "lt");
filters.Add(pkFilterGE);
filters.Add(pkFilterLT);
// Also split any words in this term for later word search
string[] splitwords = cleanFilterPKey.Split(" ", StringSplitOptions.RemoveEmptyEntries);
wordsToFind.AddRange(splitwords);
}
}
_logger.LogInformation("GetByReverseFilter: getting keyphraseEntities for words=" + oFilter.keyphrases);
Stopwatch st = new Stopwatch();
st.Start();
List <POCO.RecordAssociationKeyPhraseReverse> keyphraseEntities = new List <POCO.RecordAssociationKeyPhraseReverse>();
DataFactory.DataConfig dataConfig = Utils.GetDataConfig();
keyphraseEntities = DataFactory.KeyPhrase.GetReverseKeyPhrases(dataConfig, filters);
st.Stop();
_logger.LogInformation("GetByReverseFilter: keyphraseEntities=" + keyphraseEntities.Count.ToString() + " timer=" + st.ElapsedMilliseconds.ToString() + "ms");
//// Now get reverse keyphrase words
//List<POCO.RecordAssociationKeyPhraseReverseWord> words = new List<POCO.RecordAssociationKeyPhraseReverseWord>();
//if (wordsToFind.Count>0)
//{
// // Find matches for each word and add to our keyphrase set
// foreach (string w in wordsToFind)
// {
// // Create filter and search
// List<DataFactory.Filter> wordfilter = new List<DataFactory.Filter>();
// DataFactory.Filter word = new DataFactory.Filter("PartitionKey", w, "eq");
// wordfilter.Add(word);
// _logger.LogInformation("GetByReverseFilter: getting word matches for word=" + w);
// st.Restart();
// string nextPageId = string.Empty;
// List<POCO.RecordAssociationKeyPhraseReverseWord> wordMatches = DataFactory.KeyPhrase.GetReverseKeyPhraseWords(dataConfig, wordfilter, string.Empty, 2000, out nextPageId);
// st.Stop();
// _logger.LogInformation("GetByReverseFilter: word=" + w + " wordMatches=" + wordMatches.Count.ToString() + " timer=" + st.ElapsedMilliseconds.ToString() + "ms");
// words.AddRange(wordMatches);
// }
//}
//// Check if we have any word matches
//if (words.Count>0)
//{
// foreach(POCO.RecordAssociationKeyPhraseReverseWord word in words)
// {
// // Split the rowkey
// string[] key = word.RowKey.Split("|||");
// if (key.Length==3)
// {
// // Create a new keyphrase reverse object using this word match
// // multi-part key format is [record assoc uri] [key phrase] [key phrase location]
// POCO.RecordAssociationKeyPhraseReverse kpreverse = new POCO.RecordAssociationKeyPhraseReverse(key[1], key[0], key[2]);
// keyphraseEntities.Add(kpreverse);
// }
// }
//}
// Return only distinct keyphrases
_logger.LogInformation("GetByReverseFilter: creating return data");
st.Restart();
foreach (POCO.RecordAssociationKeyPhraseReverse kp in keyphraseEntities)
{
if (!keyPhrases.Contains(kp.PartitionKey))
{
keyPhrases.Add(kp.PartitionKey);
}
}
keyPhrases.Sort((x, y) => String.Compare(x, y));
// Serialize
entityAsJson = JsonConvert.SerializeObject(keyPhrases, Formatting.Indented);
st.Stop();
_logger.LogInformation("GetByReverseFilter: created return data timer=" + st.ElapsedMilliseconds.ToString() + "ms");
}
catch (Exception ex)
{
string exceptionMsg = "KeyPhrase GET exception: " + ex.Message;
//log.Info("Exception occurred extracting text from uploaded file \r\nError: " + ex.Message);
if (ex.InnerException != null)
{
exceptionMsg = exceptionMsg + "[" + ex.InnerException.Message + "]";
}
_logger.LogError(exceptionMsg);
return(StatusCode((int)System.Net.HttpStatusCode.InternalServerError));
}
ObjectResult result = new ObjectResult(entityAsJson);
return(result);
}
