public static List <POCO.FileProcessException> GetFileProcessException(DataConfig providerConfig, POCO.System system, string exceptionType)
{
string tableName = string.Empty;
List <POCO.FileProcessException> docs = new List <POCO.FileProcessException>();
// Create filter
List <Filter> filters = new List <Filter>();
Filter pkfilt = new Filter("PartitionKey", Utilities.Converters.CleanTableKey(exceptionType), "eq");
filters.Add(pkfilt);
switch (providerConfig.ProviderType)
{
case ProviderType.Azure:
// Calcuate table name based on System Id and file process result (partitionkey)
tableName = "stlpfileexception-" + Utilities.Converters.CleanTableKey(exceptionType) + "-" + system.SystemId.ToString();
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureFileProcessException> azdata = new List <AzureFileProcessException>();
AzureTableAdaptor <AzureFileProcessException> adaptor = new AzureTableAdaptor <AzureFileProcessException>();
azdata = adaptor.ReadTableData(providerConfig, tableName, combinedFilter);
foreach (var doc in azdata)
{
docs.Add(doc.Value);
}
break;
case ProviderType.Mongo:
// Calcuate table name based on System Id and file process result (partitionkey)
tableName = "fileexception-" + Utilities.Converters.CleanTableKey(exceptionType) + "-" + system.SystemId.ToString();
var collection = Utils.GetMongoCollection <MongoFileProcessException_read>(providerConfig, tableName);
FilterDefinition <MongoFileProcessException_read> filter = Utils.GenerateMongoFilter <MongoFileProcessException_read>(filters);
var documents = collection.Find(filter).ToList();
foreach (var doc in documents)
{
docs.Add(doc);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(docs);
}
public static List <POCO.LogClassification> GetLogClassification(DataConfig providerConfig, List <Filter> filters, string tableNameSuffix_yyyymm)
{
// Validate the YYYYMM suffix
if (tableNameSuffix_yyyymm.Length != 6)
{
throw new ApplicationException("Invalid size for table suffix, must be YYYYMM");
}
string tableName = string.Empty;
List <POCO.LogClassification> docs = new List <POCO.LogClassification>();
switch (providerConfig.ProviderType)
{
case ProviderType.Azure:
tableName = AzureTableNames.LogClassification + tableNameSuffix_yyyymm;
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureLogClassification> azdata = new List <AzureLogClassification>();
AzureTableAdaptor <AzureLogClassification> adaptor = new AzureTableAdaptor <AzureLogClassification>();
azdata = adaptor.ReadTableData(providerConfig, tableName, combinedFilter);
foreach (var doc in azdata)
{
docs.Add(doc.Value);
}
break;
case ProviderType.Mongo:
tableName = MongoTableNames.LogClassification + tableNameSuffix_yyyymm;
var collection = Utils.GetMongoCollection <MongoLogClassification_Read>(providerConfig, tableName);
FilterDefinition <MongoLogClassification_Read> filter = Utils.GenerateMongoFilter <MongoLogClassification_Read>(filters);
var documents = collection.Find(filter).ToList();
foreach (var doc in documents)
{
docs.Add(doc);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(docs);
}
public static POCO.OntologyMatchStatus GetOntologyMatchStatus(DataConfig providerConfig, RecordToRecordAssociation recassoc, OntologyAssigned ontology)
{
POCO.OntologyMatchStatus matchStatus = null;
// Create a filter to match the Ontology provided
List <Filter> filters = new List <Filter>();
Filter pkFilter = new Filter("PartitionKey", ontology.RowKey, "eq");
filters.Add(pkFilter);
Filter rkFilter = new Filter("RowKey", recassoc.RowKey, "eq");
filters.Add(rkFilter);
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureOntologyMatchStatus> azs = new List <AzureOntologyMatchStatus>();
AzureTableAdaptor <AzureOntologyMatchStatus> az = new AzureTableAdaptor <AzureOntologyMatchStatus>();
azs = az.ReadTableData(providerConfig, AzureTableNames.OntologyMatchStatus, combinedFilter);
if (azs.Count > 0)
{
matchStatus = azs[0].Value;
}
return(matchStatus);
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoOntologyMatchStatus>(providerConfig, MongoTableNames.OntologyMatchStatus);
FilterDefinition <MongoOntologyMatchStatus> filter = Utils.GenerateMongoFilter <MongoOntologyMatchStatus>(filters);
var documents = collection.Find(filter).ToList();
if (documents.Count > 0)
{
matchStatus = documents[0];
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(matchStatus);
}
public static List <POCO.SakaiFile> GetFiles(DataConfig providerConfig, List <Filter> filters)
{
List <POCO.SakaiFile> files = new List <POCO.SakaiFile>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureSakaiFile> azdata = new List <AzureSakaiFile>();
AzureTableAdaptor <AzureSakaiFile> adaptor = new AzureTableAdaptor <AzureSakaiFile>();
azdata = adaptor.ReadTableData(providerConfig, SakaiResource.AzureTableNames.SakaiFiles, combinedFilter);
foreach (var doc in azdata)
{
files.Add(doc.Value);
}
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoSakaiFile>(providerConfig, SakaiResource.MongoTableNames.SakaiFiles);
FilterDefinition <MongoSakaiFile> filter = Utils.GenerateMongoFilter <MongoSakaiFile>(filters);
//DEBUG output the filter values
//foreach (Castlepoint.DataFactory.Filter debugFilter in filters)
//{
// // Output the filter field names and values
// Console.WriteLine("DEBUG filter: " + debugFilter.FieldName + " : " + debugFilter.FieldValue);
//}
var documents = collection.Find(filter).ToList();
foreach (var SakaiFile in documents)
{
files.Add(SakaiFile);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(files);
}
public static List <POCO.OntologyAssigned> GetRecordAssignedOntology(DataConfig providerConfig, POCO.Record record)
{
List <POCO.OntologyAssigned> ontologies = new List <POCO.OntologyAssigned>();
List <Filter> filters = new List <Filter>();
Filter systemFilter = new Filter("PartitionKey", Utils.CleanTableKey(record.PartitionKey), "eq");
filters.Add(systemFilter);
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureOntologyAssigned> azs = new List <AzureOntologyAssigned>();
AzureTableAdaptor <AzureOntologyAssigned> az = new AzureTableAdaptor <AzureOntologyAssigned>();
azs = az.ReadTableData(providerConfig, AzureTableNames.RecordsAssignedOntology, combinedFilter);
foreach (var doc in azs)
{
ontologies.Add(doc.Value);
}
return(ontologies);
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoOntologyAssigned>(providerConfig, MongoTableNames.RecordsAssignedOntology);
FilterDefinition <MongoOntologyAssigned> filter = Utils.GenerateMongoFilter <MongoOntologyAssigned>(filters);
var documents = collection.Find(filter).ToList();
foreach (var ontology in documents)
{
ontologies.Add(ontology);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(ontologies);
}
public static List <POCO.OntologyTerm> GetOntologyTerms(DataConfig providerConfig, string ontologyUri)
{
List <POCO.OntologyTerm> ontologyTerms = new List <POCO.OntologyTerm>();
// Create a filter to match the Ontology provided
List <Filter> filters = new List <Filter>();
Filter pkFilter = new Filter("PartitionKey", ontologyUri, "eq");
filters.Add(pkFilter);
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureOntologyTerm> azs = new List <AzureOntologyTerm>();
AzureTableAdaptor <AzureOntologyTerm> az = new AzureTableAdaptor <AzureOntologyTerm>();
azs = az.ReadTableData(providerConfig, AzureTableNames.OntologyTerms, combinedFilter);
foreach (var doc in azs)
{
ontologyTerms.Add(doc.Value);
}
return(ontologyTerms);
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoOntologyTerm>(providerConfig, MongoTableNames.OntologyTerms);
FilterDefinition <MongoOntologyTerm> filter = Utils.GenerateMongoFilter <MongoOntologyTerm>(filters);
var documents = collection.Find(filter).ToList();
foreach (var ontology in documents)
{
ontologyTerms.Add(ontology);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(ontologyTerms);
}
public static List <POCO.RecordAuthorityFilter> GetAssignedForRecord(DataConfig providerConfig, POCO.Record record)
{
List <POCO.RecordAuthorityFilter> assignedrecordauthoritys = new List <POCO.RecordAuthorityFilter>();
List <Filter> filters = new List <Filter>();
Filter pkFilter = new Filter("PartitionKey", record.RowKey, "eq");
filters.Add(pkFilter);
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureRecordAuthorityFilter> azdata = new List <AzureRecordAuthorityFilter>();
AzureTableAdaptor <AzureRecordAuthorityFilter> adaptor = new AzureTableAdaptor <AzureRecordAuthorityFilter>();
azdata = adaptor.ReadTableData(providerConfig, AzureTableNames.RecordsAssignedRecordsAuthority, combinedFilter);
foreach (var doc in azdata)
{
assignedrecordauthoritys.Add(doc.Value);
}
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoRecordAuthorityFilter>(providerConfig, MongoTableNames.RecordsAssignedRecordsAuthority);
FilterDefinition <MongoRecordAuthorityFilter> filter = Utils.GenerateMongoFilter <MongoRecordAuthorityFilter>(filters);
var documents = collection.Find(filter).ToList();
foreach (var assignedra in documents)
{
assignedrecordauthoritys.Add(assignedra);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(assignedrecordauthoritys);
}
public static List <POCO.Ontology> GetOntology(DataConfig providerConfig, List <Filter> filters)
{
List <POCO.Ontology> ontologies = new List <POCO.Ontology>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureOntology> azs = new List <AzureOntology>();
AzureTableAdaptor <AzureOntology> az = new AzureTableAdaptor <AzureOntology>();
azs = az.ReadTableData(providerConfig, AzureTableNames.Ontology, combinedFilter);
foreach (var doc in azs)
{
ontologies.Add(doc.Value);
}
return(ontologies);
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoOntology>(providerConfig, MongoTableNames.Ontology);
FilterDefinition <MongoOntology> filter = Utils.GenerateMongoFilter <MongoOntology>(filters);
var documents = collection.Find(filter).ToList();
foreach (var o in documents)
{
ontologies.Add(o);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(ontologies);
}
public static List <POCO.Account.Managed> GetByFilter(DataConfig providerConfig, List <Filter> filters)
{
List <POCO.Account.Managed> auditEntries = new List <POCO.Account.Managed>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureAccountManaged> azdata = new List <AzureAccountManaged>();
AzureTableAdaptor <AzureAccountManaged> adaptor = new AzureTableAdaptor <AzureAccountManaged>();
azdata = adaptor.ReadTableData(providerConfig, POCO.TableNames.Azure.Account.ManagedAccount, combinedFilter);
foreach (var doc in azdata)
{
auditEntries.Add(doc.Value);
}
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoAccountManaged>(providerConfig, POCO.TableNames.Mongo.Account.ManagedAccount);
FilterDefinition <MongoAccountManaged> filter = Utils.GenerateMongoFilter <MongoAccountManaged>(filters);
var documents = collection.Find(filter).ToList();
foreach (var logentry in documents)
{
auditEntries.Add(logentry);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(auditEntries);
}
public static List <POCO.O365.EOFolderEntity> GetFolders(DataConfig providerConfig, List <Filter> filters)
{
List <POCO.O365.EOFolderEntity> folderEnt = new List <POCO.O365.EOFolderEntity>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureEOFolder> azdata = new List <AzureEOFolder>();
AzureTableAdaptor <AzureEOFolder> adaptor = new AzureTableAdaptor <AzureEOFolder>();
azdata = adaptor.ReadTableData(providerConfig, AzureTableNames.EOFolder, combinedFilter);
foreach (var doc in azdata)
{
folderEnt.Add(doc.Value);
}
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoEOFolderEntity>(providerConfig, MongoTableNames.EOFolder);
FilterDefinition <MongoEOFolderEntity> filter = Utils.GenerateMongoFilter <MongoEOFolderEntity>(filters);
//TODO paging
var documents = collection.Find(filter).Sort("{\"_id\":1}").Limit(1000).ToList();
foreach (var doc in documents)
{
folderEnt.Add(doc);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(folderEnt);
}
public static List <POCO.CPDiagnostic> GetDiagnosticEntries(DataConfig providerConfig, List <Filter> filters)
{
List <POCO.CPDiagnostic> diags = new List <POCO.CPDiagnostic>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureDiagnostic> azdata = new List <AzureDiagnostic>();
AzureTableAdaptor <AzureDiagnostic> adaptor = new AzureTableAdaptor <AzureDiagnostic>();
azdata = adaptor.ReadTableData(providerConfig, AzureTableNames.FileProcessing + DateTime.UtcNow.ToString(Utils.TableSuffixDateFormatYM), combinedFilter);
foreach (var doc in azdata)
{
diags.Add(doc.Value);
}
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoDiagnostic>(providerConfig, MongoTableNames.FileProcessing + DateTime.UtcNow.ToString(Utils.TableSuffixDateFormatYM));
FilterDefinition <MongoDiagnostic> filter = Utils.GenerateMongoFilter <MongoDiagnostic>(filters);
var documents = collection.Find(filter).ToList();
foreach (var keyphrase in documents)
{
diags.Add(keyphrase);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(diags);
}
public static List <POCO.SharePoint.SharePointWebInfo> GetWebInfo(DataConfig providerConfig, List <Filter> filters)
{
List <POCO.SharePoint.SharePointWebInfo> webInfo = new List <POCO.SharePoint.SharePointWebInfo>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureSharePointWebInfo> azdata = new List <AzureSharePointWebInfo>();
AzureTableAdaptor <AzureSharePointWebInfo> adaptor = new AzureTableAdaptor <AzureSharePointWebInfo>();
azdata = adaptor.ReadTableData(providerConfig, AzureTableNames.SPTracking, combinedFilter);
foreach (var doc in azdata)
{
webInfo.Add(doc.Value);
}
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoSharePointWebInfo>(providerConfig, MongoTableNames.SPTracking);
FilterDefinition <MongoSharePointWebInfo> filter = Utils.GenerateMongoFilter <MongoSharePointWebInfo>(filters);
var documents = collection.Find(filter).Sort("{\"_id\":1}").Limit(1000).ToList();
foreach (var doc in documents)
{
webInfo.Add(doc);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(webInfo);
}
public static List <POCO.RecordProcessingHistory> GetForRecord(DataConfig providerConfig, List <Filter> filters)
{
List <POCO.RecordProcessingHistory> ProcessingHistory = new List <POCO.RecordProcessingHistory>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureRecordProcessingHistory> azdata = new List <AzureRecordProcessingHistory>();
AzureTableAdaptor <AzureRecordProcessingHistory> adaptor = new AzureTableAdaptor <AzureRecordProcessingHistory>();
azdata = adaptor.ReadTableData(providerConfig, AzureTableNames.RecordProcessingHistory, combinedFilter);
foreach (var doc in azdata)
{
ProcessingHistory.Add(doc.Value);
}
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoRecordProcessingHistory>(providerConfig, MongoTableNames.RecordProcessingHistory);
FilterDefinition <MongoRecordProcessingHistory> filter = Utils.GenerateMongoFilter <MongoRecordProcessingHistory>(filters);
var documents = collection.Find(filter).ToList();
foreach (var logentry in documents)
{
ProcessingHistory.Add(logentry);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(ProcessingHistory);
}
public static List <POCO.FileAlert> GetFileAlert(DataConfig providerConfig, string tableName, List <Filter> filters)
{
List <POCO.FileAlert> batchstatus = new List <POCO.FileAlert>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureFileAlert> azdata = new List <AzureFileAlert>();
AzureTableAdaptor <AzureFileAlert> adaptor = new AzureTableAdaptor <AzureFileAlert>();
azdata = adaptor.ReadTableData(providerConfig, tableName, combinedFilter);
foreach (var doc in azdata)
{
batchstatus.Add(doc.Value);
}
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoFileAlert>(providerConfig, tableName);
FilterDefinition <MongoFileAlert> filter = Utils.GenerateMongoFilter <MongoFileAlert>(filters);
var documents = collection.Find(filter).ToList();
foreach (var doc in documents)
{
batchstatus.Add(doc);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(batchstatus);
}
public static List <POCO.LogLastAction> GetLastAction(DataConfig providerConfig, List <Filter> filters)
{
List <POCO.LogLastAction> docs = new List <POCO.LogLastAction>();
switch (providerConfig.ProviderType)
{
case ProviderType.Azure:
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureLastAction> azdata = new List <AzureLastAction>();
AzureTableAdaptor <AzureLastAction> adaptor = new AzureTableAdaptor <AzureLastAction>();
azdata = adaptor.ReadTableData(providerConfig, AzureTableNames.LogLastAction, combinedFilter);
foreach (var doc in azdata)
{
docs.Add(doc.Value);
}
break;
case ProviderType.Mongo:
var collection = Utils.GetMongoCollection <MongoLastAction>(providerConfig, MongoTableNames.LogLastAction);
FilterDefinition <MongoLastAction> filter = Utils.GenerateMongoFilter <MongoLastAction>(filters);
var documents = collection.Find(filter).ToList();
foreach (var doc in documents)
{
docs.Add(doc);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(docs);
}
public static List <POCO.RecordAssociationKeyPhraseReverse> GetReverseKeyPhrases(DataConfig providerConfig, List <Filter> filters)
{
List <POCO.RecordAssociationKeyPhraseReverse> keyphrases = new List <POCO.RecordAssociationKeyPhraseReverse>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureRecordAssociationKeyPhraseReverse> azdata = new List <AzureRecordAssociationKeyPhraseReverse>();
AzureTableAdaptor <AzureRecordAssociationKeyPhraseReverse> adaptor = new AzureTableAdaptor <AzureRecordAssociationKeyPhraseReverse>();
azdata = adaptor.ReadTableData(providerConfig, Record.AzureTableNames.RecordAssociationKeyPhraseReverse, combinedFilter);
foreach (var doc in azdata)
{
keyphrases.Add(doc.Value);
}
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoRecordAssociationKeyPhraseReverse>(providerConfig, Record.MongoTableNames.RecordAssociationKeyPhraseReverse);
FilterDefinition <MongoRecordAssociationKeyPhraseReverse> filter = Utils.GenerateMongoFilter <MongoRecordAssociationKeyPhraseReverse>(filters);
var documents = collection.Find(filter).ToList();
foreach (var keyphrase in documents)
{
keyphrases.Add(keyphrase);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(keyphrases);
}
public static List <POCO.System> GetSystems(DataConfig providerConfig, List <Filter> filters)
{
List <POCO.System> systems = new List <POCO.System>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureSystem> azs = new List <AzureSystem>();
AzureTableAdaptor <AzureSystem> az = new AzureTableAdaptor <AzureSystem>();
azs = az.ReadTableData(providerConfig, System.TableNames.System, combinedFilter);
foreach (var doc in azs)
{
systems.Add(doc.Value);
}
return(systems);
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoSystem>(providerConfig, System.TableNames.System);
FilterDefinition <MongoSystem> filter = Utils.GenerateMongoFilter <MongoSystem>(filters);
var documents = collection.Find(filter).ToList();
foreach (var system in documents)
{
systems.Add(system);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(systems);
}
public static List <POCO.CaptureRegexMatch> GetRegexMatches(DataConfig providerConfig, List <DataFactory.Filter> filters)
{
List <POCO.CaptureRegexMatch> regexMatches = new List <POCO.CaptureRegexMatch>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureRegexMatch> azs = new List <AzureRegexMatch>();
AzureTableAdaptor <AzureRegexMatch> az = new AzureTableAdaptor <AzureRegexMatch>();
azs = az.ReadTableData(providerConfig, AzureTableNames.RecordAssociationRegexMatch, combinedFilter);
foreach (var doc in azs)
{
regexMatches.Add(doc.Value);
}
return(regexMatches);
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoRegexMatch>(providerConfig, MongoTableNames.RecordAssociationRegexMatch);
FilterDefinition <MongoRegexMatch> filter = Utils.GenerateMongoFilter <MongoRegexMatch>(filters);
var documents = collection.Find(filter).ToList();
foreach (var ontology in documents)
{
regexMatches.Add(ontology);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(regexMatches);
}
public static List <POCO.SakaiFolder> GetFolders(DataConfig providerConfig, List <Filter> filters)
{
List <POCO.SakaiFolder> folders = new List <POCO.SakaiFolder>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureSakaiFolder> azdata = new List <AzureSakaiFolder>();
AzureTableAdaptor <AzureSakaiFolder> adaptor = new AzureTableAdaptor <AzureSakaiFolder>();
azdata = adaptor.ReadTableData(providerConfig, SakaiResource.AzureTableNames.SakaiFolders, combinedFilter);
foreach (var doc in azdata)
{
folders.Add(doc.Value);
}
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoSakaiFolder>(providerConfig, SakaiResource.MongoTableNames.SakaiFolders);
FilterDefinition <MongoSakaiFolder> filter = Utils.GenerateMongoFilter <MongoSakaiFolder>(filters);
var documents = collection.Find(filter).ToList();
foreach (var ntfsfolder in documents)
{
folders.Add(ntfsfolder);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(folders);
}
public static List <POCO.ProcessingBatchStatus> GetProcessingStatus(DataConfig providerConfig)
{
List <POCO.ProcessingBatchStatus> processingStatus = new List <POCO.ProcessingBatchStatus>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
List <AzureProcessingBatchStatus> azs = new List <AzureProcessingBatchStatus>();
AzureTableAdaptor <AzureProcessingBatchStatus> az = new AzureTableAdaptor <AzureProcessingBatchStatus>();
azs = az.ReadTableData(providerConfig, AzureTableNames.SystemProcessingStatus);
foreach (var doc in azs)
{
processingStatus.Add(doc.Value);
}
return(processingStatus);
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoProcessingBatchStatus>(providerConfig, MongoTableNames.SystemProcessingStatus);
var documents = collection.Find(new BsonDocument()).ToList();
foreach (var doc in documents)
{
processingStatus.Add(doc);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(processingStatus);
}
public static List <POCO.SystemNodeConfig> GetAllBatchNodeConfig(DataConfig providerConfig)
{
List <POCO.SystemNodeConfig> systems = new List <POCO.SystemNodeConfig>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
List <AzureSystemBatchConfig> azs = new List <AzureSystemBatchConfig>();
AzureTableAdaptor <AzureSystemBatchConfig> az = new AzureTableAdaptor <AzureSystemBatchConfig>();
azs = az.ReadTableData(providerConfig, POCO.TableNames.Azure.System.BatchConfig);
foreach (var doc in azs)
{
systems.Add(doc.Value);
}
return(systems);
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoSystemBatchConfig>(providerConfig, POCO.TableNames.Mongo.System.BatchConfig);
var documents = collection.Find(new BsonDocument()).ToList();
foreach (var system in documents)
{
systems.Add(system);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(systems);
}
public static List <POCO.RecordAuthorityKeyPhrase> GetKeyPhrases(DataConfig providerConfig, List <POCO.RecordAuthorityFilter> raFilters)
{
List <POCO.RecordAuthorityKeyPhrase> keyphrases = new List <POCO.RecordAuthorityKeyPhrase>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedAzureFilter = string.Empty;
foreach (POCO.RecordAuthorityFilter raFilter in raFilters)
{
List <Filter> currentFilters = new List <Filter>();
if (raFilter.RASchemaUri != null && raFilter.RASchemaUri != "")
{
Filter raschemaFilter = new Filter("PartitionKey", raFilter.RASchemaUri, "eq");
currentFilters.Add(raschemaFilter);
}
if (raFilter.Function != null && raFilter.Function != "")
{
Filter functionFilter = new Filter("Function", raFilter.Function, "eq");
currentFilters.Add(functionFilter);
}
if (raFilter.Class != null && raFilter.Class != "")
{
Filter classFilter = new Filter("Class", raFilter.Class, "eq");
currentFilters.Add(classFilter);
}
// AND the current filters
string andFilter = Utils.GenerateAzureFilter(currentFilters);
// OR with the combined filter
if (combinedAzureFilter == string.Empty)
{
combinedAzureFilter = andFilter;
}
else
{
combinedAzureFilter = TableQuery.CombineFilters(combinedAzureFilter, TableOperators.Or, andFilter);
}
}
List <AzureRecordAuthorityKeyPhrase> azdata = new List <AzureRecordAuthorityKeyPhrase>();
AzureTableAdaptor <AzureRecordAuthorityKeyPhrase> adaptor = new AzureTableAdaptor <AzureRecordAuthorityKeyPhrase>();
azdata = adaptor.ReadTableData(providerConfig, AzureTableNames.RecordAuthorityKeyphrases, combinedAzureFilter);
foreach (var doc in azdata)
{
keyphrases.Add(doc.Value);
}
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoRecordAuthorityKeyPhrase>(providerConfig, MongoTableNames.RecordAuthorityKeyphrases);
FilterDefinition <MongoRecordAuthorityKeyPhrase> combinedMongoFilter = null;
foreach (POCO.RecordAuthorityFilter raFilter in raFilters)
{
List <Filter> currentFilters = new List <Filter>();
if (raFilter.RASchemaUri != null && raFilter.RASchemaUri != "")
{
Filter raschemaFilter = new Filter("PartitionKey", raFilter.RASchemaUri, "eq");
currentFilters.Add(raschemaFilter);
}
if (raFilter.Function != null && raFilter.Function != "")
{
Filter functionFilter = new Filter("Function", raFilter.Function, "eq");
currentFilters.Add(functionFilter);
}
if (raFilter.Class != null && raFilter.Class != "")
{
Filter classFilter = new Filter("Class", raFilter.Class, "eq");
currentFilters.Add(classFilter);
}
// AND the current filters
FilterDefinition <MongoRecordAuthorityKeyPhrase> andFilter = Utils.GenerateMongoFilter <MongoRecordAuthorityKeyPhrase>(currentFilters);
// OR with the combined filter
if (combinedMongoFilter == null)
{
combinedMongoFilter = andFilter;
}
else
{
combinedMongoFilter = combinedMongoFilter | andFilter;
}
}
var documents = collection.Find(combinedMongoFilter).ToList();
foreach (var keyphrase in documents)
{
keyphrases.Add(keyphrase);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(keyphrases);
}
public static List <POCO.RecordAuthorityFunctionActivityEntry> GetEntries(DataConfig providerConfig, List <POCO.RecordAuthorityFilter> raFilters)
{
List <POCO.RecordAuthorityFunctionActivityEntry> raitems = new List <POCO.RecordAuthorityFunctionActivityEntry>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedAzureFilter = string.Empty;
foreach (POCO.RecordAuthorityFilter raFilter in raFilters)
{
List <Filter> currentFilters = new List <Filter>();
if (raFilter.RASchemaUri != null && raFilter.RASchemaUri != "")
{
Filter raschemaFilter = new Filter("PartitionKey", raFilter.RASchemaUri, "eq");
currentFilters.Add(raschemaFilter);
}
if (raFilter.Function != null && raFilter.Function != "")
{
Filter functionFilter = new Filter("Function", raFilter.Function, "eq");
currentFilters.Add(functionFilter);
}
if (raFilter.Class != null && raFilter.Class != "")
{
Filter classFilter = new Filter("Class", raFilter.Class, "eq");
currentFilters.Add(classFilter);
}
// AND the current filters
string andFilter = Utils.GenerateAzureFilter(currentFilters);
// OR with the combined filter
if (combinedAzureFilter == string.Empty)
{
combinedAzureFilter = andFilter;
}
else
{
combinedAzureFilter = TableQuery.CombineFilters(combinedAzureFilter, TableOperators.Or, andFilter);
}
}
//if (combinedAzureFilter == null || combinedAzureFilter==string.Empty)
//{
// throw new InvalidOperationException("Schema filter has not been provided. Cannot retrieve Record Authority information.");
//}
List <AzureRecordAuthorityFunctionActivityEntry> azdata = new List <AzureRecordAuthorityFunctionActivityEntry>();
AzureTableAdaptor <AzureRecordAuthorityFunctionActivityEntry> adaptor = new AzureTableAdaptor <AzureRecordAuthorityFunctionActivityEntry>();
azdata = adaptor.ReadTableData(providerConfig, AzureTableNames.RecordAuthority, combinedAzureFilter);
foreach (var doc in azdata)
{
raitems.Add(doc.Value);
}
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoRecordAuthorityFunctionActivityEntry>(providerConfig, MongoTableNames.RecordAuthority);
FilterDefinition <MongoRecordAuthorityFunctionActivityEntry> combinedFilter = null;
List <Filter> filters = new List <Filter>();
foreach (POCO.RecordAuthorityFilter raf in raFilters)
{
List <Filter> thisFilter = new List <Filter>();
if (!string.IsNullOrEmpty(raf.RASchemaUri))
{
Filter fschema = new Filter("PartitionKey", raf.RASchemaUri, "eq");
thisFilter.Add(fschema);
}
if (!string.IsNullOrEmpty(raf.Function))
{
Filter ffunction = new Filter("Function", raf.Function, "eq");
thisFilter.Add(ffunction);
}
if (!string.IsNullOrEmpty(raf.Class))
{
Filter fclass = new Filter("Class", raf.Class, "eq");
thisFilter.Add(fclass);
}
// AND the filters
FilterDefinition <MongoRecordAuthorityFunctionActivityEntry> andFilter = Utils.GenerateMongoFilter <MongoRecordAuthorityFunctionActivityEntry>(thisFilter);
// OR with the combined filter
if (combinedFilter == null)
{
combinedFilter = andFilter;
}
else
{
combinedFilter = combinedFilter | andFilter;
}
}
if (combinedFilter == null)
{
// Set to blank filter
combinedFilter = Utils.GenerateMongoFilter <MongoRecordAuthorityFunctionActivityEntry>(filters);
//throw new InvalidOperationException("Schema filter has not been provided. Cannot retrieve Record Authority information.");
}
var documents = collection.Find(combinedFilter).ToList();
foreach (var raitem in documents)
{
raitems.Add(raitem);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(raitems);
}
public static List <POCO.RecordAssociationNamedEntityReverse> GetReverseNamedEntities(DataConfig providerConfig, List <Filter> filters, string namedEntityLocation)
{
List <POCO.RecordAssociationNamedEntityReverse> namedents = new List <POCO.RecordAssociationNamedEntityReverse>();
string tableName = string.Empty;
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
switch (namedEntityLocation)
{
case "filename":
{
tableName = AzureTableNames.RecordAssociationNamedEntityReverse;
break;
}
case "body":
{
tableName = AzureTableNames.RecordAssociationNamedEntityReverse;
break;
}
default:
throw new ApplicationException("Named entity location not recognised: " + namedEntityLocation);
}
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureRecordAssociationNamedEntityReverse> azdata = new List <AzureRecordAssociationNamedEntityReverse>();
AzureTableAdaptor <AzureRecordAssociationNamedEntityReverse> adaptor = new AzureTableAdaptor <AzureRecordAssociationNamedEntityReverse>();
azdata = adaptor.ReadTableData(providerConfig, tableName, combinedFilter);
foreach (var doc in azdata)
{
namedents.Add(doc.Value);
}
break;
case "internal.mongodb":
switch (namedEntityLocation)
{
case "filename":
{
tableName = MongoTableNames.RecordAssociationNamedEntityReverse;
break;
}
case "body":
{
tableName = MongoTableNames.RecordAssociationNamedEntityReverse;
break;
}
default:
throw new ApplicationException("Named entity location not recognised: " + namedEntityLocation);
}
var collection = Utils.GetMongoCollection <MongoRecordAssociationNamedEntityReverse>(providerConfig, tableName);
FilterDefinition <MongoRecordAssociationNamedEntityReverse> filter = Utils.GenerateMongoFilter <MongoRecordAssociationNamedEntityReverse>(filters);
var documents = collection.Find(filter).ToList();
foreach (var keyphrase in documents)
{
namedents.Add(keyphrase);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(namedents);
}
public static string GetMIMEType(DataConfig providerConfig, POCO.System system, POCO.Record record, RecordToRecordAssociation recordAssocEntity, ILogger logger)
{
string mimeType = string.Empty;
List <DataFactory.Filter> filters = new List <Filter>();
DataFactory.Filter pkFilter = new Filter("PartitionKey", recordAssocEntity.RowKey, "eq");
filters.Add(pkFilter);
//DataFactory.Filter rkFilter = new Filter("RowKey", , "eq");
//filters.Add(rkFilter);
string nextPageId = string.Empty;
string tableName = string.Empty;
// Check which System the file is from
switch (system.Type)
{
case SystemType.SharePoint2010:
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
tableName = TableNames.Azure.SharePoint.SPFile;
break;
case "internal.mongodb":
tableName = TableNames.Mongo.SharePoint.SPFile;
break;
}
break;
case SystemType.SharePoint2013:
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
tableName = TableNames.Azure.SharePoint.SPFile;
break;
case "internal.mongodb":
tableName = TableNames.Mongo.SharePoint.SPFile;
break;
}
break;
case SystemType.SharePointOnline:
case SystemType.SharePointOneDrive:
case SystemType.SharePointTeam:
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
tableName = TableNames.Azure.SharePointOnline.SPFile;
break;
case "internal.mongodb":
tableName = TableNames.Mongo.SharePointOnline.SPFile;
break;
}
break;
case SystemType.NTFSShare:
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
tableName = TableNames.Azure.NTFS.NTFSFiles;
break;
case "internal.mongodb":
tableName = TableNames.Mongo.NTFS.NTFSFiles;
break;
}
break;
case SystemType.SakaiAlliance:
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
tableName = TableNames.Azure.Sakai.SakaiFiles;
break;
case "internal.mongodb":
tableName = TableNames.Mongo.Sakai.SakaiFiles;
break;
}
break;
default:
throw new NotImplementedException();
break;
}
List <POCO.CPFileMIMEType> mimetypes = new List <POCO.CPFileMIMEType>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureFileMIMEType> azdata = new List <AzureFileMIMEType>();
AzureTableAdaptor <AzureFileMIMEType> adaptor = new AzureTableAdaptor <AzureFileMIMEType>();
azdata = adaptor.ReadTableData(providerConfig, tableName, combinedFilter);
foreach (var doc in azdata)
{
mimetypes.Add(doc.Value);
}
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoFileMIMEType>(providerConfig, tableName);
FilterDefinition <MongoFileMIMEType> filter = Utils.GenerateMongoFilter <MongoFileMIMEType>(filters);
var documents = collection.Find(filter).ToList();
foreach (var doc in documents)
{
mimetypes.Add(doc);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
if (mimetypes.Count > 0)
{
mimeType = mimetypes[0].MIMEType;
}
return(mimeType);
}
public static List <POCO.SakaiFile> GetFiles(DataConfig providerConfig, List <Filter> filters, string thisPageId, int rowLimit, out string nextPageId)
{
nextPageId = string.Empty;
List <POCO.SakaiFile> files = new List <POCO.SakaiFile>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
throw new NotImplementedException();
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureSakaiFile> azdata = new List <AzureSakaiFile>();
AzureTableAdaptor <AzureSakaiFile> adaptor = new AzureTableAdaptor <AzureSakaiFile>();
azdata = adaptor.ReadTableData(providerConfig, SakaiResource.AzureTableNames.SakaiFiles, combinedFilter);
foreach (var doc in azdata)
{
files.Add(doc.Value);
}
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoSakaiFile>(providerConfig, SakaiResource.MongoTableNames.SakaiFiles);
// Add an _id filter if a page has been requested
if (thisPageId != null && thisPageId != string.Empty)
{
filters.Insert(0, new Filter("_id", thisPageId, "gt"));
}
FilterDefinition <MongoSakaiFile> filter = Utils.GenerateMongoFilter <MongoSakaiFile>(filters);
//DEBUG output the filter values
//foreach (Castlepoint.DataFactory.Filter debugFilter in filters)
//{
// // Output the filter field names and values
// Console.WriteLine("DEBUG filter: " + debugFilter.FieldName + " : " + debugFilter.FieldValue);
//}
var documents = collection.Find(filter).Sort("{\"_id\":1}").Limit(rowLimit).ToList();
foreach (var SakaiFile in documents)
{
files.Add(SakaiFile);
}
// Get the next page id
if (documents.Count == rowLimit)
{
// Set the next page id
nextPageId = documents[documents.Count - 1]._id.ToString();
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(files);
}
