public static List <POCO.SakaiFileBatchStatus> GetFileBatchStatus(DataConfig providerConfig, List <Filter> filters)
{
List <POCO.SakaiFileBatchStatus> fileBatchStatus = new List <POCO.SakaiFileBatchStatus>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureSakaiFileBatchStatus> azdata = new List <AzureSakaiFileBatchStatus>();
AzureTableAdaptor <AzureSakaiFileBatchStatus> adaptor = new AzureTableAdaptor <AzureSakaiFileBatchStatus>();
azdata = adaptor.ReadTableData(providerConfig, SakaiResource.AzureTableNames.SakaiFilesBatchStatus, combinedFilter);
foreach (var doc in azdata)
{
fileBatchStatus.Add(doc.Value);
}
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoSakaiFileBatchStatus>(providerConfig, SakaiResource.MongoTableNames.SakaiFilesBatchStatus);
FilterDefinition <MongoSakaiFileBatchStatus> filter = Utils.GenerateMongoFilter <MongoSakaiFileBatchStatus>(filters);
var documents = collection.Find(filter).ToList();
foreach (var status in documents)
{
fileBatchStatus.Add(status);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(fileBatchStatus);
}
public static List <POCO.SystemNodeConfig> GetAllBatchNodeConfig(DataConfig providerConfig)
{
List <POCO.SystemNodeConfig> systems = new List <POCO.SystemNodeConfig>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
List <AzureSystemBatchConfig> azs = new List <AzureSystemBatchConfig>();
AzureTableAdaptor <AzureSystemBatchConfig> az = new AzureTableAdaptor <AzureSystemBatchConfig>();
azs = az.ReadTableData(providerConfig, POCO.TableNames.Azure.System.BatchConfig);
foreach (var doc in azs)
{
systems.Add(doc.Value);
}
return(systems);
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoSystemBatchConfig>(providerConfig, POCO.TableNames.Mongo.System.BatchConfig);
var documents = collection.Find(new BsonDocument()).ToList();
foreach (var system in documents)
{
systems.Add(system);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(systems);
}
public static List <POCO.ProcessingBatchStatus> GetProcessingStatus(DataConfig providerConfig)
{
List <POCO.ProcessingBatchStatus> processingStatus = new List <POCO.ProcessingBatchStatus>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
List <AzureProcessingBatchStatus> azs = new List <AzureProcessingBatchStatus>();
AzureTableAdaptor <AzureProcessingBatchStatus> az = new AzureTableAdaptor <AzureProcessingBatchStatus>();
azs = az.ReadTableData(providerConfig, AzureTableNames.SystemProcessingStatus);
foreach (var doc in azs)
{
processingStatus.Add(doc.Value);
}
return(processingStatus);
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoProcessingBatchStatus>(providerConfig, MongoTableNames.SystemProcessingStatus);
var documents = collection.Find(new BsonDocument()).ToList();
foreach (var doc in documents)
{
processingStatus.Add(doc);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(processingStatus);
}
public static List <POCO.O365.EOMessageEntity> GetMessages(DataConfig providerConfig, List <Filter> filters, string thisPageId, int rowLimit, out string nextPageId)
{
nextPageId = string.Empty;
List <POCO.O365.EOMessageEntity> files = new List <POCO.O365.EOMessageEntity>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
{
string combinedFilter = Utils.GenerateAzureFilter(filters);
TableContinuationToken thisPageToken = null;
if (thisPageId != null && thisPageId != string.Empty)
{
thisPageToken = Newtonsoft.Json.JsonConvert.DeserializeObject <TableContinuationToken>(thisPageId);
}
TableContinuationToken nextPageToken = null;
List <AzureEOMessageEntity> azdata = new List <AzureEOMessageEntity>();
AzureTableAdaptor <AzureEOMessageEntity> adaptor = new AzureTableAdaptor <AzureEOMessageEntity>();
azdata = adaptor.ReadTableDataWithToken(providerConfig, AzureTableNames.EOMessage, combinedFilter, rowLimit, thisPageToken, out nextPageToken);
foreach (var doc in azdata)
{
files.Add(doc.Value);
}
// Check if there is a next page token available
if (nextPageToken != null)
{
nextPageId = Newtonsoft.Json.JsonConvert.SerializeObject(nextPageToken);
}
break;
}
case "internal.mongodb":
{
var lastId = "";
var collection = Utils.GetMongoCollection <MongoEOMessageEntity>(providerConfig, MongoTableNames.EOMessage);
// Add an _id filter if a page has been requested
if (thisPageId != null && thisPageId != string.Empty)
{
filters.Insert(0, new Filter("_id", thisPageId, "gt"));
}
FilterDefinition <MongoEOMessageEntity> filter = Utils.GenerateMongoFilter <MongoEOMessageEntity>(filters);
var documents = collection.Find(filter).Sort("{\"_id\":1}").Limit(rowLimit).ToList();
foreach (var file in documents)
{
files.Add(file);
lastId = file._id.ToString();
}
// Check if there are more pages of data
if (files.Count == rowLimit)
{
// Return the _id of the last row
nextPageId = lastId;
}
break;
}
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(files);
}
public static List <POCO.RecordAssociationNamedEntityReverse> GetReverseNamedEntities(DataConfig providerConfig, List <Filter> filters, string namedEntityLocation)
{
List <POCO.RecordAssociationNamedEntityReverse> namedents = new List <POCO.RecordAssociationNamedEntityReverse>();
string tableName = string.Empty;
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
switch (namedEntityLocation)
{
case "filename":
{
tableName = AzureTableNames.RecordAssociationNamedEntityReverse;
break;
}
case "body":
{
tableName = AzureTableNames.RecordAssociationNamedEntityReverse;
break;
}
default:
throw new ApplicationException("Named entity location not recognised: " + namedEntityLocation);
}
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureRecordAssociationNamedEntityReverse> azdata = new List <AzureRecordAssociationNamedEntityReverse>();
AzureTableAdaptor <AzureRecordAssociationNamedEntityReverse> adaptor = new AzureTableAdaptor <AzureRecordAssociationNamedEntityReverse>();
azdata = adaptor.ReadTableData(providerConfig, tableName, combinedFilter);
foreach (var doc in azdata)
{
namedents.Add(doc.Value);
}
break;
case "internal.mongodb":
switch (namedEntityLocation)
{
case "filename":
{
tableName = MongoTableNames.RecordAssociationNamedEntityReverse;
break;
}
case "body":
{
tableName = MongoTableNames.RecordAssociationNamedEntityReverse;
break;
}
default:
throw new ApplicationException("Named entity location not recognised: " + namedEntityLocation);
}
var collection = Utils.GetMongoCollection <MongoRecordAssociationNamedEntityReverse>(providerConfig, tableName);
FilterDefinition <MongoRecordAssociationNamedEntityReverse> filter = Utils.GenerateMongoFilter <MongoRecordAssociationNamedEntityReverse>(filters);
var documents = collection.Find(filter).ToList();
foreach (var keyphrase in documents)
{
namedents.Add(keyphrase);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(namedents);
}
public static List <POCO.SakaiFile> GetFiles(DataConfig providerConfig, List <Filter> filters, string thisPageId, int rowLimit, out string nextPageId)
{
nextPageId = string.Empty;
List <POCO.SakaiFile> files = new List <POCO.SakaiFile>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
throw new NotImplementedException();
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureSakaiFile> azdata = new List <AzureSakaiFile>();
AzureTableAdaptor <AzureSakaiFile> adaptor = new AzureTableAdaptor <AzureSakaiFile>();
azdata = adaptor.ReadTableData(providerConfig, SakaiResource.AzureTableNames.SakaiFiles, combinedFilter);
foreach (var doc in azdata)
{
files.Add(doc.Value);
}
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoSakaiFile>(providerConfig, SakaiResource.MongoTableNames.SakaiFiles);
// Add an _id filter if a page has been requested
if (thisPageId != null && thisPageId != string.Empty)
{
filters.Insert(0, new Filter("_id", thisPageId, "gt"));
}
FilterDefinition <MongoSakaiFile> filter = Utils.GenerateMongoFilter <MongoSakaiFile>(filters);
//DEBUG output the filter values
//foreach (Castlepoint.DataFactory.Filter debugFilter in filters)
//{
// // Output the filter field names and values
// Console.WriteLine("DEBUG filter: " + debugFilter.FieldName + " : " + debugFilter.FieldValue);
//}
var documents = collection.Find(filter).Sort("{\"_id\":1}").Limit(rowLimit).ToList();
foreach (var SakaiFile in documents)
{
files.Add(SakaiFile);
}
// Get the next page id
if (documents.Count == rowLimit)
{
// Set the next page id
nextPageId = documents[documents.Count - 1]._id.ToString();
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(files);
}
public static List <POCO.RecordAuthorityFunctionActivityEntry> GetEntries(DataConfig providerConfig, List <POCO.RecordAuthorityFilter> raFilters)
{
List <POCO.RecordAuthorityFunctionActivityEntry> raitems = new List <POCO.RecordAuthorityFunctionActivityEntry>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedAzureFilter = string.Empty;
foreach (POCO.RecordAuthorityFilter raFilter in raFilters)
{
List <Filter> currentFilters = new List <Filter>();
if (raFilter.RASchemaUri != null && raFilter.RASchemaUri != "")
{
Filter raschemaFilter = new Filter("PartitionKey", raFilter.RASchemaUri, "eq");
currentFilters.Add(raschemaFilter);
}
if (raFilter.Function != null && raFilter.Function != "")
{
Filter functionFilter = new Filter("Function", raFilter.Function, "eq");
currentFilters.Add(functionFilter);
}
if (raFilter.Class != null && raFilter.Class != "")
{
Filter classFilter = new Filter("Class", raFilter.Class, "eq");
currentFilters.Add(classFilter);
}
// AND the current filters
string andFilter = Utils.GenerateAzureFilter(currentFilters);
// OR with the combined filter
if (combinedAzureFilter == string.Empty)
{
combinedAzureFilter = andFilter;
}
else
{
combinedAzureFilter = TableQuery.CombineFilters(combinedAzureFilter, TableOperators.Or, andFilter);
}
}
//if (combinedAzureFilter == null || combinedAzureFilter==string.Empty)
//{
// throw new InvalidOperationException("Schema filter has not been provided. Cannot retrieve Record Authority information.");
//}
List <AzureRecordAuthorityFunctionActivityEntry> azdata = new List <AzureRecordAuthorityFunctionActivityEntry>();
AzureTableAdaptor <AzureRecordAuthorityFunctionActivityEntry> adaptor = new AzureTableAdaptor <AzureRecordAuthorityFunctionActivityEntry>();
azdata = adaptor.ReadTableData(providerConfig, AzureTableNames.RecordAuthority, combinedAzureFilter);
foreach (var doc in azdata)
{
raitems.Add(doc.Value);
}
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoRecordAuthorityFunctionActivityEntry>(providerConfig, MongoTableNames.RecordAuthority);
FilterDefinition <MongoRecordAuthorityFunctionActivityEntry> combinedFilter = null;
List <Filter> filters = new List <Filter>();
foreach (POCO.RecordAuthorityFilter raf in raFilters)
{
List <Filter> thisFilter = new List <Filter>();
if (!string.IsNullOrEmpty(raf.RASchemaUri))
{
Filter fschema = new Filter("PartitionKey", raf.RASchemaUri, "eq");
thisFilter.Add(fschema);
}
if (!string.IsNullOrEmpty(raf.Function))
{
Filter ffunction = new Filter("Function", raf.Function, "eq");
thisFilter.Add(ffunction);
}
if (!string.IsNullOrEmpty(raf.Class))
{
Filter fclass = new Filter("Class", raf.Class, "eq");
thisFilter.Add(fclass);
}
// AND the filters
FilterDefinition <MongoRecordAuthorityFunctionActivityEntry> andFilter = Utils.GenerateMongoFilter <MongoRecordAuthorityFunctionActivityEntry>(thisFilter);
// OR with the combined filter
if (combinedFilter == null)
{
combinedFilter = andFilter;
}
else
{
combinedFilter = combinedFilter | andFilter;
}
}
if (combinedFilter == null)
{
// Set to blank filter
combinedFilter = Utils.GenerateMongoFilter <MongoRecordAuthorityFunctionActivityEntry>(filters);
//throw new InvalidOperationException("Schema filter has not been provided. Cannot retrieve Record Authority information.");
}
var documents = collection.Find(combinedFilter).ToList();
foreach (var raitem in documents)
{
raitems.Add(raitem);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(raitems);
}
public static List <POCO.RecordAuthorityKeyPhrase> GetKeyPhrases(DataConfig providerConfig, List <POCO.RecordAuthorityFilter> raFilters)
{
List <POCO.RecordAuthorityKeyPhrase> keyphrases = new List <POCO.RecordAuthorityKeyPhrase>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedAzureFilter = string.Empty;
foreach (POCO.RecordAuthorityFilter raFilter in raFilters)
{
List <Filter> currentFilters = new List <Filter>();
if (raFilter.RASchemaUri != null && raFilter.RASchemaUri != "")
{
Filter raschemaFilter = new Filter("PartitionKey", raFilter.RASchemaUri, "eq");
currentFilters.Add(raschemaFilter);
}
if (raFilter.Function != null && raFilter.Function != "")
{
Filter functionFilter = new Filter("Function", raFilter.Function, "eq");
currentFilters.Add(functionFilter);
}
if (raFilter.Class != null && raFilter.Class != "")
{
Filter classFilter = new Filter("Class", raFilter.Class, "eq");
currentFilters.Add(classFilter);
}
// AND the current filters
string andFilter = Utils.GenerateAzureFilter(currentFilters);
// OR with the combined filter
if (combinedAzureFilter == string.Empty)
{
combinedAzureFilter = andFilter;
}
else
{
combinedAzureFilter = TableQuery.CombineFilters(combinedAzureFilter, TableOperators.Or, andFilter);
}
}
List <AzureRecordAuthorityKeyPhrase> azdata = new List <AzureRecordAuthorityKeyPhrase>();
AzureTableAdaptor <AzureRecordAuthorityKeyPhrase> adaptor = new AzureTableAdaptor <AzureRecordAuthorityKeyPhrase>();
azdata = adaptor.ReadTableData(providerConfig, AzureTableNames.RecordAuthorityKeyphrases, combinedAzureFilter);
foreach (var doc in azdata)
{
keyphrases.Add(doc.Value);
}
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoRecordAuthorityKeyPhrase>(providerConfig, MongoTableNames.RecordAuthorityKeyphrases);
FilterDefinition <MongoRecordAuthorityKeyPhrase> combinedMongoFilter = null;
foreach (POCO.RecordAuthorityFilter raFilter in raFilters)
{
List <Filter> currentFilters = new List <Filter>();
if (raFilter.RASchemaUri != null && raFilter.RASchemaUri != "")
{
Filter raschemaFilter = new Filter("PartitionKey", raFilter.RASchemaUri, "eq");
currentFilters.Add(raschemaFilter);
}
if (raFilter.Function != null && raFilter.Function != "")
{
Filter functionFilter = new Filter("Function", raFilter.Function, "eq");
currentFilters.Add(functionFilter);
}
if (raFilter.Class != null && raFilter.Class != "")
{
Filter classFilter = new Filter("Class", raFilter.Class, "eq");
currentFilters.Add(classFilter);
}
// AND the current filters
FilterDefinition <MongoRecordAuthorityKeyPhrase> andFilter = Utils.GenerateMongoFilter <MongoRecordAuthorityKeyPhrase>(currentFilters);
// OR with the combined filter
if (combinedMongoFilter == null)
{
combinedMongoFilter = andFilter;
}
else
{
combinedMongoFilter = combinedMongoFilter | andFilter;
}
}
var documents = collection.Find(combinedMongoFilter).ToList();
foreach (var keyphrase in documents)
{
keyphrases.Add(keyphrase);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(keyphrases);
}
public static string GetMIMEType(DataConfig providerConfig, POCO.System system, POCO.Record record, RecordToRecordAssociation recordAssocEntity, ILogger logger)
{
string mimeType = string.Empty;
List <DataFactory.Filter> filters = new List <Filter>();
DataFactory.Filter pkFilter = new Filter("PartitionKey", recordAssocEntity.RowKey, "eq");
filters.Add(pkFilter);
//DataFactory.Filter rkFilter = new Filter("RowKey", , "eq");
//filters.Add(rkFilter);
string nextPageId = string.Empty;
string tableName = string.Empty;
// Check which System the file is from
switch (system.Type)
{
case SystemType.SharePoint2010:
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
tableName = TableNames.Azure.SharePoint.SPFile;
break;
case "internal.mongodb":
tableName = TableNames.Mongo.SharePoint.SPFile;
break;
}
break;
case SystemType.SharePoint2013:
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
tableName = TableNames.Azure.SharePoint.SPFile;
break;
case "internal.mongodb":
tableName = TableNames.Mongo.SharePoint.SPFile;
break;
}
break;
case SystemType.SharePointOnline:
case SystemType.SharePointOneDrive:
case SystemType.SharePointTeam:
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
tableName = TableNames.Azure.SharePointOnline.SPFile;
break;
case "internal.mongodb":
tableName = TableNames.Mongo.SharePointOnline.SPFile;
break;
}
break;
case SystemType.NTFSShare:
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
tableName = TableNames.Azure.NTFS.NTFSFiles;
break;
case "internal.mongodb":
tableName = TableNames.Mongo.NTFS.NTFSFiles;
break;
}
break;
case SystemType.SakaiAlliance:
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
tableName = TableNames.Azure.Sakai.SakaiFiles;
break;
case "internal.mongodb":
tableName = TableNames.Mongo.Sakai.SakaiFiles;
break;
}
break;
default:
throw new NotImplementedException();
break;
}
List <POCO.CPFileMIMEType> mimetypes = new List <POCO.CPFileMIMEType>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
List <AzureFileMIMEType> azdata = new List <AzureFileMIMEType>();
AzureTableAdaptor <AzureFileMIMEType> adaptor = new AzureTableAdaptor <AzureFileMIMEType>();
azdata = adaptor.ReadTableData(providerConfig, tableName, combinedFilter);
foreach (var doc in azdata)
{
mimetypes.Add(doc.Value);
}
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoFileMIMEType>(providerConfig, tableName);
FilterDefinition <MongoFileMIMEType> filter = Utils.GenerateMongoFilter <MongoFileMIMEType>(filters);
var documents = collection.Find(filter).ToList();
foreach (var doc in documents)
{
mimetypes.Add(doc);
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
if (mimetypes.Count > 0)
{
mimeType = mimetypes[0].MIMEType;
}
return(mimeType);
}
public static List <POCO.NTFSFile> GetFiles(DataConfig providerConfig, List <Filter> filters, string thisPageId, int rowLimit, out string nextPageId)
{
nextPageId = string.Empty;
List <POCO.NTFSFile> files = new List <POCO.NTFSFile>();
switch (providerConfig.ProviderType)
{
case "azure.tableservice":
string combinedFilter = Utils.GenerateAzureFilter(filters);
TableContinuationToken thisPageToken = null;
if (thisPageId != null && thisPageId != string.Empty)
{
thisPageToken = Newtonsoft.Json.JsonConvert.DeserializeObject <TableContinuationToken>(thisPageId);
}
TableContinuationToken nextPageToken = null;
List <AzureNTFSFile> azdata = new List <AzureNTFSFile>();
AzureTableAdaptor <AzureNTFSFile> adaptor = new AzureTableAdaptor <AzureNTFSFile>();
azdata = adaptor.ReadTableDataWithToken(providerConfig, NTFS.AzureTableNames.NTFSFiles, combinedFilter, rowLimit, thisPageToken, out nextPageToken);
foreach (var doc in azdata)
{
files.Add(doc.Value);
}
// Check if there is a next page token available
if (nextPageToken != null)
{
nextPageId = Newtonsoft.Json.JsonConvert.SerializeObject(nextPageToken);
}
break;
//throw new NotImplementedException();
//string combinedFilter = Utils.GenerateAzureFilter(filters);
//List<AzureNTFSFile> azdata = new List<AzureNTFSFile>();
//AzureTableAdaptor<AzureNTFSFile> adaptor = new AzureTableAdaptor<AzureNTFSFile>();
//azdata = adaptor.ReadTableData(providerConfig, NTFS.AzureTableNames.NTFSFiles, combinedFilter);
//foreach (var doc in azdata)
//{
// files.Add(doc.Value);
//}
break;
case "internal.mongodb":
var collection = Utils.GetMongoCollection <MongoNTFSFile>(providerConfig, NTFS.MongoTableNames.NTFSFiles);
// Add an _id filter if a page has been requested
if (thisPageId != null && thisPageId != string.Empty)
{
filters.Insert(0, new Filter("_id", thisPageId, "gt"));
}
FilterDefinition <MongoNTFSFile> filter = Utils.GenerateMongoFilter <MongoNTFSFile>(filters);
//DEBUG output the filter values
//foreach (Castlepoint.DataFactory.Filter debugFilter in filters)
//{
// // Output the filter field names and values
// Console.WriteLine("DEBUG filter: " + debugFilter.FieldName + " : " + debugFilter.FieldValue);
//}
var documents = collection.Find(filter).Sort("{\"_id\":1}").Limit(rowLimit).ToList();
foreach (var NTFSFile in documents)
{
files.Add(NTFSFile);
}
// Get the next page id
if (documents.Count == rowLimit)
{
// Set the next page id
nextPageId = documents[documents.Count - 1]._id.ToString();
}
break;
default:
throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
}
return(files);
}