public static List <POCO.SakaiFileBatchStatus> GetFileBatchStatus(DataConfig providerConfig, List <Filter> filters) { List <POCO.SakaiFileBatchStatus> fileBatchStatus = new List <POCO.SakaiFileBatchStatus>(); switch (providerConfig.ProviderType) { case "azure.tableservice": string combinedFilter = Utils.GenerateAzureFilter(filters); List <AzureSakaiFileBatchStatus> azdata = new List <AzureSakaiFileBatchStatus>(); AzureTableAdaptor <AzureSakaiFileBatchStatus> adaptor = new AzureTableAdaptor <AzureSakaiFileBatchStatus>(); azdata = adaptor.ReadTableData(providerConfig, SakaiResource.AzureTableNames.SakaiFilesBatchStatus, combinedFilter); foreach (var doc in azdata) { fileBatchStatus.Add(doc.Value); } break; case "internal.mongodb": var collection = Utils.GetMongoCollection <MongoSakaiFileBatchStatus>(providerConfig, SakaiResource.MongoTableNames.SakaiFilesBatchStatus); FilterDefinition <MongoSakaiFileBatchStatus> filter = Utils.GenerateMongoFilter <MongoSakaiFileBatchStatus>(filters); var documents = collection.Find(filter).ToList(); foreach (var status in documents) { fileBatchStatus.Add(status); } break; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return(fileBatchStatus); }
public static List <POCO.SystemNodeConfig> GetAllBatchNodeConfig(DataConfig providerConfig) { List <POCO.SystemNodeConfig> systems = new List <POCO.SystemNodeConfig>(); switch (providerConfig.ProviderType) { case "azure.tableservice": List <AzureSystemBatchConfig> azs = new List <AzureSystemBatchConfig>(); AzureTableAdaptor <AzureSystemBatchConfig> az = new AzureTableAdaptor <AzureSystemBatchConfig>(); azs = az.ReadTableData(providerConfig, POCO.TableNames.Azure.System.BatchConfig); foreach (var doc in azs) { systems.Add(doc.Value); } return(systems); case "internal.mongodb": var collection = Utils.GetMongoCollection <MongoSystemBatchConfig>(providerConfig, POCO.TableNames.Mongo.System.BatchConfig); var documents = collection.Find(new BsonDocument()).ToList(); foreach (var system in documents) { systems.Add(system); } break; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return(systems); }
public static List <POCO.ProcessingBatchStatus> GetProcessingStatus(DataConfig providerConfig) { List <POCO.ProcessingBatchStatus> processingStatus = new List <POCO.ProcessingBatchStatus>(); switch (providerConfig.ProviderType) { case "azure.tableservice": List <AzureProcessingBatchStatus> azs = new List <AzureProcessingBatchStatus>(); AzureTableAdaptor <AzureProcessingBatchStatus> az = new AzureTableAdaptor <AzureProcessingBatchStatus>(); azs = az.ReadTableData(providerConfig, AzureTableNames.SystemProcessingStatus); foreach (var doc in azs) { processingStatus.Add(doc.Value); } return(processingStatus); case "internal.mongodb": var collection = Utils.GetMongoCollection <MongoProcessingBatchStatus>(providerConfig, MongoTableNames.SystemProcessingStatus); var documents = collection.Find(new BsonDocument()).ToList(); foreach (var doc in documents) { processingStatus.Add(doc); } break; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return(processingStatus); }
public static List <POCO.O365.EOMessageEntity> GetMessages(DataConfig providerConfig, List <Filter> filters, string thisPageId, int rowLimit, out string nextPageId) { nextPageId = string.Empty; List <POCO.O365.EOMessageEntity> files = new List <POCO.O365.EOMessageEntity>(); switch (providerConfig.ProviderType) { case "azure.tableservice": { string combinedFilter = Utils.GenerateAzureFilter(filters); TableContinuationToken thisPageToken = null; if (thisPageId != null && thisPageId != string.Empty) { thisPageToken = Newtonsoft.Json.JsonConvert.DeserializeObject <TableContinuationToken>(thisPageId); } TableContinuationToken nextPageToken = null; List <AzureEOMessageEntity> azdata = new List <AzureEOMessageEntity>(); AzureTableAdaptor <AzureEOMessageEntity> adaptor = new AzureTableAdaptor <AzureEOMessageEntity>(); azdata = adaptor.ReadTableDataWithToken(providerConfig, AzureTableNames.EOMessage, combinedFilter, rowLimit, thisPageToken, out nextPageToken); foreach (var doc in azdata) { files.Add(doc.Value); } // Check if there is a next page token available if (nextPageToken != null) { nextPageId = Newtonsoft.Json.JsonConvert.SerializeObject(nextPageToken); } break; } case "internal.mongodb": { var lastId = ""; var collection = Utils.GetMongoCollection <MongoEOMessageEntity>(providerConfig, MongoTableNames.EOMessage); // Add an _id filter if a page has been requested if (thisPageId != null && thisPageId != string.Empty) { filters.Insert(0, new Filter("_id", thisPageId, "gt")); } FilterDefinition <MongoEOMessageEntity> filter = Utils.GenerateMongoFilter <MongoEOMessageEntity>(filters); var documents = collection.Find(filter).Sort("{\"_id\":1}").Limit(rowLimit).ToList(); foreach (var file in documents) { files.Add(file); lastId = file._id.ToString(); } // Check if there are more pages of data if (files.Count == rowLimit) { // Return the _id of the last row nextPageId = lastId; } break; } default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return(files); }
public static List <POCO.RecordAssociationNamedEntityReverse> GetReverseNamedEntities(DataConfig providerConfig, List <Filter> filters, string namedEntityLocation) { List <POCO.RecordAssociationNamedEntityReverse> namedents = new List <POCO.RecordAssociationNamedEntityReverse>(); string tableName = string.Empty; switch (providerConfig.ProviderType) { case "azure.tableservice": switch (namedEntityLocation) { case "filename": { tableName = AzureTableNames.RecordAssociationNamedEntityReverse; break; } case "body": { tableName = AzureTableNames.RecordAssociationNamedEntityReverse; break; } default: throw new ApplicationException("Named entity location not recognised: " + namedEntityLocation); } string combinedFilter = Utils.GenerateAzureFilter(filters); List <AzureRecordAssociationNamedEntityReverse> azdata = new List <AzureRecordAssociationNamedEntityReverse>(); AzureTableAdaptor <AzureRecordAssociationNamedEntityReverse> adaptor = new AzureTableAdaptor <AzureRecordAssociationNamedEntityReverse>(); azdata = adaptor.ReadTableData(providerConfig, tableName, combinedFilter); foreach (var doc in azdata) { namedents.Add(doc.Value); } break; case "internal.mongodb": switch (namedEntityLocation) { case "filename": { tableName = MongoTableNames.RecordAssociationNamedEntityReverse; break; } case "body": { tableName = MongoTableNames.RecordAssociationNamedEntityReverse; break; } default: throw new ApplicationException("Named entity location not recognised: " + namedEntityLocation); } var collection = Utils.GetMongoCollection <MongoRecordAssociationNamedEntityReverse>(providerConfig, tableName); FilterDefinition <MongoRecordAssociationNamedEntityReverse> filter = Utils.GenerateMongoFilter <MongoRecordAssociationNamedEntityReverse>(filters); var documents = collection.Find(filter).ToList(); foreach (var keyphrase in documents) { namedents.Add(keyphrase); } break; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return(namedents); }
public static List <POCO.SakaiFile> GetFiles(DataConfig providerConfig, List <Filter> filters, string thisPageId, int rowLimit, out string nextPageId) { nextPageId = string.Empty; List <POCO.SakaiFile> files = new List <POCO.SakaiFile>(); switch (providerConfig.ProviderType) { case "azure.tableservice": throw new NotImplementedException(); string combinedFilter = Utils.GenerateAzureFilter(filters); List <AzureSakaiFile> azdata = new List <AzureSakaiFile>(); AzureTableAdaptor <AzureSakaiFile> adaptor = new AzureTableAdaptor <AzureSakaiFile>(); azdata = adaptor.ReadTableData(providerConfig, SakaiResource.AzureTableNames.SakaiFiles, combinedFilter); foreach (var doc in azdata) { files.Add(doc.Value); } break; case "internal.mongodb": var collection = Utils.GetMongoCollection <MongoSakaiFile>(providerConfig, SakaiResource.MongoTableNames.SakaiFiles); // Add an _id filter if a page has been requested if (thisPageId != null && thisPageId != string.Empty) { filters.Insert(0, new Filter("_id", thisPageId, "gt")); } FilterDefinition <MongoSakaiFile> filter = Utils.GenerateMongoFilter <MongoSakaiFile>(filters); //DEBUG output the filter values //foreach (Castlepoint.DataFactory.Filter debugFilter in filters) //{ // // Output the filter field names and values // Console.WriteLine("DEBUG filter: " + debugFilter.FieldName + " : " + debugFilter.FieldValue); //} var documents = collection.Find(filter).Sort("{\"_id\":1}").Limit(rowLimit).ToList(); foreach (var SakaiFile in documents) { files.Add(SakaiFile); } // Get the next page id if (documents.Count == rowLimit) { // Set the next page id nextPageId = documents[documents.Count - 1]._id.ToString(); } break; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return(files); }
public static List <POCO.RecordAuthorityFunctionActivityEntry> GetEntries(DataConfig providerConfig, List <POCO.RecordAuthorityFilter> raFilters) { List <POCO.RecordAuthorityFunctionActivityEntry> raitems = new List <POCO.RecordAuthorityFunctionActivityEntry>(); switch (providerConfig.ProviderType) { case "azure.tableservice": string combinedAzureFilter = string.Empty; foreach (POCO.RecordAuthorityFilter raFilter in raFilters) { List <Filter> currentFilters = new List <Filter>(); if (raFilter.RASchemaUri != null && raFilter.RASchemaUri != "") { Filter raschemaFilter = new Filter("PartitionKey", raFilter.RASchemaUri, "eq"); currentFilters.Add(raschemaFilter); } if (raFilter.Function != null && raFilter.Function != "") { Filter functionFilter = new Filter("Function", raFilter.Function, "eq"); currentFilters.Add(functionFilter); } if (raFilter.Class != null && raFilter.Class != "") { Filter classFilter = new Filter("Class", raFilter.Class, "eq"); currentFilters.Add(classFilter); } // AND the current filters string andFilter = Utils.GenerateAzureFilter(currentFilters); // OR with the combined filter if (combinedAzureFilter == string.Empty) { combinedAzureFilter = andFilter; } else { combinedAzureFilter = TableQuery.CombineFilters(combinedAzureFilter, TableOperators.Or, andFilter); } } //if (combinedAzureFilter == null || combinedAzureFilter==string.Empty) //{ // throw new InvalidOperationException("Schema filter has not been provided. Cannot retrieve Record Authority information."); //} List <AzureRecordAuthorityFunctionActivityEntry> azdata = new List <AzureRecordAuthorityFunctionActivityEntry>(); AzureTableAdaptor <AzureRecordAuthorityFunctionActivityEntry> adaptor = new AzureTableAdaptor <AzureRecordAuthorityFunctionActivityEntry>(); azdata = adaptor.ReadTableData(providerConfig, AzureTableNames.RecordAuthority, combinedAzureFilter); foreach (var doc in azdata) { raitems.Add(doc.Value); } break; case "internal.mongodb": var collection = Utils.GetMongoCollection <MongoRecordAuthorityFunctionActivityEntry>(providerConfig, MongoTableNames.RecordAuthority); FilterDefinition <MongoRecordAuthorityFunctionActivityEntry> combinedFilter = null; List <Filter> filters = new List <Filter>(); foreach (POCO.RecordAuthorityFilter raf in raFilters) { List <Filter> thisFilter = new List <Filter>(); if (!string.IsNullOrEmpty(raf.RASchemaUri)) { Filter fschema = new Filter("PartitionKey", raf.RASchemaUri, "eq"); thisFilter.Add(fschema); } if (!string.IsNullOrEmpty(raf.Function)) { Filter ffunction = new Filter("Function", raf.Function, "eq"); thisFilter.Add(ffunction); } if (!string.IsNullOrEmpty(raf.Class)) { Filter fclass = new Filter("Class", raf.Class, "eq"); thisFilter.Add(fclass); } // AND the filters FilterDefinition <MongoRecordAuthorityFunctionActivityEntry> andFilter = Utils.GenerateMongoFilter <MongoRecordAuthorityFunctionActivityEntry>(thisFilter); // OR with the combined filter if (combinedFilter == null) { combinedFilter = andFilter; } else { combinedFilter = combinedFilter | andFilter; } } if (combinedFilter == null) { // Set to blank filter combinedFilter = Utils.GenerateMongoFilter <MongoRecordAuthorityFunctionActivityEntry>(filters); //throw new InvalidOperationException("Schema filter has not been provided. Cannot retrieve Record Authority information."); } var documents = collection.Find(combinedFilter).ToList(); foreach (var raitem in documents) { raitems.Add(raitem); } break; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return(raitems); }
public static List <POCO.RecordAuthorityKeyPhrase> GetKeyPhrases(DataConfig providerConfig, List <POCO.RecordAuthorityFilter> raFilters) { List <POCO.RecordAuthorityKeyPhrase> keyphrases = new List <POCO.RecordAuthorityKeyPhrase>(); switch (providerConfig.ProviderType) { case "azure.tableservice": string combinedAzureFilter = string.Empty; foreach (POCO.RecordAuthorityFilter raFilter in raFilters) { List <Filter> currentFilters = new List <Filter>(); if (raFilter.RASchemaUri != null && raFilter.RASchemaUri != "") { Filter raschemaFilter = new Filter("PartitionKey", raFilter.RASchemaUri, "eq"); currentFilters.Add(raschemaFilter); } if (raFilter.Function != null && raFilter.Function != "") { Filter functionFilter = new Filter("Function", raFilter.Function, "eq"); currentFilters.Add(functionFilter); } if (raFilter.Class != null && raFilter.Class != "") { Filter classFilter = new Filter("Class", raFilter.Class, "eq"); currentFilters.Add(classFilter); } // AND the current filters string andFilter = Utils.GenerateAzureFilter(currentFilters); // OR with the combined filter if (combinedAzureFilter == string.Empty) { combinedAzureFilter = andFilter; } else { combinedAzureFilter = TableQuery.CombineFilters(combinedAzureFilter, TableOperators.Or, andFilter); } } List <AzureRecordAuthorityKeyPhrase> azdata = new List <AzureRecordAuthorityKeyPhrase>(); AzureTableAdaptor <AzureRecordAuthorityKeyPhrase> adaptor = new AzureTableAdaptor <AzureRecordAuthorityKeyPhrase>(); azdata = adaptor.ReadTableData(providerConfig, AzureTableNames.RecordAuthorityKeyphrases, combinedAzureFilter); foreach (var doc in azdata) { keyphrases.Add(doc.Value); } break; case "internal.mongodb": var collection = Utils.GetMongoCollection <MongoRecordAuthorityKeyPhrase>(providerConfig, MongoTableNames.RecordAuthorityKeyphrases); FilterDefinition <MongoRecordAuthorityKeyPhrase> combinedMongoFilter = null; foreach (POCO.RecordAuthorityFilter raFilter in raFilters) { List <Filter> currentFilters = new List <Filter>(); if (raFilter.RASchemaUri != null && raFilter.RASchemaUri != "") { Filter raschemaFilter = new Filter("PartitionKey", raFilter.RASchemaUri, "eq"); currentFilters.Add(raschemaFilter); } if (raFilter.Function != null && raFilter.Function != "") { Filter functionFilter = new Filter("Function", raFilter.Function, "eq"); currentFilters.Add(functionFilter); } if (raFilter.Class != null && raFilter.Class != "") { Filter classFilter = new Filter("Class", raFilter.Class, "eq"); currentFilters.Add(classFilter); } // AND the current filters FilterDefinition <MongoRecordAuthorityKeyPhrase> andFilter = Utils.GenerateMongoFilter <MongoRecordAuthorityKeyPhrase>(currentFilters); // OR with the combined filter if (combinedMongoFilter == null) { combinedMongoFilter = andFilter; } else { combinedMongoFilter = combinedMongoFilter | andFilter; } } var documents = collection.Find(combinedMongoFilter).ToList(); foreach (var keyphrase in documents) { keyphrases.Add(keyphrase); } break; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return(keyphrases); }
public static string GetMIMEType(DataConfig providerConfig, POCO.System system, POCO.Record record, RecordToRecordAssociation recordAssocEntity, ILogger logger) { string mimeType = string.Empty; List <DataFactory.Filter> filters = new List <Filter>(); DataFactory.Filter pkFilter = new Filter("PartitionKey", recordAssocEntity.RowKey, "eq"); filters.Add(pkFilter); //DataFactory.Filter rkFilter = new Filter("RowKey", , "eq"); //filters.Add(rkFilter); string nextPageId = string.Empty; string tableName = string.Empty; // Check which System the file is from switch (system.Type) { case SystemType.SharePoint2010: switch (providerConfig.ProviderType) { case "azure.tableservice": tableName = TableNames.Azure.SharePoint.SPFile; break; case "internal.mongodb": tableName = TableNames.Mongo.SharePoint.SPFile; break; } break; case SystemType.SharePoint2013: switch (providerConfig.ProviderType) { case "azure.tableservice": tableName = TableNames.Azure.SharePoint.SPFile; break; case "internal.mongodb": tableName = TableNames.Mongo.SharePoint.SPFile; break; } break; case SystemType.SharePointOnline: case SystemType.SharePointOneDrive: case SystemType.SharePointTeam: switch (providerConfig.ProviderType) { case "azure.tableservice": tableName = TableNames.Azure.SharePointOnline.SPFile; break; case "internal.mongodb": tableName = TableNames.Mongo.SharePointOnline.SPFile; break; } break; case SystemType.NTFSShare: switch (providerConfig.ProviderType) { case "azure.tableservice": tableName = TableNames.Azure.NTFS.NTFSFiles; break; case "internal.mongodb": tableName = TableNames.Mongo.NTFS.NTFSFiles; break; } break; case SystemType.SakaiAlliance: switch (providerConfig.ProviderType) { case "azure.tableservice": tableName = TableNames.Azure.Sakai.SakaiFiles; break; case "internal.mongodb": tableName = TableNames.Mongo.Sakai.SakaiFiles; break; } break; default: throw new NotImplementedException(); break; } List <POCO.CPFileMIMEType> mimetypes = new List <POCO.CPFileMIMEType>(); switch (providerConfig.ProviderType) { case "azure.tableservice": string combinedFilter = Utils.GenerateAzureFilter(filters); List <AzureFileMIMEType> azdata = new List <AzureFileMIMEType>(); AzureTableAdaptor <AzureFileMIMEType> adaptor = new AzureTableAdaptor <AzureFileMIMEType>(); azdata = adaptor.ReadTableData(providerConfig, tableName, combinedFilter); foreach (var doc in azdata) { mimetypes.Add(doc.Value); } break; case "internal.mongodb": var collection = Utils.GetMongoCollection <MongoFileMIMEType>(providerConfig, tableName); FilterDefinition <MongoFileMIMEType> filter = Utils.GenerateMongoFilter <MongoFileMIMEType>(filters); var documents = collection.Find(filter).ToList(); foreach (var doc in documents) { mimetypes.Add(doc); } break; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } if (mimetypes.Count > 0) { mimeType = mimetypes[0].MIMEType; } return(mimeType); }
public static List <POCO.NTFSFile> GetFiles(DataConfig providerConfig, List <Filter> filters, string thisPageId, int rowLimit, out string nextPageId) { nextPageId = string.Empty; List <POCO.NTFSFile> files = new List <POCO.NTFSFile>(); switch (providerConfig.ProviderType) { case "azure.tableservice": string combinedFilter = Utils.GenerateAzureFilter(filters); TableContinuationToken thisPageToken = null; if (thisPageId != null && thisPageId != string.Empty) { thisPageToken = Newtonsoft.Json.JsonConvert.DeserializeObject <TableContinuationToken>(thisPageId); } TableContinuationToken nextPageToken = null; List <AzureNTFSFile> azdata = new List <AzureNTFSFile>(); AzureTableAdaptor <AzureNTFSFile> adaptor = new AzureTableAdaptor <AzureNTFSFile>(); azdata = adaptor.ReadTableDataWithToken(providerConfig, NTFS.AzureTableNames.NTFSFiles, combinedFilter, rowLimit, thisPageToken, out nextPageToken); foreach (var doc in azdata) { files.Add(doc.Value); } // Check if there is a next page token available if (nextPageToken != null) { nextPageId = Newtonsoft.Json.JsonConvert.SerializeObject(nextPageToken); } break; //throw new NotImplementedException(); //string combinedFilter = Utils.GenerateAzureFilter(filters); //List<AzureNTFSFile> azdata = new List<AzureNTFSFile>(); //AzureTableAdaptor<AzureNTFSFile> adaptor = new AzureTableAdaptor<AzureNTFSFile>(); //azdata = adaptor.ReadTableData(providerConfig, NTFS.AzureTableNames.NTFSFiles, combinedFilter); //foreach (var doc in azdata) //{ // files.Add(doc.Value); //} break; case "internal.mongodb": var collection = Utils.GetMongoCollection <MongoNTFSFile>(providerConfig, NTFS.MongoTableNames.NTFSFiles); // Add an _id filter if a page has been requested if (thisPageId != null && thisPageId != string.Empty) { filters.Insert(0, new Filter("_id", thisPageId, "gt")); } FilterDefinition <MongoNTFSFile> filter = Utils.GenerateMongoFilter <MongoNTFSFile>(filters); //DEBUG output the filter values //foreach (Castlepoint.DataFactory.Filter debugFilter in filters) //{ // // Output the filter field names and values // Console.WriteLine("DEBUG filter: " + debugFilter.FieldName + " : " + debugFilter.FieldValue); //} var documents = collection.Find(filter).Sort("{\"_id\":1}").Limit(rowLimit).ToList(); foreach (var NTFSFile in documents) { files.Add(NTFSFile); } // Get the next page id if (documents.Count == rowLimit) { // Set the next page id nextPageId = documents[documents.Count - 1]._id.ToString(); } break; default: throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType); } return(files); }