public static List <POCO.SakaiFileBatchStatus> GetFileBatchStatus(DataConfig providerConfig, List <Filter> filters)
        {
            List <POCO.SakaiFileBatchStatus> fileBatchStatus = new List <POCO.SakaiFileBatchStatus>();

            switch (providerConfig.ProviderType)
            {
            case "azure.tableservice":

                string combinedFilter = Utils.GenerateAzureFilter(filters);

                List <AzureSakaiFileBatchStatus> azdata = new List <AzureSakaiFileBatchStatus>();
                AzureTableAdaptor <AzureSakaiFileBatchStatus> adaptor = new AzureTableAdaptor <AzureSakaiFileBatchStatus>();
                azdata = adaptor.ReadTableData(providerConfig, SakaiResource.AzureTableNames.SakaiFilesBatchStatus, combinedFilter);

                foreach (var doc in azdata)
                {
                    fileBatchStatus.Add(doc.Value);
                }

                break;

            case "internal.mongodb":
                var collection = Utils.GetMongoCollection <MongoSakaiFileBatchStatus>(providerConfig, SakaiResource.MongoTableNames.SakaiFilesBatchStatus);
                FilterDefinition <MongoSakaiFileBatchStatus> filter = Utils.GenerateMongoFilter <MongoSakaiFileBatchStatus>(filters);

                var documents = collection.Find(filter).ToList();

                foreach (var status in documents)
                {
                    fileBatchStatus.Add(status);
                }
                break;

            default:
                throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
            }

            return(fileBatchStatus);
        }
Beispiel #2
0
        public static List <POCO.SystemNodeConfig> GetAllBatchNodeConfig(DataConfig providerConfig)
        {
            List <POCO.SystemNodeConfig> systems = new List <POCO.SystemNodeConfig>();

            switch (providerConfig.ProviderType)
            {
            case "azure.tableservice":
                List <AzureSystemBatchConfig> azs             = new List <AzureSystemBatchConfig>();
                AzureTableAdaptor <AzureSystemBatchConfig> az = new AzureTableAdaptor <AzureSystemBatchConfig>();
                azs = az.ReadTableData(providerConfig, POCO.TableNames.Azure.System.BatchConfig);

                foreach (var doc in azs)
                {
                    systems.Add(doc.Value);
                }

                return(systems);


            case "internal.mongodb":
                var collection = Utils.GetMongoCollection <MongoSystemBatchConfig>(providerConfig, POCO.TableNames.Mongo.System.BatchConfig);

                var documents = collection.Find(new BsonDocument()).ToList();

                foreach (var system in documents)
                {
                    systems.Add(system);
                }
                break;

            default:
                throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
            }

            return(systems);
        }
        public static List <POCO.ProcessingBatchStatus> GetProcessingStatus(DataConfig providerConfig)
        {
            List <POCO.ProcessingBatchStatus> processingStatus = new List <POCO.ProcessingBatchStatus>();

            switch (providerConfig.ProviderType)
            {
            case "azure.tableservice":
                List <AzureProcessingBatchStatus> azs             = new List <AzureProcessingBatchStatus>();
                AzureTableAdaptor <AzureProcessingBatchStatus> az = new AzureTableAdaptor <AzureProcessingBatchStatus>();
                azs = az.ReadTableData(providerConfig, AzureTableNames.SystemProcessingStatus);

                foreach (var doc in azs)
                {
                    processingStatus.Add(doc.Value);
                }

                return(processingStatus);


            case "internal.mongodb":
                var collection = Utils.GetMongoCollection <MongoProcessingBatchStatus>(providerConfig, MongoTableNames.SystemProcessingStatus);

                var documents = collection.Find(new BsonDocument()).ToList();

                foreach (var doc in documents)
                {
                    processingStatus.Add(doc);
                }
                break;

            default:
                throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
            }

            return(processingStatus);
        }
Beispiel #4
0
        public static List <POCO.O365.EOMessageEntity> GetMessages(DataConfig providerConfig, List <Filter> filters, string thisPageId, int rowLimit, out string nextPageId)
        {
            nextPageId = string.Empty;

            List <POCO.O365.EOMessageEntity> files = new List <POCO.O365.EOMessageEntity>();

            switch (providerConfig.ProviderType)
            {
            case "azure.tableservice":
            {
                string combinedFilter = Utils.GenerateAzureFilter(filters);
                TableContinuationToken thisPageToken = null;
                if (thisPageId != null && thisPageId != string.Empty)
                {
                    thisPageToken = Newtonsoft.Json.JsonConvert.DeserializeObject <TableContinuationToken>(thisPageId);
                }
                TableContinuationToken      nextPageToken        = null;
                List <AzureEOMessageEntity> azdata               = new List <AzureEOMessageEntity>();
                AzureTableAdaptor <AzureEOMessageEntity> adaptor = new AzureTableAdaptor <AzureEOMessageEntity>();
                azdata = adaptor.ReadTableDataWithToken(providerConfig, AzureTableNames.EOMessage, combinedFilter, rowLimit, thisPageToken, out nextPageToken);

                foreach (var doc in azdata)
                {
                    files.Add(doc.Value);
                }

                // Check if there is a next page token available
                if (nextPageToken != null)
                {
                    nextPageId = Newtonsoft.Json.JsonConvert.SerializeObject(nextPageToken);
                }

                break;
            }

            case "internal.mongodb":
            {
                var lastId     = "";
                var collection = Utils.GetMongoCollection <MongoEOMessageEntity>(providerConfig, MongoTableNames.EOMessage);

                // Add an _id filter if a page has been requested
                if (thisPageId != null && thisPageId != string.Empty)
                {
                    filters.Insert(0, new Filter("_id", thisPageId, "gt"));
                }


                FilterDefinition <MongoEOMessageEntity> filter = Utils.GenerateMongoFilter <MongoEOMessageEntity>(filters);

                var documents = collection.Find(filter).Sort("{\"_id\":1}").Limit(rowLimit).ToList();

                foreach (var file in documents)
                {
                    files.Add(file);
                    lastId = file._id.ToString();
                }

                // Check if there are more pages of data
                if (files.Count == rowLimit)
                {
                    // Return the _id of the last row
                    nextPageId = lastId;
                }

                break;
            }

            default:
                throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
            }

            return(files);
        }
Beispiel #5
0
        public static List <POCO.RecordAssociationNamedEntityReverse> GetReverseNamedEntities(DataConfig providerConfig, List <Filter> filters, string namedEntityLocation)
        {
            List <POCO.RecordAssociationNamedEntityReverse> namedents = new List <POCO.RecordAssociationNamedEntityReverse>();

            string tableName = string.Empty;

            switch (providerConfig.ProviderType)
            {
            case "azure.tableservice":

                switch (namedEntityLocation)
                {
                case "filename":
                {
                    tableName = AzureTableNames.RecordAssociationNamedEntityReverse;
                    break;
                }

                case "body":
                {
                    tableName = AzureTableNames.RecordAssociationNamedEntityReverse;
                    break;
                }

                default:
                    throw new ApplicationException("Named entity location not recognised: " + namedEntityLocation);
                }

                string combinedFilter = Utils.GenerateAzureFilter(filters);

                List <AzureRecordAssociationNamedEntityReverse> azdata = new List <AzureRecordAssociationNamedEntityReverse>();
                AzureTableAdaptor <AzureRecordAssociationNamedEntityReverse> adaptor = new AzureTableAdaptor <AzureRecordAssociationNamedEntityReverse>();
                azdata = adaptor.ReadTableData(providerConfig, tableName, combinedFilter);

                foreach (var doc in azdata)
                {
                    namedents.Add(doc.Value);
                }

                break;

            case "internal.mongodb":
                switch (namedEntityLocation)
                {
                case "filename":
                {
                    tableName = MongoTableNames.RecordAssociationNamedEntityReverse;
                    break;
                }

                case "body":
                {
                    tableName = MongoTableNames.RecordAssociationNamedEntityReverse;
                    break;
                }

                default:
                    throw new ApplicationException("Named entity location not recognised: " + namedEntityLocation);
                }


                var collection = Utils.GetMongoCollection <MongoRecordAssociationNamedEntityReverse>(providerConfig, tableName);

                FilterDefinition <MongoRecordAssociationNamedEntityReverse> filter = Utils.GenerateMongoFilter <MongoRecordAssociationNamedEntityReverse>(filters);

                var documents = collection.Find(filter).ToList();

                foreach (var keyphrase in documents)
                {
                    namedents.Add(keyphrase);
                }

                break;

            default:
                throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
            }

            return(namedents);
        }
        public static List <POCO.SakaiFile> GetFiles(DataConfig providerConfig, List <Filter> filters, string thisPageId, int rowLimit, out string nextPageId)
        {
            nextPageId = string.Empty;
            List <POCO.SakaiFile> files = new List <POCO.SakaiFile>();

            switch (providerConfig.ProviderType)
            {
            case "azure.tableservice":

                throw new NotImplementedException();

                string combinedFilter = Utils.GenerateAzureFilter(filters);

                List <AzureSakaiFile> azdata = new List <AzureSakaiFile>();
                AzureTableAdaptor <AzureSakaiFile> adaptor = new AzureTableAdaptor <AzureSakaiFile>();
                azdata = adaptor.ReadTableData(providerConfig, SakaiResource.AzureTableNames.SakaiFiles, combinedFilter);

                foreach (var doc in azdata)
                {
                    files.Add(doc.Value);
                }

                break;

            case "internal.mongodb":
                var collection = Utils.GetMongoCollection <MongoSakaiFile>(providerConfig, SakaiResource.MongoTableNames.SakaiFiles);

                // Add an _id filter if a page has been requested
                if (thisPageId != null && thisPageId != string.Empty)
                {
                    filters.Insert(0, new Filter("_id", thisPageId, "gt"));
                }

                FilterDefinition <MongoSakaiFile> filter = Utils.GenerateMongoFilter <MongoSakaiFile>(filters);

                //DEBUG output the filter values
                //foreach (Castlepoint.DataFactory.Filter debugFilter in filters)
                //{
                //    // Output the filter field names and values
                //    Console.WriteLine("DEBUG filter: " + debugFilter.FieldName + " : " + debugFilter.FieldValue);
                //}
                var documents = collection.Find(filter).Sort("{\"_id\":1}").Limit(rowLimit).ToList();


                foreach (var SakaiFile in documents)
                {
                    files.Add(SakaiFile);
                }

                // Get the next page id
                if (documents.Count == rowLimit)
                {
                    // Set the next page id
                    nextPageId = documents[documents.Count - 1]._id.ToString();
                }


                break;

            default:
                throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
            }

            return(files);
        }
Beispiel #7
0
        public static List <POCO.RecordAuthorityFunctionActivityEntry> GetEntries(DataConfig providerConfig, List <POCO.RecordAuthorityFilter> raFilters)
        {
            List <POCO.RecordAuthorityFunctionActivityEntry> raitems = new List <POCO.RecordAuthorityFunctionActivityEntry>();

            switch (providerConfig.ProviderType)
            {
            case "azure.tableservice":


                string combinedAzureFilter = string.Empty;

                foreach (POCO.RecordAuthorityFilter raFilter in raFilters)
                {
                    List <Filter> currentFilters = new List <Filter>();
                    if (raFilter.RASchemaUri != null && raFilter.RASchemaUri != "")
                    {
                        Filter raschemaFilter = new Filter("PartitionKey", raFilter.RASchemaUri, "eq");
                        currentFilters.Add(raschemaFilter);
                    }
                    if (raFilter.Function != null && raFilter.Function != "")
                    {
                        Filter functionFilter = new Filter("Function", raFilter.Function, "eq");
                        currentFilters.Add(functionFilter);
                    }
                    if (raFilter.Class != null && raFilter.Class != "")
                    {
                        Filter classFilter = new Filter("Class", raFilter.Class, "eq");
                        currentFilters.Add(classFilter);
                    }

                    // AND the current filters
                    string andFilter = Utils.GenerateAzureFilter(currentFilters);

                    // OR with the combined filter
                    if (combinedAzureFilter == string.Empty)
                    {
                        combinedAzureFilter = andFilter;
                    }
                    else
                    {
                        combinedAzureFilter = TableQuery.CombineFilters(combinedAzureFilter, TableOperators.Or, andFilter);
                    }
                }

                //if (combinedAzureFilter == null || combinedAzureFilter==string.Empty)
                //{
                //    throw new InvalidOperationException("Schema filter has not been provided. Cannot retrieve Record Authority information.");
                //}

                List <AzureRecordAuthorityFunctionActivityEntry> azdata = new List <AzureRecordAuthorityFunctionActivityEntry>();
                AzureTableAdaptor <AzureRecordAuthorityFunctionActivityEntry> adaptor = new AzureTableAdaptor <AzureRecordAuthorityFunctionActivityEntry>();
                azdata = adaptor.ReadTableData(providerConfig, AzureTableNames.RecordAuthority, combinedAzureFilter);

                foreach (var doc in azdata)
                {
                    raitems.Add(doc.Value);
                }

                break;

            case "internal.mongodb":
                var collection = Utils.GetMongoCollection <MongoRecordAuthorityFunctionActivityEntry>(providerConfig, MongoTableNames.RecordAuthority);

                FilterDefinition <MongoRecordAuthorityFunctionActivityEntry> combinedFilter = null;
                List <Filter> filters = new List <Filter>();
                foreach (POCO.RecordAuthorityFilter raf in raFilters)
                {
                    List <Filter> thisFilter = new List <Filter>();
                    if (!string.IsNullOrEmpty(raf.RASchemaUri))
                    {
                        Filter fschema = new Filter("PartitionKey", raf.RASchemaUri, "eq");
                        thisFilter.Add(fschema);
                    }
                    if (!string.IsNullOrEmpty(raf.Function))
                    {
                        Filter ffunction = new Filter("Function", raf.Function, "eq");
                        thisFilter.Add(ffunction);
                    }
                    if (!string.IsNullOrEmpty(raf.Class))
                    {
                        Filter fclass = new Filter("Class", raf.Class, "eq");
                        thisFilter.Add(fclass);
                    }

                    // AND the filters
                    FilterDefinition <MongoRecordAuthorityFunctionActivityEntry> andFilter = Utils.GenerateMongoFilter <MongoRecordAuthorityFunctionActivityEntry>(thisFilter);

                    // OR with the combined filter
                    if (combinedFilter == null)
                    {
                        combinedFilter = andFilter;
                    }
                    else
                    {
                        combinedFilter = combinedFilter | andFilter;
                    }
                }

                if (combinedFilter == null)
                {
                    // Set to blank filter
                    combinedFilter = Utils.GenerateMongoFilter <MongoRecordAuthorityFunctionActivityEntry>(filters);
                    //throw new InvalidOperationException("Schema filter has not been provided. Cannot retrieve Record Authority information.");
                }

                var documents = collection.Find(combinedFilter).ToList();

                foreach (var raitem in documents)
                {
                    raitems.Add(raitem);
                }
                break;

            default:
                throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
            }

            return(raitems);
        }
Beispiel #8
0
        public static List <POCO.RecordAuthorityKeyPhrase> GetKeyPhrases(DataConfig providerConfig, List <POCO.RecordAuthorityFilter> raFilters)
        {
            List <POCO.RecordAuthorityKeyPhrase> keyphrases = new List <POCO.RecordAuthorityKeyPhrase>();

            switch (providerConfig.ProviderType)
            {
            case "azure.tableservice":

                string combinedAzureFilter = string.Empty;

                foreach (POCO.RecordAuthorityFilter raFilter in raFilters)
                {
                    List <Filter> currentFilters = new List <Filter>();
                    if (raFilter.RASchemaUri != null && raFilter.RASchemaUri != "")
                    {
                        Filter raschemaFilter = new Filter("PartitionKey", raFilter.RASchemaUri, "eq");
                        currentFilters.Add(raschemaFilter);
                    }
                    if (raFilter.Function != null && raFilter.Function != "")
                    {
                        Filter functionFilter = new Filter("Function", raFilter.Function, "eq");
                        currentFilters.Add(functionFilter);
                    }
                    if (raFilter.Class != null && raFilter.Class != "")
                    {
                        Filter classFilter = new Filter("Class", raFilter.Class, "eq");
                        currentFilters.Add(classFilter);
                    }

                    // AND the current filters
                    string andFilter = Utils.GenerateAzureFilter(currentFilters);

                    // OR with the combined filter
                    if (combinedAzureFilter == string.Empty)
                    {
                        combinedAzureFilter = andFilter;
                    }
                    else
                    {
                        combinedAzureFilter = TableQuery.CombineFilters(combinedAzureFilter, TableOperators.Or, andFilter);
                    }
                }
                List <AzureRecordAuthorityKeyPhrase> azdata = new List <AzureRecordAuthorityKeyPhrase>();
                AzureTableAdaptor <AzureRecordAuthorityKeyPhrase> adaptor = new AzureTableAdaptor <AzureRecordAuthorityKeyPhrase>();
                azdata = adaptor.ReadTableData(providerConfig, AzureTableNames.RecordAuthorityKeyphrases, combinedAzureFilter);

                foreach (var doc in azdata)
                {
                    keyphrases.Add(doc.Value);
                }

                break;

            case "internal.mongodb":
                var collection = Utils.GetMongoCollection <MongoRecordAuthorityKeyPhrase>(providerConfig, MongoTableNames.RecordAuthorityKeyphrases);

                FilterDefinition <MongoRecordAuthorityKeyPhrase> combinedMongoFilter = null;

                foreach (POCO.RecordAuthorityFilter raFilter in raFilters)
                {
                    List <Filter> currentFilters = new List <Filter>();
                    if (raFilter.RASchemaUri != null && raFilter.RASchemaUri != "")
                    {
                        Filter raschemaFilter = new Filter("PartitionKey", raFilter.RASchemaUri, "eq");
                        currentFilters.Add(raschemaFilter);
                    }
                    if (raFilter.Function != null && raFilter.Function != "")
                    {
                        Filter functionFilter = new Filter("Function", raFilter.Function, "eq");
                        currentFilters.Add(functionFilter);
                    }
                    if (raFilter.Class != null && raFilter.Class != "")
                    {
                        Filter classFilter = new Filter("Class", raFilter.Class, "eq");
                        currentFilters.Add(classFilter);
                    }

                    // AND the current filters
                    FilterDefinition <MongoRecordAuthorityKeyPhrase> andFilter = Utils.GenerateMongoFilter <MongoRecordAuthorityKeyPhrase>(currentFilters);

                    // OR with the combined filter
                    if (combinedMongoFilter == null)
                    {
                        combinedMongoFilter = andFilter;
                    }
                    else
                    {
                        combinedMongoFilter = combinedMongoFilter | andFilter;
                    }
                }

                var documents = collection.Find(combinedMongoFilter).ToList();

                foreach (var keyphrase in documents)
                {
                    keyphrases.Add(keyphrase);
                }
                break;

            default:
                throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
            }

            return(keyphrases);
        }
Beispiel #9
0
        public static string GetMIMEType(DataConfig providerConfig, POCO.System system, POCO.Record record, RecordToRecordAssociation recordAssocEntity, ILogger logger)
        {
            string mimeType = string.Empty;

            List <DataFactory.Filter> filters = new List <Filter>();

            DataFactory.Filter pkFilter = new Filter("PartitionKey", recordAssocEntity.RowKey, "eq");
            filters.Add(pkFilter);
            //DataFactory.Filter rkFilter = new Filter("RowKey", , "eq");
            //filters.Add(rkFilter);

            string nextPageId = string.Empty;

            string tableName = string.Empty;

            // Check which System the file is from
            switch (system.Type)
            {
            case SystemType.SharePoint2010:
                switch (providerConfig.ProviderType)
                {
                case "azure.tableservice":
                    tableName = TableNames.Azure.SharePoint.SPFile;
                    break;

                case "internal.mongodb":
                    tableName = TableNames.Mongo.SharePoint.SPFile;
                    break;
                }
                break;

            case SystemType.SharePoint2013:
                switch (providerConfig.ProviderType)
                {
                case "azure.tableservice":
                    tableName = TableNames.Azure.SharePoint.SPFile;
                    break;

                case "internal.mongodb":
                    tableName = TableNames.Mongo.SharePoint.SPFile;
                    break;
                }
                break;

            case SystemType.SharePointOnline:
            case SystemType.SharePointOneDrive:
            case SystemType.SharePointTeam:
                switch (providerConfig.ProviderType)
                {
                case "azure.tableservice":
                    tableName = TableNames.Azure.SharePointOnline.SPFile;
                    break;

                case "internal.mongodb":
                    tableName = TableNames.Mongo.SharePointOnline.SPFile;
                    break;
                }
                break;

            case SystemType.NTFSShare:
                switch (providerConfig.ProviderType)
                {
                case "azure.tableservice":
                    tableName = TableNames.Azure.NTFS.NTFSFiles;
                    break;

                case "internal.mongodb":
                    tableName = TableNames.Mongo.NTFS.NTFSFiles;
                    break;
                }
                break;

            case SystemType.SakaiAlliance:
                switch (providerConfig.ProviderType)
                {
                case "azure.tableservice":
                    tableName = TableNames.Azure.Sakai.SakaiFiles;
                    break;

                case "internal.mongodb":
                    tableName = TableNames.Mongo.Sakai.SakaiFiles;
                    break;
                }
                break;

            default:
                throw new NotImplementedException();
                break;
            }

            List <POCO.CPFileMIMEType> mimetypes = new List <POCO.CPFileMIMEType>();

            switch (providerConfig.ProviderType)
            {
            case "azure.tableservice":

                string combinedFilter = Utils.GenerateAzureFilter(filters);

                List <AzureFileMIMEType> azdata = new List <AzureFileMIMEType>();
                AzureTableAdaptor <AzureFileMIMEType> adaptor = new AzureTableAdaptor <AzureFileMIMEType>();
                azdata = adaptor.ReadTableData(providerConfig, tableName, combinedFilter);

                foreach (var doc in azdata)
                {
                    mimetypes.Add(doc.Value);
                }

                break;

            case "internal.mongodb":
                var collection = Utils.GetMongoCollection <MongoFileMIMEType>(providerConfig, tableName);

                FilterDefinition <MongoFileMIMEType> filter = Utils.GenerateMongoFilter <MongoFileMIMEType>(filters);

                var documents = collection.Find(filter).ToList();

                foreach (var doc in documents)
                {
                    mimetypes.Add(doc);
                }
                break;

            default:
                throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
            }

            if (mimetypes.Count > 0)
            {
                mimeType = mimetypes[0].MIMEType;
            }

            return(mimeType);
        }
Beispiel #10
0
        public static List <POCO.NTFSFile> GetFiles(DataConfig providerConfig, List <Filter> filters, string thisPageId, int rowLimit, out string nextPageId)
        {
            nextPageId = string.Empty;
            List <POCO.NTFSFile> files = new List <POCO.NTFSFile>();

            switch (providerConfig.ProviderType)
            {
            case "azure.tableservice":


                string combinedFilter = Utils.GenerateAzureFilter(filters);
                TableContinuationToken thisPageToken = null;
                if (thisPageId != null && thisPageId != string.Empty)
                {
                    thisPageToken = Newtonsoft.Json.JsonConvert.DeserializeObject <TableContinuationToken>(thisPageId);
                }
                TableContinuationToken            nextPageToken = null;
                List <AzureNTFSFile>              azdata        = new List <AzureNTFSFile>();
                AzureTableAdaptor <AzureNTFSFile> adaptor       = new AzureTableAdaptor <AzureNTFSFile>();
                azdata = adaptor.ReadTableDataWithToken(providerConfig, NTFS.AzureTableNames.NTFSFiles, combinedFilter, rowLimit, thisPageToken, out nextPageToken);

                foreach (var doc in azdata)
                {
                    files.Add(doc.Value);
                }

                // Check if there is a next page token available
                if (nextPageToken != null)
                {
                    nextPageId = Newtonsoft.Json.JsonConvert.SerializeObject(nextPageToken);
                }

                break;

                //throw new NotImplementedException();

                //string combinedFilter = Utils.GenerateAzureFilter(filters);

                //List<AzureNTFSFile> azdata = new List<AzureNTFSFile>();
                //AzureTableAdaptor<AzureNTFSFile> adaptor = new AzureTableAdaptor<AzureNTFSFile>();
                //azdata = adaptor.ReadTableData(providerConfig, NTFS.AzureTableNames.NTFSFiles, combinedFilter);

                //foreach (var doc in azdata)
                //{
                //    files.Add(doc.Value);
                //}

                break;

            case "internal.mongodb":
                var collection = Utils.GetMongoCollection <MongoNTFSFile>(providerConfig, NTFS.MongoTableNames.NTFSFiles);

                // Add an _id filter if a page has been requested
                if (thisPageId != null && thisPageId != string.Empty)
                {
                    filters.Insert(0, new Filter("_id", thisPageId, "gt"));
                }

                FilterDefinition <MongoNTFSFile> filter = Utils.GenerateMongoFilter <MongoNTFSFile>(filters);

                //DEBUG output the filter values
                //foreach (Castlepoint.DataFactory.Filter debugFilter in filters)
                //{
                //    // Output the filter field names and values
                //    Console.WriteLine("DEBUG filter: " + debugFilter.FieldName + " : " + debugFilter.FieldValue);
                //}
                var documents = collection.Find(filter).Sort("{\"_id\":1}").Limit(rowLimit).ToList();


                foreach (var NTFSFile in documents)
                {
                    files.Add(NTFSFile);
                }

                // Get the next page id
                if (documents.Count == rowLimit)
                {
                    // Set the next page id
                    nextPageId = documents[documents.Count - 1]._id.ToString();
                }


                break;

            default:
                throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
            }

            return(files);
        }