//public GetDocumentsResponse GetDocuments() //{ // GetDocumentsResponse response = new GetDocumentsResponse(); // try // { // IEnumerable<DocumentView> documents = _documentRepository.FindAll() // .ConvertToDocumentViews(); // response.DocumentViews = documents; // } // catch (Exception ex) // { // } // return response; //} public GetDocumentsResponse GetDocumentsBy(Guid customerID, IList <Sort> sort) { GetDocumentsResponse response = new GetDocumentsResponse(); try { Infrastructure.Querying.Query query = new Infrastructure.Querying.Query(); Criterion criterion = new Criterion("Customer.ID", customerID, CriteriaOperator.Equal); query.Add(criterion); Response <Document> documents = _documentRepository.FindBy(query, -1, -1, sort); IEnumerable <DocumentView> documentViews = documents.data.ConvertToDocumentViews(); // اصلاح آدرس عکسها و پاس دادن آدرس جدید foreach (var document in documentViews) { document.Photo = document.Photo.Replace(@"\", "/").Substring(document.Photo.IndexOf("data"));; } response.DocumentViews = documentViews; } catch (Exception ex) { } return(response); }
public ICollectionReader GetDocuments(ICollection<string> documentKeys) { if (documentKeys == null) throw new ArgumentNullException("documentKeys"); if (documentKeys.Count < 1) throw new ArgumentException("No DocumentKey specified"); List<JSONDocument> documents = new List<JSONDocument>(); foreach (string documentKey in documentKeys) { if (documentKey != null) { JSONDocument jdoc = new JSONDocument(); jdoc.Key = documentKey; documents.Add(jdoc); } } GetDocumentsOperation getDocumentsOperation = new GetDocumentsOperation(); getDocumentsOperation.DocumentIds = documents.Cast<IJSONDocument>().ToList(); getDocumentsOperation.Database = _database.DatabaseName; getDocumentsOperation.Collection = _collectionName; GetDocumentsResponse getDocumentsResponse = (GetDocumentsResponse)_database.ExecutionMapper.GetDocuments(getDocumentsOperation); if (getDocumentsResponse.IsSuccessfull) { CollectionReader reader = new CollectionReader((DataChunk)getDocumentsResponse.DataChunk, _database.ExecutionMapper, _database.DatabaseName, _collectionName); return reader; } else throw new Exception("Operation failed Error: " + Common.ErrorHandling.ErrorMessages.GetErrorMessage(getDocumentsResponse.ErrorCode, getDocumentsResponse.ErrorParams)); }
public ActionResult <GetDocumentsResponse> GetDocuments() { var response = new GetDocumentsResponse(); var documents = documentService.GetDocuments(); response.Documents = documents.Select(x => new Document { Location = x.Location, Name = x.Name, FileSize = x.FileSize, documentId = x.Id, SortOrder = x.SortOrder }).ToList(); return(response); }
public JsonResult Document_Read(Guid customerID, string sort) { GetDocumentsResponse response = new GetDocumentsResponse(); #region Access Check bool hasPermission = GetEmployee().IsGuaranteed("Document_Read"); if (!hasPermission) { ModelState.AddModelError("", "AccessDenied"); return(Json(response.DocumentViews, JsonRequestBehavior.AllowGet)); } #endregion response = _documentService.GetDocumentsBy(customerID, ConvertJsonToObject(sort)); return(Json(response.DocumentViews, JsonRequestBehavior.AllowGet)); }
public static void TestDotnerApiBloomFilters(string clientName, string accessKey) { double falsePositiveRate = 0.00001; string baseUri = "https://op-dhs-prod-read-nus.azurewebsites.net/";// "https://op-dhs-sandbox-read.azurewebsites.net/"; // IDocumentHostingService dhsClient = new DocumentHostingServiceClient(new Uri(baseUri), clientName, accessKey); IList <BloomFilter> result = new List <BloomFilter>(); IList <string> assetIds = new List <string>(); Dictionary <string, List <string> > conflicts = new Dictionary <string, List <string> >(StringComparer.OrdinalIgnoreCase); IList <GetDepotResponse> depots = dhsClient.GetAllDepotsBySiteBasePath("docs", "docs.microsoft.com/dotnet/", null, CancellationToken.None).Result; foreach (GetDepotResponse depot in depots) { if (depot.SystemMetadata.GetValueOrDefault <bool>(MetadataConstants.ActiveKey)) { string depotName = depot.DepotName; string continueAt = null; Console.WriteLine($"{depotName} Start."); IEnumerable <GetDocumentResponse> allDocuments = new List <GetDocumentResponse>(); int i = 0; do { for (int retry = 0; retry < 3; i++) { try { GetDocumentsResponse documents = dhsClient.GetDocumentsPaginated(depot.DepotName, "en-us", "live", true, continueAt, null, null, CancellationToken.None).Result; continueAt = documents.ContinueAt; allDocuments = allDocuments.Concat(documents.Documents); break; } catch (Exception e) { Console.WriteLine(e); Console.WriteLine($"Retry for {retry + 1} times"); } } i++; Console.WriteLine($"{i:000} .................."); }while (!string.IsNullOrEmpty(continueAt)); Console.WriteLine($"{depotName} Size: {allDocuments.Count()}."); var bloomFilter = new BloomFilter(allDocuments.Count(), falsePositiveRate); foreach (var document in allDocuments) { bloomFilter.Add(document.AssetId); assetIds.Add(document.AssetId); } Console.WriteLine($"{depotName} Bloom Filter Size: {bloomFilter.BitLength / 1024 / 8} KB."); result.Add(bloomFilter); Console.WriteLine($"{depotName} Done."); } } using (StreamWriter file = new StreamWriter(@"output.json", true)) { file.WriteLine(JsonConvert.SerializeObject(result)); } int onlyOneCount = 0; var conflictHashDict = new Dictionary <int, int>(); foreach (var assetId in assetIds) { var conflictCount = result.Where(r => r.Contains(assetId)).Count(); if (conflictCount == 1) { onlyOneCount++; } else { if (conflictHashDict.ContainsKey(conflictCount)) { conflictHashDict[conflictCount] += 1; } else { conflictHashDict[conflictCount] = 1; } } } Console.WriteLine($"Only one count: {onlyOneCount}, total count: {assetIds.Count()}"); Console.WriteLine($"Duplicate count: {conflictHashDict.Keys.Count}"); var output = conflictHashDict.OrderBy(p => p.Value).Reverse(); using (StreamWriter file = new StreamWriter(@"conflictCount.txt", true)) { foreach (var item in output) { file.WriteLine($"{item.Key}: {item.Value}"); } } Console.ReadLine(); }
public static void GenerateDotnetApiBloomFilter(string clientName, string accessKey) { double falsePositiveRate = 0.00001; string baseUri = "https://op-dhs-prod-read-nus.azurewebsites.net/";// "https://op-dhs-sandbox-read.azurewebsites.net/"; // IDocumentHostingService dhsClient = new DocumentHostingServiceClient(new Uri(baseUri), clientName, accessKey); IList <DepotBloomFilter> result = new List <DepotBloomFilter>(); Dictionary <string, List <string> > conflicts = new Dictionary <string, List <string> >(StringComparer.OrdinalIgnoreCase); IList <GetDepotResponse> depots = dhsClient.GetAllDepotsBySiteBasePath("docs", "docs.microsoft.com/dotnet/", null, CancellationToken.None).Result; foreach (GetDepotResponse depot in depots.Skip(20).Take(1)) { if (depot.SystemMetadata.GetValueOrDefault <bool>(MetadataConstants.ActiveKey)) { string depotName = depot.DepotName; string continueAt = null; Console.WriteLine($"{depotName} Start."); IEnumerable <GetDocumentResponse> allDocuments = new List <GetDocumentResponse>(); int i = 0; do { for (int retry = 0; retry < 3; i++) { try { GetDocumentsResponse documents = dhsClient.GetDocumentsPaginated(depot.DepotName, "en-us", "live", true, continueAt, null, null, CancellationToken.None).Result; continueAt = documents.ContinueAt; allDocuments = allDocuments.Concat(documents.Documents); break; } catch (Exception e) { Console.WriteLine(e); Console.WriteLine($"Retry for {retry + 1} times"); } } i++; Console.WriteLine($"{i:000} .................."); }while (!string.IsNullOrEmpty(continueAt)); Console.WriteLine($"{depotName} Size: {allDocuments.Count()}."); var bloomFilter = new BloomFilter(allDocuments.Count(), falsePositiveRate); foreach (var document in allDocuments) { bloomFilter.Add(document.AssetId); } Console.WriteLine($"{depotName} Bloom Filter Size: {bloomFilter.BitLength / 1024 / 8} KB."); result.Add(new DepotBloomFilter { DepotName = depotName, BloomFilter = bloomFilter.BitArray, Count = allDocuments.Count(), FalsePositiveRate = falsePositiveRate }); Console.WriteLine($"{depotName} Done."); } } /* * using (StreamWriter file = new StreamWriter(@"output.json", true)) * { * file.WriteLine(JsonConvert.SerializeObject(result)); * } */ MemoryStream ms = new MemoryStream(); using (BsonWriter writer = new BsonWriter(ms)) { JsonSerializer serializer = new JsonSerializer(); serializer.Serialize(writer, result); } Console.ReadLine(); }
private static void GetMergedRepo(IDocumentHostingService sandboxClient, IDocumentHostingService prodClient) { string[] depotNames = { //"MSDN.azuredotnet", //"MSDN.aspnetAPIDocs", //"MSDN.coredocs-demo", // Prod "MSDN.aspnetAPIDocs", "VS.core-docs", }; CancellationToken cancellationToken = new CancellationToken(); foreach (var depotName in depotNames) { Console.WriteLine(depotName); string continueAt = null; int count = 0; while (true) { GetDocumentsResponse documents = prodClient.GetDocumentsPaginated(depotName, "en-us", "master", false, continueAt, null, 100, cancellationToken).Result; continueAt = documents.ContinueAt; count += documents.Documents.Count; Console.WriteLine($"{count}, {DateTime.Now:HH:mm:ss tt zz}"); foreach (var document in documents.Documents) { var key = document.AssetId; if (key.StartsWith(HardCodedPrefix)) { key = key.Substring(HardCodedPrefix.Length); } document.AssetId = key; if (!DocumentDict.ContainsKey(key)) { DocumentDict.Add(key, new List <GetDocumentResponse>()); } DocumentDict[key].Add(document); } var putDocumentsRequest = new PutDocumentsRequest(); putDocumentsRequest.Documents.AddRange(documents.Documents.Select(d => new PutDocumentsRequestItem { AssetId = d.AssetId, ProductVersion = d.ProductVersion, ContentSourceUri = GetCorrectedCanonicalUrl(d.ContentUri, d.AssetId), //ContentSourceUri = d.ContentUri, Locale = d.Locale, Metadata = d.Metadata })); sandboxClient.PutDocuments(newDepotName, "master", putDocumentsRequest, null, cancellationToken).Wait(); if (string.IsNullOrEmpty(continueAt)) { break; } } } Console.WriteLine("Merge comeplete. Press any key to continue..."); }