static void LaunchParallelDataExtraction(int CurrentDocCount, BlobContainerFileProvider fileProvider) { // Delete any existing files foreach (BlobContainerFileInfo file in (BlobContainerDirectoryContents)fileProvider.GetDirectoryContents(null)) { if (file.Name.StartsWith(SourceIndexName)) { file.DeleteBlob(); } } // Launch output in parallel string IDFieldName = GetIDFieldName(); int FileCounter = 0; for (int batch = 0; batch <= (CurrentDocCount / MaxBatchSize); batch += ParallelizedJobs) { List <Task> tasks = new List <Task>(); for (int job = 0; job < ParallelizedJobs; job++) { FileCounter++; int fileCounter = FileCounter; if ((fileCounter - 1) * MaxBatchSize < CurrentDocCount) { Console.WriteLine("Writing {0} docs to {1}", MaxBatchSize, SourceIndexName + fileCounter + ".json"); tasks.Add(Task.Factory.StartNew(() => ExportToJSON((fileCounter - 1) * MaxBatchSize, IDFieldName, SourceIndexName + fileCounter + ".json", fileProvider) )); } } Task.WaitAll(tasks.ToArray()); // Wait for all the exports to complete } return; }
static void ImportFromJSON(BlobContainerFileProvider fileProvider) { // Take JSON file and import this as-is to target index Uri ServiceUri = new Uri("https://" + SourceSearchServiceName + ".search.windows.net"); HttpClient HttpClient = new HttpClient(); HttpClient.DefaultRequestHeaders.Add("api-key", SourceAPIKey); try { // Delete any existing files foreach (BlobContainerFileInfo file in (BlobContainerDirectoryContents)fileProvider.GetDirectoryContents(null)) { if (file.Name.StartsWith(SourceIndexName)) { Console.WriteLine("Uploading documents from file {0}", file.Name); Uri uri = new Uri(ServiceUri, "/indexes/" + TargetIndexName + "/docs/index"); using (var ms = (MemoryStream)file.CreateReadStream()) { string json = ms.ToString(); HttpResponseMessage response = AzureSearchHelper.SendSearchRequest(HttpClient, HttpMethod.Post, uri, json); response.EnsureSuccessStatusCode(); } } } //foreach (string fileName in Directory.GetFiles(Directory.GetCurrentDirectory(), SourceIndexName + "*.json")) //{ // Console.WriteLine("Uploading documents from file {0}", fileName); // string json = File.ReadAllText(fileName); // Uri uri = new Uri(ServiceUri, "/indexes/" + TargetIndexName + "/docs/index"); // HttpResponseMessage response = AzureSearchHelper.SendSearchRequest(HttpClient, HttpMethod.Post, uri, json); // response.EnsureSuccessStatusCode(); //} } catch (Exception ex) { Console.WriteLine("Error: {0}", ex.Message.ToString()); } }