static void LaunchParallelDataExtraction(int CurrentDocCount, BlobContainerFileProvider fileProvider)
{
// Delete any existing files
foreach (BlobContainerFileInfo file in (BlobContainerDirectoryContents)fileProvider.GetDirectoryContents(null))
{
if (file.Name.StartsWith(SourceIndexName))
{
file.DeleteBlob();
}
}
// Launch output in parallel
string IDFieldName = GetIDFieldName();
int FileCounter = 0;
for (int batch = 0; batch <= (CurrentDocCount / MaxBatchSize); batch += ParallelizedJobs)
{
List <Task> tasks = new List <Task>();
for (int job = 0; job < ParallelizedJobs; job++)
{
FileCounter++;
int fileCounter = FileCounter;
if ((fileCounter - 1) * MaxBatchSize < CurrentDocCount)
{
Console.WriteLine("Writing {0} docs to {1}", MaxBatchSize, SourceIndexName + fileCounter + ".json");
tasks.Add(Task.Factory.StartNew(() =>
ExportToJSON((fileCounter - 1) * MaxBatchSize, IDFieldName, SourceIndexName + fileCounter + ".json", fileProvider)
));
}
}
Task.WaitAll(tasks.ToArray()); // Wait for all the exports to complete
}
return;
}
static void ImportFromJSON(BlobContainerFileProvider fileProvider)
{
// Take JSON file and import this as-is to target index
Uri ServiceUri = new Uri("https://" + SourceSearchServiceName + ".search.windows.net");
HttpClient HttpClient = new HttpClient();
HttpClient.DefaultRequestHeaders.Add("api-key", SourceAPIKey);
try
{
// Delete any existing files
foreach (BlobContainerFileInfo file in (BlobContainerDirectoryContents)fileProvider.GetDirectoryContents(null))
{
if (file.Name.StartsWith(SourceIndexName))
{
Console.WriteLine("Uploading documents from file {0}", file.Name);
Uri uri = new Uri(ServiceUri, "/indexes/" + TargetIndexName + "/docs/index");
using (var ms = (MemoryStream)file.CreateReadStream())
{
string json = ms.ToString();
HttpResponseMessage response = AzureSearchHelper.SendSearchRequest(HttpClient, HttpMethod.Post, uri, json);
response.EnsureSuccessStatusCode();
}
}
}
//foreach (string fileName in Directory.GetFiles(Directory.GetCurrentDirectory(), SourceIndexName + "*.json"))
//{
// Console.WriteLine("Uploading documents from file {0}", fileName);
// string json = File.ReadAllText(fileName);
// Uri uri = new Uri(ServiceUri, "/indexes/" + TargetIndexName + "/docs/index");
// HttpResponseMessage response = AzureSearchHelper.SendSearchRequest(HttpClient, HttpMethod.Post, uri, json);
// response.EnsureSuccessStatusCode();
//}
}
catch (Exception ex)
{
Console.WriteLine("Error: {0}", ex.Message.ToString());
}
}
