protected async Task DeleteTestDataAsync()
{
var response = (JObject)await this.TenancyService.GetTenantAsync(this.ClientTenantId);
Tenancy.Client.Models.Tenant clientTenant = JsonConvert.DeserializeObject <Tenancy.Client.Models.Tenant>(response.ToString());
var tenant = new Tenant(clientTenant.Id, clientTenant.Name, this.PropertyBagFactory.Create(clientTenant.Properties));
BlobContainerClient claimPermissionsContainer =
await this.TenantBlobContainerClientFactory.GetBlobContainerClientFromTenantAsync(
tenant,
"StorageConfiguration__claimpermissions",
"StorageConfigurationV3__claimpermissions");
BlobContainerClient resourceAccessRuleSetsContainer =
await this.TenantBlobContainerClientFactory.GetBlobContainerClientFromTenantAsync(
tenant,
"StorageConfiguration__resourceaccessrulesets",
"StorageConfigurationV3__resourceaccessrulesets");
foreach (BlobItem blob in claimPermissionsContainer.GetBlobs())
{
await claimPermissionsContainer.DeleteBlobAsync(blob.Name);
}
foreach (BlobItem blob in resourceAccessRuleSetsContainer.GetBlobs())
{
await claimPermissionsContainer.DeleteBlobAsync(blob.Name);
}
}
public async Task <IActionResult> DeleteMap(int id)
{
string currentUser = User.FindFirstValue(ClaimTypes.NameIdentifier);
Map mapToDelete = await _context.Maps.FindAsync(id);
if (mapToDelete.UserId != currentUser)
{
return(BadRequest());
}
try
{
BlobContainerClient containerClient = blobServiceClient.GetBlobContainerClient(currentUser);
await containerClient.DeleteBlobAsync(mapToDelete.FileName);
}
catch (Exception exc)
{
_logger.AddSystemLog($"An Exception occured while deleting a map file: {exc}");
return(StatusCode(500));
}
_context.Remove(mapToDelete);
await _context.SaveChangesAsync();
return(Ok());
}
private async Task <Result> CleanContainerFolderStorageAsync(string container, string folder,
CancellationToken token)
{
var containerClient = new BlobContainerClient(_storageOptions.ConnectionString, container);
var blobClient = containerClient.GetBlobClient(folder);
if (!await blobClient.ExistsAsync(token))
{
return(Success());
}
var success = true;
string reason = null;
await foreach (var blob in containerClient.GetBlobsAsync(prefix: folder, cancellationToken: token))
{
var response = await containerClient.DeleteBlobAsync(blob.Name, cancellationToken : token);
if (response.Status >= 400)
{
success = false;
reason = response.ReasonPhrase;
}
}
return(success ? Success() : Failure(reason));
}
/// <summary>
/// Deletes the previously recorded checkpoints if the current checkpoint blob path (corresponding to the input event hub) has changed.
/// </summary>
public async Task ResetCheckpointsAsync()
{
try
{
_log.LogTrace($"Entering {nameof(ResetCheckpointsAsync)}...");
foreach (BlobItem blob in _storageClient.GetBlobs(states: BlobStates.All, prefix: _blobCheckpointPrefix, cancellationToken: CancellationToken.None))
{
if (!blob.Name.Contains(_blobPath, StringComparison.OrdinalIgnoreCase))
{
try
{
await _storageClient.DeleteBlobAsync(blob.Name, cancellationToken : CancellationToken.None);
_log.LogTrace($"Blob checkpoint path changed to {_blobPath}. Deleted checkpoint {blob.Name}.");
}
#pragma warning disable CA1031
catch (Exception ex)
#pragma warning restore CA1031
{
_log.LogError(new Exception($"Unable to delete checkpoint {blob.Name} with error {ex.Message}"));
}
}
}
_log.LogTrace($"Exiting {nameof(ResetCheckpointsAsync)}.");
}
#pragma warning disable CA1031
catch (Exception ex)
#pragma warning restore CA1031
{
_log.LogError(new Exception($"Unable to reset checkpoints. {ex.Message}"));
}
}
public static async Task AutoCleanup(
[TimerTrigger("0 0 0 * * *", RunOnStartup = true)] TimerInfo timer
, [Blob("backups", FileAccess.Read)] BlobContainerClient blobReader
, [Blob("backups", FileAccess.Write)] BlobContainerClient backupClient
, [Blob("archives", FileAccess.Write)] BlobContainerClient archiveClient
)
{
var pages = blobReader.GetBlobsAsync();
var blobsToMove = new List <BlobItem>();
var current = DateTime.Now;
await foreach (var blob in pages)
{
var date = DateTime.ParseExact(blob.Name.Replace(".zip", string.Empty),
dateFormat,
null);
if ((date - current).Days > 7)
{
blobsToMove.Add(blob);
}
}
foreach (var blob in blobsToMove)
{
await using var stream = (await blobReader.GetBlobClient(blob.Name).DownloadStreamingAsync()).Value.Content;
string blobName = $"Weekly-{blob.Name}";
var newBlob = await archiveClient.UploadBlobAsync(blobName, stream);
await archiveClient.GetBlobClient(blobName).SetAccessTierAsync(AccessTier.Archive);
await backupClient.DeleteBlobAsync(blob.Name);
}
}
public async Task DeleteBlobAsync(string containername
, string blobname)
{
BlobContainerClient containerClient =
this.service.GetBlobContainerClient(containername);
await containerClient.DeleteBlobAsync(blobname);
}
public async Task ClearStorage()
{
foreach (var blob in _solutionIdsToGuids.Values.SelectMany(directory => _blobContainer.GetBlobs(prefix: directory)))
{
await _blobContainer.DeleteBlobAsync(blob.Name);
}
}
public override async Task DeleteDirectoryAsync(string path, CancellationToken cancellationToken = default)
{
await GetDirectoryAsync(path, cancellationToken);
path = PrependRootPath(path);
path = path.EndsWith("/") ? path : path + "/";
try
{
await foreach (var item in client.GetBlobsAsync(BlobTraits.None, BlobStates.None, path))
{
await client.DeleteBlobAsync(item.Name, DeleteSnapshotsOption.IncludeSnapshots, cancellationToken : cancellationToken);
}
}
catch (Exception exception)
{
throw Exception(exception);
}
}
/// <summary>
/// Delete picture thumbs
/// </summary>
/// <param name="picture">Picture</param>
protected override async Task DeletePictureThumbs(Picture picture)
{
string filter = string.Format("{0}", picture.Id);
var blobs = container.GetBlobs(Azure.Storage.Blobs.Models.BlobTraits.All, Azure.Storage.Blobs.Models.BlobStates.All, filter);
foreach (var blob in blobs)
{
await container.DeleteBlobAsync(blob.Name);
}
}
public static async Task DeleteDirectoryAsync(string dir)
{
dir = RemoveContainerFromPathIfPresent(dir);
foreach (var blobItem in BlobContainerClient.GetBlobs())
{
if (blobItem.Name.StartsWith(dir))
{
await BlobContainerClient.DeleteBlobAsync(blobItem.Name);
}
}
}
private static async Async.Task AssertCanCRUD(Uri sasUrl)
{
var client = new BlobContainerClient(sasUrl);
await client.UploadBlobAsync("blob", new BinaryData("content")); // create
var b = Assert.Single(await client.GetBlobsAsync().ToListAsync()); // list
using (var s = await client.GetBlobClient(b.Name).OpenReadAsync())
using (var sr = new StreamReader(s)) {
Assert.Equal("content", await sr.ReadToEndAsync()); // read
}
await client.DeleteBlobAsync("blob"); // delete
}
public async Task <IActionResult> ClearBlobs()
{
try
{
await foreach (var blob in _blobClient.GetBlobsAsync(BlobTraits.Metadata))
{
await _blobClient.DeleteBlobAsync(blob.Name);
}
}
catch
{
}
return(Ok("Deleted all Blobs"));
}
private async Task <IActionResult> DeleteOldImageBlobIfNotEqual(WikiPage wikiPage)
{
BlobContainerClient containerClient = blobServiceClient.GetBlobContainerClient(wikiPage.UserId);
BlobClient blobClient = containerClient.GetBlobClient(wikiPage.PageName);
var blobProps = await blobClient.GetPropertiesAsync();
int oldSize = Convert.ToInt32(blobProps.Value.ContentLength);
if (oldSize != wikiPage.ImageFile.Length)
{
await containerClient.DeleteBlobAsync(wikiPage.PageName);
}
return(Ok());
}
public async Task DeleteFile(string fileName, string containerName)
{
BlobContainerClient container = blobServiceClient.GetBlobContainerClient(containerName);
try
{
await container.DeleteBlobAsync(fileName);
}
catch (RequestFailedException e)
{
Console.WriteLine(e.Message);
Console.ReadLine();
throw;
}
}
public async Task <bool> RemovePhotoAsync(string blobPath)
{
var typeContainerClient = new BlobContainerClient(_connectionString, "photos");
var blobs = typeContainerClient.GetBlobsAsync(prefix: blobPath);
await foreach (var blob in blobs)
{
var result = await typeContainerClient.DeleteBlobAsync(blob.Name, DeleteSnapshotsOption.IncludeSnapshots);
if ((HttpStatusCode)result.Status != HttpStatusCode.Accepted)
{
return(false);
}
}
return(true);
}
private static async Task UpdateAzureBlobStorageAsync(int count, ILogger logger)
{
logger.LogInformation("Start rebuilding search index");
logger.LogInformation("Retrieving data urls.");
var dataUrls = GetDataUrls(count);
// container name "index-storage"
var blobContainerClient = new BlobContainerClient(
Environment.GetEnvironmentVariable("AzureBlogStorageConnectionString"),
Environment.GetEnvironmentVariable("IndexContainerName"));
logger.LogInformation("Clear blob storage");
await foreach (var blob in blobContainerClient.GetBlobsAsync())
{
await blobContainerClient.DeleteBlobAsync(blob.Name, DeleteSnapshotsOption.IncludeSnapshots);
}
logger.LogInformation("Upload data files into blob storage.");
dataUrls.ToList().ForEach(async dataUrl =>
{
using var httpClient = new HttpClient();
logger.LogInformation($"Retrieving data for url: {dataUrl}");
var response = await httpClient.GetAsync(dataUrl);
if (response.StatusCode != System.Net.HttpStatusCode.OK)
{
logger.LogInformation($"Received response with status code {response.StatusCode} for request with url {dataUrl}. skipping...");
return;
}
try
{
await blobContainerClient.UploadBlobAsync($"{Guid.NewGuid()}.json", await response.Content.ReadAsStreamAsync());
}
catch (RequestFailedException)
{
logger.LogInformation("Blob did not change, skipping upload");
}
});
}
/// <summary>
/// Deletes the previously recorded checkpoints if the current checkpoint blob path (corresponding to the input event hub) has changed.
/// </summary>
/// <param name="cancellationToken">A cancellation token</param>
/// <returns><see cref="Task"/></returns>
public async Task ResetCheckpointsAsync(CancellationToken cancellationToken = default)
{
try
{
_logger.LogTrace($"Entering {nameof(ResetCheckpointsAsync)}...");
var hasEventHubChanged = false;
var blobs = _storageClient.GetBlobs(states: BlobStates.All, prefix: _blobCheckpointPrefix, cancellationToken: cancellationToken);
foreach (BlobItem blob in blobs)
{
if (!blob.Name.Contains(_blobPath, StringComparison.OrdinalIgnoreCase))
{
try
{
await _storageClient.DeleteBlobAsync(blob.Name, cancellationToken : cancellationToken);
_logger.LogTrace($"Blob checkpoint path changed to {_blobPath}. Deleted checkpoint {blob.Name}.");
hasEventHubChanged = true;
}
catch (Exception ex)
{
_logger.LogError(new StorageCheckpointClientException($"Unable to delete checkpoint {blob.Name} with error {ex.Message}", ex));
}
}
}
if (blobs.Count() == 0 || hasEventHubChanged)
{
_logger.LogMetric(EventMetrics.EventHubChanged(_blobPath.Replace(_blobCheckpointPrefix, string.Empty)), 1);
}
_logger.LogTrace($"Exiting {nameof(ResetCheckpointsAsync)}.");
}
catch (Exception ex)
{
_logger.LogError(new StorageCheckpointClientException($"Unable to reset checkpoints. {ex.Message}", ex));
}
}
public async Task <ActionResult> DeleteAll()
{
try
{
InitClient();
await foreach (BlobItem blob in blobContainer.GetBlobsAsync())
{
if (blob.Properties.BlobType == BlobType.Block)
{
await blobContainer.DeleteBlobAsync(blob.Name);
}
}
return(RedirectToAction("Index"));
}
catch (Exception ex)
{
ViewData["message"] = ex.Message;
ViewData["trace"] = ex.StackTrace;
return(View("Error"));
}
}
public Task <Response> DeleteBlobAsync(string blobName, string containerName)
{
_blobContainerClient = new BlobContainerClient(_connectionString, containerName);
return(_blobContainerClient.DeleteBlobAsync(blobName));
}
private static async Task RunAsync(CrawledSubscriptionList subscriptionList, bool reindex, bool pullLatest, bool randomReindex, bool uploadToAzure, string startingRepoName, string outputPath)
{
var reindexIntervalInDays = 28;
var today = DateTime.Today;
var connectionString = GetAzureStorageConnectionString();
// TODO: We should avoid having to use a temp directory
var tempDirectory = Path.Combine(Path.GetTempPath(), "ghcrawler");
if (Directory.Exists(tempDirectory))
{
Directory.Delete(tempDirectory, recursive: true);
}
Directory.CreateDirectory(tempDirectory);
var cacheContainerName = "cache";
var cacheContainerClient = new BlobContainerClient(connectionString, cacheContainerName);
if (!reindex || startingRepoName is not null)
{
var startingBlobName = $"{startingRepoName}.crcache";
var reachedStartingBlob = false;
await foreach (var blob in cacheContainerClient.GetBlobsAsync())
{
if (!subscriptionList.Contains(blob.Name.Replace(".crcache", "")))
{
continue;
}
if (blob.Name == startingBlobName)
{
reachedStartingBlob = true;
}
if (reachedStartingBlob)
{
continue;
}
Console.WriteLine($"Downloading {blob.Name}...");
var localPath = Path.Combine(tempDirectory, blob.Name);
var localDirectory = Path.GetDirectoryName(localPath);
Directory.CreateDirectory(localDirectory);
var blobClient = new BlobClient(connectionString, cacheContainerName, blob.Name);
await blobClient.DownloadToAsync(localPath);
}
}
var factory = CreateGitHubClientFactory();
var client = await factory.CreateAsync();
var jsonOptions = new JsonSerializerOptions()
{
WriteIndented = true
};
var repos = new List <CrawledRepo>();
var reachedStartingRepo = reindex && startingRepoName is null;
foreach (var org in subscriptionList.Orgs)
{
var orgDirectory = Path.Join(tempDirectory, org);
Directory.CreateDirectory(orgDirectory);
var existingRepos = Directory.GetFiles(orgDirectory, "*.crcache")
.Select(p => Path.GetFileNameWithoutExtension(p));
if (!pullLatest)
{
Console.WriteLine($"Loading repos for {org}...");
foreach (var repoName in existingRepos)
{
var blobName = $"{repoName}.crcache";
var repoPath = Path.Join(orgDirectory, blobName);
var repo = await CrawledRepo.LoadAsync(repoPath);
if (repo is not null)
{
repos.Add(repo);
}
}
}
else
{
Console.WriteLine($"Requesting repos for {org}...");
var availableRepos = await RequestReposAsync(factory, client, org);
var deletedRepos = existingRepos.ToHashSet(StringComparer.OrdinalIgnoreCase);
deletedRepos.ExceptWith(availableRepos.Select(r => r.Name));
foreach (var deletedRepo in deletedRepos)
{
var blobName = $"{org}/{deletedRepo}.crcache";
var repoPath = Path.Join(tempDirectory, blobName);
Console.WriteLine($"Deleting local file {blobName}...");
File.Delete(repoPath);
if (uploadToAzure)
{
Console.WriteLine($"Deleting Azure blob {blobName}...");
await cacheContainerClient.DeleteBlobAsync(blobName);
}
}
foreach (var repo in availableRepos)
{
if (!subscriptionList.Contains(org, repo.Name))
{
continue;
}
var blobName = $"{org}/{repo.Name}.crcache";
var repoPath = Path.Join(tempDirectory, blobName);
if (string.Equals($"{org}/{repo.Name}", startingRepoName, StringComparison.OrdinalIgnoreCase))
{
reachedStartingRepo = true;
}
CrawledRepo crawledRepo;
try
{
crawledRepo = await CrawledRepo.LoadAsync(repoPath);
}
catch (JsonException)
{
Console.WriteLine($"WARNING: Couldn't parse {blobName}");
crawledRepo = null;
}
if (crawledRepo is null)
{
crawledRepo = new CrawledRepo
{
Id = repo.Id,
Org = org,
Name = repo.Name
};
}
crawledRepo.IsArchived = repo.Archived;
crawledRepo.Size = repo.Size;
repos.Add(crawledRepo);
var repoIsDueForReindexing = crawledRepo.LastReindex is null ||
crawledRepo.LastReindex?.AddDays(reindexIntervalInDays) <= today;
if (reachedStartingRepo)
{
Console.WriteLine($"Marking {repo.FullName} to be re-indexed because we reached the starting repo {startingRepoName}.");
}
if (repoIsDueForReindexing)
{
if (crawledRepo.LastReindex is null)
{
Console.WriteLine($"Marking {repo.FullName} to be re-indexed because it was never fully indexed.");
}
else
{
Console.WriteLine($"Marking {repo.FullName} to be re-indexed because it was more than {reindexIntervalInDays} days ago, on {crawledRepo.LastReindex}.");
}
}
if (reachedStartingRepo || repoIsDueForReindexing)
{
crawledRepo.Clear();
}
}
}
}
// We want to ensure that all repos are fully-reindexed at least once every four weeks.
// That means we need to reindex at least #Repos / 28 per day.
//
// On top of that, we need to ensure that all repos which were never fully indexed (e.g.
// they are new or were forced to be reindexed) are also reindexed.
if (randomReindex)
{
var reposThatNeedReindexing = repos.Where(r => r.LastReindex is null).ToHashSet();
var minimumNumberOfReposToBeReindexed = (int)Math.Ceiling(repos.Count / (float)reindexIntervalInDays);
var numberOfReposThatNeedReindexing = reposThatNeedReindexing.Count;
if (numberOfReposThatNeedReindexing < minimumNumberOfReposToBeReindexed)
{
// OK, there are fewer repos that need reindexing than what we want to reindex
// per day. So let's randomly pick some repos to reindex.
var remainingRepos = repos.Except(reposThatNeedReindexing).ToList();
var choiceCount = minimumNumberOfReposToBeReindexed - numberOfReposThatNeedReindexing;
var random = new Random();
for (var choice = 0; choice < choiceCount; choice++)
{
var i = random.Next(0, remainingRepos.Count);
var repo = remainingRepos[i];
Console.WriteLine($"Marking {repo.FullName} to be re-indexed because it was randomly chosen.");
repo.Clear();
reposThatNeedReindexing.Add(repo);
remainingRepos.RemoveAt(i);
}
}
}
if (pullLatest)
{
Console.WriteLine($"Listing events...");
var eventStore = new GitHubEventStore(connectionString);
var events = await eventStore.ListAsync();
Console.WriteLine($"Crawling {repos.Count:N0} repos, fully reindexing {repos.Count(r => r.LastReindex is null):N0} repos...");
foreach (var crawledRepo in repos)
{
var blobName = $"{crawledRepo.FullName}.crcache";
var repoPath = Path.Join(tempDirectory, blobName);
var since = crawledRepo.IncrementalUpdateStart;
var messages = new List <GitHubEventMessage>();
if (since is null)
{
Console.WriteLine($"Crawling {crawledRepo.FullName}...");
}
else
{
var toBeDownloaded = events.Where(n => string.Equals(n.Org, crawledRepo.Org, StringComparison.OrdinalIgnoreCase) &&
string.Equals(n.Repo, crawledRepo.Name, StringComparison.OrdinalIgnoreCase))
.ToArray();
if (toBeDownloaded.Any())
{
Console.WriteLine($"Loading {toBeDownloaded.Length:N0} events for {crawledRepo.FullName}...");
var i = 0;
var lastPercent = 0;
foreach (var name in toBeDownloaded)
{
var percent = (int)Math.Ceiling((float)i / toBeDownloaded.Length * 100);
i++;
if (percent % 10 == 0)
{
if (percent != lastPercent)
{
Console.Write($"{percent}%...");
}
lastPercent = percent;
}
var payload = await eventStore.LoadAsync(name);
var headers = payload.Headers.ToDictionary(kv => kv.Key, kv => new StringValues(kv.Value.ToArray()));
var body = payload.Body;
var message = GitHubEventMessage.Parse(headers, body);
messages.Add(message);
}
Console.WriteLine("done.");
}
Console.WriteLine($"Crawling {crawledRepo.FullName} since {since}...");
}
if (crawledRepo.LastReindex is null)
{
crawledRepo.LastReindex = DateTimeOffset.UtcNow;
}
crawledRepo.AreaOwners = await GetAreaOwnersAsync(crawledRepo.Org, crawledRepo.Name);
var currentLabels = await RequestLabelsAsync(factory, client, crawledRepo.Org, crawledRepo.Name);
SyncLabels(crawledRepo, currentLabels, out var labelById);
var currentMilestones = await RequestMilestonesAsync(factory, client, crawledRepo.Org, crawledRepo.Name);
SyncMilestones(crawledRepo, currentMilestones, out var milestoneById);
// NOTE: GitHub's Issues.GetAllForeRepository() doesn't include issues that were transferred
//
// That's the good part. The bad part is that for the new repository where
// it shows up, we have no way of knowing which repo it came from and which
// number it used to have (even when looking at the issues timeline data),
// so we can't remove the issue from the source repo.
//
// However, since we're persisting GitHub events we received, we'll can look
// up which issues were transferred and remove them from the repo. This avoids
// having to wait until we fully reindex the repo.
//
// Note, we remove transferred issues before pulling issues in case the issues
// were being transferred back; it seems GitHub is reusing the numbers in that
// case.
foreach (var message in messages.Where(m => m.Body.Action == "transferred"))
{
Console.WriteLine($"Removing {message.Body?.Repository?.FullName}#{message.Body?.Issue?.Number}: {message.Body?.Issue?.Title}");
var number = message.Body?.Issue?.Number;
if (number is not null)
{
crawledRepo.Issues.Remove(number.Value);
}
}
foreach (var issue in await RequestIssuesAsync(factory, client, crawledRepo.Org, crawledRepo.Name, since))
{
var crawledIssue = ConvertIssue(crawledRepo, issue, labelById, milestoneById);
crawledRepo.Issues[issue.Number] = crawledIssue;
}
foreach (var pullRequest in await RequestPullRequestsAsync(factory, client, crawledRepo.Org, crawledRepo.Name, since))
{
if (crawledRepo.Issues.TryGetValue(pullRequest.Number, out var issue))
{
UpdateIssue(issue, pullRequest);
}
// TODO: Get PR reviews
// TODO: Get PR commits
// TODO: Get PR status
}
await crawledRepo.SaveAsync(repoPath);
if (uploadToAzure)
{
Console.WriteLine($"Uploading {blobName} to Azure...");
var repoClient = new BlobClient(connectionString, cacheContainerName, blobName);
await repoClient.UploadAsync(repoPath, overwrite : true);
if (since is null)
{
var eventsToBeDeleted = events.Where(e => string.Equals($"{e.Org}/{e.Repo}", crawledRepo.FullName, StringComparison.OrdinalIgnoreCase))
.ToArray();
Console.WriteLine($"Deleting {eventsToBeDeleted.Length:N0} events for {crawledRepo.FullName}...");
foreach (var e in eventsToBeDeleted)
{
await eventStore.DeleteAsync(e);
}
}
}
}
}
foreach (var repo in repos)
{
var milestones = repo.Milestones.ToHashSet();
var labels = repo.Labels.ToHashSet();
foreach (var issue in repo.Issues.Values)
{
foreach (var label in issue.Labels.Where(l => !labels.Contains(l)))
{
Console.Error.WriteLine($"error: {repo.FullName}#{issue.Number}: label '{label.Name}' doesn't exist");
}
if (issue.Milestone is not null && !milestones.Contains(issue.Milestone))
{
Console.Error.WriteLine($"error: {repo.FullName}#{issue.Number}: milestone '{issue.Milestone.Title}' doesn't exist");
}
}
}
Console.WriteLine("Creating trie...");
var trie = new CrawledTrie <CrawledIssue>();
foreach (var repo in repos)
{
foreach (var issue in repo.Issues.Values)
{
trie.Add(issue);
}
}
Console.WriteLine("Creating index...");
var index = new CrawledIndex()
{
Repos = repos.ToList(),
Trie = trie
};
var indexName = "index.cicache";
var indexPath = string.IsNullOrEmpty(outputPath)
? Path.Join(tempDirectory, indexName)
: outputPath;
await index.SaveAsync(indexPath);
if (uploadToAzure)
{
Console.WriteLine("Uploading index to Azure...");
var indexClient = new BlobClient(connectionString, "index", indexName);
await indexClient.UploadAsync(indexPath, overwrite : true);
}
Console.WriteLine("Deleting temp files...");
Directory.Delete(tempDirectory, recursive: true);
}
static async Task Main(string[] args)
{
string sourceFolder = null;
string repoName = null;
string blobContainerSasUrl = null;
var options = new OptionSet
{
{ "i=", "The source folder", i => sourceFolder = i },
{ "n=", "The repo name", n => repoName = n },
{ "o=", "The destination blob container url, can also be in the BLOB_CONTAINER_URL environment variable", o => blobContainerSasUrl = o },
};
List <string> extra = options.Parse(args);
if (extra.Any())
{
Fatal($"Unexpected argument {extra.First()}");
}
if (string.IsNullOrEmpty(sourceFolder))
{
Fatal("Missing argument -i");
}
if (string.IsNullOrEmpty(repoName))
{
Fatal("Missing argument -n");
}
if (string.IsNullOrEmpty(blobContainerSasUrl))
{
blobContainerSasUrl = Environment.GetEnvironmentVariable("BLOB_CONTAINER_URL");
}
if (string.IsNullOrEmpty(blobContainerSasUrl))
{
Fatal("Missing argument -o");
}
var containerClient = new BlobContainerClient(new Uri(blobContainerSasUrl));
string newBlobName = $"{repoName}/{DateTime.UtcNow:O}.tar.gz";
BlobClient newBlobClient = containerClient.GetBlobClient(newBlobName);
Console.WriteLine($"Uploading folder {sourceFolder} to blob {new UriBuilder(newBlobClient.Uri) {Fragment = "", Query = ""}.Uri.AbsoluteUri}");
await using (var outputFileStream = new MemoryStream())
{
await using (var gzoStream = new GZipOutputStream(outputFileStream)
{
IsStreamOwner = false
})
using (var tarArchive = TarArchive.CreateOutputTarArchive(gzoStream, Encoding.UTF8))
{
string sourceRoot = Path.GetFullPath(sourceFolder).Replace('\\', '/').TrimEnd('/');
void AddEntry(string path)
{
string normalizedPath = Path.GetFullPath(path).Replace('\\', '/');
var e = TarEntry.CreateEntryFromFile(path);
e.Name = normalizedPath.Substring(sourceRoot.Length).TrimStart('/');
Console.WriteLine($"Adding {path} as {e.Name}");
tarArchive.WriteEntry(e, false);
}
void AddFolder(string path)
{
AddEntry(path);
foreach (string file in Directory.GetFiles(path))
{
AddEntry(file);
}
foreach (string dir in Directory.GetDirectories(path))
{
AddFolder(dir);
}
}
AddFolder(sourceRoot);
}
outputFileStream.Position = 0;
await newBlobClient.UploadAsync(outputFileStream);
}
Console.WriteLine("Cleaning up old blobs");
List <BlobItem> blobs = containerClient.GetBlobs(prefix: repoName + "/").ToList();
List <BlobItem> toDelete = blobs.OrderByDescending(b => b.Name).Skip(10).ToList();
foreach (BlobItem d in toDelete)
{
Console.WriteLine($"Deleting blob {d.Name}");
await containerClient.DeleteBlobAsync(d.Name);
}
Console.WriteLine("Finished.");
}
public async Task <Response> DeleteBlobAsync(string blobName)
{
return(await _client.DeleteBlobAsync(blobName));
}
public async Task <DeletionResult> DeletePhotoAsync(string publicId)
{
await _containerClient.DeleteBlobAsync(publicId);
return(new DeletionResult());
}
public async Task DeleteAsync(string file)
{
await client.DeleteBlobAsync(file);
}
/// <inheritdoc/>
public Task <OperationResult> DeleteAsync(string name)
{
return(SafeExecuteAsync(() => container.DeleteBlobAsync(name), name));
}
public async Task DeleteBlobAsync(string blobName)
{
await _blobContainerClient.DeleteBlobAsync(blobName, DeleteSnapshotsOption.IncludeSnapshots);
}