protected void btn_Upload_Click(object sender, EventArgs e)
{
try
{
if (CheckAllRequiredShapeFilesUploaded())
{
BlobAccess blobAccess = new BlobAccess();
var container = blobAccess.UploadShapeFiles(ShapeFileIDUpload.PostedFile.InputStream, ShapeFileIDUpload.FileName);
container = blobAccess.UploadShapeFiles(FileUploadDescribe.PostedFile.InputStream, FileUploadDescribe.FileName);
container = blobAccess.UploadShapeFiles(FileUploadProjection.PostedFile.InputStream, FileUploadProjection.FileName);
container = blobAccess.UploadShapeFiles(FileUploadShapeIndex.PostedFile.InputStream, FileUploadShapeIndex.FileName);
//saving the file
((TextBox)lvwgroup.InsertItem.FindControl("ShapeFileIDTextBox")).Text = Path.GetFileNameWithoutExtension(ShapeFileIDUpload.FileName);
var k = ((TextBox)lvwgroup.InsertItem.FindControl("GroupNameTextBox")).Text;
BindGroupListParentCombobox(lvwgroup.InsertItem, GetGroups());
lblSuccess.Text = String.Empty;
}
else
{
lblSuccess.Text = "Please upload Shape Files Correctly.";
}
}
catch (Exception ex)
{
throw ex;
}
}
static List <Tuple <int, string, string> > GetGroupIdWithKeysAndShapeIds()
{
GroupService grpService = new GroupService();
BlobAccess blobAccess = new BlobAccess();
var container = blobAccess.LoadShapeFiles();
LocalResource myStorage = RoleEnvironment.GetLocalResource("LocalStorageWorkerRole");
List <Tuple <int, string, string> > GrpIdGroupKeyAndShapePaths = new List <Tuple <int, string, string> >();
GroupService.ParentGroup.ForEach(ParentGroup =>
{
if (!String.IsNullOrWhiteSpace(ParentGroup.ShapeFileID) && ParentGroup.NotifySubgroups) //
{
string shapeIndex = LocalPath(myStorage, ParentGroup.ShapeFileID + ".shx");
string describeFile = LocalPath(myStorage, ParentGroup.ShapeFileID + ".dbf");
string shapeFile = LocalPath(myStorage, ParentGroup.ShapeFileID + ".shp");
string projectionFile = LocalPath(myStorage, ParentGroup.ShapeFileID + ".prj");
if (File.Exists(shapeIndex))
{
File.Delete(shapeIndex);
}
if (File.Exists(describeFile))
{
File.Delete(describeFile);
}
if (File.Exists(shapeFile))
{
File.Delete(shapeFile);
}
if (File.Exists(projectionFile))
{
File.Delete(projectionFile);
}
var ShxBlockBlobReference = container.GetBlockBlobReference(GetPath(FileType.IndexFile, ParentGroup.ShapeFileID));
ShxBlockBlobReference.DownloadToFile(shapeIndex, FileMode.Create);
var DescribeBlockBlobReference = container.GetBlockBlobReference(GetPath(FileType.DescribeFile, ParentGroup.ShapeFileID));
DescribeBlockBlobReference.DownloadToFile(describeFile, FileMode.Create);
var ShapeBlockBlobReference = container.GetBlockBlobReference(GetPath(FileType.ShapeFile, ParentGroup.ShapeFileID));
ShapeBlockBlobReference.DownloadToFile(shapeFile, FileMode.Create);
var BlockBlobReference = container.GetBlockBlobReference(GetPath(FileType.ProjectionFile, ParentGroup.ShapeFileID));
BlockBlobReference.DownloadToFile(projectionFile, FileMode.Create);
GrpIdGroupKeyAndShapePaths.Add(new Tuple <int, string, string>(ParentGroup.GroupID, ParentGroup.SubGroupIdentificationKey, Path.Combine(myStorage.RootPath, ParentGroup.ShapeFileID)));
}
});
return(GrpIdGroupKeyAndShapePaths);
}
private static void TestBlobManipulation()
{
blob = new BlobAccess(connection);
byte[] byteArray = Encoding.ASCII.GetBytes("Data for the file");
var stream = new MemoryStream(byteArray);
var metadata = new NameValueCollection();
metadata.Add("Created", DateTime.UtcNow.ToString());
metadata.Add("Type", "Any old type");
const string myFile = "private/brett/testfile.txt";
blob.Create(myFile, stream, "text/plain", metadata);
BlobAttributes attr = blob.GetAttributes(myFile);
}
private static void TestLeasing()
{
blob = new BlobAccess(connection);
//string id1 = Acquire();
//string id2 = Acquire();
//Console.WriteLine("Released: " + blob.LeaseRelease(file, id1));
//id2 = Acquire();
//Console.WriteLine("Released: " + blob.LeaseRelease(file, id2));
//id1 = Acquire();
//Console.WriteLine("Released: " + blob.LeaseRelease(file, id1));
var store = CloudStorageAccount.Parse(connection);
var client = new CloudBlobClient(store.BlobEndpoint, store.Credentials);
CloudBlobContainer container = client.GetContainerReference("test");
CloudBlob cb = container.GetBlobReference("folder/Daily.txt");
using (var lease = new AutoRenewLease(cb))
{
//lease.RenewIntervalSec = 5;
if (lease.HasLease)
{
Console.WriteLine("I have the leaseId: " + lease.LeaseId + " at: " + DateTime.Now.ToLongTimeString());
//string id = Acquire();
//cb.ReleaseLease(lease.LeaseId);
//id = Acquire();
//cb.ReleaseLease(id);
Thread.Sleep(62 * 1000);
Console.WriteLine("I have the leaseId: " + lease.LeaseId + " at: " + DateTime.Now.ToLongTimeString());
Thread.Sleep(30 * 1000);
Console.WriteLine("I have the leaseId: " + lease.LeaseId + " at: " + DateTime.Now.ToLongTimeString());
Thread.Sleep(10 * 1000);
Console.WriteLine("I have the leaseId: " + lease.LeaseId + " at: " + DateTime.Now.ToLongTimeString());
Console.WriteLine("Letting go of lease at: " + DateTime.Now.ToLongTimeString());
}
else
{
Console.WriteLine("Could not get lease at: " + DateTime.Now.ToLongTimeString());
}
}
Console.ReadKey();
}
public async Task OnPostSavePicture()
{
if (Upload != null)
{
List <Links> linksSperated = new List <Links>();
if (Name != null)
{
if (links == "")
{
string[] SplitLinks = links.Split(',');
foreach (string element in SplitLinks)
{
string[] temp = element.Split(' ');
linksSperated.Add(new Links {
link = temp[1], name = temp[0]
});
}
}
try
{
await SingleTon.GetSQLConnector().AddNewItemToContainerAsync(new Entities.Project()
{
Name = this.Name, SmallDescriptiom = this.smallDescription, Description = this.Description, ImagePath = await BlobAccess.UploadImage(Upload.OpenReadStream(), Upload.FileName), TrelloLink = this.TrelloLink, GithubLink = this.Github, OtherLinks = linksSperated.ToArray(), Id = Name + Guid.NewGuid().ToString()
});
}
catch (Exception ex)
{ Console.WriteLine(ex.Message); }
}
//testTextURI = await BlobAccess.UploadImage(Upload.OpenReadStream(), Upload.FileName);
}
}
private static async Task <KeyValuePair <string, BlobTransferStatistics> > CopyContainerAsync(this CloudBlobContainer sourceContainer, CloudBlobClient targetClient, BlobCopyOptions copyOptions, Func <bool> stopCalled)
{
var targetContainerName = sourceContainer.Name;
if (stopCalled())
{
return(sourceContainer.Name.PairWithValue(BlobTransferStatistics.Default.Concat(new[] { $"copy stopped on {targetContainerName}" })));
}
return(await await sourceContainer.CreateIfNotExistTargetContainerForCopyAsync(targetClient, targetContainerName,
async (targetContainer, findExistingAsync, renewAccessAsync, releaseAccessAsync) =>
{
EastFiveAzureStorageBackupService.Log.Info($"starting {targetContainerName}");
try
{
var existingTargetBlobs = await findExistingAsync(stopCalled);
EastFiveAzureStorageBackupService.Log.Info($"{existingTargetBlobs.Value.Count} blobs already backed up for {targetContainerName}");
var pair = default(BlobContinuationToken).PairWithValue(BlobTransferStatistics.Default.Concat(existingTargetBlobs.Key));
BlobAccess access = default(BlobAccess);
Func <Task <BlobAccess> > renewWhenExpiredAsync =
async() =>
{
if (access.expiresUtc < DateTime.UtcNow + TimeSpan.FromMinutes(5))
{
access = await renewAccessAsync(copyOptions.accessPeriod);
await Task.Delay(TimeSpan.FromSeconds(10)); // let settle in so first copy will be ok
}
return access;
};
while (true)
{
if (stopCalled())
{
return sourceContainer.Name.PairWithValue(pair.Value.Concat(new[] { $"copy stopped on {sourceContainer.Name}" }));
}
pair = await await sourceContainer.FindNextBlobSegmentAsync <Task <KeyValuePair <BlobContinuationToken, BlobTransferStatistics> > >(pair.Key,
async(token, blobs) =>
{
var stats = await blobs.CopyBlobsWithContainerKeyAsync(targetContainer, existingTargetBlobs.Value,
() => pair.Value.calc.GetNextInterval(copyOptions.minIntervalCheckCopy, copyOptions.maxIntervalCheckCopy),
copyOptions.maxTotalWaitForCopy, renewWhenExpiredAsync, copyOptions.maxBlobConcurrencyPerContainer, stopCalled);
blobs = null;
return token.PairWithValue(pair.Value.Concat(stats));
},
why => default(BlobContinuationToken).PairWithValue(pair.Value.Concat(new[] { why })).ToTask());
pair.Value.LogProgress(copyOptions.minIntervalCheckCopy, copyOptions.maxIntervalCheckCopy,
msg => EastFiveAzureStorageBackupService.Log.Info($"(progress) {targetContainerName} -> {msg}"));
if (default(BlobContinuationToken) == pair.Key)
{
if (pair.Value.retries.Any())
{
EastFiveAzureStorageBackupService.Log.Info($"retrying {targetContainerName}");
var copyRetries = copyOptions.copyRetries;
existingTargetBlobs = await findExistingAsync(stopCalled);
EastFiveAzureStorageBackupService.Log.Info($"{existingTargetBlobs.Value.Count} blobs already backed up for {targetContainerName}");
pair = default(BlobContinuationToken).PairWithValue(pair.Value.Concat(existingTargetBlobs.Key)); // just copies errors
var checkCopyCompleteAfter = pair.Value.calc.GetNextInterval(copyOptions.minIntervalCheckCopy, copyOptions.maxIntervalCheckCopy);
while (copyRetries-- > 0)
{
if (stopCalled())
{
pair = default(BlobContinuationToken).PairWithValue(pair.Value.Concat(new[] { $"copy stopped on {sourceContainer.Name}" }));
break;
}
var stats = await pair.Value.retries
.Select(x => x.Key)
.ToArray()
.CopyBlobsWithContainerKeyAsync(targetContainer, existingTargetBlobs.Value,
() => pair.Value.calc.GetNextInterval(copyOptions.minIntervalCheckCopy, copyOptions.maxIntervalCheckCopy),
copyOptions.maxTotalWaitForCopy, renewWhenExpiredAsync, copyOptions.maxBlobConcurrencyPerContainer, stopCalled);
pair = default(BlobContinuationToken).PairWithValue(new BlobTransferStatistics
{
calc = pair.Value.calc.Concat(stats.calc),
errors = pair.Value.errors.Concat(stats.errors).ToArray(),
successes = pair.Value.successes + stats.successes,
retries = stats.retries,
failures = pair.Value.failures.Concat(stats.failures).ToArray()
});
if (!pair.Value.retries.Any())
{
break;
}
}
}
pair.Value.LogProgress(copyOptions.minIntervalCheckCopy, copyOptions.maxIntervalCheckCopy,
msg => EastFiveAzureStorageBackupService.Log.Info($"finished {targetContainerName} -> {msg}"),
BlobTransferStatistics.logFrequency);
return sourceContainer.Name.PairWithValue(pair.Value);
}
}
}
catch (Exception e)
{
return sourceContainer.Name.PairWithValue(BlobTransferStatistics.Default.Concat(new [] { $"Exception copying container, Detail: {e.Message}" }));
}
finally
{
await releaseAccessAsync();
EastFiveAzureStorageBackupService.Log.Info($"done with {targetContainerName}");
}
},
why => sourceContainer.Name.PairWithValue(BlobTransferStatistics.Default.Concat(new[] { why })).ToTask()));
}
private static async Task <TResult> CreateIfNotExistTargetContainerForCopyAsync <TResult>(this CloudBlobContainer sourceContainer,
CloudBlobClient targetClient, string targetContainerName,
Func <CloudBlobContainer, Func <Func <bool>, Task <KeyValuePair <string[], IDictionary <string, SparseCloudBlob> > > >, Func <TimeSpan, Task <BlobAccess> >, Func <Task>, TResult> onSuccess, Func <string, TResult> onFailure)
{
try
{
var targetContainer = targetClient.GetContainerReference(targetContainerName);
var context = new OperationContext();
var exists = await targetContainer.ExistsAsync(BlobCopyOptions.requestOptions, context);
if (!exists)
{
var createPermissions = await sourceContainer.GetPermissionsAsync(EmptyCondition, BlobCopyOptions.requestOptions, context);
await targetContainer.CreateAsync(createPermissions.PublicAccess, BlobCopyOptions.requestOptions, context);
var metadataModified = false;
foreach (var item in sourceContainer.Metadata)
{
if (!targetContainer.Metadata.ContainsKey(item.Key) || targetContainer.Metadata[item.Key] != item.Value)
{
targetContainer.Metadata[item.Key] = item.Value;
metadataModified = true;
}
}
if (metadataModified)
{
await targetContainer.SetMetadataAsync(EmptyCondition, BlobCopyOptions.requestOptions, context);
}
}
var keyName = $"{sourceContainer.ServiceClient.Credentials.AccountName}-{targetContainerName}-access";
return(onSuccess(targetContainer,
(stopCalled) =>
{
return targetContainer.FindAllBlobsAsync(stopCalled,
blobs => new string[] { }.PairWithValue(blobs),
(why, partialBlobList) => new[] { why }.PairWithValue(partialBlobList));
},
async(sourceAccessWindow) =>
{
try
{
var renewContext = new OperationContext();
var permissions = await sourceContainer.GetPermissionsAsync(EmptyCondition, BlobCopyOptions.requestOptions, renewContext);
permissions.SharedAccessPolicies.Clear();
var access = new BlobAccess
{
key = keyName,
expiresUtc = DateTime.UtcNow.Add(sourceAccessWindow)
};
permissions.SharedAccessPolicies.Add(access.key, new SharedAccessBlobPolicy
{
SharedAccessExpiryTime = access.expiresUtc,
Permissions = SharedAccessBlobPermissions.Read
});
await sourceContainer.SetPermissionsAsync(permissions, EmptyCondition, BlobCopyOptions.requestOptions, renewContext);
return access;
}
catch (Exception e)
{
return new BlobAccess {
error = $"Error renewing access policy on container, Detail: {e.Message}"
};
}
},
async() =>
{
try
{
var releaseContext = new OperationContext();
var permissions = await sourceContainer.GetPermissionsAsync(EmptyCondition, BlobCopyOptions.requestOptions, releaseContext);
permissions.SharedAccessPolicies.Clear();
await sourceContainer.SetPermissionsAsync(permissions, EmptyCondition, BlobCopyOptions.requestOptions, releaseContext);
}
catch (Exception e)
{
// Failure here is no big deal as the container will still be usable
return;
}
}));
}
catch (Exception e)
{
return(onFailure($"Exception preparing container for copy, Detail: {e.Message}"));
}
}
