public static async Task <bool> DeleteBlobs(BlobDirectory dirInfo)
{
var continuationToken = dirInfo.NewBlobContinuationToken();
do
{
var response = await dirInfo.ListBlobsSegmentedAsync(continuationToken);
continuationToken = response.ContinuationToken;
var results = response.Results;
foreach (var item in results)
{
var subdir = item.ToBlobDirectory();
if (!Object.ReferenceEquals(subdir, null))
{
await DeleteBlobs(subdir);
}
else
{
var file = item.ToBlockBlob();
await file.DeleteAsync();
}
}
}while (continuationToken != null);
return(true);
}
private static async Task SyncOneEntryObject(
Tuple <JObject, List <int>, int, string> tuple,
BlobDirectory dir, String entry, List <int> statuses)
{
if (!Object.ReferenceEquals(tuple.Item2, null) && tuple.Item2.Count > 0)
{
var dirProviders = dir.GetAllProviders();
if (Object.ReferenceEquals(statuses, null))
{
statuses = dirProviders.ConvertAll(x => 0);
}
var content = tuple.Item1.ToString();
foreach (var i in tuple.Item2)
{
try
{
if (statuses[i] >= 0)
{
var item = dirProviders[i].GetBlockBlobReference(entry);
await item.UploadTextAsync(content);
Console.WriteLine($"Sync {dir.Name}/{entry} for {dirProviders[i].Provider.Name}");
statuses[i] = 1;
}
}
catch
{
var provider = dirProviders[i].Provider.Name;
Console.WriteLine($"Sync {entry} failed, the provider {provider} is not operational");
statuses[i] = -1;
}
}
}
}
public static async Task <JObject> FetchBlobs(BlobDirectory dirInfo, JToken metadata, String current, int maxItem = int.MaxValue)
{
var entryList = await Blob.ListBlobsAsync(dirInfo);
var blockList = new List <BlockBlob>();
// How many items are BlockBlob?
foreach (var item in entryList)
{
if (item.GetType() == typeof(CloudBlockBlob))
{
var blob = item.ToBlockBlob();
await blob.FetchAttributesAsync();
blockList.Add(blob);
}
}
if (blockList.Count > maxItem && maxItem > 0)
{
// Need to filter.
var dicSorted = new SortedDictionary <Int64, BlockBlob>();
foreach (var item in blockList)
{
var ticks = item.Properties.LastModified.Value.Ticks;
if (String.Compare(item.Name, current, true) == 0)
{
ticks = DateTime.MaxValue.Ticks;
}
if (Json.ContainsKey(item.Name, metadata))
{
ticks += TimeSpan.TicksPerDay * 10000;
}
dicSorted[ticks] = item;
}
blockList.Clear();
foreach (var pair in dicSorted.Reverse())
{
blockList.Add(pair.Value);
if (blockList.Count >= maxItem)
{
break;
}
}
}
var retDic = new JObject();
foreach (var item in blockList)
{
var strValue = await item.DownloadTextAsync();
var fullpathname = item.Name;
var pathes = fullpathname.Split(new Char[] { '/' });
var lastName = pathes[pathes.Length - 1];
retDic[lastName] = strValue;
}
return(retDic);
}
public static async Task <List <ICustomizedBlobItem> > ListBlobsAsync(BlobDirectory dirInfo)
{
var continuationToken = dirInfo.NewBlobContinuationToken();
List <ICustomizedBlobItem> results = new List <ICustomizedBlobItem>();
do
{
var response = await dirInfo.ListBlobsSegmentedAsync(continuationToken);
continuationToken = response.ContinuationToken;
results.AddRange(response.Results);
}while (continuationToken != null);
return(results);
}
public virtual String GetBaseName(BlobDirectory dir)
{
var curName = Name;
var dirName = dir.Name;
var idx = 0;
for (int start_idx = 0; start_idx < dirName.Length; start_idx++)
{
var full_idx = curName.IndexOf(dirName.Substring(start_idx), StringComparison.OrdinalIgnoreCase);
if (full_idx >= 0)
{
idx = full_idx + dirName.Length - start_idx;
break;
}
}
if (idx < curName.Length && curName[idx] == '/')
{
idx++;
}
return(curName.Substring(idx));
}
/// <summary>
/// Downloads and sync one entry (base on timestamp across multiple cloud provider).
/// </summary>
/// <returns>A Tuple, which contains: 1) one entry, 2) index of the current entry, 3) eTag </returns>
/// <param name="dir">Dir.</param>
/// <param name="entry">Entry.</param>
/// <param name="statuses">Statuses.</param>
/// <param name="timeTag">Time tag.</param>
public static async Task <Tuple <JObject, int, string> > DownloadAndSyncOneEntry(BlobDirectory dir, String entry, List <int> statuses, String timeTag = Constant.JSonKeyTime)
{
var tuple = await GetConsistentJObject(dir, entry, timeTag);
await SyncOneEntryObject(tuple, dir, entry, statuses);
return(new Tuple <JObject, int, string>(tuple.Item1, tuple.Item3, tuple.Item4));
}
/// <summary>
/// Gets the consistent JObject, this is used for JObject that are stored at multiple cloud provider without the hash (for consistency).
/// A certain key, usually time, is used to identity the freshness of the JObject, the object with the latest time is considered the latest.
/// </summary>
/// <returns>The consistent JObject, a list of blob that needs to be updated, current blob, and eTag (if exists) </returns>
/// <param name="dir">directory.</param>
/// <param name="tag">path .</param>
public static async Task <Tuple <JObject, List <int>, int, string> > GetConsistentJObject(BlobDirectory dir, String entry, String timeTag = Constant.JSonKeyTime)
{
var allDirs = dir.GetAllProviders();
var allBlobs = allDirs.ConvertAll(x => x.GetBlockBlobReference(entry));
var allTasks = allBlobs.ConvertAll(x => x.DownloadTextAsync());
try
{
await Task.WhenAll(allTasks);
} catch {};
List <Tuple <String, int> > curStrs = new List <Tuple <string, int> >();
for (int i = 0; i < allBlobs.Count; i++)
{
var task = allTasks[i];
var curBlob = allBlobs[i];
String str = null;
if (task.Status == TaskStatus.RanToCompletion)
{
str = task.Result;
}
curStrs.Add(new Tuple <String, int>(str, i));
}
;
// Nothing
if (curStrs.Count <= 0)
{
return(new Tuple <JObject, List <int>, int, string>(new JObject(), null, -1, null));
}
var curObjs = curStrs.ConvertAll(x => JsonUtils.TryParse(x.Item1));
var modTicks = curObjs.ConvertAll(x => JsonUtils.GetType <Int64>(timeTag, x, 0L));
// Console.WriteLine($"Consistency object {entry} ticks: {String.Join( ',', modTicks.ConvertAll(x => x.ToString()) )}");
var bestObj = LatestModified(modTicks);
if (bestObj < 0)
{
bestObj = 0;
}
var retObj = curObjs[bestObj];
var retIdx = curStrs[bestObj].Item2;
var eTag = allBlobs[retIdx].ETag;
var lst = new List <int>();
for (int i = 0; i < modTicks.Count; i++)
{
if (modTicks[i] < modTicks[bestObj])
{
lst.Add(curStrs[i].Item2);
}
}
if (lst.Count <= 0)
{
lst = null;
}
return(new Tuple <JObject, List <int>, int, string>(retObj, lst, retIdx, eTag));
}
