Esempio n. 1
0
        public static async Task <bool> DeleteBlobs(BlobDirectory dirInfo)
        {
            var continuationToken = dirInfo.NewBlobContinuationToken();

            do
            {
                var response = await dirInfo.ListBlobsSegmentedAsync(continuationToken);

                continuationToken = response.ContinuationToken;
                var results = response.Results;
                foreach (var item in results)
                {
                    var subdir = item.ToBlobDirectory();
                    if (!Object.ReferenceEquals(subdir, null))
                    {
                        await DeleteBlobs(subdir);
                    }
                    else
                    {
                        var file = item.ToBlockBlob();
                        await file.DeleteAsync();
                    }
                }
            }while (continuationToken != null);
            return(true);
        }
        private static async Task SyncOneEntryObject(
            Tuple <JObject, List <int>, int, string> tuple,
            BlobDirectory dir, String entry, List <int> statuses)
        {
            if (!Object.ReferenceEquals(tuple.Item2, null) && tuple.Item2.Count > 0)
            {
                var dirProviders = dir.GetAllProviders();
                if (Object.ReferenceEquals(statuses, null))
                {
                    statuses = dirProviders.ConvertAll(x => 0);
                }

                var content = tuple.Item1.ToString();
                foreach (var i in tuple.Item2)
                {
                    try
                    {
                        if (statuses[i] >= 0)
                        {
                            var item = dirProviders[i].GetBlockBlobReference(entry);
                            await item.UploadTextAsync(content);

                            Console.WriteLine($"Sync {dir.Name}/{entry} for {dirProviders[i].Provider.Name}");
                            statuses[i] = 1;
                        }
                    }
                    catch
                    {
                        var provider = dirProviders[i].Provider.Name;
                        Console.WriteLine($"Sync {entry} failed, the provider {provider} is not operational");
                        statuses[i] = -1;
                    }
                }
            }
        }
Esempio n. 3
0
        public static async Task <JObject> FetchBlobs(BlobDirectory dirInfo, JToken metadata, String current, int maxItem = int.MaxValue)
        {
            var entryList = await Blob.ListBlobsAsync(dirInfo);

            var blockList = new List <BlockBlob>();

            // How many items are BlockBlob?
            foreach (var item in entryList)
            {
                if (item.GetType() == typeof(CloudBlockBlob))
                {
                    var blob = item.ToBlockBlob();
                    await blob.FetchAttributesAsync();

                    blockList.Add(blob);
                }
            }
            if (blockList.Count > maxItem && maxItem > 0)
            {
                // Need to filter.
                var dicSorted = new SortedDictionary <Int64, BlockBlob>();
                foreach (var item in blockList)
                {
                    var ticks = item.Properties.LastModified.Value.Ticks;
                    if (String.Compare(item.Name, current, true) == 0)
                    {
                        ticks = DateTime.MaxValue.Ticks;
                    }
                    if (Json.ContainsKey(item.Name, metadata))
                    {
                        ticks += TimeSpan.TicksPerDay * 10000;
                    }
                    dicSorted[ticks] = item;
                }
                blockList.Clear();
                foreach (var pair in dicSorted.Reverse())
                {
                    blockList.Add(pair.Value);
                    if (blockList.Count >= maxItem)
                    {
                        break;
                    }
                }
            }
            var retDic = new JObject();

            foreach (var item in blockList)
            {
                var strValue = await item.DownloadTextAsync();

                var fullpathname = item.Name;
                var pathes       = fullpathname.Split(new Char[] { '/' });
                var lastName     = pathes[pathes.Length - 1];
                retDic[lastName] = strValue;
            }
            return(retDic);
        }
Esempio n. 4
0
        public static async Task <List <ICustomizedBlobItem> > ListBlobsAsync(BlobDirectory dirInfo)
        {
            var continuationToken = dirInfo.NewBlobContinuationToken();
            List <ICustomizedBlobItem> results = new List <ICustomizedBlobItem>();

            do
            {
                var response = await dirInfo.ListBlobsSegmentedAsync(continuationToken);

                continuationToken = response.ContinuationToken;
                results.AddRange(response.Results);
            }while (continuationToken != null);
            return(results);
        }
        public virtual String GetBaseName(BlobDirectory dir)
        {
            var curName = Name;
            var dirName = dir.Name;
            var idx     = 0;

            for (int start_idx = 0; start_idx < dirName.Length; start_idx++)
            {
                var full_idx = curName.IndexOf(dirName.Substring(start_idx), StringComparison.OrdinalIgnoreCase);
                if (full_idx >= 0)
                {
                    idx = full_idx + dirName.Length - start_idx;
                    break;
                }
            }
            if (idx < curName.Length && curName[idx] == '/')
            {
                idx++;
            }
            return(curName.Substring(idx));
        }
        /// <summary>
        /// Downloads and sync one entry (base on timestamp across multiple cloud provider).
        /// </summary>
        /// <returns>A Tuple, which contains: 1) one entry, 2) index of the current entry, 3) eTag </returns>
        /// <param name="dir">Dir.</param>
        /// <param name="entry">Entry.</param>
        /// <param name="statuses">Statuses.</param>
        /// <param name="timeTag">Time tag.</param>
        public static async Task <Tuple <JObject, int, string> > DownloadAndSyncOneEntry(BlobDirectory dir, String entry, List <int> statuses, String timeTag = Constant.JSonKeyTime)
        {
            var tuple = await GetConsistentJObject(dir, entry, timeTag);

            await SyncOneEntryObject(tuple, dir, entry, statuses);

            return(new Tuple <JObject, int, string>(tuple.Item1, tuple.Item3, tuple.Item4));
        }
        /// <summary>
        /// Gets the consistent JObject, this is used for JObject that are stored at multiple cloud provider without the hash (for consistency).
        /// A certain  key, usually time, is used to identity the freshness of the JObject, the object with the latest time is considered the latest.
        /// </summary>
        /// <returns>The consistent JObject, a list of blob that needs to be updated, current blob, and eTag (if exists) </returns>
        /// <param name="dir">directory.</param>
        /// <param name="tag">path .</param>
        public static async Task <Tuple <JObject, List <int>, int, string> > GetConsistentJObject(BlobDirectory dir, String entry, String timeTag = Constant.JSonKeyTime)
        {
            var allDirs  = dir.GetAllProviders();
            var allBlobs = allDirs.ConvertAll(x => x.GetBlockBlobReference(entry));
            var allTasks = allBlobs.ConvertAll(x => x.DownloadTextAsync());

            try
            {
                await Task.WhenAll(allTasks);
            } catch {};

            List <Tuple <String, int> > curStrs = new List <Tuple <string, int> >();

            for (int i = 0; i < allBlobs.Count; i++)
            {
                var    task    = allTasks[i];
                var    curBlob = allBlobs[i];
                String str     = null;
                if (task.Status == TaskStatus.RanToCompletion)
                {
                    str = task.Result;
                }
                curStrs.Add(new Tuple <String, int>(str, i));
            }
            ;
            // Nothing
            if (curStrs.Count <= 0)
            {
                return(new Tuple <JObject, List <int>, int, string>(new JObject(), null, -1, null));
            }
            var curObjs  = curStrs.ConvertAll(x => JsonUtils.TryParse(x.Item1));
            var modTicks = curObjs.ConvertAll(x => JsonUtils.GetType <Int64>(timeTag, x, 0L));
            // Console.WriteLine($"Consistency object {entry} ticks: {String.Join( ',', modTicks.ConvertAll(x => x.ToString()) )}");
            var bestObj = LatestModified(modTicks);

            if (bestObj < 0)
            {
                bestObj = 0;
            }
            var retObj = curObjs[bestObj];
            var retIdx = curStrs[bestObj].Item2;
            var eTag   = allBlobs[retIdx].ETag;
            var lst    = new List <int>();

            for (int i = 0; i < modTicks.Count; i++)
            {
                if (modTicks[i] < modTicks[bestObj])
                {
                    lst.Add(curStrs[i].Item2);
                }
            }

            if (lst.Count <= 0)
            {
                lst = null;
            }

            return(new Tuple <JObject, List <int>, int, string>(retObj, lst, retIdx, eTag));
        }