예제 #1
0
        public void WriteObject(CollectTask task, object data)
        {
            checkConnection();

            IObjectWriter writer = writerFactory.CreateWriter(task);
            writer.WriteData(task, data);
        }
        public IObjectWriter CreateWriter(CollectTask task)
        {
            if (task.SocialNetwork == SocialNetwork.VKontakte)
            {
                switch (task.Method)
                {
                    case "groups.getById":
                        return new VkGroupsDbObjectWriter(this.sqlConn);
                    case "wall.getComments":
                        return new VkCommentsDbObjectWriter(this.sqlConn);
                    case "friends.get":
                        return new VkFriendsDbObjectWriter(this.sqlConn);
                    case "likes.getList":
                        return new VkLikesDbObjectWriter(this.sqlConn);
                    case "users.get":
                        return new VkUsersDbObjectWriter(this.sqlConn);
                    case "wall.get":
                    case "wall.getReposts":
                        return new VkPostsDbObjectWriter(this.sqlConn);
                    case "groups.getMembers":
                    case "groups.get":
                    case "users.getSubscriptions":
                        return new VkUserGroupsDbObjectWriter(this.sqlConn);
                    default:
                        throw new NotSupportedException(task.ToString());
                }
            }

            throw new NotSupportedException(task.ToString());
        }
        protected void collectData(CollectTask collectTask)
        {
            var result = apiHelper.GetResult(collectTask);

            if (result != null)
            {
                var messageToSend = new CollectTaskResult();
                messageToSend.Result = result;
                messageToSend.Task = collectTask;

                sendResult(messageToSend);
            }
        }
        public void WriteData(CollectTask task, object data)
        {
            setFieldsFromTask(task);

            var items = convertToObjectsList(data);

            if (!items.Any())
                return;

            // remove duplicates
            var uniqItems = items
                .GroupBy(o => getItemKey(o))
                .Select(g => g.First());

            // Group items for batch insert
            var groupedItems = uniqItems
                .Select((item, index) => new { Index = index, Item = item })
                .GroupBy(di => di.Index / batchSize , e => e.Item, (key, elem) => elem)
                //.Select(group => group.Select(gi => gi.Item))
                ;

            var dataTable = createDataTable();

            // First, try to insert group in batch mode
            foreach (var group in groupedItems)
            {
                try
                {
                    fillTableAndDoBulkCopy(group, dataTable);
                }
                catch (SqlException) // Exception - insert row-by-row
                {
                    Trace.TraceWarning("Bulk exception: insert needed items");

                    // Let's insert only needed items
                    var itemsToInsert = excludeExistingItems(group);
                    var dupItems = itemsToInsert.GroupBy(o => o.ToString()).Where(g => g.Count() > 1).Select(g => g.Key).ToList();
                    var dupGroup = group.GroupBy(o => o.ToString()).Where(g => g.Count() > 1).Select(g => g.Key).ToList();

                    if (itemsToInsert.Any())
                    {
                        fillTableAndDoBulkCopy(itemsToInsert, dataTable);
                    }
                    else
                    {
                        Trace.TraceWarning("Bulk exception: no needed items");
                    }
                }
            }
        }
예제 #5
0
        protected static CloudQueueMessage createMessage(SocialNetwork network, string method, string param)
        {
            var task = new CollectTask();
            task.SocialNetwork = network;
            task.Method = method;
            task.Params = param;

            var bFrmt = new BinaryFormatter();
            var outputStream = new MemoryStream();
            bFrmt.Serialize(outputStream, task);
            var result = outputStream.ToArray();

            var message = new CloudQueueMessage(result);
            return message;
        }
 protected override void setFieldsFromTask(CollectTask task)
 {
 }
 protected abstract void setFieldsFromTask(CollectTask task);
 protected override void setFieldsFromTask(CollectTask task)
 {
     this.paramOne = long.Parse(task.Params);
     if (task.Method == "groups.getMembers") // in list - user ids
     {
         this.isParamOneUserId = false;
     }
     if (task.Method == "groups.get" || task.Method == "users.getSubscriptions") // in list - group ids
     {
         this.isParamOneUserId = true;
     }
 }
 protected override void setFieldsFromTask(CollectTask task)
 {
     this.userId = long.Parse(task.Params);
 }
 protected override void setFieldsFromTask(CollectTask task)
 {
     var ids = task.Params.Split('_');
     this.ownerId = long.Parse(ids[0]);
     this.postId = long.Parse(ids[1]);
 }