public void WriteObject(CollectTask task, object data) { checkConnection(); IObjectWriter writer = writerFactory.CreateWriter(task); writer.WriteData(task, data); }
public IObjectWriter CreateWriter(CollectTask task) { if (task.SocialNetwork == SocialNetwork.VKontakte) { switch (task.Method) { case "groups.getById": return new VkGroupsDbObjectWriter(this.sqlConn); case "wall.getComments": return new VkCommentsDbObjectWriter(this.sqlConn); case "friends.get": return new VkFriendsDbObjectWriter(this.sqlConn); case "likes.getList": return new VkLikesDbObjectWriter(this.sqlConn); case "users.get": return new VkUsersDbObjectWriter(this.sqlConn); case "wall.get": case "wall.getReposts": return new VkPostsDbObjectWriter(this.sqlConn); case "groups.getMembers": case "groups.get": case "users.getSubscriptions": return new VkUserGroupsDbObjectWriter(this.sqlConn); default: throw new NotSupportedException(task.ToString()); } } throw new NotSupportedException(task.ToString()); }
protected void collectData(CollectTask collectTask) { var result = apiHelper.GetResult(collectTask); if (result != null) { var messageToSend = new CollectTaskResult(); messageToSend.Result = result; messageToSend.Task = collectTask; sendResult(messageToSend); } }
public void WriteData(CollectTask task, object data) { setFieldsFromTask(task); var items = convertToObjectsList(data); if (!items.Any()) return; // remove duplicates var uniqItems = items .GroupBy(o => getItemKey(o)) .Select(g => g.First()); // Group items for batch insert var groupedItems = uniqItems .Select((item, index) => new { Index = index, Item = item }) .GroupBy(di => di.Index / batchSize , e => e.Item, (key, elem) => elem) //.Select(group => group.Select(gi => gi.Item)) ; var dataTable = createDataTable(); // First, try to insert group in batch mode foreach (var group in groupedItems) { try { fillTableAndDoBulkCopy(group, dataTable); } catch (SqlException) // Exception - insert row-by-row { Trace.TraceWarning("Bulk exception: insert needed items"); // Let's insert only needed items var itemsToInsert = excludeExistingItems(group); var dupItems = itemsToInsert.GroupBy(o => o.ToString()).Where(g => g.Count() > 1).Select(g => g.Key).ToList(); var dupGroup = group.GroupBy(o => o.ToString()).Where(g => g.Count() > 1).Select(g => g.Key).ToList(); if (itemsToInsert.Any()) { fillTableAndDoBulkCopy(itemsToInsert, dataTable); } else { Trace.TraceWarning("Bulk exception: no needed items"); } } } }
protected static CloudQueueMessage createMessage(SocialNetwork network, string method, string param) { var task = new CollectTask(); task.SocialNetwork = network; task.Method = method; task.Params = param; var bFrmt = new BinaryFormatter(); var outputStream = new MemoryStream(); bFrmt.Serialize(outputStream, task); var result = outputStream.ToArray(); var message = new CloudQueueMessage(result); return message; }
protected override void setFieldsFromTask(CollectTask task) { }
protected abstract void setFieldsFromTask(CollectTask task);
protected override void setFieldsFromTask(CollectTask task) { this.paramOne = long.Parse(task.Params); if (task.Method == "groups.getMembers") // in list - user ids { this.isParamOneUserId = false; } if (task.Method == "groups.get" || task.Method == "users.getSubscriptions") // in list - group ids { this.isParamOneUserId = true; } }
protected override void setFieldsFromTask(CollectTask task) { this.userId = long.Parse(task.Params); }
protected override void setFieldsFromTask(CollectTask task) { var ids = task.Params.Split('_'); this.ownerId = long.Parse(ids[0]); this.postId = long.Parse(ids[1]); }