public void WriteObject(CollectTask task, object data)
{
checkConnection();
IObjectWriter writer = writerFactory.CreateWriter(task);
writer.WriteData(task, data);
}
public IObjectWriter CreateWriter(CollectTask task)
{
if (task.SocialNetwork == SocialNetwork.VKontakte)
{
switch (task.Method)
{
case "groups.getById":
return new VkGroupsDbObjectWriter(this.sqlConn);
case "wall.getComments":
return new VkCommentsDbObjectWriter(this.sqlConn);
case "friends.get":
return new VkFriendsDbObjectWriter(this.sqlConn);
case "likes.getList":
return new VkLikesDbObjectWriter(this.sqlConn);
case "users.get":
return new VkUsersDbObjectWriter(this.sqlConn);
case "wall.get":
case "wall.getReposts":
return new VkPostsDbObjectWriter(this.sqlConn);
case "groups.getMembers":
case "groups.get":
case "users.getSubscriptions":
return new VkUserGroupsDbObjectWriter(this.sqlConn);
default:
throw new NotSupportedException(task.ToString());
}
}
throw new NotSupportedException(task.ToString());
}
protected void collectData(CollectTask collectTask)
{
var result = apiHelper.GetResult(collectTask);
if (result != null)
{
var messageToSend = new CollectTaskResult();
messageToSend.Result = result;
messageToSend.Task = collectTask;
sendResult(messageToSend);
}
}
public void WriteData(CollectTask task, object data)
{
setFieldsFromTask(task);
var items = convertToObjectsList(data);
if (!items.Any())
return;
// remove duplicates
var uniqItems = items
.GroupBy(o => getItemKey(o))
.Select(g => g.First());
// Group items for batch insert
var groupedItems = uniqItems
.Select((item, index) => new { Index = index, Item = item })
.GroupBy(di => di.Index / batchSize , e => e.Item, (key, elem) => elem)
//.Select(group => group.Select(gi => gi.Item))
;
var dataTable = createDataTable();
// First, try to insert group in batch mode
foreach (var group in groupedItems)
{
try
{
fillTableAndDoBulkCopy(group, dataTable);
}
catch (SqlException) // Exception - insert row-by-row
{
Trace.TraceWarning("Bulk exception: insert needed items");
// Let's insert only needed items
var itemsToInsert = excludeExistingItems(group);
var dupItems = itemsToInsert.GroupBy(o => o.ToString()).Where(g => g.Count() > 1).Select(g => g.Key).ToList();
var dupGroup = group.GroupBy(o => o.ToString()).Where(g => g.Count() > 1).Select(g => g.Key).ToList();
if (itemsToInsert.Any())
{
fillTableAndDoBulkCopy(itemsToInsert, dataTable);
}
else
{
Trace.TraceWarning("Bulk exception: no needed items");
}
}
}
}
protected static CloudQueueMessage createMessage(SocialNetwork network, string method, string param)
{
var task = new CollectTask();
task.SocialNetwork = network;
task.Method = method;
task.Params = param;
var bFrmt = new BinaryFormatter();
var outputStream = new MemoryStream();
bFrmt.Serialize(outputStream, task);
var result = outputStream.ToArray();
var message = new CloudQueueMessage(result);
return message;
}
protected override void setFieldsFromTask(CollectTask task)
{
}
protected abstract void setFieldsFromTask(CollectTask task);
protected override void setFieldsFromTask(CollectTask task)
{
this.paramOne = long.Parse(task.Params);
if (task.Method == "groups.getMembers") // in list - user ids
{
this.isParamOneUserId = false;
}
if (task.Method == "groups.get" || task.Method == "users.getSubscriptions") // in list - group ids
{
this.isParamOneUserId = true;
}
}
protected override void setFieldsFromTask(CollectTask task)
{
this.userId = long.Parse(task.Params);
}
protected override void setFieldsFromTask(CollectTask task)
{
var ids = task.Params.Split('_');
this.ownerId = long.Parse(ids[0]);
this.postId = long.Parse(ids[1]);
}