public String Add([FromBody] CorpusContentDto obj)
{
var content = new CorpusContent();
try
{
content = new CorpusContent()
{
Id = -1,
CorpusId = obj.CorpusId,
Name = obj.Name,
Content = Encoding.ASCII.GetBytes(obj.Content),
Type = obj.Type,
Hash = ScraperUtilities._hashContent(Encoding.ASCII.GetBytes(obj.Content)),
DownloadDate = DateTime.Now,
ScraperType = "Manual Insert",
};
m_context.CorpusContentRepository.Add(content);
}
catch (Exception e)
{
return(e.ToString());
}
return("All Good");
}
/// <summary>
/// creates a corpus content from a tweet
/// </summary>
/// <param name="Name"></param>
/// <param name="Type"></param>
/// <param name="ScraperGuid"></param>
/// <param name="ScraperType"></param>
/// <param name="tweet"></param>
/// <param name="m_context"></param>
public static void addCorpusContent(string Name, string Type,
Guid ScraperGuid, string ScraperType, ITweet tweet, ICorpusContext m_context, int corpusId)
{
CorpusContent corpContent = new CorpusContent();
corpContent.CorpusId = corpusId;
corpContent.Name = Name;
corpContent.Type = Type;
corpContent.ScraperGuid = ScraperGuid;
corpContent.ScraperType = ScraperType;
corpContent.Content = Encoding.ASCII.GetBytes(tweet.Text);
corpContent.DownloadDate = tweet.CreatedAt;
corpContent.URL = tweet.Url;
if (tweet.Coordinates != null) //may be null if tweet does not have a location
{
corpContent.Lat = (float)tweet.Coordinates.Latitude;
corpContent.Long = (float)tweet.Coordinates.Longitude;
}
corpContent.TweetID = tweet.Id;
corpContent.AuthorName = tweet.CreatedBy.Name;
//corpContent.Hashtags = tweet.Hashtags;
corpContent.Language = tweet.Language.GetType().FullName;
// corpContent.Source = source;
corpContent.Hash = hashContent(Encoding.ASCII.GetBytes(tweet.Text));
m_context.CorpusContentRepository.Add(corpContent);
}
public CorpusContent GetById(long id)
{
CorpusContent content = null;
using (var conn = this.Connection())
{
/* TODO: Support two different calls, one for fetching
* flyweight objects and one for heavyweight objects */
IEnumerable <CorpusContent> result =
conn.Query <CorpusContent>(@"
SELECT Id, Hash, Name, Type,
ScraperType,
DownloadDate,
DownloadURL
FROM la.CorpusContent
WHERE Id=@Id
", new { Id = id });
if (result.Any())
{
Debug.Assert(result.Count() == 1);
content = result.First();
}
}
return(content);
}
public void Delete(CorpusContent content)
{
Debug.Assert(content.Id != -1);
using (var conn = this.Connection())
{
conn.Execute(@"
DELETE FROM la.CorpusContent
WHERE Id=@Id
", new { Id = content.Id });
/* TODO: Schedule garbage collection in the Merkle tree in case
* ContentBlob associated with this corpus content has been
* orphaned */
}
}
/// <summary>
/// Creates corpus content for a project Gutenberg text file
/// </summary>
/// <param name="Name"></param>
/// <param name="Type"></param>
/// <param name="ScraperGuid"></param>
/// <param name="ScraperType"></param>
/// <param name="DownloadDate"></param>
/// <param name="DownloadURL"></param>
/// <param name="Content"></param>
/// <param name="m_context"></param>
public static void addCorpusContent(string Name, string Type,
Guid ScraperGuid, string ScraperType, DateTime DownloadDate, string DownloadURL,
byte[] Content, ICorpusContext m_context, int corpusId)
{
CorpusContent corpContent = new CorpusContent();
corpContent.CorpusId = corpusId;
corpContent.Name = Name;
corpContent.Type = Type;
corpContent.ScraperGuid = ScraperGuid;
corpContent.ScraperType = ScraperType;
corpContent.DownloadDate = DownloadDate;
corpContent.URL = DownloadURL;
corpContent.Content = Content;
corpContent.Hash = hashContent(Content);
m_context.CorpusContentRepository.Add(corpContent);
}
/* Public methods */
public void Add(CorpusContent content)
{
Debug.Assert(content.Id == -1);
using (var conn = this.Connection())
{
using (IDbTransaction tran = conn.BeginTransaction())
{
try
{
conn.Execute(@"
INSERT INTO la.CorpusContent
(CorpusId, Hash, Name, Type, ScraperGuid, ScraperType, DownloadDate,
DownloadURL, Long, Lat )
VALUES ( @CorpusId, @Hash, @Name, @Type, @ScraperGuid, @ScraperType, @DownloadDate,
@DownloadURL, @Long, @Lat)
", new
{
CorpusId = content.CorpusId,
Hash = content.Hash,
Name = content.Name,
Type = content.Type,
ScraperGuid = content.ScraperGuid,
ScraperType = content.ScraperType,
DownloadDate = content.DownloadDate,
DownloadUrl = content.URL,
Long = content.Long,
Lat = content.Lat
}, transaction: tran);
conn.Execute(@"IF NOT EXISTS
(SELECT 1 FROM la.MerkleNode
WHERE Hash = @Hash)BEGIN
INSERT INTO la.MerkleNode
(Hash, Type, Pinned)
VALUES (@Hash, @Type, @Pinned)
END ", new
{
Hash = content.Hash,
Type = "CorpusContent",
Pinned = 0,
}, transaction: tran
);
conn.Execute(@"IF NOT EXISTS
(SELECT 1 FROM la.ContentBlob
WHERE Hash = @Hash)BEGIN
INSERT INTO la.ContentBlob
(Hash, Contents)
VALUES (@Hash, @Contents)
END",
new
{
Hash = content.Hash,
Contents = System.Text.Encoding.UTF8.GetString(content.Content),
}, transaction: tran);
tran.Commit();
}
catch (SqlException e)
{
tran.Rollback();
for (int i = 0; i < e.Errors.Count; ++i)
{
Debug.WriteLine("SQL Error: " + e.Errors[i].ToString());
}
throw;
}
/* TODO: Check for flyweight CorpusContent objects */
/* TODO: Make sure the contents are somehow added to the Merkle
* tree as a ContentBlob */
/* TODO: If we also add a ContentBlob here, it would be nice to
* do everything as a single transaction */
}
}
}
public void Update(CorpusContent content)
{
Debug.Assert(content.Id != -1);
/* TODO */
}
