public USGSItemActor() { Receive <ProcessUSGSItem>(e => { var item = XMLSerializeHelper.Deserialize <FeedEntry>(e.item.ToString()); var usgs = new FeedDataStructures.USGSItem(); var html = new HtmlDocument(); html.LoadHtml(item.Summary.Text); var orig = html.DocumentNode.SelectNodes("//dd").Select(z => z.InnerText).First(); var dt = DateTimeOffset.Parse(orig.Substring(0, orig.Length - 3)); usgs.id = item.Id.Substring(item.Id.LastIndexOf(":") + 1); usgs.partionKey = usgs.id + ":" + dt.ToString("yyyy-MM-dd"); usgs.published = dt; usgs.updated = DateTimeOffset.Parse(item.Updated); usgs.magnitude = processMag(item.Title); usgs.point = processPoint(item.Point); usgs.elevation = float.Parse(item.Elev) / 1000f; usgs.link = item.Link.Href; usgs.originalXML = e.ToString(); usgs.title = item.Title; usgs.source = e.path.Substring(0, e.path.IndexOf('.')); if (updateIfNew(usgs)) { Program.cdb.UpsertDocument(new CommonDataFormat() { id = Guid.NewGuid().ToString(), partionKey = "usgs", source = "usgs", title = usgs.title, extra = usgs.summary, point = usgs.point, pubDate = usgs.published, sourceId = usgs.id, sourcePk = usgs.partionKey }, "commondata").Wait(); if ((DateTimeOffset.UtcNow - usgs.updated).TotalSeconds > 10) { Sender.Tell(new processRecent(usgs)); } } Sender.Tell(new itemProcessed()); }); }
public processRecent(FeedDataStructures.USGSItem item) { Item = item; }
private bool updateIfNew(FeedDataStructures.USGSItem usgs) { FeedOptions queryOptions = new FeedOptions { MaxItemCount = -1, EnableCrossPartitionQuery = true }; // get updated to see if (1) in the DB and (2) if so, if this is newer than db version var qryUpd = Program.cdb.GetDocumentQuery <USGSItem>("usgs") .Where(z => z.id == usgs.id && z.partionKey == usgs.partionKey) .Select(z => z.updated); var vals = qryUpd.ToList(); if (Program.forceSave) { vals.Clear(); } // not there, create both main and archive if (!vals.Any()) { Console.WriteLine(" adding " + usgs.id); Program.cdb.UpsertDocument(usgs, "usgs").Wait(); var arch = new USGSItemArchive() { id = "archive:" + usgs.id, partionKey = usgs.partionKey, updates = new List <FeedDataStructures.USGSItem>() { usgs } }; Program.cdb.UpsertDocument(arch, "usgs").Wait(); return(true); } else if (vals.First() != usgs.updated) { Console.WriteLine(" updating " + usgs.id); // is in the DB, and the updated is different than the value in the DB var archiveQuery = Program.cdb.GetDocumentQuery <USGSItemArchive>("usgs") .Where(z => z.id == "archive:" + usgs.id && z.partionKey == usgs.partionKey); var archives = archiveQuery.ToList(); if (archives.Any()) { if (!archives.First().updates.Any(z => z.updated == usgs.updated)) { archives.First().updates.Add(usgs); } Program.cdb.UpsertDocument(usgs, "usgs").Wait(); Program.cdb.UpsertDocument(archives.First(), "usgs").Wait(); // submit to EventHub for further processing return(true); } } return(false); }