public override global::System.Data.DataSet Clone()
{
DataSetSummary cln = ((DataSetSummary)(base.Clone()));
cln.InitVars();
cln.SchemaSerializationMode = this.SchemaSerializationMode;
return(cln);
}
public void UpdateManifest(DataSetSummary dataSet)
{
MongoAgent conn = MongoAgent.GetInstance(Config);
BsonDocument query = new BsonDocument("_id", dataSet._id);
BsonDocument doc = BsonDocument.Parse(JsonConvert.SerializeObject(dataSet));
doc.SetElement(new BsonElement("_id", dataSet._id));
conn.UpdateCollectionAsync(Config.ManifestCollection, query, doc);
}
public static global::System.Xml.Schema.XmlSchemaComplexType GetTypedDataSetSchema(global::System.Xml.Schema.XmlSchemaSet xs)
{
DataSetSummary ds = new DataSetSummary();
global::System.Xml.Schema.XmlSchemaComplexType type = new global::System.Xml.Schema.XmlSchemaComplexType();
global::System.Xml.Schema.XmlSchemaSequence sequence = new global::System.Xml.Schema.XmlSchemaSequence();
global::System.Xml.Schema.XmlSchemaAny any = new global::System.Xml.Schema.XmlSchemaAny();
any.Namespace = ds.Namespace;
sequence.Items.Add(any);
type.Particle = sequence;
global::System.Xml.Schema.XmlSchema dsSchema = ds.GetSchemaSerializable();
if (xs.Contains(dsSchema.TargetNamespace))
{
global::System.IO.MemoryStream s1 = new global::System.IO.MemoryStream();
global::System.IO.MemoryStream s2 = new global::System.IO.MemoryStream();
try {
global::System.Xml.Schema.XmlSchema schema = null;
dsSchema.Write(s1);
for (global::System.Collections.IEnumerator schemas = xs.Schemas(dsSchema.TargetNamespace).GetEnumerator(); schemas.MoveNext();)
{
schema = ((global::System.Xml.Schema.XmlSchema)(schemas.Current));
s2.SetLength(0);
schema.Write(s2);
if ((s1.Length == s2.Length))
{
s1.Position = 0;
s2.Position = 0;
for (; ((s1.Position != s1.Length) &&
(s1.ReadByte() == s2.ReadByte()));)
{
;
}
if ((s1.Position == s1.Length))
{
return(type);
}
}
}
}
finally {
if ((s1 != null))
{
s1.Close();
}
if ((s2 != null))
{
s2.Close();
}
}
}
xs.Add(dsSchema);
return(type);
}
public List <DataSetSummary> ParsingData(MongoAgent conn)
{
Logger.Info("Start parsing today's manifest");
List <DataSetSummary> summary = new List <DataSetSummary>();
try
{
StreamReader sr = new StreamReader(Path.Combine(Config.LocalFolder, LocalFileName));
JObject jsonStr = JObject.Parse(sr.ReadToEnd());
foreach (JToken token in jsonStr.SelectToken("dataset").Children())
{
//Deserialize the token from JSON to object
DataSetSummary dataSetSum = JsonConvert.DeserializeObject <DataSetSummary>(token.First.ToString());
dataSetSum.token = token.Path;
//Compare with the last record to decide if it needs to be download today.
BsonDocument query = new BsonDocument("identifier", dataSetSum.identifier);
List <BsonDocument> list = conn.ReadCollection(Config.ManifestCollection, query);
if (list.Count == 0)
{
//If there is no record of such identifier, it means this is a new file type. It needs to be downloaded and insert into database.
summary.Add(dataSetSum);
conn.InsertCollectionAsync(Config.ManifestCollection, BsonDocument.Parse(token.First.ToString()));
}
else
{
//Always only check the top 1 record to decide if a downloading need to be performed or not.
DataSetSummary old = BsonSerializer.Deserialize <DataSetSummary>(list[0]);
if (DateTime.Parse(dataSetSum.last_updated) > DateTime.Parse(old.last_updated))
{
dataSetSum._id = old._id;
summary.Add(dataSetSum);
//update performing.
//query = new BsonDocument("_id", old._id);
//BsonDocument doc = BsonDocument.Parse(JsonConvert.SerializeObject(dataSetSum));
//doc.SetElement(new BsonElement("_id", old._id));
//conn.UpdateCollectionAsync(Configurations.ManifestCollection, query, doc);
}
}
}
Logger.Info("Finish parsing today's manifest.");
Logger.Info("There are " + summary.Count + " datasets need to update.");
}
catch (Exception e)
{
Logger.Error(e.Message);
return(new List <DataSetSummary>());
}
return(summary);
}
public async Task <bool> ProcessingData(DataSetSummary dataSummary)
{
try
{
Logger.Info("Task " + dataSummary.identifier + " start.");
FileHandler handler = new FileHandler(dataSummary.accessURL);
//string downloadedfile = string.Empty;
int Counter = 3;
string downloadedfile = await handler.Download(Path.Combine(Config.LocalFolder, dataSummary.identifier));
while (Counter-- >= 0)
{
//string downloadedfile = await handler.Download(Path.Combine(configurations.LocalFolder, dataSummary.identifier));
if (!string.IsNullOrEmpty(downloadedfile) && !downloadedfile.Equals("Failed"))
{
string extractedFile = handler.UnZipping(downloadedfile, Path.Combine(Config.LocalFolder, dataSummary.identifier));
if (!String.IsNullOrEmpty(extractedFile))
{
ParsingData(extractedFile, dataSummary.identifier, Config.DebugMode);
}
}
else
{
if (Counter == 0)
{
break;
}
System.Threading.Thread.Sleep(Config.RetryInterval);
Logger.Info("Retry downloading " + dataSummary.identifier);
string strIncomplete = Path.Combine(Config.LocalFolder, dataSummary.identifier, dataSummary.accessURL.Substring(dataSummary.accessURL.LastIndexOf("/") + 1));
if (File.Exists(strIncomplete))
{
File.Delete(strIncomplete);
}
downloadedfile = handler.Download(Path.Combine(Config.LocalFolder, dataSummary.identifier)).Result;
continue;
}
Logger.Info("Task " + dataSummary.identifier + " end.");
return(true);
}
Logger.Info("Downloading " + dataSummary.identifier + " failed.");
return(false);
}
catch (Exception E)
{
Logger.Error(E.Message, E);
return(false);
}
}
public static global::System.Xml.Schema.XmlSchemaComplexType GetTypedTableSchema(global::System.Xml.Schema.XmlSchemaSet xs)
{
global::System.Xml.Schema.XmlSchemaComplexType type = new global::System.Xml.Schema.XmlSchemaComplexType();
global::System.Xml.Schema.XmlSchemaSequence sequence = new global::System.Xml.Schema.XmlSchemaSequence();
DataSetSummary ds = new DataSetSummary();
global::System.Xml.Schema.XmlSchemaAny any1 = new global::System.Xml.Schema.XmlSchemaAny();
any1.Namespace = "http://www.w3.org/2001/XMLSchema";
any1.MinOccurs = new decimal(0);
any1.MaxOccurs = decimal.MaxValue;
any1.ProcessContents = global::System.Xml.Schema.XmlSchemaContentProcessing.Lax;
sequence.Items.Add(any1);
global::System.Xml.Schema.XmlSchemaAny any2 = new global::System.Xml.Schema.XmlSchemaAny();
any2.Namespace = "urn:schemas-microsoft-com:xml-diffgram-v1";
any2.MinOccurs = new decimal(1);
any2.ProcessContents = global::System.Xml.Schema.XmlSchemaContentProcessing.Lax;
sequence.Items.Add(any2);
global::System.Xml.Schema.XmlSchemaAttribute attribute1 = new global::System.Xml.Schema.XmlSchemaAttribute();
attribute1.Name = "namespace";
attribute1.FixedValue = ds.Namespace;
type.Attributes.Add(attribute1);
global::System.Xml.Schema.XmlSchemaAttribute attribute2 = new global::System.Xml.Schema.XmlSchemaAttribute();
attribute2.Name = "tableTypeName";
attribute2.FixedValue = "SummaryDataTable";
type.Attributes.Add(attribute2);
type.Particle = sequence;
global::System.Xml.Schema.XmlSchema dsSchema = ds.GetSchemaSerializable();
if (xs.Contains(dsSchema.TargetNamespace))
{
global::System.IO.MemoryStream s1 = new global::System.IO.MemoryStream();
global::System.IO.MemoryStream s2 = new global::System.IO.MemoryStream();
try {
global::System.Xml.Schema.XmlSchema schema = null;
dsSchema.Write(s1);
for (global::System.Collections.IEnumerator schemas = xs.Schemas(dsSchema.TargetNamespace).GetEnumerator(); schemas.MoveNext();)
{
schema = ((global::System.Xml.Schema.XmlSchema)(schemas.Current));
s2.SetLength(0);
schema.Write(s2);
if ((s1.Length == s2.Length))
{
s1.Position = 0;
s2.Position = 0;
for (; ((s1.Position != s1.Length) &&
(s1.ReadByte() == s2.ReadByte()));)
{
;
}
if ((s1.Position == s1.Length))
{
return(type);
}
}
}
}
finally {
if ((s1 != null))
{
s1.Close();
}
if ((s2 != null))
{
s2.Close();
}
}
}
xs.Add(dsSchema);
return(type);
}
