using System; using System.Data; using System.Collections; using System.Collections.Generic; using System.Collections.Specialized; using System.Text; using System.IO; using System.Xml; using System.Diagnostics; using System.Timers; using System.Net.NetworkInformation; using System.Threading; using System.Globalization; namespace WmData { public class WmDataLink { protected static CultureInfo cultureInfoUs = new CultureInfo("en-US"); protected WmGenericData data; protected WmDataLinkRole currentRole; protected List dataLinkServers; protected List syncObjectTypes; protected List syncTableNames; protected string thisServerAddress; protected DateTime lastSyncMoment = new DateTime(1985, 9, 10); protected int syncIntervalInSeconds = 60; protected System.Timers.Timer currentRoleWatcherTimer; protected System.Timers.Timer workTimer; public WmDataLink(WmGenericData data, string thisServerAddress, List dataLinkServerAddresses, int syncIntervalInSeconds, ListsyncObjectTypes) { this.data = data; this.currentRole = WmDataLinkRole.UNAVAILABLE; this.dataLinkServers = new List(); this.thisServerAddress = thisServerAddress; this.syncIntervalInSeconds = syncIntervalInSeconds; this.syncObjectTypes = syncObjectTypes; this.syncTableNames = new List(); foreach (Type syncObjectType in syncObjectTypes) { syncTableNames.Add(WmDataTools.getTableName(syncObjectType)); WmIntermediateObject intermediateSyncObject = data.IntermediateObjectManager.getIntermediateObject(syncObjectType, true); DataTable table = intermediateSyncObject.DataTable; // let it create a data table up front } foreach (string dataLinkServerAddress in dataLinkServerAddresses) { string dataLinkServerAddressNoTld = dataLinkServerAddress.Replace(".net", "").Replace(".eu", "").Replace(".nl", "").Replace(".com", ""); string thisServerAddressNoTld = thisServerAddress.Replace(".net", "").Replace(".eu", "").Replace(".nl", "").Replace(".com", ""); bool isLocalServer = dataLinkServerAddressNoTld.Equals(thisServerAddressNoTld); WmDataLinkServer dataLinkServer = new WmDataLinkServer(dataLinkServerAddress, isLocalServer, data); dataLinkServers.Add(dataLinkServer); } currentRoleWatcherTimer = new System.Timers.Timer(1000); currentRoleWatcherTimer.AutoReset = true; currentRoleWatcherTimer.Elapsed += new ElapsedEventHandler(currentRoleWatcherTimer_Elapsed); currentRoleWatcherTimer.Start(); workTimer = new System.Timers.Timer(1000); workTimer.AutoReset = true; workTimer.Elapsed += new ElapsedEventHandler(workTimer_Elapsed); workTimer.Start(); } /// /// Data link role change event handler with the data link role change event arguments /// public delegate void DataLinkRoleChangedEventHandler(object sender, WmDataLinkRoleChangedEventArguments e); /// /// Data change event, fires when something is added, updated or deleted /// public event DataLinkRoleChangedEventHandler DataLinkRoleChanged; protected void currentRoleWatcherTimer_Elapsed(object sender, ElapsedEventArgs e) { currentRoleWatcherTimer.Stop(); determineCurrentRole(); currentRoleWatcherTimer.Start(); } protected void workTimer_Elapsed(object sender, ElapsedEventArgs e) { workTimer.Stop(); work(); workTimer.Start(); } /// /// This functions contains the logic that determines the current role of this DataLink instance /// it triggers dataLinkRoleChanged events in case of changed external conditions. /// protected void determineCurrentRole() { if (CurrentRole == WmDataLinkRole.ACTIVE) { // check if there is an internet connection (that we may continue to be active) if (!WmDataTools.isHostAvailableViaPing("google.com")) { // if google can't be pinged: Thread.Sleep(30000); // sleep for a tiny while, it might be a single glitch if (!WmDataTools.isHostAvailableViaPing("microsoft.com") && !WmDataTools.isHostAvailableViaPing("google.com") && !WmDataTools.isHostAvailableViaPing("apple.com") && !WmDataTools.isInternetConnectionAvailable()) { // try our friends @ Microsoft, Google and Apple // if we'll reach this point, both google AND microsoft cannot be reached, we no assume there is NO internet connection available changeCurrentRole(WmDataLinkRole.UNAVAILABLE, "No internet connection available after multiple checks (Code 01)."); } } // check for a conflicting ACTIVE server role WmDataLinkServer activeServer = getExternalActiveServer(false); if (activeServer != null) { string otherActiveServerName = activeServer.ServerUrl.Replace("http://", "").Replace("/datalink/",""); // there is an other active server that is 'earlier' in the line of servers than this server // we should resign our active role and turn back to backup. changeCurrentRole(WmDataLinkRole.UNAVAILABLE, "Conflicting active server detected, server " + otherActiveServerName+" also has the ACTIVE role (Code 02)."); } Thread.Sleep(5000); // give it some extra air } else if (CurrentRole == WmDataLinkRole.UNAVAILABLE) { // check if there is an internet connection if (WmDataTools.isHostAvailableViaPing("google.com") || WmDataTools.isHostAvailableViaPing("microsoft.com")) { // if we can reach Microsoft or Google // check if there is an other server out there WmDataLinkServer activeServer = getExternalActiveServer(true); // if there is another active server: follow that one by becomming backup if (activeServer != null) { changeCurrentRole(WmDataLinkRole.BACKUP, "Normal procedure from UNAVAILABLE to BACKUP (Code 03)."); } else { // if there is NO active server, become active changeCurrentRole(WmDataLinkRole.ACTIVE, "Normal procedure from UNAVAILABLE to ACTIVE (Code 04)."); } } else { // no internet connection available, we'll do a little nap and wait for it to return Thread.Sleep(10000); } } else if (CurrentRole == WmDataLinkRole.BACKUP) { // check if there is another active server WmDataLinkServer activeServer = getExternalActiveServer(true); // if NO other active server found determine if we should become active if (activeServer == null) { // it may be that the active server is experiencing a tiny network glitch. We'll wait 15 seconds and try again: Thread.Sleep(15000); activeServer = getExternalActiveServer(true); if (activeServer == null) { // we still have no active server, wait an additional 10 seconds and try again: Thread.Sleep(10000); activeServer = getExternalActiveServer(true); if (activeServer == null) { // Before we become active, we should be certain that we'll have a working internet connection // we'll test that by pinging both google and microsoft AND requesting the google home page if ((WmDataTools.isHostAvailableViaPing("google.com") || WmDataTools.isHostAvailableViaPing("microsoft.com")) && WmDataTools.isInternetConnectionAvailable()) { // Allright, we're ready to go: changeCurrentRole(WmDataLinkRole.ACTIVE, "Lost contact with the former ACTIVE server, this server is coming from BACKUP to ACTIVE (Code 06)."); } else { // there is no internet connection available, we should go into unavailable mode: changeCurrentRole(WmDataLinkRole.UNAVAILABLE, "Lost contact with the former ACTIVE server, but this server has no internet connection. Therefore going in UNAVAILABLE mode (Code 07)."); } } // else: there is an active server: do nothing. (after 15+10 seconds) }// else: there is an active server: do nothing (after 15 seconds) }// else: there is an active server: do nothing } else if (CurrentRole == WmDataLinkRole.DISABLED) { // we'll stay disabled (just to be explicit) } } public void changeCurrentRole(WmDataLinkRole newRole, string reason) { if (newRole == WmDataLinkRole.ACTIVE) { // this server wants to become active server, do an additional check if no other server became active by now: int randomWaitInterval = new Random().Next(500); Thread.Sleep(randomWaitInterval); // to make it less likely that two servers become active at the same time if (getExternalActiveServer(true) != null) { // yes there is another server return; // stop this function, the server will not become active } } Trace.WriteLine(thisServerAddress + " now assumes role " + newRole+". "+reason); currentRole = newRole; if (DataLinkRoleChanged != null) { DataLinkRoleChanged(this, new WmDataLinkRoleChangedEventArguments(newRole,reason)); } } protected void work() { if (currentRole == WmDataLinkRole.BACKUP) { if (lastSyncMoment.AddSeconds(syncIntervalInSeconds) < DateTime.Now) { WmDataLinkServer activeServer = getExternalActiveServer(true); if (activeServer != null) { syncWithServer(activeServer); lastSyncMoment = DateTime.Now; } } } else if(currentRole == WmDataLinkRole.ACTIVE) { calculateChecksums(); } // else no work for that role } protected void calculateChecksums() { if(CurrentRole == WmDataLinkRole.ACTIVE){ foreach (Type objectType in syncObjectTypes) { string tableName = WmDataTools.getTableName(objectType); DateTime lastTransactionMomentUtc = new DateTime(data.TransactionManager.getLastTransactionMomentUtcTicksOnTable(tableName)); if (lastTransactionMomentUtc.AddMinutes(3) < DateTime.UtcNow) { Trace.WriteLine("WmDataLink: Calculating checksum for dormant table " + tableName); data.TransactionManager.registerChecksumTransaction(tableName); } } } } protected void syncWithServer(WmDataLinkServer server) { // Trace.WriteLine("WmDataLink: server " + thisServerAddress + " now syncing with " + server.ServerUrl); Stopwatch stopwatch = Stopwatch.StartNew(); List inconsistentTableNames = new List(); // Step 1: process transactions since last sync moment long lastSyncMomentUtcTicks = Status.LastSyncMomentUtcTicks; WmDataTransaction[] transactionsSinceLastSync = server.getTransactionsSinceLastSync(lastSyncMomentUtcTicks); if (transactionsSinceLastSync.Length > 0) { foreach (WmDataTransaction transactionSinceLastSync in transactionsSinceLastSync) { Trace.WriteLine("WmDataLink: processing transaction " + transactionSinceLastSync.LocalMomentUtcTicks + " on table " + transactionSinceLastSync.TableName+" ("+transactionSinceLastSync.DataRowAction.ToString()+")"); if (syncTableNames.IndexOf(transactionSinceLastSync.TableName)!=-1 && inconsistentTableNames.IndexOf(transactionSinceLastSync.TableName) == -1) { // if this transaction is not for an inconsitent table try { data.TransactionManager.importTransaction(transactionSinceLastSync); // import it long currentLocalChecksum = data.getChecksum(transactionSinceLastSync.TableName); if (currentLocalChecksum != transactionSinceLastSync.TableChecksumAfterThisTransaction) { Trace.WriteLine("WmDataLink: table " + transactionSinceLastSync.TableName + " on server " + thisServerAddress + " is different compared to " + server.ServerUrl + ". Local checksum " + currentLocalChecksum + " remote checksum " + transactionSinceLastSync.TableChecksumAfterThisTransaction + "@" + transactionSinceLastSync.LocalMomentUtcTicks + ". Marked table as inconsistent.)"); inconsistentTableNames.Add(transactionSinceLastSync.TableName); } } catch (Exception error) { Trace.WriteLine("WmDataLink: Error while processing transaction " + transactionSinceLastSync.LocalMomentUtcTicks + " on table " + transactionSinceLastSync.TableName + " (" + transactionSinceLastSync.DataRowAction.ToString() + "). "+error.Message+" Marking table for a resync."); inconsistentTableNames.Add(transactionSinceLastSync.TableName); } } } Trace.WriteLine("WmDataLink: Finished processing " + transactionsSinceLastSync.Length + " transactions"); long newLastSyncMoment = transactionsSinceLastSync[transactionsSinceLastSync.Length - 1].LocalMomentUtcTicks; WmDataLinkStatus status = Status; status.LastSyncMomentUtcTicks = newLastSyncMoment; data.updateObject(status); if (inconsistentTableNames.Count > 0) { Trace.WriteLine("WmDataLink: Now processing " + inconsistentTableNames.Count + " inconsistent table(s)"); } else { Trace.WriteLine("WmDataLink: All tables are consistent."); } // Step 2: Import inconsistent tables foreach (string inconsistentTableName in inconsistentTableNames) { reSyncTable(server, inconsistentTableName); } } stopwatch.Stop(); Trace.WriteLine("WmDataLink: completed syncing with " + server.ServerUrl+" in "+stopwatch.Elapsed.ToString()+" ("+stopwatch.ElapsedMilliseconds+"ms)"); } public void reSyncTable(WmDataLinkServer server, string tableName) { Stopwatch stopwatch = Stopwatch.StartNew(); Trace.WriteLine("WmDataLink: resyncing table " + tableName); Type objectType = data.IntermediateObjectManager.getExistingIntermediateObjectByTableName(tableName).ObjectType; DataTable localTable = data.getDataTable(objectType); DataRow[] allLocalDataRows = data.getDataRows(objectType, null); foreach (DataRow localRow in allLocalDataRows) { localTable.ImportRow(localRow); } Trace.WriteLine("WmDataLink: created a virtual representation of table " + tableName); DataTable remoteDataTable = server.getRemoteDataTable(tableName); DataRow[] allRemoteDataRows = remoteDataTable.Select(); Trace.WriteLine("WmDataLink: retrieved remote table " + tableName); int totalRows = allRemoteDataRows.Length; int currentRow = 0; double percentageDone = 0; double lastPercentageShown = 0; foreach (DataRow remoteDataRow in allRemoteDataRows) { currentRow++; percentageDone = Math.Round(((double)currentRow / totalRows) * 100, 2); if (lastPercentageShown + 5 < percentageDone) { Trace.WriteLine("WmDataLink: resyncing entire table '" + tableName + "' " + percentageDone + "% completed"); lastPercentageShown = percentageDone; } DataRow localDataRow = localTable.Rows.Find(remoteDataRow["id"]); if (localDataRow == null) { // this is a new row, import it to our local data: data.importDataRow(objectType, remoteDataRow, true, -1); Trace.WriteLine("WmDataLink: resyncing entire table '" + tableName + "': imported remote row id="+remoteDataRow["id"]); } else { // compare the two rows: bool rowChanged = false; foreach (DataColumn column in localTable.Columns) { object localValue = localDataRow[column.ColumnName] ; object remoteValue = remoteDataRow[column.ColumnName]; if(localValue!= null && (localValue.GetType().Equals(typeof(sbyte)) || localValue.GetType().Equals(typeof(bool)) )){ localValue = Convert.ToBoolean(localValue); if (remoteValue.GetType().Equals(typeof(DBNull))) { remoteValue = false; } else { remoteValue = Convert.ToBoolean(remoteValue) ; } } if (!WmDataTools.compareObjects(localValue, remoteValue)) { data.updateDataRow(objectType, remoteDataRow,-1); Trace.WriteLine("WmDataLink: resyncing entire table '" + tableName + "': updated row id=" + remoteDataRow["id"] + " because column " + column.ColumnName + " was different: " + localValue + "/" + remoteValue + " types are " + localValue.GetType() + " " + remoteValue.GetType()); rowChanged = true; break; } } if (!rowChanged) { // Trace.WriteLine("WmDataLink: resyncing entire table '" + tableName + "': ignoring row id=" + remoteDataRow["id"]); } } } foreach (DataRow localRow in allLocalDataRows) { DataRow remoteRow = remoteDataTable.Rows.Find(localRow["id"]); if (remoteRow == null) { // the local data row has become obsolete (remotly deleted) data.deleteDataRow(objectType,localRow,-1); // Trace.WriteLine("WmDataLink: resyncing entire table '" + tableName + "': deleted row id=" + localRow["id"]); } } stopwatch.Stop(); Trace.WriteLine("WmDataLink: resynced entire table '" + tableName + "' from " + server.ServerUrl + " containing " + remoteDataTable.Rows.Count + " records in "+stopwatch.Elapsed.ToString()+" ("+stopwatch.ElapsedMilliseconds+"ms)"); } /// /// Returns a non local active server or null in case none exits /// protected WmDataLinkServer getExternalActiveServer(bool scanAllServers) { foreach (WmDataLinkServer server in DataLinkServers) { if (!server.IsLocalServer && server.getCurrentRole() == WmDataLinkRole.ACTIVE) { return server; } if (!scanAllServers) { if (server.IsLocalServer && server.getCurrentRole() == WmDataLinkRole.ACTIVE) { // we found an active server, but the server is local. return null; // we'll stop our search since we've found an active server } } } // no active server found, return null: return null; } /// /// Returns the active data server, or null if none could be found /// public WmDataLinkServer getActiveServer() { foreach (WmDataLinkServer server in DataLinkServers) { if (server.getCurrentRole() == WmDataLinkRole.ACTIVE) { return server; } } return null; } public List DataLinkServers { get { return dataLinkServers; } } public List SyncObjectTypes { get { return syncObjectTypes; } set { syncObjectTypes = value; } } public WmDataLinkRole CurrentRole { get { return currentRole; } } public string getTransactionsAsXmlSince(long sinceMomentUtcTicks) { DataRow[] transactionRows = data.TransactionManager.getTransactionsAsDataRowsSince(sinceMomentUtcTicks); DataTable transactionTable = data.getDataTable(typeof(WmDataTransaction)); for (int d = 0; d < transactionRows.Length; d++) { if (syncTableNames.IndexOf((string)transactionRows[d]["tablename"]) != -1) { transactionTable.ImportRow(transactionRows[d]); } } StringWriter writer = new StringWriter(); transactionTable.WriteXml(writer, XmlWriteMode.WriteSchema, false); return writer.ToString(); } public string getDataTableAsXml(Type objectType) { DataTable dataTable = data.getDataTable(objectType); bool didChangeCulture = false; CultureInfo originalCultureInfo = Thread.CurrentThread.CurrentCulture; if (!Thread.CurrentThread.CurrentCulture.Name.Equals("en-US")) { Thread.CurrentThread.CurrentCulture = cultureInfoUs; } if (dataTable != null) { StringWriter writer = new StringWriter(); writer.NewLine = System.Environment.NewLine; DataRow[] allDataRows = data.getDataRows(objectType, null); for (int d = 0; d < allDataRows.Length; d++) { NameValueCollection nameValueCollection = WmDataTools.getNameValueCollectionFromDataRow(allDataRows[d]); string rowAsQueryString= WmDataTools.getQueryStringFromNameValueCollection(nameValueCollection); writer.WriteLine(rowAsQueryString); } return writer.ToString(); } else { return ""; } if (didChangeCulture) { Thread.CurrentThread.CurrentCulture = originalCultureInfo; } } public long LastSyncMomentUtcTicks { get { return this.Status.LastSyncMomentUtcTicks; } } public WmDataLinkStatus Status { get { WmDataLinkStatus status =(WmDataLinkStatus) data.getObject(typeof(WmDataLinkStatus), 1); if (status == null) { status = new WmDataLinkStatus(); status.Id = 1; status.LastSyncMomentUtcTicks = new DateTime(1985, 9, 10).Ticks; data.addObject(status, true); return Status; } else { return status; } } } public string ThisServerAddress { get { return thisServerAddress; } } } public enum WmDataLinkRole { ACTIVE=1, BACKUP=2, DISABLED=-1, UNAVAILABLE=-2 } }