better clock synchronization, including p2p sync

direct/src/distributed/ClientRepository.py

@@ -59,7 +59,7 @@ class ClientRepository(ConnectionRepository.ConnectionRepository):
         precisely measured and may drift slightly after startup, but
         it should be accurate plus or minus a couple of seconds.
         """
-        return time.time() + self.cr.getServerDelta()
+        return time.time() + self.serverDelta
 
     def parseDcFile(self, dcFileName):
         self.dcFile = DCFile()

direct/src/distributed/ClockDelta.py

@@ -24,6 +24,18 @@ NetworkTimePrecision = 100.0
 # These values are derived from the above.
 NetworkTimeMask = (1 << NetworkTimeBits) - 1
 NetworkTimeTopBits = 32 - NetworkTimeBits
+MaxTimeDelta = (NetworkTimeMask / 2.0) / NetworkTimePrecision
+
+# This is the maximum number of seconds by which we expect our clock
+# (or the server's clock) to drift over an hour.
+ClockDriftPerHour = 1.0   # Is this generous enough?
+
+# And the above, scaled into a per-second value.
+ClockDriftPerSecond = ClockDriftPerHour / 3600.0
+
+# How many seconds to insist on waiting before accepting a second
+# resync request from another client.
+P2PResyncDelay = 10.0
 
 class ClockDelta(DirectObject.DirectObject):
     """
@@ -38,33 +50,171 @@ class ClockDelta(DirectObject.DirectObject):
     def __init__(self):
         self.globalClock = ClockObject.getGlobalClock()
 
+        # self.delta is the relative delta from our clock to the
+        # server's clock.
         self.delta = 0
+
+        # self.uncertainty represents the number of seconds plus or
+        # minus in which we are confident our delta matches the
+        # server's actual time.  The initial value, None, represents
+        # infinity--we have no idea.
+        self.uncertainty = None
+
+        # self.lastResync is the time at which self.uncertainty
+        # was measured.  It is important to remember because our
+        # uncertainty increases over time (due to relative clock
+        # drift).
+        self.lastResync = 0.0
+        
         self.accept("resetClock", self.__resetClock)
 
+    def getDelta(self):
+        return self.delta
+
+    def getUncertainty(self):
+        # Returns our current uncertainty with our clock measurement,
+        # as a number of seconds plus or minus.  Returns None,
+        # representing infinite uncertainty, if we have never received
+        # a time measurement.
+        
+        if self.uncertainty == None:
+            return None
+        
+        now = self.globalClock.getRealTime()
+        elapsed = now - self.lastResync
+        return self.uncertainty + elapsed * ClockDriftPerSecond
+
+    def getLastResync(self):
+        # Returns the local time at which we last resynchronized the
+        # clock delta.
+        return self.lastResync
+    
     def __resetClock(self, timeDelta):
         """
         this is called when the global clock gets adjusted
         timeDelta is equal to the amount of time, in seconds,
         that has been added to the global clock
         """
-        self.notify.debug("adjusting timebase by %f seconds" % timeDelta)
+        assert(self.notify.debug("adjusting timebase by %f seconds" % timeDelta))
         # adjust our timebase by the same amount
         self.delta += timeDelta
 
-    def resynchronize(self, localTime, networkTime):
-        """resynchronize(self, float localTime, uint32 networkTime)
-
-        Resets the relative delta so that the indicated networkTime
-        and localTime map to the same instant.  The return value is
-        the amount by which the clock changes, in seconds.
+    def clear(self):
+        """
+        Throws away any previous synchronization information.
+        """
+        self.delta = 0
+        self.uncertainty = None
+        self.lastResync = 0.0
+
+    def resynchronize(self, localTime, networkTime, newUncertainty,
+                      trustNew = 1):
+        """resynchronize(self, float localTime, int32 networkTime,
+                         float newUncertainty)
+
+        Accepts a new networkTime value, which is understood to
+        represent the same moment as localTime, plus or minus
+        uncertainty seconds.  Improves our current notion of the time
+        delta accordingly.
         """
         newDelta = (float(localTime) -
                     (float(networkTime) / NetworkTimePrecision))
-        change = newDelta - self.delta
-        self.delta = newDelta
+        self.newDelta(localTime, newDelta, newUncertainty)
 
-        return change
+    def peerToPeerResync(self, avId, timestamp, serverTime, uncertainty):
+        """
+        Accepts an AI time and uncertainty value from another client,
+        along with a local timestamp value of the message from this
+        client which prompted the other client to send us its delta
+        information.
+
+        The return value is true if the other client's measurement was
+        reasonably close to our own, or false if the other client's
+        time estimate was wildly divergent from our own; the return
+        value is negative if the test was not even considered (because
+        it happened too soon after another recent request).
+        """
+        
+        now = self.globalClock.getRealTime()
+        if now - self.lastResync < P2PResyncDelay:
+            # We can't process this request; it came in on the heels
+            # of some other request, and our local timestamp may have
+            # been resynced since then: ergo, the timestamp in this
+            # request is meaningless.
+            assert(self.notify.debug("Ignoring request for resync from %s within %.3f s." % (avId, now - self.lastResync)))
+            return -1
+            
+        # The timestamp value will be a timestamp that we sent out
+        # previously, echoed back to us.  Therefore we can confidently
+        # convert it back into our local time, even though we suspect
+        # our clock delta might be off.
+        local = self.networkToLocalTime(timestamp, now)
+        elapsed = now - local
+        delta = (local + now) / 2.0 - serverTime
+
+        gotSync = 0
+        if elapsed <= 0 or elapsed > P2PResyncDelay:
+            # The elapsed time must be positive (the local timestamp
+            # must be in the past), and shouldn't be more than
+            # P2PResyncDelay.  If it does not meet these requirements,
+            # it must be very old indeed, or someone is playing tricks
+            # on us.
+            self.notify.info("Ignoring old request for resync from %s." % (avId))
+        else:
+            # Now the other client has told us his delta and uncertainty
+            # information, which was generated somewhere in the range
+            # [-elapsed, 0] seconds ago.  That means our complete window
+            # is wider by that amount.
+            self.notify.info("Got sync +/- %.3f s, elapsed %.3f s, from %s." % (uncertainty, elapsed, avId))
+            delta -= elapsed / 2.0
+            uncertainty += elapsed / 2.0
+        
+            gotSync = self.newDelta(local, delta, uncertainty, trustNew = 0)
 
+        return gotSync
+
+    def newDelta(self, localTime, newDelta, newUncertainty,
+                 trustNew = 1):
+        """
+        Accepts a new delta and uncertainty pair, understood to
+        represent time as of localTime.  Improves our current notion
+        of the time delta accordingly.  The return value is true if
+        the new measurement was used, false if it was discarded.
+        """
+        oldUncertainty = self.getUncertainty()
+        if oldUncertainty != None:
+            assert(self.notify.debug('previous delta at %.3f s, +/- %.3f s.' % (self.delta, oldUncertainty)))
+            assert(self.notify.debug('new delta at %.3f s, +/- %.3f s.' % (newDelta, newUncertainty)))
+            # Our previous measurement was self.delta +/- oldUncertainty;
+            # our new measurement is newDelta +/- newUncertainty.  Take
+            # the intersection of both.
+            
+            oldLow = self.delta - oldUncertainty
+            oldHigh = self.delta + oldUncertainty
+            newLow = newDelta - newUncertainty
+            newHigh = newDelta + newUncertainty
+
+            low = max(oldLow, newLow)
+            high = min(oldHigh, newHigh)
+                      
+            # If there is no intersection, whoops!  Either the old
+            # measurement or the new measurement is completely wrong.
+            if low > high:
+                if not trustNew:
+                    self.notify.info('discarding new delta.')
+                    return 0
+                
+                self.notify.info('discarding previous delta.')
+            else:
+                newDelta = (low + high) / 2.0
+                newUncertainty = (high - low) / 2.0
+                assert(self.notify.debug('intersection at %.3f s, +/- %.3f s.' % (newDelta, newUncertainty)))
+        
+        self.delta = newDelta
+        self.uncertainty = newUncertainty
+        self.lastResync = localTime
+
+        return 1
 
     ### Primary interface functions ###
 
@@ -147,11 +297,7 @@ class ClockDelta(DirectObject.DirectObject):
         now = self.globalClock.getFrameTime()
         dt = now - self.networkToLocalTime(networkTime, now, bits=bits)
 
-        if (dt >= 0.0):
-            return dt
-        else:
-            self.notify.debug('negative clock delta: %.3f' % dt)
-            return 0.0
+        return max(dt, 0.0)
 
 
 

direct/src/distributed/DistributedSmoothNode.py

@@ -283,17 +283,20 @@ class DistributedSmoothNode(DistributedNode.DistributedNode):
         
         now = globalClock.getFrameTime()
         local = globalClockDelta.networkToLocalTime(timestamp, now)
-        chug = globalClock.getRealTime() - now
+        real = globalClock.getRealTime()
+        chug = real - now
 
         # Sanity check the timestamp from the other avatar.  It should
         # be just slightly in the past, but it might be off by as much
         # as this frame's amount of time forward or back.
         howFarFuture = local - now
         if howFarFuture - chug >= MaxFuture:
-            # Too far off; resync both of us.
-            if self.cr.timeManager != None:
-                self.cr.timeManager.synchronize("Packets from %d off by %.1f s" % (self.doId, howFarFuture))
-            self.d_suggestResync(self.cr.localToonDoId)
+            # Too far off; advise the other client of our clock information.
+            if globalClockDelta.getUncertainty() != None:
+                self.d_suggestResync(self.cr.localToonDoId, timestamp,
+                                     globalClockDelta.getRealNetworkTime(),
+                                     real - globalClockDelta.getDelta(),
+                                     globalClockDelta.getUncertainty())
         
         self.smoother.setTimestamp(local)
         self.smoother.markPosition()
@@ -334,16 +337,48 @@ class DistributedSmoothNode(DistributedNode.DistributedNode):
 
     ### Monitor clock sync ###
 
-    def d_suggestResync(self, avId):
-        self.sendUpdate("suggestResync", [avId])
+    def d_suggestResync(self, avId, timestampA, timestampB, serverTime, uncertainty):
+        self.sendUpdate("suggestResync", [avId, timestampA, timestampB, serverTime, uncertainty])
         
-    def suggestResync(self, avId):
+    def suggestResync(self, avId, timestampA, timestampB, serverTime, uncertainty):
         """suggestResync(self, avId)
 
         This message is sent from one client to another when the other
         client receives a timestamp from this client that is so far
         out of date as to suggest that one or both clients needs to
-        resynchronize with the AI.
+        resynchronize their clock information.
+        """
+        result = \
+               self.peerToPeerResync(avId, timestampA, serverTime, uncertainty)
+        if result >= 0 and \
+           globalClockDelta.getUncertainty() != None:
+            other = self.cr.doId2do.get(avId)
+            if other and hasattr(other, "d_returnResync"):
+                real = globalClock.getRealTime()
+                other.d_returnResync(self.cr.localToonDoId, timestampB,
+                                     real - globalClockDelta.getDelta(),
+                                     globalClockDelta.getUncertainty())
+        
+
+    def d_returnResync(self, avId, timestampB, serverTime, uncertainty):
+        self.sendUpdate("returnResync", [avId, timestampB, serverTime, uncertainty])
+        
+    def returnResync(self, avId, timestampB, serverTime, uncertainty):
+        """returnResync(self, avId)
+
+        A reply sent by a client whom we recently sent suggestResync
+        to, this reports the client's new delta information so we can
+        adjust our clock as well.
         """
-        if self.cr.timeManager != None:
-            self.cr.timeManager.synchronize("suggested by %d" % (avId))
+        self.peerToPeerResync(avId, timestampB, serverTime, uncertainty)
+
+    def peerToPeerResync(self, avId, timestamp, serverTime, uncertainty):
+        gotSync = globalClockDelta.peerToPeerResync(avId, timestamp, serverTime, uncertainty)
+
+        # If we didn't get anything useful from the other client,
+        # maybe our clock is just completely hosed.  Go ask the AI.
+        if not gotSync:
+            if self.cr.timeManager != None:
+                self.cr.timeManager.synchronize("suggested by %d" % (avId))
+
+        return gotSync