feat(network): Revise runahead from measured latency and frame pacing

Core/GameEngine/CMakeLists.txt

@@ -505,6 +505,7 @@ set(GAMEENGINE_SRC
 #    Include/GameLogic/WeaponStatus.h
     Include/GameNetwork/Connection.h
     Include/GameNetwork/ConnectionManager.h
+    Include/GameNetwork/ConnectionStats.h
     Include/GameNetwork/DisconnectManager.h
     Include/GameNetwork/DownloadManager.h
     Include/GameNetwork/FileTransfer.h
@@ -1099,6 +1100,7 @@ set(GAMEENGINE_SRC
     Source/GameLogic/System/RankInfo.cpp
     Source/GameNetwork/Connection.cpp
     Source/GameNetwork/ConnectionManager.cpp
+    Source/GameNetwork/ConnectionStats.cpp
     Source/GameNetwork/DisconnectManager.cpp
     Source/GameNetwork/DownloadManager.cpp
     Source/GameNetwork/FileTransfer.cpp

Core/GameEngine/Include/GameNetwork/Connection.h

@@ -39,11 +39,9 @@
 #include "GameNetwork/User.h"
 #include "GameNetwork/Transport.h"
 #include "GameNetwork/NetPacket.h"
-
+#include "GameNetwork/ConnectionStats.h"
 #if defined(GENERALS_ONLINE)
-#define CONNECTION_LATENCY_HISTORY_LENGTH 10
-#else
-#define CONNECTION_LATENCY_HISTORY_LENGTH 200
+#include "GameNetwork/NetworkSettings.h"
 #endif
 
 class Connection : public MemoryPoolObject
@@ -66,6 +64,13 @@ class Connection : public MemoryPoolObject
 	void setUser(User *user);
 	User *getUser();
 	void setFrameGrouping(time_t frameGrouping);
+#if defined(GENERALS_ONLINE)
+	void setNetworkSettings(const NetworkSettings &settings, Bool unreliableTransport) { m_networkSettings = settings; m_unreliableTransport = unreliableTransport; }
+#endif
+	void setSlot(Int slot) { m_slot = slot; }
+	void setShortGrouping(Bool shortGrouping);	///< Send at one frame while the run ahead is close to its floor.
+	Int getRedundancyDepth() const { return m_redundancyDepth; }
+	time_t getFrameGrouping() const { return m_frameGrouping; }	///< The send interval in use, shorter than the configured one while redundancy is on.
 
 	void sendNetCommandMsg(NetCommandMsg *msg, UnsignedByte relay);
 
@@ -80,12 +85,27 @@ class Connection : public MemoryPoolObject
 	void setQuitting();
 	Bool isQuitting() { return m_isQuitting; }
 
+	const ConnectionStats& getStats() const { return m_stats; }
+
 #if defined(RTS_DEBUG)
 	void debugPrintCommands();
 #endif
 
 protected:
-	void doRetryMetrics();
+#if defined(GENERALS_ONLINE)
+	NetworkSettings m_networkSettings;
+	Bool m_unreliableTransport;
+#endif
+
+	void resetRedundancy();
+	void updateRedundancy(time_t curtime, Bool &active, time_t &window, Int &depth);
+	Bool hasLossEvidence();	///< A packet was lost in either direction lately, or nothing says otherwise
+	Bool getOutgoingLoss(float &lossPercent, Int &longestRun);	///< FALSE when the peer sent no packet feedback
+	void noteLossEvent(time_t now, Bool burst);
+	void setRedundancyDepth(Int depth, Int cap, time_t now);
+	Bool isQueueBuilding(time_t now);	///< Round trip at 2x its low while packets are lost: a full queue
+	void applyFrameGrouping();
+	void addRedundantCommands(NetPacket *packet, time_t curtime, time_t window, Int depth);
 
 	Bool m_isQuitting;
 	UnsignedInt m_quitTime;
@@ -95,11 +115,17 @@ class Connection : public MemoryPoolObject
 
 	NetCommandList *m_netCommandList;
 	time_t m_retryTime;						///< The time between sending retry packets for this connection.  Time is in milliseconds.
-	Real m_averageLatency;			///< The average time between sending a command and receiving an ACK.
-	Real m_latencies[CONNECTION_LATENCY_HISTORY_LENGTH];	///< List of the last 100 latencies.
+	ConnectionStats m_stats;			///< Latency, jitter and loss of this connection.
+
+	Int m_redundancyDepth;					///< How many later packets each un-acked command is repeated in, 0 on a clean connection.
+	Int m_redundancyCap;						///< The deepest depth, from the longest burst to repair and the send interval.
+	time_t m_lastLossEventTime;			///< What the decay counts from.
+	time_t m_lastDepthChangeTime;		///< A burst needs a round trip to show whether the new depth helped.
+	Bool m_queueSuspected;				///< A full queue on the path, redundancy steps down instead of up.
+	Int m_slot;
 
+	Bool m_shortGrouping;
+	time_t m_configuredFrameGrouping;
 	time_t m_frameGrouping;				///< The minimum time between packet sends.
 	time_t m_lastTimeSent;				///< The time of the last packet send.
-	Int m_numRetries;							///< The number of retries for the last second.
-	time_t m_retryMetricsTime;		///< The start time of the current retry metrics thing.
 };

Core/GameEngine/Include/GameNetwork/ConnectionManager.h

@@ -37,6 +37,9 @@
 #include "GameNetwork/FrameMetrics.h"
 #include "GameNetwork/NetworkDefs.h"
 #include "GameNetwork/DisconnectManager.h"
+#if defined(GENERALS_ONLINE)
+#include "GameNetwork/NetworkSettings.h"
+#endif
 
 class GameInfo;
 class NetCommandWrapperList;
@@ -56,6 +59,7 @@ class ConnectionManager
 	virtual void update(Bool isInGame);			///< Service the Connections being managed by this instance.
 
 #if defined(GENERALS_ONLINE)
+	const NetworkSettings &getNetworkSettings() const { return m_networkSettings; }
 	void SeedLatencyData(int highestLatency)
 	{
 		for (int i = 0; i < MAX_SLOTS; ++i) {
@@ -108,6 +112,7 @@ class ConnectionManager
 	UnsignedInt getPacketRouterSlot();																///< Returns the current packet router's slot.
 	PlayerLeaveCode disconnectPlayer(Int slot);												///< Disconnect this player immediately.  This should only be called by the disconnect manager.
 #if defined(GENERALS_ONLINE)
+	void playerClosedGracefully(Int64 userID);
 	PlayerLeaveCode disconnectPlayer(int64_t userID);												///< Disconnect this player immediately.  This should only be called by the disconnect manager.
 #endif
 	void disconnectLocalPlayer();																			///< Does whatever is necessary to get TheNetwork to realize that it should be leaving the game now.
@@ -147,13 +152,17 @@ class ConnectionManager
 
 	void notifyOthersOfCurrentFrame(Int frame);
 	void sendFrameDataToPlayer(UnsignedInt playerID, UnsignedInt startingFrame);
-	void sendSingleFrameToPlayer(UnsignedInt playerID, UnsignedInt frame);
+	void sendSingleFrameToPlayer(UnsignedInt playerID, UnsignedInt frame, Bool ownOnly = FALSE);
 	void notifyOthersOfNewFrame(UnsignedInt frame);
 
 	UnsignedInt getNextPacketRouterSlot(UnsignedInt playerID); ///< returns the packet router player that comes after the given player.
 
 	Int getAverageFPS();
 	Int getSlotAverageFPS(Int slot);
+#if defined(GENERALS_ONLINE)
+	void noteStalling(Bool stalling, UnsignedInt frame);	///< Called every network update with whether the game is waiting for commands.
+#endif
+	const ConnectionStats* getConnectionStats(Int slot) const;	///< Returns nullptr when there is no connection for the slot.
 
 #if defined(RTS_DEBUG)
 	void debugPrintConnectionCommands();
@@ -165,6 +174,10 @@ class ConnectionManager
 	Int getPingsReceived();
 
 private:
+#if defined(GENERALS_ONLINE)
+	NetworkSettings m_networkSettings;
+#endif
+
 	void doRelay();
 	void doKeepAlive();
 	void sendRemoteCommand(NetCommandRef *msg);
@@ -192,6 +205,11 @@ class ConnectionManager
 	Real getMaximumLatency(); ///< Returns the average of the two highest average latencies between players.
 
 	void requestFrameDataResend(Int playerID, UnsignedInt frame); ///< request of this player that he send the specified frame's data.
+	UnsignedShort generateImpersonatedCommandID(Int playerID, UnsignedInt frame);
+#if !RETAIL_COMPATIBLE_NETWORKING
+	void askForOutPlayerFrame(Int slot, UnsignedInt frame);
+	Int getSlotOfAddress(UnsignedInt addr, UnsignedShort port);
+#endif
 
 	// The connections are set up like the slot list. The connection corresponding to the local
 	// player's position in the slot list will be null.  Connections corresponding to slots that
@@ -212,6 +230,14 @@ class ConnectionManager
 
 	NetCommandList *m_pendingCommands;
 	NetCommandList *m_relayedCommands;
+	UnsignedShort m_lastSeenCommandID[MAX_SLOTS];
+	Bool m_haveLastSeenCommandID[MAX_SLOTS];
+#if !RETAIL_COMPATIBLE_NETWORKING
+	Int m_currentSenderSlot;	///< The slot of the peer the packet being read came from, -1 when not known.
+	Int m_outFrameAskedSlot;
+	UnsignedInt m_outFrameAskedFrame;
+	UnsignedInt m_outFrameAskedTime;
+#endif
 
 	FrameMetrics m_frameMetrics;
 
@@ -226,6 +252,126 @@ class ConnectionManager
 	UnsignedInt m_smallestPacketArrivalCushion;
 	Bool m_didSelfSlug;
 
+#if defined(GENERALS_ONLINE)
+	// the last command ids seen per sender, so a redundant copy of a command that is not synchronized is dropped
+	enum { RECENT_ID_COUNT = 256 };
+	struct RecentCommandIds
+	{
+		Bool valid;
+		UnsignedShort highest;
+		Bool seen[RECENT_ID_COUNT];
+	};
+	RecentCommandIds m_recentIds[MAX_SLOTS];
+
+	Int m_runAheadLowerCount;
+	Int m_runAheadLowerPeak;
+	Bool isDuplicateCommand(NetCommandMsg *msg);
+
+	Int slotForUserID(Int64 userID) const;
+	Bool isPeerActive(Int slot) const;
+	void onDirectFrameAck(UnsignedInt frame, Int peerSlot);
+	std::map<UnsignedInt, UnsignedInt> m_frameAckedMasks;	///< Frame -> the peers that acked its frame info so far.
+	Bool isDirectSendActive();
+	Bool m_directSendActive;
+	time_t m_directSendCheckedAt;
+	UnsignedInt m_formerRouterMask;	///< Slots that were the packet router in this match, their last packets can still be on the way.
+	Bool isCommandFromSender(NetCommandMsg *msg, Int senderSlot);
+	Bool noteSecurityDrop();
+	Bool m_loggedSenderUnavailable;
+	Int m_securityDrops;
+	void resetRecentCommandIds();
+
+	// Frames with commands and chat that a peer passes on for another player need that player's signature.
+	// Every peer keeps the signatures it gets as long as the frames, so whoever holds a frame holds its proof.
+	enum { SIGNATURE_BYTES = 64, SIGNATURE_WAIT_MS = 3000, HELD_CHAT_COUNT = 8 };
+	struct FrameSignature
+	{
+		Bool used;
+		UnsignedInt frame;
+		Bool relayed;			///< a command or the frame info arrived from a peer other than the author
+		Bool haveSignature;
+		Bool verified;
+		Int relayedBy;
+		time_t waitingSince;	///< when the frame was complete without a signature, 0 when not waiting
+		UnsignedByte signature[SIGNATURE_BYTES];
+	};
+	struct HeldChat
+	{
+		NetChatCommandMsg *msg;
+		Int relayedBy;
+		time_t since;
+	};
+	struct ChatSignature
+	{
+		Bool used;
+		UnsignedInt author;
+		UnsignedShort commandID;
+		time_t since;
+		UnsignedByte signature[SIGNATURE_BYTES];
+	};
+	enum { RELAYED_WRAPPER_COUNT = 8 };
+	struct RelayedWrapper
+	{
+		Bool used;
+		UnsignedShort wrappedCommandID;
+		Int relayedBy;
+	};
+	std::vector<FrameSignature> m_frameSignatures[MAX_SLOTS];	///< Indexed like the frame data, by frame modulo FRAME_DATA_LENGTH.
+	HeldChat m_heldChats[HELD_CHAT_COUNT];
+	ChatSignature m_chatSignatures[HELD_CHAT_COUNT];
+	RelayedWrapper m_relayedWrappers[MAX_SLOTS][RELAYED_WRAPPER_COUNT];
+	Int m_nextRelayedWrapper[MAX_SLOTS];
+	Int64 m_signerUserID[MAX_SLOTS];	///< The user id per slot, kept after the player leaves to verify its last frames.
+	UnsignedInt m_lastFrameInfoSent;	///< The last frame our frame info went out for, our data up to it is final.
+	void clearSignatureState(Bool releaseChats);
+	void requestPassedOnFrameAgain(Int slot, UnsignedInt frame, Int passedOnBy);
+	FrameSignature *findFrameSignature(Int slot, UnsignedInt frame);
+	FrameSignature &frameSignatureFor(Int slot, UnsignedInt frame);
+	Int64 signerUserID(Int slot);
+	void noteFramePassedOn(Int author, UnsignedInt frame, Int relayedBy, Bool changesFrame);
+	void noteWrapperPassedOn(NetCommandMsg *wrapper, Int relayedBy);
+	Int wrapperPassedOnBy(NetCommandMsg *msg);
+	Bool buildSignedFrame(Int slot, UnsignedInt frame, std::vector<UnsignedByte> &out);
+	Bool buildSignedCommand(Int slot, NetCommandMsg *msg, std::vector<UnsignedByte> &out);
+	void signLocalFrame(NetFrameCommandMsg *frameInfo);
+	void signLocalChat(NetChatCommandMsg *msg);
+	Bool buildSignedLeave(Int slot, std::vector<UnsignedByte> &out);
+public:
+	Bool signLeaveNotice(NetDisconnectPlayerCommandMsg *msg);	///< The leaving player signs its leave notice.
+	Bool verifyLeaveNotice(NetDisconnectPlayerCommandMsg *msg);	///< The notice carries the signature of the player it says left.
+private:
+	void attachStoredSignature(Int author, NetFrameCommandMsg *frameInfo);
+	Bool isPassedOnFrameAuthentic(Int slot, UnsignedInt frame);
+	void storeFrameSignature(Int author, UnsignedInt frame, const UnsignedByte *signature, Bool fromAuthor);
+	void processSignatures(NetSignaturesCommandMsg *msg);
+	Bool holdPassedOnChat(NetChatCommandMsg *msg);
+	void releaseHeldChat(Int index, const UnsignedByte *signature);
+	void expireHeldChats(time_t now);
+
+	ArrivalCushionStats m_cushionStats[MAX_SLOTS];
+	StallStats m_stallStats;
+	Bool m_stallActive;
+	time_t m_stallStartTime;
+	UnsignedInt m_stallMissingMask;
+	Int m_lastKnownRunAhead;
+	Bool m_runAheadStillInitial;		///< No run ahead change has taken effect yet, the first decisions apply without the decrease hold.
+	Int m_pendingRunAhead;				///< Router: the run ahead of the last command it issued.
+	UnsignedInt m_pendingExecFrame;		///< Router: the execution frame of that command.
+	Int applyDecreaseHold(Int oldRunAhead, Int askedRunAhead, Int decreaseAfter);
+public:
+	void noteRunAheadApplied(Int oldRunAhead, Int newRunAhead);	///< The run ahead command took effect on this frame.
+private:
+
+	Bool m_shortGrouping;				///< The run ahead is within 2 frames of its floor, connections send at one frame
+	Int getRunAheadFloor() const;
+	time_t getEffectiveDecisionIntervalMs(Int runAhead) const;	///< The longer of the configured floor and 2 x the run ahead in ms
+	void updateShortGrouping();
+	Int m_frameGroupingMs;
+public:
+	Bool getOverlayInfo(Int slot, NetworkOverlayInfo &info);	///< Fills what the overlay and the connection rating need about one slot.
+protected:
+#endif
+
 	// -----------------------------------------------------------------------------
 	FileCommandMap s_fileCommandMap;
 	FileMaskMap s_fileRecipientMaskMap;