feat(network): Measure connection latency, jitter and loss with packet feedback

Core/GameEngine/CMakeLists.txt

@@ -505,6 +505,7 @@ set(GAMEENGINE_SRC
 #    Include/GameLogic/WeaponStatus.h
     Include/GameNetwork/Connection.h
     Include/GameNetwork/ConnectionManager.h
+    Include/GameNetwork/ConnectionStats.h
     Include/GameNetwork/DisconnectManager.h
     Include/GameNetwork/DownloadManager.h
     Include/GameNetwork/FileTransfer.h
@@ -1099,6 +1100,7 @@ set(GAMEENGINE_SRC
     Source/GameLogic/System/RankInfo.cpp
     Source/GameNetwork/Connection.cpp
     Source/GameNetwork/ConnectionManager.cpp
+    Source/GameNetwork/ConnectionStats.cpp
     Source/GameNetwork/DisconnectManager.cpp
     Source/GameNetwork/DownloadManager.cpp
     Source/GameNetwork/FileTransfer.cpp

Core/GameEngine/Include/GameNetwork/Connection.h

@@ -39,12 +39,7 @@
 #include "GameNetwork/User.h"
 #include "GameNetwork/Transport.h"
 #include "GameNetwork/NetPacket.h"
-
-#if defined(GENERALS_ONLINE)
-#define CONNECTION_LATENCY_HISTORY_LENGTH 10
-#else
-#define CONNECTION_LATENCY_HISTORY_LENGTH 200
-#endif
+#include "GameNetwork/ConnectionStats.h"
 
 class Connection : public MemoryPoolObject
 {
@@ -80,13 +75,13 @@ class Connection : public MemoryPoolObject
 	void setQuitting();
 	Bool isQuitting() { return m_isQuitting; }
 
+	const ConnectionStats& getStats() const { return m_stats; }
+
 #if defined(RTS_DEBUG)
 	void debugPrintCommands();
 #endif
 
 protected:
-	void doRetryMetrics();
-
 	Bool m_isQuitting;
 	UnsignedInt m_quitTime;
 
@@ -95,11 +90,8 @@ class Connection : public MemoryPoolObject
 
 	NetCommandList *m_netCommandList;
 	time_t m_retryTime;						///< The time between sending retry packets for this connection.  Time is in milliseconds.
-	Real m_averageLatency;			///< The average time between sending a command and receiving an ACK.
-	Real m_latencies[CONNECTION_LATENCY_HISTORY_LENGTH];	///< List of the last 100 latencies.
+	ConnectionStats m_stats;			///< Latency, jitter and loss of this connection.
 
 	time_t m_frameGrouping;				///< The minimum time between packet sends.
 	time_t m_lastTimeSent;				///< The time of the last packet send.
-	Int m_numRetries;							///< The number of retries for the last second.
-	time_t m_retryMetricsTime;		///< The start time of the current retry metrics thing.
 };

Core/GameEngine/Include/GameNetwork/ConnectionManager.h

@@ -37,6 +37,9 @@
 #include "GameNetwork/FrameMetrics.h"
 #include "GameNetwork/NetworkDefs.h"
 #include "GameNetwork/DisconnectManager.h"
+#if defined(GENERALS_ONLINE)
+#include "GameNetwork/NetworkSettings.h"
+#endif
 
 class GameInfo;
 class NetCommandWrapperList;
@@ -56,6 +59,7 @@ class ConnectionManager
 	virtual void update(Bool isInGame);			///< Service the Connections being managed by this instance.
 
 #if defined(GENERALS_ONLINE)
+	const NetworkSettings &getNetworkSettings() const { return m_networkSettings; }
 	void SeedLatencyData(int highestLatency)
 	{
 		for (int i = 0; i < MAX_SLOTS; ++i) {
@@ -154,6 +158,10 @@ class ConnectionManager
 
 	Int getAverageFPS();
 	Int getSlotAverageFPS(Int slot);
+#if defined(GENERALS_ONLINE)
+	void noteStalling(Bool stalling, UnsignedInt frame);	///< Called every network update with whether the game is waiting for commands.
+#endif
+	const ConnectionStats* getConnectionStats(Int slot) const;	///< Returns nullptr when there is no connection for the slot.
 
 #if defined(RTS_DEBUG)
 	void debugPrintConnectionCommands();
@@ -165,6 +173,10 @@ class ConnectionManager
 	Int getPingsReceived();
 
 private:
+#if defined(GENERALS_ONLINE)
+	NetworkSettings m_networkSettings;
+#endif
+
 	void doRelay();
 	void doKeepAlive();
 	void sendRemoteCommand(NetCommandRef *msg);
@@ -226,6 +238,14 @@ class ConnectionManager
 	UnsignedInt m_smallestPacketArrivalCushion;
 	Bool m_didSelfSlug;
 
+#if defined(GENERALS_ONLINE)
+	ArrivalCushionStats m_cushionStats[MAX_SLOTS];
+	StallStats m_stallStats;
+	Bool m_stallActive;
+	time_t m_stallStartTime;
+	UnsignedInt m_stallMissingMask;
+#endif
+
 	// -----------------------------------------------------------------------------
 	FileCommandMap s_fileCommandMap;
 	FileMaskMap s_fileRecipientMaskMap;

Core/GameEngine/Include/GameNetwork/ConnectionStats.h

@@ -0,0 +1,162 @@
+/*
+**	Command & Conquer Generals Zero Hour(tm)
+**	Copyright 2025 Electronic Arts Inc.
+**
+**	This program is free software: you can redistribute it and/or modify
+**	it under the terms of the GNU General Public License as published by
+**	the Free Software Foundation, either version 3 of the License, or
+**	(at your option) any later version.
+**
+**	This program is distributed in the hope that it will be useful,
+**	but WITHOUT ANY WARRANTY; without even the implied warranty of
+**	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+**	GNU General Public License for more details.
+**
+**	You should have received a copy of the GNU General Public License
+**	along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <time.h>
+
+// Fixed size ring of items with a time member, oldest first, that drops the items older than a window.
+template <typename T, Int N>
+class TimedRing
+{
+public:
+	TimedRing() { clear(); }
+
+	void clear() { m_head = 0; m_count = 0; }
+	Int size() const { return m_count; }
+	const T& operator[](Int i) const { return m_items[(m_head + i) % N]; }
+
+	void push(const T& item)
+	{
+		if (m_count == N)
+		{
+			m_head = (m_head + 1) % N;
+			--m_count;
+		}
+		m_items[(m_head + m_count) % N] = item;
+		++m_count;
+	}
+
+	void prune(time_t now, time_t windowMs)
+	{
+		while (m_count > 0 && now - m_items[m_head].time > windowMs)
+		{
+			m_head = (m_head + 1) % N;
+			--m_count;
+		}
+	}
+
+private:
+	T m_items[N];
+	Int m_head;
+	Int m_count;
+};
+
+// Tracks latency, jitter and loss of one connection over a sliding time window.
+class ConnectionStats
+{
+public:
+	ConnectionStats();
+
+	void reset();
+	void noteAcked(Real rttMs, time_t now);	///< A command that was acked on its first send.
+	void noteRetry(time_t now);							///< A command that was not acked in time and had to be sent again.
+	void noteLateAck(time_t now);						///< A retried command was acked by its earlier send after all, the retry was not a loss.
+	void update(time_t now);								///< Drop the samples that have left the window.
+
+	Real getLatency() const { return m_srtt; }
+	Real getMinLatency() const;
+	Real getLatencyPercentile(Int percent, time_t since = 0) const;	///< Round trip that this share of the samples in the window, from since on, stays within
+	Real getJitter() const { return m_jitter; }
+	Real getLossPercent() const;						///< Share of retries the earlier send was not acked for
+	Real getLateAckPercent() const;					///< Share of retries whose earlier send was acked after the retry
+	Real getRetryPercent() const;						///< Share of retries, lost and late acked together
+	Real getLatePercent() const;						///< Share of acks that took more than twice the smoothed round trip time.
+	Int getSampleCount() const { return m_samples.size(); }
+
+private:
+	enum
+	{
+		MAX_SAMPLES = 1024,	///< Covers the window at 60 acked frames per second.
+		MAX_RETRIES = 1024	///< As many as acks, under heavy loss a smaller ring undercounts the retries of the window.
+	};
+
+	struct Sample
+	{
+		time_t time;
+		Real rtt;
+		Bool late;
+	};
+
+	struct Stamp
+	{
+		time_t time;
+	};
+
+	TimedRing<Sample, MAX_SAMPLES> m_samples;
+	TimedRing<Stamp, MAX_RETRIES> m_retries;
+	TimedRing<Stamp, MAX_RETRIES> m_lateAcks;
+
+	Real m_srtt;
+	Real m_jitter;
+	Real m_previousRtt;
+	Bool m_hasRtt;
+};
+
+// Tracks how many frames before its execution frame the frame commands of one sender arrive,
+// over the same sliding window as ConnectionStats. Zero or less means the frame had to wait.
+class ArrivalCushionStats
+{
+public:
+	ArrivalCushionStats();
+
+	void reset();
+	void note(Int cushion, time_t now);
+	void update(time_t now);
+
+	Int getSampleCount() const { return m_samples.size(); }
+	Int getPercentile(Int percent) const;			///< Cushion that percent of the samples are at or above the lowest value of.
+
+private:
+	enum { MAX_SAMPLES = 1024 };
+
+	struct Sample
+	{
+		time_t time;
+		Int cushion;
+	};
+
+	TimedRing<Sample, MAX_SAMPLES> m_samples;
+};
+
+// Frames the game wanted to advance but had to wait for commands, over the same sliding window.
+class StallStats
+{
+public:
+	StallStats();
+
+	void reset();
+	void note(Int durationMs, UnsignedInt missingSlotMask, time_t now);	///< One finished stall and the slots whose commands were missing when it began.
+	void update(time_t now);
+
+	Int getCount() const { return m_events.size(); }
+	Int getTotalMs() const;
+	Int getCountForSlot(Int slot) const;
+
+private:
+	enum { MAX_EVENTS = 512 };
+
+	struct Event
+	{
+		time_t time;
+		Int ms;
+		UnsignedInt slotMask;
+	};
+
+	TimedRing<Event, MAX_EVENTS> m_events;
+};

Core/GameEngine/Include/GameNetwork/FrameMetrics.h

@@ -59,6 +59,9 @@ class FrameMetrics {
 	Real *m_fpsList;								///< A record of how many game logic frames per second there were for the last 60 seconds.
 	time_t m_lastFpsTimeThing;																///< The time when the last fps entry started being recorded.
 	Int m_fpsListIndex;																				///< Index into the array of the current fps list entry being measured.
+#if defined(GENERALS_ONLINE)
+	UnsignedInt m_fpsListLength;																						///< Entries m_fpsList holds, the history length can change after construction.
+#endif
 	Real m_averageFps;																				///< The current average logic fps, computed just like m_averageLatency below but with the fps numbers.
 
 	// round trip time to packet router variables.

Core/GameEngine/Include/GameNetwork/NetCommandRef.h

@@ -62,12 +62,16 @@ class NetCommandRef : public MemoryPoolObject
 	time_t getTimeLastSent() const;
 	void setTimeLastSent(time_t timeLastSent);
 
+	Bool isRetried() const;
+	void setRetried(Bool retried);
+
 protected:
 	NetCommandMsg *m_msg;
 	NetCommandRef *m_next;
 	NetCommandRef *m_prev;
 	UnsignedByte m_relay; ///< Need this in the command reference since the relay value will be different depending on where this particular reference is being sent.
 	time_t m_timeLastSent;
+	Bool m_retried; ///< Set once this command was sent again, so its last send time is not its first.
 
 #ifdef DEBUG_NETCOMMANDREF
 	UnsignedInt m_id;
@@ -138,6 +142,16 @@ inline void NetCommandRef::setTimeLastSent(time_t timeLastSent)
 	m_timeLastSent = timeLastSent;
 }
 
+inline Bool NetCommandRef::isRetried() const
+{
+	return m_retried;
+}
+
+inline void NetCommandRef::setRetried(Bool retried)
+{
+	m_retried = retried;
+}
+
 /**
  * Set the send relay for this reference of the command.
  */

Core/GameEngine/Include/GameNetwork/NetworkDefs.h

@@ -38,7 +38,7 @@ static constexpr const Int MIN_LOGIC_FRAMES = 5;
 static constexpr const Int MAX_FRAMES_AHEAD = 128;
 
 #if defined(GENERALS_ONLINE)
-static Int MIN_RUNAHEAD = 6;
+extern Int MIN_RUNAHEAD;
 #else
 static constexpr const Int MIN_RUNAHEAD = 4;
 #endif
@@ -56,6 +56,9 @@ enum ConnectionNumbers CPP_11(: Int)
 
 static constexpr const Int MAX_SLOTS = MAX_PLAYER + 1;
 
+// Sliding window over which connection latency, jitter, loss and cushion are evaluated.
+static constexpr const Int NETWORK_STATS_WINDOW_MS = 10000;
+
 #pragma pack(push, 1)
 struct TransportMessageHeader
 {
@@ -207,7 +210,12 @@ enum PlayerLeaveCode CPP_11(: Int) {
 };
 
 // Magic number for identifying a Generals packet.
+#if defined(GENERALS_ONLINE)
+// 0xF00E: packets carry the sequence envelope; builds without it fail on the magic number
+static constexpr const UnsignedShort GENERALS_MAGIC_NUMBER = 0xF00E;
+#else
 static constexpr const UnsignedShort GENERALS_MAGIC_NUMBER = 0xF00D;
+#endif
 
 // The number of fps history entries.
 //static constexpr const Int NETWORK_FPS_HISTORY_LENGTH = 30;

Core/GameEngine/Include/GameNetwork/NetworkInterface.h

@@ -100,6 +100,7 @@ class NetworkInterface : public SubsystemInterface
 
 	virtual Int getAverageFPS() = 0;
 	virtual Int getSlotAverageFPS(Int slot) = 0;
+	virtual const ConnectionStats* getConnectionStats(Int slot) const = 0;	///< Latency, jitter and loss of the connection to a slot, nullptr when there is none.
 
 #if defined(GENERALS_ONLINE)
 	virtual void SeedLatencyData(int highestLatency) = 0;

Core/GameEngine/Include/GameNetwork/NetworkSettings.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#if defined(GENERALS_ONLINE)
+
+#include <string>
+
+// Per-match game networking policy, independent of online-service lifetime.
+struct NetworkSettings
+{
+	struct Send
+	{
+		int frames_per_packet = 2;      // send interval in frames
+		bool flush_every_frame = true;  // flush GNS after each frame
+	} send;
+	struct InputDelay
+	{
+		int min_frames = 4;
+		int decision_interval_ms = 5000;
+		int fps_history_samples = 30;   // per-second FPS samples averaged
+		int slack_percent = 30;         // percent added to the measured round trip, also sets the start run ahead and the rating
+
+		struct Formula
+		{
+			int latency_history_samples = 200; // in-band ping samples averaged
+		} formula;
+	} input_delay;
+
+	static NetworkSettings LocalDefaults()
+	{
+		return NetworkSettings();
+	}
+};
+
+#endif