diff --git a/Core/GameEngine/CMakeLists.txt b/Core/GameEngine/CMakeLists.txt index e6aee6da85f..0b0cc58d095 100644 --- a/Core/GameEngine/CMakeLists.txt +++ b/Core/GameEngine/CMakeLists.txt @@ -505,6 +505,7 @@ set(GAMEENGINE_SRC # Include/GameLogic/WeaponStatus.h Include/GameNetwork/Connection.h Include/GameNetwork/ConnectionManager.h + Include/GameNetwork/ConnectionStats.h Include/GameNetwork/DisconnectManager.h Include/GameNetwork/DownloadManager.h Include/GameNetwork/FileTransfer.h @@ -1099,6 +1100,7 @@ set(GAMEENGINE_SRC Source/GameLogic/System/RankInfo.cpp Source/GameNetwork/Connection.cpp Source/GameNetwork/ConnectionManager.cpp + Source/GameNetwork/ConnectionStats.cpp Source/GameNetwork/DisconnectManager.cpp Source/GameNetwork/DownloadManager.cpp Source/GameNetwork/FileTransfer.cpp diff --git a/Core/GameEngine/Include/GameNetwork/Connection.h b/Core/GameEngine/Include/GameNetwork/Connection.h index 9a26a5fee63..0c77332ab1b 100644 --- a/Core/GameEngine/Include/GameNetwork/Connection.h +++ b/Core/GameEngine/Include/GameNetwork/Connection.h @@ -39,12 +39,7 @@ #include "GameNetwork/User.h" #include "GameNetwork/Transport.h" #include "GameNetwork/NetPacket.h" - -#if defined(GENERALS_ONLINE) -#define CONNECTION_LATENCY_HISTORY_LENGTH 10 -#else -#define CONNECTION_LATENCY_HISTORY_LENGTH 200 -#endif +#include "GameNetwork/ConnectionStats.h" class Connection : public MemoryPoolObject { @@ -80,13 +75,13 @@ class Connection : public MemoryPoolObject void setQuitting(); Bool isQuitting() { return m_isQuitting; } + const ConnectionStats& getStats() const { return m_stats; } + #if defined(RTS_DEBUG) void debugPrintCommands(); #endif protected: - void doRetryMetrics(); - Bool m_isQuitting; UnsignedInt m_quitTime; @@ -95,11 +90,8 @@ class Connection : public MemoryPoolObject NetCommandList *m_netCommandList; time_t m_retryTime; ///< The time between sending retry packets for this connection. Time is in milliseconds. - Real m_averageLatency; ///< The average time between sending a command and receiving an ACK. - Real m_latencies[CONNECTION_LATENCY_HISTORY_LENGTH]; ///< List of the last 100 latencies. + ConnectionStats m_stats; ///< Latency, jitter and loss of this connection. time_t m_frameGrouping; ///< The minimum time between packet sends. time_t m_lastTimeSent; ///< The time of the last packet send. - Int m_numRetries; ///< The number of retries for the last second. - time_t m_retryMetricsTime; ///< The start time of the current retry metrics thing. }; diff --git a/Core/GameEngine/Include/GameNetwork/ConnectionManager.h b/Core/GameEngine/Include/GameNetwork/ConnectionManager.h index 4e1693455b5..5812389f3f0 100644 --- a/Core/GameEngine/Include/GameNetwork/ConnectionManager.h +++ b/Core/GameEngine/Include/GameNetwork/ConnectionManager.h @@ -37,6 +37,9 @@ #include "GameNetwork/FrameMetrics.h" #include "GameNetwork/NetworkDefs.h" #include "GameNetwork/DisconnectManager.h" +#if defined(GENERALS_ONLINE) +#include "GameNetwork/NetworkSettings.h" +#endif class GameInfo; class NetCommandWrapperList; @@ -56,6 +59,7 @@ class ConnectionManager virtual void update(Bool isInGame); ///< Service the Connections being managed by this instance. #if defined(GENERALS_ONLINE) + const NetworkSettings &getNetworkSettings() const { return m_networkSettings; } void SeedLatencyData(int highestLatency) { for (int i = 0; i < MAX_SLOTS; ++i) { @@ -154,6 +158,10 @@ class ConnectionManager Int getAverageFPS(); Int getSlotAverageFPS(Int slot); +#if defined(GENERALS_ONLINE) + void noteStalling(Bool stalling, UnsignedInt frame); ///< Called every network update with whether the game is waiting for commands. +#endif + const ConnectionStats* getConnectionStats(Int slot) const; ///< Returns nullptr when there is no connection for the slot. #if defined(RTS_DEBUG) void debugPrintConnectionCommands(); @@ -165,6 +173,10 @@ class ConnectionManager Int getPingsReceived(); private: +#if defined(GENERALS_ONLINE) + NetworkSettings m_networkSettings; +#endif + void doRelay(); void doKeepAlive(); void sendRemoteCommand(NetCommandRef *msg); @@ -226,6 +238,19 @@ class ConnectionManager UnsignedInt m_smallestPacketArrivalCushion; Bool m_didSelfSlug; +#if defined(GENERALS_ONLINE) + ArrivalCushionStats m_cushionStats[MAX_SLOTS]; + StallStats m_stallStats; + Bool m_stallActive; + time_t m_stallStartTime; + UnsignedInt m_stallMissingMask; + Int m_lastKnownRunAhead; + Int m_frameGroupingMs; +public: + Bool getOverlayInfo(Int slot, NetworkOverlayInfo &info); ///< Fills what the overlay and the connection rating need about one slot. +protected: +#endif + // ----------------------------------------------------------------------------- FileCommandMap s_fileCommandMap; FileMaskMap s_fileRecipientMaskMap; diff --git a/Core/GameEngine/Include/GameNetwork/ConnectionStats.h b/Core/GameEngine/Include/GameNetwork/ConnectionStats.h new file mode 100644 index 00000000000..8e3789a1a43 --- /dev/null +++ b/Core/GameEngine/Include/GameNetwork/ConnectionStats.h @@ -0,0 +1,182 @@ +/* +** Command & Conquer Generals Zero Hour(tm) +** Copyright 2025 Electronic Arts Inc. +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation, either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program. If not, see . +*/ + +#pragma once + +#include + +// Fixed size ring of items with a time member, oldest first, that drops the items older than a window. +template +class TimedRing +{ +public: + TimedRing() { clear(); } + + void clear() { m_head = 0; m_count = 0; } + Int size() const { return m_count; } + const T& operator[](Int i) const { return m_items[(m_head + i) % N]; } + + void push(const T& item) + { + if (m_count == N) + { + m_head = (m_head + 1) % N; + --m_count; + } + m_items[(m_head + m_count) % N] = item; + ++m_count; + } + + void prune(time_t now, time_t windowMs) + { + while (m_count > 0 && now - m_items[m_head].time > windowMs) + { + m_head = (m_head + 1) % N; + --m_count; + } + } + +private: + T m_items[N]; + Int m_head; + Int m_count; +}; + +// Tracks latency, jitter and loss of one connection over a sliding time window. +class ConnectionStats +{ +public: + ConnectionStats(); + + void reset(); + void noteAcked(Real rttMs, time_t now); ///< A command that was acked on its first send. + void noteRetry(time_t now); ///< A command that was not acked in time and had to be sent again. + void noteLateAck(time_t now); ///< A retried command was acked by its earlier send after all, the retry was not a loss. + void update(time_t now); ///< Drop the samples that have left the window. + + Real getLatency() const { return m_srtt; } + Real getMinLatency() const; + Real getLatencyPercentile(Int percent, time_t since = 0) const; ///< Round trip that this share of the samples in the window, from since on, stays within + Real getJitter() const { return m_jitter; } + Real getLossPercent() const; ///< Share of retries the earlier send was not acked for + Real getLateAckPercent() const; ///< Share of retries whose earlier send was acked after the retry + Real getRetryPercent() const; ///< Share of retries, lost and late acked together + Real getLatePercent() const; ///< Share of acks that took more than twice the smoothed round trip time. + Int getSampleCount() const { return m_samples.size(); } + +private: + enum + { + MAX_SAMPLES = 1024, ///< Covers the window at 60 acked frames per second. + MAX_RETRIES = 1024 ///< As many as acks, under heavy loss a smaller ring undercounts the retries of the window. + }; + + struct Sample + { + time_t time; + Real rtt; + Bool late; + }; + + struct Stamp + { + time_t time; + }; + + TimedRing m_samples; + TimedRing m_retries; + TimedRing m_lateAcks; + + Real m_srtt; + Real m_jitter; + Real m_previousRtt; + Bool m_hasRtt; +}; + +// Tracks how many frames before its execution frame the frame commands of one sender arrive, +// over the same sliding window as ConnectionStats. Zero or less means the frame had to wait. +class ArrivalCushionStats +{ +public: + ArrivalCushionStats(); + + void reset(); + void note(Int cushion, time_t now); + void update(time_t now); + + Int getSampleCount() const { return m_samples.size(); } + Int getPercentile(Int percent) const; ///< Cushion that percent of the samples are at or above the lowest value of. + +private: + enum { MAX_SAMPLES = 1024 }; + + struct Sample + { + time_t time; + Int cushion; + }; + + TimedRing m_samples; +}; + +// Frames the game wanted to advance but had to wait for commands, kept for a minute and counted over a window. +class StallStats +{ +public: + StallStats(); + + void reset(); + void note(Int durationMs, UnsignedInt missingSlotMask, time_t now); ///< One finished stall and the slots whose commands were missing when it began. + void update(time_t now); + + Int getCount(time_t now, Int windowMs) const; + Int getCountForSlot(Int slot, time_t now, Int windowMs) const; + +private: + enum { MAX_EVENTS = 512, HISTORY_MS = 60000 }; + + struct Event + { + time_t time; + Int ms; + UnsignedInt slotMask; + }; + + TimedRing m_events; +}; + +// What the network overlay and the connection rating need to know about one slot beyond its ConnectionStats. +struct NetworkOverlayInfo +{ + NetworkOverlayInfo() + : cushionSamples(0), cushionP5(0), cushionP50(0) + , stallsWindow(0), stallsMinute(0), stalledNow(FALSE), waitingOnSlot(FALSE) + , slotInBandMs(-1), runAhead(0), frameGroupingMs(0) {} + + Int cushionSamples; ///< Frames before execution that the frame commands of this slot arrived, over the stats window. + Int cushionP5; + Int cushionP50; + Int stallsWindow; ///< Stalls the local game had to wait on this slot for, over the stats window. + Int stallsMinute; ///< The same over the last minute. + Bool stalledNow; ///< The local game is waiting for commands right now. + Bool waitingOnSlot; ///< ... and this slot is one it is waiting on. + Int slotInBandMs; ///< In-band round trip this slot contributes to the run ahead, -1 if unknown. + + Int runAhead; ///< Frames. + Int frameGroupingMs; ///< The send interval. +}; diff --git a/Core/GameEngine/Include/GameNetwork/FrameMetrics.h b/Core/GameEngine/Include/GameNetwork/FrameMetrics.h index 13967e49164..fdf4edca419 100644 --- a/Core/GameEngine/Include/GameNetwork/FrameMetrics.h +++ b/Core/GameEngine/Include/GameNetwork/FrameMetrics.h @@ -59,6 +59,9 @@ class FrameMetrics { Real *m_fpsList; ///< A record of how many game logic frames per second there were for the last 60 seconds. time_t m_lastFpsTimeThing; ///< The time when the last fps entry started being recorded. Int m_fpsListIndex; ///< Index into the array of the current fps list entry being measured. +#if defined(GENERALS_ONLINE) + UnsignedInt m_fpsListLength; ///< Entries m_fpsList holds, the history length can change after construction. +#endif Real m_averageFps; ///< The current average logic fps, computed just like m_averageLatency below but with the fps numbers. // round trip time to packet router variables. diff --git a/Core/GameEngine/Include/GameNetwork/NetCommandRef.h b/Core/GameEngine/Include/GameNetwork/NetCommandRef.h index 4a3ae4226ca..86603e10f3e 100644 --- a/Core/GameEngine/Include/GameNetwork/NetCommandRef.h +++ b/Core/GameEngine/Include/GameNetwork/NetCommandRef.h @@ -62,12 +62,16 @@ class NetCommandRef : public MemoryPoolObject time_t getTimeLastSent() const; void setTimeLastSent(time_t timeLastSent); + Bool isRetried() const; + void setRetried(Bool retried); + protected: NetCommandMsg *m_msg; NetCommandRef *m_next; NetCommandRef *m_prev; UnsignedByte m_relay; ///< Need this in the command reference since the relay value will be different depending on where this particular reference is being sent. time_t m_timeLastSent; + Bool m_retried; ///< Set once this command was sent again, so its last send time is not its first. #ifdef DEBUG_NETCOMMANDREF UnsignedInt m_id; @@ -138,6 +142,16 @@ inline void NetCommandRef::setTimeLastSent(time_t timeLastSent) m_timeLastSent = timeLastSent; } +inline Bool NetCommandRef::isRetried() const +{ + return m_retried; +} + +inline void NetCommandRef::setRetried(Bool retried) +{ + m_retried = retried; +} + /** * Set the send relay for this reference of the command. */ diff --git a/Core/GameEngine/Include/GameNetwork/NetworkDefs.h b/Core/GameEngine/Include/GameNetwork/NetworkDefs.h index 6295493ea7d..af159f71874 100644 --- a/Core/GameEngine/Include/GameNetwork/NetworkDefs.h +++ b/Core/GameEngine/Include/GameNetwork/NetworkDefs.h @@ -38,7 +38,7 @@ static constexpr const Int MIN_LOGIC_FRAMES = 5; static constexpr const Int MAX_FRAMES_AHEAD = 128; #if defined(GENERALS_ONLINE) -static Int MIN_RUNAHEAD = 6; +extern Int MIN_RUNAHEAD; #else static constexpr const Int MIN_RUNAHEAD = 4; #endif @@ -56,6 +56,9 @@ enum ConnectionNumbers CPP_11(: Int) static constexpr const Int MAX_SLOTS = MAX_PLAYER + 1; +// Sliding window over which connection latency, jitter, loss and cushion are evaluated. +static constexpr const Int NETWORK_STATS_WINDOW_MS = 10000; + #pragma pack(push, 1) struct TransportMessageHeader { @@ -207,7 +210,12 @@ enum PlayerLeaveCode CPP_11(: Int) { }; // Magic number for identifying a Generals packet. +#if defined(GENERALS_ONLINE) +// 0xF00E: packets carry the sequence envelope; builds without it fail on the magic number +static constexpr const UnsignedShort GENERALS_MAGIC_NUMBER = 0xF00E; +#else static constexpr const UnsignedShort GENERALS_MAGIC_NUMBER = 0xF00D; +#endif // The number of fps history entries. //static constexpr const Int NETWORK_FPS_HISTORY_LENGTH = 30; diff --git a/Core/GameEngine/Include/GameNetwork/NetworkInterface.h b/Core/GameEngine/Include/GameNetwork/NetworkInterface.h index 606429450dd..2e28de9e1d2 100644 --- a/Core/GameEngine/Include/GameNetwork/NetworkInterface.h +++ b/Core/GameEngine/Include/GameNetwork/NetworkInterface.h @@ -100,8 +100,10 @@ class NetworkInterface : public SubsystemInterface virtual Int getAverageFPS() = 0; virtual Int getSlotAverageFPS(Int slot) = 0; + virtual const ConnectionStats* getConnectionStats(Int slot) const = 0; ///< Latency, jitter and loss of the connection to a slot, nullptr when there is none. #if defined(GENERALS_ONLINE) + virtual Bool getOverlayInfo(Int slot, NetworkOverlayInfo &info) = 0; virtual void SeedLatencyData(int highestLatency) = 0; virtual bool IsSlugging() = 0; #endif diff --git a/Core/GameEngine/Include/GameNetwork/NetworkSettings.h b/Core/GameEngine/Include/GameNetwork/NetworkSettings.h new file mode 100644 index 00000000000..af102d9109d --- /dev/null +++ b/Core/GameEngine/Include/GameNetwork/NetworkSettings.h @@ -0,0 +1,34 @@ +#pragma once + +#if defined(GENERALS_ONLINE) + +#include + +// Per-match game networking policy, independent of online-service lifetime. +struct NetworkSettings +{ + struct Send + { + int frames_per_packet = 2; // send interval in frames + bool flush_every_frame = true; // flush GNS after each frame + } send; + struct InputDelay + { + int min_frames = 4; + int decision_interval_ms = 5000; + int fps_history_samples = 30; // per-second FPS samples averaged + int slack_percent = 30; // percent added to the measured round trip, also sets the start run ahead and the rating + + struct Formula + { + int latency_history_samples = 200; // in-band ping samples averaged + } formula; + } input_delay; + + static NetworkSettings LocalDefaults() + { + return NetworkSettings(); + } +}; + +#endif diff --git a/Core/GameEngine/Source/GameClient/MessageStream/CommandXlat.cpp b/Core/GameEngine/Source/GameClient/MessageStream/CommandXlat.cpp index 231a6f355b4..adb5bb72c8d 100644 --- a/Core/GameEngine/Source/GameClient/MessageStream/CommandXlat.cpp +++ b/Core/GameEngine/Source/GameClient/MessageStream/CommandXlat.cpp @@ -54,6 +54,7 @@ #include "Common/OptionPreferences.h" #include "GameClient/InGameUI.h" +#include "GameClient/Keyboard.h" #include "GameClient/CommandXlat.h" #include "GameClient/DebugDisplay.h" #include "GameClient/Drawable.h" @@ -3936,7 +3937,7 @@ GameMessageDisposition CommandTranslator::translateGameMessage(const GameMessage } else if (key == KEY_F10) { - NGMP_OnlineServicesManager::ToggleAdvancedNetworkStats(); + NGMP_OnlineServicesManager::ToggleNetworkOverlay((TheKeyboard != nullptr && TheKeyboard->isShift()) ? 2 : 1); disp = DESTROY_MESSAGE; } diff --git a/Core/GameEngine/Source/GameNetwork/Connection.cpp b/Core/GameEngine/Source/GameNetwork/Connection.cpp index 8ba8a3e2dcd..2831f12c436 100644 --- a/Core/GameEngine/Source/GameNetwork/Connection.cpp +++ b/Core/GameEngine/Source/GameNetwork/Connection.cpp @@ -48,12 +48,6 @@ Connection::Connection() { m_frameGrouping = 1; m_isQuitting = false; m_quitTime = 0; - m_averageLatency = 0.0f; - Int i; - for(i = 0; i < CONNECTION_LATENCY_HISTORY_LENGTH; i++) - { - m_latencies[i] = 0.0f; - } } /** @@ -84,13 +78,7 @@ void Connection::init() { m_lastTimeSent = 0; m_frameGrouping = 1; - m_numRetries = 0; - m_retryMetricsTime = 0; - - for (Int i = 0; i < CONNECTION_LATENCY_HISTORY_LENGTH; ++i) { - m_latencies[i] = 0; - } - m_averageLatency = 0; + m_stats.reset(); m_isQuitting = FALSE; m_quitTime = 0; } @@ -261,6 +249,8 @@ UnsignedInt Connection::doSend() { time_t curtime = timeGetTime(); Bool couldQueue = TRUE; + m_stats.update(curtime); + // Do this check first, since it's an important fail-safe if (m_isQuitting && curtime > m_quitTime + MaxQuitFlushTime) { @@ -271,7 +261,7 @@ UnsignedInt Connection::doSend() { if ((curtime - m_lastTimeSent) < m_frameGrouping) { // DEBUG_LOG(("not sending packet, time = %d, m_lastFrameSent = %d, m_frameGrouping = %d", curtime, m_lastTimeSent, m_frameGrouping)); - NetworkLog(ELogVerbosity::LOG_DEBUG, "not sending packet, time = %d, m_lastFrameSent = %d, m_frameGrouping = %d", curtime, m_lastTimeSent, m_frameGrouping); + NetworkLog(ELogVerbosity::LOG_DEBUG, "not sending packet, time = %I64d, m_lastFrameSent = %I64d, m_frameGrouping = %I64d", (Int64)curtime, (Int64)m_lastTimeSent, (Int64)m_frameGrouping); return 0; } @@ -297,9 +287,9 @@ UnsignedInt Connection::doSend() { // the msg command was added to the packet. if (CommandRequiresAck(msg->getCommand())) { if (timeLastSent != -1) { - ++m_numRetries; + m_stats.noteRetry(curtime); + msg->setRetried(TRUE); } - doRetryMetrics(); msg->setTimeLastSent(curtime); } else { m_netCommandList->removeMessage(msg); @@ -376,13 +366,16 @@ NetCommandRef * Connection::processAck(UnsignedShort commandID, UnsignedByte ori } #endif - Int index = temp->getCommand()->getID() % CONNECTION_LATENCY_HISTORY_LENGTH; - m_averageLatency -= ((Real)(m_latencies[index])) / CONNECTION_LATENCY_HISTORY_LENGTH; - Real lat = timeGetTime() - temp->getTimeLastSent(); - - NetworkLog(ELogVerbosity::LOG_DEBUG, "Latency calc is %f, avg is %f", lat, m_averageLatency); - m_averageLatency += lat / CONNECTION_LATENCY_HISTORY_LENGTH; - m_latencies[index] = lat; + // A retried command was last sent after its first send, so its round trip time would be too short. + const time_t now = timeGetTime(); + if (temp->isRetried()) { + // an ack sooner after the retry than any round trip so far answers the earlier send: the retry went out for a late ack, not a loss + if ((m_stats.getSampleCount() > 0) && ((now - temp->getTimeLastSent()) < (time_t)(0.9f * m_stats.getMinLatency()))) { + m_stats.noteLateAck(now); + } + } else { + m_stats.noteAcked((Real)(now - temp->getTimeLastSent()), now); + } #if defined(RTS_DEBUG) if (doDebug == TRUE) { @@ -398,19 +391,6 @@ void Connection::setFrameGrouping(time_t frameGrouping) { // m_retryTime = frameGrouping * 4; } -void Connection::doRetryMetrics() { - static Int numSeconds = 0; - time_t curTime = timeGetTime(); - - if ((curTime - m_retryMetricsTime) > 10000) { - m_retryMetricsTime = curTime; - ++numSeconds; -// DEBUG_LOG(("Retries in the last 10 seconds = %d, average latency = %fms", m_numRetries, m_averageLatency)); - m_numRetries = 0; -// m_retryTime = m_averageLatency * 1.5; - } -} - #if defined(RTS_DEBUG) void Connection::debugPrintCommands() { NetCommandRef *ref = m_netCommandList->getFirstMessage(); diff --git a/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp b/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp index f347db5a83f..abe73435b7a 100644 --- a/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp +++ b/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp @@ -59,6 +59,10 @@ #include "../NetworkMesh.h" #include "../ngmp_interfaces.h" +#if defined(GENERALS_ONLINE) +Int MIN_RUNAHEAD = 4; +#endif + static Bool hasValidTransferFileExtension(const AsciiString& filePath) { static const char* const validExtensions[] = { @@ -272,6 +276,23 @@ ConnectionManager::ConnectionManager() */ void ConnectionManager::init() { +#if defined(GENERALS_ONLINE) + m_networkSettings = NetworkSettings::LocalDefaults(); + NGMP_OnlineServicesManager *services = NGMP_OnlineServicesManager::GetInstance(); + if (TheLAN == nullptr && TheNGMPGame != nullptr && services != nullptr) { + const ServiceConfig &overrides = services->GetServiceConfig(); + m_networkSettings.send = overrides.send; + m_networkSettings.input_delay = overrides.input_delay; + } + const NetworkSettings::InputDelay &runAhead = m_networkSettings.input_delay; + TheWritableGlobalData->m_networkFPSHistoryLength = (std::max)(1, runAhead.fps_history_samples); + TheWritableGlobalData->m_networkLatencyHistoryLength = (std::max)(1, runAhead.formula.latency_history_samples); + TheWritableGlobalData->m_networkRunAheadSlack = runAhead.slack_percent; + TheWritableGlobalData->m_networkRunAheadMetricsTime = runAhead.decision_interval_ms; + MIN_RUNAHEAD = runAhead.min_frames; + FRAME_GROUPING_CAP = (int)(m_networkSettings.send.frames_per_packet * (float)(1000.f / GENERALS_ONLINE_HIGH_FPS_LIMIT)); +#endif + // if (m_transport == nullptr) { // m_transport = new Transport; // } @@ -318,6 +339,13 @@ void ConnectionManager::init() m_latencyAverages[i] = 0.0; // using zero since all floating point standards should be able to specify 0.0 accurately. } m_smallestPacketArrivalCushion = -1; +#if defined(GENERALS_ONLINE) + m_stallActive = FALSE; + m_stallStartTime = 0; + m_stallMissingMask = 0; + m_lastKnownRunAhead = 0; + m_frameGroupingMs = 0; +#endif m_frameMetrics.init(); @@ -389,10 +417,10 @@ void ConnectionManager::reset() #endif m_packetRouterSlot = -1; - for (i = 0; i < TheGlobalData->m_networkFPSHistoryLength; ++i) { + for (i = 0; i < (UnsignedInt)MAX_SLOTS; ++i) { m_fpsAverages[i] = -1; } - for (i = 0; i < TheGlobalData->m_networkLatencyHistoryLength; ++i) { + for (i = 0; i < (UnsignedInt)MAX_SLOTS; ++i) { m_latencyAverages[i] = 0.0; } @@ -981,6 +1009,11 @@ void ConnectionManager::processFrameInfo(NetFrameCommandMsg *msg) { if (m_frameData[playerID] != nullptr) { // DEBUG_LOG(("ConnectionManager::processFrameInfo - player %d, frame %d, command count %d, received on frame %d", playerID, msg->getExecutionFrame(), msg->getCommandCount(), TheGameLogic->getFrame())); m_frameData[playerID]->setFrameCommandCount(msg->getExecutionFrame(), msg->getCommandCount()); +#if defined(GENERALS_ONLINE) + if ((playerID != (UnsignedInt)m_localSlot) && (TheGameLogic != nullptr)) { + m_cushionStats[playerID].note((Int)msg->getExecutionFrame() - (Int)TheGameLogic->getFrame(), timeGetTime()); + } +#endif } } } @@ -1381,6 +1414,10 @@ void ConnectionManager::updateRunAhead(Int oldRunAhead, Int frameRate, Bool didS static time_t lasttimesent = 0; time_t curTime = timeGetTime(); +#if defined(GENERALS_ONLINE) + m_lastKnownRunAhead = oldRunAhead; +#endif + if ((lasttimesent == 0) || ((curTime - lasttimesent) > TheGlobalData->m_networkRunAheadMetricsTime)) { if (m_localSlot == m_packetRouterSlot) { // We are the packet router, time to compute a new run ahead for this game. @@ -1995,11 +2032,14 @@ void ConnectionManager::setFrameGrouping(time_t frameGrouping) { // This is probably ok since the packet router should have the fastest connection of all // the players in the game. - NetworkLog(ELogVerbosity::LOG_DEBUG, "ConnectionManager::setFrameGrouping Frame grouping is %d, is packet router %d", frameGrouping, m_localSlot == m_packetRouterSlot); + NetworkLog(ELogVerbosity::LOG_DEBUG, "ConnectionManager::setFrameGrouping Frame grouping is %I64d, is packet router %d", (Int64)frameGrouping, m_localSlot == m_packetRouterSlot); if (m_localSlot == m_packetRouterSlot) { frameGrouping = frameGrouping / 2; } +#if defined(GENERALS_ONLINE) + m_frameGroupingMs = (Int)frameGrouping; +#endif for (Int i = 0; i < MAX_SLOTS; ++i) { if (m_connections[i] != nullptr) { m_connections[i]->setFrameGrouping(frameGrouping); @@ -2721,6 +2761,13 @@ Int ConnectionManager::getSlotAverageFPS(Int slot) { return m_fpsAverages[slot]; } +const ConnectionStats* ConnectionManager::getConnectionStats(Int slot) const { + if ((slot < 0) || (slot >= MAX_SLOTS) || (m_connections[slot] == nullptr)) { + return nullptr; + } + return &m_connections[slot]->getStats(); +} + #if defined(RTS_DEBUG) void ConnectionManager::debugPrintConnectionCommands() { DEBUG_LOG_LEVEL(DEBUG_LEVEL_NET, ("ConnectionManager::debugPrintConnectionCommands - begin commands")); @@ -2785,7 +2832,7 @@ void ConnectionManager::sendFrameDataToPlayer(UnsignedInt playerID, UnsignedInt } void ConnectionManager::sendSingleFrameToPlayer(UnsignedInt playerID, UnsignedInt frame) { - if ((TheGameLogic->getFrame() - FRAMES_TO_KEEP) > frame) { + if (frame + FRAMES_TO_KEEP < TheGameLogic->getFrame()) { DEBUG_LOG_LEVEL(DEBUG_LEVEL_NET, ("ConnectionManager::sendSingleFrameToPlayer - player %d requested frame %d when we are on frame %d, this is too far in the past.", playerID, frame, TheGameLogic->getFrame())); return; } @@ -2852,3 +2899,49 @@ void ConnectionManager::requestFrameDataResend(Int playerID, UnsignedInt frame) msg->detach(); } + +#if defined(GENERALS_ONLINE) +void ConnectionManager::noteStalling(Bool stalling, UnsignedInt frame) { + time_t now = timeGetTime(); + + if (stalling && !m_stallActive) { + // list the senders whose commands for this frame are missing + m_stallMissingMask = 0; + for (Int i = 0; i < MAX_SLOTS; ++i) { + if ((m_frameData[i] != nullptr) && (m_frameData[i]->getIsQuitting() == FALSE) + && (m_frameData[i]->getFrameCommandCount(frame) != m_frameData[i]->getCommandCount(frame))) { + m_stallMissingMask |= (1u << i); + } + } + m_stallActive = TRUE; + m_stallStartTime = now; + } else if (!stalling && m_stallActive) { + m_stallActive = FALSE; + m_stallStats.note((Int)(now - m_stallStartTime), m_stallMissingMask, now); + } +} + +Bool ConnectionManager::getOverlayInfo(Int slot, NetworkOverlayInfo &info) { + time_t now = timeGetTime(); + m_stallStats.update(now); + + info = NetworkOverlayInfo(); + info.stalledNow = m_stallActive; + info.runAhead = m_lastKnownRunAhead; + info.frameGroupingMs = m_frameGroupingMs; + + if (slot >= 0 && slot < MAX_SLOTS) { + info.stallsWindow = m_stallStats.getCountForSlot(slot, now, NETWORK_STATS_WINDOW_MS); + info.stallsMinute = m_stallStats.getCountForSlot(slot, now, 60000); + info.waitingOnSlot = m_stallActive && ((m_stallMissingMask & (1u << slot)) != 0); + if (isPlayerConnected(slot)) { + info.slotInBandMs = (Int)(m_latencyAverages[slot] * 1000.0f); + } + m_cushionStats[slot].update(now); + info.cushionSamples = m_cushionStats[slot].getSampleCount(); + info.cushionP5 = m_cushionStats[slot].getPercentile(5); + info.cushionP50 = m_cushionStats[slot].getPercentile(50); + } + return TRUE; +} +#endif diff --git a/Core/GameEngine/Source/GameNetwork/ConnectionStats.cpp b/Core/GameEngine/Source/GameNetwork/ConnectionStats.cpp new file mode 100644 index 00000000000..08db5856949 --- /dev/null +++ b/Core/GameEngine/Source/GameNetwork/ConnectionStats.cpp @@ -0,0 +1,319 @@ +/* +** Command & Conquer Generals Zero Hour(tm) +** Copyright 2025 Electronic Arts Inc. +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation, either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program. If not, see . +*/ + +#include "PreRTS.h" // This must go first in EVERY cpp file in the GameEngine + +#include "GameNetwork/ConnectionStats.h" +#include "GameNetwork/NetworkDefs.h" + +#include + +ConnectionStats::ConnectionStats() +{ + reset(); +} + +void ConnectionStats::reset() +{ + m_samples.clear(); + m_retries.clear(); + m_lateAcks.clear(); + m_srtt = 0.0f; + m_jitter = 0.0f; + m_previousRtt = 0.0f; + m_hasRtt = FALSE; +} + +void ConnectionStats::noteAcked(Real rttMs, time_t now) +{ + if (rttMs < 0.0f) + { + rttMs = 0.0f; + } + + Bool late = FALSE; + if (m_hasRtt) + { + // RFC 3550 interarrival jitter and the TCP smoothed round trip time. + const Real delta = rttMs - m_previousRtt; + const Real deviation = delta < 0.0f ? -delta : delta; + // late = more than 2 x srtt, floored at 20 ms + const Real lateMargin = m_srtt > 20.0f ? m_srtt : 20.0f; + late = (rttMs - m_srtt) > lateMargin; + m_jitter += (deviation - m_jitter) / 16.0f; + m_srtt += (rttMs - m_srtt) / 8.0f; + } + else + { + m_srtt = rttMs; + m_hasRtt = TRUE; + } + m_previousRtt = rttMs; + + Sample sample; + sample.time = now; + sample.rtt = rttMs; + sample.late = late; + m_samples.push(sample); +} + +void ConnectionStats::noteRetry(time_t now) +{ + Stamp stamp; + stamp.time = now; + m_retries.push(stamp); +} + +void ConnectionStats::noteLateAck(time_t now) +{ + Stamp stamp; + stamp.time = now; + m_lateAcks.push(stamp); +} + +void ConnectionStats::update(time_t now) +{ + m_samples.prune(now, NETWORK_STATS_WINDOW_MS); + m_retries.prune(now, NETWORK_STATS_WINDOW_MS); + m_lateAcks.prune(now, NETWORK_STATS_WINDOW_MS); +} + +Real ConnectionStats::getMinLatency() const +{ + if (m_samples.size() == 0) + { + return 0.0f; + } + + Real result = m_samples[0].rtt; + for (Int i = 1; i < m_samples.size(); ++i) + { + if (m_samples[i].rtt < result) + { + result = m_samples[i].rtt; + } + } + return result; +} + +Real ConnectionStats::getLatencyPercentile(Int percent, time_t since) const +{ + Real sorted[MAX_SAMPLES]; + Int count = 0; + for (Int i = 0; i < m_samples.size(); ++i) + { + if (m_samples[i].time >= since) + { + sorted[count] = m_samples[i].rtt; + ++count; + } + } + if (count == 0) + { + return 0.0f; + } + std::sort(sorted, sorted + count); + + Int index = (count - 1) * percent / 100; + if (index < 0) + { + index = 0; + } + if (index >= count) + { + index = count - 1; + } + return sorted[index]; +} + +Real ConnectionStats::getRetryPercent() const +{ + const Int total = m_samples.size() + m_retries.size(); + if (total == 0) + { + return 0.0f; + } + + return 100.0f * m_retries.size() / total; +} + +Real ConnectionStats::getLateAckPercent() const +{ + const Int total = m_samples.size() + m_retries.size(); + if (total == 0) + { + return 0.0f; + } + + const Int lateAcks = m_lateAcks.size() < m_retries.size() ? m_lateAcks.size() : m_retries.size(); + return 100.0f * lateAcks / total; +} + +Real ConnectionStats::getLossPercent() const +{ + const Int total = m_samples.size() + m_retries.size(); + if (total == 0) + { + return 0.0f; + } + + const Int lost = m_retries.size() > m_lateAcks.size() ? m_retries.size() - m_lateAcks.size() : 0; + return 100.0f * lost / total; +} + +Real ConnectionStats::getLatePercent() const +{ + if (m_samples.size() == 0) + { + return 0.0f; + } + + Int lateCount = 0; + for (Int i = 0; i < m_samples.size(); ++i) + { + if (m_samples[i].late) + { + ++lateCount; + } + } + return 100.0f * lateCount / m_samples.size(); +} + +ArrivalCushionStats::ArrivalCushionStats() +{ + reset(); +} + +void ArrivalCushionStats::reset() +{ + m_samples.clear(); +} + +void ArrivalCushionStats::note(Int cushion, time_t now) +{ + update(now); + + Sample sample; + sample.time = now; + sample.cushion = cushion; + m_samples.push(sample); +} + +void ArrivalCushionStats::update(time_t now) +{ + m_samples.prune(now, NETWORK_STATS_WINDOW_MS); +} + +Int ArrivalCushionStats::getPercentile(Int percent) const +{ + if (m_samples.size() == 0) + { + return 0; + } + + // counting sort over a clamped range, the values are small frame counts + enum { LOW = -64, RANGE = 256 }; + Int counts[RANGE]; + for (Int r = 0; r < RANGE; ++r) + { + counts[r] = 0; + } + + for (Int i = 0; i < m_samples.size(); ++i) + { + Int cushion = m_samples[i].cushion; + if (cushion < LOW) + { + cushion = LOW; + } + if (cushion > LOW + RANGE - 1) + { + cushion = LOW + RANGE - 1; + } + ++counts[cushion - LOW]; + } + + Int target = (m_samples.size() * percent + 99) / 100; + if (target < 1) + { + target = 1; + } + + Int seen = 0; + for (Int q = 0; q < RANGE; ++q) + { + seen += counts[q]; + if (seen >= target) + { + return q + LOW; + } + } + return LOW + RANGE - 1; +} + +StallStats::StallStats() +{ + reset(); +} + +void StallStats::reset() +{ + m_events.clear(); +} + +void StallStats::note(Int durationMs, UnsignedInt missingSlotMask, time_t now) +{ + update(now); + + Event event; + event.time = now; + event.ms = durationMs; + event.slotMask = missingSlotMask; + m_events.push(event); +} + +void StallStats::update(time_t now) +{ + m_events.prune(now, HISTORY_MS); +} + +Int StallStats::getCount(time_t now, Int windowMs) const +{ + Int count = 0; + for (Int i = 0; i < m_events.size(); ++i) + { + if (now - m_events[i].time <= windowMs) + { + ++count; + } + } + return count; +} + +Int StallStats::getCountForSlot(Int slot, time_t now, Int windowMs) const +{ + Int count = 0; + for (Int i = 0; i < m_events.size(); ++i) + { + if ((m_events[i].slotMask & (1u << slot)) && now - m_events[i].time <= windowMs) + { + ++count; + } + } + return count; +} diff --git a/Core/GameEngine/Source/GameNetwork/DisconnectManager.cpp b/Core/GameEngine/Source/GameNetwork/DisconnectManager.cpp index 9d2f1b58574..a59a6fcbc7c 100644 --- a/Core/GameEngine/Source/GameNetwork/DisconnectManager.cpp +++ b/Core/GameEngine/Source/GameNetwork/DisconnectManager.cpp @@ -214,7 +214,7 @@ void DisconnectManager::updateDisconnectStatus(ConnectionManager *conMgr) { m_haveNotifiedOtherPlayersOfCurrentFrame = TRUE; } - DEBUG_LOG(("DisconnectManager::updateDisconnectStatus - curTime = %d, m_timeOfDisconnectScreenOn = %d, curTime - m_timeOfDisconnectScreenOn = %d", curTime, m_timeOfDisconnectScreenOn, curTime - m_timeOfDisconnectScreenOn)); + DEBUG_LOG(("DisconnectManager::updateDisconnectStatus - curTime = %I64d, m_timeOfDisconnectScreenOn = %I64d, curTime - m_timeOfDisconnectScreenOn = %I64d", (Int64)curTime, (Int64)m_timeOfDisconnectScreenOn, (Int64)(curTime - m_timeOfDisconnectScreenOn))); if (m_timeOfDisconnectScreenOn != 0) { if ((curTime - m_timeOfDisconnectScreenOn) > TheGlobalData->m_networkDisconnectScreenNotifyTime) { diff --git a/Core/GameEngine/Source/GameNetwork/FrameMetrics.cpp b/Core/GameEngine/Source/GameNetwork/FrameMetrics.cpp index 0de0dd2096c..abc805ddb71 100644 --- a/Core/GameEngine/Source/GameNetwork/FrameMetrics.cpp +++ b/Core/GameEngine/Source/GameNetwork/FrameMetrics.cpp @@ -61,14 +61,17 @@ FrameMetrics::FrameMetrics() m_fpsList = NEW Real[TheGlobalData->m_networkFPSHistoryLength]; +#if defined(GENERALS_ONLINE) + m_fpsListLength = TheGlobalData->m_networkFPSHistoryLength; +#endif } FrameMetrics::~FrameMetrics() { - delete m_fpsList; + delete[] m_fpsList; m_fpsList = NULL; #if !defined(USE_NEW_FRAMEMETRIC_LOGIC) - delete m_latencyList; + delete[] m_latencyList; m_latencyList = NULL; delete[] m_pendingLatencies; @@ -115,6 +118,15 @@ void FrameMetrics::init() { #endif m_minimumCushion = -1; +#if defined(GENERALS_ONLINE) + // the game network settings can change the history length after construction + if (m_fpsListLength != TheGlobalData->m_networkFPSHistoryLength) { + delete[] m_fpsList; + m_fpsList = NEW Real[TheGlobalData->m_networkFPSHistoryLength]; + m_fpsListLength = TheGlobalData->m_networkFPSHistoryLength; + } +#endif + UnsignedInt i = 0; for (; i < TheGlobalData->m_networkFPSHistoryLength; ++i) { #if defined(GENERALS_ONLINE_HIGH_FPS_SERVER) @@ -228,8 +240,8 @@ void FrameMetrics::processLatencyResponse(UnsignedInt frame) { m_averageLatency += kvPaAir.second; } - NetworkLog(ELogVerbosity::LOG_DEBUG, "Avg latency is: %f / %d = %f", m_averageLatency, m_mapLatenciesLookup.size(), m_averageLatency /= m_mapLatenciesLookup.size()); m_averageLatency /= m_mapLatenciesLookup.size(); + NetworkLog(ELogVerbosity::LOG_DEBUG, "Avg latency is: %f over %d samples", m_averageLatency, (int)m_mapLatenciesLookup.size()); if (timeDiff > 1000) @@ -280,4 +292,16 @@ void FrameMetrics::SeedLatencyData(int latency) { m_averageFps = GENERALS_ONLINE_HIGH_FPS_LIMIT; m_averageLatency = latency / 1000.f; + +#if defined(USE_NEW_FRAMEMETRIC_LOGIC) + // the next response recomputes the average from these maps, so they must carry the seed too + for (std::unordered_map::iterator it = m_mapLatenciesLookup.begin(); it != m_mapLatenciesLookup.end(); ++it) + { + it->second = m_averageLatency; + } + for (std::map::iterator it = m_mapLatenciesSorted.begin(); it != m_mapLatenciesSorted.end(); ++it) + { + it->second = m_averageLatency; + } +#endif } diff --git a/Core/GameEngine/Source/GameNetwork/NetCommandRef.cpp b/Core/GameEngine/Source/GameNetwork/NetCommandRef.cpp index ba59cab5694..3c470673176 100644 --- a/Core/GameEngine/Source/GameNetwork/NetCommandRef.cpp +++ b/Core/GameEngine/Source/GameNetwork/NetCommandRef.cpp @@ -49,6 +49,7 @@ NetCommandRef::NetCommandRef(NetCommandMsg *msg) m_prev = nullptr; m_relay = 0; m_timeLastSent = -1; + m_retried = FALSE; #ifdef DEBUG_NETCOMMANDREF m_id = ++refNum; diff --git a/Core/GameEngine/Source/GameNetwork/Network.cpp b/Core/GameEngine/Source/GameNetwork/Network.cpp index a149bca7179..d5e6b6ada93 100644 --- a/Core/GameEngine/Source/GameNetwork/Network.cpp +++ b/Core/GameEngine/Source/GameNetwork/Network.cpp @@ -165,6 +165,10 @@ class Network : public NetworkInterface virtual Int getAverageFPS() override { return m_conMgr->getAverageFPS(); } virtual Int getSlotAverageFPS(Int slot) override; + virtual const ConnectionStats* getConnectionStats(Int slot) const override; +#if defined(GENERALS_ONLINE) + virtual Bool getOverlayInfo(Int slot, NetworkOverlayInfo &info) override; +#endif virtual void attachTransport(Transport *transport) override; virtual void initTransport() override; @@ -791,6 +795,12 @@ void Network::update() QueryPerformanceCounter((LARGE_INTEGER*)&curTime); m_isStalling = curTime >= m_nextFrameTime; } + +#if defined(GENERALS_ONLINE) + if (m_conMgr != nullptr && m_localStatus == NETLOCALSTATUS_INGAME) { + m_conMgr->noteStalling(m_isStalling, TheGameLogic->getFrame()); + } +#endif } void Network::liteupdate() { @@ -1094,6 +1104,22 @@ Int Network::getSlotAverageFPS(Int slot) { return -1; } +const ConnectionStats* Network::getConnectionStats(Int slot) const { + if (m_conMgr != nullptr) { + return m_conMgr->getConnectionStats(slot); + } + return nullptr; +} + +#if defined(GENERALS_ONLINE) +Bool Network::getOverlayInfo(Int slot, NetworkOverlayInfo &info) { + if (m_conMgr != nullptr) { + return m_conMgr->getOverlayInfo(slot, info); + } + return FALSE; +} +#endif + #if defined(RTS_DEBUG) void Network::toggleNetworkOn() { if (m_networkOn == TRUE) { diff --git a/Generals/Code/GameEngineDevice/Source/W3DDevice/GameClient/W3DDisplay.cpp b/Generals/Code/GameEngineDevice/Source/W3DDevice/GameClient/W3DDisplay.cpp index 38595af3fcf..6ef77a53e79 100644 --- a/Generals/Code/GameEngineDevice/Source/W3DDevice/GameClient/W3DDisplay.cpp +++ b/Generals/Code/GameEngineDevice/Source/W3DDevice/GameClient/W3DDisplay.cpp @@ -1446,6 +1446,20 @@ void W3DDisplay::gatherDebugStats() // Network performance stats unibuffer.format(L"Run Ahead: %d, Net FPS: %d, Packet arrival cushion: %d", TheNetwork->getRunAhead(), TheNetwork->getFrameRate(), TheNetwork->getPacketArrivalCushion()); + + // per-connection latency, jitter and loss + for (Int slot = 0; slot < MAX_SLOTS; ++slot) { + const ConnectionStats* stats = TheNetwork->getConnectionStats(slot); + if (stats != nullptr && stats->getSampleCount() > 0) { + UnicodeString statsstr; + if (stats->getLatePercent() > 0.0f) { + statsstr.format(L" P%d %.0fms j%.1f l%.0f/%.0f%%", slot, stats->getLatency(), stats->getJitter(), stats->getLossPercent(), stats->getLatePercent()); + } else { + statsstr.format(L" P%d %.0fms j%.1f l%.0f%%", slot, stats->getLatency(), stats->getJitter(), stats->getLossPercent()); + } + unibuffer.concat(statsstr); + } + } m_displayStrings[NetStats]->setText( unibuffer ); // Client frame rate averages for all players in the game. This only works right for the packet router. diff --git a/GeneralsMD/Code/GameEngine/CMakeLists.txt b/GeneralsMD/Code/GameEngine/CMakeLists.txt index a84c0935e98..ee275be1383 100644 --- a/GeneralsMD/Code/GameEngine/CMakeLists.txt +++ b/GeneralsMD/Code/GameEngine/CMakeLists.txt @@ -1140,7 +1140,10 @@ set(GAMEENGINE_SRC Include/GameNetwork/GeneralsOnline/DiscordRichPresence.h Include/GameNetwork/GeneralsOnline/Vendor/DiscordRPC/discord_rpc.h Include/GameNetwork/GeneralsOnline/NetworkBitstream.h + Include/GameNetwork/GeneralsOnline/ConnectionRating.h Include/GameNetwork/GeneralsOnline/NetworkMesh.h + Include/GameNetwork/GeneralsOnline/NetBytes.h + Include/GameNetwork/GeneralsOnline/PacketFeedback.h Include/GameNetwork/GeneralsOnline/NetworkPacket.h Include/GameNetwork/GeneralsOnline/NextGenMP_defines.h Include/GameNetwork/GeneralsOnline/NGMPGame.h @@ -1166,7 +1169,9 @@ set(GAMEENGINE_SRC Source/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.cpp Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp + Source/GameNetwork/GeneralsOnline/ConnectionRating.cpp Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp + Source/GameNetwork/GeneralsOnline/PacketFeedback.cpp Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp Source/GameNetwork/GeneralsOnline/OnlineServices_Auth.cpp Source/GameNetwork/GeneralsOnline/OnlineServices_LobbyInterface.cpp diff --git a/GeneralsMD/Code/GameEngine/Include/GameClient/InGameUI.h b/GeneralsMD/Code/GameEngine/Include/GameClient/InGameUI.h index d5798a48ec7..872a4dab664 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameClient/InGameUI.h +++ b/GeneralsMD/Code/GameEngine/Include/GameClient/InGameUI.h @@ -615,6 +615,7 @@ class InGameUI : public SubsystemInterface, public Snapshot void drawRenderFps(Int& x, Int& y); void drawSystemTime(Int& x, Int& y); void drawGameTime(); + void drawNetworkOverlay(Int mode); ///< 1 = player view, 2 = dev view void drawPlayerInfoList(); void drawObserverStats(Int &x, Int &y); Bool m_observerStatsHidden = false; // hide/show observer overlay diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/ConnectionRating.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/ConnectionRating.h new file mode 100644 index 00000000000..1225735f622 --- /dev/null +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/ConnectionRating.h @@ -0,0 +1,77 @@ +#pragma once + +#include "Lib/BaseType.h" +#include "GameNetwork/ConnectionStats.h" + +// One overall rating of a connection, shared by the lobby slot indicator and the in-game overlay. +// It is the worse of two axes: delay (the input delay the link forces, in ms) and stability (stalls per minute). +// 5 bars: delay <= 150 ms and no stall in the last minute; 1 bar: delay above 500 ms or 6 or more stalls per minute. +// Colors: 4-5 green, 2-3 amber, 1 red. Red needs one of (RedReason): blackout, loss, delay or stalls, see RedReason in the .cpp. +// Stalls a healthy link does not explain wait on the peer's frame rate: slow PC, at most 3 bars and never red for it. +// Too few samples (10 probes, 20 command acks) give no rating. +class ConnectionRating +{ +public: + enum { BAR_COUNT = 5 }; + enum { SEVERITY_OK = 0, SEVERITY_WARN = 1, SEVERITY_BAD = 2 }; + + struct Inputs + { + Inputs(); + + Real delayMs; ///< Input delay the link forces. + Real stallsPerMinute; ///< Measured and predicted, whichever is higher. + Real measuredStallsPerMinute; ///< Stalls of the last minute in a match, negative in the lobby. + Real lossPercent; ///< Packet loss over the window from enough feedback reports, negative if unknown. + Int silentMs; ///< Since the last packet of the peer, negative if unknown. + Int silentLimitMs; ///< Silence that makes the link red, at least 2.5 probe intervals. + Bool stallInWindow; ///< A stall in the last 10 s stats window. + Bool slowPc; ///< The measured stalls are the peer's frame rate, not the link. + }; + + struct State + { + State() : bars(0), delayBars(0), stabilityBars(0), delayMs(0), stallsPerMinute(0.0f), raiseSinceMs(0), redReason(nullptr), slowPc(FALSE) {} + + Int bars; ///< 1..BAR_COUNT, 0 until rated. + Int delayBars; + Int stabilityBars; + Int delayMs; + Real stallsPerMinute; + UnsignedInt raiseSinceMs; ///< Since when the rating asks for more bars than shown, 0 if it does not. + const char *redReason; ///< Why the link is red, nullptr if it is not. + Bool slowPc; ///< The peer's PC is the bottleneck, the link is fine. + }; + + /// lossPercent is the packet loss from the packet feedback, negative to use the loss of the stats. + static Inputs InputsFromProbe(const ConnectionStats &probe, Int slackPercent, Int sendIntervalMs, Real lossPercent); + + static Inputs InputsFromMatch(const ConnectionStats &cmd, const NetworkOverlayInfo &info, Int slackPercent, Int sendIntervalMs, Real lossPercent); + + /// Fewer bars show at once, more only after the rating asked for them for a few seconds. + static void Update(State &state, const Inputs &inputs, UnsignedInt nowMs); + + static Int Severity(Int bars); + + static const char *RedReason(const Inputs &inputs); + + /// Draws the signal bars, level is bars - 1. + static void DrawBars(Int left, Int top, Int size, Int level); + + /// The spinner shown while a connection is not rated yet. + static void DrawSpinner(Int left, Int top, Int size); + + /// The indicator of a rating as ComputeConnectionRating returns it: the spinner below 1 bar, else the bars. + static void DrawIndicator(Int left, Int top, Int size, Int bars); +}; + +// Draws between begin and end of a display batch, unless the caller already batches. +class ScopedDisplayBatch +{ +public: + ScopedDisplayBatch(); + ~ScopedDisplayBatch(); + +private: + Bool m_ownBatch; +}; diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPManager.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPManager.h index 57a4c498928..d2d5c80f753 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPManager.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPManager.h @@ -54,7 +54,7 @@ class HTTPManager // NOTE: set bDisableServiceAuth for endpoints that must be called with something other than the session token (e.g. the refresh endpoint, which needs the refresh token), otherwise the session token overwrites any Authorization header passed in inHeaders void SendPOSTRequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, const char* szPostData, std::function completionCallback, std::function progressCallback = nullptr, int timeoutMS = -1, bool bDisableServiceAuth = false); void SendPUTRequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, const char* szData, std::function completionCallback, std::function progressCallback = nullptr, int timeoutMS = -1); - void SendS3PUTRequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, std::vector vecBuffer, std::function completionCallback, std::function progressCallback = nullptr, int timeoutMS = -1); + void SendS3PUTRequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, std::vector vecBuffer, std::function completionCallback, std::function progressCallback = nullptr, int timeoutMS = -1, int maxSendBytesPerSec = 0); void SendDELETERequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, const char* szData, std::function completionCallback, std::function progressCallback = nullptr, int timeoutMS = -1); void Shutdown(); diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.h index f5b901aade5..7dbf977a136 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.h @@ -36,6 +36,11 @@ class HTTPRequest m_bAppendAuthIfPresent = false; } + void SetMaxSendSpeed(int bytesPerSec) + { + m_maxSendBytesPerSec = bytesPerSec; + } + void OnResponsePartialWrite(std::uint8_t* pBuffer, size_t numBytes); bool HasStarted() const { return m_bIsStarted; } @@ -77,6 +82,8 @@ class HTTPRequest bool m_bAppendAuthIfPresent = true; + int m_maxSendBytesPerSec = 0; // 0 = unlimited + EIPProtocolVersion m_protover; int m_timeoutMS = 5000; diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMPGame.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMPGame.h index 0d11c6a51b3..1275b1bc5c8 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMPGame.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMPGame.h @@ -79,6 +79,19 @@ class NGMPGame : public GameInfo virtual ~NGMPGame(); virtual void reset(void); + Int getSlotOfUser(int64_t userID) const + { + for (Int slotNum = 0; slotNum < MAX_SLOTS; ++slotNum) + { + const NGMPGameSlot* pSlot = (const NGMPGameSlot*)getConstSlot(slotNum); + if (pSlot != nullptr && pSlot->m_userID == userID) + { + return slotNum; + } + } + return -1; + } + bool HasCommittedOutcome() const { return m_bHasCommittedOutcome; } void SetHasCommittedOutcome() { m_bHasCommittedOutcome = true; } diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMP_include.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMP_include.h index 342de5a1087..b22c0ced24e 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMP_include.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMP_include.h @@ -10,7 +10,8 @@ enum class ELogVerbosity enum class ENetworkChannel : BYTE { NETWORK_CHANNEL_GAME = 0, - NETWORK_CHANNEL_AC = 1 + NETWORK_CHANNEL_AC = 1, + NETWORK_CHANNEL_PING = 2 }; #define CHECK_MAIN_THREAD assert(std::this_thread::get_id() == NGMP_OnlineServicesManager::g_MainThreadID); diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetBytes.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetBytes.h new file mode 100644 index 00000000000..c173e617a29 --- /dev/null +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetBytes.h @@ -0,0 +1,16 @@ +#pragma once + +#include + +// Little endian wire values, the game runs on little endian targets only + +inline void WriteU16(uint8_t* out, uint16_t value) +{ + out[0] = static_cast(value & 0xFF); + out[1] = static_cast(value >> 8); +} + +inline uint16_t ReadU16(const uint8_t* in) +{ + return static_cast(in[0] | (in[1] << 8)); +} diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h index fe3c7ac9c6c..3faae1def69 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h @@ -6,6 +6,10 @@ #include #include #include "PluginInterfaces.h" +#include "Lib/BaseType.h" +#include "GameNetwork/ConnectionStats.h" +#include "GameNetwork/GeneralsOnline/ConnectionRating.h" +#include "GameNetwork/GeneralsOnline/PacketFeedback.h" class NetRoom_ChatMessagePacket; @@ -50,6 +54,12 @@ class PlayerConnection void UpdateLatencyHistogram(); + void TickProbe(int64_t nowMs); + void HandleProbe(const uint8_t* payload, size_t len, int64_t arrivalMs); + void ResetProbe(); + static int64_t GetArrivalMs(const SteamNetworkingMessage_t* msg); ///< transport receive time in steady_clock ms + ConnectionStats m_probeStats; + void Close(); bool IsIPV4(); @@ -66,7 +76,7 @@ class PlayerConnection m_State != EConnectionState::CONNECTION_DISCONNECTED; } - int Recv(SteamNetworkingMessage_t** pMsg); + int Recv(SteamNetworkingMessage_t** pMsg, int maxMessages = 255); int GetHighestHistoricalLatency() { @@ -83,7 +93,8 @@ class PlayerConnection } std::vector m_vecLatencyHistory; - std::vector m_vecQualityHistory; + float GetSustainedLossPercent(int windowMs); ///< The worse direction of the packet feedback loss from at least 3 reports, -1 if unknown. + std::string GetStats(); std::string GetConnectionType(); @@ -96,20 +107,26 @@ class PlayerConnection EConnectionState m_State = EConnectionState::NOT_CONNECTED; - int64_t pingSent = -1; - int m_SignallingAttempts = 0; // when signalling for this connection started, for connect-time logging int64_t m_connectStartedMs = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); int GetLatency(); + PacketFeedback m_feedback; + size_t ConsumeEnvelope(const uint8_t* data, size_t len); ///< envelope size, 0 if malformed + int64_t m_lastPacketMs = 0; ///< steady_clock ms of the last enveloped packet, 0 before the first + int SendEnveloped(ENetworkChannel channel, const uint8_t* payload, size_t len, int sendFlags); ///< returns an EResult + bool GetOutgoingLoss(int windowMs, PacketFeedback::Summary& summary); ///< our packets to the peer, false without feedback + bool GetIncomingLoss(int windowMs, PacketFeedback::Summary& summary); ///< the peer's packets to us int GetJitter(); - float GetConnectionQuality(); - int ComputeConnectionScore(); + int ComputeConnectionRating(); ///< 1..5 bars, -1 while it is not known yet; cached for 250 ms. + int UpdateConnectionRating(); + int64_t m_nextRatingMs = 0; + int m_ratingResult = -1; std::chrono::steady_clock::time_point m_connectedSinceTime = (std::chrono::steady_clock::time_point::min)(); - float m_smoothedScore = -1.0f; + ConnectionRating::State m_rating; // Only set for Steam connections HSteamNetConnection m_hSteamConnection = k_HSteamNetConnection_Invalid; @@ -118,6 +135,22 @@ class PlayerConnection std::string m_strMiddlewareID = std::string("NOT SET"); void LiteUpdateForAC(); + +private: + static constexpr size_t k_probeRingSize = 64; + static constexpr uint16_t k_noProbeSeq = 0xFFFF; + + struct OutstandingProbe + { + uint16_t seq = k_noProbeSeq; // k_noProbeSeq marks a free slot + int64_t sendTimeMs = 0; + }; + + OutstandingProbe m_probeRing[k_probeRingSize]; // indexed by seq % k_probeRingSize + uint16_t m_nextProbeSeq = 0; + uint16_t m_lastPeerSeq = k_noProbeSeq; // last probe received from the peer + int64_t m_lastPeerSeqRecvMs = 0; + int64_t m_nextProbeMs = 0; }; struct LobbyMemberEntry; diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NextGenMP_defines.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NextGenMP_defines.h index b78109bc4c4..d7ac3365899 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NextGenMP_defines.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NextGenMP_defines.h @@ -89,11 +89,7 @@ void showNotificationBox(AsciiString nick, UnicodeString message, bool bPlaySoun #define GENERALS_ONLINE_HIGH_FPS_RENDER 1 // This is optional on 30fps, but will boost/unlock the framerate, similar to gentool #endif -#if defined(GENERALS_ONLINE_HIGH_FPS_SERVER) -static int FRAME_GROUPING_CAP = 32; -#else -static int FRAME_GROUPING_CAP = 64; -#endif +extern int FRAME_GROUPING_CAP; //#define GENERALS_ONLINE_ENABLE_CONTROVERSIAL_NON_RETAIL_CHANGES 1 diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h index 7d1419d2ee1..87c12c5af1c 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h @@ -1,9 +1,13 @@ #pragma once +#include "GameNetwork/NetworkSettings.h" #include "NGMP_include.h" +#include +#include #include #include +#include class HTTPManager; @@ -28,6 +32,7 @@ struct S3ScreenshotEntry std::vector vecBytes; std::string strSignedURI; EScreenshotType screenshotType; + std::chrono::steady_clock::time_point notBefore; // upload is held back until this time bool operator==(const S3ScreenshotEntry& other) const { @@ -308,51 +313,173 @@ struct RegionResponse struct ServiceConfig { - bool retry_signalling = false; - bool use_mapped_port = true; - int min_run_ahead_frames = 4; - int ra_update_frequency_frames = 10; - bool relay_all_traffic = false; - int ra_slack_percent = 20; - int frame_grouping_frames = 2; - bool enable_host_migration = true; - - bool network_do_immediate_flush_per_frame = true; - int network_send_flags = -1; - - int network_latency_logic_model = 0; - - bool use_default_config = false; - int ra_slack_override_percent_in_default = 10; - bool do_probes = true; - bool do_replay_upload = true; - - int network_mesh_histogram_duration = 20000; - + // Peer connections: signalling, NAT traversal, latency probes, and the per-connection ping history + struct Connection + { + bool relay_only = false; + int ice_engine = 2; // ICE library: 0 = library default, 1 = native, 2 = WebRTC; the key also takes "default", "native" or "webrtc" + bool use_mapped_port = true; + bool retry_signalling = true; + bool anticheat_transport = false; // game traffic through the anticheat plugin's transport + int probe_lobby_interval_ms = 250; // time between latency probes sent to each peer while not in a match + int probe_match_interval_ms = 1000; // the same in a match, the command acks already measure the round trip + int latency_history_ms = 20000; + } connection; + + // Not read from the service: the live values + int network_latency_logic_model = 2; bool ibra_ra_tweaks = false; float ibra_minslack_default = 0.25f; float ibra_maxslack_default = 1.f; - float ibra_minslack_greaterthan300ms = 0.35f; float ibra_maxslack_greaterthan300ms = 1.f; - float ibra_minslack_greaterthan200ms = 0.3f; float ibra_maxslack_greaterthan200ms = 1.f; + using Send = NetworkSettings::Send; + using InputDelay = NetworkSettings::InputDelay; + Send send; + InputDelay input_delay; - int screenshot_width = 557; - int screenshot_height = 333; + struct Lobby + { + bool host_migration = false; + } lobby; - // ICE client used for P2P NAT traversal: 0 = library default, 1 = native, 2 = WebRTC. - int ice_implementation = 2; + struct Telemetry + { + bool screenshot_probes = true; + bool replay_upload = true; + int screenshot_width = 960; + int screenshot_height = 600; + int screenshot_upload_bytes_per_sec = 131072; // 128 KB/s + int screenshot_slot_offset_ms = 2000; + } telemetry; +}; - bool use_new_networking = false; +// Reads key into value when present; a missing key keeps the default +template +inline void ReadServiceConfigValue(const nlohmann::json& j, const char* key, T& value) +{ + if (!j.is_object()) + { + throw std::invalid_argument("Service config sections must be objects"); + } + const auto it = j.find(key); + if (it != j.end() && !it->is_null()) + { + value = it->get(); + } +} - NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(ServiceConfig, retry_signalling, use_mapped_port, min_run_ahead_frames, ra_update_frequency_frames, relay_all_traffic, - ra_slack_percent, frame_grouping_frames, enable_host_migration, network_do_immediate_flush_per_frame, network_send_flags, network_latency_logic_model, - use_default_config, ra_slack_override_percent_in_default, do_probes, do_replay_upload, network_mesh_histogram_duration, - ibra_ra_tweaks, ibra_minslack_default, ibra_maxslack_default, ibra_minslack_greaterthan300ms, ibra_maxslack_greaterthan300ms, ibra_minslack_greaterthan200ms, ibra_maxslack_greaterthan200ms, - screenshot_width, screenshot_height, ice_implementation, use_new_networking) -}; +// The ICE library by name or by number +inline void ReadIceEngine(const nlohmann::json& j, int& value) +{ + if (!j.is_object()) + { + throw std::invalid_argument("Service config sections must be objects"); + } + const auto it = j.find("ice_engine"); + if (it == j.end() || it->is_null()) + { + return; + } + if (it->is_string()) + { + const std::string name = it->get(); + if (name == "default") + { + value = 0; + } + else if (name == "native") + { + value = 1; + } + else if (name == "webrtc") + { + value = 2; + } + return; + } + value = it->get(); +} + +// A key the service sets replaces the compiled default, a missing key keeps it +inline void ReadServiceConfig(const nlohmann::json& j, ServiceConfig& c) +{ + if (j.contains("connection")) + { + const nlohmann::json& s = j["connection"]; + ReadServiceConfigValue(s, "relay_only", c.connection.relay_only); + ReadIceEngine(s, c.connection.ice_engine); + ReadServiceConfigValue(s, "use_mapped_port", c.connection.use_mapped_port); + ReadServiceConfigValue(s, "retry_signalling", c.connection.retry_signalling); + ReadServiceConfigValue(s, "anticheat_transport", c.connection.anticheat_transport); + ReadServiceConfigValue(s, "probe_lobby_interval_ms", c.connection.probe_lobby_interval_ms); + ReadServiceConfigValue(s, "probe_match_interval_ms", c.connection.probe_match_interval_ms); + ReadServiceConfigValue(s, "latency_history_ms", c.connection.latency_history_ms); + } + if (j.contains("send")) + { + const nlohmann::json& s = j["send"]; + ReadServiceConfigValue(s, "frames_per_packet", c.send.frames_per_packet); + ReadServiceConfigValue(s, "flush_every_frame", c.send.flush_every_frame); + } + if (j.contains("input_delay")) + { + const nlohmann::json& s = j["input_delay"]; + ReadServiceConfigValue(s, "min_frames", c.input_delay.min_frames); + ReadServiceConfigValue(s, "decision_interval_ms", c.input_delay.decision_interval_ms); + ReadServiceConfigValue(s, "fps_history_samples", c.input_delay.fps_history_samples); + ReadServiceConfigValue(s, "slack_percent", c.input_delay.slack_percent); + if (s.contains("formula")) + { + const nlohmann::json& f = s["formula"]; + ReadServiceConfigValue(f, "latency_history_samples", c.input_delay.formula.latency_history_samples); + } + } + if (j.contains("lobby")) + { + ReadServiceConfigValue(j["lobby"], "host_migration", c.lobby.host_migration); + } + if (j.contains("telemetry")) + { + const nlohmann::json& s = j["telemetry"]; + ReadServiceConfigValue(s, "screenshot_probes", c.telemetry.screenshot_probes); + ReadServiceConfigValue(s, "replay_upload", c.telemetry.replay_upload); + ReadServiceConfigValue(s, "screenshot_width", c.telemetry.screenshot_width); + ReadServiceConfigValue(s, "screenshot_height", c.telemetry.screenshot_height); + ReadServiceConfigValue(s, "screenshot_upload_bytes_per_sec", c.telemetry.screenshot_upload_bytes_per_sec); + ReadServiceConfigValue(s, "screenshot_slot_offset_ms", c.telemetry.screenshot_slot_offset_ms); + } +} + +// Keeps values from the service inside the ranges the game can run with +inline void ClampServiceConfig(ServiceConfig& c) +{ + c.connection.latency_history_ms = (std::max)(c.connection.latency_history_ms, 1000); + c.connection.probe_lobby_interval_ms = std::clamp(c.connection.probe_lobby_interval_ms, 50, 5000); + c.connection.probe_match_interval_ms = std::clamp(c.connection.probe_match_interval_ms, 50, 10000); + + c.input_delay.min_frames = std::clamp(c.input_delay.min_frames, 1, 64); + c.input_delay.slack_percent = std::clamp(c.input_delay.slack_percent, 0, 200); + c.input_delay.decision_interval_ms = std::clamp(c.input_delay.decision_interval_ms, 100, 60000); + c.input_delay.fps_history_samples = std::clamp(c.input_delay.fps_history_samples, 1, 128); + c.input_delay.formula.latency_history_samples = std::clamp(c.input_delay.formula.latency_history_samples, 128, 3600); // FrameMetrics drops pending acks when the window is smaller than the frames in flight + c.send.frames_per_packet = std::clamp(c.send.frames_per_packet, 1, 16); + + c.telemetry.screenshot_width = std::clamp(c.telemetry.screenshot_width, 16, 4096); + c.telemetry.screenshot_height = std::clamp(c.telemetry.screenshot_height, 16, 4096); +} + +inline void from_json(const nlohmann::json& j, ServiceConfig& c) +{ + if (!j.is_object()) + { + throw std::invalid_argument("Service config must be an object"); + } + ReadServiceConfig(j, c); + ClampServiceConfig(c); +} class NGMP_OnlineServicesManager { @@ -534,9 +661,10 @@ class NGMP_OnlineServicesManager static void CaptureScreenshot(bool bResizeForTransmit, std::function)> cbOnDataAvailable); static void CaptureScreenshotForProbe(EScreenshotType screenshotType, std::string strURI); - static bool g_bAdvancedNetworkStats; - static void ToggleAdvancedNetworkStats() { g_bAdvancedNetworkStats = !g_bAdvancedNetworkStats; } - static bool IsAdvancedNetworkStatsEnabled() { return g_bAdvancedNetworkStats; } + // the network overlay: 0 off, 1 player view (F10), 2 dev view (Shift+F10) + static int g_networkOverlayMode; + static void ToggleNetworkOverlay(int mode) { g_networkOverlayMode = (g_networkOverlayMode == mode) ? 0 : mode; } + static int GetNetworkOverlayMode() { return g_networkOverlayMode; } void OnLogin(ELoginResult loginResult, const char* szWSAddr, std::function fnWebsocketConnectedCallback); @@ -586,10 +714,10 @@ class NGMP_OnlineServicesManager ServiceConfig& GetServiceConfig() { return m_ServiceConfig; } public: - void CacheScreenshotBytes_StartMatch(std::vector& vecData) + void CacheScreenshotBytes_StartMatch(std::vector vecData) { std::scoped_lock ssLock(m_ScreenshotMutex); - m_vecCachedScreenshotBytes_MatchStart = vecData; + m_vecCachedScreenshotBytes_MatchStart = std::move(vecData); } void CacheScreenshotBytes_EndMatch(uint64_t matchID, std::vector data); diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/PacketFeedback.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/PacketFeedback.h new file mode 100644 index 00000000000..c5fc1ba138e --- /dev/null +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/PacketFeedback.h @@ -0,0 +1,70 @@ +#pragma once + +#include +#include +#include + +// Per-peer packet envelope after the channel byte: u16 sequence, u8 flags (bit 7 always set, bit 0 = feedback record follows), +// optional feedback { u16 highest seen, u8 expected, u8 received, u8 longest lost run } over the last 64 sequence numbers, +// sent at most every 250 ms. +class PacketFeedback +{ +public: + enum + { + ENVELOPE_BASE_SIZE = 3, + FEEDBACK_SIZE = 5, + ENVELOPE_MAX_SIZE = ENVELOPE_BASE_SIZE + FEEDBACK_SIZE + }; + + struct Summary + { + Summary() : samples(0), lossPercent(0.0f), longestRun(0) {} + + int samples; ///< Reports in the window, 0 when there is nothing to say. + float lossPercent; + int longestRun; ///< Longest run of consecutive lost packets in any report. + }; + + PacketFeedback(); + + void Reset(); + + size_t WriteEnvelope(uint8_t* out, int64_t nowMs); + + /// The envelope written last never left this machine, its sequence number is not a lost packet. + void CancelEnvelope(); + + /// Returns the envelope size, 0 if malformed. + size_t ReadEnvelope(const uint8_t* in, size_t len, int64_t nowMs); + + Summary GetOutgoing(int64_t nowMs, int windowMs) const; + + Summary GetIncoming(int64_t nowMs, int windowMs) const; + +private: + struct Report + { + int64_t timeMs; + uint8_t expected; + uint8_t received; + uint8_t longestRun; + }; + + static void Push(std::deque& reports, const Report& report, int64_t nowMs); + static Summary Summarize(const std::deque& reports, int64_t nowMs, int windowMs); + + enum { MAX_WINDOW_MS = 60000, FEEDBACK_INTERVAL_MS = 250 }; + + uint16_t m_nextSequence; + uint32_t m_sentCount; + + bool m_haveHighest; + uint16_t m_highest; + uint64_t m_mask; ///< Bit n is set when highest - n was received. + uint8_t m_span; ///< How many sequence numbers of the window have been seen, at most 64. + int64_t m_lastFeedbackMs; + + std::deque m_outgoing; ///< reports from the peer about our packets + std::deque m_incoming; ///< taken when we send our record +}; diff --git a/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GUICallbacks/Menus/WOLGameSetupMenu.cpp b/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GUICallbacks/Menus/WOLGameSetupMenu.cpp index d8c0b2c2eba..b04c4c1180d 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GUICallbacks/Menus/WOLGameSetupMenu.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GUICallbacks/Menus/WOLGameSetupMenu.cpp @@ -262,10 +262,6 @@ enum }; static Int connectionIndicatorState[MAX_SLOTS]; -static const Int CONNECTION_LEVEL_COUNT = 5; -static const Int connectionLevelThresholds[CONNECTION_LEVEL_COUNT - 1] = { 40, 60, 75, 90 }; ///< score needed for 2..5 bars -static const Int CONNECTION_LEVEL_HYSTERESIS = 3; - WindowLayout *WOLMapSelectLayout = NULL; void PopBackToLobby() @@ -515,10 +511,11 @@ static void playerTooltip(GameWindow *window, } bool bIsConnected = false; - int connectionScore = -1; + int connectionBars = -1; int connectionLatency = -1; int connectionJitter = -1; - int connectionQualityPct = -1; + int connectionLossPct = -1; + int connectionDelayMs = -1; std::string strConnectionType = ""; LobbyMemberEntry member = pLobbyInterface->GetRoomMemberFromID(slot->m_userID); @@ -540,11 +537,12 @@ static void playerTooltip(GameWindow *window, { bIsConnected = false; } - connectionScore = pConnection->ComputeConnectionScore(); + connectionBars = pConnection->ComputeConnectionRating(); connectionLatency = pConnection->GetLatency(); connectionJitter = pConnection->GetJitter(); - float rawQuality = pConnection->GetConnectionQuality(); - connectionQualityPct = (rawQuality >= 0.0f) ? static_cast(rawQuality * 100.0f) : -1; + const float sustainedLoss = pConnection->GetSustainedLossPercent(60000); + connectionLossPct = (sustainedLoss >= 0.0f) ? static_cast(sustainedLoss + 0.5f) : -1; + connectionDelayMs = (connectionBars > 0) ? pConnection->m_rating.delayMs : -1; } } } @@ -562,19 +560,26 @@ static void playerTooltip(GameWindow *window, } else if (bIsConnected) { - UnicodeString scoreStr, latencyStr, jitterStr, qualityStr; - if (connectionScore >= 0) scoreStr.format(L"%d%%", connectionScore); else scoreStr = L"Unknown"; + UnicodeString latencyStr, jitterStr, qualityStr, delayStr; if (connectionLatency >= 0) latencyStr.format(L"%d ms", connectionLatency); else latencyStr = L"Unknown"; + if (connectionDelayMs >= 0) + { + delayStr.format(L"%d ms", connectionDelayMs); + } + else + { + delayStr = L"Unknown"; + } if (connectionJitter >= 0) jitterStr.format(L"%d ms", connectionJitter); else jitterStr = L"Unknown"; - if (connectionQualityPct >= 0) qualityStr.format(L"%d%%", connectionQualityPct); else qualityStr = L"Unknown"; + if (connectionLossPct >= 0) qualityStr.format(L"%d%%", connectionLossPct); else qualityStr = L"Unknown"; #if defined(WORLD_SERIES_CLIENT) - playerInfo.format(L"\nConnection State: Connected (%hs)\nConnection Score: %s\nLatency: %s\nJitter: %s\nReliability: %s\nRegion: %hs\nOverall Elo Rating: %d (in %d matches)\nWS Elo Rating: %d\nWins: %d\nLosses: %d\nDisconnects: %d\nFavorite Army: %s", - strConnectionType.c_str(), scoreStr.str(), latencyStr.str(), jitterStr.str(), qualityStr.str(), + playerInfo.format(L"\nConnection State: Connected (%hs)\nPing (round trip): %s\nPacket loss: %s\nJitter: %s\nExpected input delay: %s\nRegion: %hs\nOverall Elo Rating: %d (in %d matches)\nWS Elo Rating: %d\nWins: %d\nLosses: %d\nDisconnects: %d\nFavorite Army: %s", + strConnectionType.c_str(), latencyStr.str(), qualityStr.str(), jitterStr.str(), delayStr.str(), member.region.c_str(), stats.elo_rating, stats.elo_num_matches, stats.monthly_elo_rating, totalWins, totalLosses, totalDiscons, favoriteSide.str()); #else - playerInfo.format(L"\nConnection State: Connected (%hs)\nConnection Score: %s\nLatency: %s\nJitter: %s\nReliability: %s\nRegion: %hs\nOverall Elo Rating: %d (in %d matches)\nMonthly Elo Rating: %d\nWins: %d\nLosses: %d\nDisconnects: %d\nFavorite Army: %s", - strConnectionType.c_str(), scoreStr.str(), latencyStr.str(), jitterStr.str(), qualityStr.str(), + playerInfo.format(L"\nConnection State: Connected (%hs)\nPing (round trip): %s\nPacket loss: %s\nJitter: %s\nExpected input delay: %s\nRegion: %hs\nOverall Elo Rating: %d (in %d matches)\nMonthly Elo Rating: %d\nWins: %d\nLosses: %d\nDisconnects: %d\nFavorite Army: %s", + strConnectionType.c_str(), latencyStr.str(), qualityStr.str(), jitterStr.str(), delayStr.str(), member.region.c_str(), stats.elo_rating, stats.elo_num_matches, stats.monthly_elo_rating, totalWins, totalLosses, totalDiscons, favoriteSide.str()); #endif } @@ -1399,26 +1404,6 @@ void WOLDisplayGameOptions() } -// ----------------------------------------------------------------------------------------- -// The Bad munkee slot list displaying function -//------------------------------------------------------------------------------------------------- -static Int WOLUpdateConnectionLevel(Int level, Int score) -{ - if (level < 0) - { - level = 0; - while (level < CONNECTION_LEVEL_COUNT - 1 && score >= connectionLevelThresholds[level]) - ++level; - return level; - } - - while (level < CONNECTION_LEVEL_COUNT - 1 && score >= connectionLevelThresholds[level] + CONNECTION_LEVEL_HYSTERESIS) - ++level; - while (level > 0 && score < connectionLevelThresholds[level - 1] - CONNECTION_LEVEL_HYSTERESIS) - --level; - return level; -} - static void WOLRefreshConnectionIndicators(void) { NGMP_OnlineServices_LobbyInterface* pLobbyInterface = NGMP_OnlineServicesManager::GetInterface(); @@ -1445,7 +1430,7 @@ static void WOLRefreshConnectionIndicators(void) genericPingWindow[i]->winHide(FALSE); EConnectionState connectionState = EConnectionState::NOT_CONNECTED; - int connectionScore = -1; + int connectionBars = -1; if (pMesh != nullptr) { @@ -1453,17 +1438,17 @@ static void WOLRefreshConnectionIndicators(void) if (pConnection != nullptr) { connectionState = pConnection->GetState(); - connectionScore = pConnection->ComputeConnectionScore(); + connectionBars = pConnection->ComputeConnectionRating(); } } Int& state = connectionIndicatorState[i]; if (connectionState == EConnectionState::CONNECTION_FAILED || connectionState == EConnectionState::CONNECTION_DISCONNECTED) state = CONNECTION_INDICATOR_WARNING; - else if (connectionState != EConnectionState::CONNECTED_DIRECT || connectionScore < 0) + else if (connectionState != EConnectionState::CONNECTED_DIRECT || connectionBars < 0) state = CONNECTION_INDICATOR_CONNECTING; else - state = WOLUpdateConnectionLevel(state, connectionScore); + state = connectionBars - 1; // the rating is 1..5 bars, the state is the level 0..4 } } @@ -1484,6 +1469,8 @@ static void WOLConnectionIndicatorDraw(GameWindow *window, WinInstanceData *inst if (slot < 0) return; + ScopedDisplayBatch batch; + Int x, y, width, height; window->winGetScreenPosition(&x, &y); window->winGetSize(&width, &height); @@ -1492,26 +1479,11 @@ static void WOLConnectionIndicatorDraw(GameWindow *window, WinInstanceData *inst const Int left = x + (width - size) / 2; const Int top = y + (height - size) / 2; const Real centerX = left + size * 0.5f; - const Real centerY = top + size * 0.5f; const Int state = connectionIndicatorState[slot]; if (state == CONNECTION_INDICATOR_CONNECTING) { - // Faint full ring with a solid arc rotating around it, drawn as short segments. - const Int segmentCount = 24; - const Int arcSegments = 7; - const Real radius = size * 0.4f; - const Real rotation = (timeGetTime() % 900) * 2.0f * PI / 900.0f; - const Color ringColor = GameMakeColor(47, 55, 168, 70); // Zero Hour UI blue - const Color arcColor = GameMakeColor(47, 55, 168, 255); - for (Int segment = 0; segment < segmentCount; ++segment) - { - const Real startAngle = rotation + segment * 2.0f * PI / segmentCount; - const Real endAngle = startAngle + 2.0f * PI / segmentCount; - TheDisplay->drawLine((Int)(centerX + std::cos(startAngle) * radius), (Int)(centerY + std::sin(startAngle) * radius), - (Int)(centerX + std::cos(endAngle) * radius), (Int)(centerY + std::sin(endAngle) * radius), - 3.0f, segment < arcSegments ? arcColor : ringColor); - } + ConnectionRating::DrawSpinner(left, top, size); } else if (state == CONNECTION_INDICATOR_WARNING) { @@ -1528,21 +1500,7 @@ static void WOLConnectionIndicatorDraw(GameWindow *window, WinInstanceData *inst } else { - Color levelColor = GameMakeColor(220, 40, 40, 255); - if (state >= 3) - levelColor = GameMakeColor(0, 200, 0, 255); - else if (state == 2) - levelColor = GameMakeColor(255, 190, 0, 255); - const Color emptyColor = GameMakeColor(90, 90, 90, 200); - - const Int gap = 1; - const Int barWidth = max(1, (size - gap * (CONNECTION_LEVEL_COUNT - 1)) / CONNECTION_LEVEL_COUNT); - for (Int bar = 0; bar < CONNECTION_LEVEL_COUNT; ++bar) - { - const Int barHeight = max(1, size * (bar + 1) / CONNECTION_LEVEL_COUNT); - TheDisplay->drawFillRect(left + bar * (barWidth + gap), top + size - barHeight, barWidth, barHeight, - bar <= state ? levelColor : emptyColor); - } + ConnectionRating::DrawBars(left, top, size, state); } } @@ -2510,7 +2468,7 @@ void WOLGameSetupMenuUpdate( WindowLayout * layout, void *userData) if (pLobbyInterface != nullptr) { ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); - bool bHostMigrationEnabledOnService = serviceConf.enable_host_migration; + bool bHostMigrationEnabledOnService = serviceConf.lobby.host_migration; if (bHostMigrationEnabledOnService) { diff --git a/GeneralsMD/Code/GameEngine/Source/GameClient/InGameUI.cpp b/GeneralsMD/Code/GameEngine/Source/GameClient/InGameUI.cpp index b833f4c59d1..68954303707 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameClient/InGameUI.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameClient/InGameUI.cpp @@ -101,6 +101,11 @@ #include "GameNetwork/NetworkInterface.h" extern NetworkInterface * TheNetwork; #include + +namespace +{ + void overlayReleaseCache(); +} #endif @@ -2354,6 +2359,9 @@ void InGameUI::freeMessageResources() void InGameUI::freeCustomUiResources() { +#if defined(GENERALS_ONLINE) + overlayReleaseCache(); +#endif TheDisplayStringManager->freeDisplayString(m_networkLatencyString); m_networkLatencyString = nullptr; TheDisplayStringManager->freeDisplayString(m_renderFpsString); @@ -7171,99 +7179,561 @@ void InGameUI::drawSystemTime(Int& x, Int& y) } } -void InGameUI::drawGameTime() +#if defined(GENERALS_ONLINE) +namespace { - // draw connections - if (NGMP_OnlineServicesManager::IsAdvancedNetworkStatsEnabled()) + enum OverlayColumn + { + COL_SLOT, COL_NAME, COL_BARS, COL_PATH, COL_PROBE, COL_CMD, COL_JIT, COL_LOST, COL_LATE, + COL_CUSHION, COL_SAMPLES, COL_STALLS, COL_OUT, COL_IN, + COL_COUNT + }; + + const Int k_playerViewTop = 500; + const Real k_devViewTopFraction = 0.10f; // the dev table sits higher to clear the control bar + const Int k_nameMaxChars = 14; + const Int k_devViewHeaderLines = 2; + const Int k_devViewFooterLines = 1; + + const wchar_t *const s_overlayHeaders[COL_COUNT] = { - if (TheNGMPGame != nullptr) + L"Slot", L"Player", L"Bars", L"Path", L"Probe", L"Cmd", L"Jit", L"Lost%", L"Late%", + L"Cush5/50", L"n", L"Stalls", L"OutL%", L"InL%" + }; + + + struct OverlayRow + { + OverlayRow() : used(FALSE), bars(-1), slowPc(FALSE), direct(FALSE), pingMs(0), lossPercent(0.0f), latePercent(0.0f) { - NetworkMesh* pMesh = NGMP_OnlineServicesManager::GetNetworkMesh(); + for (Int c = 0; c < COL_COUNT; ++c) + { + severity[c] = ConnectionRating::SEVERITY_OK; + } + } - NGMP_OnlineServices_LobbyInterface* pLobbyInterface = NGMP_OnlineServicesManager::GetInterface(); + Bool used; + UnicodeString name; + Int bars; + Bool slowPc; + Bool direct; + Int pingMs; + Real lossPercent; + Real latePercent; + NetworkOverlayInfo info; + UnicodeString cell[COL_COUNT]; + Int severity[COL_COUNT]; + }; - if (pMesh != nullptr && pLobbyInterface != nullptr) + Int overlayPercentSeverity(Real percent) + { + if (percent >= 5.0f) + { + return ConnectionRating::SEVERITY_BAD; + } + if (percent >= 0.5f) + { + return ConnectionRating::SEVERITY_WARN; + } + return ConnectionRating::SEVERITY_OK; + } + + Color overlayColor(Int severity) + { + if (severity >= ConnectionRating::SEVERITY_BAD) + { + return GameMakeColor(255, 90, 80, 255); + } + if (severity == ConnectionRating::SEVERITY_WARN) + { + return GameMakeColor(255, 195, 50, 255); + } + return GameMakeColor(235, 235, 235, 255); + } + + Color overlayDim() + { + return GameMakeColor(165, 165, 165, 255); + } + + // the overlay owns one DisplayString per text slot, so a slot whose text is unchanged keeps its built sentence and + // quads. a shared string would rebuild a sentence surface and texture on every draw, which is very slow on some drivers + const Int k_overlayRefreshMs = 250; + const Int k_strMeasure = 0; + const Int k_strHeaderBase = 1; + const Int k_strHeaderCount = 8; + const Int k_strRowBase = k_strHeaderBase + k_strHeaderCount; + const Int k_strRowCount = 8; + const Int k_strDevHeadBase = k_strRowBase + MAX_SLOTS * k_strRowCount; + const Int k_strDevCellBase = k_strDevHeadBase + COL_COUNT; + const Int k_strFooterBase = k_strDevCellBase + MAX_SLOTS * COL_COUNT; + const Int k_strFooterCount = 8; + const Int k_strTotal = k_strFooterBase + k_strFooterCount; + + struct OverlayCache + { + OverlayCache() : valid(FALSE), mode(0), refreshedMs(0), lineH(12), rowCount(0) + { + for (Int i = 0; i < k_strTotal; ++i) + { + strings[i] = nullptr; + } + for (Int c = 0; c < COL_COUNT; ++c) { - //std::vector& vecMembers = pLobbyInterface->GetMembersListForCurrentRoom(); + headerCell[c] = s_overlayHeaders[c]; + } + } - ISteamNetworkingSockets* pSteamNetSockets = SteamNetworkingSockets(); - if (!pSteamNetSockets) - { - return; - } + Bool valid; + Int mode; + UnsignedInt refreshedMs; + Int lineH; + Int rowCount; + NetworkOverlayInfo header; + UnicodeString waitingOn; + UnicodeString headerCell[COL_COUNT]; + OverlayRow rows[MAX_SLOTS]; + DisplayString *strings[k_strTotal]; + }; + + OverlayCache *s_overlayCache = nullptr; + GameFont *s_overlayFont = nullptr; + + void overlayReleaseCache() + { + if (s_overlayCache == nullptr) + { + return; + } + for (Int i = 0; i < k_strTotal; ++i) + { + if (s_overlayCache->strings[i] != nullptr && TheDisplayStringManager != nullptr) + { + TheDisplayStringManager->freeDisplayString(s_overlayCache->strings[i]); + } + } + delete s_overlayCache; + s_overlayCache = nullptr; + } + + DisplayString *overlayString(Int index) + { + DisplayString *&pString = s_overlayCache->strings[index]; + if (pString == nullptr) + { + pString = TheDisplayStringManager->newDisplayString(); + } + pString->setFont(s_overlayFont); + return pString; + } + + void overlaySetText(DisplayString *pString, const UnicodeString &text) + { + if (!(text == pString->getText())) + { + pString->setText(text); + } + } - int i = 0; - auto& allConnections = pMesh->GetAllConnections(); - if (allConnections.empty()) + Int overlayTextWidth(Int index, const UnicodeString &text) + { + DisplayString *pString = overlayString(index); + overlaySetText(pString, text); + return pString->getWidth(); + } + + Int overlayDrawText(Int index, Int x, Int y, const UnicodeString &text, Color color, Color dropColor) + { + DisplayString *pString = overlayString(index); + overlaySetText(pString, text); + pString->draw(x, y, color, dropColor); + return pString->getWidth(); + } + + Int overlayTableWidth(const Int *colWidth, const Bool *colShown, Int gap) + { + Int total = 0; + Int shown = 0; + for (Int c = 0; c < COL_COUNT; ++c) + { + if (colShown[c]) + { + total += colWidth[c]; + ++shown; + } + } + return total + (shown > 1 ? (shown - 1) * gap : 0); + } +} +#endif + +void InGameUI::drawNetworkOverlay(Int mode) +{ +#if defined(GENERALS_ONLINE) + if (TheNGMPGame == nullptr || TheNetwork == nullptr) + { + return; + } + + NetworkMesh* pMesh = NGMP_OnlineServicesManager::GetNetworkMesh(); + NGMP_OnlineServices_LobbyInterface* pLobbyInterface = NGMP_OnlineServicesManager::GetInterface(); + if (pMesh == nullptr || pLobbyInterface == nullptr) + { + return; + } + + if (s_overlayCache == nullptr) + { + s_overlayCache = new OverlayCache(); + } + OverlayCache& cache = *s_overlayCache; + s_overlayFont = m_gameTimeString->getFont(); + + const Color dropColor = m_gameTimeDropColor; + const Color neutral = overlayColor(ConnectionRating::SEVERITY_OK); + const Color dim = overlayDim(); + + const UnsignedInt nowMs = timeGetTime(); + const Bool refresh = !cache.valid || cache.mode != mode || nowMs - cache.refreshedMs >= (UnsignedInt)k_overlayRefreshMs; + if (refresh) + { + cache.valid = TRUE; + cache.mode = mode; + cache.refreshedMs = nowMs; + + Int lineW, lineH; + DisplayString *pMeasure = overlayString(k_strMeasure); + overlaySetText(pMeasure, UnicodeString(L"Wg")); + pMeasure->getSize(&lineW, &lineH); + cache.lineH = (std::max)(lineH, 12); + TheNetwork->getOverlayInfo(-1, cache.header); + + for (Int slot = 0; slot < MAX_SLOTS; ++slot) + { + cache.rows[slot] = OverlayRow(); + } + cache.rowCount = 0; + cache.waitingOn.clear(); + OverlayRow (&rows)[MAX_SLOTS] = cache.rows; + Int& rowCount = cache.rowCount; + UnicodeString& waitingOn = cache.waitingOn; + + for (auto& connection : pMesh->GetAllConnections()) + { + PlayerConnection& conn = connection.second; + if (!conn.IsValid()) + { + continue; + } + + LobbyMemberEntry member = pLobbyInterface->GetRoomMemberFromID(connection.first); + if (member.display_name.empty() || member.m_SlotIndex < 0 || member.m_SlotIndex >= MAX_SLOTS) + { + continue; + } + + const Int slot = member.m_SlotIndex; + OverlayRow& row = rows[slot]; + row.used = TRUE; + row.name.set(from_utf8(member.display_name).c_str()); + row.bars = conn.ComputeConnectionRating(); + row.slowPc = conn.m_rating.slowPc; + row.direct = conn.IsDirect(); + row.pingMs = conn.GetLatency(); + ++rowCount; + + const ConnectionStats* pCmdStats = TheNetwork->getConnectionStats(slot); + const ConnectionStats& probeStats = conn.m_probeStats; + const Bool hasCmd = (pCmdStats != nullptr && pCmdStats->getSampleCount() > 0); + row.lossPercent = hasCmd ? pCmdStats->getLossPercent() : probeStats.getLossPercent(); + row.latePercent = hasCmd ? pCmdStats->getLatePercent() : 0.0f; + TheNetwork->getOverlayInfo(slot, row.info); + + if (row.info.waitingOnSlot) + { + if (!waitingOn.isEmpty()) { - return; + waitingOn.concat(L", "); } + waitingOn.concat(row.name); + } - for (auto& connection : allConnections) - { - if (!connection.second.IsValid()) - { - continue; - } + if (mode != 2) + { + continue; + } - LobbyMemberEntry lobbyMember = pLobbyInterface->GetRoomMemberFromID(connection.first); + UnicodeString* cell = row.cell; + Int* sev = row.severity; + cell[COL_SLOT].format(L"%d", slot); + cell[COL_NAME] = row.name; + cell[COL_NAME].truncateTo(k_nameMaxChars); + if (row.bars > 0) + { + cell[COL_BARS].format(L"%d", row.bars); + sev[COL_BARS] = ConnectionRating::Severity(row.bars); + } + else + { + cell[COL_BARS] = L"-"; + } + cell[COL_PATH].format(L"%hs %hs", conn.IsIPV4() ? "IPv4" : "IPv6", row.direct ? "direct" : "relay"); - const int k_nLanes = 1; - SteamNetConnectionRealTimeStatus_t status; - SteamNetConnectionRealTimeLaneStatus_t laneStatus[k_nLanes]; - - if (!TheNetwork) - { - continue; - } + if (probeStats.getSampleCount() > 0) + { + cell[COL_PROBE].format(L"%.0f", probeStats.getLatency()); + } + else + { + cell[COL_PROBE] = L"-"; + } - EResult res = pSteamNetSockets->GetConnectionRealTimeStatus(connection.second.m_hSteamConnection, &status, k_nLanes, laneStatus); + if (hasCmd) + { + const Real jitter = pCmdStats->getJitter(); + cell[COL_CMD].format(L"%.0f", pCmdStats->getLatency()); + cell[COL_JIT].format(L"%.1f", jitter); + sev[COL_JIT] = (jitter >= 60.0f) ? ConnectionRating::SEVERITY_BAD : ((jitter >= 30.0f) ? ConnectionRating::SEVERITY_WARN : ConnectionRating::SEVERITY_OK); + cell[COL_LOST].format(L"%.0f", pCmdStats->getLossPercent()); + sev[COL_LOST] = overlayPercentSeverity(pCmdStats->getLossPercent()); + cell[COL_LATE].format(L"%.0f", pCmdStats->getLatePercent()); + sev[COL_LATE] = overlayPercentSeverity(pCmdStats->getLatePercent()); + } + else + { + cell[COL_CMD] = L"-"; + cell[COL_JIT] = L"-"; + cell[COL_LOST] = L"-"; + cell[COL_LATE] = L"-"; + } - if (res == k_EResultNoConnection || lobbyMember.display_name.empty()) - { - continue; - } + if (row.info.cushionSamples > 0) + { + cell[COL_CUSHION].format(L"%d/%d", row.info.cushionP5, row.info.cushionP50); + sev[COL_CUSHION] = (row.info.cushionP5 <= 0) ? ConnectionRating::SEVERITY_BAD : ConnectionRating::SEVERITY_OK; + } + else + { + cell[COL_CUSHION] = L"-"; + } + cell[COL_SAMPLES].format(L"%d", row.info.cushionSamples); + cell[COL_STALLS].format(L"%d", row.info.stallsWindow); + sev[COL_STALLS] = (row.info.stallsWindow >= 3 && !row.slowPc) ? ConnectionRating::SEVERITY_BAD : ((row.info.stallsWindow > 0) ? ConnectionRating::SEVERITY_WARN : ConnectionRating::SEVERITY_OK); + + // out is how our packets fare to the peer, in is the peer's to us + PacketFeedback::Summary feedbackOut; + PacketFeedback::Summary feedbackIn; + if (conn.GetOutgoingLoss(NETWORK_STATS_WINDOW_MS, feedbackOut)) + { + cell[COL_OUT].format(L"%.1f", feedbackOut.lossPercent); + sev[COL_OUT] = overlayPercentSeverity(feedbackOut.lossPercent); + } + else + { + cell[COL_OUT] = L"-"; + } + if (conn.GetIncomingLoss(NETWORK_STATS_WINDOW_MS, feedbackIn)) + { + cell[COL_IN].format(L"%.1f", feedbackIn.lossPercent); + sev[COL_IN] = overlayPercentSeverity(feedbackIn.lossPercent); + } + else + { + cell[COL_IN] = L"-"; + } + } + } - int avgFPS = TheNetwork->getSlotAverageFPS(lobbyMember.m_SlotIndex); - - UnicodeString netString; - netString.format(L"\n[usr %s|%d][%hs %hs][AVGFPS: %d] Lat: %i, QL: %.2f, QR: %.2f OutP/s: %.2f, OutB/s: %.2f, InP/s: %.2f, InB/s: %.2f, SR %i PU: %d, PR: %d, NACK: %d, QT: %I64d", - from_utf8(lobbyMember.display_name).c_str(), - (int)res, - connection.second.IsIPV4() ? "IPv4" : "IPv6", - connection.second.IsDirect() ? "Direct" : "Relay", - avgFPS, - status.m_nPing, - status.m_flConnectionQualityLocal, - status.m_flConnectionQualityRemote, - status.m_flOutPacketsPerSec, - status.m_flOutBytesPerSec, - status.m_flInPacketsPerSec, - status.m_flInBytesPerSec, - status.m_nSendRateBytesPerSecond, - status.m_cbPendingUnreliable, - status.m_cbPendingReliable, - status.m_cbSentUnackedReliable, - status.m_usecQueueTime); - - int w, h; - m_gameTimeString->getSize(&w, &h); - - bool bIsHighQuality = true; - if (avgFPS < (GENERALS_ONLINE_HIGH_FPS_LIMIT - 1) || status.m_cbSentUnackedReliable >= 1000 || (status.m_flConnectionQualityLocal != -1.f && status.m_flConnectionQualityLocal < 1.f) || (status.m_flConnectionQualityRemote != -1.f && status.m_flConnectionQualityRemote < 1.f)) - { - bIsHighQuality = false; - } + const Int lineH = cache.lineH; + const Int segmentGap = (lineH * 3) / 4; + const Int displayWidth = TheDisplay->getWidth(); + const Int displayHeight = TheDisplay->getHeight(); - m_gameTimeString->setText(netString); - m_gameTimeString->draw(0, 500 + (i * h / 2), bIsHighQuality ? m_colorGood : m_colorBad, m_gameTimeDropColor); - ++i; - } + const Int left = 0; + + OverlayRow (&rows)[MAX_SLOTS] = cache.rows; + const Int rowCount = cache.rowCount; + const UnicodeString& waitingOn = cache.waitingOn; + const NetworkOverlayInfo& header = cache.header; + + const Int tableLines = (mode == 2) ? (k_devViewHeaderLines + rowCount + k_devViewFooterLines) : (1 + rowCount); + Int y = (mode == 2) ? (Int)(displayHeight * k_devViewTopFraction) : k_playerViewTop; + y = (std::min)(y, displayHeight - 4 - tableLines * lineH); + y = (std::max)(y, 0); + + const Int delayMs = header.runAhead * 1000 / GENERALS_ONLINE_HIGH_FPS_LIMIT; + { + Int x = left; + UnicodeString text; + text.format(L"Input delay %d ms", delayMs); + x += overlayDrawText(k_strHeaderBase + 0, x, y, text, header.stalledNow ? overlayColor(ConnectionRating::SEVERITY_BAD) : neutral, dropColor) + segmentGap; + + if (header.stalledNow && !waitingOn.isEmpty()) + { + text.format(L"waiting on: %ls", waitingOn.str()); + overlayDrawText(k_strHeaderBase + 3, x, y, text, overlayColor(ConnectionRating::SEVERITY_BAD), dropColor); + } + y += lineH; + } + + if (mode != 2) + { + Int nameWidth = 0; + for (Int slot = 0; slot < MAX_SLOTS; ++slot) + { + if (rows[slot].used) + { + nameWidth = (std::max)(nameWidth, overlayTextWidth(k_strRowBase + slot * k_strRowCount, rows[slot].name)); + } + } + + Int barRowY[MAX_SLOTS]; + Int barRowBars[MAX_SLOTS]; + Int barRowCount = 0; + for (Int slot = 0; slot < MAX_SLOTS; ++slot) + { + const OverlayRow& row = rows[slot]; + if (!row.used) + { + continue; + } + + // the indicators of all rows are drawn in one batch afterwards, text draws must not sit inside it + barRowY[barRowCount] = y; + barRowBars[barRowCount] = row.bars; + ++barRowCount; + + const Int rowStr = k_strRowBase + slot * k_strRowCount; + Int x = left + lineH + 8; + UnicodeString text; + overlayDrawText(rowStr, x, y, row.name, neutral, dropColor); + x += nameWidth + segmentGap; + text.format(L"ping %d ms", row.pingMs); + x += overlayDrawText(rowStr + 1, x, y, text, neutral, dropColor) + segmentGap; + text = row.direct ? L"direct" : L"relay"; + x += overlayDrawText(rowStr + 2, x, y, text, dim, dropColor) + segmentGap; + if (row.lossPercent >= 0.5f) + { + text.format(L"loss %.0f%%", row.lossPercent); + x += overlayDrawText(rowStr + 3, x, y, text, overlayColor(overlayPercentSeverity(row.lossPercent)), dropColor) + segmentGap; + } + if (row.latePercent >= 0.5f) + { + text.format(L"late %.0f%%", row.latePercent); + x += overlayDrawText(rowStr + 4, x, y, text, overlayColor(overlayPercentSeverity(row.latePercent)), dropColor) + segmentGap; + } + if (row.slowPc) + { + text = L"slow PC"; + x += overlayDrawText(rowStr + 5, x, y, text, overlayColor(ConnectionRating::SEVERITY_WARN), dropColor) + segmentGap; + } + else if (row.info.stallsWindow > 0) + { + text.format(L"stalls %d", row.info.stallsWindow); + x += overlayDrawText(rowStr + 5, x, y, text, overlayColor(row.info.stallsWindow >= 3 ? ConnectionRating::SEVERITY_BAD : ConnectionRating::SEVERITY_WARN), dropColor) + segmentGap; + } + y += lineH; + } + + if (barRowCount > 0) + { + ScopedDisplayBatch batch; + for (Int barRow = 0; barRow < barRowCount; ++barRow) + { + ConnectionRating::DrawIndicator(left, barRowY[barRow], lineH, barRowBars[barRow]); } + } + return; + } + // dev view: one row per peer, every column drawn at its own x as the font is proportional + Int colWidth[COL_COUNT]; + Bool colShown[COL_COUNT]; + Int colX[COL_COUNT]; + for (Int c = 0; c < COL_COUNT; ++c) + { + colShown[c] = TRUE; + colWidth[c] = overlayTextWidth(k_strDevHeadBase + c, cache.headerCell[c]); + for (Int slot = 0; slot < MAX_SLOTS; ++slot) + { + if (rows[slot].used) + { + colWidth[c] = (std::max)(colWidth[c], overlayTextWidth(k_strDevCellBase + slot * COL_COUNT + c, rows[slot].cell[c])); + } + } + } + + Int tableGap = segmentGap; + if (overlayTableWidth(colWidth, colShown, tableGap) > displayWidth - 8) + { + tableGap = (std::max)(4, lineH / 3); + } + const Int optionalColumns[3] = { COL_OUT, COL_IN, COL_BARS }; + for (Int o = 0; o < 3 && overlayTableWidth(colWidth, colShown, tableGap) > displayWidth - 8; ++o) + { + colShown[optionalColumns[o]] = FALSE; + } + + { + Int x = left; + for (Int c = 0; c < COL_COUNT; ++c) + { + colX[c] = x; + if (colShown[c]) + { + x += colWidth[c] + tableGap; + } } } + for (Int c = 0; c < COL_COUNT; ++c) + { + if (colShown[c]) + { + overlayDrawText(k_strDevHeadBase + c, colX[c], y, cache.headerCell[c], dim, dropColor); + } + } + y += lineH; + + for (Int slot = 0; slot < MAX_SLOTS; ++slot) + { + const OverlayRow& row = rows[slot]; + if (!row.used) + { + continue; + } + for (Int c = 0; c < COL_COUNT; ++c) + { + if (colShown[c]) + { + overlayDrawText(k_strDevCellBase + slot * COL_COUNT + c, colX[c], y, row.cell[c], overlayColor(row.severity[c]), dropColor); + } + } + y += lineH; + } + y += lineH / 2; + + { + UnicodeString text; + text.format(L"R %d frames (%d ms) send interval %d ms", header.runAhead, delayMs, header.frameGroupingMs); + overlayDrawText(k_strFooterBase + 0, left, y, text, neutral, dropColor); + } +#endif +} + +void InGameUI::drawGameTime() +{ + if (NGMP_OnlineServicesManager::GetNetworkOverlayMode() != 0) + { + drawNetworkOverlay(NGMP_OnlineServicesManager::GetNetworkOverlayMode()); + } +#if defined(GENERALS_ONLINE) + else + { + overlayReleaseCache(); + } +#endif + Int currentFrame = TheGameLogic->getFrame(); Int gameSeconds = (Int)(SECONDS_PER_LOGICFRAME_REAL * currentFrame); Int hours = gameSeconds / 60 / 60; diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/ConnectionRating.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/ConnectionRating.cpp new file mode 100644 index 00000000000..397c129250e --- /dev/null +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/ConnectionRating.cpp @@ -0,0 +1,275 @@ +#include "GameNetwork/GeneralsOnline/ConnectionRating.h" +#include "GameNetwork/NetworkDefs.h" +#include "GameClient/Color.h" +#include "GameClient/Display.h" + +#include +#include + +// upper limit of each band, index 0 is the limit for 5 bars, above the last is 1 bar +static const Real k_delayLimitsMs[ConnectionRating::BAR_COUNT - 1] = { 150.0f, 225.0f, 300.0f, 500.0f }; +static const Real k_stallLimitsPerMinute[ConnectionRating::BAR_COUNT - 1] = { 0.05f, 0.5f, 1.0f, 6.0f }; + +// Red (1 bar) only for a link that breaks the match, every other weakness is amber at worst: +static const Real k_redDelayMs = k_delayLimitsMs[ConnectionRating::BAR_COUNT - 2]; // input delay the run ahead would need +static const Real k_redStallsPerMinute = k_stallLimitsPerMinute[ConnectionRating::BAR_COUNT - 2]; // measured in a match, a prediction alone is never red +static const Real k_redLossPercent = 20.0f; // sustained packet loss, more than the redundancy repairs +static const Int k_redSilentMs = 1000; // no packet at all for this long, a blackout + +// Measured stalls of a link whose loss and tail predict almost none are the peer's slow frame rate +static const Real k_slowPcMinStalls = k_redStallsPerMinute; +static const Real k_slowPcMaxLinkStalls = 1.0f; +static const Int k_slowPcMaxBars = 3; + +static const Real k_hysteresis = 0.1f; // a band is left only when the value is this much past its limit + +static const UnsignedInt k_raiseHoldMs = 3000; // more bars are shown only after the rating asked for them this long + +// predicted stalls/min = 2.1 x loss% x 110 / (rtt + 60) + 0.1 x max(0, tail - max(100 ms, 0.2 x rtt)), tail = p90 round trip - smoothed round trip, up to 500 ms +static Real PredictStallsPerMinute(const ConnectionStats &stats, Real lossPercent) +{ + const Real tail = std::clamp(stats.getLatencyPercentile(90) - stats.getLatency(), 0.0f, 500.0f); + const Real headroom = std::max(100.0f, 0.2f * stats.getLatency()); + const Real excess = std::max(0.0f, tail - headroom); + const Real loss = (lossPercent >= 0.0f) ? lossPercent : stats.getRetryPercent(); + return 2.1f * loss * 110.0f / (stats.getLatency() + 60.0f) + 0.1f * excess; +} + +static Real EstimateDelayMs(Real roundTripMs, Int slackPercent, Int sendIntervalMs) +{ + return roundTripMs * (1.0f + slackPercent / 100.0f) + sendIntervalMs; +} + +ConnectionRating::Inputs::Inputs() + : delayMs(0.0f), stallsPerMinute(0.0f), measuredStallsPerMinute(-1.0f), lossPercent(-1.0f), silentMs(-1), silentLimitMs(k_redSilentMs), stallInWindow(FALSE), slowPc(FALSE) +{ +} + +ConnectionRating::Inputs ConnectionRating::InputsFromProbe(const ConnectionStats &probe, Int slackPercent, Int sendIntervalMs, Real lossPercent) +{ + Inputs inputs; + // the p75 round trip, not the smoothed one a few spikes pull up + inputs.delayMs = EstimateDelayMs(probe.getLatencyPercentile(75), slackPercent, sendIntervalMs); + inputs.stallsPerMinute = PredictStallsPerMinute(probe, lossPercent); + return inputs; +} + +ConnectionRating::Inputs ConnectionRating::InputsFromMatch(const ConnectionStats &cmd, const NetworkOverlayInfo &info, Int slackPercent, Int sendIntervalMs, Real lossPercent) +{ + Inputs inputs; + const Real roundTripMs = (info.slotInBandMs >= 0) ? (Real)info.slotInBandMs : cmd.getLatency(); + inputs.delayMs = EstimateDelayMs(roundTripMs, slackPercent, sendIntervalMs); + + Real stallsPerMinute = PredictStallsPerMinute(cmd, lossPercent); + if ((Real)info.stallsMinute >= k_slowPcMinStalls && stallsPerMinute < k_slowPcMaxLinkStalls) + { + inputs.stallsPerMinute = stallsPerMinute; + inputs.slowPc = TRUE; + return inputs; + } + if ((Real)info.stallsMinute > stallsPerMinute) + { + stallsPerMinute = (Real)info.stallsMinute; // the minute window holds one minute of stalls + } + inputs.stallsPerMinute = stallsPerMinute; + inputs.measuredStallsPerMinute = (Real)info.stallsMinute; + inputs.stallInWindow = info.stallsWindow > 0; + return inputs; +} + +static Int RateBand(Int previousBars, Real value, const Real *limits) +{ + const Int count = ConnectionRating::BAR_COUNT; + + Int raw = 1; + for (Int bars = count; bars >= 2; --bars) + { + if (value <= limits[count - bars]) + { + raw = bars; + break; + } + } + + if (previousBars < 1) + { + return raw; + } + + Int bars = previousBars; + while (bars < count && value <= limits[count - (bars + 1)] * (1.0f - k_hysteresis)) + { + ++bars; + } + while (bars > 1 && value > limits[count - bars] * (1.0f + k_hysteresis)) + { + --bars; + } + return bars; +} + +void ConnectionRating::Update(State &state, const Inputs &inputs, UnsignedInt nowMs) +{ + Real stalls = inputs.stallsPerMinute; + if (inputs.stallInWindow) + { + // a stall in the 10 s window rules out the two top bands + stalls = std::max(stalls, 0.6f); + } + else if (inputs.measuredStallsPerMinute > 0.0f) + { + // a stall in the last minute rules out the top band + stalls = std::max(stalls, 0.3f); + } + + state.delayBars = RateBand(state.delayBars, inputs.delayMs, k_delayLimitsMs); + state.stabilityBars = RateBand(state.stabilityBars, stalls, k_stallLimitsPerMinute); + Int rated = std::min(state.delayBars, state.stabilityBars); + state.redReason = RedReason(inputs); + if (state.redReason != nullptr) + { + rated = 1; + } + else if (rated < 2) + { + rated = 2; + } + state.slowPc = inputs.slowPc; + if (state.slowPc && rated > k_slowPcMaxBars) + { + rated = k_slowPcMaxBars; + } + if (state.bars < 1 || rated < state.bars) + { + state.bars = rated; + state.raiseSinceMs = 0; + } + else if (rated > state.bars) + { + // a noisy input crosses a limit back and forth, it must stay better for the whole hold + if (state.raiseSinceMs == 0) + { + state.raiseSinceMs = nowMs | 1; + } + else if (nowMs - state.raiseSinceMs >= k_raiseHoldMs) + { + state.bars = rated; + state.raiseSinceMs = 0; + } + } + else + { + state.raiseSinceMs = 0; + } + state.delayMs = static_cast(inputs.delayMs); + state.stallsPerMinute = stalls; +} + +const char *ConnectionRating::RedReason(const Inputs &inputs) +{ + if (inputs.silentMs >= inputs.silentLimitMs) + { + return "blackout"; + } + if (inputs.lossPercent >= k_redLossPercent) + { + return "loss"; + } + if (inputs.delayMs > k_redDelayMs) + { + return "delay"; + } + if (inputs.measuredStallsPerMinute >= k_redStallsPerMinute) + { + return "stalls"; + } + return nullptr; +} + +Int ConnectionRating::Severity(Int bars) +{ + if (bars >= 4) + { + return SEVERITY_OK; + } + if (bars >= 2) + { + return SEVERITY_WARN; + } + return SEVERITY_BAD; +} + +void ConnectionRating::DrawSpinner(Int left, Int top, Int size) +{ + // a faint full ring with a solid arc rotating around it, drawn as short segments + const Real twoPi = 6.28318531f; + const Int segmentCount = 24; + const Int arcSegments = 7; + const Real radius = size * 0.4f; + const Real centerX = left + size * 0.5f; + const Real centerY = top + size * 0.5f; + const Real rotation = (timeGetTime() % 900) * twoPi / 900.0f; + const Color ringColor = GameMakeColor(47, 55, 168, 70); // Zero Hour UI blue + const Color arcColor = GameMakeColor(47, 55, 168, 255); + for (Int segment = 0; segment < segmentCount; ++segment) + { + const Real startAngle = rotation + segment * twoPi / segmentCount; + const Real endAngle = startAngle + twoPi / segmentCount; + TheDisplay->drawLine((Int)(centerX + std::cos(startAngle) * radius), (Int)(centerY + std::sin(startAngle) * radius), + (Int)(centerX + std::cos(endAngle) * radius), (Int)(centerY + std::sin(endAngle) * radius), + 3.0f, segment < arcSegments ? arcColor : ringColor); + } +} + +void ConnectionRating::DrawIndicator(Int left, Int top, Int size, Int bars) +{ + if (bars < 1) + { + DrawSpinner(left, top, size); + } + else + { + DrawBars(left, top, size, bars - 1); + } +} + +ScopedDisplayBatch::ScopedDisplayBatch() + : m_ownBatch(!TheDisplay->isBatching()) +{ + if (m_ownBatch) + { + TheDisplay->beginBatch(); + } +} + +ScopedDisplayBatch::~ScopedDisplayBatch() +{ + if (m_ownBatch) + { + TheDisplay->endBatch(); + } +} + +void ConnectionRating::DrawBars(Int left, Int top, Int size, Int level) +{ + Color levelColor = GameMakeColor(220, 40, 40, 255); + const Int severity = Severity(level + 1); + if (severity == SEVERITY_OK) + { + levelColor = GameMakeColor(0, 200, 0, 255); + } + else if (severity == SEVERITY_WARN) + { + levelColor = GameMakeColor(255, 190, 0, 255); + } + const Color emptyColor = GameMakeColor(90, 90, 90, 200); + + const Int gap = 1; + const Int barWidth = std::max(1, (size - gap * (BAR_COUNT - 1)) / BAR_COUNT); + for (Int bar = 0; bar < BAR_COUNT; ++bar) + { + const Int barHeight = std::max(1, size * (bar + 1) / BAR_COUNT); + TheDisplay->drawFillRect(left + bar * (barWidth + gap), top + size - barHeight, barWidth, barHeight, + bar <= level ? levelColor : emptyColor); + } +} diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp index a420a04cda6..e476ced3bb7 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp @@ -43,13 +43,14 @@ void HTTPManager::SendPUTRequest(const char* szURI, EIPProtocolVersion protover, } -void HTTPManager::SendS3PUTRequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, std::vector vecBuffer, std::function completionCallback, std::function progressCallback /*= nullptr*/, int timeoutMS /*= -1*/) +void HTTPManager::SendS3PUTRequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, std::vector vecBuffer, std::function completionCallback, std::function progressCallback /*= nullptr*/, int timeoutMS /*= -1*/, int maxSendBytesPerSec /*= 0*/) { CHECK_MAIN_THREAD; HTTPRequest* pRequest = PlatformCreateRequest(EHTTPVerb::HTTP_VERB_PUT, protover, szURI, inHeaders, completionCallback, progressCallback, timeoutMS); pRequest->DisableServiceAuth(); - pRequest->SetPostDataBuffer(vecBuffer); + pRequest->SetMaxSendSpeed(maxSendBytesPerSec); + pRequest->SetPostDataBuffer(std::move(vecBuffer)); m_vecRequestsPendingStart.push_back(pRequest); } diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp index 02786d7f3d9..3c77e8f4039 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp @@ -298,6 +298,11 @@ void HTTPRequest::PlatformStartRequest() } } + if (m_maxSendBytesPerSec > 0) + { + curl_easy_setopt(m_pCURL, CURLOPT_MAX_SEND_SPEED_LARGE, (curl_off_t)m_maxSendBytesPerSec); + } + // needed for PUT etc if (m_httpVerb == EHTTPVerb::HTTP_VERB_PUT) { diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMPGame.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMPGame.cpp index 62f7b703cba..490bfcf0e33 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMPGame.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMPGame.cpp @@ -13,6 +13,12 @@ #include "GameClient/View.h" #include "../NextGenMP_defines.h" +#if defined(GENERALS_ONLINE_HIGH_FPS_SERVER) +int FRAME_GROUPING_CAP = 32; +#else +int FRAME_GROUPING_CAP = 64; +#endif + NGMPGameSlot::NGMPGameSlot() { GameSlot(); @@ -411,34 +417,6 @@ void NGMPGame::launchGame(void) //TheWritableGlobalData->m_networkPlayerTimeoutTime = 60000; //TheWritableGlobalData->m_networkDisconnectScreenNotifyTime = 2500; - // process service config - NGMP_OnlineServicesManager* pOnlineServicesMgr = NGMP_OnlineServicesManager::GetInstance(); - if (pOnlineServicesMgr != nullptr) - { - ServiceConfig& serviceConf = pOnlineServicesMgr->GetServiceConfig(); - - if (serviceConf.use_default_config) - { - TheWritableGlobalData->m_networkFPSHistoryLength = 30; - TheWritableGlobalData->m_networkLatencyHistoryLength = 200; - TheWritableGlobalData->m_networkRunAheadSlack = serviceConf.ra_slack_override_percent_in_default; // normally 10 - TheWritableGlobalData->m_networkRunAheadMetricsTime = 5000; - } - else - { - TheWritableGlobalData->m_networkFPSHistoryLength = 10; - TheWritableGlobalData->m_networkLatencyHistoryLength = 10; - - MIN_RUNAHEAD = serviceConf.min_run_ahead_frames; - - TheWritableGlobalData->m_networkRunAheadSlack = serviceConf.ra_slack_percent; - TheWritableGlobalData->m_networkRunAheadMetricsTime = serviceConf.ra_update_frequency_frames * (float)(1000.f / GENERALS_ONLINE_HIGH_FPS_LIMIT); - - FRAME_GROUPING_CAP = serviceConf.frame_grouping_frames * (float)(1000.f / GENERALS_ONLINE_HIGH_FPS_LIMIT); - } - } - - #if defined(GENERALS_ONLINE_HIGH_FPS_RENDER) TheWritableGlobalData->m_horizontalScrollSpeedFactor = NGMP_OnlineServicesManager::Settings.Camera_MoveSpeedRatio(); TheWritableGlobalData->m_verticalScrollSpeedFactor = NGMP_OnlineServicesManager::Settings.Camera_MoveSpeedRatio(); diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp index 5957924e4c2..d1890a35bb7 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp @@ -1,10 +1,12 @@ #include "GameNetwork/GeneralsOnline/NetworkMesh.h" #include "GameNetwork/GeneralsOnline/NGMP_include.h" +#include "GameNetwork/GeneralsOnline/NetBytes.h" #include "GameNetwork/GeneralsOnline/NGMP_interfaces.h" #include #include "GameNetwork/NetworkDefs.h" #include "GameNetwork/NetworkInterface.h" +#include "Common/GlobalData.h" #include "GameLogic/GameLogic.h" #include "../OnlineServices_RoomsInterface.h" #include "../json.hpp" @@ -19,6 +21,11 @@ bool g_bForceRelay = false; UnsignedInt m_exeCRCOriginal = 0; +static int64_t NowSteadyMs() +{ + return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); +} + // Pool for deferred deletion of ConnectionSignaling objects; avoids "delete this" races during // async Steam callbacks. static std::mutex g_pendingDeletionMutex; @@ -114,7 +121,7 @@ void OnSteamNetConnectionStatusChanged(SteamNetConnectionStatusChangedCallback_t const bool bPeerLeft = pJoinOrderLobby != nullptr && pJoinOrderLobby->IsJoinOrderKnown() && !pJoinOrderLobby->IsLobbyMember(userID); // a match can't leave its lobby, so keep repairing the link until the service drops the player const bool bInMatch = TheGameLogic != nullptr && TheGameLogic->isInInternetGame(); - bool bShouldRetry = serviceConf.retry_signalling && ((bInMatch && !bPeerLeft) || (!bWeJoinedLater && !bPeerLeft) || signallingAttemptsBeforeDisconnect < numSignallingAttempts); + bool bShouldRetry = serviceConf.connection.retry_signalling && ((bInMatch && !bPeerLeft) || (!bWeJoinedLater && !bPeerLeft) || signallingAttemptsBeforeDisconnect < numSignallingAttempts); bool bWasError = pInfo->m_info.m_eState == k_ESteamNetworkingConnectionState_ProblemDetectedLocally || pInfo->m_info.m_eEndReason != k_ESteamNetConnectionEnd_App_Generic; plrConnection.SetDisconnected(bWasError, pMesh, bShouldRetry && bWasError); @@ -716,12 +723,12 @@ NetworkMesh::NetworkMesh() m_strTurnTokenString = std::format("{},{}", m_strTurnToken.c_str(), m_strTurnToken.c_str()); ServiceConfig& serviceConf = pOnlineServicesMgr->GetServiceConfig(); - m_iceEnable = (g_bForceRelay || serviceConf.relay_all_traffic) + m_iceEnable = (g_bForceRelay || serviceConf.connection.relay_only) ? k_nSteamNetworkingConfig_P2P_Transport_ICE_Enable_Relay : k_nSteamNetworkingConfig_P2P_Transport_ICE_Enable_All; // 0 = library default, 1 = native, 2 = WebRTC - m_iceImplementation = (serviceConf.ice_implementation >= 0 && serviceConf.ice_implementation <= 2) ? serviceConf.ice_implementation : 2; + m_iceImplementation = (serviceConf.connection.ice_engine >= 0 && serviceConf.connection.ice_engine <= 2) ? serviceConf.connection.ice_engine : 2; NetworkLog(ELogVerbosity::LOG_RELEASE, "NetworkMesh: using ICE implementation %d (0=default, 1=native, 2=WebRTC)", m_iceImplementation); m_pSignaling = new CSignalingClient(SteamNetworkingSockets()); @@ -772,7 +779,7 @@ NetworkMesh::NetworkMesh() void NetworkMesh::Flush() { ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); - bool bDoImmediateFlushPerFrame = serviceConf.network_do_immediate_flush_per_frame; + bool bDoImmediateFlushPerFrame = serviceConf.send.flush_every_frame; if (bDoImmediateFlushPerFrame) { @@ -952,7 +959,7 @@ void NetworkMesh::StartConnectionSignalling(const char* szMiddlewareID, int64_t int g_nLocalPort = 0; - int g_nVirtualPortRemote = serviceConf.use_mapped_port ? preferredPort : 0; + int g_nVirtualPortRemote = serviceConf.connection.use_mapped_port ? preferredPort : 0; // Our remote and local port don't match, so we need to set it explicitly if (g_nVirtualPortRemote != g_nLocalPort) @@ -1171,10 +1178,13 @@ void NetworkMesh::Tick() m_pSignaling->Poll(); } + const int64_t nowMs = NowSteadyMs(); + // update connection histograms for (auto& kvPair : m_mapConnections) { PlayerConnection& conn = kvPair.second; + conn.TickProbe(nowMs); conn.UpdateLatencyHistogram(); } @@ -1248,6 +1258,12 @@ void PlayerConnection::LiteUpdateForAC() msg->Release(); continue; } + else if (netChannel == ENetworkChannel::NETWORK_CHANNEL_PING) + { + HandleProbe(vecData.data() + sizeof(ENetworkChannel), numBytes - sizeof(ENetworkChannel), GetArrivalMs(msg)); + msg->Release(); + continue; + } } else if (numBytes != -1 && numBytes < sizeof(ENetworkChannel)) { @@ -1264,6 +1280,160 @@ void PlayerConnection::LiteUpdateForAC() } } +// Probe payload after the envelope, little endian: u16 seq, u16 echoSeq, u16 echoHoldMs +static constexpr size_t k_probePayloadSize = 6; + +void PlayerConnection::ResetProbe() +{ + m_probeStats.reset(); + for (OutstandingProbe& probe : m_probeRing) + { + probe = OutstandingProbe(); + } + m_nextProbeSeq = 0; + m_lastPeerSeq = k_noProbeSeq; + m_lastPeerSeqRecvMs = 0; + m_nextProbeMs = 0; +} + +void PlayerConnection::TickProbe(int64_t nowMs) +{ + int pingIntervalMs = 250; + if (NGMP_OnlineServicesManager::GetInstance() != nullptr) + { + const ServiceConfig::Connection& connection = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig().connection; + pingIntervalMs = (TheNetwork != nullptr) ? connection.probe_match_interval_ms : connection.probe_lobby_interval_ms; + } + + // an echo rides on the next probe, so it is allowed one probe interval late + const int64_t timeoutMs = (std::max)(static_cast(1000), static_cast(m_probeStats.getLatency() * 4.0f)) + pingIntervalMs; + for (OutstandingProbe& probe : m_probeRing) + { + if (probe.seq != k_noProbeSeq && nowMs - probe.sendTimeMs > timeoutMs) + { + m_probeStats.noteRetry(static_cast(nowMs)); + probe.seq = k_noProbeSeq; + } + } + + m_probeStats.update(static_cast(nowMs)); + + if (nowMs < m_nextProbeMs || m_State != EConnectionState::CONNECTED_DIRECT) + { + return; + } + + m_nextProbeMs = nowMs + pingIntervalMs; + + if (AnticheatPlugInterface::DoesACPluginProvideSecureGameTransport()) + { + // the plugin transport has no per-peer receive path for the ping channel, so nothing could be echoed back + return; + } + + if (m_hSteamConnection == k_HSteamNetConnection_Invalid) + { + return; + } + + const uint16_t seq = m_nextProbeSeq; + m_nextProbeSeq = (m_nextProbeSeq == k_noProbeSeq - 1) ? 0 : static_cast(m_nextProbeSeq + 1); + + uint16_t echoHoldMs = 0; + if (m_lastPeerSeq != k_noProbeSeq) + { + echoHoldMs = static_cast((std::min)(nowMs - m_lastPeerSeqRecvMs, static_cast(k_noProbeSeq - 1))); + } + + BYTE packet[k_probePayloadSize]; + WriteU16(packet + 0, seq); + WriteU16(packet + 2, m_lastPeerSeq); + WriteU16(packet + 4, echoHoldMs); + + if (SendEnveloped(ENetworkChannel::NETWORK_CHANNEL_PING, packet, sizeof(packet), k_nSteamNetworkingSend_UnreliableNoNagle) != k_EResultOK) + { + return; + } + + // the slot is still in use: that probe was never echoed in a full ring turn + OutstandingProbe& slot = m_probeRing[seq % k_probeRingSize]; + if (slot.seq != k_noProbeSeq) + { + m_probeStats.noteRetry(static_cast(nowMs)); + } + slot.seq = seq; + slot.sendTimeMs = nowMs; +} + +int64_t PlayerConnection::GetArrivalMs(const SteamNetworkingMessage_t* msg) +{ + const int64_t nowMs = NowSteadyMs(); + if (msg == nullptr || SteamNetworkingUtils() == nullptr || msg->m_usecTimeReceived <= 0) + { + return nowMs; + } + + // how long the message waited in the transport's queue for the main loop + const int64_t queuedMs = (SteamNetworkingUtils()->GetLocalTimestamp() - msg->m_usecTimeReceived) / 1000; + return nowMs - (std::max)(static_cast(0), queuedMs); +} + +void PlayerConnection::HandleProbe(const uint8_t* payload, size_t len, int64_t arrivalMs) +{ + const size_t envelopeSize = (payload != nullptr) ? ConsumeEnvelope(payload, len) : 0; + if (envelopeSize == 0) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[PROBE] Dropping probe without a valid packet envelope from user %lld, size %zu", m_userID, len); + return; + } + payload += envelopeSize; + len -= envelopeSize; + + if (payload == nullptr || len < k_probePayloadSize) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[PROBE] Dropping malformed probe of size %zu from user %lld", len, m_userID); + return; + } + + // the arrival time, a slow frame on either side is no network delay + const int64_t nowMs = arrivalMs; + const uint16_t peerSeq = ReadU16(payload); + const uint16_t echoSeq = ReadU16(payload + 2); + const uint16_t echoHoldMs = ReadU16(payload + 4); + + // a reordered older probe must not move the echo back + if (peerSeq != k_noProbeSeq && (m_lastPeerSeq == k_noProbeSeq || static_cast(peerSeq - m_lastPeerSeq) < 0x8000)) + { + m_lastPeerSeq = peerSeq; + m_lastPeerSeqRecvMs = nowMs; + } + + if (echoSeq == k_noProbeSeq) + { + return; + } + + OutstandingProbe& acked = m_probeRing[echoSeq % k_probeRingSize]; + if (acked.seq != echoSeq) + { + // already resolved or timed out + return; + } + + const int64_t rttMs = (std::max)(static_cast(0), nowMs - acked.sendTimeMs - echoHoldMs); + m_probeStats.noteAcked(static_cast(rttMs), static_cast(nowMs)); + acked.seq = k_noProbeSeq; + + // older un-echoed probes are dropped, not lost: packet feedback measures the loss + for (OutstandingProbe& probe : m_probeRing) + { + if (probe.seq != k_noProbeSeq && static_cast(echoSeq - probe.seq) < 0x8000) + { + probe.seq = k_noProbeSeq; + } + } +} + PlayerConnection::PlayerConnection(int64_t userID, HSteamNetConnection hSteamConnection) { m_userID = userID; @@ -1327,57 +1497,14 @@ int PlayerConnection::SendGamePacket(void* pBuffer, uint32_t totalDataSize) return (int)k_EResultFail; } - ENetworkChannel netChannel = ENetworkChannel::NETWORK_CHANNEL_GAME; - std::vector vecData; - vecData.resize(totalDataSize + sizeof(ENetworkChannel)); - memcpy(vecData.data() + sizeof(ENetworkChannel), pBuffer, totalDataSize); - vecData[0] = (BYTE)netChannel; - int sendFlags = k_nSteamNetworkingSend_Reliable | k_nSteamNetworkingSend_AutoRestartBrokenSession; // default from last patch - ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); - int netSendFlags = serviceConf.network_send_flags; - NetworkLog(ELogVerbosity::LOG_DEBUG, "[GAME PACKET] Sending msg of size %ld to user %lld\n", totalDataSize, m_userID); - EResult r = SteamNetworkingSockets()->SendMessageToConnection( - m_hSteamConnection, vecData.data(), vecData.size(), sendFlags, nullptr); - + const int r = SendEnveloped(ENetworkChannel::NETWORK_CHANNEL_GAME, static_cast(pBuffer), totalDataSize, sendFlags); if (r != k_EResultOK) { - if (netSendFlags != -1) - { - if (netSendFlags == 0) - { - sendFlags = k_nSteamNetworkingSend_Unreliable; - } - else if (netSendFlags == 1) - { - sendFlags = k_nSteamNetworkingSend_UnreliableNoNagle; - } - else if (netSendFlags == 2) - { - sendFlags = k_nSteamNetworkingSend_UnreliableNoDelay; - } - else if (netSendFlags == 3) - { - sendFlags = k_nSteamNetworkingSend_Reliable; - } - else if (netSendFlags == 4) - { - sendFlags = k_nSteamNetworkingSend_ReliableNoNagle; - } - } - - NetworkLog(ELogVerbosity::LOG_DEBUG, "[GAME PACKET] Sending msg of size %ld to user %lld\n", totalDataSize, m_userID); - EResult r = SteamNetworkingSockets()->SendMessageToConnection( - m_hSteamConnection, pBuffer, (int)totalDataSize, sendFlags, nullptr); - - if (r != k_EResultOK) - { - NetworkLog(ELogVerbosity::LOG_RELEASE, "[GAME PACKET] Failed to send, err code was %d", r); - } - - return (int)r; + NetworkLog(ELogVerbosity::LOG_RELEASE, "[GAME PACKET] Failed to send, err code was %d", r); + return r; } } @@ -1424,12 +1551,12 @@ void PlayerConnection::UpdateLatencyHistogram() if (NGMP_OnlineServicesManager::GetInstance() != nullptr) { ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); - histogram_duration = serviceConf.network_mesh_histogram_duration; + histogram_duration = serviceConf.connection.latency_history_ms; } - // update latency history - int currLatency = GetLatency(); + // update latency history, from the mesh probe once it has samples + int currLatency = (m_probeStats.getSampleCount() > 0) ? static_cast(m_probeStats.getLatency()) : GetLatency(); #if defined(GENERALS_ONLINE_HIGH_FPS_SERVER) const int connectionHistoryLength = histogram_duration / 16; // ~20 sec worth of frames at 60fps (default) #else @@ -1441,38 +1568,6 @@ void PlayerConnection::UpdateLatencyHistogram() m_vecLatencyHistory.erase(m_vecLatencyHistory.begin()); } m_vecLatencyHistory.push_back(currLatency); - - // Sample connection quality into rolling history. - // Prefer SNS local quality, then remote quality, then fall back to manual in/out packet rate ratio. - if (m_hSteamConnection != k_HSteamNetConnection_Invalid) - { - SteamNetConnectionRealTimeStatus_t status; - if (SteamNetworkingSockets()->GetConnectionRealTimeStatus(m_hSteamConnection, &status, 0, nullptr) == k_EResultOK) - { - float sample = -1.0f; - if (status.m_flConnectionQualityLocal >= 0.0f) - { - sample = status.m_flConnectionQualityLocal; - } - else if (status.m_flConnectionQualityRemote >= 0.0f) - { - sample = status.m_flConnectionQualityRemote; - } - else if (status.m_flOutPacketsPerSec > 0.0f) - { - sample = status.m_flInPacketsPerSec / status.m_flOutPacketsPerSec; - } - - if (sample > 1.0f) sample = 1.0f; - - if (sample >= 0.0f) - { - if (m_vecQualityHistory.size() >= connectionHistoryLength) - m_vecQualityHistory.erase(m_vecQualityHistory.begin()); - m_vecQualityHistory.push_back(sample); - } - } - } } void PlayerConnection::Close() @@ -1509,12 +1604,12 @@ bool PlayerConnection::IsIPV4() return info.m_addrRemote.IsIPv4(); } -int PlayerConnection::Recv(SteamNetworkingMessage_t** pMsg) +int PlayerConnection::Recv(SteamNetworkingMessage_t** pMsg, int maxMessages) { int r = -1; if (m_hSteamConnection != k_HSteamNetConnection_Invalid) { - r = SteamNetworkingSockets()->ReceiveMessagesOnConnection(m_hSteamConnection, pMsg, 255); + r = SteamNetworkingSockets()->ReceiveMessagesOnConnection(m_hSteamConnection, pMsg, maxMessages); NetworkLog(ELogVerbosity::LOG_DEBUG, "[DISC] Recv Result %d from user %lld", r, m_userID); } else @@ -1528,14 +1623,25 @@ int PlayerConnection::Recv(SteamNetworkingMessage_t** pMsg) std::string PlayerConnection::GetStats() { + // lead with the probe numbers, or just the reported ping while there are no samples yet + char szSummary[64] = { 0 }; + if (m_probeStats.getSampleCount() > 0) + { + snprintf(szSummary, sizeof(szSummary), "Lat %d j%.0f L%.0f%%\n", static_cast(m_probeStats.getLatency()), m_probeStats.getJitter(), m_probeStats.getLossPercent()); + } + else + { + snprintf(szSummary, sizeof(szSummary), "Lat %d\n", GetLatency()); + } + if (m_hSteamConnection == k_HSteamNetConnection_Invalid) - return "(disconnected)"; + return std::string(szSummary) + "(disconnected)"; char szBuf[2048] = { 0 }; int ret = SteamNetworkingSockets()->GetDetailedConnectionStatus(m_hSteamConnection, szBuf, 2048); NetworkLog(ELogVerbosity::LOG_RELEASE, "[STEAM] PlayerConnection::GetStats returned %d", ret); - return std::string(szBuf); + return std::string(szSummary) + szBuf; } @@ -1577,7 +1683,9 @@ void PlayerConnection::UpdateState(EConnectionState newState, NetworkMesh* pOwni if (newState == EConnectionState::CONNECTED_DIRECT && m_State != EConnectionState::CONNECTED_DIRECT) { m_connectedSinceTime = std::chrono::steady_clock::now(); - m_smoothedScore = -1.0f; + m_rating = ConnectionRating::State(); + m_nextRatingMs = 0; + ResetProbe(); } m_State = newState; @@ -1688,6 +1796,66 @@ int PlayerConnection::GetLatency() return -1; } +size_t PlayerConnection::ConsumeEnvelope(const uint8_t* data, size_t len) +{ + const int64_t nowMs = NowSteadyMs(); + m_lastPacketMs = nowMs; + return m_feedback.ReadEnvelope(data, len, nowMs); +} + +int PlayerConnection::SendEnveloped(ENetworkChannel channel, const uint8_t* payload, size_t len, int sendFlags) +{ + const int64_t nowMs = NowSteadyMs(); + + std::vector vecData; + vecData.resize(sizeof(ENetworkChannel) + PacketFeedback::ENVELOPE_MAX_SIZE + len); + vecData[0] = static_cast(channel); + const size_t envelopeSize = m_feedback.WriteEnvelope(vecData.data() + sizeof(ENetworkChannel), nowMs); + memcpy(vecData.data() + sizeof(ENetworkChannel) + envelopeSize, payload, len); + vecData.resize(sizeof(ENetworkChannel) + envelopeSize + len); + + EResult r = SteamNetworkingSockets()->SendMessageToConnection(m_hSteamConnection, vecData.data(), static_cast(vecData.size()), sendFlags, nullptr); + if (r != k_EResultOK) + { + // once more with the same packet, it keeps its sequence number + r = SteamNetworkingSockets()->SendMessageToConnection(m_hSteamConnection, vecData.data(), static_cast(vecData.size()), sendFlags, nullptr); + } + if (r != k_EResultOK) + { + m_feedback.CancelEnvelope(); + } + return static_cast(r); +} + +bool PlayerConnection::GetOutgoingLoss(int windowMs, PacketFeedback::Summary& summary) +{ + const int64_t nowMs = NowSteadyMs(); + summary = m_feedback.GetOutgoing(nowMs, windowMs); + return summary.samples > 0; +} + +bool PlayerConnection::GetIncomingLoss(int windowMs, PacketFeedback::Summary& summary) +{ + const int64_t nowMs = NowSteadyMs(); + summary = m_feedback.GetIncoming(nowMs, windowMs); + return summary.samples > 0; +} + +float PlayerConnection::GetSustainedLossPercent(int windowMs) +{ + // one report of a short window is no trend + static constexpr int k_minFeedbackReports = 3; + PacketFeedback::Summary outLoss; + PacketFeedback::Summary inLoss; + GetOutgoingLoss(windowMs, outLoss); + GetIncomingLoss(windowMs, inLoss); + if (outLoss.samples < k_minFeedbackReports && inLoss.samples < k_minFeedbackReports) + { + return -1.0f; + } + return (std::max)((outLoss.samples >= k_minFeedbackReports) ? outLoss.lossPercent : 0.0f, (inLoss.samples >= k_minFeedbackReports) ? inLoss.lossPercent : 0.0f); +} + int PlayerConnection::GetJitter() { int sumDelta = 0; @@ -1716,20 +1884,20 @@ int PlayerConnection::GetJitter() return sumDelta / count; } -float PlayerConnection::GetConnectionQuality() +int PlayerConnection::ComputeConnectionRating() { - if (!m_vecQualityHistory.empty()) + // the overlay and the lobby ask every frame, the percentile sorts inside do not change that fast + const int64_t ratingNowMs = NowSteadyMs(); + if (ratingNowMs < m_nextRatingMs) { - float sum = 0.0f; - for (float r : m_vecQualityHistory) - sum += r; - return sum / static_cast(m_vecQualityHistory.size()); + return m_ratingResult; } - - return -1.0f; + m_nextRatingMs = ratingNowMs + 250; + m_ratingResult = UpdateConnectionRating(); + return m_ratingResult; } -int PlayerConnection::ComputeConnectionScore() +int PlayerConnection::UpdateConnectionRating() { static constexpr std::chrono::milliseconds k_warmupDuration{ 1000 }; if (m_connectedSinceTime == (std::chrono::steady_clock::time_point::min)() || @@ -1738,65 +1906,71 @@ int PlayerConnection::ComputeConnectionScore() return -1; } - // TODO_EOS: need to impl jitter etc again - const int latency = GetLatency(); - const int jitter = GetJitter(); - const float quality = GetConnectionQuality(); // packet delivery ratio [0..1] + if (NGMP_OnlineServicesManager::GetInstance() == nullptr) + { + return -1; + } + const ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); + const int slackPercent = serviceConf.input_delay.slack_percent; + int sendIntervalMs = serviceConf.send.frames_per_packet * 1000 / GENERALS_ONLINE_HIGH_FPS_LIMIT; - static constexpr float k_subScoreFloor = 0.3f; - static constexpr float k_latencyWeight = 0.30f; - static constexpr float k_jitterWeight = 0.25f; - static constexpr float k_reliabilityWeight = 0.45f; + // the command connections are indexed by game slot, find the slot of this user + int slot = -1; + const ConnectionStats* pCmd = nullptr; + if (TheNetwork != nullptr && TheNGMPGame != nullptr) + { + slot = TheNGMPGame->getSlotOfUser(m_userID); + pCmd = TheNetwork->getConnectionStats(slot); + } - float weightedLogSum = 0.0f; - float activeWeightSum = 0.0f; + ConnectionRating::Inputs inputs; + NetworkOverlayInfo info; - if (latency >= 0) + float feedbackLoss = -1.0f; + static constexpr int k_minCmdSamples = 20; + const bool inMatch = (pCmd != nullptr && pCmd->getSampleCount() >= k_minCmdSamples); + const int feedbackWindowMs = inMatch ? NETWORK_STATS_WINDOW_MS : 60000; + PacketFeedback::Summary outLoss; + PacketFeedback::Summary inLoss; + const bool hasOut = GetOutgoingLoss(feedbackWindowMs, outLoss); + const bool hasIn = GetIncomingLoss(feedbackWindowMs, inLoss); + if (hasOut || hasIn) { - // 10ms and below are treated as full score. Above that, roughly: - // 400ms -> composite 75, 800ms -> composite 50 when other metrics are perfect. - int effectiveLatency = (std::max)(latency - 10, 0); - float latFactor = std::clamp(1.0f - static_cast(effectiveLatency) / 1590.0f, 0.0f, 1.0f); - float latencyScore = (std::max)(std::powf(latFactor, 4.545f), k_subScoreFloor); - weightedLogSum += k_latencyWeight * std::logf(latencyScore); - activeWeightSum += k_latencyWeight; + feedbackLoss = (std::max)(hasOut ? outLoss.lossPercent : 0.0f, hasIn ? inLoss.lossPercent : 0.0f); } + const float sustainedLoss = GetSustainedLossPercent(feedbackWindowMs); - if (jitter >= 0) + if (inMatch && TheNetwork->getOverlayInfo(slot, info)) { - // 50ms -> composite 75, 100ms -> composite 50 when other metrics are perfect. - float jitFactor = std::clamp(1.0f - static_cast(jitter) / 200.0f, 0.0f, 1.0f); - float jitterScore = (std::max)(std::powf(jitFactor, 2.632f), k_subScoreFloor); - weightedLogSum += k_jitterWeight * std::logf(jitterScore); - activeWeightSum += k_jitterWeight; + if (info.frameGroupingMs > 0) + { + sendIntervalMs = info.frameGroupingMs; + } + inputs = ConnectionRating::InputsFromMatch(*pCmd, info, slackPercent, sendIntervalMs, feedbackLoss); } - - if (quality >= 0.0f) + else if (m_probeStats.getSampleCount() >= 10) { - // 90% -> composite 75, 80% -> composite 50 when other metrics are perfect. - float relFactor = std::clamp(2.5f * quality - 1.5f, 0.0f, 1.0f); - float reliabilityScore = (std::max)(std::powf(relFactor, 2.5f), k_subScoreFloor); - weightedLogSum += k_reliabilityWeight * std::logf(reliabilityScore); - activeWeightSum += k_reliabilityWeight; + inputs = ConnectionRating::InputsFromProbe(m_probeStats, slackPercent, sendIntervalMs, feedbackLoss); } - - if (activeWeightSum <= 0.0f) + else { - return -1; + // a gap in the samples keeps the last rating, the indicator must not blink out + return (m_rating.bars > 0) ? m_rating.bars : -1; } - float composite = std::expf(weightedLogSum / activeWeightSum); - float rawScore = composite * 100.0f; - - static constexpr float k_scoreSmoothingAlpha = 0.15f; - if (m_smoothedScore < 0.0f) + inputs.lossPercent = sustainedLoss; + if (m_lastPacketMs > 0) { - m_smoothedScore = rawScore; + const int64_t nowSteadyMs = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + inputs.silentMs = static_cast((std::min)(nowSteadyMs - m_lastPacketMs, static_cast(3600000))); } - else { - m_smoothedScore += k_scoreSmoothingAlpha * (rawScore - m_smoothedScore); + // silence must outlast the probe interval of the current phase, probes may be the only traffic + const ServiceConfig::Connection& pingConfig = serviceConf.connection; + const int pingIntervalMs = (TheNetwork != nullptr) ? pingConfig.probe_match_interval_ms : pingConfig.probe_lobby_interval_ms; + inputs.silentLimitMs = (std::max)(inputs.silentLimitMs, pingIntervalMs * 5 / 2); } - return static_cast(std::round(m_smoothedScore)); + ConnectionRating::Update(m_rating, inputs, timeGetTime()); + return m_rating.bars; } diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NextGenTransport.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NextGenTransport.cpp index d6032342218..78fac2e4447 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NextGenTransport.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NextGenTransport.cpp @@ -332,8 +332,26 @@ Bool NextGenTransport::doRecv(void) std::map& connections = pMesh->GetAllConnections(); for (auto& kvPair : connections) { + // Only take as many messages as there are free slots, the rest stay queued in GNS + int numFreeSlots = 0; + for (int iSlot = 0; iSlot < MAX_MESSAGES; ++iSlot) + { + if (!m_inBufferOccupied[iSlot] || m_inBuffer[iSlot].length == 0) + { + ++numFreeSlots; + } + } + + if (numFreeSlots == 0) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, + "Game Packet Recv: m_inBuffer is full, leaving packets from user %lld queued", + static_cast(kvPair.second.m_userID)); + continue; + } + SteamNetworkingMessage_t* pMsg[255] = { nullptr }; - int numPackets = kvPair.second.Recv(pMsg); + int numPackets = kvPair.second.Recv(pMsg, std::min(numFreeSlots, static_cast(std::size(pMsg)))); if (numPackets <= 0) continue; @@ -381,12 +399,28 @@ Bool NextGenTransport::doRecv(void) msg->Release(); continue; } + else if (netChannel == ENetworkChannel::NETWORK_CHANNEL_PING) + { + kvPair.second.HandleProbe(reinterpret_cast(vecPacketDataWithoutHeader.data()), vecPacketDataWithoutHeader.size(), PlayerConnection::GetArrivalMs(msg)); + msg->Release(); + continue; + } else if (netChannel == ENetworkChannel::NETWORK_CHANNEL_GAME) { NetworkLog(ELogVerbosity::LOG_DEBUG, "[GAME PACKET] Received message of size %u from user %lld", vecPacketDataWithoutHeader.size(), static_cast(kvPair.second.m_userID)); + // the packet envelope: sequence number and receiver feedback + const size_t envelopeSize = kvPair.second.ConsumeEnvelope(reinterpret_cast(vecPacketDataWithoutHeader.data()), vecPacketDataWithoutHeader.size()); + if (envelopeSize == 0) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Game Packet Recv: Dropping packet without a valid envelope from user %lld, probably another game version", static_cast(kvPair.second.m_userID)); + msg->Release(); + continue; + } + vecPacketDataWithoutHeader.erase(vecPacketDataWithoutHeader.begin(), vecPacketDataWithoutHeader.begin() + envelopeSize); + // Must at least contain the header if (vecPacketDataWithoutHeader.size() < sizeof(TransportMessageHeader)) { @@ -565,6 +599,12 @@ Bool NextGenTransport::doRecv(void) bRet = TRUE; } } + else + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Game Packet Recv: Dropping packet with unknown channel %u from user %lld", static_cast(vecDataWithHeader[0]), static_cast(kvPair.second.m_userID)); + msg->Release(); + continue; + } } else if (vecDataWithHeader.size() != -1) { diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp index cfe0835efd4..29ae73bc5bd 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp @@ -23,7 +23,10 @@ #include "GameNetwork/GeneralsOnline/Vendor/stb_image/stb_image_write.h" #include "GameNetwork/GeneralsOnline/Vendor/stb_image/stb_image_resize.h" #include "GameClient/GameText.h" +#include "GameLogic/GameLogic.h" #include +#include +#include extern "C" { @@ -39,7 +42,7 @@ std::mutex NGMP_OnlineServicesManager::m_ScreenshotMutex; std::vector NGMP_OnlineServicesManager::m_vecGuardedSSData; -bool NGMP_OnlineServicesManager::g_bAdvancedNetworkStats; +int NGMP_OnlineServicesManager::g_networkOverlayMode; NetworkMesh* NGMP_OnlineServicesManager::GetNetworkMesh() { @@ -99,6 +102,34 @@ void NGMP_OnlineServicesManager::GetAndParseServiceConfig(std::function(); + NGMP_OnlineServices_LobbyInterface* pLobbyInterface = NGMP_OnlineServicesManager::GetInterface(); + if (pAuthInterface != nullptr && pLobbyInterface != nullptr) + { + const int64_t localUserID = pAuthInterface->GetUserID(); + for (const LobbyMemberEntry& lobbyMember : pLobbyInterface->GetMembersListForCurrentRoom()) + { + if (lobbyMember.user_id == localUserID) + { + slotIndex = lobbyMember.m_SlotIndex < MAX_SLOTS ? lobbyMember.m_SlotIndex : 0; + break; + } + } + } + + const int slotOffsetMS = std::clamp(NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig().telemetry.screenshot_slot_offset_ms, 0, 10000); + + static thread_local std::mt19937 rng{ std::random_device{}() }; + const int jitterMS = std::uniform_int_distribution(0, 500)(rng); + + return std::chrono::steady_clock::now() + std::chrono::milliseconds(slotIndex * slotOffsetMS + jitterMS); +} + void NGMP_OnlineServicesManager::CaptureScreenshotForProbe(EScreenshotType screenshotType, std::string strURI) { NGMP_OnlineServicesManager* pOnlineServicesMgr = NGMP_OnlineServicesManager::GetInstance(); @@ -106,7 +137,7 @@ void NGMP_OnlineServicesManager::CaptureScreenshotForProbe(EScreenshotType scree { ServiceConfig& serviceConf = pOnlineServicesMgr->GetServiceConfig(); - if (serviceConf.do_probes) + if (serviceConf.telemetry.screenshot_probes) { CHECK_MAIN_THREAD; @@ -114,7 +145,8 @@ void NGMP_OnlineServicesManager::CaptureScreenshotForProbe(EScreenshotType scree if (pLobbyInterface != nullptr) { const uint64_t matchID = pLobbyInterface->GetCurrentMatchID(); - NGMP_OnlineServicesManager::GetInstance()->CaptureScreenshot(true, [strURI = std::move(strURI), screenshotType, matchID](std::vector vecData) + const std::chrono::steady_clock::time_point uploadTime = GetScreenshotUploadTime(); + NGMP_OnlineServicesManager::GetInstance()->CaptureScreenshot(true, [strURI = std::move(strURI), screenshotType, matchID, uploadTime](std::vector vecData) { CHECK_WORKER_THREAD; @@ -127,7 +159,7 @@ void NGMP_OnlineServicesManager::CaptureScreenshotForProbe(EScreenshotType scree // certain screenshots require caching for later upload when we have a valid match id, so we store them if (screenshotType == EScreenshotType::SCREENSHOT_TYPE_LOADSCREEN) { - NGMP_OnlineServicesManager::GetInstance()->CacheScreenshotBytes_StartMatch(vecData); + NGMP_OnlineServicesManager::GetInstance()->CacheScreenshotBytes_StartMatch(std::move(vecData)); } else if (screenshotType == EScreenshotType::SCREENSHOT_TYPE_SCORESCREEN) { @@ -143,7 +175,8 @@ void NGMP_OnlineServicesManager::CaptureScreenshotForProbe(EScreenshotType scree newEntry.vecBytes = std::move(vecData); newEntry.strSignedURI = strURI; newEntry.screenshotType = screenshotType; - m_vecGuardedSSData.push_back(newEntry); + newEntry.notBefore = uploadTime; + m_vecGuardedSSData.push_back(std::move(newEntry)); } }); } @@ -242,7 +275,7 @@ void NGMP_OnlineServicesManager::CommitReplay(AsciiString absoluteReplayPath) { ServiceConfig& serviceConf = pOnlineServicesMgr->GetServiceConfig(); - if (serviceConf.do_replay_upload) + if (serviceConf.telemetry.replay_upload) { NGMP_OnlineServices_LobbyInterface* pLobbyInterface = NGMP_OnlineServicesManager::GetInterface(); const uint64_t matchID = pLobbyInterface == nullptr ? 0 : pLobbyInterface->GetCurrentMatchID(); @@ -579,7 +612,6 @@ void NGMP_OnlineServicesManager::CaptureScreenshot(bool bResizeForTransmit, std: } SurfaceClass* surface = DX8Wrapper::_Get_DX8_Back_Buffer(); - LPDIRECT3DSURFACE8 surf = nullptr; SurfaceClass* surfaceCopy = nullptr; void* pBits = nullptr; IDirect3DSurface8* pDXsurf = nullptr; @@ -599,119 +631,96 @@ void NGMP_OnlineServicesManager::CaptureScreenshot(bool bResizeForTransmit, std: { DX8Wrapper::_Copy_DX8_Rects(surface->Peek_D3D_Surface(), NULL, 0, surfaceCopy->Peek_D3D_Surface(), NULL); - HRESULT hr; + // gather all our data + int pitch = 0; + pBits = surfaceCopy->Lock(&pitch); - D3DDISPLAYMODE mode; - if (SUCCEEDED(hr = DX8Wrapper::_Get_D3D_Device8()->GetDisplayMode(&mode))) + if (pBits != nullptr) { - if (SUCCEEDED(hr = DX8Wrapper::_Get_D3D_Device8()->CreateImageSurface(mode.Width, mode.Height, - D3DFMT_A8R8G8B8, &surf))) - { - if (SUCCEEDED(hr = DX8Wrapper::_Get_D3D_Device8()->GetFrontBuffer(surf))) + int width = surfaceDesc.Width; + int height = surfaceDesc.Height; + + // Copy pixel data into an owned buffer before spawning the thread so + // the surface can be safely unlocked on the main thread immediately after. + const uint8_t* pSurfaceBytes = static_cast(pBits); + std::vector pixelData(pSurfaceBytes, pSurfaceBytes + height * pitch); + + // process on thread - track the thread so we can join it during shutdown + std::thread* pNewThread = new std::thread([cbOnDataAvailable, width, height, pixelData = std::move(pixelData), pitch, bResizeForTransmit]() mutable { - // gather all our data - int pitch = 0; - pBits = surfaceCopy->Lock(&pitch); + CHECK_WORKER_THREAD; - if (pBits != nullptr) - { - int width = surfaceDesc.Width; - int height = surfaceDesc.Height; - - // Copy pixel data into an owned buffer before spawning the thread so - // the surface can be safely unlocked on the main thread immediately after. - std::vector pixelData(height * pitch); - memcpy(pixelData.data(), pBits, height * pitch); - - // process on thread - track the thread so we can join it during shutdown - std::thread* pNewThread = new std::thread([cbOnDataAvailable, width, height, pixelData = std::move(pixelData), pitch, bResizeForTransmit]() - { - CHECK_WORKER_THREAD; - - unsigned char* rgbData = new unsigned char[width * height * 3]; - - std::vector vecData; - - int finalWidth = width; - int finalHeight = height; - - for (int y = 0; y < height; ++y) { - const uint8_t* row = pixelData.data() + y * pitch; - int rowOffset = y * width * 3; - int srcOffset = 0; - for (int x = 0; x < width; ++x, srcOffset += 4) - { - int dstIndex = rowOffset + x * 3; - rgbData[dstIndex + 0] = row[srcOffset + 2]; // R - rgbData[dstIndex + 1] = row[srcOffset + 1]; // G - rgbData[dstIndex + 2] = row[srcOffset + 0]; // B - } - } - - // resize - unsigned char* pBufferToWrite = rgbData; - if (bResizeForTransmit) - { - ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); - int new_width = serviceConf.screenshot_width; - int new_height = serviceConf.screenshot_height; - int channels = 3; - unsigned char* resized = new unsigned char[new_width * new_height * channels]; - - stbir_resize_uint8(rgbData, width, height, 0, - resized, new_width, new_height, 0, - channels - ); - - // update data - finalWidth = new_width; - finalHeight = new_height; - pBufferToWrite = resized; - } - // end resize - - stbi_write_jpg_to_func([](void* context, void* data, int size) - { - std::vector* buffer = static_cast*>(context); - buffer->insert(buffer->end(), (unsigned char*)data, (unsigned char*)data + size); - }, &vecData, finalWidth, finalHeight, 3, pBufferToWrite, bResizeForTransmit ? 0 : 90); - - // cleanup - if (bResizeForTransmit) - { - delete[] pBufferToWrite; // This is 'resized' - pBufferToWrite = nullptr; - } - - delete[] rgbData; - rgbData = nullptr; - - // invoke cb - if (cbOnDataAvailable != nullptr) - { - cbOnDataAvailable(vecData); - } - } - ); + const int channels = 3; + int finalWidth = width; + int finalHeight = height; - // Store the thread so we can join it during shutdown - // SECURITY FIX: Capture manager pointer before spawning thread to avoid TOCTOU race - NGMP_OnlineServicesManager* pMgr = NGMP_OnlineServicesManager::GetInstance(); - if (pMgr != nullptr) + // Pack the BGRA rows to tight RGB in place. The write position never passes the read position. + uint8_t* pixels = pixelData.data(); + for (int y = 0; y < height; ++y) { - std::scoped_lock lock(pMgr->m_mutexScreenshotThreads); - pMgr->m_vecScreenshotThreads.push_back(pNewThread); + const uint8_t* src = pixels + y * pitch; + uint8_t* dst = pixels + y * width * channels; + for (int x = 0; x < width; ++x, src += 4, dst += channels) + { + const uint8_t blue = src[0]; + dst[0] = src[2]; // R + dst[1] = src[1]; // G + dst[2] = blue; // B + } } - else + + // resize + std::vector resized; + if (bResizeForTransmit) { - // Manager was destroyed, cannot store thread. Thread will leak but won't crash. - NetworkLog(ELogVerbosity::LOG_RELEASE, "[Screenshot] Manager destroyed before thread could be registered"); + ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); + finalWidth = serviceConf.telemetry.screenshot_width; + finalHeight = serviceConf.telemetry.screenshot_height; + resized.resize(finalWidth * finalHeight * channels); + + stbir_resize_uint8(pixels, width, height, 0, + resized.data(), finalWidth, finalHeight, 0, + channels + ); + + // the capture is no longer needed + std::vector().swap(pixelData); } + // end resize + + // room for a typical JPEG, so the write callback rarely has to grow it + std::vector vecData; + vecData.reserve(256 * 1024); + + stbi_write_jpg_to_func([](void* context, void* data, int size) + { + std::vector* buffer = static_cast*>(context); + buffer->insert(buffer->end(), (unsigned char*)data, (unsigned char*)data + size); + }, &vecData, finalWidth, finalHeight, channels, bResizeForTransmit ? resized.data() : pixelData.data(), bResizeForTransmit ? 0 : 90); - bSucceeded = true; + // invoke cb + if (cbOnDataAvailable != nullptr) + { + cbOnDataAvailable(std::move(vecData)); } } + ); + + // Store the thread so we can join it during shutdown + // SECURITY FIX: Capture manager pointer before spawning thread to avoid TOCTOU race + NGMP_OnlineServicesManager* pMgr = NGMP_OnlineServicesManager::GetInstance(); + if (pMgr != nullptr) + { + std::scoped_lock lock(pMgr->m_mutexScreenshotThreads); + pMgr->m_vecScreenshotThreads.push_back(pNewThread); } + else + { + // Manager was destroyed, cannot store thread. Thread will leak but won't crash. + NetworkLog(ELogVerbosity::LOG_RELEASE, "[Screenshot] Manager destroyed before thread could be registered"); + } + + bSucceeded = true; } } } @@ -719,18 +728,9 @@ void NGMP_OnlineServicesManager::CaptureScreenshot(bool bResizeForTransmit, std: // clean everything up, whether we succeeded or not - // release the image surface - if (surf != nullptr) - { - surf->Release(); - //delete surf; - surf = nullptr; - } - // unlock if (surface != nullptr) { - surface->Unlock(); surface->Release_Ref(); surface = nullptr; } @@ -953,20 +953,35 @@ void NGMP_OnlineServicesManager::Tick() std::scoped_lock ssLock(m_ScreenshotMutex); + const int uploadBytesPerSec = std::clamp(m_ServiceConfig.telemetry.screenshot_upload_bytes_per_sec, 16384, 10485760); + const std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now(); + const bool bMatchOver = TheGameLogic == nullptr || !TheGameLogic->isInMultiplayerGame(); // uploads no longer compete with game traffic + // screenshot types that already have a presigned URL - for (S3ScreenshotEntry screenshotEntry : m_vecGuardedSSData) + for (auto it = m_vecGuardedSSData.begin(); it != m_vecGuardedSSData.end();) { + S3ScreenshotEntry& screenshotEntry = *it; + const bool bUnthrottled = bMatchOver || screenshotEntry.screenshotType == EScreenshotType::SCREENSHOT_TYPE_SCORESCREEN; + + // not this client's turn to upload yet + if (!bUnthrottled && screenshotEntry.notBefore > now) + { + ++it; + continue; + } + // NOTE: Screenshot types start and end of match are captures and cached in memory until the server tells us where to upload them, so we need to wait for the upload URI if ((screenshotEntry.screenshotType == EScreenshotType::SCREENSHOT_TYPE_LOADSCREEN || screenshotEntry.screenshotType == EScreenshotType::SCREENSHOT_TYPE_SCORESCREEN) && screenshotEntry.strSignedURI.empty()) { + it = m_vecGuardedSSData.erase(it); continue; } else // we have the data we need, send away { std::map mapHeaders; mapHeaders["Content-Type"] = "image/jpeg"; - NGMP_OnlineServicesManager::GetInstance()->GetHTTPManager()->SendS3PUTRequest(screenshotEntry.strSignedURI.c_str(), EIPProtocolVersion::DONT_CARE, mapHeaders, screenshotEntry.vecBytes, [=](bool bSuccess, int statusCode, std::string strBody, HTTPRequest* pReq) + NGMP_OnlineServicesManager::GetInstance()->GetHTTPManager()->SendS3PUTRequest(screenshotEntry.strSignedURI.c_str(), EIPProtocolVersion::DONT_CARE, mapHeaders, std::move(screenshotEntry.vecBytes), [=](bool bSuccess, int statusCode, std::string strBody, HTTPRequest* pReq) { #if _DEBUG if (statusCode != 200) @@ -975,12 +990,11 @@ void NGMP_OnlineServicesManager::Tick() } #endif NetworkLog(ELogVerbosity::LOG_RELEASE, "Screenshot upload, result: %d", statusCode); - }, nullptr, HTTP_UPLOAD_TIMEOUT); + }, nullptr, HTTP_UPLOAD_TIMEOUT, bUnthrottled ? 0 : uploadBytesPerSec); } - - } - m_vecGuardedSSData.clear(); + it = m_vecGuardedSSData.erase(it); + } // screenshots and replays that are cached and awaiting S3 url, and now have said URL if (!m_vecCachedScreenshotBytes_MatchStart.empty()) // we have data waiting @@ -990,9 +1004,10 @@ void NGMP_OnlineServicesManager::Tick() // queue it S3ScreenshotEntry newEntry; newEntry.screenshotType = EScreenshotType::SCREENSHOT_TYPE_LOADSCREEN; - newEntry.vecBytes = m_vecCachedScreenshotBytes_MatchStart; - newEntry.strSignedURI = m_strCachedScreenshot_MatchStart_S3URI; - m_vecGuardedSSData.push_back(newEntry); + newEntry.vecBytes = std::move(m_vecCachedScreenshotBytes_MatchStart); + newEntry.strSignedURI = std::move(m_strCachedScreenshot_MatchStart_S3URI); + newEntry.notBefore = GetScreenshotUploadTime(); + m_vecGuardedSSData.push_back(std::move(newEntry)); // clear data m_vecCachedScreenshotBytes_MatchStart = std::vector(); @@ -1023,7 +1038,7 @@ void NGMP_OnlineServicesManager::Tick() // do the upload std::map mapHeaders; mapHeaders["Content-Type"] = "application/octet-stream"; - NGMP_OnlineServicesManager::GetInstance()->GetHTTPManager()->SendS3PUTRequest(m_cachedReplayUpload.signedURI.c_str(), EIPProtocolVersion::DONT_CARE, mapHeaders, m_cachedReplayUpload.bytes, [=](bool bSuccess, int statusCode, std::string strBody, HTTPRequest* pReq) + NGMP_OnlineServicesManager::GetInstance()->GetHTTPManager()->SendS3PUTRequest(m_cachedReplayUpload.signedURI.c_str(), EIPProtocolVersion::DONT_CARE, mapHeaders, std::move(m_cachedReplayUpload.bytes), [=](bool bSuccess, int statusCode, std::string strBody, HTTPRequest* pReq) { #if _DEBUG if (statusCode != 200) diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/PacketFeedback.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/PacketFeedback.cpp new file mode 100644 index 00000000000..64c659e9799 --- /dev/null +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/PacketFeedback.cpp @@ -0,0 +1,179 @@ +#include "GameNetwork/GeneralsOnline/PacketFeedback.h" +#include "GameNetwork/GeneralsOnline/NetBytes.h" + +#include + +PacketFeedback::PacketFeedback() +{ + Reset(); +} + +void PacketFeedback::Reset() +{ + m_nextSequence = 0; + m_sentCount = 0; + m_haveHighest = false; + m_highest = 0; + m_mask = 0; + m_span = 0; + m_lastFeedbackMs = 0; + m_outgoing.clear(); + m_incoming.clear(); +} + +size_t PacketFeedback::WriteEnvelope(uint8_t* out, int64_t nowMs) +{ + WriteU16(out, m_nextSequence); + ++m_nextSequence; + ++m_sentCount; + out[2] = 0x80; + + if (!m_haveHighest || (m_lastFeedbackMs != 0 && nowMs - m_lastFeedbackMs < FEEDBACK_INTERVAL_MS)) + { + return ENVELOPE_BASE_SIZE; + } + + int received = 0; + int longestRun = 0; + int run = 0; + for (int bit = 0; bit < m_span; ++bit) + { + if (m_mask & (static_cast(1) << bit)) + { + ++received; + run = 0; + } + else + { + ++run; + longestRun = (std::max)(longestRun, run); + } + } + + out[2] |= 0x01; + WriteU16(out + 3, m_highest); + out[5] = m_span; + out[6] = static_cast(received); + out[7] = static_cast(longestRun); + + Report report; + report.timeMs = nowMs; + report.expected = m_span; + report.received = static_cast(received); + report.longestRun = static_cast(longestRun); + Push(m_incoming, report, nowMs); + + m_lastFeedbackMs = nowMs; + return ENVELOPE_MAX_SIZE; +} + +void PacketFeedback::CancelEnvelope() +{ + if (m_sentCount > 0) + { + --m_nextSequence; + --m_sentCount; + } +} + +size_t PacketFeedback::ReadEnvelope(const uint8_t* in, size_t len, int64_t nowMs) +{ + if (in == nullptr || len < ENVELOPE_BASE_SIZE || (in[2] & 0x80) == 0) + { + return 0; + } + + const bool hasFeedback = (in[2] & 0x01) != 0; + if (hasFeedback && len < ENVELOPE_MAX_SIZE) + { + return 0; + } + + const uint16_t sequence = ReadU16(in); + if (!m_haveHighest) + { + m_haveHighest = true; + m_highest = sequence; + m_mask = 1; + m_span = 1; + } + else + { + const int diff = static_cast(static_cast(sequence - m_highest)); + if (diff > 0) + { + m_mask = (diff >= 64) ? 0 : (m_mask << diff); + m_mask |= 1; + m_highest = sequence; + m_span = static_cast((std::min)(64, m_span + diff)); + } + else if (diff < 0 && -diff < 64) + { + m_mask |= static_cast(1) << (-diff); + } + } + + if (!hasFeedback) + { + return ENVELOPE_BASE_SIZE; + } + + Report report; + report.timeMs = nowMs; + report.expected = in[5]; + report.received = in[6]; + report.longestRun = in[7]; + if (report.expected > 64 || report.received > report.expected) + { + return ENVELOPE_MAX_SIZE; // implausible, the packet is still consumed + } + + // the peer cannot have expected more of our packets than we sent + report.expected = static_cast((std::min)(report.expected, m_sentCount)); + report.received = (std::min)(report.received, report.expected); + Push(m_outgoing, report, nowMs); + return ENVELOPE_MAX_SIZE; +} + +void PacketFeedback::Push(std::deque& reports, const Report& report, int64_t nowMs) +{ + reports.push_back(report); + while (!reports.empty() && nowMs - reports.front().timeMs > MAX_WINDOW_MS) + { + reports.pop_front(); + } +} + +PacketFeedback::Summary PacketFeedback::Summarize(const std::deque& reports, int64_t nowMs, int windowMs) +{ + Summary summary; + int expected = 0; + int received = 0; + for (const Report& report : reports) + { + if (nowMs - report.timeMs > windowMs) + { + continue; + } + ++summary.samples; + expected += report.expected; + received += report.received; + summary.longestRun = (std::max)(summary.longestRun, static_cast(report.longestRun)); + } + + if (expected > 0) + { + summary.lossPercent = 100.0f * static_cast(expected - received) / static_cast(expected); + } + return summary; +} + +PacketFeedback::Summary PacketFeedback::GetOutgoing(int64_t nowMs, int windowMs) const +{ + return Summarize(m_outgoing, nowMs, windowMs); +} + +PacketFeedback::Summary PacketFeedback::GetIncoming(int64_t nowMs, int windowMs) const +{ + return Summarize(m_incoming, nowMs, windowMs); +} diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/PluginInterfaces.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/PluginInterfaces.cpp index 7d60712e8cc..c62f5f7fcf9 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/PluginInterfaces.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/PluginInterfaces.cpp @@ -507,7 +507,7 @@ bool AnticheatPlugInterface::DoesACPluginProvideSecureGameTransport() { #if defined(AC_ENABLED) ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); - const bool bEnabledByConfig = serviceConf.use_new_networking; + const bool bEnabledByConfig = serviceConf.connection.anticheat_transport; // The plugin makes this same decision internally in places the game cannot // reach - notably inside the EOS anti-cheat OnMessageToPeer callback, which diff --git a/GeneralsMD/Code/GameEngineDevice/Source/W3DDevice/GameClient/W3DDisplay.cpp b/GeneralsMD/Code/GameEngineDevice/Source/W3DDevice/GameClient/W3DDisplay.cpp index 5eda9635d85..e69e94431ab 100644 --- a/GeneralsMD/Code/GameEngineDevice/Source/W3DDevice/GameClient/W3DDisplay.cpp +++ b/GeneralsMD/Code/GameEngineDevice/Source/W3DDevice/GameClient/W3DDisplay.cpp @@ -1545,6 +1545,20 @@ void W3DDisplay::gatherDebugStats() // Network performance stats unibuffer.format(L"Run Ahead: %d, Net FPS: %d, Packet arrival cushion: %d", TheNetwork->getRunAhead(), TheNetwork->getFrameRate(), TheNetwork->getPacketArrivalCushion()); + + // per-connection latency, jitter and loss + for (Int slot = 0; slot < MAX_SLOTS; ++slot) { + const ConnectionStats* stats = TheNetwork->getConnectionStats(slot); + if (stats != nullptr && stats->getSampleCount() > 0) { + UnicodeString statsstr; + if (stats->getLatePercent() > 0.0f) { + statsstr.format(L" P%d %.0fms j%.1f l%.0f/%.0f%%", slot, stats->getLatency(), stats->getJitter(), stats->getLossPercent(), stats->getLatePercent()); + } else { + statsstr.format(L" P%d %.0fms j%.1f l%.0f%%", slot, stats->getLatency(), stats->getJitter(), stats->getLossPercent()); + } + unibuffer.concat(statsstr); + } + } m_displayStrings[NetStats]->setText( unibuffer ); // Client frame rate averages for all players in the game. This only works right for the packet router.